From: Ant Zucaro Date: Sat, 6 Feb 2016 21:21:44 +0000 (-0500) Subject: Make the active players/servers/maps use the cache. X-Git-Url: https://git.rm.cloudns.org/?a=commitdiff_plain;h=5e7c58ce773acd93dbd00900fcab0caa3e959909;p=xonotic%2Fxonstat.git Make the active players/servers/maps use the cache. Instead of fetching from the base data, the active players/server/maps views will fetch from the so-called "materialized view" tables instead. These tables are precomputed and will be MUCH faster, performance-wise. Regular Beaker caching remains the same for these, and the paginated pages now only support clicking "more" until the list is exhausted. Perhaps in the future they will be fetched via JSON and appended to the page in a "never-ending scroll" thing. --- diff --git a/xonstat/__init__.py b/xonstat/__init__.py index 1609f4d..5466efd 100644 --- a/xonstat/__init__.py +++ b/xonstat/__init__.py @@ -101,14 +101,14 @@ def main(global_config, **settings): config.add_route("player_weaponstats_data_json", "/player/{id:\d+}/weaponstats.json") config.add_view(player_weaponstats_data_json, route_name="player_weaponstats_data_json", renderer="jsonp") - config.add_route("top_players_by_time", "/topactive") - config.add_view(top_players_by_time, route_name="top_players_by_time", renderer="top_players_by_time.mako") + config.add_route("top_players_index", "/topactive") + config.add_view(top_players_index, route_name="top_players_index", renderer="top_players_by_time.mako") - config.add_route("top_servers_by_players", "/topservers") - config.add_view(top_servers_by_players, route_name="top_servers_by_players", renderer="top_servers_by_players.mako") + config.add_route("top_servers_index", "/topservers") + config.add_view(top_servers_index, route_name="top_servers_index", renderer="top_servers_by_players.mako") - config.add_route("top_maps_by_times_played", "/topmaps") - config.add_view(top_maps_by_times_played, route_name="top_maps_by_times_played", renderer="top_maps_by_times_played.mako") + config.add_route("top_maps_index", "/topmaps") + config.add_view(top_maps_index, route_name="top_maps_index", renderer="top_maps_by_times_played.mako") # GAME ROUTES config.add_route("game_info", "/game/{id:\d+}") diff --git a/xonstat/models.py b/xonstat/models.py index e34a295..0ae5a52 100644 --- a/xonstat/models.py +++ b/xonstat/models.py @@ -389,6 +389,45 @@ class PlayerCapTime(object): } +class ActivePlayer(object): + def __init__(self, sort_order=None, player_id=None, nick=None, + alivetime=None): + self.sort_order = sort_order + self.player_id = player_id + self.nick = nick + self.alivetime = alivetime + + def nick_html_colors(self): + return html_colors(self.nick) + + def __repr__(self): + return "" % (self.sort_order, self.player_id) + + +class ActiveServer(object): + def __init__(self, sort_order=None, server_id=None, server_name=None, + games=None): + self.sort_order = sort_order + self.server_id = server_id + self.server_name = server_name + self.games = games + + def __repr__(self): + return "" % (self.sort_order, self.server_id) + + +class ActiveMap(object): + def __init__(self, sort_order=None, map_id=None, map_name=None, + games=None): + self.sort_order = sort_order + self.map_id = map_id + self.map_name = map_name + self.games = games + + def __repr__(self): + return "" % (self.sort_order, self.map_id) + + def initialize_db(engine=None): DBSession.configure(bind=engine) Base.metadata.bind = engine @@ -417,6 +456,9 @@ def initialize_db(engine=None): team_game_stats_table = MetaData.tables['team_game_stats'] player_game_anticheats_table = MetaData.tables['player_game_anticheats'] player_groups_table = MetaData.tables['player_groups'] + active_players_table = MetaData.tables['active_players_mv'] + active_servers_table = MetaData.tables['active_servers_mv'] + active_maps_table = MetaData.tables['active_maps_mv'] # now map the tables and the objects together mapper(PlayerAchievement, achievements_table) @@ -438,3 +480,6 @@ def initialize_db(engine=None): mapper(TeamGameStat, team_game_stats_table) mapper(PlayerGameAnticheat, player_game_anticheats_table) mapper(PlayerGroups, player_groups_table) + mapper(ActivePlayer, active_players_table) + mapper(ActiveServer, active_servers_table) + mapper(ActiveMap, active_maps_table) diff --git a/xonstat/templates/main_index.mako b/xonstat/templates/main_index.mako index 5f4fa79..3405039 100644 --- a/xonstat/templates/main_index.mako +++ b/xonstat/templates/main_index.mako @@ -73,7 +73,7 @@ ##### ACTIVE PLAYERS #####
-
Most Active Players
+
Most Active Players
@@ -83,18 +83,12 @@ - <% i = 1 %> - % for (player_id, nick, alivetime) in top_players: + % for tp in top_players: - - % if player_id != '-': - - % else: - - % endif - + + + - <% i = i+1 %> % endfor
${i}${nick|n}${nick|n}${alivetime}${tp.sort_order}${tp.nick_html_colors()|n}${tp.alivetime}
@@ -103,7 +97,7 @@ ##### ACTIVE SERVERS #####
-
Most Active Servers
+
Most Active Servers
@@ -113,18 +107,12 @@ - <% i = 1 %> - % for (server_id, name, count) in top_servers: + % for ts in top_servers: - - % if server_id != '-': - - % else: - - % endif - + + + - <% i = i+1 %> % endfor
${i}${name}${name}${count}${ts.sort_order}${ts.server_name}${ts.games}
@@ -133,7 +121,7 @@ ##### ACTIVE MAPS #####
-
Most Active Maps
+
Most Active Maps
@@ -143,23 +131,18 @@ - <% i = 1 %> - % for (map_id, name, count) in top_maps: + % for tm in top_maps: - - % if map_id != '-': - - % else: - - % endif - + + + - <% i = i+1 %> % endfor
${i}${name}${name}${count}${tm.sort_order}${tm.map_name}${tm.games}
+
*Most active stats are from the past 7 days diff --git a/xonstat/templates/top_maps_by_times_played.mako b/xonstat/templates/top_maps_by_times_played.mako index ceb32f8..4a8f358 100644 --- a/xonstat/templates/top_maps_by_times_played.mako +++ b/xonstat/templates/top_maps_by_times_played.mako @@ -1,6 +1,5 @@ <%inherit file="base.mako"/> <%namespace name="nav" file="nav.mako" /> -<%namespace file="navlinks.mako" import="navlinks" /> <%block name="navigation"> ${nav.nav('maps')} @@ -10,8 +9,11 @@ Active Maps Index -% if not top_maps: -

Sorry, no maps yet. Get playing!

+% if not top_maps and start is not None: +

Sorry, no more maps!

+ +% elif not top_maps and start is None: +

No active maps found. Yikes, get playing!

% else: ##### ACTIVE SERVERS ##### @@ -26,30 +28,29 @@ - ##### this is to get around the actual row_number/rank of the map not being in the actual query - <% i = 1 + (top_maps.page-1) * 25%> - % for (map_id, name, count) in top_maps: + % for tm in top_maps: - ${i} - % if map_id != '-': - ${name} - % else: - ${name} - % endif - ${count} + ${tm.sort_order} + ${tm.map_name} + ${tm.sort_order} - <% i = i+1 %> % endfor - *figures are from the past 7 days +

Note: these figures are from the past 7 days

-
-
- ${navlinks("top_maps_by_times_played", top_maps.page, top_maps.last_page)} -
+% if len(top_maps) == 20: +
+
+
+
+% endif % endif diff --git a/xonstat/templates/top_players_by_time.mako b/xonstat/templates/top_players_by_time.mako index 37a47e9..9f75ff6 100644 --- a/xonstat/templates/top_players_by_time.mako +++ b/xonstat/templates/top_players_by_time.mako @@ -1,6 +1,5 @@ <%inherit file="base.mako"/> <%namespace name="nav" file="nav.mako" /> -<%namespace file="navlinks.mako" import="navlinks" /> <%block name="navigation"> ${nav.nav('players')} @@ -10,8 +9,11 @@ Active Players Index -% if not top_players: -

Sorry, no players yet. Get playing!

+% if not top_players and start is not None: +

Sorry, no more players!

+ +% elif not top_players and start is None: +

No active players found. Yikes, get playing!

% else: ##### ACTIVE PLAYERS ##### @@ -27,29 +29,29 @@ - ##### this is to get around the actual row_number/rank of the player not being in the actual query - <% i = 1 + (top_players.page-1) * 25%> - % for (player_id, nick, alivetime) in top_players.items: + % for tp in top_players: - ${i} - % if player_id != '-': - ${nick|n} - % else: - ${nick|n} - % endif - ${alivetime} + ${tp.sort_order} + ${tp.nick_html_colors()|n} + ${tp.alivetime} - <% i = i+1 %> % endfor - *figures are from the past 7 days +

Note: these figures are from the past 7 days

-
-
- ${navlinks("top_players_by_time", top_players.page, top_players.last_page)} -
+% if len(top_players) == 20: +
+
+
+
+% endif + % endif diff --git a/xonstat/templates/top_servers_by_players.mako b/xonstat/templates/top_servers_by_players.mako index ea6912f..4adba45 100644 --- a/xonstat/templates/top_servers_by_players.mako +++ b/xonstat/templates/top_servers_by_players.mako @@ -1,6 +1,5 @@ <%inherit file="base.mako"/> <%namespace name="nav" file="nav.mako" /> -<%namespace file="navlinks.mako" import="navlinks" /> <%block name="navigation"> ${nav.nav('servers')} @@ -10,8 +9,11 @@ Active Servers Index -% if not top_servers: -

Sorry, no servers yet. Get playing!

+% if not top_servers and start is not None: +

Sorry, no more servers!

+ +% elif not top_servers and start is None: +

No active servers found. Yikes, get playing!

% else: ##### ACTIVE SERVERS ##### @@ -26,30 +28,29 @@ - ##### this is to get around the actual row_number/rank of the server not being in the actual query - <% i = 1 + (top_servers.page-1) * 25%> - % for (server_id, name, count) in top_servers.items: + % for ts in top_servers: - ${i} - % if server_id != '-': - ${name} - % else: - ${name} - % endif - ${count} + ${ts.sort_order} + ${ts.server_name} + ${ts.games} - <% i = i+1 %> % endfor - *figures are from the past 7 days +

Note: these figures are from the past 7 days

-
-
- ${navlinks("top_servers_by_players", top_servers.page, top_servers.last_page)} -
+% if len(top_servers) == 20: +
+
+
+
+% endif % endif diff --git a/xonstat/views/__init__.py b/xonstat/views/__init__.py index d293a83..4ca6718 100644 --- a/xonstat/views/__init__.py +++ b/xonstat/views/__init__.py @@ -26,8 +26,8 @@ from xonstat.views.search import search_json from xonstat.views.exceptions import notfound -from xonstat.views.main import main_index, top_players_by_time, top_servers_by_players -from xonstat.views.main import top_servers_by_players, top_maps_by_times_played +from xonstat.views.main import main_index, top_players_index, top_servers_index +from xonstat.views.main import top_maps_index from xonstat.views.admin import forbidden, login, merge diff --git a/xonstat/views/main.py b/xonstat/views/main.py index 1e937bf..98973a7 100644 --- a/xonstat/views/main.py +++ b/xonstat/views/main.py @@ -111,125 +111,58 @@ def get_ranks(game_type_cd): return ranks -def top_players_by_time_q(cutoff_days): - """ - Query for the top players by the amount of time played during a date range. - - Games older than cutoff_days days old are ignored. - """ - - # only games played during this range are considered - right_now = datetime.utcnow() - cutoff_dt = right_now - timedelta(days=cutoff_days) - - top_players_q = DBSession.query(Player.player_id, Player.nick, - func.sum(PlayerGameStat.alivetime)).\ - filter(Player.player_id == PlayerGameStat.player_id).\ - filter(Player.player_id > 2).\ - filter(expr.between(PlayerGameStat.create_dt, cutoff_dt, right_now)).\ - order_by(expr.desc(func.sum(PlayerGameStat.alivetime))).\ - group_by(Player.nick).\ - group_by(Player.player_id) - - return top_players_q - - @cache_region('hourly_term') -def get_top_players_by_time(cutoff_days): +def get_top_players_by_time(limit=None, start=None): """ The top players by the amount of time played during a date range. - - Games older than cutoff_days days old are ignored. """ - # how many to retrieve - count = 10 - - # only games played during this range are considered - right_now = datetime.utcnow() - cutoff_dt = right_now - timedelta(days=cutoff_days) - - top_players_q = top_players_by_time_q(cutoff_days) + q = DBSession.query(ActivePlayer) - top_players = top_players_q.limit(count).all() + if start is not None: + q = q.filter(ActivePlayer.sort_order >= start) - top_players = [(player_id, html_colors(nick), score) \ - for (player_id, nick, score) in top_players] + q = q.order_by(ActivePlayer.sort_order) - return top_players + if limit is not None: + q = q.limit(limit) - -def top_servers_by_players_q(cutoff_days): - """ - Query to get the top servers by the amount of players active - during a date range. - - Games older than cutoff_days days old are ignored. - """ - # only games played during this range are considered - right_now = datetime.utcnow() - cutoff_dt = right_now - timedelta(days=cutoff_days) - - top_servers_q = DBSession.query(Server.server_id, Server.name, - func.count()).\ - filter(Game.server_id==Server.server_id).\ - filter(expr.between(Game.create_dt, cutoff_dt, right_now)).\ - order_by(expr.desc(func.count(Game.game_id))).\ - group_by(Server.server_id).\ - group_by(Server.name) - - return top_servers_q + return q.all() @cache_region('hourly_term') -def get_top_servers_by_players(cutoff_days): +def get_top_servers_by_games(limit=None, start=None): """ - The top servers by the amount of players active during a date range. - - Games older than cutoff_days days old are ignored. + The top servers by the number of games played during a date range. """ - # how many to retrieve - count = 10 - - top_servers = top_servers_by_players_q(cutoff_days).limit(count).all() - - return top_servers + q = DBSession.query(ActiveServer) + if start is not None: + q = q.filter(ActiveServer.sort_order >= start) -def top_maps_by_times_played_q(cutoff_days): - """ - Query to retrieve the top maps by the amount of times it was played - during a date range. - - Games older than cutoff_days days old are ignored. - """ - # only games played during this range are considered - right_now = datetime.utcnow() - cutoff_dt = right_now - timedelta(days=cutoff_days) + q = q.order_by(ActiveServer.sort_order) - top_maps_q = DBSession.query(Game.map_id, Map.name, - func.count()).\ - filter(Map.map_id==Game.map_id).\ - filter(expr.between(Game.create_dt, cutoff_dt, right_now)).\ - order_by(expr.desc(func.count())).\ - group_by(Game.map_id).\ - group_by(Map.name) + if limit is not None: + q = q.limit(limit) - return top_maps_q + return q.all() @cache_region('hourly_term') -def get_top_maps_by_times_played(cutoff_days): +def get_top_maps_by_games(limit=None, start=None): """ - The top maps by the amount of times it was played during a date range. - - Games older than cutoff_days days old are ignored. + The top maps by the number of games played during a date range. """ - # how many to retrieve - count = 10 + q = DBSession.query(ActiveMap) + + if start is not None: + q = q.filter(ActiveMap.sort_order >= start) + + q = q.order_by(ActiveMap.sort_order) - top_maps = top_maps_by_times_played_q(cutoff_days).limit(count).all() + if limit is not None: + q = q.limit(limit) - return top_maps + return q.all() def _main_index_data(request): @@ -258,13 +191,13 @@ def _main_index_data(request): back_then = datetime.utcnow() - timedelta(days=leaderboard_lifetime) # top players by playing time - top_players = get_top_players_by_time(leaderboard_lifetime) + top_players = get_top_players_by_time(10) - # top servers by number of total players played - top_servers = get_top_servers_by_players(leaderboard_lifetime) + # top servers by number of games + top_servers = get_top_servers_by_games(10) # top maps by total times played - top_maps = get_top_maps_by_times_played(leaderboard_lifetime) + top_maps = get_top_maps_by_games(10) # recent games played in descending order rgs = recent_games_q(cutoff=back_then).limit(recent_games_count).all() @@ -284,22 +217,7 @@ def main_index(request): """ Display the main page information. """ - mainindex_data = _main_index_data(request) - - # FIXME: code clone, should get these from _main_index_data - leaderboard_count = 10 - recent_games_count = 20 - - for i in range(leaderboard_count-len(mainindex_data['top_players'])): - mainindex_data['top_players'].append(('-', '-', '-')) - - for i in range(leaderboard_count-len(mainindex_data['top_servers'])): - mainindex_data['top_servers'].append(('-', '-', '-')) - - for i in range(leaderboard_count-len(mainindex_data['top_maps'])): - mainindex_data['top_maps'].append(('-', '-', '-')) - - return mainindex_data + return _main_index_data(request) def main_index_json(request): @@ -309,43 +227,61 @@ def main_index_json(request): return [{'status':'not implemented'}] -def top_players_by_time(request): - current_page = request.params.get('page', 1) - - cutoff_days = int(request.registry.settings.\ - get('xonstat.leaderboard_lifetime', 30)) - - top_players_q = top_players_by_time_q(cutoff_days) - - top_players = Page(top_players_q, current_page, items_per_page=25, url=page_url) - - top_players.items = [(player_id, html_colors(nick), score) \ - for (player_id, nick, score) in top_players.items] +def top_players_index(request): + try: + start = int(request.params.get('start', None)) + except: + start = None - return {'top_players':top_players} + top_players = get_top_players_by_time(20, start) + # building a query string + query = {} + if len(top_players) > 1: + query['start'] = top_players[-1].sort_order + 1 -def top_servers_by_players(request): - current_page = request.params.get('page', 1) + return { + 'top_players':top_players, + 'query':query, + 'start':start, + } - cutoff_days = int(request.registry.settings.\ - get('xonstat.leaderboard_lifetime', 30)) - top_servers_q = top_servers_by_players_q(cutoff_days) +def top_servers_index(request): + try: + start = int(request.params.get('start', None)) + except: + start = None - top_servers = Page(top_servers_q, current_page, items_per_page=25, url=page_url) + top_servers = get_top_servers_by_games(20, start) - return {'top_servers':top_servers} + # building a query string + query = {} + if len(top_servers) > 1: + query['start'] = top_servers[-1].sort_order + 1 + return { + 'top_servers':top_servers, + 'query':query, + 'start':start, + } -def top_maps_by_times_played(request): - current_page = request.params.get('page', 1) - cutoff_days = int(request.registry.settings.\ - get('xonstat.leaderboard_lifetime', 30)) +def top_maps_index(request): + try: + start = int(request.params.get('start', None)) + except: + start = None - top_maps_q = top_maps_by_times_played_q(cutoff_days) + top_maps = get_top_maps_by_games(20, start) - top_maps = Page(top_maps_q, current_page, items_per_page=25, url=page_url) + # building a query string + query = {} + if len(top_maps) > 1: + query['start'] = top_maps[-1].sort_order + 1 - return {'top_maps':top_maps} + return { + 'top_maps':top_maps, + 'query':query, + 'start':start, + }