Added protobuf stats. (#329)

* Added protobuf stats.

* Update ChartData.ts

* Added tests for proto change

* Fixed tests.

Added mocks

* fix bug.

* fix imports

* Moved everything to use one class for pathing

* Address comments

Fix import error

* Fixed issues with tests

* imports

* Fixed all imports!

* fixed bug with tests.

Author: dtracers

backend/blueprints/api.py

@@ -2,7 +2,6 @@
 import os
 from functools import wraps
 
-from carball.analysis.utils import proto_manager
 from flask import render_template, url_for, redirect, request, jsonify, send_from_directory, Blueprint, current_app, \
     Response
 from google.protobuf.json_format import MessageToJson
@@ -12,9 +11,8 @@
 from backend.blueprints.spa_api.service_layers.utils import with_session
 from backend.database.objects import Game, PlayerGame
 from backend.database.wrapper.query_filter_builder import QueryFilterBuilder
-from backend.utils.cloud_handler import download_proto
 from backend.utils.psyonix_api_handler import get_rank, tier_div_to_string
-from backend.database.utils.file_manager import parsed_directory, get_proto_path, get_replay_path
+from backend.utils.file_manager import FileManager
 
 bp = Blueprint('apiv1', __name__, url_prefix='/api/v1')
 
@@ -153,52 +151,44 @@ def api_v1_get_ranks():
 def api_v1_get_stats(session=None):
     # TODO: stats?
     ct = session.query(Game).count()
-    dct = len([f for f in os.listdir(current_app.config[parsed_directory]) if f.endswith('pts')])
+    dct = len([f for f in os.listdir(FileManager.get_default_parse_folder()) if f.endswith('pts')])
     return jsonify({'db_count': ct, 'count': dct})
 
 
 @bp.route('/replay/<id_>')
 @key_required
 def api_v1_get_replay_info(id_):
-    pickle_path = get_proto_path(current_app, id_)
-    if not os.path.isfile(pickle_path):
-        g = download_proto(id_)
-    else:
-        try:
-            with open(pickle_path, 'rb') as f:
-                g = proto_manager.ProtobufManager.read_proto_out_from_file(f)
-        except Exception as e:
-            return jsonify({'error': 'Error opening game: ' + str(e)})
+    proto = FileManager.get_proto(id_)
 
     response = Response(
-        response=convert_proto_to_json(g),
+        response=convert_proto_to_json(proto),
         status=200,
         mimetype='application/json'
     )
 
     return response
 
     game = session.query(Game).filter(Game.hash == id_).first()
-    data = {'datetime': g.datetime, 'map': g.map, 'mmrs': game.mmrs, 'ranks': game.ranks, 'name': g.name,
-            'hash': game.hash, 'version': g.replay_version, 'id': g.id, 'frames': len(g.frames)}
-    data = player_interface(data, g, game.mmrs, game.ranks)
-    data = team_interface(data, g)
-    data = score_interface(data, g)
-    data = goals_interface(data, g)
+    data = {'datetime': proto.datetime, 'map': proto.map, 'mmrs': game.mmrs, 'ranks': game.ranks, 'name': proto.name,
+            'hash': game.hash, 'version': proto.replay_version, 'id': proto.id, 'frames': len(proto.frames)}
+    data = player_interface(data, proto, game.mmrs, game.ranks)
+    data = team_interface(data, proto)
+    data = score_interface(data, proto)
+    data = goals_interface(data, proto)
     return jsonify(data)
 
 
 @bp.route('/parsed/list')
 @key_required
 def api_v1_list_parsed_replays():
-    fs = os.listdir(current_app.config[parsed_directory])
+    fs = os.listdir(FileManager.get_default_parse_folder())
     return jsonify(fs)
 
 
 @bp.route('/parsed/<path:fn>')
 @key_required
 def api_v1_download_parsed(fn):
-    return send_from_directory(current_app.config[parsed_directory], fn, as_attachment=True)
+    return send_from_directory(FileManager.get_default_parse_folder(), fn, as_attachment=True)
 
 
 @bp.route('/rank/<id_>')

backend/blueprints/spa_api/service_layers/homepage/twitch.py

@@ -39,7 +39,10 @@ def __init__(self, streams: List[Stream]):
 
     @classmethod
     def create(cls):
-        streams = cls.get_streams()
+        try:
+            streams = cls.get_streams()
+        except KeyError as e:
+            return cls([])
         objs = []
         for stream in streams:
             objs.append(Stream(stream['user_name'], game='Rocket League', title=stream['title'],

backend/blueprints/spa_api/service_layers/replay/basic_stats.py

@@ -1,11 +1,10 @@
 from typing import List
 
-from flask import current_app
-
-from backend.database.wrapper.chart.player_chart_metadata import player_stats_metadata
+from backend.database.wrapper.chart.player_chart_metadata import player_group_stats_metadata
 from backend.database.wrapper.chart.stat_point import OutputChartData
 from backend.database.wrapper.chart.team_chart_metadata import team_stats_metadata
 from backend.database.wrapper.stats.chart_stats_wrapper import ChartStatsWrapper
+from backend.database.wrapper.stats.shared_stats_wrapper import SharedStatsWrapper
 
 wrapper = ChartStatsWrapper()
 
@@ -14,11 +13,12 @@ class PlayerStatsChart:
     @staticmethod
     def create_from_id(id_: str) -> List[OutputChartData]:
         wrapped_player_games = wrapper.get_chart_stats_for_player(id_)
-        return wrapper.wrap_chart_stats(wrapped_player_games, player_stats_metadata)
+        protobuf_stats = wrapper.get_protobuf_stats(id_)
+        all_basic_stats = SharedStatsWrapper.merge_stats(wrapped_player_games, protobuf_stats)
+        return wrapper.wrap_chart_stats(all_basic_stats, player_group_stats_metadata)
 
 
 class TeamStatsChart:
     @staticmethod
     def create_from_id(id_: str) -> List[OutputChartData]:
-        wrapped_team_games = wrapper.get_chart_stats_for_team(id_)
-        return wrapper.wrap_chart_stats(wrapped_team_games, team_stats_metadata)
+        return wrapper.wrap_chart_stats(wrapper.get_chart_stats_for_team(id_), team_stats_metadata)

backend/blueprints/spa_api/service_layers/replay/groups.py

@@ -2,8 +2,6 @@
 
 from typing import List
 
-from flask import current_app
-
 from backend.database.wrapper import player_wrapper
 from backend.database.wrapper.chart.chart_data import ChartData, ChartDataPoint
 from backend.database.wrapper.chart.player_chart_metadata import player_group_stats_metadata

backend/blueprints/spa_api/service_layers/replay/heatmaps.py

@@ -2,17 +2,16 @@
 
 import numpy as np
 from carball.generated.api.game_pb2 import Game
-from flask import current_app
 
-from backend.database.utils.file_manager import get_pandas, get_proto
+from backend.utils.file_manager import FileManager
 
 
 class ReplayHeatmaps:
 
     @staticmethod
     def create_from_id(id_: str, type_="positioning") -> 'ReplayHeatmaps':
-        data_frame = get_pandas(current_app, id_)
-        protobuf_game = get_proto(current_app, id_)
+        data_frame = FileManager.get_pandas(id_)
+        protobuf_game = FileManager.get_proto(id_)
         # output = generate_heatmaps(data_frame, protobuf_game, type="hits")
         width = 400 / 500
         step = 350.0

backend/blueprints/spa_api/service_layers/replay/replay_positions.py

@@ -1,8 +1,6 @@
 from typing import List
 
-from flask import current_app
-
-from backend.database.utils.file_manager import get_pandas, get_proto
+from backend.utils.file_manager import FileManager
 from backend.blueprints.spa_api.service_layers.replay.replay_player import ReplayPlayer
 
 
@@ -20,8 +18,8 @@ def __init__(self, id_: str,
 
     @staticmethod
     def create_from_id(id_: str) -> 'ReplayPositions':
-        data_frame = get_pandas(current_app, id_)
-        protobuf_game = get_proto(current_app, id_)
+        data_frame = FileManager.get_pandas(id_)
+        protobuf_game = FileManager.get_proto(id_)
 
         cs = ['pos_x', 'pos_y', 'pos_z']
         rot_cs = ['rot_x', 'rot_y', 'rot_z']

backend/blueprints/spa_api/spa_api.py

@@ -19,18 +19,15 @@
 from backend.blueprints.spa_api.service_layers.replay.heatmaps import ReplayHeatmaps
 from backend.blueprints.spa_api.service_layers.replay.predicted_ranks import PredictedRank
 from backend.blueprints.spa_api.service_layers.replay.visibility import ReplayVisibility
-from backend.blueprints.spa_api.service_layers.replay.visualizations import Visualizations
 from backend.blueprints.spa_api.utils.query_param_definitions import upload_file_query_params, \
     replay_search_query_params, progression_query_params, playstyle_query_params, visibility_params, convert_to_enum
 from backend.database.startup import lazy_get_redis
-from backend.database.utils.file_manager import get_replay_path
 from backend.tasks.add_replay import create_replay_task, parsed_replay_processing
 from backend.utils.checks import log_error
 from backend.utils.global_functions import get_current_user_id
 from backend.blueprints.spa_api.service_layers.replay.visualizations import Visualizations
-from backend.database.utils.file_manager import get_replay_path
 from backend.tasks.update import update_self
-
+from backend.utils.file_manager import FileManager
 
 try:
     import config
@@ -306,7 +303,7 @@ def api_download_group():
 @bp.route('/replay/<id_>/download')
 def download_replay(id_):
     filename = id_ + ".replay"
-    path = get_replay_path(current_app, id_)
+    path = FileManager.get_replay_path(id_)
     if os.path.isfile(path):
         return send_from_directory(current_app.config['REPLAY_DIR'], filename, as_attachment=True)
     elif config is not None and hasattr(config, 'GCP_BUCKET_URL'):

backend/database/utils/dynamic_field_manager.py

@@ -7,19 +7,21 @@
 
 
 class DynamicFieldResult:
-    def __init__(self, field_name):
+    def __init__(self, field_name: str):
         self.field_name = field_name
 
 
 class ProtoFieldResult(DynamicFieldResult):
-    def __init__(self, nested_parents, field_name, field_descriptor, nested_names=None):
+    def __init__(self, nested_parents: List[str], field_name: str, field_descriptor, nested_names: List[str] = None):
         super().__init__(field_name)
         self.field_descriptor = field_descriptor
         self.nested_parents = nested_parents
         self.nested_names = nested_names
 
 
-def get_proto_fields_as_flatten_list(proto_message: message, nested_parents=None, nested_fields=None) -> List[ProtoFieldResult]:
+def get_proto_fields_as_flatten_list(proto_message: message,
+                                     nested_parents: List[str] = None,
+                                     nested_fields: List[str] = None) -> List[ProtoFieldResult]:
     """
     Gets the result of a flatten version of all fields in the protobuf object.
     :param proto_message:  A class representing what we want to get the fields from.  It is not an instance.
@@ -53,12 +55,14 @@ def filter_proto_fields(proto_field_list: List[ProtoFieldResult], blacklist_fiel
     result_list = []
     for item in proto_field_list:
         allowed = True
-        for name in blacklist_field_names:
-            if name == item.field_name:
-                allowed = False
-        for message in blacklist_message_types:
-            if message in item.nested_parents:
-                allowed = False
+        if blacklist_field_names is not None:
+            for name in blacklist_field_names:
+                if name == item.field_name:
+                    allowed = False
+        if blacklist_message_types is not None:
+            for message in blacklist_message_types:
+                if message in item.nested_parents:
+                    allowed = False
         if allowed:
             result_list.append(item)
     return result_list
@@ -73,22 +77,58 @@ def get_db_proto_union(proto_field_list: List[ProtoFieldResult], db_object: DBOb
     return result_list
 
 
-def create_and_filter_proto_field(proto_message: message, blacklist_field_names: List[str],
-                                  blacklist_message_types: List[str], db_object: DBObjectBase) -> List[ProtoFieldResult]:
+def get_whitelist_proto_union(proto_field_list: List[ProtoFieldResult],
+                              whitelist_field_name: [str],
+                              whitelist_message_type: [str]) -> List[ProtoFieldResult]:
+    result_list = []
+    allowed = False
+    for item in proto_field_list:
+        if whitelist_field_name is not None and item.field_name in whitelist_field_name:
+            allowed = True
+        if whitelist_message_type is not None:
+            for field_type in whitelist_message_type:
+                if field_type in item.nested_parents:
+                    allowed = True
+        if allowed:
+            result_list.append(item)
+    return result_list
+
+
+def create_and_filter_proto_field(proto_message: message,
+                                  blacklist_field_names: List[str]=None,
+                                  blacklist_message_types: List[str]=None,
+                                  db_object: DBObjectBase=None,
+                                  whitelist_field_names: List[str]=None,
+                                  whitelist_message_types: List[str]=None) -> List[ProtoFieldResult]:
     """
     Creates a flatten list of the union of protobuf and db objects.
+
+    First this limits to just the whitelist if it exists
+    Then filters out the blacklist
+    Then limits to what exists in the db if that exists.
     :param proto_message: The protobuf class not an object we are grabbing fields from.
     :param blacklist_field_names: Fields we do not want included.
     :param blacklist_message_types: Message types we do not want included.
     :param db_object: The database object that is being unioned with the protobuf
+    :param whitelist_field_names A list of field names that are the only ones allowed.
+    :param whitelist_message_types A list of field types that are the only ones allowed.
     :return: A list of fields that can be used to go between protobuf and the database.
     """
-    list = get_proto_fields_as_flatten_list(proto_message)
-    list = filter_proto_fields(list, blacklist_field_names, blacklist_message_types)
-    return get_db_proto_union(list, db_object)
+    field_list = get_proto_fields_as_flatten_list(proto_message)
+
+    if whitelist_field_names is not None or whitelist_message_types is not None:
+        field_list = get_whitelist_proto_union(field_list, whitelist_field_names, whitelist_message_types)
+
+    if blacklist_field_names is not None or blacklist_message_types is not None:
+        field_list = filter_proto_fields(field_list, blacklist_field_names, blacklist_message_types)
+
+    if db_object is not None:
+        field_list = get_db_proto_union(field_list, db_object)
+
+    return field_list
 
 
-def get_proto_values(proto_object, fields: List[ProtoFieldResult]):
+def get_proto_values(proto_object, fields: List[ProtoFieldResult]) -> List[any]:
     """
     Returns all proto values specified by a list of ProtoFieldResult
     :param proto_object:

backend/database/utils/file_manager.py

@@ -31,7 +31,8 @@ class ChartSubcatagory(Enum):
 
 
 class ChartStatsMetadata:
-    def __init__(self, stat_name: str, type_: ChartType, subcategory: ChartSubcatagory):
+    def __init__(self, stat_name: str, type_: ChartType, subcategory: ChartSubcatagory, is_protobuf=False):
+        self.is_protobuf = is_protobuf
         self.stat_name = stat_name
         self.type = type_.name
         self.subcategory = subcategory.name.replace('_', ' ')