Add supports for OpenPAYGO Metrics

Author: benmod

README.md

@@ -192,14 +192,14 @@ metrics_request = MetricsRequestHandler(
       auth_method=AuthMethod.RECURSIVE_DATA_AUTH
 )
 
-metrics_requestset_timestamp(1611583070)
-metrics_requestset_data({
+metrics_request.set_timestamp(1611583070)
+metrics_request.set_data({
   "token_count": 3,
   "tampered": False,
   "firmware_version": "1.2.3"
 })
 # Here we assume that the data we send is separated by 60 seconds as per the data format
-metrics_requestset_historical_data([
+metrics_request.set_historical_data([
   {
       "panel_voltage": 12.31,
       "battery_voltage": 12.32,
@@ -213,7 +213,7 @@ metrics_requestset_historical_data([
       "battery_current": -1.21,
   }
 ])
-payload = metrics_requestget_condensed_request_payload()
+payload = metrics_request.get_condensed_request_payload()
 
 # We can now proceed to send the payload to the URL
 # It looks something like `{"sn":"aaa111222","df":1234,"ts":1611583070,"d":[3,false,"1.2.3"],"hd":[[12.31,12.32,1.23,-1.23],[12.3,12.31,1.22,-1.21]],"a":"raa5cb1fda302cf94e"}`
@@ -232,39 +232,44 @@ You can use the `MetricsResponseHandler` object to process your OpenPAYGO Metric
 - `metrics_payload` (required): The OpenPAYGO Metrics payload, as a string containing the JSON payload. 
 - `secret_key` (optional): The secret key provided as a string containing 32 hexadecimal characters (e.g. `dac86b1a29ab82edc5fbbc41ec9530f6`)
 - `data_format` (optional): The data format, provided as dictionnary matching the data format object specifications. 
+- `last_request_count` (optional): The request count of the last valid request (used for avoiding request replay)
+- `last_request_timestamp` (optional): The timestamp of the last valid request (used for avoiding request replay)
 
 It provides the following methods:
 - `get_device_serial()`: Returns the serial number of the device as a string. 
-- `set_device_parameters(secret_key, data_format)`: Used to set the device data required for proper processing of the request in the handler if it was not set initially, which is often the case as the serial number is usually required to fetch that data. It will return `ValueError` if either of the parameters is invalid. 
-- `is_auth_valid()`: Returns `true` if the authentication provided is valid or `false` if not. 
+- `set_device_parameters(secret_key, data_format, last_request_count, last_request_timestamp)`: Used to set the device data required for proper processing of the request in the handler if it was not set initially, which is often the case as the serial number is usually required to fetch that data. It will return `ValueError` if either of the parameters is invalid. 
+- `is_auth_valid()`: Returns `true` if the authentication provided is valid or `false` if not. Note that it checks both that the signature is valid and that the `request_count` or `timestamp` are more recent than the one provided in the device parameters. 
 - `get_simple_metrics()`: Returns the metrics provided in the simple expanded format. It will also convert relative timestamps into explicit timestamps for easier processing. 
 - `expects_token_answer()`: Return `true` if the payload requested tokens in the answer. You can set the tokens to be returned by calling `add_tokens_to_answer(token_list)` with `token_list` being a list of token strings. 
 - `expects_time_answer()`: Return `true` if the payload requested either relative time or absolute time in the answer. You can set the time to be returned by calling `add_time_to_answer(target_datetime)` with `target_datetime` being a datetime object. The function will automatically provide it in the correct format based on the request.  
 - `add_settings_to_answer(settings_dict)`: Will add the provided settings dictionnary to the answer. 
 - `add_extra_data_to_answer(extra_data_dict)`: Will add the provided extra data dictionnary to the answer. 
-- `get_answer()`: Will return the answer as a string based on the request and the data added to answer, it will automatically handle the authentication and fomatting. 
+- `add_new_base_url_to_answer(new_base_url)`: Will tell the device to change the base URL to send the data to. 
+- `get_answer_payload()`: Will return the answer as a string based on the request and the data added to answer, it will automatically handle the authentication and fomatting. 
 
 
 **Example - Full Request flow from server side:**
 
 ```python
-from openpaygo import MetricsHandler
-from my_db_service import get_device, store_metric
+from openpaygo import MetricsResponseHandler
+from my_db_service import get_device, get_data_format, store_metric
 
 
 @app.route('/dd')
 def device_data():
   # We load the metrics
   try:
-    metrics = MetricsHandler(request.data)
+    metrics = MetricsResponseHandler(request.data)
   except ValueError as e:
     return {'error': 'Invalid data format'}, 400
   # We get the serial number and load the device data from our storage
   device = get_device(serial=metrics.get_device_serial())
+  # We get the data format if needed from our storage
+  data_format = get_data_format(id=metrics.get_data_format_id()) if not metrics.data_format_available() else None
   # We set the device parameters in the metrics handler
   metrics.set_device_parameters(
     secret_key=device.secret_key,
-    data_format=device.data_format
+    data_format=data_format
   )
   # We check the authentication
   if not metrics.is_auth_valid():
@@ -274,6 +279,7 @@ def device_data():
   # We store the metrics in our database
   for metric_data in simple_data.get('data'):
     store_metric(name=metric_data['name'], value=metric_data['value'])
+  # Here the handler automatically computed the timestamp for each step
   for time_step in simple_data.get('historical_data'):
     for historical_metric_data in time_step:
       store_metric(name=metric_data['name'], value=metric_data['value'], time=time_step['timestamp'])
@@ -285,16 +291,19 @@ def device_data():
   # We can add extra data
   metrics.add_settings_to_answer({'language': 'fr-FR'})
   # The handler handles the signature, etc.
-  return metrics.get_answer(), 200
+  return metrics.get_answer_payload(), 200
 ```
 
 
 ## Changelog
 
 ### 2023-10-09 - v0.3.0
 - Fix token generation issue
+- Add support for OpenPAYGO Metrics Request Generation
+- Add support for OpenPAYGO Metrics Request Decoding
+- Add support for OpenPAYGO Metrics Response Generation
 
 ### 2023-10-03 - v0.2.0
 - First working version published on PyPI
 - Has support for OpenPAYGO Token
-- Has working CI for pushing to PyPI
+- Has working CI for pushing to PyPI

openpaygo/__init__.py

@@ -11,4 +11,4 @@ def generate_token(**kwargs):
 
 
 def decode_token(**kwargs):
-    return OpenPAYGOTokenDecoder.decode_token(**kwargs)
+    return OpenPAYGOTokenDecoder.decode_token(**kwargs)

openpaygo/metrics_request.py

@@ -54,13 +54,19 @@ def get_condensed_request_payload(self):
 
     def get_condensed_request_dict(self):
         if not self.data_format:
-            raise ValueError('No data format provided for condensed request')
-        condensed_request = self.request_dict
+            raise ValueError('No Data Format provided for condensed request')
+        data_order = self.data_format.get('data_order')
+        if self.data and not data_order:
+            raise ValueError('Data Format does not contain data_order')
+        historical_data_order = self.data_format.get('historical_data_order')
+        if self.historical_data and not historical_data_order:
+            raise ValueError('Data Format does not contain historical_data_order')
+        condensed_request = copy.deepcopy(self.request_dict)
         condensed_request['data'] = []
         condensed_request['historical_data'] = []
         # We add the data
         data_copy = copy.deepcopy(self.data)
-        for var in self.data_format.get('data_order'):
+        for var in data_order:
             condensed_request['data'].append(data_copy.pop(var) if var in data_copy else None)
         if len(data_copy) > 0:
             raise ValueError('Additional variables not present in the data format: '+str(data_copy))
@@ -69,7 +75,7 @@ def get_condensed_request_dict(self):
         historical_data_copy = copy.deepcopy(self.historical_data)
         for time_step in historical_data_copy:
             time_step_data = []
-            for var in self.data_format.get('historical_data_order'):
+            for var in historical_data_order:
                 time_step_data.append(time_step.pop(var) if var in time_step else None)
             if len(time_step) > 0:
                 raise ValueError('Additional variables not present in the historical data format: '+str(time_step))

openpaygo/metrics_response.py

@@ -1,49 +1,184 @@
 import json
-
+from .metrics_shared import OpenPAYGOMetricsShared
+import copy
+from datetime import datetime, timedelta
 
 class MetricsResponseHandler(object):
 
-    def __init__(self, received_metrics, data_format=None, secret_key=None):
+    def __init__(self, received_metrics, data_format=None, secret_key=None, last_request_count=None, last_request_timestamp=None):
         self.received_metrics = received_metrics
-        self.metrics_dict = json.loads(received_metrics)
-        self.answer_dict = {}
+        self.request_dict = json.loads(received_metrics)
+        # We convert the base variable names to simple
+        self.request_dict = OpenPAYGOMetricsShared.convert_dict_keys_to_simple(self.request_dict)
+        # We add the reception timestamp if not timestamp was provided
+        if not self.request_dict.get('timestamp'):
+            self.timestamp = int(datetime.now().timestamp())
+        else:
+            self.timestamp = self.request_dict.get('timestamp')
+        self.response_dict = {}
+        self.secret_key = secret_key
+        self.data_format = data_format
+        self.last_request_count = last_request_count
+        self.last_request_timestamp = last_request_timestamp
+        if not self.data_format and self.request_dict.get('data_format'):
+            self.data_format = self.request_dict.get('data_format')
 
     def get_device_serial(self):
-        return self.received_metrics.get('sn', self.received_metrics.get('serial_number'))
+        return self.request_dict.get('serial_number')
+    
+    def get_data_format_id(self):
+        return self.request_dict.get('data_format_id')
+    
+    def data_format_available(self):
+        return self.data_format != None
 
     def set_device_parameters(self, secret_key, data_format):
-        pass
+        self.secret_key = secret_key
+        self.data_format = data_format
 
     def is_auth_valid(self):
-        pass
+        auth_string = self.request_dict.get('auth', None)
+        if not auth_string:
+            return True
+        elif not self.secret_key:
+            raise ValueError('Secret key is required to check the auth.')
+        self.auth_method = auth_string[:2]
+        new_signature = OpenPAYGOMetricsShared.generate_request_signature_from_data(self.request_dict, self.auth_method, self.secret_key)
+        if auth_string == new_signature:
+            request_count = self.request_dict.get('request_count')
+            if request_count and self.last_request_count and request_count <= self.last_request_count:
+                return False
+            timestamp = self.request_dict.get('timestamp')
+            if timestamp and self.last_request_timestamp and timestamp <= self.last_request_timestamp:
+                return False
+            return True
+        return False
 
     def get_simple_metrics(self):
-        pass
+        # We start the process by making a copy of the dict to work with
+        simple_dict = copy.deepcopy(self.request_dict)
+        simple_dict.pop('auth') if 'auth' in simple_dict else None # We remove the auth
+        # We process the data and replace it
+        simple_dict['data'] = self._get_simple_data()
+        # We process the historical data
+        simple_dict['historical_data'] = self._get_simple_historical_data()
+        # We fill in the timestamps for each time step
+        simple_dict['historical_data'] = self._fill_timestamp_in_historical_data(simple_dict['historical_data'])
+        return simple_dict
 
     def expects_token_answer(self):
-        pass
+        data = self._get_simple_data()
+        return data.get('token_count') is not None
 
     def add_tokens_to_answer(self, token_list):
-        pass
+        self.response_dict['token_list'] = token_list
 
     def expects_time_answer(self):
-        pass
+        data = self._get_simple_data()
+        if data.get('active_until_timestamp_requested', False) or data.get('active_seconds_left_requested', False):
+            return True
+        return False
 
     def add_time_to_answer(self, target_datetime):
-        pass
+        data = self._get_simple_data()
+        if data.get('active_until_timestamp_requested', False):
+            self.response_dict['active_until_timestamp'] = target_datetime.timestamp()
+        elif data.get('active_seconds_left_requested', False):
+            self.response_dict['active_seconds_left'] = (datetime.now() - target_datetime).total_seconds()
+        else:
+            raise ValueError('No time requested')
+        
+    def add_new_base_url_to_answer(self, new_base_url):
+        self.add_settings_to_answer({'base_url': new_base_url})
 
     def add_settings_to_answer(self, settings_dict):
-        pass
+        if not self.response_dict.get('settings'):
+            self.response_dict['settings'] = {}
+        self.response_dict['settings'].update(settings_dict)
 
     def add_extra_data_to_answer(self, extra_data_dict):
-        pass
+        if not self.response_dict.get('extra_data'):
+            self.response_dict['extra_data'] = {}
+        self.response_dict['extra_data'].update(extra_data_dict)
 
-    def get_answer(self):
-        pass
+    def get_answer_payload(self):
+        payload = self.get_answer_dict()
+        return OpenPAYGOMetricsShared.convert_to_metrics_json(payload)
 
-    def _check_data_auth(self):
-        data = self.metrics_dict.get('data', None)
-        data_string = json.dumps(data, separators=(',', ':')) if data else ''
-        historical_data = self.metrics_dict.get('historical_data', None)
-        historical_data_string = json.dumps(historical_data, separators=(',', ':')) if historical_data else ''
+    def get_answer_dict(self):
+        # If there is not data format, we just return the full response
+        condensed_answer = copy.deepcopy(self.response_dict)
+        condensed_answer['auth'] = OpenPAYGOMetricsShared.generate_response_signature_from_data(
+            serial_number=self.request_dict.get('serial_number'),
+            request_count=self.request_dict.get('request_count'),
+            data=condensed_answer,
+            timestamp=self.request_dict.get('timestamp'),
+            secret_key=self.secret_key
+        )
+        return OpenPAYGOMetricsShared.convert_dict_keys_to_condensed(condensed_answer)
 
+    def _get_simple_data(self):
+        data = copy.deepcopy(self.request_dict.get('data'))
+        # If no data or not condensed in list, we just return it
+        if not data:
+            return {}
+        if not isinstance(data, list):
+            return data
+        data_order = self.data_format.get('data_order')
+        if not data_order:
+            raise ValueError('Data Format does not contain data_order')
+        clean_data = {}
+        data_len = len(data)
+        for idx, var in enumerate(data_order):
+            clean_data[var] = data[idx] if idx < data_len else None
+        data = data[data_len:]
+        if len(data) > 0:
+            raise ValueError('Additional variables not present in the data format: '+str(data))
+        return OpenPAYGOMetricsShared.convert_dict_keys_to_simple(clean_data)
+    
+    def _get_simple_historical_data(self):
+        historical_data = copy.deepcopy(self.request_dict.get('historical_data'))
+        if not historical_data:
+            return []
+        historical_data_order = self.data_format.get('historical_data_order')
+        clean_historical_data = []
+        for time_step in historical_data:
+            time_step_data = {}
+            if isinstance(time_step, list):
+                if not historical_data_order:
+                    raise ValueError('Data Format does not contain historical_data_order')
+                timse_step_len = len(time_step)
+                for idx, var in enumerate(historical_data_order):
+                    if idx < timse_step_len:
+                        time_step_data[var] = time_step[idx]
+                time_step = time_step[timse_step_len:]
+                if len(time_step) > 0:
+                    raise ValueError('Additional variables not present in the historical data format: '+str(time_step))
+            elif isinstance(time_step, dict):
+                for key in time_step:
+                    if key.isdigit() and int(key) < len(historical_data_order):
+                        time_step_data[historical_data_order[int(key)]] = time_step[key]
+                    else:
+                        time_step_data[key] = time_step[key]
+            else:
+                raise ValueError('Invalid historical data step type: '+str(time_step))
+            clean_historical_data.append(time_step_data)
+        return clean_historical_data
+    
+    def _fill_timestamp_in_historical_data(self, historical_data):
+        if self.request_dict.get('data_collection_timestamp'):
+            last_timestamp = datetime.fromtimestamp(self.request_dict.get('data_collection_timestamp'))
+        else:
+            last_timestamp = datetime.fromtimestamp(self.timestamp)
+        for idx, time_step in enumerate(historical_data):
+            if time_step.get('relative_time') is not None:
+                last_timestamp = last_timestamp + timedelta(seconds=int(time_step.get('relative_time')))
+                historical_data[idx]['timestamp'] = int(last_timestamp.timestamp())
+                del historical_data[idx]['relative_time']
+            elif time_step.get('timestamp'):
+                last_timestamp = datetime.fromtimestamp(time_step.get('timestamp'))
+            else:
+                if idx != 0:
+                    last_timestamp = last_timestamp + timedelta(seconds=int(self.data_format.get('historical_data_interval')))
+                historical_data[idx]['timestamp'] = int(last_timestamp.timestamp())
+        return historical_data