Merge pull request #96 from pitangainnovare/impl/include-params-days-to-go-back

Impl/include params days to go back

Author: pitangainnovare

VERSION

@@ -1 +1 @@
-1.9.1
+1.10.0

core/utils/date_utils.py

@@ -36,6 +36,11 @@ def get_date_range_str(from_date_str: str = None, until_date_str: str = None, da
     """
     Get the date range to be used in the queries.
 
+    If both from_date_str and until_date_str are provided, they will be used.
+    If only one is provided, it will be used as the start or end date, and the other will be calculated based on a 7-day range.
+    If neither is provided, the function will default to the last 7 days from today.
+    If days_to_go_back is provided, it will override the from_date_str and until_date_str.
+
     Args:
         from_date_str: Date to start the query.
         until_date_str: Date to end the query.

log_manager/tasks.py

@@ -169,14 +169,15 @@ def _fetch_validation_parameters(collection, default_buffer_size=0.1, default_sa
 
 
 @celery_app.task(bind=True, name=_('Check missing log files'))
-def task_check_missing_logs_for_date_range(self, collections=[], from_date=None, until_date=None, user_id=None, username=None):
+def task_check_missing_logs_for_date_range(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None):
     """
     Task to check for missing log files in the defined date range.
 
     Parameters:
         collections (list, optional): List of collection acronyms. Defaults to [].
         from_date (str, optional): The start date for log discovery in YYYY-MM-DD format. Defaults to None.
         until_date (str, optional): The end date for log discovery in YYYY-MM-DD format. Defaults to None.
+        days_to_go_back (int, optional): The number of days to go back from today for log discovery. Defaults to None.
         user_id (int, optional): The ID of the user initiating the task. Defaults to None.
         username (str, optional): The username of the user initiating the task. Defaults to None.
 
@@ -186,9 +187,11 @@ def task_check_missing_logs_for_date_range(self, collections=[], from_date=None,
     """
     user = _get_user(self.request, username=username, user_id=user_id)
 
+    from_date_str, until_date_str = date_utils.get_date_range_str(from_date, until_date, days_to_go_back)
+
     for col in collections or Collection.acron3_list():
         collection = Collection.objects.get(acron3=col)
-        for date in date_utils.get_date_objs_from_date_range(from_date, until_date):
+        for date in date_utils.get_date_objs_from_date_range(from_date_str, until_date_str):
             logging.info(f'Checking missings logs for collection {col} and date {date}')
             _check_missing_logs_for_date(user, collection, date)
 
@@ -214,8 +217,8 @@ def _check_missing_logs_for_date(user, collection, date):
 
 
 @celery_app.task(bind=True, name=_('Generate log files count report'))
-def task_log_files_count_status_report(self, collections=[], from_date=None, until_date=None, user_id=None, username=None):
-    from_date, until_date = date_utils.get_date_range_str(from_date, until_date)
+def task_log_files_count_status_report(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None):
+    from_date, until_date = date_utils.get_date_range_str(from_date, until_date, days_to_go_back)
 
     from_date_obj = date_utils.get_date_obj(from_date)
     until_date_obj = date_utils.get_date_obj(until_date)

metrics/tasks.py

@@ -48,21 +48,22 @@
 
 
 @celery_app.task(bind=True, name=_('Compute access'), timelimit=-1)
-def task_parse_logs(self, collections=[], from_date=None, until_date=None, user_id=None, username=None):
+def task_parse_logs(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None):
     """
     Parses log files associated with a given collection.
 
     Args:
         collections (list, optional): List of collection acronyms to parse logs for. Defaults to all collections.
         from_date (str, optional): Start date for log parsing in 'YYYY-MM-DD' format. Defaults to None.
         until_date (str, optional): End date for log parsing in 'YYYY-MM-DD' format. Defaults to None.
+        days_to_go_back (int, optional): Number of days to go back from the current date to parse logs. Defaults to None.
         user_id
         username
 
     Returns:
         None.
     """
-    from_date, until_date = get_date_range_str(from_date, until_date)
+    from_date, until_date = get_date_range_str(from_date, until_date, days_to_go_back)
 
     from_date_obj = get_date_obj(from_date)
     until_date_obj = get_date_obj(until_date)
@@ -373,7 +374,7 @@ def task_delete_documents_by_key(self, keys, values, index_name=None, user_id=No
         logging.error(f"Failed to delete documents with keys {keys} and values {values} from index {index_name}: {e}")
 
 
-@celery_app.task(bind=True, name=_('Compute metrics'), timelimit=-1)
+@celery_app.task(bind=True, name=_('Index metrics'), timelimit=-1)
 def task_index_documents(self, collections=[], from_date=None, until_date=None, days_to_go_back=None, user_id=None, username=None, bulk_size=5000, replace=False):
     """
     Task to compute and index metrics for specified collections within a given date range.