Merge branch 'fix-duplicated-results' into 'enterprise'

fix(scoring): flush empty list of results and breakdown items

See merge request dkinternal/testgen/dataops-testgen!221

Author: ci bot

testgen/commands/run_refresh_score_cards_results.py

@@ -2,7 +2,7 @@
 import logging
 import time
 
-from testgen.common.models import with_database_session
+from testgen.common.models import get_current_session, with_database_session
 from testgen.common.models.scores import (
     SCORE_CATEGORIES,
     ScoreCard,
@@ -36,6 +36,7 @@ def run_refresh_score_cards_results(
         LOG.exception("CurrentStep: Stopping scorecards results refresh after unexpected error")
         return
 
+    db_session = get_current_session()
     for definition in definitions:
         LOG.info(
             "CurrentStep: Refreshing results for scorecard %s in project %s",
@@ -45,6 +46,10 @@ def run_refresh_score_cards_results(
 
         try:
             fresh_score_card = definition.as_score_card()
+            definition.results = []
+            definition.breakdown = []
+            db_session.flush([definition])
+
             definition.results = _score_card_to_results(fresh_score_card)
             definition.breakdown = _score_definition_to_results_breakdown(definition)
             if add_history_entry:

testgen/common/logs.py

@@ -33,7 +33,7 @@ def configure_logging(
 
             file_handler = ConcurrentTimedRotatingFileHandler(
                 get_log_full_path(),
-                when="D",
+                when="MIDNIGHT",
                 interval=1,
                 backupCount=int(settings.LOG_FILE_MAX_QTY),
             )

testgen/common/models/scores.py

@@ -361,17 +361,22 @@ def recalculate_scores_history(self) -> None:
         )
         overall_scores = get_current_session().execute(query).mappings().all()
         current_history: dict[tuple[datetime, str, str], ScoreDefinitionResultHistoryEntry] = {}
-        for entry in self.history:
-            current_history[(entry.last_run_time, entry.category,)] = entry
-
         renewed_history: dict[tuple[datetime, str, str], float] = {}
+
         for scores in overall_scores:
             renewed_history[(scores["last_run_time"], "score",)] = scores["score"]
             renewed_history[(scores["last_run_time"], "cde_score",)] = scores["cde_score"]
 
+        for entry in self.history:
+            entry_key = (entry.last_run_time, entry.category,)
+            if entry_key in renewed_history:
+                current_history[entry_key] = entry
+
         for key, entry in current_history.items():
             entry.score = renewed_history[key]
 
+        self.history = list(current_history.values())
+
     def _get_raw_query_filters(self, cde_only: bool = False, prefix: str | None = None) -> list[str]:
         values_by_field = defaultdict(list)
         for filter_ in self.filters:

testgen/ui/views/score_explorer.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from io import BytesIO
 from typing import ClassVar
 
@@ -246,7 +247,7 @@ def save_score_definition(_) -> None:
         latest_run = max(
             profiling_queries.get_latest_run_date(project_code),
             test_run_queries.get_latest_run_date(project_code),
-            key=lambda run: getattr(run, "run_time", 0),
+            key=lambda run: getattr(run, "run_time", datetime.min),
         )
 
         refresh_kwargs = {