Merge branch 'luis/history_fingerprint_tests' into 'enterprise'

Initial history update/Stale_Table test

See merge request dkinternal/testgen/dataops-testgen!288

Author: ci bot

testgen/commands/queries/execute_tests_query.py

@@ -143,9 +143,11 @@ def GetTestsNonCAT(self) -> tuple[str, dict]:
             query = CleanSQL(query)
         return query, params
 
-    def AddTestRecordtoTestRunTable(self) -> tuple[str, dict]:
-        # Runs on App database
-        return self._get_query("ex_write_test_record_to_testrun_table.sql")
+    def GetHistoricThresholdUpdate(self) -> tuple[str, dict]:
+        query, params = self._get_query("ex_update_history_threshold_last_n.sql")
+        if self._use_clean:
+            query = CleanSQL(query)
+        return query, params
 
     def PushTestRunStatusUpdateSQL(self) -> tuple[str, dict]:
         # Runs on App database

testgen/commands/queries/generate_tests_query.py

@@ -2,7 +2,8 @@
 from typing import ClassVar, TypedDict
 
 from testgen.common import CleanSQL, date_service, read_template_sql_file
-from testgen.common.database.database_service import get_queries_for_command, replace_params
+from testgen.common.database.database_service import replace_params
+from testgen.common.read_file import get_template_files
 
 LOG = logging.getLogger("testgen")
 
@@ -67,11 +68,35 @@ def GetTestTypesSQL(self) -> tuple[str, dict]:
 
     def GetTestDerivationQueriesAsList(self, template_directory: str) -> list[tuple[str, dict]]:
         # Runs on App database
-        params = self._get_params()
-        queries = get_queries_for_command(template_directory, params)
-        if self._use_clean:
-            queries = [ CleanSQL(query) for query in queries ]
-        return [ (query, params) for query in queries ]
+        generic_template_directory = template_directory
+        flavor_template_directory = f"flavors.{self.sql_flavor}.{template_directory}"
+
+        query_templates = {}
+        try:
+            for query_file in get_template_files(r"^.*sql$", generic_template_directory):
+                query_templates[query_file.name] = generic_template_directory
+        except:
+            LOG.debug(
+                f"query template '{generic_template_directory}' directory does not exist",
+                exc_info=True,
+                stack_info=True,
+            )
+
+        try:
+            for query_file in get_template_files(r"^.*sql$", flavor_template_directory):
+                query_templates[query_file.name] = flavor_template_directory
+        except:
+            LOG.debug(
+                f"query template '{generic_template_directory}' directory does not exist",
+                exc_info=True,
+                stack_info=True,
+            )
+
+        queries = []
+        for filename, sub_directory in query_templates.items():
+            queries.append(self._get_query(filename, sub_directory=sub_directory))
+
+        return queries
 
     def GetTestQueriesFromGenericFile(self) -> tuple[str, dict]:
         # Runs on App database

testgen/commands/run_execute_tests.py

@@ -68,6 +68,10 @@ def run_test_queries(
     clsExecute.process_id = process_service.get_current_process_id()
 
     try:
+        # Update Historic Test Thresholds
+        LOG.info("CurrentStep: Updating Historic Test Thresholds")
+        execute_db_queries([clsExecute.GetHistoricThresholdUpdate()])
+
         # Retrieve non-CAT Queries
         LOG.info("CurrentStep: Retrieve Non-CAT Queries")
         lstTestSet = fetch_dict_from_db(*clsExecute.GetTestsNonCAT())

testgen/common/models/test_definition.py

@@ -65,6 +65,8 @@ class TestDefinitionSummary(EntityMinimal):
     match_groupby_names: str
     match_having_condition: str
     custom_query: str
+    history_calculation: str
+    history_lookback: int
     test_active: str
     test_definition_status: str
     severity: str
@@ -177,6 +179,8 @@ class TestDefinition(Entity):
     match_subset_condition: str = Column(NullIfEmptyString)
     match_groupby_names: str = Column(NullIfEmptyString)
     match_having_condition: str = Column(NullIfEmptyString)
+    history_calculation: str = Column(NullIfEmptyString)
+    history_lookback: int = Column(ZeroIfEmptyInteger, default=0)
     test_mode: str = Column(String)
     custom_query: str = Column(QueryString)
     test_active: bool = Column(YNString, default="Y")

testgen/common/models/test_run.py

@@ -44,6 +44,7 @@ class TestRunSummary(EntityMinimal):
     warning_ct: int
     failed_ct: int
     error_ct: int
+    log_ct: int
     dismissed_ct: int
     dq_score_testing: float
 
@@ -68,6 +69,7 @@ class TestRun(Entity):
     failed_ct: int = Column(Integer)
     warning_ct: int = Column(Integer)
     error_ct: int = Column(Integer)
+    log_ct: int = Column(Integer)
     table_ct: int = Column(Integer)
     column_ct: int = Column(Integer)
     column_failed_ct: int = Column(Integer)
@@ -163,6 +165,13 @@ def select_summary(
                         ELSE 0
                     END
                 ) AS error_ct,
+                SUM(
+                    CASE
+                        WHEN COALESCE(disposition, 'Confirmed') = 'Confirmed'
+                        AND result_status = 'Log' THEN 1
+                        ELSE 0
+                    END
+                ) AS log_ct,
                 SUM(
                     CASE
                         WHEN COALESCE(disposition, 'Confirmed') IN ('Dismissed', 'Inactive') THEN 1
@@ -185,6 +194,7 @@ def select_summary(
             run_results.warning_ct,
             run_results.failed_ct,
             run_results.error_ct,
+            run_results.log_ct,
             run_results.dismissed_ct,
             test_runs.dq_score_test_run AS dq_score_testing
         FROM test_runs

testgen/common/models/test_suite.py

@@ -43,6 +43,7 @@ class TestSuiteSummary(EntityMinimal):
     last_run_warning_ct: int
     last_run_failed_ct: int
     last_run_error_ct: int
+    last_run_log_ct: int
     last_run_dismissed_ct: int
 
 
@@ -122,6 +123,13 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N
                         ELSE 0
                     END
                 ) AS error_ct,
+                SUM(
+                    CASE
+                        WHEN COALESCE(test_results.disposition, 'Confirmed') = 'Confirmed'
+                        AND test_results.result_status = 'Log' THEN 1
+                        ELSE 0
+                    END
+                ) AS log_ct,
                 SUM(
                     CASE
                         WHEN COALESCE(test_results.disposition, 'Confirmed') IN ('Dismissed', 'Inactive') THEN 1
@@ -161,6 +169,7 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N
             last_run.warning_ct AS last_run_warning_ct,
             last_run.failed_ct AS last_run_failed_ct,
             last_run.error_ct AS last_run_error_ct,
+            last_run.log_ct AS last_run_log_ct,
             last_run.dismissed_ct AS last_run_dismissed_ct
         FROM test_suites AS suites
         LEFT JOIN last_run

testgen/template/dbsetup/030_initialize_new_schema_structure.sql

@@ -200,6 +200,8 @@ CREATE TABLE test_definitions (
    match_subset_condition VARCHAR(500),
    match_groupby_names    VARCHAR,
    match_having_condition VARCHAR(500),
+   history_calculation    VARCHAR(20),
+   history_lookback       INTEGER,
    test_mode              VARCHAR(20),
    custom_query           VARCHAR,
    test_active            VARCHAR(10) DEFAULT 'Y':: CHARACTER VARYING,
@@ -443,33 +445,35 @@ CREATE TABLE data_column_chars (
 );
 
 CREATE TABLE test_types (
-   id                      VARCHAR,
-   test_type               VARCHAR(200) NOT NULL
-      CONSTRAINT cat_tests_test_type_pk
-         PRIMARY KEY,
-   test_name_short         VARCHAR(30),
-   test_name_long          VARCHAR(100),
-   test_description        VARCHAR(1000),
-   except_message          VARCHAR(1000),
-   measure_uom             VARCHAR(100),
-   measure_uom_description VARCHAR(200),
-   selection_criteria      TEXT,
-   dq_score_prevalence_formula TEXT,
-   dq_score_risk_factor       TEXT,
-   column_name_prompt      TEXT,
-   column_name_help        TEXT,
-   default_parm_columns    TEXT,
-   default_parm_values     TEXT,
-   default_parm_prompts    TEXT,
-   default_parm_help       TEXT,
-   default_severity        VARCHAR(10),
-   run_type                VARCHAR(10),
-   test_scope              VARCHAR,
-   dq_dimension            VARCHAR(50),
-   health_dimension        VARCHAR(50),
-   threshold_description   VARCHAR(200),
-   usage_notes             VARCHAR,
-   active                  VARCHAR
+   id                            VARCHAR,
+   test_type                     VARCHAR(200) NOT NULL
+                                    CONSTRAINT cat_tests_test_type_pk
+                                    PRIMARY KEY,
+   test_name_short               VARCHAR(30),
+   test_name_long                VARCHAR(100),
+   test_description              VARCHAR(1000),
+   except_message                VARCHAR(1000),
+   measure_uom                   VARCHAR(100),
+   measure_uom_description       VARCHAR(200),
+   selection_criteria            TEXT,
+   dq_score_prevalence_formula   TEXT,
+   dq_score_risk_factor          TEXT,
+   column_name_prompt            TEXT,
+   column_name_help              TEXT,
+   default_parm_columns          TEXT,
+   default_parm_values           TEXT,
+   default_parm_prompts          TEXT,
+   default_parm_help             TEXT,
+   default_severity              VARCHAR(10),
+   run_type                      VARCHAR(10),
+   test_scope                    VARCHAR,
+   dq_dimension                  VARCHAR(50),
+   health_dimension              VARCHAR(50),
+   threshold_description         VARCHAR(200),
+   result_visualization          VARCHAR(50) DEFAULT 'line_chart',
+   result_visualization_params   TEXT DEFAULT NULL,
+   usage_notes                   VARCHAR,
+   active                        VARCHAR
 );
 
 CREATE TABLE test_templates (
@@ -505,6 +509,7 @@ CREATE TABLE test_runs (
    failed_ct               INTEGER,
    warning_ct              INTEGER,
    error_ct                INTEGER,
+   log_ct                  INTEGER,
    table_ct                INTEGER,
    column_ct               INTEGER,
    column_failed_ct        INTEGER,
@@ -538,6 +543,7 @@ CREATE TABLE test_results (
    severity               VARCHAR(10),
    result_status          VARCHAR(10),
    result_message         VARCHAR(1000),
+   result_signal          VARCHAR(1000),
    result_measure         VARCHAR(1000),
    threshold_value        VARCHAR(1000),
    result_error_data      VARCHAR(4000),