Merge branch 'aarthy/qa-fixes' into 'enterprise'

fix: bugs in test results diff and table freshness

See merge request dkinternal/testgen/dataops-testgen!360

Author: ci bot

testgen/commands/run_test_execution.py

@@ -82,6 +82,10 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r
 
         sql_generator = TestExecutionSQL(connection, table_group, test_run)
 
+        # Update the thresholds before retrieving the test definitions in the next steps
+        LOG.info("Updating historic test thresholds")
+        execute_db_queries([sql_generator.update_historic_thresholds()])
+
         LOG.info("Retrieving active test definitions in test suite")
         test_defs = fetch_dict_from_db(*sql_generator.get_active_test_definitions())
         test_defs = [TestExecutionDef(**item) for item in test_defs]
@@ -100,9 +104,6 @@ def run_test_execution(test_suite_id: str | UUID, username: str | None = None, r
             )
 
             if valid_test_defs:
-                LOG.info("Updating historic test thresholds")
-                execute_db_queries([sql_generator.update_historic_thresholds()])
-
                 column_types = {(col.schema_name, col.table_name, col.column_name): col.column_type for col in data_chars}
                 for td in valid_test_defs:
                     td.column_type = column_types.get((td.schema_name, td.table_name, td.column_name))

testgen/common/models/test_result.py

@@ -2,10 +2,9 @@
 from collections import defaultdict
 from uuid import UUID, uuid4
 
-from sqlalchemy import Column, Enum, ForeignKey, String, or_, select
+from sqlalchemy import Boolean, Column, Enum, ForeignKey, String, and_, or_, select
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.orm import aliased
-from sqlalchemy.sql.functions import coalesce
 
 from testgen.common.models import get_current_session
 from testgen.common.models.entity import Entity
@@ -29,13 +28,17 @@ class TestResult(Entity):
 
     test_suite_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("test_suites.id"), nullable=False)
     test_run_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("test_runs.id"), nullable=False)
+
     test_definition_id: UUID = Column(postgresql.UUID(as_uuid=True), ForeignKey("test_definitions.id"), nullable=False)
     test_type: str = Column(String, ForeignKey("test_types.test_type"), nullable=False)
+    auto_gen: bool = Column(Boolean)
+
+    schema_name: str = Column(String, nullable=False)
+    table_name: str = Column(String)
+    column_names: str = Column(String)
 
     status: TestResultStatus = Column("result_status", Enum(TestResultStatus))
-    message: str = Column("result_message", String, nullable=False)
-    table_name: str = Column(String, nullable=False)
-    column_names: str = Column(String, nullable=False)
+    message: str = Column("result_message", String)
 
     # Note: not all table columns are implemented by this entity
 
@@ -44,9 +47,27 @@ def diff(cls, test_run_id_a: UUID, test_run_id_b: UUID) -> list[TestResultDiffTy
         alias_a = aliased(cls)
         alias_b = aliased(cls)
         query = select(
-            alias_a.status, alias_b.status, coalesce(alias_a.test_definition_id, alias_b.test_definition_id),
+            alias_a.status, alias_b.status, alias_b.test_definition_id,
         ).join(
-            alias_b, (alias_a.test_definition_id == alias_b.test_definition_id), isouter=True, full=True,
+            alias_b,
+            or_(
+                and_(
+                    alias_a.auto_gen.is_(True),
+                    alias_b.auto_gen.is_(True),
+                    alias_a.test_suite_id == alias_b.test_suite_id,
+                    alias_a.schema_name == alias_b.schema_name,
+                    alias_a.table_name.isnot_distinct_from(alias_b.table_name),
+                    alias_a.column_names.isnot_distinct_from(alias_b.column_names),
+                    alias_a.test_type == alias_b.test_type,
+                ),
+                and_(
+                    alias_a.auto_gen.isnot(True),
+                    alias_b.auto_gen.isnot(True),
+                    alias_a.test_definition_id == alias_b.test_definition_id,
+                ),
+            ),
+            isouter=True,
+            full=True,
         ).where(
             or_(alias_a.test_run_id == test_run_id_a, alias_a.test_run_id.is_(None)),
             or_(alias_b.test_run_id == test_run_id_b, alias_b.test_run_id.is_(None)),

testgen/template/dbsetup_test_types/test_types_CUSTOM.yaml

@@ -23,7 +23,7 @@ test_types:
   default_parm_prompts: |-
     Custom SQL Query Returning Error Records
   default_parm_help: |-
-    Query should return records indicating one or more errors. The test passes if no records are returned. Results of the query will be shown when you click `Review Source Data` for a failed test, so be sure to include enough data in your results to follow-up. \n\nA query can refer to any tables in the database. You must hard-code the schema or use `{DATA_SCHEMA}` to represent the schema defined for the Table Group.
+    Query should return records indicating one or more errors. The test passes if no records are returned. Results of the query will be shown when you click `Review Source Data` for a failed test, so be sure to include enough data in your results to follow-up. A query can refer to any tables in the database. You must hard-code the schema or use `{DATA_SCHEMA}` to represent the schema defined for the Table Group.
   default_severity: Fail
   run_type: QUERY
   test_scope: custom

testgen/template/execution/update_historic_thresholds.sql

@@ -1,27 +1,71 @@
-WITH stats AS (
-  SELECT
-    d.id AS test_definition_id,
-    COALESCE(
-      MIN(r.result_signal) FILTER (WHERE d.history_calculation = 'Value'),
-      MIN(r.result_signal::NUMERIC) FILTER (WHERE d.history_calculation = 'Minimum')::VARCHAR,
-      MAX(r.result_signal::NUMERIC) FILTER (WHERE d.history_calculation = 'Maximum')::VARCHAR,
-      SUM(r.result_signal::NUMERIC) FILTER (WHERE d.history_calculation = 'Sum')::VARCHAR,
-      AVG(r.result_signal::NUMERIC) FILTER (WHERE d.history_calculation = 'Average')::VARCHAR
-    ) as calc_signal
-  FROM test_definitions d
-  INNER JOIN LATERAL (
-    SELECT result_signal
-    FROM test_results tr
-    WHERE tr.test_definition_id = d.id
-    ORDER BY tr.test_time DESC
-    LIMIT CASE WHEN d.history_calculation = 'Value' THEN 1 ELSE d.history_lookback END
-  ) AS r ON TRUE
-  WHERE d.test_suite_id    = :TEST_SUITE_ID
-    AND d.test_active      = 'Y'
-    AND d.history_lookback IS NOT NULL
-  GROUP BY d.id, d.history_calculation, d.history_lookback
+WITH filtered_defs AS (
+  -- Step 1: Filter definitions first to minimize join surface area
+  SELECT id,
+    test_suite_id,
+    schema_name,
+    table_name,
+    column_name,
+    test_type,
+    history_calculation,
+    CASE WHEN history_calculation = 'Value' THEN 1 ELSE COALESCE(history_lookback, 1) END AS lookback
+  FROM test_definitions
+  WHERE test_suite_id = :TEST_SUITE_ID
+    AND test_active = 'Y'
+    AND history_calculation IS NOT NULL
+    AND history_lookback IS NOT NULL
+),
+normalized_results AS (
+  -- Step 2: Normalize definition IDs for autogenerated tests
+  SELECT CASE
+      WHEN r.auto_gen THEN d.id
+      ELSE r.test_definition_id
+    END AS test_definition_id,
+    r.test_time,
+    r.result_signal
+  FROM test_results r
+    LEFT JOIN filtered_defs d ON r.auto_gen = TRUE
+    AND r.test_suite_id = d.test_suite_id
+    AND r.schema_name = d.schema_name
+    AND r.table_name IS NOT DISTINCT FROM d.table_name
+    AND r.column_names IS NOT DISTINCT FROM d.column_name
+    AND r.test_type = d.test_type
+  WHERE r.test_suite_id = :TEST_SUITE_ID
+),
+ranked_results AS (
+  -- Step 3: Use a Window Function to get the N most recent results
+  SELECT n.test_definition_id,
+    n.result_signal,
+    CASE
+      WHEN n.result_signal ~ '^-?[0-9]*\.?[0-9]+$' THEN n.result_signal::NUMERIC
+      ELSE NULL
+    END AS signal_numeric,
+    ROW_NUMBER() OVER (PARTITION BY n.test_definition_id ORDER BY n.test_time DESC) AS rank
+  FROM normalized_results n
+  WHERE n.test_definition_id IN (SELECT id FROM filtered_defs)
+),
+stats AS (
+  -- Step 4: Aggregate only the rows within the lookback range
+  SELECT d.id AS test_definition_id,
+    d.history_calculation,
+    MAX(CASE WHEN rr.rank = 1 THEN rr.result_signal END) AS val,
+    MIN(rr.signal_numeric) AS min,
+    MAX(rr.signal_numeric) AS max,
+    SUM(rr.signal_numeric) AS sum,
+    AVG(rr.signal_numeric) AS avg
+  FROM filtered_defs d
+    JOIN ranked_results rr ON d.id = rr.test_definition_id
+  WHERE rr.rank <= d.lookback
+  GROUP BY d.id,
+    d.history_calculation
 )
 UPDATE test_definitions t
-SET baseline_value = s.calc_signal
+SET baseline_value = CASE
+    WHEN s.history_calculation = 'Value' THEN s.val
+    WHEN s.history_calculation = 'Minimum' THEN s.min::VARCHAR
+    WHEN s.history_calculation = 'Maximum' THEN s.max::VARCHAR
+    WHEN s.history_calculation = 'Sum' THEN s.sum::VARCHAR
+    WHEN s.history_calculation = 'Average' THEN s.avg::VARCHAR
+    ELSE NULL
+  END
 FROM stats s
-WHERE t.id = s.test_definition_id;
+WHERE t.id = s.test_definition_id;

testgen/ui/components/frontend/js/pages/profiling_runs.js

@@ -72,7 +72,7 @@ const ProfilingRuns = (/** @type Properties */ props) => {
     Streamlit.setFrameHeight(1);
     window.testgen.isPage = true;
 
-    const columns = ['5%', '15%', '20%', '20%', '30%', '10%'];
+    const columns = ['5%', '20%', '15%', '20%', '30%', '10%'];
     const userCanEdit = getValue(props.permissions)?.can_edit ?? false;
 
     const pageIndex = van.state(0);
@@ -118,7 +118,7 @@ const ProfilingRuns = (/** @type Properties */ props) => {
                 () => profilingRuns.val.length
                 ? div(
                     div(
-                        { class: 'table pb-0' },
+                        { class: 'table pb-0', style: 'overflow-y: auto;' },
                         () => {
                             const selectedItems = profilingRuns.val.filter(i => selectedRuns[i.id]?.val ?? false);
                             const someRunSelected = selectedItems.length > 0;
@@ -157,7 +157,7 @@ const ProfilingRuns = (/** @type Properties */ props) => {
                                 }
 
                                 return span(
-                                    { style: `flex: ${columns[0]}` },
+                                    { style: `flex: 0 0 ${columns[0]}` },
                                     userCanEdit
                                         ? Checkbox({
                                             checked: allSelected,
@@ -169,23 +169,23 @@ const ProfilingRuns = (/** @type Properties */ props) => {
                                 );
                             },
                             span(
-                                { style: `flex: ${columns[1]}` },
+                                { style: `flex: 0 0 ${columns[1]}` },
                                 'Start Time | Table Group',
                             ),
                             span(
-                                { style: `flex: ${columns[2]}` },
+                                { style: `flex: 0 0 ${columns[2]}` },
                                 'Status | Duration',
                             ),
                             span(
-                                { style: `flex: ${columns[3]}` },
+                                { style: `flex: 0 0 ${columns[3]}` },
                                 'Schema',
                             ),
                             span(
-                                { style: `flex: ${columns[4]}`, class: 'tg-profiling-runs--issues' },
+                                { style: `flex: 0 0 ${columns[4]}`, class: 'tg-profiling-runs--issues' },
                                 'Hygiene Issues',
                             ),
                             span(
-                                { style: `flex: ${columns[5]}` },
+                                { style: `flex: 0 0 ${columns[5]}` },
                                 'Profiling Score',
                             ),
                         ),
@@ -287,7 +287,7 @@ const ProfilingRunItem = (
         { class: 'table-row flex-row', 'data-testid': 'profiling-run-item' },
         userCanEdit
             ? div(
-                { style: `flex: ${columns[0]}; font-size: 16px;` },
+                { style: `flex: 0 0 ${columns[0]}; font-size: 16px;` },
                 Checkbox({
                     checked: selected,
                     onChange: (checked) => selected.val = checked,
@@ -296,15 +296,15 @@ const ProfilingRunItem = (
             )
             : '',
         div(
-            { style: `flex: ${columns[1]}` },
+            { style: `flex: 0 0 ${columns[1]}; max-width: ${columns[1]}; word-wrap: break-word;` },
             div({ 'data-testid': 'profiling-run-item-starttime' }, formatTimestamp(item.profiling_starttime)),
             div(
                 { class: 'text-caption mt-1', 'data-testid': 'profiling-run-item-tablegroup' },
                 item.table_groups_name,
             ),
         ),
         div(
-            { style: `flex: ${columns[2]}` },
+            { style: `flex: 0 0 ${columns[2]};  max-width: ${columns[2]};` },
             div(
                 { class: 'flex-row' },
                 ProfilingRunStatus(item),
@@ -337,7 +337,7 @@ const ProfilingRunItem = (
                 ),
         ),
         div(
-            { style: `flex: ${columns[3]}` },
+            { style: `flex: 0 0 ${columns[3]}; max-width: ${columns[3]};` },
             div({ 'data-testid': 'profiling-run-item-schema' }, item.table_group_schema),
             div(
                 {
@@ -369,7 +369,7 @@ const ProfilingRunItem = (
             }) : null,
         ),
         div(
-            { class: 'pr-3 tg-profiling-runs--issues', style: `flex: ${columns[4]}` },
+            { class: 'pr-3 tg-profiling-runs--issues', style: `flex: 0 0 ${columns[4]};  max-width: ${columns[4]};` },
             item.anomaly_ct ? SummaryCounts({
                 items: [
                     { label: 'Definite', value: item.anomalies_definite_ct, color: 'red' },
@@ -389,7 +389,7 @@ const ProfilingRunItem = (
             }) : null,
         ),
         div(
-            { style: `flex: ${columns[5]}; font-size: 16px;` },
+            { style: `flex: 0 0 ${columns[5]};  max-width: ${columns[5]}; font-size: 16px;` },
             item.column_ct && item.dq_score_profiling
                 ? item.dq_score_profiling
                 : '--',

testgen/ui/views/test_definitions.py

@@ -483,11 +483,12 @@ def show_test_form(
     if dynamic_attributes_labels_raw:
         dynamic_attributes_labels = dynamic_attributes_labels_raw.split(",")
 
-    dynamic_attributes_help_raw = selected_test_type_row["default_parm_help"]
-    if not dynamic_attributes_help_raw:
-        dynamic_attributes_help_raw = "No help is available"
     # Split on pipe -- could contain commas
-    dynamic_attributes_help = dynamic_attributes_help_raw.split("|")
+    dynamic_attributes_help = (
+        selected_test_type_row["default_parm_help"].split("|")
+        if selected_test_type_row["default_parm_help"]
+        else None
+    )
 
     if mode == "edit":
         st.text_input(label="Test Type", value=test_type_display, disabled=True),
@@ -700,7 +701,7 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator):
         help_text = (
             dynamic_attributes_help[index]
             if dynamic_attributes_help and len(dynamic_attributes_help) > index
-            else "Help text is not available."
+            else None
         )
 
         if attribute == "custom_query":
@@ -710,7 +711,7 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator):
             elif test_type == "CUSTOM":
                 custom_query_placeholder = "EXAMPLE:  SELECT product, SUM(qty_sold) as sum_sold, SUM(qty_shipped) as qty_shipped \n FROM {DATA_SCHEMA}.sales_history \n GROUP BY product \n HAVING SUM(qty_shipped) > SUM(qty_sold)"
 
-            test_definition[attribute] = st.text_area(
+            test_definition[attribute] = container.text_area(
                 label=label_text,
                 value=custom_query,
                 placeholder=custom_query_placeholder,
@@ -766,6 +767,7 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator):
     if test_scope != "tablegroup":
         st.divider()
 
+    mid_container = st.container()
     mid_left_column, mid_right_column = st.columns([0.5, 0.5])
 
     if has_match_attributes:
@@ -775,7 +777,7 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator):
                 render_dynamic_attribute(f"match_{attribute}", mid_right_column)
 
     if "custom_query" in dynamic_attributes:
-        render_dynamic_attribute("custom_query", mid_left_column)
+        render_dynamic_attribute("custom_query", mid_container)
 
     total_length = len(leftover_attributes)
     half_length = round(total_length / 2)
@@ -953,13 +955,13 @@ def validate_form(test_scope, test_definition, column_name_label):
 
 def prompt_for_test_type():
 
-    col0, col1, col2, col3, col4, col5 = st.columns([0.1, 0.2, 0.2, 0.2, 0.2, 0.1])
+    col0, col1, col2, col3, col4 = st.columns([0.2, 0.2, 0.2, 0.2, 0.2])
     col0.write("Show Types")
 
-    include_referential=col1.checkbox(":green[⧉] Referential", True),
-    include_table=col2.checkbox(":green[⊞] Table", True),
-    include_column=col3.checkbox(":green[≣] Column", True),
-    include_custom=col4.checkbox(":green[⛭] Custom", True),
+    include_referential=col1.checkbox(":green[⧉] Referential", True)
+    include_table=col2.checkbox(":green[⊞] Table", True)
+    include_column=col3.checkbox(":green[≣] Column", True)
+    include_custom=col4.checkbox(":green[⛭] Custom", True)
     # always exclude tablegroup scopes from showing
     include_all = not any([include_referential, include_table, include_column, include_custom])
 

testgen/ui/views/test_results.py

@@ -600,7 +600,7 @@ def render_selected_details(
                 if selected_item["measure_uom_description"]:
                     st.caption(selected_item["measure_uom_description"])
                 if selected_item["result_message"]:
-                    st.caption(selected_item["result_message"])
+                    st.caption(selected_item["result_message"].replace("*", "\\*"))
                 fm.render_grid_select(dfh, show_hist_columns, selection_mode="disabled", key="test_history")
             with pg_col2:
                 ut_tab1, ut_tab2 = st.tabs(["History", "Test Definition"])
@@ -809,8 +809,9 @@ def source_data_dialog(selected_row):
     st.markdown("#### Test Parameters")
     testgen.caption(selected_row["input_parameters"], styles="max-height: 75px; overflow: auto;")
 
-    st.markdown("#### Result Detail")
-    st.caption(selected_row["result_message"])
+    if selected_row["result_message"]:
+        st.markdown("#### Result Detail")
+        st.caption(selected_row["result_message"].replace("*", "\\*"))
 
     st.markdown("#### SQL Query")
     if selected_row["test_type"] == "CUSTOM":