Merge branch 'aarthy/qa-fixes' into 'enterprise'

misc: add example config for sso and tls

See merge request dkinternal/testgen/dataops-testgen!299

Author: ci bot

testgen/__main__.py

@@ -369,7 +369,7 @@ def quick_start(
 
     click.echo("loading initial data")
     run_quick_start_increment(0)
-    minutes_offset = -30*24*60*3
+    minutes_offset = -30*24*60 # 1 month ago
     table_group_id="0ea85e17-acbe-47fe-8394-9970725ad37d"
 
     click.echo(f"run-profile with table_group_id: {table_group_id}")
@@ -387,7 +387,7 @@ def quick_start(
 
     for iteration in range(1, 4):
         click.echo(f"Running iteration: {iteration} / 3")
-        minutes_offset = -30*24*60 * (3-iteration)
+        minutes_offset = -10*24*60 * (3-iteration)
         run_quick_start_increment(iteration)
         run_execution_steps(settings.PROJECT_KEY, settings.DEFAULT_TEST_SUITE_KEY, minutes_offset=minutes_offset)
 

testgen/template/dbsetup/050_populate_new_schema_metadata.sql

@@ -158,7 +158,7 @@ VALUES  ('1004', 'Alpha_Trunc', 'Alpha Truncation', 'Maximum character count con
         ('1508', 'Timeframe_Combo_Gain', 'Timeframe No Drops', 'Latest timeframe has at least all value combinations from prior period', 'Tests that column values in most recent time-window include at least same as prior time window', 'Column values in most recent time-window don''t include all values in prior window.', 'Mismatched values', NULL, NULL, '({RESULT_MEASURE}-{THRESHOLD_VALUE})::FLOAT/NULLIF({RECORD_CT}::FLOAT, 0)', '1.0', 'Categorical Column List', 'Specify one or more Categorical columns, separated by commas. Make sure not to use continuous measurements here. Do not use numeric values unless they represent discrete categories.', 'window_date_column,window_days,subset_condition', NULL, 'Date Column for Time Windows,Time Window in Days,Record Subset Condition', 'The date column used to define the time windows. This must be a DATE or DATETIME type.|Length in days of the time window. The test will compare the most recent period of days to the prior period of the same duration.|Condition defining a subset of records in main table to evaluate, written like a condition within a SQL WHERE clause - OPTIONAL', 'Fail', 'QUERY', 'referential', 'Consistency', 'Data Drift', 'Expected count of missing value combinations', 'This test checks a single transactional table to verify that categorical values or combinations that are present in the most recent time window you define include at least all those found in the prior time window of the same duration. Missing values in the latest time window will trigger the test to fail. New values are permitted. Use this test to confirm that codes or categories are not lost across successive time periods in a transactional table.', 'Y', 'line_chart', NULL),
         ('1509', 'Timeframe_Combo_Match', 'Timeframe Match', 'Column value combinations from latest timeframe same as prior period', 'Tests for presence of same column values in most recent time-window vs. prior time window', 'Column values don''t match in most recent time-windows.', 'Mismatched values', NULL, NULL, '({RESULT_MEASURE}-{THRESHOLD_VALUE})::FLOAT/NULLIF({RECORD_CT}::FLOAT, 0)', '1.0', 'Categorical Column List', 'Specify one or more Categorical columns, separated by commas. Do not use continuous measurements here. Do not use numeric values unless they represent discrete categories.', 'window_date_column,window_days,subset_condition', NULL, 'Date Column for Time Windows,Time Window in Days,Record Subset Condition', NULL, 'Fail', 'QUERY', 'referential', 'Consistency', 'Data Drift', 'Expected count of non-matching value combinations', 'This test checks a single transactional table (such as a fact table) to verify that categorical values or combinations that are present in the most recent time window you define match those found in the prior time window of the same duration. New or missing values in the latest time window will trigger the test to fail. Use this test to confirm the consistency in the occurrence of codes or categories across successive time periods in a transactional table.', 'Y', 'line_chart', NULL),
         ('1510', 'Dupe_Rows', 'Duplicate Rows', 'Rows are not duplicated in table', 'Tests for the absence of duplicate rows based on unique combination of column values', 'Column value combinations are duplicated in the table.', 'Duplicate records', NULL, NULL, '(({RESULT_MEASURE}-{THRESHOLD_VALUE}))::FLOAT/NULLIF({RECORD_CT}::FLOAT, 0)', '1.0', 'null', 'null', 'groupby_names', NULL, 'Columns to Compare', 'List of columns in the table that define a duplicate record when the combination of values is repeated on multiple rows', 'Fail', 'QUERY', 'table', 'Uniqueness', 'Schema Drift', 'Expected count of duplicate value combinations', 'This test verifies that combinations of values are not repeated within the table. By default when auto-generated, the test considers all columns to protect against duplication of entire rows. If you know the minimum columns that should constitute a unique record, such as a set of ID''s, you should use those to make the test as sensitive as possible. Alternatively, if you know of columns you can always exclude, such as file_date or refresh_snapshot_id, remove them to tighten the test somewhat.', 'Y', 'line_chart', NULL),
-        ('1511', 'Stale_Table', 'Stale Table', 'Stale Table Not Updated', 'Confirms whether table has been updated based on data fingerprint', 'Table has not been updated.', 'Was Change Detected', NULL, 'TEMPLATE', '(({RESULT_MEASURE}-{THRESHOLD_VALUE}))::FLOAT/NULLIF({RECORD_CT}::FLOAT, 0)', '0.0', 'null', 'null', 'history_calculation,history_lookback,subset_condition,custom_query', NULL, 'History Aggregate,History Lookback,Record Subset Condition,Fingerprint Expression', 'Aggregate calculation to be performed on the N lookback results|Last N tests to use for history aggregate calculation|Condition defining a subset of records in main table|String expression combining key column measures into a distinct representation of table state', 'Log', 'QUERY', 'table', 'Recency', 'Recency', 'Most recent prior table fingerprint', 'This test compares the current table fingerprint, calculated signature of column contents, to confirm that the table has been updated. The table fingerprint is derived from a set of values and aggregates from columns most likely to change. This test allows you to track the schedule and frequency of updates and refreshes to the table.', 'Y', 'binary_chart', '{"legend":{"labels":{"0":"Stale","1":"Updated"}}}')
+        ('1511', 'Table_Freshness', 'Table Freshness', 'Stale Table Not Updated', 'Confirms whether table has been updated based on data fingerprint', 'Table has not been updated.', 'Was Change Detected', NULL, 'TEMPLATE', '(({RESULT_MEASURE}-{THRESHOLD_VALUE}))::FLOAT/NULLIF({RECORD_CT}::FLOAT, 0)', '0.0', 'null', 'null', 'history_calculation,history_lookback,subset_condition,custom_query', NULL, 'History Aggregate,History Lookback,Record Subset Condition,Fingerprint Expression', 'Aggregate calculation to be performed on the N lookback results|Last N tests to use for history aggregate calculation|Condition defining a subset of records in main table|String expression combining key column measures into a distinct representation of table state', 'Log', 'QUERY', 'table', 'Recency', 'Recency', 'Most recent prior table fingerprint', 'This test compares the current table fingerprint, calculated signature of column contents, to confirm that the table has been updated. The table fingerprint is derived from a set of values and aggregates from columns most likely to change. This test allows you to track the schedule and frequency of updates and refreshes to the table.', 'Y', 'binary_chart', '{"legend":{"labels":{"0":"Stale","1":"Updated"}}}')
 ;
 
 
@@ -171,7 +171,7 @@ VALUES  ('Monitor', 'Recency'),
         ('Monitor', 'Daily_Record_Ct'),
         ('Monitor', 'Monthly_Rec_Ct'),
         ('Monitor', 'Weekly_Rec_Ct'),
-        ('Monitor', 'Stale_Table');
+        ('Monitor', 'Table_Freshness');
 
 
 TRUNCATE TABLE test_templates;
@@ -232,11 +232,11 @@ VALUES  ('2001', 'Combo_Match', 'redshift', 'ex_data_match_generic.sql'),
         ('2410', 'Aggregate_Balance_Range', 'databricks', 'ex_aggregate_match_range_generic.sql'),
         ('2411', 'Dupe_Rows', 'databricks', 'ex_dupe_rows_generic.sql'),
 
-        ('2012', 'Stale_Table', 'redshift', 'ex_table_changed_generic.sql'),
-        ('2112', 'Stale_Table', 'snowflake', 'ex_table_changed_generic.sql'),
-        ('2212', 'Stale_Table', 'mssql', 'ex_table_changed_generic.sql'),
-        ('2312', 'Stale_Table', 'postgresql', 'ex_table_changed_generic.sql'),
-        ('2412', 'Stale_Table', 'databricks', 'ex_table_changed_generic.sql')
+        ('2012', 'Table_Freshness', 'redshift', 'ex_table_changed_generic.sql'),
+        ('2112', 'Table_Freshness', 'snowflake', 'ex_table_changed_generic.sql'),
+        ('2212', 'Table_Freshness', 'mssql', 'ex_table_changed_generic.sql'),
+        ('2312', 'Table_Freshness', 'postgresql', 'ex_table_changed_generic.sql'),
+        ('2412', 'Table_Freshness', 'databricks', 'ex_table_changed_generic.sql')
 ;
 
 TRUNCATE TABLE cat_test_conditions;

testgen/template/flavors/databricks/gen_query_tests/gen_table_changed_test.sql

@@ -24,7 +24,7 @@ locked AS       (SELECT schema_name, table_name
                   FROM test_definitions
 				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
                    AND test_suite_id = '{TEST_SUITE_ID}'
-				       AND test_type = 'Stale_Table'
+				       AND test_type = 'Table_Freshness'
                    AND lock_refresh = 'Y'),
 -- IDs - TOP 2
 id_cols
@@ -126,7 +126,7 @@ newtests
       GROUP BY profile_run_id, schema_name, table_name)
 SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
        n.profile_run_id,
-       'Stale_Table' AS test_type,
+       'Table_Freshness' AS test_type,
        '{TEST_SUITE_ID}' AS test_suite_id,
        n.schema_name, n.table_name,
        0 as skip_errors, 'Y' as test_active,
@@ -139,7 +139,7 @@ SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
        fingerprint as custom_query
 FROM newtests n
 INNER JOIN test_types t
-   ON ('Stale_Table' = t.test_type
+   ON ('Table_Freshness' = t.test_type
   AND   'Y' = t.active)
 LEFT JOIN generation_sets s
    ON (t.test_type = s.test_type

testgen/template/flavors/mssql/gen_query_tests/gen_table_changed_test.sql

@@ -24,7 +24,7 @@ locked AS       (SELECT schema_name, table_name
                   FROM test_definitions
 				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
                    AND test_suite_id = '{TEST_SUITE_ID}'
-				       AND test_type = 'Stale_Table'
+				       AND test_type = 'Table_Freshness'
                    AND lock_refresh = 'Y'),
 -- IDs - TOP 2
 id_cols
@@ -131,7 +131,7 @@ newtests AS (
 )
 SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
        n.profile_run_id,
-       'Stale_Table' AS test_type,
+       'Table_Freshness' AS test_type,
        '{TEST_SUITE_ID}' AS test_suite_id,
        n.schema_name, n.table_name,
        0 as skip_errors, 'Y' as test_active,
@@ -144,7 +144,7 @@ SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
        fingerprint as custom_query
 FROM newtests n
 INNER JOIN test_types t
-   ON ('Stale_Table' = t.test_type
+   ON ('Table_Freshness' = t.test_type
   AND   'Y' = t.active)
 LEFT JOIN generation_sets s
    ON (t.test_type = s.test_type

testgen/template/gen_query_tests/gen_table_changed_test.sql

@@ -24,7 +24,7 @@ locked AS       (SELECT schema_name, table_name
                   FROM test_definitions
 				     WHERE table_groups_id = '{TABLE_GROUPS_ID}'::UUID
                    AND test_suite_id = '{TEST_SUITE_ID}'
-				       AND test_type = 'Stale_Table'
+				       AND test_type = 'Table_Freshness'
                    AND lock_refresh = 'Y'),
 -- IDs - TOP 2
 id_cols
@@ -126,7 +126,7 @@ newtests
       GROUP BY profile_run_id, schema_name, table_name)
 SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
        n.profile_run_id,
-       'Stale_Table' AS test_type,
+       'Table_Freshness' AS test_type,
        '{TEST_SUITE_ID}' AS test_suite_id,
        n.schema_name, n.table_name,
        0 as skip_errors, 'Y' as test_active,
@@ -139,7 +139,7 @@ SELECT '{TABLE_GROUPS_ID}'::UUID as table_groups_id,
        fingerprint as custom_query
 FROM newtests n
 INNER JOIN test_types t
-   ON ('Stale_Table' = t.test_type
+   ON ('Table_Freshness' = t.test_type
   AND   'Y' = t.active)
 LEFT JOIN generation_sets s
    ON (t.test_type = s.test_type

testgen/ui/views/test_definitions.py

@@ -669,7 +669,7 @@ def render_dynamic_attribute(attribute: str, container: DeltaGenerator):
     submit = bottom_left_column.button("Save")
 
     if submit:
-        if validate_form(test_scope, test_type, test_definition, column_name_label):
+        if validate_form(test_scope, test_definition, column_name_label):
             if mode == "edit":
                 test_definition["id"] = selected_test_def["id"]
             TestDefinition(**test_definition).save()
@@ -793,13 +793,7 @@ def copy_move_test_dialog(
         time.sleep(1)
         st.rerun()
 
-def validate_form(test_scope, test_type, test_definition, column_name_label):
-    if test_type == "Condition_Flag" and not test_definition["threshold_value"]:
-        st.error("Threshold Error Count is a required field.")
-        return False
-    if not test_definition["test_type"]:
-        st.error("Test Type is a required field.")
-        return False
+def validate_form(test_scope, test_definition, column_name_label):
     if test_scope in ["column", "referential", "custom"] and not test_definition["column_name"]:
         st.error(f"{column_name_label} is a required field.")
         return False

testgen/ui/views/test_results.py

@@ -734,7 +734,7 @@ def render_binary_chart(data: pd.DataFrame, **params: dict) -> None:
     legend_labels = params.get("legend", {}).get("labels") or {"0": "0", "1": "1"}
 
     history["test_start"] = history["test_date"].apply(datetime.fromisoformat)
-    history["test_end"] = history["test_start"].apply(lambda start: start + timedelta(seconds=30))
+    history["test_end"] = history["test_start"].apply(lambda start: start + timedelta(seconds=60))
     history["formatted_test_date"] = history["test_date"].apply(lambda date_str: datetime.fromisoformat(date_str).strftime("%I:%M:%S %p, %d/%m/%Y"))
     history["result_measure_with_status"] = history.apply(lambda row: f"{legend_labels[str(int(row['result_measure']))]} ({row['result_status']})", axis=1)