Merge branch 'qa-fixes' into 'enterprise'

Misc QA fixes + doc updates

See merge request dkinternal/testgen/dataops-testgen!446

Author: ci bot

testgen/common/models/test_suite.py

@@ -101,7 +101,7 @@ def select_minimal_where(
 
     @classmethod
     @st.cache_data(show_spinner=False)
-    def select_summary(cls, project_code: str, table_group_id: str | UUID | None = None) -> Iterable[TestSuiteSummary]:
+    def select_summary(cls, project_code: str, table_group_id: str | UUID | None = None, test_suite_name: str | None = None) -> Iterable[TestSuiteSummary]:
         if table_group_id and not is_uuid4(table_group_id):
             return []
 
@@ -199,9 +199,10 @@ def select_summary(cls, project_code: str, table_group_id: str | UUID | None = N
         WHERE suites.is_monitor IS NOT TRUE
             AND suites.project_code = :project_code
             {"AND suites.table_groups_id = :table_group_id" if table_group_id else ""}
+            {"AND suites.test_suite ILIKE :test_suite_name" if test_suite_name else ""}
         ORDER BY LOWER(suites.test_suite);
         """
-        params = {"project_code": project_code, "table_group_id": table_group_id}
+        params = {"project_code": project_code, "table_group_id": table_group_id, "test_suite_name": f"%{test_suite_name}%" if test_suite_name else None}
         db_session = get_current_session()
         results = db_session.execute(text(query), params).mappings().all()
         return [TestSuiteSummary(**row) for row in results]

testgen/template/flavors/bigquery/profiling/project_profiling_query.sql

@@ -99,6 +99,7 @@ SELECT
     WHEN SAFE_DIVIDE(SUM(CASE WHEN REGEXP_CONTAINS(`{COL_NAME}`, r'^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$')
     THEN 1 END), COUNT(`{COL_NAME}`)) > 0.8 THEN 'PHONE_USA'
     WHEN SAFE_DIVIDE(SUM(CASE WHEN REGEXP_CONTAINS(`{COL_NAME}`, r'^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$')
+              AND `{COL_NAME}` NOT LIKE '%://%'
     THEN 1 END), COUNT(`{COL_NAME}`)) > 0.9 THEN 'EMAIL'
     WHEN SAFE_DIVIDE(SUM(CASE WHEN TRANSLATE(`{COL_NAME}`, '012345678', '999999999') IN ('99999', '999999999', '99999-9999')
     THEN 1 END), COUNT(`{COL_NAME}`)) > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/databricks/profiling/project_profiling_query.sql

@@ -100,7 +100,8 @@ SELECT
          THEN 1 END) AS FLOAT)/CAST(COUNT(`{COL_NAME}`) AS FLOAT) > 0.9 THEN 'STATE_USA'
     WHEN CAST(SUM( CASE WHEN `{COL_NAME}` RLIKE '\\+1\\s*\\(?\\d{3}\\)?[-. ]*\\d{3}[-. ]*\\d{4}'
          THEN 1 END) AS FLOAT)/CAST(COUNT(`{COL_NAME}`) AS FLOAT) > 0.9 THEN 'PHONE_USA'
-    WHEN CAST(SUM( CASE WHEN `{COL_NAME}` RLIKE '[_a-zA-Z0-9.-]+@[a-zA-Z0-9.-]+.[a-zA-Z][a-zA-Z]+'
+    WHEN CAST(SUM( CASE WHEN `{COL_NAME}` RLIKE '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$'
+              AND `{COL_NAME}` NOT LIKE '%://%'
          THEN 1 END) AS FLOAT)/CAST(COUNT(`{COL_NAME}`) AS FLOAT) > 0.9 THEN 'EMAIL'
     WHEN CAST(SUM( CASE WHEN TRANSLATE(`{COL_NAME}`,'012345678','999999999') IN ('99999', '999999999', '99999-9999')
          THEN 1 END) AS FLOAT)/CAST(COUNT(`{COL_NAME}`) AS FLOAT) > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/mssql/profiling/project_profiling_query.sql

@@ -110,6 +110,7 @@ SELECT
                      OR ("{COL_NAME}" LIKE '[+]1%[0-9][0-9][0-9][-. ][0-9][0-9][0-9][-. ][0-9][0-9][0-9][0-9]' AND "{COL_NAME}" NOT LIKE '%[^0-9+-]%')
          THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'PHONE_USA'
     WHEN CAST(SUM(CASE WHEN "{COL_NAME}" LIKE '%[_a-zA-Z0-9.-]%@%[a-zA-Z0-9.-]%.[a-zA-Z][a-zA-Z]%'
+              AND "{COL_NAME}" NOT LIKE '%://%'
          THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'EMAIL'
     WHEN CAST(SUM(CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
          THEN 1 END) AS FLOAT)/CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/oracle/profiling/project_profiling_query.sql

@@ -159,6 +159,7 @@ FROM (
       WHEN SUM(CASE WHEN REGEXP_LIKE("{COL_NAME}", '^(\+1|1)?[ .-]?(\([2-9][0-9]{2}\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$')
            THEN 1 ELSE 0 END) / NULLIF(COUNT("{COL_NAME}"), 0) > 0.8 THEN 'PHONE_USA'
       WHEN SUM(CASE WHEN REGEXP_LIKE("{COL_NAME}", '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$')
+                AND "{COL_NAME}" NOT LIKE '%://%'
            THEN 1 ELSE 0 END) / NULLIF(COUNT("{COL_NAME}"), 0) > 0.9 THEN 'EMAIL'
       WHEN SUM(CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
            THEN 1 ELSE 0 END) / NULLIF(COUNT("{COL_NAME}"), 0) > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/redshift/profiling/project_profiling_query.sql

@@ -70,6 +70,7 @@ SELECT
     WHEN SUM(CASE WHEN "{COL_NAME}" ~ '^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$'
          THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'PHONE_USA'
     WHEN SUM(CASE WHEN "{COL_NAME}" ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$'
+              AND "{COL_NAME}" NOT LIKE '%://%'
          THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'EMAIL'
     WHEN SUM(CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
          THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/redshift_spectrum/profiling/project_profiling_query.sql

@@ -70,6 +70,7 @@ SELECT
     WHEN SUM(CASE WHEN "{COL_NAME}" ~ '^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$'
          THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.8 THEN 'PHONE_USA'
     WHEN SUM(CASE WHEN "{COL_NAME}" ~ '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$'
+              AND "{COL_NAME}" NOT LIKE '%://%'
          THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'EMAIL'
     WHEN SUM(CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
          THEN 1 END)::FLOAT/COUNT("{COL_NAME}")::FLOAT > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/sap_hana/profiling/project_profiling_query.sql

@@ -87,6 +87,7 @@ SELECT
     WHEN SUM(CASE WHEN "{COL_NAME}" LIKE_REGEXPR '^(\+1|1)?[ .-]?(\([2-9][0-9]{2}\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$'
          THEN 1 ELSE 0 END) / NULLIF(COUNT("{COL_NAME}"), 0) > 0.8 THEN 'PHONE_USA'
     WHEN SUM(CASE WHEN "{COL_NAME}" LIKE_REGEXPR '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$'
+              AND "{COL_NAME}" NOT LIKE '%://%'
          THEN 1 ELSE 0 END) / NULLIF(COUNT("{COL_NAME}"), 0) > 0.9 THEN 'EMAIL'
     WHEN SUM(CASE WHEN REPLACE_REGEXPR('[0-9]' IN "{COL_NAME}" WITH '9') IN ('99999', '999999999', '99999-9999')
          THEN 1 ELSE 0 END) / NULLIF(COUNT("{COL_NAME}"), 0) > 0.9 THEN 'ZIP_USA'

testgen/template/flavors/snowflake/profiling/project_profiling_query.sql

@@ -79,6 +79,7 @@ SELECT
     WHEN CAST(SUM(CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^(\\+1|1)?[ .-]?(\\([2-9][0-9]{2}\\)|[2-9][0-9]{2})[ .-]?[2-9][0-9]{2}[ .-]?[0-9]{4}$')
                         THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.8 THEN 'PHONE_USA'
     WHEN CAST(SUM(CASE WHEN REGEXP_LIKE("{COL_NAME}"::VARCHAR, '^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\\.[A-Za-z]{2,}$')
+                        AND "{COL_NAME}"::VARCHAR NOT LIKE '%://%'
                         THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'EMAIL'
     WHEN CAST(SUM(CASE WHEN TRANSLATE("{COL_NAME}",'012345678','999999999') IN ('99999', '999999999', '99999-9999')
                         THEN 1 END) AS FLOAT) / CAST(COUNT("{COL_NAME}") AS FLOAT) > 0.9 THEN 'ZIP_USA'

testgen/ui/components/frontend/js/data_profiling/metadata_tags.js

@@ -15,7 +15,7 @@ import { Attribute } from '../components/attribute.js';
 import { Input } from '../components/input.js';
 import { Icon } from '../components/icon.js';
 import { withTooltip } from '../components/tooltip.js';
-import { emitEvent } from '../utils.js';
+import { emitEvent, loadStylesheet } from '../utils.js';
 import { RadioGroup } from '../components/radio_group.js';
 import { Checkbox } from '../components/checkbox.js';
 import { capitalize } from '../display_utils.js';
@@ -26,7 +26,7 @@ import { Alert } from '../components/alert.js';
 
 const { div, span } = van.tags;
 
-const attributeWidth = 300;
+const attributeWidth = 250;
 const descriptionWidth = 932;
 const multiEditWidth = 400;
 
@@ -79,10 +79,12 @@ const TAG_HELP = {
  * @returns
  */
 const MetadataTagsCard = (props, item) => {
+    loadStylesheet('metadata-tags', stylesheet);
+
     const title = `${item.type} Tags `;
     const attributes = [
         'critical_data_element',
-        ...(item.type === 'column' ? ['excluded_data_element', 'pii_flag'] : []),
+        ...(item.type === 'column' ? ['pii_flag', 'excluded_data_element'] : []),
         'description',
         ...TAG_KEYS,
     ].map(key => {
@@ -235,33 +237,40 @@ const InheritedIcon = (/** @type string */ inheritedFrom) => withTooltip(
  * @returns
  */
 const CdeDisplay = (value, isColumn, isInherited) => {
+    if (value) {
+        return div(
+            { style: `width: ${attributeWidth}px` },
+            span(
+                { class: 'flex-row fx-gap-1 metadata-badge cde' },
+                Icon({ size: 24, classes: 'text-purple' }, 'star'),
+                span(isColumn ? 'Critical data element' : 'All critical data elements'),
+                (isColumn && isInherited) ? InheritedIcon('table') : null,
+            ),
+        );
+    }
     return span(
         { class: 'flex-row fx-gap-1', style: `width: ${attributeWidth}px` },
-        Icon(
-            { size: value ? 24 : 20, classes: value ? 'text-purple' : 'text-disabled' },
-            value ? 'star' : 'cancel',
-        ),
-        span(
-            { class: value ? '' : 'text-secondary' },
-            isColumn
-                ? (value ? 'Critical data element' : 'Not a critical data element')
-                : (value ? 'All critical data elements' : 'Not all critical data elements'),
-        ),
+        Icon({ size: 20, classes: 'text-disabled' }, 'cancel'),
+        span({ class: 'text-secondary' }, isColumn ? 'Not a critical data element' : 'Not all critical data elements'),
         (isColumn && isInherited) ? InheritedIcon('table') : null,
     );
 }
 
 const XdeDisplay = (/** @type boolean */ value) => {
+    if (value) {
+        return div(
+            { style: `width: ${attributeWidth}px` },
+            span(
+                { class: 'flex-row fx-gap-1 metadata-badge xde' },
+                Icon({ size: 20, classes: 'text-brown' }, 'visibility_off'),
+                span('Excluded data element'),
+            ),
+        );
+    }
     return span(
         { class: 'flex-row fx-gap-1', style: `width: ${attributeWidth}px` },
-        Icon(
-            { size: 20, classes: value ? 'text-brown' : 'text-disabled' },
-            value ? 'visibility_off' : 'visibility',
-        ),
-        span(
-            { class: value ? '' : 'text-secondary' },
-            value ? 'Excluded data element' : 'Not an excluded data element',
-        ),
+        Icon({ size: 20, classes: 'text-disabled' }, 'visibility'),
+        span({ class: 'text-secondary' }, 'Not an excluded data element'),
     );
 }
 
@@ -273,13 +282,13 @@ const PiiDisplay = (/** @type string|null */ value) => {
             const typeLabel = pii_type_map[type];
             caption = `${pii_risk_map[risk] ?? 'Moderate'} Risk${typeLabel ? ' - ' + typeLabel : ''}${detail && detail !== typeLabel ? ' / ' + detail : ''}`;
         }
-        return span(
-            { class: 'flex-row fx-gap-1', style: `width: ${attributeWidth}px` },
-            Icon({ size: 24, classes: 'text-orange' }, 'shield_person'),
-            div(
-                { class: 'flex-column fx-gap-1' },
+        return div(
+            { style: `width: ${attributeWidth}px` },
+            span(
+                { class: 'flex-row fx-gap-1 metadata-badge pii' },
+                Icon({ size: 21, classes: 'text-orange' }, 'shield_person'),
                 span('PII data'),
-                caption ? span({ class: 'text-caption' }, caption) : null,
+                caption ? withTooltip(Icon({ size: 16 }, 'help'), { text: caption }) : null,
             ),
         );
     }
@@ -300,8 +309,8 @@ const MetadataTagsMultiEdit = (props, selectedItems) => {
 
     const attributes = [
         'critical_data_element',
-        'excluded_data_element',
         'pii_flag',
+        'excluded_data_element',
         ...TAG_KEYS,
     ].map(key => ({
         key,
@@ -451,4 +460,31 @@ const WarningDialog = (open, pendingAction, warnCde, warnPii) => {
     );
 };
 
+const stylesheet = new CSSStyleSheet();
+stylesheet.replace(`
+.metadata-badge {
+    display: inline-flex;
+    padding: 4px 12px 4px 6px;
+    border-radius: 15px;
+    height: 30px;
+    box-sizing: border-box;
+}
+
+.metadata-badge.cde {
+    background-color: rgba(171, 71, 188, 0.15);
+}
+
+.metadata-badge.cde i {
+    margin-top: -3px;
+}
+
+.metadata-badge.pii {
+    background-color: rgba(255, 152, 0, 0.15);
+}
+
+.metadata-badge.xde {
+    background-color: rgba(141, 110, 99, 0.15);
+}
+`);
+
 export { MetadataTagsCard, MetadataTagsMultiEdit, TAG_KEYS };