Skip to content

Commit ac2567c

Browse files
aarthy-dkaarthy-dk
committed
Merge branch 'aarthy/sql-fixes' into 'enterprise'
SQL fixes + Functional test improvements See merge request dkinternal/testgen/dataops-testgen!312
2 parents 4fe163a + f44230c commit ac2567c

6 files changed

Lines changed: 47 additions & 46 deletions

File tree

‎testgen/template/flavors/mssql/profiling/project_profiling_query_mssql.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ strTemplate05_else: NULL as distinct_std_value_ct,
131131
NULL as std_pattern_match,
132132
strTemplate06_A_patterns: ( SELECT LEFT(STRING_AGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats
133133
FROM (
134-
SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) + ' | ' + pattern AS pattern,
134+
SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(40)) + ' | ' + pattern AS pattern,
135135
COUNT(*) AS ct
136136
FROM ( SELECT TRANSLATE("{COL_NAME}" COLLATE Latin1_General_BIN,
137137
'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789',

‎testgen/template/flavors/mssql/profiling/templated_functions.yaml‎

Lines changed: 42 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -3,44 +3,45 @@ IS_NUM: CASE
33
ELSE 0
44
END
55

6-
IS_DATE: CASE WHEN TRY_CAST(NULLIF({$1}, '') AS float) IS NOT NULL
7-
AND LEFT(NULLIF({$1}, ''),4) BETWEEN 1800 AND 2200 THEN
8-
CASE
9-
WHEN LEN((NULLIF({$1}, ''))) > 11 THEN 0
10-
/* YYYYMMDD */
11-
WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 112) IS NOT NULL THEN 1
12-
13-
/* YYYY-MM-DD */
14-
WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 23) IS NOT NULL THEN 1
15-
16-
/* MM/DD/YYYY */
17-
WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 101) IS NOT NULL THEN 1
18-
19-
/* MM/DD/YY */
20-
WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 1) IS NOT NULL THEN 1
21-
22-
/*MM-DD-YYYY */
23-
WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 110) IS NOT NULL THEN 1
24-
25-
/*MM-DD-YY */
26-
WHEN TRY_CONVERT(DATE, NULLIF({$1}, ''), 10) IS NOT NULL THEN 1
27-
28-
29-
ELSE 0 END
30-
/*DD MMM YYYY */
31-
WHEN (TRY_CONVERT(DATE, NULLIF({$1}, ''), 106) IS NOT NULL
32-
AND LEFT(NULLIF({$1}, ''), 4) BETWEEN 1800 AND 2200)
33-
THEN 1
34-
35-
/* YYYY-MM-DD HH:MM:SS SSSSSS */
36-
WHEN (TRY_CONVERT(DATETIME2, NULLIF({$1}, ''), 121) IS NOT NULL
37-
AND LEFT(NULLIF({$1}, ''), 4) BETWEEN 1800 AND 2200)
38-
THEN 1
39-
40-
/* YYYY-MM-DD HH:MM:SS */
41-
WHEN (TRY_CONVERT(DATETIME2, NULLIF({$1}, ''), 120) IS NOT NULL
42-
AND LEFT(NULLIF({$1}, ''), 4) BETWEEN 1800 AND 2200)
43-
THEN 1
44-
ELSE 0
45-
END
46-
6+
IS_DATE: |-
7+
CASE
8+
WHEN NULLIF(LTRIM(RTRIM({$1})), '') IS NULL THEN 0
9+
WHEN (
10+
COALESCE(
11+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 112), /* YYYYMMDD */
12+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 23), /* YYYY-MM-DD */
13+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 101), /* MM/DD/YYYY */
14+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 1), /* MM/DD/YY (2-digit year windowing) */
15+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 110), /* MM-DD-YYYY */
16+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 10), /* MM-DD-YY (2-digit year windowing) */
17+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 106), /* DD MMM YYYY */
18+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 107), /* MMM dd, YYYY */
19+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 126), /* ISO 8601 no TZ: YYYY-MM-DDTHH:MM:SS[.fff] */
20+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 120), /* YYYY-MM-DD HH:MM:SS */
21+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 121), /* YYYY-MM-DD HH:MM:SS.mmm */
22+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 111), /* YYYY/MM/DD */
23+
TRY_CONVERT(datetime2, LTRIM(RTRIM({$1})), 102), /* YYYY.MM.DD */
24+
/* Normalize "YYYY-MM-DD HH:MM:SS SSSSSS" -> ISO 8601 with 'T' and '.' */
25+
TRY_CONVERT(
26+
datetime2,
27+
CASE
28+
WHEN LTRIM(RTRIM({$1})) LIKE
29+
'[0-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] [0-9]%'
30+
THEN
31+
/* Replace DT space (pos 11) with 'T', and the last space with '.' */
32+
STUFF(
33+
STUFF(LTRIM(RTRIM({$1})), 11, 1, 'T'),
34+
LEN(LTRIM(RTRIM({$1}))) - CHARINDEX(' ', REVERSE(LTRIM(RTRIM({$1})))) + 1,
35+
1,
36+
'.'
37+
)
38+
ELSE LTRIM(RTRIM({$1}))
39+
END,
40+
126
41+
)) BETWEEN '1800-01-01T00:00:00' AND '2200-12-31T23:59:59.9999999' )
42+
OR ( /* ISO 8601 with timezone: style 127 parsed as datetimeoffset, then cast */
43+
CAST(TRY_CONVERT(datetimeoffset, LTRIM(RTRIM({$1})), 127) AS datetime2)
44+
BETWEEN '1800-01-01T00:00:00' AND '2200-12-31T23:59:59.9999999' )
45+
THEN 1
46+
ELSE 0
47+
END

‎testgen/template/flavors/postgresql/profiling/project_profiling_query_postgresql.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ strTemplate05_else: NULL as distinct_std_value_ct,
107107
NULL as std_pattern_match,
108108
strTemplate06_A_patterns: ( SELECT LEFT(STRING_AGG(pattern, ' | ' ORDER BY ct DESC) , 1000) AS concat_pats
109109
FROM (
110-
SELECT CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
110+
SELECT CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern,
111111
COUNT(*) AS ct
112112
FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
113113
"{COL_NAME}", '[a-z]', 'a', 'g'),

‎testgen/template/flavors/redshift/profiling/project_profiling_query_redshift.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ strTemplate05_else: NULL as distinct_std_value_ct,
8686
NULL as date_ct,
8787
NULL as std_pattern_match,
8888
strTemplate06_A_patterns: (SELECT LEFT(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats
89-
FROM ( SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
89+
FROM ( SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern,
9090
COUNT(*) AS ct
9191
FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
9292
"{COL_NAME}", '[a-z]', 'a'),

‎testgen/template/flavors/snowflake/profiling/project_profiling_query_snowflake.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ strTemplate05_else: NULL as distinct_std_value_ct,
9494
NULL as std_pattern_match,
9595
strTemplate06_A_patterns: ( SELECT LEFT(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1000) AS concat_pats
9696
FROM (
97-
SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
97+
SELECT TOP 5 CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern,
9898
COUNT(*) AS ct
9999
FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
100100
"{COL_NAME}"::VARCHAR, '[a-z]', 'a'),

‎testgen/template/flavors/trino/profiling/project_profiling_query_trino.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,7 +107,7 @@ strTemplate05_else: NULL as distinct_std_value_ct,
107107
NULL as date_ct,
108108
NULL as std_pattern_match,
109109
strTemplate06_A_patterns: (SELECT SUBSTRING(LISTAGG(pattern, ' | ') WITHIN GROUP (ORDER BY ct DESC), 1, 1000) AS concat_pats
110-
FROM ( SELECT CAST(COUNT(*) AS VARCHAR(10)) || ' | ' || pattern AS pattern,
110+
FROM ( SELECT CAST(COUNT(*) AS VARCHAR(40)) || ' | ' || pattern AS pattern,
111111
COUNT(*) AS ct
112112
FROM ( SELECT REGEXP_REPLACE(REGEXP_REPLACE( REGEXP_REPLACE(
113113
"{COL_NAME}", '[a-z]', 'a'),

0 commit comments

Comments
 (0)