Skip to content

Commit b092836

Browse files
author
ci bot
committed
Merge branch 'TG-940' into 'enterprise'
feat(data-catalog): log column updates See merge request dkinternal/testgen/dataops-testgen!325
2 parents 5bf641c + f07de29 commit b092836

4 files changed

Lines changed: 136 additions & 76 deletions

File tree

‎testgen/template/data_chars/data_chars_update.sql‎

Lines changed: 113 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -124,23 +124,41 @@ WITH new_chars AS (
124124
run_date
125125
FROM {SOURCE_TABLE}
126126
WHERE table_groups_id = :TABLE_GROUPS_ID
127+
),
128+
update_chars AS (
129+
UPDATE data_column_chars
130+
SET ordinal_position = n.position,
131+
general_type = n.general_type,
132+
column_type = n.column_type,
133+
functional_data_type = COALESCE(n.functional_data_type, d.functional_data_type),
134+
last_mod_date = CASE WHEN n.column_type <> d.column_type THEN n.run_date ELSE d.last_mod_date END,
135+
drop_date = NULL
136+
FROM new_chars n
137+
INNER JOIN data_column_chars d ON (
138+
n.table_groups_id = d.table_groups_id
139+
AND n.schema_name = d.schema_name
140+
AND n.table_name = d.table_name
141+
AND n.column_name = d.column_name
142+
)
143+
WHERE data_column_chars.table_id = d.table_id
144+
AND data_column_chars.column_name = d.column_name
145+
RETURNING data_column_chars.*, d.column_type as old_column_type
127146
)
128-
UPDATE data_column_chars
129-
SET ordinal_position = n.position,
130-
general_type = n.general_type,
131-
column_type = n.column_type,
132-
functional_data_type = COALESCE(n.functional_data_type, d.functional_data_type),
133-
last_mod_date = CASE WHEN n.column_type <> d.column_type THEN n.run_date ELSE d.last_mod_date END,
134-
drop_date = NULL
135-
FROM new_chars n
136-
INNER JOIN data_column_chars d ON (
137-
n.table_groups_id = d.table_groups_id
138-
AND n.schema_name = d.schema_name
139-
AND n.table_name = d.table_name
140-
AND n.column_name = d.column_name
141-
)
142-
WHERE data_column_chars.table_id = d.table_id
143-
AND data_column_chars.column_name = d.column_name;
147+
INSERT INTO data_structure_log (
148+
element_id,
149+
change_date,
150+
change,
151+
old_column_type,
152+
new_column_type
153+
)
154+
SELECT u.column_id,
155+
u.last_mod_date,
156+
'M',
157+
u.old_column_type,
158+
u.column_type
159+
FROM update_chars u
160+
WHERE u.old_column_type <> u.column_type;
161+
144162

145163
-- Add new records
146164
WITH new_chars AS (
@@ -155,44 +173,58 @@ WITH new_chars AS (
155173
run_date
156174
FROM {SOURCE_TABLE}
157175
WHERE table_groups_id = :TABLE_GROUPS_ID
176+
),
177+
inserted_records AS (
178+
INSERT INTO data_column_chars (
179+
table_groups_id,
180+
schema_name,
181+
table_name,
182+
table_id,
183+
column_name,
184+
ordinal_position,
185+
general_type,
186+
column_type,
187+
functional_data_type,
188+
add_date,
189+
last_mod_date
190+
)
191+
SELECT n.table_groups_id,
192+
n.schema_name,
193+
n.table_name,
194+
dtc.table_id,
195+
n.column_name,
196+
n.position,
197+
n.general_type,
198+
n.column_type,
199+
n.functional_data_type,
200+
n.run_date,
201+
n.run_date
202+
FROM new_chars n
203+
INNER JOIN data_table_chars dtc ON (
204+
n.table_groups_id = dtc.table_groups_id
205+
AND n.schema_name = dtc.schema_name
206+
AND n.table_name = dtc.table_name
207+
)
208+
LEFT JOIN data_column_chars d ON (
209+
n.table_groups_id = d.table_groups_id
210+
AND n.schema_name = d.schema_name
211+
AND n.table_name = d.table_name
212+
AND n.column_name = d.column_name
213+
)
214+
WHERE d.table_id IS NULL
215+
RETURNING data_column_chars.*
158216
)
159-
INSERT INTO data_column_chars (
160-
table_groups_id,
161-
schema_name,
162-
table_name,
163-
table_id,
164-
column_name,
165-
ordinal_position,
166-
general_type,
167-
column_type,
168-
functional_data_type,
169-
add_date,
170-
last_mod_date
171-
)
172-
SELECT n.table_groups_id,
173-
n.schema_name,
174-
n.table_name,
175-
dtc.table_id,
176-
n.column_name,
177-
n.position,
178-
n.general_type,
179-
n.column_type,
180-
n.functional_data_type,
181-
n.run_date,
182-
n.run_date
183-
FROM new_chars n
184-
INNER JOIN data_table_chars dtc ON (
185-
n.table_groups_id = dtc.table_groups_id
186-
AND n.schema_name = dtc.schema_name
187-
AND n.table_name = dtc.table_name
188-
)
189-
LEFT JOIN data_column_chars d ON (
190-
n.table_groups_id = d.table_groups_id
191-
AND n.schema_name = d.schema_name
192-
AND n.table_name = d.table_name
193-
AND n.column_name = d.column_name
194-
)
195-
WHERE d.table_id IS NULL;
217+
INSERT INTO data_structure_log (
218+
element_id,
219+
change_date,
220+
change,
221+
new_column_type
222+
)
223+
SELECT i.column_id,
224+
i.add_date,
225+
'A',
226+
i.column_type
227+
FROM inserted_records i;
196228

197229
-- Mark dropped records
198230
WITH new_chars AS (
@@ -209,18 +241,32 @@ last_run AS (
209241
FROM {SOURCE_TABLE}
210242
WHERE table_groups_id = :TABLE_GROUPS_ID
211243
GROUP BY table_groups_id
244+
),
245+
deleted_records AS (
246+
UPDATE data_column_chars
247+
SET drop_date = l.last_run_date
248+
FROM last_run l
249+
INNER JOIN data_column_chars d ON (l.table_groups_id = d.table_groups_id)
250+
LEFT JOIN new_chars n ON (
251+
d.table_groups_id = n.table_groups_id
252+
AND d.schema_name = n.schema_name
253+
AND d.table_name = n.table_name
254+
AND d.column_name = n.column_name
255+
)
256+
WHERE data_column_chars.table_id = d.table_id
257+
AND data_column_chars.column_name = d.column_name
258+
AND d.drop_date IS NULL
259+
AND n.column_name IS NULL
260+
RETURNING data_column_chars.*
212261
)
213-
UPDATE data_column_chars
214-
SET drop_date = l.last_run_date
215-
FROM last_run l
216-
INNER JOIN data_column_chars d ON (l.table_groups_id = d.table_groups_id)
217-
LEFT JOIN new_chars n ON (
218-
d.table_groups_id = n.table_groups_id
219-
AND d.schema_name = n.schema_name
220-
AND d.table_name = n.table_name
221-
AND d.column_name = n.column_name
222-
)
223-
WHERE data_column_chars.table_id = d.table_id
224-
AND data_column_chars.column_name = d.column_name
225-
AND d.drop_date IS NULL
226-
AND n.column_name IS NULL;
262+
INSERT INTO data_structure_log (
263+
element_id,
264+
change_date,
265+
change,
266+
old_column_type
267+
)
268+
SELECT del.column_id,
269+
del.drop_date,
270+
'D',
271+
del.column_type
272+
FROM deleted_records del;

‎testgen/template/dbsetup/030_initialize_new_schema_structure.sql‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -366,15 +366,14 @@ CREATE TABLE profile_pair_rules (
366366

367367

368368
CREATE TABLE data_structure_log (
369-
project_code VARCHAR(30),
370-
connection_id BIGINT,
371-
change_date TIMESTAMP,
372-
schema_name VARCHAR(50),
373-
table_name VARCHAR(100),
374-
ordinal_position INTEGER,
375-
column_name VARCHAR(100),
376-
data_type VARCHAR(50),
377-
status VARCHAR(10)
369+
log_id UUID DEFAULT gen_random_uuid()
370+
CONSTRAINT pk_dsl_id
371+
PRIMARY KEY,
372+
element_id UUID,
373+
change_date TIMESTAMP,
374+
change VARCHAR(10),
375+
old_column_type VARCHAR(50),
376+
new_column_type VARCHAR(50)
378377
);
379378

380379
CREATE TABLE data_table_chars (

‎testgen/template/dbsetup/075_grant_role_rights.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ GRANT SELECT, INSERT, DELETE, UPDATE ON
3131
{SCHEMA_NAME}.projects,
3232
{SCHEMA_NAME}.data_table_chars,
3333
{SCHEMA_NAME}.data_column_chars,
34+
{SCHEMA_NAME}.data_structure_log,
3435
{SCHEMA_NAME}.auth_users,
3536
{SCHEMA_NAME}.score_definitions,
3637
{SCHEMA_NAME}.score_definition_criteria,

‎testgen/template/dbupgrade/0155_incremental_upgrade.sql‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
SET SEARCH_PATH TO {SCHEMA_NAME};
2+
3+
DROP TABLE data_structure_log;
4+
5+
CREATE TABLE data_structure_log (
6+
log_id UUID DEFAULT gen_random_uuid()
7+
CONSTRAINT pk_dsl_id
8+
PRIMARY KEY,
9+
element_id UUID,
10+
change_date TIMESTAMP,
11+
change VARCHAR(10),
12+
old_column_type VARCHAR(50),
13+
new_column_type VARCHAR(50)
14+
);

0 commit comments

Comments
 (0)