diff --git a/.gitignore b/.gitignore index fd8c885..fe7f651 100644 --- a/.gitignore +++ b/.gitignore @@ -1,54 +1,46 @@ -*.secret -*.env -.env* +# Secrets & environment-specific config (NEVER commit) config.local.env -local_*.sql -*_local.sql -*_secrets.sql -environments/*_local.sql -environments/*_override.sql -*.dump -*.sql.gz -*.backup -*.bak -pg_dump_* -/tmp/ -test_output/ -*.out -*.log -csv/logs/ +*.local.env +build/environments/env_dev.sql +build/environments/env_test.sql +build/environments/env_staging.sql +build/environments/env_prod.sql +!build/environments/*.example.sql +*.tfvars +!*.tfvars.example +*.pgpass +.env +.env.* +!.env.example + +# Terraform +**/.terraform/ +*.tfstate +*.tfstate.* +terraform.tfvars + +# Eval / test runtime output evals/reports/ -*_report_*.txt -*_skipped_*.csv -*_valid_*.csv -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -Thumbs.db -ehthumbs.db -Desktop.ini +**/csv/logs/ + +# Tool / session / editor folders +.abacusai/ +.claude/ +.cursor/ .vscode/ .idea/ -*.swp -*.swo -*~ -*.orig + +# Python __pycache__/ *.py[cod] -*.pyo +.pytest_cache/ .venv/ venv/ -node_modules/ -npm-debug.log* -terraform-github-repos/.terraform/ -terraform-github-repos/*.tfstate -terraform-github-repos/*.tfstate.* -terraform-github-repos/crash.log -.abacusai/ -# Terraform plan artifacts (binary, environment-specific, never commit) -*.tfplan -tfplan -.terraform.lock.hcl +# OS +.DS_Store +Thumbs.db + +# Exploratory / not-yet-wired folders +_norton_/ +extend to Oracle dbs/ diff --git a/build/csv/validator.py b/build/csv/validator.py index 729aa80..59df90b 100644 --- a/build/csv/validator.py +++ b/build/csv/validator.py @@ -1,137 +1,82 @@ -#!/usr/bin/env python3 -# ============================================================================= -# csv/validator.py — CSV Validator (Windows / Mac / Linux compatible) -# ============================================================================= -# Called by csv_loader.sh via: python3 csv/validator.py -# -# Reads environment variables: -# CSV_FILE — path to the source CSV file -# VALID_CSV — path to write accepted rows -# SKIP_FILE — path to write rejected rows with reasons -# TABLE_NAME — target table name (used for logging only) -# -# Exit codes: -# 0 — validation passed (at least one valid row found) -# 1 — validation failed (no valid rows or file error) -# ============================================================================= - -import csv -import os -import sys - -# ── Colours (work on Windows 10+ terminal and Git Bash) ────────────────────── -GREEN = '\033[0;32m' -YELLOW = '\033[1;33m' -RED = '\033[0;31m' -NC = '\033[0m' - - -def log(msg): print(f"{GREEN} [validator OK]{NC} {msg}") - - -def warn(msg): print(f"{YELLOW} [validator WARN]{NC} {msg}") - - -def err(msg): print(f"{RED} [validator ERR]{NC} {msg}", file=sys.stderr) - - -# ── Read environment variables ──────────────────────────────────────────────── -CSV_FILE = os.environ.get('CSV_FILE', '') -VALID_CSV = os.environ.get('VALID_CSV', '') -SKIP_FILE = os.environ.get('SKIP_FILE', '') -TABLE_NAME = os.environ.get('TABLE_NAME', 'unknown') - -# Validate required vars -missing = [v for v in ('CSV_FILE', 'VALID_CSV', 'SKIP_FILE') if not os.environ.get(v)] +#!/usr/bin/env python3 +# csv/validator.py — handles invalid UTF-8 per-row + delimiter auto-detect +import csv, os, sys + +GREEN, YELLOW, RED, NC = '\033[0;32m', '\033[1;33m', '\033[0;31m', '\033[0m' +def log(m): print(f"{GREEN} [validator OK]{NC} {m}") +def warn(m): print(f"{YELLOW} [validator WARN]{NC} {m}") +def err(m): print(f"{RED} [validator ERR]{NC} {m}", file=sys.stderr) + +def has_bad_bytes(cell): + try: + cell.encode("utf-8"); return False + except UnicodeEncodeError: + return True +def clean(cell): + return cell.encode("utf-8", "replace").decode("utf-8") + +CSV_FILE = os.environ.get("CSV_FILE", "") +VALID_CSV = os.environ.get("VALID_CSV", "") +SKIP_FILE = os.environ.get("SKIP_FILE", "") +FORCE_DELIM = os.environ.get("CSV_DELIMITER", "") + +missing = [v for v in ("CSV_FILE","VALID_CSV","SKIP_FILE") if not os.environ.get(v)] if missing: - err(f"Missing required environment variables: {', '.join(missing)}") - sys.exit(1) - -# ── Check source file exists ────────────────────────────────────────────────── + err(f"Missing required environment variables: {', '.join(missing)}"); sys.exit(1) if not os.path.isfile(CSV_FILE): - err(f"CSV file not found: {CSV_FILE}") - sys.exit(1) + err(f"CSV file not found: {CSV_FILE}"); sys.exit(1) +os.makedirs(os.path.dirname(VALID_CSV) or ".", exist_ok=True) +os.makedirs(os.path.dirname(SKIP_FILE) or ".", exist_ok=True) + +def detect_delimiter(path): + if FORCE_DELIM: + return FORCE_DELIM.encode().decode("unicode_escape") + try: + with open(path, "r", encoding="utf-8-sig", errors="surrogateescape", newline="") as fh: + sample = fh.read(8192) + if not sample: return "," + return csv.Sniffer().sniff(sample, delimiters=",;\t|").delimiter + except Exception: + return "," + +DELIM = detect_delimiter(CSV_FILE) +label = {",":"comma",";":"semicolon","\t":"tab","|":"pipe"}.get(DELIM, repr(DELIM)) -# ── Ensure output directories exist ────────────────────────────────────────── -os.makedirs(os.path.dirname(VALID_CSV), exist_ok=True) -os.makedirs(os.path.dirname(SKIP_FILE), exist_ok=True) - -# ── Open and validate CSV ───────────────────────────────────────────────────── try: - with open(CSV_FILE, 'r', encoding='utf-8-sig', newline='') as src, \ - open(VALID_CSV, 'w', encoding='utf-8', newline='') as vf, \ - open(SKIP_FILE, 'w', encoding='utf-8', newline='') as sf: - - reader = csv.reader(src) - writer_v = csv.writer(vf) - writer_s = csv.writer(sf) - - # ── Read and validate header ────────────────────────────────────────── + with open(CSV_FILE, "r", encoding="utf-8-sig", errors="surrogateescape", newline="") as src, \ + open(VALID_CSV, "w", encoding="utf-8", newline="") as vf, \ + open(SKIP_FILE, "w", encoding="utf-8", newline="") as sf: + reader, wv, ws = csv.reader(src, delimiter=DELIM), csv.writer(vf, delimiter=DELIM), csv.writer(sf, delimiter=DELIM) try: headers = next(reader) except StopIteration: - err("CSV file is empty — no header row found.") - sys.exit(1) - - # Strip whitespace from header names - headers = [h.strip() for h in headers] - ncols = len(headers) - + err("CSV file is empty — no header row found."); sys.exit(1) + headers = [h.strip() for h in headers]; ncols = len(headers) if ncols == 0: - err("Header row is empty.") - sys.exit(1) - - log(f"Header: {ncols} columns detected — {' | '.join(headers)}") - - # Check for duplicate column names - dupes = [h for h in set(headers) if headers.count(h) > 1] - if dupes: - warn(f"Duplicate column names: {', '.join(dupes)}") - - # Write headers to both output files - writer_v.writerow(headers) - writer_s.writerow(headers + ['_skip_reason']) - - # ── Process each data row ───────────────────────────────────────────── - valid_count = 0 - skip_count = 0 - - for line_num, row in enumerate(reader, start=2): - - # Rule 1: skip completely empty rows - if not any(cell.strip() for cell in row): - writer_s.writerow(row + ['empty row — all values blank']) - skip_count += 1 - continue - - # Rule 2: column count must match header + err("Header row is empty."); sys.exit(1) + if any(has_bad_bytes(h) for h in headers): + err("Header row contains invalid UTF-8 bytes — cannot process file."); sys.exit(1) + log(f"Delimiter: {label}") + log(f"Header: {ncols} columns — {' | '.join(headers)}") + dupes = sorted({h for h in headers if headers.count(h) > 1}) + if dupes: warn(f"Duplicate column names: {', '.join(dupes)}") + wv.writerow(headers); ws.writerow(headers + ["_skip_reason"]) + valid = skip = 0 + for n, row in enumerate(reader, start=2): + if not any(c.strip() for c in row): + ws.writerow([clean(c) for c in row] + ["empty row — all values blank"]); skip += 1; continue + if any(has_bad_bytes(c) for c in row): + ws.writerow([clean(c) for c in row] + [f"invalid UTF-8 bytes at line {n}"]); skip += 1; continue if len(row) != ncols: - reason = f'column mismatch — expected {ncols}, got {len(row)}' - writer_s.writerow(row + [reason]) - skip_count += 1 - continue - - # Valid row - writer_v.writerow(row) - valid_count += 1 - - # ── Summary ─────────────────────────────────────────────────────────── - total = valid_count + skip_count - log(f"Validation complete — {total} rows processed.") - log(f" Valid rows : {valid_count}") - - if skip_count > 0: - warn(f" Skipped rows : {skip_count} — written to: {SKIP_FILE}") - - if valid_count == 0: - err("No valid rows found. Nothing to load.") - sys.exit(1) - + ws.writerow([clean(c) for c in row] + [f"column mismatch — expected {ncols}, got {len(row)}"]); skip += 1; continue + wv.writerow(row); valid += 1 + log(f"Validation complete — {valid + skip} rows processed.") + log(f" Valid rows : {valid}") + if skip: warn(f" Skipped rows : {skip} — written to: {SKIP_FILE}") + if valid == 0: + err("No valid rows found. Nothing to load."); sys.exit(1) sys.exit(0) - except PermissionError as e: - err(f"Permission denied: {e}") - sys.exit(1) + err(f"Permission denied: {e}"); sys.exit(1) except Exception as e: - err(f"Unexpected error: {e}") - sys.exit(1) + err(f"Unexpected error: {e}"); sys.exit(1) diff --git a/build/environments/env_dev.example.sql b/build/environments/env_dev.example.sql new file mode 100644 index 0000000..7dc5aeb --- /dev/null +++ b/build/environments/env_dev.example.sql @@ -0,0 +1,11 @@ +-- env_dev.example.sql (TEMPLATE - safe to commit; no real secrets) +-- Copy to env_dev.sql (gitignored) and fill in, or inject app_password at deploy. +\set env_label DEV +\set db_name te_mgmt_dev +\set db_owner postgres +\set schema_name te_dev +\set app_user te_dev_user +\set app_password '__INJECT_AT_DEPLOY__' +\set conn_limit 10 +\set include_seed_data true +\ir ../te_core_schema.sql diff --git a/build/environments/env_dev.sql b/build/environments/env_dev.sql deleted file mode 100644 index de2c1d7..0000000 --- a/build/environments/env_dev.sql +++ /dev/null @@ -1,44 +0,0 @@ --- ============================================================================= --- ENVIRONMENT: DEV --- Usage: psql -U postgres -f environments/env_dev.sql --- ============================================================================= - --- ── ★ EDIT THESE VALUES TO RECONFIGURE THE DEV ENVIRONMENT ★ ──────────────── - -\set env_label DEV - --- Database -\set db_name te_mgmt_dev -\set db_owner postgres - --- Schema -\set schema_name te_dev - --- Application user -\set app_user te_dev_user -\set app_password Dev@Local#2025! -\set conn_limit 10 - --- Table names (change all here if you need to rename) -\set tbl_organisations organisations -\set tbl_personnel personnel -\set tbl_test_programs test_programs -\set tbl_temp_documents temp_documents -\set tbl_test_phases test_phases -\set tbl_requirements requirements -\set tbl_test_cases test_cases -\set tbl_vcrm_entries vcrm_entries -\set tbl_test_events test_events -\set tbl_test_results test_results -\set tbl_defect_reports defect_reports -\set tbl_evidence_artifacts evidence_artifacts - --- Seed data: 'true' to load realistic T&E data, 'false' for empty tables -\set include_seed_data true - --- ── END OF CONFIGURATION ───────────────────────────────────────────────────── - --- \ir (include relative) resolves relative to THIS file's directory, so --- the deploy works regardless of the caller's cwd. \i would have been --- cwd-relative and broken under `psql --file=` from any other directory. -\ir ../te_core_schema.sql diff --git a/build/environments/env_prod.sql b/build/environments/env_prod.sql deleted file mode 100644 index 41c928f..0000000 --- a/build/environments/env_prod.sql +++ /dev/null @@ -1,60 +0,0 @@ --- ============================================================================= --- ENVIRONMENT: PROD --- Usage: psql -U postgres -f environments/env_prod.sql --- --- ⚠ PRODUCTION — Run with care. --- - Seed data is DISABLED (schema only). --- - Use a secrets manager (e.g. Azure Key Vault, HashiCorp Vault) --- to inject app_password rather than storing it here. --- - Rotate the password after first deployment. --- ============================================================================= - --- ── ★ EDIT THESE VALUES TO RECONFIGURE THE PROD ENVIRONMENT ★ ─────────────── - -\set env_label PROD - --- Database -\set db_name te_mgmt_prod -\set db_owner postgres - --- Schema -\set schema_name te_prod - --- Application user -\set app_user te_prod_user -\set conn_limit 50 - --- ⚠ app_password MUST be injected from your secrets manager. --- Pass via: psql -v app_password="$(az keyvault secret show ...)" -f env_prod.sql --- The line below is INTENTIONALLY commented out — uncommenting it for --- production is a security incident. --- \set app_password 'do-not-commit-real-passwords-here' - --- Table names -\set tbl_organisations organisations -\set tbl_personnel personnel -\set tbl_test_programs test_programs -\set tbl_temp_documents temp_documents -\set tbl_test_phases test_phases -\set tbl_requirements requirements -\set tbl_test_cases test_cases -\set tbl_vcrm_entries vcrm_entries -\set tbl_test_events test_events -\set tbl_test_results test_results -\set tbl_defect_reports defect_reports -\set tbl_evidence_artifacts evidence_artifacts - --- Seed data: always false for production -\set include_seed_data false - --- ── END OF CONFIGURATION ───────────────────────────────────────────────────── - --- Fail fast if caller did not inject app_password. -\if :{?app_password} -\else - \echo 'ERROR: -v app_password= is required for env_prod.sql.' - \echo 'Example: psql -v app_password="$(az keyvault secret show ...)" -f env_prod.sql' - \quit -\endif - -\ir ../te_core_schema.sql diff --git a/build/environments/env_staging.sql b/build/environments/env_staging.sql deleted file mode 100644 index 6644311..0000000 --- a/build/environments/env_staging.sql +++ /dev/null @@ -1,53 +0,0 @@ --- ============================================================================= --- ENVIRONMENT: STAGING --- Usage: psql -U postgres -f environments/env_staging.sql --- ============================================================================= - --- ── ★ EDIT THESE VALUES TO RECONFIGURE THE STAGING ENVIRONMENT ★ ───────────── - -\set env_label STAGING - --- Database -\set db_name te_mgmt_staging -\set db_owner postgres - --- Schema -\set schema_name te_staging - --- Application user -\set app_user te_stg_user -\set conn_limit 20 - --- app_password should be injected from your secrets manager. --- Pass via: psql -v app_password="$(az keyvault secret show ...)" -f env_staging.sql --- Uncomment the next line only for non-secret local experiments: --- \set app_password 'Stg@Secure#2025!' - --- Table names -\set tbl_organisations organisations -\set tbl_personnel personnel -\set tbl_test_programs test_programs -\set tbl_temp_documents temp_documents -\set tbl_test_phases test_phases -\set tbl_requirements requirements -\set tbl_test_cases test_cases -\set tbl_vcrm_entries vcrm_entries -\set tbl_test_events test_events -\set tbl_test_results test_results -\set tbl_defect_reports defect_reports -\set tbl_evidence_artifacts evidence_artifacts - --- Seed data: false for staging — schema only, no pre-loaded data --- Load your own anonymised data snapshot after deployment if needed -\set include_seed_data false - --- ── END OF CONFIGURATION ───────────────────────────────────────────────────── - --- Fail fast if caller did not inject app_password. -\if :{?app_password} -\else - \echo 'ERROR: -v app_password= is required for env_staging.sql.' - \quit -\endif - -\ir ../te_core_schema.sql diff --git a/build/environments/env_test.sql b/build/environments/env_test.sql deleted file mode 100644 index f7357c2..0000000 --- a/build/environments/env_test.sql +++ /dev/null @@ -1,41 +0,0 @@ --- ============================================================================= --- ENVIRONMENT: TEST --- Usage: psql -U postgres -f environments/env_test.sql --- ============================================================================= - --- ── ★ EDIT THESE VALUES TO RECONFIGURE THE TEST ENVIRONMENT ★ ─────────────── - -\set env_label TEST - --- Database -\set db_name te_mgmt_test -\set db_owner postgres - --- Schema -\set schema_name te_test - --- Application user -\set app_user te_test_user -\set app_password Test@Env#2025! -\set conn_limit 15 - --- Table names -\set tbl_organisations organisations -\set tbl_personnel personnel -\set tbl_test_programs test_programs -\set tbl_temp_documents temp_documents -\set tbl_test_phases test_phases -\set tbl_requirements requirements -\set tbl_test_cases test_cases -\set tbl_vcrm_entries vcrm_entries -\set tbl_test_events test_events -\set tbl_test_results test_results -\set tbl_defect_reports defect_reports -\set tbl_evidence_artifacts evidence_artifacts - --- Seed data: full realistic seed — same as dev so tests run against known data -\set include_seed_data true - --- ── END OF CONFIGURATION ───────────────────────────────────────────────────── - -\ir ../te_core_schema.sql diff --git a/build/schema/postgresql/te_core_schema.sql b/build/schema/postgresql/te_core_schema.sql index 7dffcb2..1b42804 100644 --- a/build/schema/postgresql/te_core_schema.sql +++ b/build/schema/postgresql/te_core_schema.sql @@ -29,48 +29,44 @@ -- PHASE 1: DATABASE & USER -- ============================================================================= -\echo '>> [1/6] Creating database:' :db_name +-- Store psql variables as session parameters so DO blocks can access them. +-- (psql does not substitute :varname inside $$ dollar-quoted strings.) +SELECT + set_config('te.env_label', :'env_label', false), + set_config('te.db_name', :'db_name', false), + set_config('te.app_user', :'app_user', false), + set_config('te.app_password', :'app_password', false), + set_config('te.conn_limit', :'conn_limit', false); -DO -$$ -BEGIN - IF NOT EXISTS (SELECT FROM pg_database WHERE datname = :'db_name') THEN - PERFORM dblink_exec( - 'dbname=' || current_database(), - 'CREATE DATABASE "' || :'db_name' || '" - WITH OWNER = "' || :'db_owner' || '" - ENCODING = ''UTF8'' - TEMPLATE = template0 - CONNECTION LIMIT = -1' - ); - RAISE NOTICE '[%] Database "%" created.', :'env_label', :'db_name'; - ELSE - RAISE NOTICE '[%] Database "%" already exists — skipping.', :'env_label', :'db_name'; - END IF; -END -$$; +\echo '>> [1/6] Creating database:' :db_name +-- Database creation is handled by deploy_all.sh before this script runs. \echo '>> [2/6] Creating application user:' :app_user DO $$ +DECLARE + v_app_user TEXT := current_setting('te.app_user'); + v_app_password TEXT := current_setting('te.app_password'); + v_conn_limit INT := current_setting('te.conn_limit')::INT; + v_env TEXT := current_setting('te.env_label'); BEGIN - IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = :'app_user') THEN + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = v_app_user) THEN EXECUTE format( 'CREATE ROLE %I WITH LOGIN PASSWORD %L NOSUPERUSER NOCREATEDB NOCREATEROLE CONNECTION LIMIT %s', - :'app_user', :'app_password', :'conn_limit' + v_app_user, v_app_password, v_conn_limit ); - RAISE NOTICE '[%] Role "%" created.', :'env_label', :'app_user'; + RAISE NOTICE '[%] Role "%" created.', v_env, v_app_user; ELSE - -- Refresh password on every run (safe for non-prod) - EXECUTE format('ALTER ROLE %I PASSWORD %L', :'app_user', :'app_password'); - RAISE NOTICE '[%] Role "%" already exists — password refreshed.', :'env_label', :'app_user'; + EXECUTE format('ALTER ROLE %I PASSWORD %L CONNECTION LIMIT %s', + v_app_user, v_app_password, v_conn_limit); + RAISE NOTICE '[%] Role "%" already exists — password and conn_limit refreshed.', v_env, v_app_user; END IF; END $$; -EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', :'db_name', :'app_user'); +GRANT ALL PRIVILEGES ON DATABASE :"db_name" TO :"app_user"; -- ============================================================================= @@ -79,6 +75,24 @@ EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %I TO %I', :'db_name', :'app_us \c :"db_name" +-- Re-establish session parameters after \c opens a new connection. +SELECT + set_config('te.env_label', :'env_label', false), + set_config('te.schema_name', :'schema_name', false), + set_config('te.app_user', :'app_user', false), + set_config('te.tbl_organisations', :'tbl_organisations', false), + set_config('te.tbl_personnel', :'tbl_personnel', false), + set_config('te.tbl_test_programs', :'tbl_test_programs', false), + set_config('te.tbl_temp_documents', :'tbl_temp_documents', false), + set_config('te.tbl_test_phases', :'tbl_test_phases', false), + set_config('te.tbl_requirements', :'tbl_requirements', false), + set_config('te.tbl_test_cases', :'tbl_test_cases', false), + set_config('te.tbl_vcrm_entries', :'tbl_vcrm_entries', false), + set_config('te.tbl_test_events', :'tbl_test_events', false), + set_config('te.tbl_test_results', :'tbl_test_results', false), + set_config('te.tbl_defect_reports', :'tbl_defect_reports', false), + set_config('te.tbl_evidence_artifacts', :'tbl_evidence_artifacts', false); + \echo '>> [3/6] Setting up schema:' :schema_name CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; @@ -88,12 +102,9 @@ CREATE EXTENSION IF NOT EXISTS "dblink"; CREATE SCHEMA IF NOT EXISTS :"schema_name"; SELECT set_config('search_path', :'schema_name' || ',public', false); -EXECUTE format('GRANT USAGE ON SCHEMA %I TO %I', :'schema_name', :'app_user'); -EXECUTE format('GRANT ALL ON ALL TABLES IN SCHEMA %I TO %I', :'schema_name', :'app_user'); -EXECUTE format( - 'ALTER DEFAULT PRIVILEGES IN SCHEMA %I GRANT ALL ON TABLES TO %I', - :'schema_name', :'app_user' -); +GRANT USAGE ON SCHEMA :"schema_name" TO :"app_user"; +GRANT ALL ON ALL TABLES IN SCHEMA :"schema_name" TO :"app_user"; +ALTER DEFAULT PRIVILEGES IN SCHEMA :"schema_name" GRANT ALL ON TABLES TO :"app_user"; -- ============================================================================= @@ -449,12 +460,17 @@ $$; DO $$ DECLARE - tbl TEXT; + tbl TEXT; + v_sch TEXT := current_setting('te.schema_name'); BEGIN FOREACH tbl IN ARRAY ARRAY[ - :'tbl_organisations', :'tbl_personnel', :'tbl_test_programs', - :'tbl_temp_documents', :'tbl_test_cases', - :'tbl_test_events', :'tbl_defect_reports' + current_setting('te.tbl_organisations'), + current_setting('te.tbl_personnel'), + current_setting('te.tbl_test_programs'), + current_setting('te.tbl_temp_documents'), + current_setting('te.tbl_test_cases'), + current_setting('te.tbl_test_events'), + current_setting('te.tbl_defect_reports') ] LOOP EXECUTE format( @@ -462,9 +478,9 @@ BEGIN CREATE TRIGGER trg_updated_at BEFORE UPDATE ON %I.%I FOR EACH ROW EXECUTE FUNCTION %I.set_updated_at();', - :'schema_name', tbl, - :'schema_name', tbl, - :'schema_name' + v_sch, tbl, + v_sch, tbl, + v_sch ); END LOOP; END; @@ -476,7 +492,7 @@ $$; -- Inserted for dev and test environments; skipped for staging and prod. -- ============================================================================= -\if :'include_seed_data' = 'true' +\if :include_seed_data \echo '>> [5/6] Seeding realistic T&E data' diff --git a/extend to Oracle dbs b/extend to Oracle dbs deleted file mode 100644 index 4467f23..0000000 --- a/extend to Oracle dbs +++ /dev/null @@ -1,111 +0,0 @@ -In order to extend the current program to Oracle databases, I need to answer the following questions. - -Here is a progressive set of questions organised by phase, from planning through to completion: - ---- - -## Phase 1 — Discovery & Planning - -1. What is the source database — type, version, size, and number of tables? -2. What is the target Oracle Database version (e.g. 19c, 21c, 23ai)? -3. What is the migration deadline and what are the acceptable downtime windows? -4. Who are the key stakeholders and what are their sign-off requirements? -5. What compliance, security, or data classification requirements apply to the data being migrated? -6. Are there any data residency or sovereignty requirements (e.g. data must stay in Australia)? -7. What tools are available — Oracle Data Pump, GoldenGate, SQL Developer, AWS DMS, or custom ETL? -8. Is this a like-for-like migration or does the schema need to be restructured? - ---- - -## Phase 2 — Source Analysis - -9. How many schemas, tables, views, stored procedures, and triggers exist in the source? -10. What is the total data volume in GB/TB and how many rows per table? -11. Which tables have the highest row counts and update frequencies? -12. Are there any unsupported data types in the source that need to be mapped to Oracle equivalents? -13. What character encoding is used in the source — and does it need to convert to AL32UTF8? -14. Are there circular foreign key dependencies that could cause issues during load order? -15. Are there any tables without primary keys or with duplicate rows? -16. What indexes, sequences, and constraints exist and need to be recreated? - ---- - -## Phase 3 — Target Environment Setup - -17. What Oracle edition is being used — Standard, Enterprise, or Autonomous? -18. Is the target on-premises, Oracle Cloud Infrastructure (OCI), or a third-party cloud? -19. What tablespace layout is required — data, index, temp, undo? -20. What user accounts, roles, and privileges need to be created in Oracle? -21. What are the storage, memory (SGA/PGA), and CPU sizing requirements? -22. Is Oracle RAC (Real Application Clusters) or Data Guard required for high availability? -23. What backup and recovery strategy will be in place during migration? -24. Will Oracle Transparent Data Encryption (TDE) be enabled on the target? - ---- - -## Phase 4 — Schema Migration - -25. How will DDL be extracted from the source and converted to Oracle syntax? -26. Which data types need explicit mapping (e.g. `INT` → `NUMBER`, `VARCHAR` → `VARCHAR2`, `DATETIME` → `TIMESTAMP`)? -27. How will auto-increment columns be handled — Oracle `IDENTITY` columns or `SEQUENCE` + `TRIGGER`? -28. How will stored procedures, functions, and packages be converted to PL/SQL? -29. Will views be migrated as-is or rewritten? -30. How will indexes be created — before or after data load — and in what order? -31. Will constraints be enabled during load or disabled and re-enabled afterwards? -32. How will database links, synonyms, and public grants be handled? - ---- - -## Phase 5 — Data Migration - -33. What is the extraction method — full export, incremental, or change data capture (CDC)? -34. What is the load sequence to respect foreign key dependencies? -35. Will Oracle Data Pump (expdp/impdp), SQL*Loader, or external tables be used for loading? -36. How will large objects (BLOB, CLOB) be handled during migration? -37. How will NULL values, default values, and computed columns be handled? -38. What batch size will be used for loading to balance performance and rollback segment size? -39. How will parallel load be configured across Oracle's parallel query slaves? -40. How will the migration handle tables that are being actively written to during migration? - ---- - -## Phase 6 — Data Validation & Testing - -41. How will row counts be compared between source and target for every table? -42. How will checksums or hash comparisons be used to validate data integrity? -43. Will a sample of records be manually cross-checked between source and target? -44. How will referential integrity be verified after load (no orphaned foreign keys)? -45. How will NULL counts, distinct value counts, and min/max values be compared? -46. What business rules or domain validations need to pass before sign-off? -47. Will the application be run against the migrated Oracle database in a test environment? -48. What performance benchmarks need to be met before go-live? - ---- - -## Phase 7 — Cutover Planning - -49. What is the cutover strategy — big bang, phased, or parallel run? -50. How long is the cutover window and what triggers a rollback? -51. Who has authority to approve go-live and who approves a rollback? -52. How will the source database be put into read-only mode during final cutover? -53. How will the delta (changes since initial load) be captured and applied to Oracle? -54. How will application connection strings be switched from source to Oracle? -55. What is the communication plan for end users during cutover? -56. What monitoring is in place for the first 24–48 hours post cutover? - ---- - -## Phase 8 — Post-Migration - -57. How will Oracle AWR (Automatic Workload Repository) reports be used to tune performance? -58. Which indexes need to be rebuilt or statistics gathered after data load? -59. Are there any missing indexes identified through execution plan analysis? -60. How will Oracle's optimizer statistics be gathered and maintained going forward? -61. What is the decommission plan for the source database? -62. How will ongoing backup, patching, and maintenance be managed in Oracle? -63. What documentation needs to be produced for the migrated schema and processes? -64. What lessons learned should be captured for future migrations? - ---- - -These 64 questions cover the full Oracle migration lifecycle. Would you like me to turn any of these phases into actual implementation scripts, test cases, or a project plan?