ci: add Linux quality gate; docs: setup runbook

.github/workflows/quality-gate.yml

@@ -0,0 +1,109 @@
+name: Quality Gate
+
+on:
+  push:
+    branches:
+      - main
+      - master
+  pull_request:
+
+jobs:
+  # Fast, deterministic checks — no database, no network beyond pip.
+  free-tier:
+    name: free-tier (unit, lint, health, evals-p)
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
+
+      - name: Unit / regression / security / snapshot tests
+        run: make test-free
+
+      - name: Lint (flake8 + bandit)
+        run: make lint
+
+      - name: Component health check
+        run: make health
+
+      - name: Evals — Tier P (offline validator scenarios)
+        run: python3 evals/runner.py --tiers p --verbose
+
+  # Full integration surface: Tier I deploys env_dev.sql twice (idempotency),
+  # Tier S runs the 85-assertion SQL suite, then the PG-gated pytest tiers run.
+  integration-postgres:
+    name: integration (postgres service)
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env:
+          POSTGRES_PASSWORD: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      PGHOST: localhost
+      PGPORT: "5432"
+      PGUSER: postgres
+      PGPASSWORD: postgres
+      PGDATABASE: postgres
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install dev dependencies
+        run: pip install -r requirements-dev.txt
+
+      - name: Install PostgreSQL client
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends postgresql-client
+
+      - name: Write CI database config
+        run: |
+          cat > build/config.local.env <<'EOF'
+          DB_ENGINE="postgresql"
+          PG_HOST="localhost"
+          PG_PORT="5432"
+          PG_SUPERUSER="postgres"
+          PG_SUPERUSER_PASSWORD="postgres"
+          PG_DB_DEV="te_mgmt_dev"
+          PG_SCHEMA_DEV="te_dev"
+          PG_DB_TEST="te_mgmt_test"
+          PG_SCHEMA_TEST="te_test"
+          EOF
+
+      # env_*.sql scripts do not create their databases — deploy_all.sh does that
+      # before invoking them (see build/te_core_schema.sql:41). Mirror it here.
+      - name: Create environment databases
+        run: |
+          for db in te_mgmt_dev te_mgmt_test; do
+            if [ -z "$(psql -d postgres -tA -c "SELECT 1 FROM pg_database WHERE datname = '$db'")" ]; then
+              psql -v ON_ERROR_STOP=1 -d postgres -c \
+                "CREATE DATABASE \"$db\" WITH OWNER = postgres ENCODING = 'UTF8' TEMPLATE = template0 CONNECTION LIMIT = -1"
+            fi
+          done
+
+      - name: Evals — Tiers P, I, S
+        run: python3 evals/runner.py --tiers p,i,s --verbose
+
+      - name: Deploy test environment (for cross-env parity)
+        run: psql -v ON_ERROR_STOP=1 -f build/environments/env_test.sql
+
+      - name: Integration / e2e / parity tests
+        run: pytest -m "e2e or integration or parity" tests/ -v

ARCHITECTURE.md

@@ -49,7 +49,15 @@ The three layers can break independently, so we keep them physically separate. T
 | `tests/run_all_tests.sql` | Master SQL test orchestrator |
 | `tests/run_tests.sh` | Bash wrapper that sources `config.local.env` |
 | `tests/run_python_tests.ps1` | Windows runner — invoked by the GitHub Actions workflow |
+| `tests/conftest.py` | pytest env-var isolation between tests |
 | `tests/test_csv_validator.py` | unittest for `build/csv/validator.py` |
+| `tests/test_csv_utilise.py` | unit tests for `build/csv_utilise.sh` argument parsing |
+| `tests/test_csv_loader_arbitrary_shapes.py` | integration: arbitrary CSV shapes through loader → PG (skips without PG) |
+| `tests/test_e2e_pipeline.py` | e2e: CSV → validate → load → verify (DB half skips without PG) |
+| `tests/test_parity.py` | cross-environment row-count / schema parity (skips without PG) |
+| `tests/test_regression.py` | pinned tests for previously found bug classes |
+| `tests/test_security.py` | static credential/SQL-pattern scans |
+| `tests/test_snapshot.py` | golden-file output comparisons (`tests/snapshots/`) |
 | `tests/test_evals_runner.py` | unittest for `evals/runner.py` itself |
 
 ### `evals/` — data-driven scenarios

SETUP_RUNBOOK.md

@@ -0,0 +1,119 @@
+# SETUP RUNBOOK — PostgreDataMigrationApp
+
+Every command block below is labelled with **where to run it**. On Windows there are
+two different terminals, and the repo's `.sh` scripts only work in one of them:
+
+| Label | Prompt looks like | Use for |
+|---|---|---|
+| **Git Bash** | `user@machine MINGW64 ~` | all `.sh` scripts, `make`, day-to-day operations |
+| **PowerShell** | `PS C:\Users\...>` | `preflight.ps1`, `scripts\test.ps1`, git on Windows |
+
+On Linux / macOS / WSL2 every block runs in your normal shell.
+Azure Cloud Shell is **not** used by this runbook — it cannot reach a database on your machine.
+
+---
+
+## Phase 0 — Prerequisites (one-time per machine)
+
+Install:
+
+- **Git** (on Windows: [Git for Windows](https://gitforwindows.org/), which includes Git Bash)
+- **Python 3.10+**
+- **PostgreSQL 13+** — server running, `psql` client on PATH
+
+**▶ RUN IN: Git Bash** — verify:
+
+```bash
+git --version && python3 --version && psql --version && bash --version | head -1
+```
+
+## Phase 1 — Get the code and Python dependencies
+
+**▶ RUN IN: Git Bash**
+
+```bash
+git clone https://github.com/amar-python/PostgreDataMigrationApp.git
+cd PostgreDataMigrationApp
+pip install -r requirements-dev.txt
+```
+
+## Phase 2 — Preflight check
+
+Catches missing tools and configuration before anything half-deploys.
+
+**▶ RUN IN: Git Bash** (repo root):
+
+```bash
+bash preflight.sh
+```
+
+Windows-native alternative — **▶ RUN IN: PowerShell**: `.\preflight.ps1`
+
+## Phase 3 — Configure the database connection
+
+**▶ RUN IN: Git Bash** (repo root):
+
+```bash
+cd build
+cp config.env.example config.env
+./setup.sh          # interactive wizard: engine, host, port, credentials per environment
+cd ..
+```
+
+This writes `build/config.local.env` — the file every loader and test script reads.
+It is gitignored; never commit it.
+
+## Phase 4 — Deploy the schema
+
+PostgreSQL must be running and reachable with the credentials from Phase 3.
+
+**▶ RUN IN: Git Bash** (repo root):
+
+```bash
+export PGPASSWORD='<your postgres superuser password>'
+
+# Single environment (dev) — idempotent, safe to re-run:
+psql -U postgres -f build/environments/env_dev.sql
+
+# Or all four environments (dev / test / staging / prod):
+bash build/deploy_all.sh
+```
+
+## Phase 5 — Verify the deployment
+
+**▶ RUN IN: Git Bash** (repo root):
+
+```bash
+make test-free                          # offline unit / regression / security tests
+make lint                               # flake8 + bandit
+make health                             # structural check — every expected file present
+bash tests/run_tests.sh dev             # SQL assertion suite against the dev DB
+python3 evals/runner.py --tiers p,i,s   # evals: offline + idempotency + SQL tiers
+```
+
+All green means the environment is correctly stood up.
+
+## Phase 6 — Load and use CSV data (day-to-day)
+
+**▶ RUN IN: Git Bash** (repo root):
+
+```bash
+make csv-demo                                             # one-shot proof with bundled samples
+bash build/csv_loader.sh path/to/anything.csv --env dev   # load any CSV
+bash build/csv_utilise.sh list --env dev                  # list CSV-loaded tables
+bash build/csv_utilise.sh peek <table> --limit 5          # inspect rows
+bash build/csv_utilise.sh export <table> out.csv          # round-trip back to CSV
+bash build/csv_utilise.sh drop <table> --yes              # remove a CSV-loaded table
+```
+
+---
+
+## Troubleshooting order
+
+When a DB-dependent step fails, check in this order — it resolves most setup issues:
+
+1. Is PostgreSQL running? (`pg_isready` or `psql -c "SELECT 1"`)
+2. Is `PGPASSWORD` set **in this terminal**? (environment variables do not survive new windows)
+3. Does `build/config.local.env` exist? (Phase 3 creates it; a fresh clone does not have it)
+4. Are you in the right terminal? `.sh` scripts require Git Bash / WSL2 — they fail in
+   PowerShell and cmd with syntax errors.

build/csv/validator.py

@@ -25,18 +25,24 @@
 RED    = '\033[0;31m'
 NC     = '\033[0m'
 
+
 def log(msg):  print(f"{GREEN}  [validator OK]{NC} {msg}")
+
+
 def warn(msg): print(f"{YELLOW}  [validator WARN]{NC} {msg}")
+
+
 def err(msg):  print(f"{RED}  [validator ERR]{NC} {msg}", file=sys.stderr)
 
+
 # ── Read environment variables ────────────────────────────────────────────────
 CSV_FILE   = os.environ.get('CSV_FILE',   '')
 VALID_CSV  = os.environ.get('VALID_CSV',  '')
 SKIP_FILE  = os.environ.get('SKIP_FILE',  '')
 TABLE_NAME = os.environ.get('TABLE_NAME', 'unknown')
 
 # Validate required vars
-missing = [v for v in ('CSV_FILE','VALID_CSV','SKIP_FILE') if not os.environ.get(v)]
+missing = [v for v in ('CSV_FILE', 'VALID_CSV', 'SKIP_FILE') if not os.environ.get(v)]
 if missing:
     err(f"Missing required environment variables: {', '.join(missing)}")
     sys.exit(1)

evals/runner.py

@@ -305,7 +305,8 @@ def _can_connect_pg() -> bool:
 def _count_dev_rows() -> Dict[str, Any]:
     counts: Dict[str, Any] = {}
     for tbl in _DEV_SEED_TABLES:
-        query = 'SELECT count(*) FROM te_dev."' + tbl + '";'
+        # tbl comes from the hardcoded _DEV_SEED_TABLES constant — not injectable
+        query = 'SELECT count(*) FROM te_dev."' + tbl + '";'  # nosec B608
         r = subprocess.run(
             ["psql", "-tA", "-d", "te_mgmt_dev", "-c", query],
             env=_pg_env(),
@@ -456,6 +457,8 @@ def _run_fresh_deploy_then_tests(
 
     table_overrides = [
         "--set", "schema_name=te_dev",
+        "--set", "app_user=te_dev_user",
+        "--set", "conn_limit=10",
         "--set", "tbl_organisations=organisations",
         "--set", "tbl_personnel=personnel",
         "--set", "tbl_test_programs=test_programs",
@@ -486,14 +489,18 @@ def _run_fresh_deploy_then_tests(
     total_assertions = None
     pass_rate        = None
     for line in tests.stdout.splitlines():
-        parts = line.split()
-        if (len(parts) >= 5 and parts[0].isdigit() and parts[1].isdigit()
-                and parts[2].isdigit() and parts[3].endswith("%")):
-            try:
-                total_assertions = int(parts[0])
-                pass_rate        = float(parts[3].rstrip("%"))
-            except ValueError:
-                pass
+        # Summary row may be plain ("142 142 0 100.0% ...") or a psql table
+        # row ("142 |  142 |  0 | 0 | 100.0% | ..."); strip pipes first.
+        parts = line.replace("|", " ").split()
+        if (len(parts) >= 4 and parts[0].isdigit() and parts[1].isdigit()
+                and parts[2].isdigit()):
+            pct = next((p for p in parts[3:] if p.endswith("%")), None)
+            if pct is not None:
+                try:
+                    total_assertions = int(parts[0])
+                    pass_rate        = float(pct.rstrip("%"))
+                except ValueError:
+                    pass
     actual["total_assertions"] = total_assertions
     actual["pass_rate"]        = pass_rate
     result.actual              = actual

scripts/lint.sh

@@ -24,7 +24,7 @@ ALL_PY=("${PY_SRC[@]}" "${PY_TESTS[@]}" "${PY_SCRIPTS[@]}")
 echo -e "\n${YELLOW}=== flake8 (style + logic) ===${NC}"
 if python3 -m flake8 "${ALL_PY[@]}" \
         --max-line-length=120 \
-        --extend-ignore=E501; then
+        --extend-ignore=E501,E221,E272; then
     echo -e "${GREEN}✓ flake8 clean${NC}"
 else
     echo -e "${RED}✗ flake8 found issues — fix before merging${NC}"