From 4d6566cc5b8dee3a640cbea5aa55dd198b6b14ec Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 15 Jul 2026 06:23:45 +0000 Subject: [PATCH 1/2] ci: add Dependabot config and preserve eval reports as CI artifacts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - .github/dependabot.yml: weekly update PRs for pip (requirements-dev.txt) and the GitHub Actions used by the workflows. - quality-gate.yml: both jobs now upload evals/reports/ (including the per-run VCRM gap report) as artifacts with if: always(), so the audit trail survives the runner instead of being discarded — previously the reports were generated in CI and lost on job teardown. 30-day retention. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VBrxqChRJtxdvSpFhiUWUy --- .github/dependabot.yml | 19 +++++++++++++++++++ .github/workflows/quality-gate.yml | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..75faa4e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + # Python dev dependencies (requirements-dev.txt at repo root) + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" + + # GitHub Actions used by the workflows + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index a288262..76c30d9 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -36,6 +36,17 @@ jobs: - name: Evals — Tier P (offline validator scenarios) run: python3 evals/runner.py --tiers p --verbose + # Reports (incl. the VCRM gap report) are written to evals/reports/ and + # would otherwise vanish with the runner — keep them for auditability. + - name: Upload eval reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: eval-reports-free-tier + path: evals/reports/ + if-no-files-found: ignore + retention-days: 30 + # Full integration surface: Tier I deploys env_dev.sql twice (idempotency), # Tier S runs the 85-assertion SQL suite, then the PG-gated pytest tiers run. integration-postgres: @@ -107,3 +118,12 @@ jobs: - name: Integration / e2e / parity tests run: pytest -m "e2e or integration or parity" tests/ -v + + - name: Upload eval reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: eval-reports-integration + path: evals/reports/ + if-no-files-found: ignore + retention-days: 30 From 036ab5f6aa690d9855ea8150744891238c0e579a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 18 Jul 2026 14:19:58 +0000 Subject: [PATCH 2/2] docs: correct SQL assertion count from 85 to 142 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite has grown to 142 assertions but README (badge, features list, coverage section, checklist), ARCHITECTURE.md, and a workflow comment still said 85 — a claim dating to the initial release that went unnoticed while the suites never executed. Verified against a live deploy: 142/142. Also tightens the Tier S eval expectation (min_total_assertions 85 -> 142) so a future drop in assertion count fails the eval instead of passing silently. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VBrxqChRJtxdvSpFhiUWUy --- .github/workflows/quality-gate.yml | 2 +- ARCHITECTURE.md | 2 +- README.md | 10 +++++----- .../tier_s/01_fresh_deploy_then_all_tests_pass.json | 8 +++++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index 76c30d9..f452a27 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -48,7 +48,7 @@ jobs: retention-days: 30 # Full integration surface: Tier I deploys env_dev.sql twice (idempotency), - # Tier S runs the 85-assertion SQL suite, then the PG-gated pytest tiers run. + # Tier S runs the 142-assertion SQL suite, then the PG-gated pytest tiers run. integration-postgres: name: integration (postgres service) runs-on: ubuntu-latest diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 93895a6..b85abb8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -45,7 +45,7 @@ The three layers can break independently, so we keep them physically separate. T | Path | What it is | |------|-----------| | `tests/framework/test_framework.sql` | Assertion library + results table | -| `tests/suites/test_01..05_*.sql` | 85 SQL assertions across 5 suites | +| `tests/suites/test_01..05_*.sql` | 142 SQL assertions across 5 suites | | `tests/run_all_tests.sql` | Master SQL test orchestrator | | `tests/run_tests.sh` | Bash wrapper that sources `config.local.env` | | `tests/run_python_tests.ps1` | Windows runner — invoked by the GitHub Actions workflow | diff --git a/README.md b/README.md index 9ae0a0a..00aa932 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![PostgreSQL](https://img.shields.io/badge/PostgreSQL-13%2B-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![Environments](https://img.shields.io/badge/Environments-Dev%20%7C%20Test%20%7C%20Staging%20%7C%20Prod-blue)](#environment-comparison) -[![Test Suites](https://img.shields.io/badge/Tests-5%20suites%20%7C%2085%20assertions-brightgreen)](#test-suite) +[![Test Suites](https://img.shields.io/badge/Tests-5%20suites%20%7C%20142%20assertions-brightgreen)](#test-suite) [![Evals](https://img.shields.io/badge/Evals-23%20CSV%20scenarios-brightgreen)](evals/) [![Terraform](https://img.shields.io/badge/Terraform-1.5%2B-7B42BC?logo=terraform&logoColor=white)](https://developer.hashicorp.com/terraform) @@ -20,7 +20,7 @@ This project provides a **production-grade SQL framework** to stand up a T&E man - **Test execution** — events, results, verdicts, and evidence artefacts - **Defect reporting** — deficiency reports (DRs) linked directly to failed results - **Multi-environment isolation** — separate databases, schemas, and users for Dev, Test, Staging, and Prod -- **Automated data testing** — 85 assertions across 5 SQL test suites, all written in pure PostgreSQL +- **Automated data testing** — 142 assertions across 5 SQL test suites, all written in pure PostgreSQL - **Data-driven evals** — 23 offline CSV validator scenarios plus PostgreSQL-backed idempotency and full-suite checks All names (database, schema, users, every table) are controlled by a single `\set` configuration block at the top of each environment file. Rename anything in one place and the entire script updates automatically. @@ -518,7 +518,7 @@ The `evals/` package complements the SQL and unit tests with scenario fixtures a |---|---|---| | P | `csv/validator.py` across 23 CSV edge cases, including malformed rows, BOM, CRLF, Unicode, quoted newlines, long fields, missing env vars, and invalid UTF-8 bytes | No | | I | Dev deployment idempotency by deploying twice and comparing seed row counts | Yes | -| S | Fresh Dev deploy followed by the full SQL suite, expecting all 85 assertions to pass | Yes | +| S | Fresh Dev deploy followed by the full SQL suite, expecting all 142 assertions to pass | Yes | Run examples: @@ -530,7 +530,7 @@ python evals\runner.py --only 14_quoted_newline --tiers p Each eval run writes a JSON report under `evals/reports//summary.json`; that folder is intentionally gitignored. -### Coverage — 85 assertions across 5 suites +### Coverage — 142 assertions across 5 suites | Suite | Assertions | What is tested | |---|---|---| @@ -704,7 +704,7 @@ Contributions are welcome. Please follow these steps: 1. Fork the repository 2. Create a feature branch: `git checkout -b feature/your-feature-name` 3. Make your changes and add or update tests in `tests/suites/` -4. Verify all 85 assertions still pass: `./tests/run_tests.sh dev` +4. Verify all 142 assertions still pass: `./tests/run_tests.sh dev` 5. Open a Pull Request with a clear description of what changed and why **Guidelines:** diff --git a/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json b/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json index 8f54104..1843018 100644 --- a/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json +++ b/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json @@ -1,11 +1,13 @@ { "scenario": "01_fresh_deploy_then_all_tests_pass", - "description": "Fresh Dev deploy followed by run_all_tests.sql must report 85/85 pass.", + "description": "Fresh Dev deploy followed by run_all_tests.sql must report 142/142 pass.", "expected": { "deploy_exit_code": 0, "tests_exit_code": 0, - "stdout_contains": ["ALL TESTS PASSED"], - "min_total_assertions": 85, + "stdout_contains": [ + "ALL TESTS PASSED" + ], + "min_total_assertions": 142, "min_pass_rate_percent": 100.0 } }