diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..75faa4e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + # Python dev dependencies (requirements-dev.txt at repo root) + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" + + # GitHub Actions used by the workflows + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + labels: + - "dependencies" diff --git a/.github/workflows/quality-gate.yml b/.github/workflows/quality-gate.yml index a288262..f452a27 100644 --- a/.github/workflows/quality-gate.yml +++ b/.github/workflows/quality-gate.yml @@ -36,8 +36,19 @@ jobs: - name: Evals — Tier P (offline validator scenarios) run: python3 evals/runner.py --tiers p --verbose + # Reports (incl. the VCRM gap report) are written to evals/reports/ and + # would otherwise vanish with the runner — keep them for auditability. + - name: Upload eval reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: eval-reports-free-tier + path: evals/reports/ + if-no-files-found: ignore + retention-days: 30 + # Full integration surface: Tier I deploys env_dev.sql twice (idempotency), - # Tier S runs the 85-assertion SQL suite, then the PG-gated pytest tiers run. + # Tier S runs the 142-assertion SQL suite, then the PG-gated pytest tiers run. integration-postgres: name: integration (postgres service) runs-on: ubuntu-latest @@ -107,3 +118,12 @@ jobs: - name: Integration / e2e / parity tests run: pytest -m "e2e or integration or parity" tests/ -v + + - name: Upload eval reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: eval-reports-integration + path: evals/reports/ + if-no-files-found: ignore + retention-days: 30 diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 93895a6..b85abb8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -45,7 +45,7 @@ The three layers can break independently, so we keep them physically separate. T | Path | What it is | |------|-----------| | `tests/framework/test_framework.sql` | Assertion library + results table | -| `tests/suites/test_01..05_*.sql` | 85 SQL assertions across 5 suites | +| `tests/suites/test_01..05_*.sql` | 142 SQL assertions across 5 suites | | `tests/run_all_tests.sql` | Master SQL test orchestrator | | `tests/run_tests.sh` | Bash wrapper that sources `config.local.env` | | `tests/run_python_tests.ps1` | Windows runner — invoked by the GitHub Actions workflow | diff --git a/README.md b/README.md index 9ae0a0a..00aa932 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [![PostgreSQL](https://img.shields.io/badge/PostgreSQL-13%2B-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) [![Environments](https://img.shields.io/badge/Environments-Dev%20%7C%20Test%20%7C%20Staging%20%7C%20Prod-blue)](#environment-comparison) -[![Test Suites](https://img.shields.io/badge/Tests-5%20suites%20%7C%2085%20assertions-brightgreen)](#test-suite) +[![Test Suites](https://img.shields.io/badge/Tests-5%20suites%20%7C%20142%20assertions-brightgreen)](#test-suite) [![Evals](https://img.shields.io/badge/Evals-23%20CSV%20scenarios-brightgreen)](evals/) [![Terraform](https://img.shields.io/badge/Terraform-1.5%2B-7B42BC?logo=terraform&logoColor=white)](https://developer.hashicorp.com/terraform) @@ -20,7 +20,7 @@ This project provides a **production-grade SQL framework** to stand up a T&E man - **Test execution** — events, results, verdicts, and evidence artefacts - **Defect reporting** — deficiency reports (DRs) linked directly to failed results - **Multi-environment isolation** — separate databases, schemas, and users for Dev, Test, Staging, and Prod -- **Automated data testing** — 85 assertions across 5 SQL test suites, all written in pure PostgreSQL +- **Automated data testing** — 142 assertions across 5 SQL test suites, all written in pure PostgreSQL - **Data-driven evals** — 23 offline CSV validator scenarios plus PostgreSQL-backed idempotency and full-suite checks All names (database, schema, users, every table) are controlled by a single `\set` configuration block at the top of each environment file. Rename anything in one place and the entire script updates automatically. @@ -518,7 +518,7 @@ The `evals/` package complements the SQL and unit tests with scenario fixtures a |---|---|---| | P | `csv/validator.py` across 23 CSV edge cases, including malformed rows, BOM, CRLF, Unicode, quoted newlines, long fields, missing env vars, and invalid UTF-8 bytes | No | | I | Dev deployment idempotency by deploying twice and comparing seed row counts | Yes | -| S | Fresh Dev deploy followed by the full SQL suite, expecting all 85 assertions to pass | Yes | +| S | Fresh Dev deploy followed by the full SQL suite, expecting all 142 assertions to pass | Yes | Run examples: @@ -530,7 +530,7 @@ python evals\runner.py --only 14_quoted_newline --tiers p Each eval run writes a JSON report under `evals/reports//summary.json`; that folder is intentionally gitignored. -### Coverage — 85 assertions across 5 suites +### Coverage — 142 assertions across 5 suites | Suite | Assertions | What is tested | |---|---|---| @@ -704,7 +704,7 @@ Contributions are welcome. Please follow these steps: 1. Fork the repository 2. Create a feature branch: `git checkout -b feature/your-feature-name` 3. Make your changes and add or update tests in `tests/suites/` -4. Verify all 85 assertions still pass: `./tests/run_tests.sh dev` +4. Verify all 142 assertions still pass: `./tests/run_tests.sh dev` 5. Open a Pull Request with a clear description of what changed and why **Guidelines:** diff --git a/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json b/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json index 8f54104..1843018 100644 --- a/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json +++ b/evals/expected/tier_s/01_fresh_deploy_then_all_tests_pass.json @@ -1,11 +1,13 @@ { "scenario": "01_fresh_deploy_then_all_tests_pass", - "description": "Fresh Dev deploy followed by run_all_tests.sql must report 85/85 pass.", + "description": "Fresh Dev deploy followed by run_all_tests.sql must report 142/142 pass.", "expected": { "deploy_exit_code": 0, "tests_exit_code": 0, - "stdout_contains": ["ALL TESTS PASSED"], - "min_total_assertions": 85, + "stdout_contains": [ + "ALL TESTS PASSED" + ], + "min_total_assertions": 142, "min_pass_rate_percent": 100.0 } }