From c894c69dbcc550adf09bf1839d8f890a32f51a8a Mon Sep 17 00:00:00 2001 From: 81reap Date: Thu, 29 Oct 2026 17:35:54 -0400 Subject: [PATCH] feat(tests) :: add convenient reproduce script Adds a reproduce script that runs tests against specified SQLPage versions and specified databases via docker images. Helps with #1566 Assisted-By :: Claude Opus 5.5 --- CONTRIBUTING.md | 4 ++- flake.nix | 26 +++++++++++++---- tests/end-to-end/package.json | 3 +- tests/end-to-end/reproduce.config.ts | 10 +++++++ tests/end-to-end/reproduce.sh | 43 ++++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 7 deletions(-) create mode 100644 tests/end-to-end/reproduce.config.ts create mode 100755 tests/end-to-end/reproduce.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7fd4e402..574ec089c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,9 @@ built executable. Component suites use `fixtures//{index.sql,test.ts}` an import the shared `fixture.ts` harness; root-level `*.spec.ts` files cover the official site. -To show that an issue exists before you fix it, set `SQLPAGE_RELEASE` to a release tag, such as `SQLPAGE_RELEASE=v0.47.0 npx playwright test fixtures/tracking`. Both servers then run that release's Docker image against the same fixtures, on ports 18080 and 18081, so the test fails on the release and passes on your working tree. Add `--ui` to watch the page in each step. +To show that an issue exists before you fix it, put the reproduction in a fixture and run it against releases with one command, such as `nix run .#reproduce -- repro_721 postgres v0.35.0 v0.36.1 tree`. The second argument is `sqlite` or `postgres`, and `tree` is your working tree. It prints Playwright's output for each version, then a table you can paste into the issue or pull request. Releases run from their Docker images, from v0.29.0 onwards, and Postgres from `docker compose`. Without Nix, run `npm run reproduce --` in `tests/end-to-end` instead. + +To run a whole suite against a release instead, set `SQLPAGE_RELEASE` to its tag, such as `SQLPAGE_RELEASE=v0.47.0 npx playwright test fixtures/tracking`. Both servers then run that release's Docker image against the same fixtures, on ports 18080 and 18081, so the test fails on the release and passes on your working tree. Add `--ui` to watch the page in each step. The fixtures run on in-memory SQLite. To reproduce an issue with another database, start it with `docker compose up --wait ` and set `FIXTURES_DATABASE_URL` to its connection string from the [CI matrix](./.github/workflows/ci.yml), such as `FIXTURES_DATABASE_URL='postgres://root:Password123!@127.0.0.1/sqlpage'`. Only the fixtures server uses it, so the official site keeps its own database, and it works with `SQLPAGE_RELEASE` too. diff --git a/flake.nix b/flake.nix index 431002f5c..54673c8f1 100644 --- a/flake.nix +++ b/flake.nix @@ -134,6 +134,17 @@ PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS = "true"; }; + # The Nix-built browsers find their own libraries; host libraries leaked in + # through LD_LIBRARY_PATH crash Chromium at launch. + inBrowserWorkspace = + command: + buildBrowserAssets + + '' + unset LD_LIBRARY_PATH + cd tests/end-to-end + '' + + command; + gatesFor = pkgs: lib.mapAttrs @@ -192,15 +203,20 @@ description = "Drive the components in a browser with Playwright"; packages = cargoRuntime pkgs; env = cargoEnv pkgs // playwrightEnv pkgs; - script = buildBrowserAssets + '' - # The Nix-built browsers find their own libraries; host libraries - # leaked in through LD_LIBRARY_PATH crash Chromium at launch. - unset LD_LIBRARY_PATH - cd tests/end-to-end + script = inBrowserWorkspace '' npx playwright test "$@" ''; }; + reproduce = { + description = "Run one Playwright fixture against SQLPage releases on one database, and print the results as a table"; + packages = cargoRuntime pkgs ++ [ pkgs.docker-compose ]; + env = cargoEnv pkgs // playwrightEnv pkgs; + script = inBrowserWorkspace '' + ./reproduce.sh "$@" + ''; + }; + test-examples = { description = "Run the Hurl suite of an example against its own containers"; packages = [ pkgs.hurl ]; diff --git a/tests/end-to-end/package.json b/tests/end-to-end/package.json index 5a72f3067..08f727bba 100644 --- a/tests/end-to-end/package.json +++ b/tests/end-to-end/package.json @@ -6,7 +6,8 @@ "main": "index.js", "scripts": { "test": "playwright test --reporter=line", - "gui": "playwright test --ui" + "gui": "playwright test --ui", + "reproduce": "./reproduce.sh" }, "keywords": [], "author": "", diff --git a/tests/end-to-end/reproduce.config.ts b/tests/end-to-end/reproduce.config.ts new file mode 100644 index 000000000..31125881b --- /dev/null +++ b/tests/end-to-end/reproduce.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from "@playwright/test"; +import base from "./playwright.config.ts"; + +export default defineConfig({ + ...base, + projects: base.projects?.filter((project) => project.name === "fixtures"), + webServer: [base.webServer] + .flat() + .flatMap((server) => (server?.name === "fixtures" ? [server] : [])), +}); diff --git a/tests/end-to-end/reproduce.sh b/tests/end-to-end/reproduce.sh new file mode 100755 index 000000000..4b338f75e --- /dev/null +++ b/tests/end-to-end/reproduce.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage="Usage: reproduce ..." +fixture=${1:?$usage} +database=${2:?$usage} +shift 2 +if [ $# -eq 0 ]; then + echo "$usage" >&2 + exit 2 +fi + +case $database in + sqlite) ;; + postgres) + docker compose --file ../../docker-compose.yml up --wait postgres + export FIXTURES_DATABASE_URL='postgres://root:Password123!@127.0.0.1/sqlpage' + ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac + +results=() +for version in "$@"; do + release=$version + if [ "$version" = tree ]; then + release= + fi + if SQLPAGE_RELEASE=$release npx playwright test --config reproduce.config.ts "fixtures/$fixture" --reporter=line; then + results+=("| $version | ✔ passed |") + else + results+=("| $version | ✘ failed |") + fi +done + +echo +echo "| $fixture | $database |" +echo "| --- | --- |" +printf '%s\n' "${results[@]}" +echo +echo "On commit $(git rev-parse --short HEAD): \`nix run .#reproduce -- $fixture $database $*\`"