feat(tests) :: add convenient reproduce script

CONTRIBUTING.md

@@ -66,7 +66,9 @@ built executable. Component suites use `fixtures/<suite>/{index.sql,test.ts}` an
 import the shared `fixture.ts` harness; root-level `*.spec.ts` files cover the
 official site.
 
-To show that an issue exists before you fix it, set `SQLPAGE_RELEASE` to a release tag, such as `SQLPAGE_RELEASE=v0.47.0 npx playwright test fixtures/tracking`. Both servers then run that release's Docker image against the same fixtures, on ports 18080 and 18081, so the test fails on the release and passes on your working tree. Add `--ui` to watch the page in each step.
+To show that an issue exists before you fix it, put the reproduction in a fixture and run it against releases with one command, such as `nix run .#reproduce -- repro_721 postgres v0.35.0 v0.36.1 tree`. The second argument is `sqlite` or `postgres`, and `tree` is your working tree. It prints Playwright's output for each version, then a table you can paste into the issue or pull request. Releases run from their Docker images, from v0.29.0 onwards, and Postgres from `docker compose`. Without Nix, run `npm run reproduce --` in `tests/end-to-end` instead.
+
+To run a whole suite against a release instead, set `SQLPAGE_RELEASE` to its tag, such as `SQLPAGE_RELEASE=v0.47.0 npx playwright test fixtures/tracking`. Both servers then run that release's Docker image against the same fixtures, on ports 18080 and 18081, so the test fails on the release and passes on your working tree. Add `--ui` to watch the page in each step.
 
 The fixtures run on in-memory SQLite. To reproduce an issue with another database, start it with `docker compose up --wait <database>` and set `FIXTURES_DATABASE_URL` to its connection string from the [CI matrix](./.github/workflows/ci.yml), such as `FIXTURES_DATABASE_URL='postgres://root:Password123!@127.0.0.1/sqlpage'`. Only the fixtures server uses it, so the official site keeps its own database, and it works with `SQLPAGE_RELEASE` too.
 

flake.nix

@@ -134,6 +134,17 @@
         PLAYWRIGHT_SKIP_VALIDATE_HOST_REQUIREMENTS = "true";
       };
 
+      # The Nix-built browsers find their own libraries; host libraries leaked in
+      # through LD_LIBRARY_PATH crash Chromium at launch.
+      inBrowserWorkspace =
+        command:
+        buildBrowserAssets
+        + ''
+          unset LD_LIBRARY_PATH
+          cd tests/end-to-end
+        ''
+        + command;
+
       gatesFor =
         pkgs:
         lib.mapAttrs
@@ -192,15 +203,20 @@
               description = "Drive the components in a browser with Playwright";
               packages = cargoRuntime pkgs;
               env = cargoEnv pkgs // playwrightEnv pkgs;
-              script = buildBrowserAssets + ''
-                # The Nix-built browsers find their own libraries; host libraries
-                # leaked in through LD_LIBRARY_PATH crash Chromium at launch.
-                unset LD_LIBRARY_PATH
-                cd tests/end-to-end
+              script = inBrowserWorkspace ''
                 npx playwright test "$@"
               '';
             };
 
+            reproduce = {
+              description = "Run one Playwright fixture against SQLPage releases on one database, and print the results as a table";
+              packages = cargoRuntime pkgs ++ [ pkgs.docker-compose ];
+              env = cargoEnv pkgs // playwrightEnv pkgs;
+              script = inBrowserWorkspace ''
+                ./reproduce.sh "$@"
+              '';
+            };
+
             test-examples = {
               description = "Run the Hurl suite of an example against its own containers";
               packages = [ pkgs.hurl ];

tests/end-to-end/package.json

@@ -6,7 +6,8 @@
   "main": "index.js",
   "scripts": {
     "test": "playwright test --reporter=line",
-    "gui": "playwright test --ui"
+    "gui": "playwright test --ui",
+    "reproduce": "./reproduce.sh"
   },
   "keywords": [],
   "author": "",

tests/end-to-end/reproduce.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from "@playwright/test";
+import base from "./playwright.config.ts";
+
+export default defineConfig({
+  ...base,
+  projects: base.projects?.filter((project) => project.name === "fixtures"),
+  webServer: [base.webServer]
+    .flat()
+    .flatMap((server) => (server?.name === "fixtures" ? [server] : [])),
+});

tests/end-to-end/reproduce.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage="Usage: reproduce <fixture> <sqlite|postgres> <release tag or tree>..."
+fixture=${1:?$usage}
+database=${2:?$usage}
+shift 2
+if [ $# -eq 0 ]; then
+  echo "$usage" >&2
+  exit 2
+fi
+
+case $database in
+  sqlite) ;;
+  postgres)
+    docker compose --file ../../docker-compose.yml up --wait postgres
+    export FIXTURES_DATABASE_URL='postgres://root:Password123!@127.0.0.1/sqlpage'
+    ;;
+  *)
+    echo "$usage" >&2
+    exit 2
+    ;;
+esac
+
+results=()
+for version in "$@"; do
+  release=$version
+  if [ "$version" = tree ]; then
+    release=
+  fi
+  if SQLPAGE_RELEASE=$release npx playwright test --config reproduce.config.ts "fixtures/$fixture" --reporter=line; then
+    results+=("| $version | ✔ passed |")
+  else
+    results+=("| $version | ✘ failed |")
+  fi
+done
+
+echo
+echo "| $fixture | $database |"
+echo "| --- | --- |"
+printf '%s\n' "${results[@]}"
+echo
+echo "On commit $(git rev-parse --short HEAD): \`nix run .#reproduce -- $fixture $database $*\`"