Add starter compatibility matrix e2e + version.ts floor guard

runner/.gitignore

@@ -21,3 +21,6 @@ build/
 test-results/
 playwright-report/
 .playwright/
+
+# generated starter-matrix run reports — paste into the ticket/PR instead
+docs/reports/

runner/docs/run-and-deploy.md

@@ -103,6 +103,10 @@ cascader drill-down, framework switching, and the "See in documentation" link.
   `E2E_BASE_URL=https://demos.handsontable.com E2E_LIVE=1 pnpm e2e`.
 - The API deploy workflow also does a post-deploy smoke (`GET /api/health` on
   `demos.handsontable.com` must return 200).
+- A separate, opt-in starter compatibility matrix (`pnpm e2e:matrix`, gated
+  behind `E2E_STARTER_MATRIX=1`) boots every starter at every supported
+  Handsontable major against a live instance — not part of CI, run manually.
+  See `docs/starter-compat-matrix.md`.
 
 ### Authoring app (frontend) → GitHub Actions
 

runner/docs/starter-compat-matrix.md

@@ -0,0 +1,86 @@
+# Starter compatibility matrix
+
+`e2e/starter-matrix.spec.ts` empirically verifies every starter in the
+repo-root `examples/` directory (as cataloged in `catalog.json`) against every
+Handsontable major the app claims to support (15-19) — DEV-2102 / ADR-0021
+decision 10. It boots each starter/major combination against an
+already-running instance, checks that a Handsontable grid actually renders
+(not just that the dev server responds), and records the result.
+
+It is **not** part of the deterministic PR suite (`pnpm e2e`) — it's an
+opt-in, manually-run check.
+
+## Running it
+
+```sh
+E2E_BASE_URL=https://demos.handsontable.com pnpm e2e:matrix
+pnpm e2e:matrix:report
+```
+
+- `pnpm e2e:matrix` sets `E2E_STARTER_MATRIX=1` and runs at `--workers=2`,
+  writing JSON results to `test-results/starter-matrix.json`.
+- `pnpm e2e:matrix:report` turns that JSON into a starter × major markdown
+  table, written to `docs/reports/starter-matrix-<date>.md` and printed to
+  stdout. `docs/reports/` is gitignored — a dated run snapshot isn't living
+  documentation, it's a one-off result. Paste the stdout output straight into
+  the ticket/PR instead of committing the file. It accepts multiple JSON
+  paths if you ran the matrix in chunks (see below).
+
+Against a local stack instead of prod: point `E2E_BASE_URL` at your local
+`vite --port 5173` authoring dev server, with the API worker running on 8787
+(see the "Full stack" section above) and `VITE_API_BASE` baked accordingly.
+
+### Running it in chunks
+
+The full matrix is ~80 starter×major combinations and can take well over an
+hour end to end. If you can't leave it running unattended for that long (a
+sleeping/locked laptop kills the process), run subsets with `--grep`, each to
+its own JSON file:
+
+```sh
+E2E_STARTER_MATRIX=1 E2E_BASE_URL=https://demos.handsontable.com \
+  PLAYWRIGHT_JSON_OUTPUT_NAME=/tmp/matrix-batch1.json \
+  npx playwright test e2e/starter-matrix.spec.ts --workers=2 --retries=1 \
+  --grep 'matrix: (react|vue) @' --reporter=list,json
+```
+
+**Use an absolute path outside `test-results/` for `PLAYWRIGHT_JSON_OUTPUT_NAME`
+when chunking.** Playwright wipes its default `outputDir` (`test-results/`) at
+the start of every run — a second chunk's run will silently delete the first
+chunk's JSON if both write there. Then merge:
+
+```sh
+pnpm e2e:matrix:report /tmp/matrix-batch1.json /tmp/matrix-batch2.json ...
+```
+
+## Prod concurrency cap
+
+Container-engine starters (`engine: "container"` in `catalog.json`) each boot
+a real Cloudflare Sandbox container via `POST /api/session`. Production caps
+the live-preview `Sandbox` class at **5 concurrent instances**
+(`workers/api/wrangler.jsonc`: `max_instances: 5`, separate from
+`BuilderSandbox`'s `max_instances: 3` for the unrelated demo-sharing/build
+pipeline, which this matrix never touches). The `e2e:matrix` script runs at
+`--workers=2` to leave headroom for real traffic — don't raise it without
+checking current prod load, and never run multiple `playwright test`
+invocations against prod concurrently (their worker pools stack).
+
+## What it checks, and its limits
+
+Per combo: the preview reaches "Live" (not stuck booting or in an error
+state), the Handsontable grid actually renders (`.handsontable .htCore td`
+count > 0 — catches "server responded but nothing mounted"), zero unexpected
+console/page errors, and that the requested version reached the session.
+Sandpack-engine starters additionally verify the loaded Handsontable version
+via the network request URL; container-engine starters can only confirm the
+*requested* version was pinned in `package.json` (`Handsontable.version` is
+typically unavailable in-frame for ESM bundles) — this is reported as
+`unverified`, not a failure.
+
+A major with no stable npm release yet (checked live against the npm
+registry, not the app's own sliced `/api/versions` listing) is skipped, not
+failed.
+
+See DEV-2102 (or the PR that introduced this harness) for the first run's
+findings and the resulting decision on `packages/runtime/src/version.ts`'s
+minimum-major guard — the rationale is also recorded in that file's comments.

runner/e2e/starter-matrix.spec.ts

@@ -0,0 +1,170 @@
+import { test, expect } from "@playwright/test";
+import { readFileSync } from "node:fs";
+
+// Empirical starter-compatibility matrix (DEV-2102 / ADR-0021 decision 10):
+// boots every starter in catalog.json at the latest patch of each supported
+// Handsontable major (15-19) and records what actually breaks. Opt-in via
+// E2E_STARTER_MATRIX=1 — this is not part of the deterministic PR suite; it
+// spins ~55 real Tier-2 container sessions (the live-preview `Sandbox` class)
+// and takes ~80-90 minutes.
+//
+// Prod only allows 5 concurrent live-preview containers
+// (workers/api/wrangler.jsonc: Sandbox max_instances=5, separate from
+// BuilderSandbox max_instances=3 which this test never touches). The
+// e2e:matrix script runs at --workers=2 to leave headroom for real traffic —
+// do not raise concurrency without checking current prod load.
+//
+// Run: E2E_BASE_URL=https://demos.handsontable.com pnpm e2e:matrix
+// Report: pnpm e2e:matrix:report
+
+const ENABLED = process.env.E2E_STARTER_MATRIX === "1";
+const MAJORS = [15, 16, 17, 18, 19] as const;
+
+type CatalogExample = {
+  framework: string;
+  displayName: string;
+  engine: "sandpack" | "container";
+};
+
+const catalog = JSON.parse(readFileSync(new URL("../catalog.json", import.meta.url), "utf8")) as {
+  examples: CatalogExample[];
+};
+
+// Console/page noise that isn't a real compatibility break. Extend as the
+// matrix surfaces new false positives.
+const NOISE = [
+  /non-commercial|evaluation license/i,
+  /Download the React DevTools/i,
+  /favicon\.ico/i,
+  /\[vite\] (connecting|connected)/i,
+  /ERR_BLOCKED_BY_CLIENT|third-party cookie/i,
+];
+const isKnownNoise = (message: string) => NOISE.some((re) => re.test(message));
+
+let latestByMajorPromise: Promise<Map<number, string>> | null = null;
+
+// Resolve the latest stable patch release per major straight from npm — the
+// app's own GET /api/versions slices results to 15 entries and can drop older
+// majors, so it isn't a reliable source for "latest per major".
+function resolveLatestByMajor(): Promise<Map<number, string>> {
+  latestByMajorPromise ??= (async () => {
+    const res = await fetch("https://registry.npmjs.org/handsontable", {
+      headers: { Accept: "application/vnd.npm.install-v1+json" },
+    });
+    if (!res.ok) throw new Error(`npm registry responded ${res.status}`);
+    const doc = (await res.json()) as { versions: Record<string, unknown> };
+    const best = new Map<number, string>();
+    for (const version of Object.keys(doc.versions)) {
+      if (!/^\d+\.\d+\.\d+$/.test(version)) continue;
+      const [major, minor, patch] = version.split(".").map(Number);
+      const current = best.get(major);
+      if (!current) {
+        best.set(major, version);
+        continue;
+      }
+      const [, currentMinor, currentPatch] = current.split(".").map(Number);
+      if (minor > currentMinor || (minor === currentMinor && patch > currentPatch)) {
+        best.set(major, version);
+      }
+    }
+    return best;
+  })();
+  return latestByMajorPromise;
+}
+
+for (const entry of catalog.examples) {
+  for (const major of MAJORS) {
+    test(`matrix: ${entry.framework} @ ${major} [${entry.engine}]`, async ({ page, request }, testInfo) => {
+      test.skip(!ENABLED, "set E2E_STARTER_MATRIX=1 to run the starter compatibility matrix");
+      test.setTimeout(entry.engine === "container" ? 300_000 : 150_000);
+
+      const byMajor = await resolveLatestByMajor();
+      const version = byMajor.get(major);
+      // A major with no stable npm release yet (e.g. 19 is pre-release as of
+      // writing — dist-tags carry only `next`/`rc`) isn't a starter failure,
+      // it's nothing to test yet. Skip rather than report a false breakage.
+      test.skip(!version, `no stable handsontable release published for major ${major} yet`);
+      if (!version) return;
+
+      // Annotate immediately so a failing test still reports which version it
+      // was testing — the report shouldn't say "unknown" just because the
+      // test threw before reaching the end.
+      testInfo.annotations.push({ type: "resolvedVersion", description: version }, { type: "engine", description: entry.engine });
+
+      const consoleErrors: string[] = [];
+      const pageErrors: string[] = [];
+      const htRequestVersions: string[] = [];
+      const sessions: { id: string; apiBase: string }[] = [];
+      let postedHtVersion: string | null = null;
+
+      page.on("console", (msg) => {
+        if (msg.type() === "error") consoleErrors.push(msg.text());
+      });
+      page.on("pageerror", (err) => pageErrors.push(String(err)));
+      page.on("request", (req) => {
+        const url = req.url();
+        const match = url.match(/handsontable[@/](\d+\.\d+\.\d+)/);
+        if (match) htRequestVersions.push(match[1]);
+        if (req.method() === "POST" && /\/api\/session$/.test(url)) {
+          try {
+            postedHtVersion = (JSON.parse(req.postData() ?? "{}") as { htVersion?: string }).htVersion ?? null;
+          } catch {
+            // ignore malformed payloads — absence just leaves postedHtVersion null
+          }
+        }
+      });
+      page.on("response", async (res) => {
+        if (res.request().method() === "POST" && /\/api\/session$/.test(res.url()) && res.ok()) {
+          const body = (await res.json().catch(() => null)) as { sessionId?: string } | null;
+          if (body?.sessionId) sessions.push({ id: body.sessionId, apiBase: new URL(res.url()).origin });
+        }
+      });
+
+      try {
+        await page.goto(`/?example=${encodeURIComponent(entry.framework)}&v=${version}`);
+
+        const preview = page.locator('section[aria-label="Preview"]');
+        // The status bar is the first direct child <div> of the preview
+        // section (PreviewPane.tsx) — scope to it directly rather than
+        // getByText scanning the whole section, which also matches the live
+        // boot-log <pre> when a container's dev-server error text happens to
+        // start with "Error:" (seen with mui: strict-mode violation, two
+        // elements matched).
+        const statusBar = preview.locator(":scope > div").first();
+        await expect(statusBar).toHaveText("Live", {
+          timeout: entry.engine === "container" ? 240_000 : 120_000,
+        });
+
+        // Container "Live" only means the dev server responded — the frame may
+        // still be blank or mid-hydration (Next/Nuxt/Astro). The rendered grid
+        // is the real functional check.
+        const frame = page.frameLocator('iframe[title="Demo preview"]');
+        const cells = frame.locator(".handsontable .htCore td");
+        await expect
+          .poll(async () => cells.count().catch(() => 0), { timeout: 60_000, intervals: [1_000] })
+          .toBeGreaterThan(0);
+
+        const realErrors = [...consoleErrors, ...pageErrors].filter((e) => !isKnownNoise(e));
+        expect(realErrors, `console/page errors:\n${realErrors.join("\n")}`).toEqual([]);
+
+        let detectedVersion: string | null = null;
+        if (entry.engine === "sandpack") {
+          detectedVersion = htRequestVersions.find((v) => v === version) ?? htRequestVersions[0] ?? null;
+        } else {
+          expect(postedHtVersion, "requested handsontable version reached the container session").toBe(version);
+          detectedVersion = await frame
+            .locator("body")
+            .evaluate(() => (window as unknown as { Handsontable?: { version?: string } }).Handsontable?.version ?? null)
+            .catch(() => null);
+        }
+
+        testInfo.annotations.push({ type: "detectedVersion", description: detectedVersion ?? "unverified" });
+        if (detectedVersion) expect(detectedVersion.startsWith(`${major}.`)).toBe(true);
+      } finally {
+        for (const { id, apiBase } of sessions) {
+          await request.delete(`${apiBase}/api/session/${id}`).catch(() => {});
+        }
+      }
+    });
+  }
+}

runner/package.json

@@ -16,7 +16,9 @@
     "typecheck": "pnpm -r run typecheck",
     "lint": "pnpm -r run lint",
     "test": "pnpm --filter @handsontable/demo-runtime build && node --test pipeline/*.test.mjs",
-    "e2e": "playwright test"
+    "e2e": "playwright test",
+    "e2e:matrix": "E2E_STARTER_MATRIX=1 PLAYWRIGHT_JSON_OUTPUT_NAME=test-results/starter-matrix.json playwright test e2e/starter-matrix.spec.ts --workers=2 --retries=1 --reporter=list,json",
+    "e2e:matrix:report": "node scripts/starter-matrix-report.mjs"
   },
   "devDependencies": {
     "@playwright/test": "^1.48.0",

runner/packages/runtime/src/version.ts

@@ -11,6 +11,12 @@ import semver from "semver";
 import type { FilesMap, HandsontableVersionRef } from "./types.js";
 
 export const DEFAULT_MAX_MAJOR = 19;
+// Empirically verified range is 15-19 (DEV-2102 / ADR-0021 decision 10); majors
+// below 15 have never been tested against current starters/wrappers and stay
+// out of scope. Previously only the UI-facing GET /api/versions listing
+// enforced this floor — a direct session/API call bypassing the version
+// dropdown could still request an untested major like 14.
+export const DEFAULT_MIN_MAJOR = 15;
 const MIN_BARE_NUMERIC_PKG_PR_NEW_REF = 1000;
 
 /** Dependency never rewritten: an independently versioned Handsontable plugin. */
@@ -59,6 +65,7 @@ export type ValidationResult =
 export function validateHandsontableVersion(
   value: unknown,
   maxMajor: number = DEFAULT_MAX_MAJOR,
+  minMajor: number = DEFAULT_MIN_MAJOR,
 ): ValidationResult {
   if (value === undefined || value === null) {
     return { ok: false, message: "handsontable-version is required" };
@@ -69,11 +76,11 @@ export function validateHandsontableVersion(
   const urlRef = parsePkgPrNewFromUrl(trimmed);
   if (urlRef !== null) return { ok: true, value: { ref: urlRef, pkgPrNew: true } };
 
-  const capMsg = (normalized: string) => {
+  const rangeMsg = (normalized: string) => {
     const major = semver.major(normalized);
-    return major > maxMajor
-      ? `handsontable-version major must be at most ${maxMajor}; got ${major}`
-      : null;
+    if (major > maxMajor) return `handsontable-version major must be at most ${maxMajor}; got ${major}`;
+    if (major < minMajor) return `handsontable-version major must be at least ${minMajor}; got ${major}`;
+    return null;
   };
 
   if (/^\d+$/.test(trimmed)) {
@@ -90,7 +97,7 @@ export function validateHandsontableVersion(
     const coerced = semver.coerce(trimmed);
     const normalized = coerced ? semver.valid(coerced.version, { loose: true }) : null;
     if (!normalized) return { ok: false, message: `handsontable-version could not be interpreted as semver` };
-    const err = capMsg(normalized);
+    const err = rangeMsg(normalized);
     return err ? { ok: false, message: err } : { ok: true, value: { ref: normalized, pkgPrNew: false } };
   }
 
@@ -102,7 +109,7 @@ export function validateHandsontableVersion(
   if (!normalized) {
     return { ok: false, message: `handsontable-version must be semver-valid or a pkg.pr.new id/URL` };
   }
-  const err = capMsg(normalized);
+  const err = rangeMsg(normalized);
   return err ? { ok: false, message: err } : { ok: true, value: { ref: normalized, pkgPrNew: false } };
 }
 

runner/pipeline/version.test.mjs

@@ -1,6 +1,6 @@
 import test from "node:test";
 import assert from "node:assert/strict";
-import { applyHandsontableCss } from "../packages/runtime/dist/version.js";
+import { applyHandsontableCss, validateHandsontableVersion } from "../packages/runtime/dist/version.js";
 
 const cssUrl = (version) =>
   `https://unpkg.com/handsontable@${version}/dist/handsontable.full.min.css`;
@@ -43,3 +43,32 @@ test("leaves files unchanged without a matching HTML entry or for pkg.pr.new ver
     pkgPrNewFiles,
   );
 });
+
+// DEV-2102 / ADR-0021 decision 10: majors below 15 were never empirically
+// verified, so validateHandsontableVersion rejects them by default — the same
+// floor GET /api/versions already enforces, now also covering direct
+// session/API calls that bypass the version dropdown.
+test("rejects majors below the default floor (15)", () => {
+  for (const value of ["14.5.0", "14", "14.2", "0.0.1"]) {
+    const result = validateHandsontableVersion(value);
+    assert.equal(result.ok, false, `expected ${value} to be rejected`);
+    assert.match(result.message, /must be at least 15/);
+  }
+});
+
+test("accepts majors within the default 15-19 range", () => {
+  for (const value of ["15.0.0", "17.1", "19"]) {
+    const result = validateHandsontableVersion(value);
+    assert.equal(result.ok, true, `expected ${value} to be accepted`);
+  }
+});
+
+test("pkg.pr.new refs bypass the floor (and ceiling) check", () => {
+  assert.equal(validateHandsontableVersion("1234").ok, true);
+  assert.equal(validateHandsontableVersion("https://pkg.pr.new/handsontable@abc123").ok, true);
+});
+
+test("custom minMajor/maxMajor override the defaults", () => {
+  assert.equal(validateHandsontableVersion("14.0.0", 19, 10).ok, true);
+  assert.equal(validateHandsontableVersion("9.0.0", 19, 10).ok, false);
+});