diff --git a/runner/.gitignore b/runner/.gitignore index aba96dba..0f054a14 100644 --- a/runner/.gitignore +++ b/runner/.gitignore @@ -21,3 +21,6 @@ build/ test-results/ playwright-report/ .playwright/ + +# generated starter-matrix run reports — paste into the ticket/PR instead +docs/reports/ diff --git a/runner/docs/run-and-deploy.md b/runner/docs/run-and-deploy.md index 9150905e..28399b4f 100644 --- a/runner/docs/run-and-deploy.md +++ b/runner/docs/run-and-deploy.md @@ -103,6 +103,10 @@ cascader drill-down, framework switching, and the "See in documentation" link. `E2E_BASE_URL=https://demos.handsontable.com E2E_LIVE=1 pnpm e2e`. - The API deploy workflow also does a post-deploy smoke (`GET /api/health` on `demos.handsontable.com` must return 200). +- A separate, opt-in starter compatibility matrix (`pnpm e2e:matrix`, gated + behind `E2E_STARTER_MATRIX=1`) boots every starter at every supported + Handsontable major against a live instance — not part of CI, run manually. + See `docs/starter-compat-matrix.md`. ### Authoring app (frontend) → GitHub Actions diff --git a/runner/docs/starter-compat-matrix.md b/runner/docs/starter-compat-matrix.md new file mode 100644 index 00000000..2ac15c3e --- /dev/null +++ b/runner/docs/starter-compat-matrix.md @@ -0,0 +1,86 @@ +# Starter compatibility matrix + +`e2e/starter-matrix.spec.ts` empirically verifies every starter in the +repo-root `examples/` directory (as cataloged in `catalog.json`) against every +Handsontable major the app claims to support (15-19) — DEV-2102 / ADR-0021 +decision 10. It boots each starter/major combination against an +already-running instance, checks that a Handsontable grid actually renders +(not just that the dev server responds), and records the result. + +It is **not** part of the deterministic PR suite (`pnpm e2e`) — it's an +opt-in, manually-run check. + +## Running it + +```sh +E2E_BASE_URL=https://demos.handsontable.com pnpm e2e:matrix +pnpm e2e:matrix:report +``` + +- `pnpm e2e:matrix` sets `E2E_STARTER_MATRIX=1` and runs at `--workers=2`, + writing JSON results to `test-results/starter-matrix.json`. +- `pnpm e2e:matrix:report` turns that JSON into a starter × major markdown + table, written to `docs/reports/starter-matrix-.md` and printed to + stdout. `docs/reports/` is gitignored — a dated run snapshot isn't living + documentation, it's a one-off result. Paste the stdout output straight into + the ticket/PR instead of committing the file. It accepts multiple JSON + paths if you ran the matrix in chunks (see below). + +Against a local stack instead of prod: point `E2E_BASE_URL` at your local +`vite --port 5173` authoring dev server, with the API worker running on 8787 +(see the "Full stack" section above) and `VITE_API_BASE` baked accordingly. + +### Running it in chunks + +The full matrix is ~80 starter×major combinations and can take well over an +hour end to end. If you can't leave it running unattended for that long (a +sleeping/locked laptop kills the process), run subsets with `--grep`, each to +its own JSON file: + +```sh +E2E_STARTER_MATRIX=1 E2E_BASE_URL=https://demos.handsontable.com \ + PLAYWRIGHT_JSON_OUTPUT_NAME=/tmp/matrix-batch1.json \ + npx playwright test e2e/starter-matrix.spec.ts --workers=2 --retries=1 \ + --grep 'matrix: (react|vue) @' --reporter=list,json +``` + +**Use an absolute path outside `test-results/` for `PLAYWRIGHT_JSON_OUTPUT_NAME` +when chunking.** Playwright wipes its default `outputDir` (`test-results/`) at +the start of every run — a second chunk's run will silently delete the first +chunk's JSON if both write there. Then merge: + +```sh +pnpm e2e:matrix:report /tmp/matrix-batch1.json /tmp/matrix-batch2.json ... +``` + +## Prod concurrency cap + +Container-engine starters (`engine: "container"` in `catalog.json`) each boot +a real Cloudflare Sandbox container via `POST /api/session`. Production caps +the live-preview `Sandbox` class at **5 concurrent instances** +(`workers/api/wrangler.jsonc`: `max_instances: 5`, separate from +`BuilderSandbox`'s `max_instances: 3` for the unrelated demo-sharing/build +pipeline, which this matrix never touches). The `e2e:matrix` script runs at +`--workers=2` to leave headroom for real traffic — don't raise it without +checking current prod load, and never run multiple `playwright test` +invocations against prod concurrently (their worker pools stack). + +## What it checks, and its limits + +Per combo: the preview reaches "Live" (not stuck booting or in an error +state), the Handsontable grid actually renders (`.handsontable .htCore td` +count > 0 — catches "server responded but nothing mounted"), zero unexpected +console/page errors, and that the requested version reached the session. +Sandpack-engine starters additionally verify the loaded Handsontable version +via the network request URL; container-engine starters can only confirm the +*requested* version was pinned in `package.json` (`Handsontable.version` is +typically unavailable in-frame for ESM bundles) — this is reported as +`unverified`, not a failure. + +A major with no stable npm release yet (checked live against the npm +registry, not the app's own sliced `/api/versions` listing) is skipped, not +failed. + +See DEV-2102 (or the PR that introduced this harness) for the first run's +findings and the resulting decision on `packages/runtime/src/version.ts`'s +minimum-major guard — the rationale is also recorded in that file's comments. diff --git a/runner/e2e/starter-matrix.spec.ts b/runner/e2e/starter-matrix.spec.ts new file mode 100644 index 00000000..33ac008e --- /dev/null +++ b/runner/e2e/starter-matrix.spec.ts @@ -0,0 +1,170 @@ +import { test, expect } from "@playwright/test"; +import { readFileSync } from "node:fs"; + +// Empirical starter-compatibility matrix (DEV-2102 / ADR-0021 decision 10): +// boots every starter in catalog.json at the latest patch of each supported +// Handsontable major (15-19) and records what actually breaks. Opt-in via +// E2E_STARTER_MATRIX=1 — this is not part of the deterministic PR suite; it +// spins ~55 real Tier-2 container sessions (the live-preview `Sandbox` class) +// and takes ~80-90 minutes. +// +// Prod only allows 5 concurrent live-preview containers +// (workers/api/wrangler.jsonc: Sandbox max_instances=5, separate from +// BuilderSandbox max_instances=3 which this test never touches). The +// e2e:matrix script runs at --workers=2 to leave headroom for real traffic — +// do not raise concurrency without checking current prod load. +// +// Run: E2E_BASE_URL=https://demos.handsontable.com pnpm e2e:matrix +// Report: pnpm e2e:matrix:report + +const ENABLED = process.env.E2E_STARTER_MATRIX === "1"; +const MAJORS = [15, 16, 17, 18, 19] as const; + +type CatalogExample = { + framework: string; + displayName: string; + engine: "sandpack" | "container"; +}; + +const catalog = JSON.parse(readFileSync(new URL("../catalog.json", import.meta.url), "utf8")) as { + examples: CatalogExample[]; +}; + +// Console/page noise that isn't a real compatibility break. Extend as the +// matrix surfaces new false positives. +const NOISE = [ + /non-commercial|evaluation license/i, + /Download the React DevTools/i, + /favicon\.ico/i, + /\[vite\] (connecting|connected)/i, + /ERR_BLOCKED_BY_CLIENT|third-party cookie/i, +]; +const isKnownNoise = (message: string) => NOISE.some((re) => re.test(message)); + +let latestByMajorPromise: Promise> | null = null; + +// Resolve the latest stable patch release per major straight from npm — the +// app's own GET /api/versions slices results to 15 entries and can drop older +// majors, so it isn't a reliable source for "latest per major". +function resolveLatestByMajor(): Promise> { + latestByMajorPromise ??= (async () => { + const res = await fetch("https://registry.npmjs.org/handsontable", { + headers: { Accept: "application/vnd.npm.install-v1+json" }, + }); + if (!res.ok) throw new Error(`npm registry responded ${res.status}`); + const doc = (await res.json()) as { versions: Record }; + const best = new Map(); + for (const version of Object.keys(doc.versions)) { + if (!/^\d+\.\d+\.\d+$/.test(version)) continue; + const [major, minor, patch] = version.split(".").map(Number); + const current = best.get(major); + if (!current) { + best.set(major, version); + continue; + } + const [, currentMinor, currentPatch] = current.split(".").map(Number); + if (minor > currentMinor || (minor === currentMinor && patch > currentPatch)) { + best.set(major, version); + } + } + return best; + })(); + return latestByMajorPromise; +} + +for (const entry of catalog.examples) { + for (const major of MAJORS) { + test(`matrix: ${entry.framework} @ ${major} [${entry.engine}]`, async ({ page, request }, testInfo) => { + test.skip(!ENABLED, "set E2E_STARTER_MATRIX=1 to run the starter compatibility matrix"); + test.setTimeout(entry.engine === "container" ? 300_000 : 150_000); + + const byMajor = await resolveLatestByMajor(); + const version = byMajor.get(major); + // A major with no stable npm release yet (e.g. 19 is pre-release as of + // writing — dist-tags carry only `next`/`rc`) isn't a starter failure, + // it's nothing to test yet. Skip rather than report a false breakage. + test.skip(!version, `no stable handsontable release published for major ${major} yet`); + if (!version) return; + + // Annotate immediately so a failing test still reports which version it + // was testing — the report shouldn't say "unknown" just because the + // test threw before reaching the end. + testInfo.annotations.push({ type: "resolvedVersion", description: version }, { type: "engine", description: entry.engine }); + + const consoleErrors: string[] = []; + const pageErrors: string[] = []; + const htRequestVersions: string[] = []; + const sessions: { id: string; apiBase: string }[] = []; + let postedHtVersion: string | null = null; + + page.on("console", (msg) => { + if (msg.type() === "error") consoleErrors.push(msg.text()); + }); + page.on("pageerror", (err) => pageErrors.push(String(err))); + page.on("request", (req) => { + const url = req.url(); + const match = url.match(/handsontable[@/](\d+\.\d+\.\d+)/); + if (match) htRequestVersions.push(match[1]); + if (req.method() === "POST" && /\/api\/session$/.test(url)) { + try { + postedHtVersion = (JSON.parse(req.postData() ?? "{}") as { htVersion?: string }).htVersion ?? null; + } catch { + // ignore malformed payloads — absence just leaves postedHtVersion null + } + } + }); + page.on("response", async (res) => { + if (res.request().method() === "POST" && /\/api\/session$/.test(res.url()) && res.ok()) { + const body = (await res.json().catch(() => null)) as { sessionId?: string } | null; + if (body?.sessionId) sessions.push({ id: body.sessionId, apiBase: new URL(res.url()).origin }); + } + }); + + try { + await page.goto(`/?example=${encodeURIComponent(entry.framework)}&v=${version}`); + + const preview = page.locator('section[aria-label="Preview"]'); + // The status bar is the first direct child
of the preview + // section (PreviewPane.tsx) — scope to it directly rather than + // getByText scanning the whole section, which also matches the live + // boot-log
 when a container's dev-server error text happens to
+        // start with "Error:" (seen with mui: strict-mode violation, two
+        // elements matched).
+        const statusBar = preview.locator(":scope > div").first();
+        await expect(statusBar).toHaveText("Live", {
+          timeout: entry.engine === "container" ? 240_000 : 120_000,
+        });
+
+        // Container "Live" only means the dev server responded — the frame may
+        // still be blank or mid-hydration (Next/Nuxt/Astro). The rendered grid
+        // is the real functional check.
+        const frame = page.frameLocator('iframe[title="Demo preview"]');
+        const cells = frame.locator(".handsontable .htCore td");
+        await expect
+          .poll(async () => cells.count().catch(() => 0), { timeout: 60_000, intervals: [1_000] })
+          .toBeGreaterThan(0);
+
+        const realErrors = [...consoleErrors, ...pageErrors].filter((e) => !isKnownNoise(e));
+        expect(realErrors, `console/page errors:\n${realErrors.join("\n")}`).toEqual([]);
+
+        let detectedVersion: string | null = null;
+        if (entry.engine === "sandpack") {
+          detectedVersion = htRequestVersions.find((v) => v === version) ?? htRequestVersions[0] ?? null;
+        } else {
+          expect(postedHtVersion, "requested handsontable version reached the container session").toBe(version);
+          detectedVersion = await frame
+            .locator("body")
+            .evaluate(() => (window as unknown as { Handsontable?: { version?: string } }).Handsontable?.version ?? null)
+            .catch(() => null);
+        }
+
+        testInfo.annotations.push({ type: "detectedVersion", description: detectedVersion ?? "unverified" });
+        if (detectedVersion) expect(detectedVersion.startsWith(`${major}.`)).toBe(true);
+      } finally {
+        for (const { id, apiBase } of sessions) {
+          await request.delete(`${apiBase}/api/session/${id}`).catch(() => {});
+        }
+      }
+    });
+  }
+}
diff --git a/runner/package.json b/runner/package.json
index f29794b7..7a96cf84 100644
--- a/runner/package.json
+++ b/runner/package.json
@@ -16,7 +16,9 @@
     "typecheck": "pnpm -r run typecheck",
     "lint": "pnpm -r run lint",
     "test": "pnpm --filter @handsontable/demo-runtime build && node --test pipeline/*.test.mjs",
-    "e2e": "playwright test"
+    "e2e": "playwright test",
+    "e2e:matrix": "E2E_STARTER_MATRIX=1 PLAYWRIGHT_JSON_OUTPUT_NAME=test-results/starter-matrix.json playwright test e2e/starter-matrix.spec.ts --workers=2 --retries=1 --reporter=list,json",
+    "e2e:matrix:report": "node scripts/starter-matrix-report.mjs"
   },
   "devDependencies": {
     "@playwright/test": "^1.48.0",
diff --git a/runner/packages/runtime/src/version.ts b/runner/packages/runtime/src/version.ts
index 678e4e43..06551b10 100644
--- a/runner/packages/runtime/src/version.ts
+++ b/runner/packages/runtime/src/version.ts
@@ -11,6 +11,12 @@ import semver from "semver";
 import type { FilesMap, HandsontableVersionRef } from "./types.js";
 
 export const DEFAULT_MAX_MAJOR = 19;
+// Empirically verified range is 15-19 (DEV-2102 / ADR-0021 decision 10); majors
+// below 15 have never been tested against current starters/wrappers and stay
+// out of scope. Previously only the UI-facing GET /api/versions listing
+// enforced this floor — a direct session/API call bypassing the version
+// dropdown could still request an untested major like 14.
+export const DEFAULT_MIN_MAJOR = 15;
 const MIN_BARE_NUMERIC_PKG_PR_NEW_REF = 1000;
 
 /** Dependency never rewritten: an independently versioned Handsontable plugin. */
@@ -59,6 +65,7 @@ export type ValidationResult =
 export function validateHandsontableVersion(
   value: unknown,
   maxMajor: number = DEFAULT_MAX_MAJOR,
+  minMajor: number = DEFAULT_MIN_MAJOR,
 ): ValidationResult {
   if (value === undefined || value === null) {
     return { ok: false, message: "handsontable-version is required" };
@@ -69,11 +76,11 @@ export function validateHandsontableVersion(
   const urlRef = parsePkgPrNewFromUrl(trimmed);
   if (urlRef !== null) return { ok: true, value: { ref: urlRef, pkgPrNew: true } };
 
-  const capMsg = (normalized: string) => {
+  const rangeMsg = (normalized: string) => {
     const major = semver.major(normalized);
-    return major > maxMajor
-      ? `handsontable-version major must be at most ${maxMajor}; got ${major}`
-      : null;
+    if (major > maxMajor) return `handsontable-version major must be at most ${maxMajor}; got ${major}`;
+    if (major < minMajor) return `handsontable-version major must be at least ${minMajor}; got ${major}`;
+    return null;
   };
 
   if (/^\d+$/.test(trimmed)) {
@@ -90,7 +97,7 @@ export function validateHandsontableVersion(
     const coerced = semver.coerce(trimmed);
     const normalized = coerced ? semver.valid(coerced.version, { loose: true }) : null;
     if (!normalized) return { ok: false, message: `handsontable-version could not be interpreted as semver` };
-    const err = capMsg(normalized);
+    const err = rangeMsg(normalized);
     return err ? { ok: false, message: err } : { ok: true, value: { ref: normalized, pkgPrNew: false } };
   }
 
@@ -102,7 +109,7 @@ export function validateHandsontableVersion(
   if (!normalized) {
     return { ok: false, message: `handsontable-version must be semver-valid or a pkg.pr.new id/URL` };
   }
-  const err = capMsg(normalized);
+  const err = rangeMsg(normalized);
   return err ? { ok: false, message: err } : { ok: true, value: { ref: normalized, pkgPrNew: false } };
 }
 
diff --git a/runner/pipeline/version.test.mjs b/runner/pipeline/version.test.mjs
index 591ed39b..50ba11fe 100644
--- a/runner/pipeline/version.test.mjs
+++ b/runner/pipeline/version.test.mjs
@@ -1,6 +1,6 @@
 import test from "node:test";
 import assert from "node:assert/strict";
-import { applyHandsontableCss } from "../packages/runtime/dist/version.js";
+import { applyHandsontableCss, validateHandsontableVersion } from "../packages/runtime/dist/version.js";
 
 const cssUrl = (version) =>
   `https://unpkg.com/handsontable@${version}/dist/handsontable.full.min.css`;
@@ -43,3 +43,32 @@ test("leaves files unchanged without a matching HTML entry or for pkg.pr.new ver
     pkgPrNewFiles,
   );
 });
+
+// DEV-2102 / ADR-0021 decision 10: majors below 15 were never empirically
+// verified, so validateHandsontableVersion rejects them by default — the same
+// floor GET /api/versions already enforces, now also covering direct
+// session/API calls that bypass the version dropdown.
+test("rejects majors below the default floor (15)", () => {
+  for (const value of ["14.5.0", "14", "14.2", "0.0.1"]) {
+    const result = validateHandsontableVersion(value);
+    assert.equal(result.ok, false, `expected ${value} to be rejected`);
+    assert.match(result.message, /must be at least 15/);
+  }
+});
+
+test("accepts majors within the default 15-19 range", () => {
+  for (const value of ["15.0.0", "17.1", "19"]) {
+    const result = validateHandsontableVersion(value);
+    assert.equal(result.ok, true, `expected ${value} to be accepted`);
+  }
+});
+
+test("pkg.pr.new refs bypass the floor (and ceiling) check", () => {
+  assert.equal(validateHandsontableVersion("1234").ok, true);
+  assert.equal(validateHandsontableVersion("https://pkg.pr.new/handsontable@abc123").ok, true);
+});
+
+test("custom minMajor/maxMajor override the defaults", () => {
+  assert.equal(validateHandsontableVersion("14.0.0", 19, 10).ok, true);
+  assert.equal(validateHandsontableVersion("9.0.0", 19, 10).ok, false);
+});
diff --git a/runner/scripts/starter-matrix-report.mjs b/runner/scripts/starter-matrix-report.mjs
new file mode 100644
index 00000000..7c5a872b
--- /dev/null
+++ b/runner/scripts/starter-matrix-report.mjs
@@ -0,0 +1,159 @@
+// Turn Playwright JSON reporter output from `pnpm e2e:matrix` into a
+// markdown starter × major compatibility table (DEV-2102 / ADR-0021 decision
+// 10). Accepts one or more results files (the matrix is run in chunks against
+// prod, each writing its own JSON — this merges them). Writes
+// docs/reports/starter-matrix-.md and echoes the report to stdout.
+//
+// Usage: node scripts/starter-matrix-report.mjs [path/to/results.json ...]
+//        (defaults to test-results/starter-matrix.json when no args given)
+
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const RUNNER_DIR = path.resolve(__dirname, "..");
+
+const resultsPaths = (process.argv.length > 2 ? process.argv.slice(2) : ["test-results/starter-matrix.json"]).map(
+  (p) => path.resolve(RUNNER_DIR, p),
+);
+
+const missing = resultsPaths.filter((p) => !fs.existsSync(p));
+if (missing.length > 0) {
+  console.error(`Missing results file(s):\n${missing.join("\n")}\nRun \`pnpm e2e:matrix\` first.`);
+  process.exit(1);
+}
+
+const TITLE_RE = /^matrix: (.+) @ (\d+) \[(\w+)\]$/;
+const ANSI_RE = /\x1b\[[0-9;]*m/g;
+const stripAnsi = (s) => (typeof s === "string" ? s.replace(ANSI_RE, "") : s);
+
+// combo key -> { framework, engine, results: Map }
+const combos = new Map();
+
+function walkSuites(suites) {
+  for (const suite of suites ?? []) {
+    for (const spec of suite.specs ?? []) {
+      for (const t of spec.tests ?? []) {
+        const match = TITLE_RE.exec(spec.title);
+        if (!match) continue;
+        const [, framework, majorStr, engine] = match;
+        const major = Number(majorStr);
+
+        const lastResult = t.results?.[t.results.length - 1];
+        const status = t.status; // "expected" | "unexpected" | "flaky" | "skipped"
+
+        const annotations = Object.fromEntries(
+          (t.annotations ?? []).map((a) => [a.type, a.description]),
+        );
+
+        const errorMessage = stripAnsi(lastResult?.error?.message ?? lastResult?.errors?.[0]?.message ?? null);
+
+        if (!combos.has(framework)) combos.set(framework, { framework, engine, results: new Map() });
+        // Later files win on overlap (e.g. a rerun of a previously-killed chunk).
+        combos.get(framework).results.set(major, {
+          status,
+          resolvedVersion: annotations.resolvedVersion ?? null,
+          detectedVersion: annotations.detectedVersion ?? null,
+          error: errorMessage,
+        });
+      }
+    }
+    walkSuites(suite.suites);
+  }
+}
+
+for (const resultsPath of resultsPaths) {
+  const raw = JSON.parse(fs.readFileSync(resultsPath, "utf8"));
+  walkSuites(raw.suites);
+}
+
+if (combos.size === 0) {
+  console.error("No `matrix: ...` tests found in results file — did the matrix spec actually run?");
+  process.exit(1);
+}
+
+const MAJORS = [15, 16, 17, 18, 19];
+
+const STATUS_ICON = {
+  expected: "✅",
+  flaky: "⚠️",
+  unexpected: "❌",
+  skipped: "⏭️",
+  missing: "❓",
+};
+
+function cell(result) {
+  if (!result) return STATUS_ICON.missing;
+  const icon = STATUS_ICON[result.status] ?? STATUS_ICON.missing;
+  const version = result.resolvedVersion ? ` ${result.resolvedVersion}` : "";
+  const verified = result.detectedVersion && result.detectedVersion !== "unverified" ? "v" : "";
+  return `${icon}${verified}${version}`;
+}
+
+const sortedFrameworks = [...combos.values()].sort((a, b) => a.framework.localeCompare(b.framework));
+
+const lines = [];
+lines.push(`# Starter compatibility matrix`);
+lines.push("");
+lines.push(`Generated from ${resultsPaths.map((p) => `\`${path.relative(RUNNER_DIR, p)}\``).join(", ")}.`);
+lines.push("");
+lines.push(
+  `Legend: ✅ passed, ✅v passed with in-frame version verified, ⚠️ flaky (passed on retry), ❌ failed, ⏭️ skipped, ❓ no result found.`,
+);
+lines.push("");
+lines.push(`| starter (engine) | ${MAJORS.map((m) => `${m}.x`).join(" | ")} |`);
+lines.push(`|---|${MAJORS.map(() => "---").join("|")}|`);
+for (const { framework, engine, results } of sortedFrameworks) {
+  const row = MAJORS.map((m) => cell(results.get(m)));
+  lines.push(`| ${framework} (${engine}) | ${row.join(" | ")} |`);
+}
+
+const failures = [];
+for (const { framework, engine, results } of sortedFrameworks) {
+  for (const major of MAJORS) {
+    const r = results.get(major);
+    if (r && (r.status === "unexpected" || r.status === "flaky")) {
+      failures.push({ framework, engine, major, ...r });
+    }
+  }
+}
+
+if (failures.length > 0) {
+  lines.push("");
+  lines.push(`## Failures / flaky (${failures.length})`);
+  lines.push("");
+  for (const f of failures) {
+    lines.push(`### ${f.framework} @ ${f.major} [${f.engine}] — ${f.status}`);
+    lines.push("");
+    lines.push(`- resolved version: ${f.resolvedVersion ?? "unknown"}`);
+    if (f.error) {
+      lines.push("");
+      lines.push("```");
+      lines.push(f.error.split("\n").slice(0, 15).join("\n"));
+      lines.push("```");
+    }
+    lines.push("");
+  }
+} else {
+  lines.push("");
+  lines.push("No failures.");
+}
+
+lines.push("");
+lines.push(
+  "Note: container-engine starters verify only that the requested version reached the session " +
+    "(package.json pin); in-frame `Handsontable.version` is typically unavailable for ESM bundles " +
+    "and is reported as `unverified` rather than a failure.",
+);
+
+const report = lines.join("\n") + "\n";
+
+const today = new Date().toISOString().slice(0, 10);
+const outDir = path.join(RUNNER_DIR, "docs", "reports");
+fs.mkdirSync(outDir, { recursive: true });
+const outPath = path.join(outDir, `starter-matrix-${today}.md`);
+fs.writeFileSync(outPath, report);
+
+console.log(report);
+console.error(`\nWrote ${path.relative(RUNNER_DIR, outPath)}`);