test(replay): freeze the v1-v5 replay corpus as differential oracles

.gitattributes

@@ -1 +1,2 @@
 * text=auto eol=lf
+tests/replay-corpus/** -text

biome.json

@@ -8,7 +8,8 @@
       "!!.testforge",
       "!!.omx",
       "!!graphify-out",
-      "!!site/evidence"
+      "!!site/evidence",
+      "!!tests/replay-corpus"
     ]
   },
   "formatter": {

tests/replay-corpus.test.ts

@@ -0,0 +1,82 @@
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import path from "node:path";
+import { describe, it } from "node:test";
+import { gunzipSync } from "node:zlib";
+import { PACKAGE_ROOT, runCliCase } from "./support/goldens.js";
+
+/** One real document produced by a published version, frozen with what that version decided. */
+interface CorpusEntry {
+  id: string;
+  kind: "evidence" | "qualification";
+  schemaVersion: string;
+  producer: string;
+  document: string;
+  sha256: string;
+  decision: string;
+  decisionDigest: string;
+  artifactDigest: string;
+  command: "replay" | "qualification-replay";
+  replay: Record<string, unknown>;
+  replayedBy: string[];
+}
+
+const CORPUS_ROOT = path.join(PACKAGE_ROOT, "tests", "replay-corpus");
+const corpus = JSON.parse(await readFile(path.join(CORPUS_ROOT, "corpus.json"), "utf8")) as {
+  entries: CorpusEntry[];
+};
+
+async function documentBytes(entry: CorpusEntry): Promise<Buffer> {
+  const bytes = await readFile(path.join(PACKAGE_ROOT, entry.document));
+  return entry.document.endsWith(".gz") ? gunzipSync(bytes) : bytes;
+}
+
+describe("frozen replay corpus", () => {
+  it("covers evidence manifests v1 to v5 and qualification receipts, each declared once", async () => {
+    const ids = corpus.entries.map((entry) => entry.id);
+    assert.equal(new Set(ids).size, ids.length, "duplicate corpus id");
+    const manifestVersions = new Set(
+      corpus.entries
+        .filter((entry) => entry.kind === "evidence")
+        .map((entry) => entry.schemaVersion),
+    );
+    assert.deepEqual([...manifestVersions].sort(), ["1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0"]);
+    assert.ok(corpus.entries.some((entry) => entry.kind === "qualification"));
+    const files = await readdir(path.join(CORPUS_ROOT, "documents"));
+    const declared = corpus.entries
+      .map((entry) => entry.document)
+      .filter((document) => document.startsWith("tests/replay-corpus/documents/"))
+      .map((document) => path.basename(document));
+    assert.deepEqual(files.sort(), declared.sort());
+  });
+
+  for (const entry of corpus.entries) {
+    it(`${entry.command} ${entry.id}`, { timeout: 30_000 }, async () => {
+      const bytes = await documentBytes(entry);
+      // Frozen evidence is never rewritten: the bytes are the ones the producer emitted.
+      assert.equal(`sha256:${createHash("sha256").update(bytes).digest("hex")}`, entry.sha256);
+      const document = JSON.parse(bytes.toString("utf8")) as Record<string, unknown>;
+      const decision =
+        entry.kind === "evidence"
+          ? (document.decision as { status: unknown }).status
+          : document.decision;
+      assert.equal(document.schemaVersion, entry.schemaVersion);
+      assert.equal(decision, entry.decision);
+      assert.equal(document.decisionDigest, entry.decisionDigest);
+      assert.equal(document.artifactDigest, entry.artifactDigest);
+
+      const observed = await runCliCase({
+        id: entry.id,
+        argv: [entry.command, "-"],
+        stdin: bytes.toString("utf8"),
+      });
+      assert.equal(observed.exitCode, 0, observed.stderr);
+      assert.equal(observed.stderr, "");
+      // The current build must accept the document exactly as its producer and every later
+      // published version did, with byte-identical output.
+      assert.equal(observed.stdout, `${JSON.stringify(entry.replay)}\n`);
+      assert.equal(entry.replay.valid, true);
+    });
+  }
+});

tests/replay-corpus/README.md

@@ -0,0 +1,44 @@
+# Replay corpus
+
+Real evidence manifests v1 to v5 and qualification receipts, each produced by the published
+version that introduced its contract. ADR 0002 makes them differential oracles: every later
+implementation, in any language, must replay each one with the same decision, the same digests
+and the same output.
+
+`tests/replay-corpus.test.ts` checks, for every entry of `corpus.json`:
+
+- the document bytes still hash to the recorded `sha256`;
+- its `schemaVersion`, decision, `decisionDigest` and `artifactDigest` are the producer's;
+- `dist/cli.js replay -` or `qualification-replay -`, given the document on stdin, exits 0 with
+  empty stderr and prints exactly the recorded result.
+
+## Entries
+
+| Entries | Contract | Producer |
+| --- | --- | --- |
+| `evidence-v1` | manifest 1.0.0, `node:test` | `assertledger@1.0.0` |
+| `evidence-v2` | manifest 2.0.0, `node:test` | `assertledger@1.1.0` |
+| `evidence-v3` | manifest 3.0.0, `bun-test` | `assertledger@1.2.0` |
+| `evidence-v4` | manifest 4.0.0, `bun-test-designated` | `assertledger@1.5.0` |
+| `evidence-v5` | manifest 5.0.0, `bun-test-designated` | `assertledger@2.1.0` |
+| `qualification-1.5.0-*` | receipt 1.0.0, `node-test` and `command` adapters | `assertledger@1.5.0` |
+| `qualification-2.1.0-*` | receipt 1.0.0, same plans | `assertledger@2.1.0` |
+| `evidence-v4-container-*` | manifest 4.0.0, container backend | the 12 runs of `benchmarks/container-runtime/2026-10-11`, read in place |
+
+The documents in `documents/` were produced on Windows x64 with Node 22.23.2 and Bun 1.4.2, in
+trusted-local mode (`verify` or `qualify` with `--allow-unsafe-execution`), from the inputs kept in
+`inputs/<id>/`. Each `request.json` there is the request as run, with its root rewritten to the
+relative directory beside it; the root is not sealed in any document. The v3 to v5 manifests seal
+the resolved Bun executable path, a machine path, because it is part of the producer's evidence.
+
+`replayedBy` lists the published versions, from the producer onwards, whose replay gave this exact
+result when the corpus was frozen, plus `recorded` for the replay stored with a benchmark run. That
+cross-version check ran once, with the packages installed from npm; CI replays only the current
+build.
+
+## Changing the corpus
+
+Never edit, re-encode or reformat a document: Biome and TypeScript skip this directory and Git does
+not normalize it. A new contract version adds entries produced by the version that introduces it.
+A test failure here means the current build no longer replays a historical document as it was
+decided; fix the build, not the corpus.