From cbd43c653057855921111e8db650aca84e73e7a5 Mon Sep 17 00:00:00 2001 From: Guillaume Fossier Date: Mon, 26 Oct 2026 20:29:28 +0100 Subject: [PATCH] test(replay): freeze the v1-v5 replay corpus as differential oracles Real evidence manifests v1 to v5 and qualification receipts, each produced by the published version that introduced its contract, with the 12 container v4 benchmark manifests read in place. The test checks the frozen bytes, the producer's decision and digests, and the current build's replay output byte for byte. Refs #84 Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 1 + biome.json | 3 +- tests/replay-corpus.test.ts | 82 +++ tests/replay-corpus/README.md | 44 ++ tests/replay-corpus/corpus.json | 516 ++++++++++++++++++ .../documents/evidence-v1.manifest.json | 1 + .../documents/evidence-v2.manifest.json | 1 + .../documents/evidence-v3.manifest.json | 1 + .../documents/evidence-v4.manifest.json | 1 + .../documents/evidence-v5.manifest.json | 1 + ...qualification-1.5.0-node-test.receipt.json | 1 + ...alification-1.5.0-propagation.receipt.json | 1 + ...qualification-2.1.0-node-test.receipt.json | 1 + ...alification-2.1.0-propagation.receipt.json | 1 + .../evidence-v1/repository/package.json | 5 + .../evidence-v1/repository/src/is-even.js | 3 + .../evidence-v1/repository/tests/base.test.js | 6 + .../inputs/evidence-v1/request.json | 109 ++++ .../evidence-v2/repository/package.json | 5 + .../evidence-v2/repository/src/is-even.js | 3 + .../evidence-v2/repository/tests/base.test.js | 6 + .../inputs/evidence-v2/request.json | 109 ++++ .../evidence-v3/repository/package.json | 1 + .../evidence-v3/repository/src/subject.ts | 1 + .../evidence-v3/repository/tests/base.test.ts | 2 + .../inputs/evidence-v3/request.json | 109 ++++ .../evidence-v4/repository/package.json | 1 + .../evidence-v4/repository/src/subject.ts | 1 + .../evidence-v4/repository/tests/base.test.ts | 2 + .../repository/tests/parity.test.ts | 7 + .../inputs/evidence-v4/request.json | 102 ++++ .../evidence-v5/repository/package.json | 1 + .../evidence-v5/repository/src/subject.ts | 1 + .../evidence-v5/repository/tests/base.test.ts | 2 + .../repository/tests/parity.test.ts | 7 + .../inputs/evidence-v5/request.json | 107 ++++ .../request.json | 117 ++++ .../source/public.test.mjs | 3 + .../request.json | 107 ++++ .../source/leaf.mjs | 3 + .../request.json | 117 ++++ .../source/public.test.mjs | 3 + .../request.json | 107 ++++ .../source/leaf.mjs | 3 + tsconfig.json | 3 +- 45 files changed, 1706 insertions(+), 2 deletions(-) create mode 100644 tests/replay-corpus.test.ts create mode 100644 tests/replay-corpus/README.md create mode 100644 tests/replay-corpus/corpus.json create mode 100644 tests/replay-corpus/documents/evidence-v1.manifest.json create mode 100644 tests/replay-corpus/documents/evidence-v2.manifest.json create mode 100644 tests/replay-corpus/documents/evidence-v3.manifest.json create mode 100644 tests/replay-corpus/documents/evidence-v4.manifest.json create mode 100644 tests/replay-corpus/documents/evidence-v5.manifest.json create mode 100644 tests/replay-corpus/documents/qualification-1.5.0-node-test.receipt.json create mode 100644 tests/replay-corpus/documents/qualification-1.5.0-propagation.receipt.json create mode 100644 tests/replay-corpus/documents/qualification-2.1.0-node-test.receipt.json create mode 100644 tests/replay-corpus/documents/qualification-2.1.0-propagation.receipt.json create mode 100644 tests/replay-corpus/inputs/evidence-v1/repository/package.json create mode 100644 tests/replay-corpus/inputs/evidence-v1/repository/src/is-even.js create mode 100644 tests/replay-corpus/inputs/evidence-v1/repository/tests/base.test.js create mode 100644 tests/replay-corpus/inputs/evidence-v1/request.json create mode 100644 tests/replay-corpus/inputs/evidence-v2/repository/package.json create mode 100644 tests/replay-corpus/inputs/evidence-v2/repository/src/is-even.js create mode 100644 tests/replay-corpus/inputs/evidence-v2/repository/tests/base.test.js create mode 100644 tests/replay-corpus/inputs/evidence-v2/request.json create mode 100644 tests/replay-corpus/inputs/evidence-v3/repository/package.json create mode 100644 tests/replay-corpus/inputs/evidence-v3/repository/src/subject.ts create mode 100644 tests/replay-corpus/inputs/evidence-v3/repository/tests/base.test.ts create mode 100644 tests/replay-corpus/inputs/evidence-v3/request.json create mode 100644 tests/replay-corpus/inputs/evidence-v4/repository/package.json create mode 100644 tests/replay-corpus/inputs/evidence-v4/repository/src/subject.ts create mode 100644 tests/replay-corpus/inputs/evidence-v4/repository/tests/base.test.ts create mode 100644 tests/replay-corpus/inputs/evidence-v4/repository/tests/parity.test.ts create mode 100644 tests/replay-corpus/inputs/evidence-v4/request.json create mode 100644 tests/replay-corpus/inputs/evidence-v5/repository/package.json create mode 100644 tests/replay-corpus/inputs/evidence-v5/repository/src/subject.ts create mode 100644 tests/replay-corpus/inputs/evidence-v5/repository/tests/base.test.ts create mode 100644 tests/replay-corpus/inputs/evidence-v5/repository/tests/parity.test.ts create mode 100644 tests/replay-corpus/inputs/evidence-v5/request.json create mode 100644 tests/replay-corpus/inputs/qualification-1.5.0-node-test/request.json create mode 100644 tests/replay-corpus/inputs/qualification-1.5.0-node-test/source/public.test.mjs create mode 100644 tests/replay-corpus/inputs/qualification-1.5.0-propagation/request.json create mode 100644 tests/replay-corpus/inputs/qualification-1.5.0-propagation/source/leaf.mjs create mode 100644 tests/replay-corpus/inputs/qualification-2.1.0-node-test/request.json create mode 100644 tests/replay-corpus/inputs/qualification-2.1.0-node-test/source/public.test.mjs create mode 100644 tests/replay-corpus/inputs/qualification-2.1.0-propagation/request.json create mode 100644 tests/replay-corpus/inputs/qualification-2.1.0-propagation/source/leaf.mjs diff --git a/.gitattributes b/.gitattributes index 6313b56..98d74a3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ * text=auto eol=lf +tests/replay-corpus/** -text diff --git a/biome.json b/biome.json index c689cbf..2c2879e 100644 --- a/biome.json +++ b/biome.json @@ -8,7 +8,8 @@ "!!.testforge", "!!.omx", "!!graphify-out", - "!!site/evidence" + "!!site/evidence", + "!!tests/replay-corpus" ] }, "formatter": { diff --git a/tests/replay-corpus.test.ts b/tests/replay-corpus.test.ts new file mode 100644 index 0000000..179c96c --- /dev/null +++ b/tests/replay-corpus.test.ts @@ -0,0 +1,82 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { describe, it } from "node:test"; +import { gunzipSync } from "node:zlib"; +import { PACKAGE_ROOT, runCliCase } from "./support/goldens.js"; + +/** One real document produced by a published version, frozen with what that version decided. */ +interface CorpusEntry { + id: string; + kind: "evidence" | "qualification"; + schemaVersion: string; + producer: string; + document: string; + sha256: string; + decision: string; + decisionDigest: string; + artifactDigest: string; + command: "replay" | "qualification-replay"; + replay: Record; + replayedBy: string[]; +} + +const CORPUS_ROOT = path.join(PACKAGE_ROOT, "tests", "replay-corpus"); +const corpus = JSON.parse(await readFile(path.join(CORPUS_ROOT, "corpus.json"), "utf8")) as { + entries: CorpusEntry[]; +}; + +async function documentBytes(entry: CorpusEntry): Promise { + const bytes = await readFile(path.join(PACKAGE_ROOT, entry.document)); + return entry.document.endsWith(".gz") ? gunzipSync(bytes) : bytes; +} + +describe("frozen replay corpus", () => { + it("covers evidence manifests v1 to v5 and qualification receipts, each declared once", async () => { + const ids = corpus.entries.map((entry) => entry.id); + assert.equal(new Set(ids).size, ids.length, "duplicate corpus id"); + const manifestVersions = new Set( + corpus.entries + .filter((entry) => entry.kind === "evidence") + .map((entry) => entry.schemaVersion), + ); + assert.deepEqual([...manifestVersions].sort(), ["1.0.0", "2.0.0", "3.0.0", "4.0.0", "5.0.0"]); + assert.ok(corpus.entries.some((entry) => entry.kind === "qualification")); + const files = await readdir(path.join(CORPUS_ROOT, "documents")); + const declared = corpus.entries + .map((entry) => entry.document) + .filter((document) => document.startsWith("tests/replay-corpus/documents/")) + .map((document) => path.basename(document)); + assert.deepEqual(files.sort(), declared.sort()); + }); + + for (const entry of corpus.entries) { + it(`${entry.command} ${entry.id}`, { timeout: 30_000 }, async () => { + const bytes = await documentBytes(entry); + // Frozen evidence is never rewritten: the bytes are the ones the producer emitted. + assert.equal(`sha256:${createHash("sha256").update(bytes).digest("hex")}`, entry.sha256); + const document = JSON.parse(bytes.toString("utf8")) as Record; + const decision = + entry.kind === "evidence" + ? (document.decision as { status: unknown }).status + : document.decision; + assert.equal(document.schemaVersion, entry.schemaVersion); + assert.equal(decision, entry.decision); + assert.equal(document.decisionDigest, entry.decisionDigest); + assert.equal(document.artifactDigest, entry.artifactDigest); + + const observed = await runCliCase({ + id: entry.id, + argv: [entry.command, "-"], + stdin: bytes.toString("utf8"), + }); + assert.equal(observed.exitCode, 0, observed.stderr); + assert.equal(observed.stderr, ""); + // The current build must accept the document exactly as its producer and every later + // published version did, with byte-identical output. + assert.equal(observed.stdout, `${JSON.stringify(entry.replay)}\n`); + assert.equal(entry.replay.valid, true); + }); + } +}); diff --git a/tests/replay-corpus/README.md b/tests/replay-corpus/README.md new file mode 100644 index 0000000..9578696 --- /dev/null +++ b/tests/replay-corpus/README.md @@ -0,0 +1,44 @@ +# Replay corpus + +Real evidence manifests v1 to v5 and qualification receipts, each produced by the published +version that introduced its contract. ADR 0002 makes them differential oracles: every later +implementation, in any language, must replay each one with the same decision, the same digests +and the same output. + +`tests/replay-corpus.test.ts` checks, for every entry of `corpus.json`: + +- the document bytes still hash to the recorded `sha256`; +- its `schemaVersion`, decision, `decisionDigest` and `artifactDigest` are the producer's; +- `dist/cli.js replay -` or `qualification-replay -`, given the document on stdin, exits 0 with + empty stderr and prints exactly the recorded result. + +## Entries + +| Entries | Contract | Producer | +| --- | --- | --- | +| `evidence-v1` | manifest 1.0.0, `node:test` | `assertledger@1.0.0` | +| `evidence-v2` | manifest 2.0.0, `node:test` | `assertledger@1.1.0` | +| `evidence-v3` | manifest 3.0.0, `bun-test` | `assertledger@1.2.0` | +| `evidence-v4` | manifest 4.0.0, `bun-test-designated` | `assertledger@1.5.0` | +| `evidence-v5` | manifest 5.0.0, `bun-test-designated` | `assertledger@2.1.0` | +| `qualification-1.5.0-*` | receipt 1.0.0, `node-test` and `command` adapters | `assertledger@1.5.0` | +| `qualification-2.1.0-*` | receipt 1.0.0, same plans | `assertledger@2.1.0` | +| `evidence-v4-container-*` | manifest 4.0.0, container backend | the 12 runs of `benchmarks/container-runtime/2026-10-11`, read in place | + +The documents in `documents/` were produced on Windows x64 with Node 22.23.2 and Bun 1.4.2, in +trusted-local mode (`verify` or `qualify` with `--allow-unsafe-execution`), from the inputs kept in +`inputs//`. Each `request.json` there is the request as run, with its root rewritten to the +relative directory beside it; the root is not sealed in any document. The v3 to v5 manifests seal +the resolved Bun executable path, a machine path, because it is part of the producer's evidence. + +`replayedBy` lists the published versions, from the producer onwards, whose replay gave this exact +result when the corpus was frozen, plus `recorded` for the replay stored with a benchmark run. That +cross-version check ran once, with the packages installed from npm; CI replays only the current +build. + +## Changing the corpus + +Never edit, re-encode or reformat a document: Biome and TypeScript skip this directory and Git does +not normalize it. A new contract version adds entries produced by the version that introduces it. +A test failure here means the current build no longer replays a historical document as it was +decided; fix the build, not the corpus. diff --git a/tests/replay-corpus/corpus.json b/tests/replay-corpus/corpus.json new file mode 100644 index 0000000..a4874b1 --- /dev/null +++ b/tests/replay-corpus/corpus.json @@ -0,0 +1,516 @@ +{ + "schemaVersion": "1.0.0", + "entries": [ + { + "id": "evidence-v1", + "kind": "evidence", + "schemaVersion": "1.0.0", + "producer": "assertledger@1.0.0", + "document": "tests/replay-corpus/documents/evidence-v1.manifest.json", + "sha256": "sha256:2020a9371a2c8b07cb0af1f8c344a94a3bce8975bc753a2f4b6b9e6de6a4c935", + "input": "tests/replay-corpus/inputs/evidence-v1", + "decision": "VERIFIED", + "decisionDigest": "sha256:55cbeb05e93f1ab4aeb25aa665424e9d9649d3a9e16929bb9217ca8326ab5945", + "artifactDigest": "sha256:c8dcd35f527478978a71fab22640e4752b945896bfe8429c1ddeb31627550abd", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.0.0", + "1.1.0", + "1.2.0", + "1.5.0", + "2.1.0" + ] + }, + { + "id": "evidence-v2", + "kind": "evidence", + "schemaVersion": "2.0.0", + "producer": "assertledger@1.1.0", + "document": "tests/replay-corpus/documents/evidence-v2.manifest.json", + "sha256": "sha256:41b037c363e7b5ffa174428ffa2cc32f76a35f7602c8b89ceead58924e3d2ed0", + "input": "tests/replay-corpus/inputs/evidence-v2", + "decision": "VERIFIED", + "decisionDigest": "sha256:45ede1c32965769529cedba0c100ec01d9f3dc3f33c0edbcfbbc05689515d687", + "artifactDigest": "sha256:c47a80ec7fcdc3a42326ef33d6a0e8f8cf46f24ff7804b3ca24ead90fcf1a6fb", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.1.0", + "1.2.0", + "1.5.0", + "2.1.0" + ] + }, + { + "id": "evidence-v3", + "kind": "evidence", + "schemaVersion": "3.0.0", + "producer": "assertledger@1.2.0", + "document": "tests/replay-corpus/documents/evidence-v3.manifest.json", + "sha256": "sha256:86163df9c7bb0660118af1bf0a15821fa224459a42919a519125f5b1ba825a54", + "input": "tests/replay-corpus/inputs/evidence-v3", + "decision": "VERIFIED", + "decisionDigest": "sha256:28395039370d3d217191bae9ad37071da44932aab01c1f49a41aef52f311dcc5", + "artifactDigest": "sha256:754cbd939d6d696971f60894b2380f62e78a209531ca91e6b91585c135e49b5f", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.2.0", + "1.5.0", + "2.1.0" + ] + }, + { + "id": "evidence-v4", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@1.5.0", + "document": "tests/replay-corpus/documents/evidence-v4.manifest.json", + "sha256": "sha256:f8716b74cfcffd00ef751e436d39a1974d947d221a9e16fb69714a4cc07da48c", + "input": "tests/replay-corpus/inputs/evidence-v4", + "decision": "VERIFIED", + "decisionDigest": "sha256:67f1f61e2c942377b1ba444294a7c1da0e6a1c17c436b78f31dc6a698e289d6f", + "artifactDigest": "sha256:a1bfd2732f29c6b07b050dd80fc374890d562ed1ea68474d9e196574d252b944", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0" + ] + }, + { + "id": "evidence-v5", + "kind": "evidence", + "schemaVersion": "5.0.0", + "producer": "assertledger@2.1.0", + "document": "tests/replay-corpus/documents/evidence-v5.manifest.json", + "sha256": "sha256:9360e2c8441f86be7a464eb6a74087d00c587939074ad3fb1a15332334f14d66", + "input": "tests/replay-corpus/inputs/evidence-v5", + "decision": "VERIFIED", + "decisionDigest": "sha256:a7a0f658f161bd507e3209d49c6f23249a09c8d8425cbbd4cdfb4fc6fb1c6330", + "artifactDigest": "sha256:f4694b7c59a97b37982366bf068f8949c5c2f1b714515a643db1e0b23cd65b23", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "2.1.0" + ] + }, + { + "id": "qualification-1.5.0-node-test", + "kind": "qualification", + "schemaVersion": "1.0.0", + "producer": "assertledger@1.5.0", + "document": "tests/replay-corpus/documents/qualification-1.5.0-node-test.receipt.json", + "sha256": "sha256:e763fc107693f24f386eb2e6300976420c3bf13e0f8e766eae6fbe76e4f91acd", + "input": "tests/replay-corpus/inputs/qualification-1.5.0-node-test", + "decision": "QUALIFIED", + "decisionDigest": "sha256:f78849ed84082ec4da7086ed0e7a852c86eb64a6f250e2a9a302afd41c57490b", + "artifactDigest": "sha256:e336a92c28fe78831e501b40d521822410f7038fbfeb26740561fc43726e4ea9", + "command": "qualification-replay", + "replay": { + "valid": true, + "integrityValid": true, + "semanticsValid": true, + "domainValid": true, + "reexecuted": false, + "producerAuthenticated": false + }, + "replayedBy": [ + "1.5.0", + "2.1.0" + ] + }, + { + "id": "qualification-1.5.0-propagation", + "kind": "qualification", + "schemaVersion": "1.0.0", + "producer": "assertledger@1.5.0", + "document": "tests/replay-corpus/documents/qualification-1.5.0-propagation.receipt.json", + "sha256": "sha256:5d815004a1df63df192b731349753701ff49d4f26e1e10993801d2e2a8291a57", + "input": "tests/replay-corpus/inputs/qualification-1.5.0-propagation", + "decision": "QUALIFIED", + "decisionDigest": "sha256:75e0fff422b61aed00775d2b2dfa7b33752e7e84a30b4ca8fb94619d592f5935", + "artifactDigest": "sha256:7d72a9ea6443ba3a9b809d6fe4628f225dc8b61850dc423bbdf09dd601bbaf02", + "command": "qualification-replay", + "replay": { + "valid": true, + "integrityValid": true, + "semanticsValid": true, + "domainValid": true, + "reexecuted": false, + "producerAuthenticated": false + }, + "replayedBy": [ + "1.5.0", + "2.1.0" + ] + }, + { + "id": "qualification-2.1.0-node-test", + "kind": "qualification", + "schemaVersion": "1.0.0", + "producer": "assertledger@2.1.0", + "document": "tests/replay-corpus/documents/qualification-2.1.0-node-test.receipt.json", + "sha256": "sha256:87968cac0ab7c6d50b61311fd8a4d877c192c8ef7af7bf237ddde8e75bebf5a2", + "input": "tests/replay-corpus/inputs/qualification-2.1.0-node-test", + "decision": "QUALIFIED", + "decisionDigest": "sha256:f78849ed84082ec4da7086ed0e7a852c86eb64a6f250e2a9a302afd41c57490b", + "artifactDigest": "sha256:fce88cdbdd2a97e19ec81f8774d928adfe6e9592cb5ccdf8fcb3906834afdb26", + "command": "qualification-replay", + "replay": { + "valid": true, + "integrityValid": true, + "semanticsValid": true, + "domainValid": true, + "reexecuted": false, + "producerAuthenticated": false + }, + "replayedBy": [ + "2.1.0" + ] + }, + { + "id": "qualification-2.1.0-propagation", + "kind": "qualification", + "schemaVersion": "1.0.0", + "producer": "assertledger@2.1.0", + "document": "tests/replay-corpus/documents/qualification-2.1.0-propagation.receipt.json", + "sha256": "sha256:e78ea5eff0fe3f65788b5ff8191649359551e43a12dae33f0d03f998bbffbdef", + "input": "tests/replay-corpus/inputs/qualification-2.1.0-propagation", + "decision": "QUALIFIED", + "decisionDigest": "sha256:75e0fff422b61aed00775d2b2dfa7b33752e7e84a30b4ca8fb94619d592f5935", + "artifactDigest": "sha256:277cb7080167b892fecb68501c1bcd8e22af62482dca71d525e52f509bd669bb", + "command": "qualification-replay", + "replay": { + "valid": true, + "integrityValid": true, + "semanticsValid": true, + "domainValid": true, + "reexecuted": false, + "producerAuthenticated": false + }, + "replayedBy": [ + "2.1.0" + ] + }, + { + "id": "evidence-v4-container-matched-before-1", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@2.0.2 with the trace-only patch of baseline-producer.json", + "document": "benchmarks/container-runtime/2026-10-11/matched-before-1.manifest.json.gz", + "sha256": "sha256:b33cae2cfe0759968136342871deb7d1bb4d3af1167aa2dff76216e0865b7b5d", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:0720ec06e5fda010bbaa24a824935f72089af30a2e6fed8ccf69b86f4e5b0037", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-matched-before-2", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@2.0.2 with the trace-only patch of baseline-producer.json", + "document": "benchmarks/container-runtime/2026-10-11/matched-before-2.manifest.json.gz", + "sha256": "sha256:9217fb31e0668374112405ea5f8ca03fe1c6639557f360ac8df729d479e445a8", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:896eea182aa327ce4913cf1f35b922b775ac007406fa15ad35f54d8f40e0130d", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-matched-before-3", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@2.0.2 with the trace-only patch of baseline-producer.json", + "document": "benchmarks/container-runtime/2026-10-11/matched-before-3.manifest.json.gz", + "sha256": "sha256:759d8c54e3939a4bd7ad4864acfba04d2660e8eb1a943207cbec41c57c28fcbe", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:5a1f4b9c9a15fdc44a472f5cca1c8872fa3fe8aaaa6a70881958f3f7a2e67f88", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-matched-after-1", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "the candidate build measured in benchmarks/container-runtime/2026-10-11", + "document": "benchmarks/container-runtime/2026-10-11/matched-after-1.manifest.json.gz", + "sha256": "sha256:fd7c291a5f6ef089f444ed5f26b13f9c2abe90cc8bb46a457e25422e6cb79c61", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:fed2fbb5df0f131428afee69ca3e005292bc244e2f04bc78cc6201c33d1611c1", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-matched-after-2", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "the candidate build measured in benchmarks/container-runtime/2026-10-11", + "document": "benchmarks/container-runtime/2026-10-11/matched-after-2.manifest.json.gz", + "sha256": "sha256:e2c299c55788c90b2b092013a165ffeb5ec2b6d28b907bc2ac917a5d3d6f45f0", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:3ea1672197dc9db9927c8507bf49d5f4e1483e1fb4d0165f22585d248301e5e9", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-matched-after-3", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "the candidate build measured in benchmarks/container-runtime/2026-10-11", + "document": "benchmarks/container-runtime/2026-10-11/matched-after-3.manifest.json.gz", + "sha256": "sha256:977ca542aad5cd04cd052a064952eb19308c6274a6906aede08273f6a6f14670", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:c94329171a87bf61fb65c692d3553075b5f0fdcc66fa905f4fe1fae5ac1d73f2", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-rigorous-before-1", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@2.0.2 with the trace-only patch of baseline-producer.json", + "document": "benchmarks/container-runtime/2026-10-11/rigorous-before-1.manifest.json.gz", + "sha256": "sha256:aa4f5290bea19731a166f631c3ef60564faf53e1b618ff858ed1749970c77d1f", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:74e497d1455d20ec8b2bbfc1e780426b3845163f44e885980e7553933c51982c", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-rigorous-before-2", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@2.0.2 with the trace-only patch of baseline-producer.json", + "document": "benchmarks/container-runtime/2026-10-11/rigorous-before-2.manifest.json.gz", + "sha256": "sha256:1e8b6884c789d9eb6bf2e0b1ef0c8c0dd079b690c01bda3f7da9b24f3eded085", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:648cd1555fb718bd96797524f02de4e6daf9f906fe7efbaf2dc1bf862a0ae3fe", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-rigorous-before-3", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "assertledger@2.0.2 with the trace-only patch of baseline-producer.json", + "document": "benchmarks/container-runtime/2026-10-11/rigorous-before-3.manifest.json.gz", + "sha256": "sha256:1dc7b6c34c7a02c439fcde923a7fcfc89bb40958aee1e7af23577d46c7df1aa7", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:3f00cc27f2af6bc9212ce5e661375eab38ae821e4881b4b40524b7160a3042f9", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-rigorous-after-1", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "the candidate build measured in benchmarks/container-runtime/2026-10-11", + "document": "benchmarks/container-runtime/2026-10-11/rigorous-after-1.manifest.json.gz", + "sha256": "sha256:195b0f653ad95c3d9ac3552a9563bba6d698a59a805c46114233904d53c9bf2a", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:a5852c85a663ebee6455dd4594fc1a408b862b0fd91c2028814f837bc02688ca", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-rigorous-after-2", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "the candidate build measured in benchmarks/container-runtime/2026-10-11", + "document": "benchmarks/container-runtime/2026-10-11/rigorous-after-2.manifest.json.gz", + "sha256": "sha256:ceb5a8e508abaf2cf1f56eb878f2b518aedc13b0c76ea6e08ee02c3fdee68e54", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:f7e88d5a8b94da7b70b50879a7165baae9b9101f2a63524a77ed0c7b8180ee52", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + }, + { + "id": "evidence-v4-container-rigorous-after-3", + "kind": "evidence", + "schemaVersion": "4.0.0", + "producer": "the candidate build measured in benchmarks/container-runtime/2026-10-11", + "document": "benchmarks/container-runtime/2026-10-11/rigorous-after-3.manifest.json.gz", + "sha256": "sha256:7e44534d3f12c77028be9a26d6fc7d6d5ae9de4be4d6e0a2f1c47d1b92ddddbe", + "decision": "VERIFIED", + "decisionDigest": "sha256:cced6d46aeb465e69494a67cb422bebdfe09fd6485af9796143705dd2ae8a482", + "artifactDigest": "sha256:bd59ff3925a37c271e03200cab4adad559dbd18b78e94f8e184b750ed40feccd", + "command": "replay", + "replay": { + "valid": true, + "schemaValid": true, + "decisionDigestValid": true, + "artifactDigestValid": true, + "decisionSemanticsValid": true + }, + "replayedBy": [ + "1.5.0", + "2.1.0", + "recorded" + ] + } + ] +} diff --git a/tests/replay-corpus/documents/evidence-v1.manifest.json b/tests/replay-corpus/documents/evidence-v1.manifest.json new file mode 100644 index 0000000..b024058 --- /dev/null +++ b/tests/replay-corpus/documents/evidence-v1.manifest.json @@ -0,0 +1 @@ +{"schemaVersion":"1.0.0","repositoryDigest":"sha256:c2bc0c14f30b015305d49ec47cf02c89fe7d9f679829ee34ce66d6ae1789b9db","evidenceContext":{"engine":{"name":"testforge","version":"0.1.0"},"adapter":{"name":"node-test","version":"22.23.2","configuration":{"kind":"node-test","profile":{"profileId":"node-test","profileVersion":"1.0.0","official":true,"capabilities":{"detectsCollectionFailure":false,"detectsCompileFailure":false,"attributesPerAssertionFailure":true},"reporterDigest":"sha256:1ac7b13724d344a9dad8221e38e7c75eecda20344617c7a61083dff6eda0f2a1"},"requestedExecutable":"node","resolvedExecutable":"C:\\ProgramData\\nvm\\v22.23.2\\node.exe","executableDigest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4","nodeVersion":"22.23.2","runtimePreflight":{"protocolVersion":"1.0.0","assertionProbe":{"outcome":"ASSERTION_FAILURE","attributed":true,"candidateTestsDiscovered":1,"testsDiscovered":2,"candidateFailureCount":1,"nonCandidateFailureCount":0,"candidateSyntaxFailureCount":0,"candidateFailuresAllAssertions":true,"processExitedNonZero":true},"genericThrowProbe":{"outcome":"PROCESS_CRASH","attributed":false,"candidateTestsDiscovered":1,"testsDiscovered":2,"candidateFailureCount":1,"nonCandidateFailureCount":0,"candidateSyntaxFailureCount":0,"candidateFailuresAllAssertions":false,"processExitedNonZero":true}},"arguments":["--test","--","tests/base.test.js"]}},"execution":{"isolation":"UNSANDBOXED","environmentAllowlist":["PATH","SystemRoot","TEMP","TMP","WINDIR"],"budgets":{"maximumCandidates":4,"maximumWorlds":4,"maximumExecutions":24,"maximumRepositoryFiles":10000,"maximumRepositoryBytes":100000000,"maximumWorldOverlayBytes":1000000,"maximumCandidateBytes":16384,"maximumTotalCandidateBytes":65536,"timeoutMsPerExecution":5000,"maximumOutputBytes":65536},"candidateRoots":["tests/candidates"]},"worlds":[{"id":"neutral-bitwise-equivalence","provenance":"example:reviewed-equivalent","digest":"sha256:ae1fd036ffbd99521321564fc68e0b1081c3be75e663cf04fd7855f11c0a03bf"},{"id":"reference","provenance":"example:correct-implementation","digest":"sha256:9d778a13de5848add9b5272caca7e9ec4d791b450fb275201c5c1467539cf532"},{"id":"target-parity-inversion","provenance":"example:known-fault","digest":"sha256:f9e2b0224a307d73d6c19239222e8844b6bf59961230084853661e3659d3306d"}]},"policy":{"policyVersion":"1.0.0","requiredAttempts":2,"minimumTargetWeightPermille":1000,"maximumSelectedCandidates":1,"acceptedTargetOutcomes":["ASSERTION_FAILURE"]},"worlds":[{"id":"neutral-bitwise-equivalence","kind":"NEUTRAL","required":true,"weight":0},{"id":"reference","kind":"REFERENCE","required":true,"weight":0},{"id":"target-parity-inversion","kind":"TARGET","required":true,"weight":1}],"candidates":[{"id":"strong","digest":"sha256:1a4d873324e1ab33fbe43945d55a65d5b8cb2d18e643ce615c82320ce8bc8a61","sizeBytes":176,"status":"ELIGIBLE","killedTargetIds":["target-parity-inversion"],"targetWeightKilled":1,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["strong:reference:1","strong:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"PASSED","evidenceRunIds":["strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]}],"reasonCodes":["POLICY_SATISFIED"]},{"id":"weak","digest":"sha256:cf01079f493e1bf31869db244cf90330cad233c2c749c2e3bfa8e832a9c254c4","sizeBytes":194,"status":"WEAK_ORACLE","killedTargetIds":[],"targetWeightKilled":0,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["weak:reference:1","weak:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"FAILED","evidenceRunIds":["weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":["TARGET_STRENGTH_INSUFFICIENT"]}],"reasonCodes":["TARGET_STRENGTH_INSUFFICIENT"]}],"observations":[{"runId":"control:neutral-bitwise-equivalence:1","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":152,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:neutral-bitwise-equivalence:2","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":148,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:1","candidateId":null,"worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":167,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:2","candidateId":null,"worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":155,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:1","candidateId":null,"worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":180,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:2","candidateId":null,"worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":159,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:neutral-bitwise-equivalence:1","candidateId":"strong","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":167,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:neutral-bitwise-equivalence:2","candidateId":"strong","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":172,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:reference:1","candidateId":"strong","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":166,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:reference:2","candidateId":"strong","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":190,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:target-parity-inversion:1","candidateId":"strong","worldId":"target-parity-inversion","attempt":1,"outcome":"ASSERTION_FAILURE","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":191,"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:target-parity-inversion:2","candidateId":"strong","worldId":"target-parity-inversion","attempt":2,"outcome":"ASSERTION_FAILURE","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":178,"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:neutral-bitwise-equivalence:1","candidateId":"weak","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":172,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:neutral-bitwise-equivalence:2","candidateId":"weak","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":181,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:reference:1","candidateId":"weak","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":163,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:reference:2","candidateId":"weak","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":157,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:target-parity-inversion:1","candidateId":"weak","worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":173,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:target-parity-inversion:2","candidateId":"weak","worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":170,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"decision":{"status":"VERIFIED","selectedCandidateIds":["strong"],"reasonCodes":["POLICY_SATISFIED"]},"decisionDigest":"sha256:55cbeb05e93f1ab4aeb25aa665424e9d9649d3a9e16929bb9217ca8326ab5945","artifactDigest":"sha256:c8dcd35f527478978a71fab22640e4752b945896bfe8429c1ddeb31627550abd","adapter":{"kind":"node-test"},"isolation":{"kind":"trusted-local","level":"UNSANDBOXED","acknowledgedUnsafeExecution":true},"limitations":["UNSANDBOXED trusted-local execution cannot safely contain hostile candidate code.","Process-tree termination is best effort and depends on host operating-system facilities."]} diff --git a/tests/replay-corpus/documents/evidence-v2.manifest.json b/tests/replay-corpus/documents/evidence-v2.manifest.json new file mode 100644 index 0000000..d88f58a --- /dev/null +++ b/tests/replay-corpus/documents/evidence-v2.manifest.json @@ -0,0 +1 @@ +{"schemaVersion":"2.0.0","repositoryDigest":"sha256:c2bc0c14f30b015305d49ec47cf02c89fe7d9f679829ee34ce66d6ae1789b9db","evidenceContext":{"engine":{"name":"testforge","version":"0.1.0"},"adapter":{"name":"node-test","version":"22.23.2","configuration":{"kind":"node-test","profile":{"profileId":"node-test","profileVersion":"1.0.0","official":true,"capabilities":{"detectsCollectionFailure":false,"detectsCompileFailure":false,"attributesPerAssertionFailure":true},"reporterDigest":"sha256:1ac7b13724d344a9dad8221e38e7c75eecda20344617c7a61083dff6eda0f2a1"},"requestedExecutable":"node","resolvedExecutable":"C:\\ProgramData\\nvm\\v22.23.2\\node.exe","executableDigest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4","nodeVersion":"22.23.2","runtimePreflight":{"protocolVersion":"1.0.0","assertionProbe":{"outcome":"ASSERTION_FAILURE","attributed":true,"candidateTestsDiscovered":1,"testsDiscovered":2,"candidateFailureCount":1,"nonCandidateFailureCount":0,"candidateSyntaxFailureCount":0,"candidateFailuresAllAssertions":true,"processExitedNonZero":true},"genericThrowProbe":{"outcome":"PROCESS_CRASH","attributed":false,"candidateTestsDiscovered":1,"testsDiscovered":2,"candidateFailureCount":1,"nonCandidateFailureCount":0,"candidateSyntaxFailureCount":0,"candidateFailuresAllAssertions":false,"processExitedNonZero":true}},"arguments":["--test","--","tests/base.test.js"]}},"execution":{"isolation":"UNSANDBOXED","environmentAllowlist":["PATH","SystemRoot","TEMP","TMP","WINDIR"],"budgets":{"maximumCandidates":4,"maximumWorlds":4,"maximumExecutions":24,"maximumRepositoryFiles":10000,"maximumRepositoryBytes":100000000,"maximumWorldOverlayBytes":1000000,"maximumCandidateBytes":16384,"maximumTotalCandidateBytes":65536,"timeoutMsPerExecution":5000,"maximumOutputBytes":65536},"candidateRoots":["tests/candidates"],"backend":{"kind":"trusted-local","level":"UNSANDBOXED"}},"worlds":[{"id":"neutral-bitwise-equivalence","provenance":"example:reviewed-equivalent","digest":"sha256:ae1fd036ffbd99521321564fc68e0b1081c3be75e663cf04fd7855f11c0a03bf"},{"id":"reference","provenance":"example:correct-implementation","digest":"sha256:9d778a13de5848add9b5272caca7e9ec4d791b450fb275201c5c1467539cf532"},{"id":"target-parity-inversion","provenance":"example:known-fault","digest":"sha256:f9e2b0224a307d73d6c19239222e8844b6bf59961230084853661e3659d3306d"}]},"policy":{"policyVersion":"1.0.0","requiredAttempts":2,"minimumTargetWeightPermille":1000,"maximumSelectedCandidates":1,"acceptedTargetOutcomes":["ASSERTION_FAILURE"]},"worlds":[{"id":"neutral-bitwise-equivalence","kind":"NEUTRAL","required":true,"weight":0},{"id":"reference","kind":"REFERENCE","required":true,"weight":0},{"id":"target-parity-inversion","kind":"TARGET","required":true,"weight":1}],"candidates":[{"id":"strong","digest":"sha256:1a4d873324e1ab33fbe43945d55a65d5b8cb2d18e643ce615c82320ce8bc8a61","sizeBytes":176,"status":"ELIGIBLE","killedTargetIds":["target-parity-inversion"],"targetWeightKilled":1,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["strong:reference:1","strong:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"PASSED","evidenceRunIds":["strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]}],"reasonCodes":["POLICY_SATISFIED"]},{"id":"weak","digest":"sha256:cf01079f493e1bf31869db244cf90330cad233c2c749c2e3bfa8e832a9c254c4","sizeBytes":194,"status":"WEAK_ORACLE","killedTargetIds":[],"targetWeightKilled":0,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["weak:reference:1","weak:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"FAILED","evidenceRunIds":["weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":["TARGET_STRENGTH_INSUFFICIENT"]}],"reasonCodes":["TARGET_STRENGTH_INSUFFICIENT"]}],"observations":[{"runId":"control:neutral-bitwise-equivalence:1","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":161,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:neutral-bitwise-equivalence:2","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":151,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:1","candidateId":null,"worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":146,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:2","candidateId":null,"worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":141,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:1","candidateId":null,"worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":144,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:2","candidateId":null,"worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":143,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:neutral-bitwise-equivalence:1","candidateId":"strong","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":170,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:neutral-bitwise-equivalence:2","candidateId":"strong","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":159,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:reference:1","candidateId":"strong","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":173,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:reference:2","candidateId":"strong","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":172,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:target-parity-inversion:1","candidateId":"strong","worldId":"target-parity-inversion","attempt":1,"outcome":"ASSERTION_FAILURE","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":163,"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:target-parity-inversion:2","candidateId":"strong","worldId":"target-parity-inversion","attempt":2,"outcome":"ASSERTION_FAILURE","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":166,"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:neutral-bitwise-equivalence:1","candidateId":"weak","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":159,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:neutral-bitwise-equivalence:2","candidateId":"weak","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":149,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:reference:1","candidateId":"weak","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":169,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:reference:2","candidateId":"weak","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":157,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:target-parity-inversion:1","candidateId":"weak","worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":157,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:target-parity-inversion:2","candidateId":"weak","worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":155,"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"decision":{"status":"VERIFIED","selectedCandidateIds":["strong"],"reasonCodes":["POLICY_SATISFIED"]},"decisionDigest":"sha256:45ede1c32965769529cedba0c100ec01d9f3dc3f33c0edbcfbbc05689515d687","artifactDigest":"sha256:c47a80ec7fcdc3a42326ef33d6a0e8f8cf46f24ff7804b3ca24ead90fcf1a6fb","adapter":{"kind":"node-test"},"isolation":{"kind":"trusted-local","level":"UNSANDBOXED","acknowledgedUnsafeExecution":true},"limitations":["UNSANDBOXED trusted-local execution cannot safely contain hostile candidate code.","Process-tree termination is best effort and depends on host operating-system facilities."]} diff --git a/tests/replay-corpus/documents/evidence-v3.manifest.json b/tests/replay-corpus/documents/evidence-v3.manifest.json new file mode 100644 index 0000000..14c28c7 --- /dev/null +++ b/tests/replay-corpus/documents/evidence-v3.manifest.json @@ -0,0 +1 @@ +{"schemaVersion":"3.0.0","repositoryDigest":"sha256:a504ca2134e51b116623c0c5f768cb1c1cbf68faef5bef0940ce2022e75d07ec","evidenceContext":{"engine":{"name":"testforge","version":"0.1.0"},"adapter":{"name":"bun-test","version":"1.4.2","configuration":{"kind":"bun-test","profile":{"profileId":"bun-test","profileVersion":"1.2.0","official":true,"bunVersion":"1.4.2","bunRevision":"744846f844374847c902b5e7fd59b4342a51ef99","capabilities":{"assertionSource":"assertledger-assertSame-issued-error","attributesPerAssertionFailure":true,"detectsCollectionFailure":false,"detectsCompileFailure":false,"reporterTransport":"signed-preload-pipe+junit","supportsContainer":false}},"requestedExecutable":"bun","resolvedExecutable":"C:\\Users\\Hokli\\scoop\\apps\\bun\\1.4.2\\bun.exe","executableDigest":"sha256:15277c59ccd6c6c20f8dc9716c2b59c1776320d606b6a8658f70be8799519ca4","bunVersion":"1.4.2","bunRevision":"744846f844374847c902b5e7fd59b4342a51ef99","driverDigest":"sha256:eab0affd561fae2596b48a9f8503d37588d2014682dc88dbac1a8bdde7b0f5cd","helperDigest":"sha256:33cf4653edceb2bde6d6d979d4149b84804c1b5bda12c2b00851d401130ea746","preloadDigest":"sha256:aa23ddc71ef9e7499042f4536a2fe0927cdaf981589640d8973874e1729c4500","runtimePreflight":{"protocolVersion":"1.0.0","probes":[{"name":"assertion","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":2,"candidateTestsDiscovered":1},{"name":"generic-throw","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":1},{"name":"caught-assertion","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":1},{"name":"replayed-assertion","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":1},{"name":"replayed-row","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":3,"candidateTestsDiscovered":2},{"name":"async-cross-test","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":3,"candidateTestsDiscovered":2},{"name":"multi-assertion-rows","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":3,"candidateTestsDiscovered":2},{"name":"operand-throw","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":1},{"name":"native-expect","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":1},{"name":"hook-failure","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0}]},"arguments":["test","--max-concurrency=1","--retry=0","--preload","./node_modules/assertledger/preload.mjs","--reporter=junit","--reporter-outfile=","tests/base.test.ts"]}},"execution":{"isolation":"UNSANDBOXED","environmentAllowlist":["PATH","SystemRoot","TEMP","TMP","WINDIR"],"budgets":{"maximumCandidates":4,"maximumWorlds":4,"maximumExecutions":24,"maximumRepositoryFiles":10000,"maximumRepositoryBytes":100000000,"maximumWorldOverlayBytes":1000000,"maximumCandidateBytes":16384,"maximumTotalCandidateBytes":65536,"timeoutMsPerExecution":10000,"maximumOutputBytes":65536},"candidateRoots":["tests/candidates"],"backend":{"kind":"trusted-local","level":"UNSANDBOXED"}},"worlds":[{"id":"neutral-bitwise-equivalence","provenance":"corpus:reviewed-equivalent","digest":"sha256:9e927e1f01bf385103eda72ed1bce77d5c344fe835dcd49dcfa20ca5bdf843ac"},{"id":"reference","provenance":"corpus:correct","digest":"sha256:c43382f86fd96a4ae1b9224e9b2297e8d8fe6a887400298d40a8510488cf0f02"},{"id":"target-parity-inversion","provenance":"corpus:known-fault","digest":"sha256:70bc1229564b8d7e41aa306f606dd5c61c2a4c87b728d5bb63b9625166efcd0a"}]},"policy":{"policyVersion":"1.0.0","requiredAttempts":2,"minimumTargetWeightPermille":1000,"maximumSelectedCandidates":1,"acceptedTargetOutcomes":["ASSERTION_FAILURE"]},"worlds":[{"id":"neutral-bitwise-equivalence","kind":"NEUTRAL","required":true,"weight":0},{"id":"reference","kind":"REFERENCE","required":true,"weight":0},{"id":"target-parity-inversion","kind":"TARGET","required":true,"weight":1}],"candidates":[{"id":"strong","digest":"sha256:94aac7e9fda9b7fc4572fd6fc839e28db5926ec8dc658aa795eab22030d159cd","sizeBytes":185,"status":"ELIGIBLE","killedTargetIds":["target-parity-inversion"],"targetWeightKilled":1,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2","strong:reference:1","strong:reference:2","strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["strong:reference:1","strong:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["strong:neutral-bitwise-equivalence:1","strong:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"PASSED","evidenceRunIds":["strong:target-parity-inversion:1","strong:target-parity-inversion:2"],"reasonCodes":[]}],"reasonCodes":["POLICY_SATISFIED"]},{"id":"weak","digest":"sha256:09190a6a559b597d4078674dbda738e43b9cfc89e58cf0092fe68ace1022e9a7","sizeBytes":203,"status":"WEAK_ORACLE","killedTargetIds":[],"targetWeightKilled":0,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2","weak:reference:1","weak:reference:2","weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["weak:reference:1","weak:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["weak:neutral-bitwise-equivalence:1","weak:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"FAILED","evidenceRunIds":["weak:target-parity-inversion:1","weak:target-parity-inversion:2"],"reasonCodes":["TARGET_STRENGTH_INSUFFICIENT"]}],"reasonCodes":["TARGET_STRENGTH_INSUFFICIENT"]}],"observations":[{"runId":"control:neutral-bitwise-equivalence:1","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":149,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:neutral-bitwise-equivalence:2","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":161,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:1","candidateId":null,"worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":163,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:2","candidateId":null,"worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":163,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:1","candidateId":null,"worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":153,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:2","candidateId":null,"worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":149,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:neutral-bitwise-equivalence:1","candidateId":"strong","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":155,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:neutral-bitwise-equivalence:2","candidateId":"strong","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":150,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:reference:1","candidateId":"strong","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":161,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:reference:2","candidateId":"strong","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":153,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:target-parity-inversion:1","candidateId":"strong","worldId":"target-parity-inversion","attempt":1,"outcome":"ASSERTION_FAILURE","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":154,"exitCode":1,"stdoutDigest":"sha256:f172aad3d46d3544adcad89e4bfef318f8c7edaaa006349784e1ed3178ed08ae","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"strong:target-parity-inversion:2","candidateId":"strong","worldId":"target-parity-inversion","attempt":2,"outcome":"ASSERTION_FAILURE","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":165,"exitCode":1,"stdoutDigest":"sha256:f172aad3d46d3544adcad89e4bfef318f8c7edaaa006349784e1ed3178ed08ae","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:neutral-bitwise-equivalence:1","candidateId":"weak","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":156,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:neutral-bitwise-equivalence:2","candidateId":"weak","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":155,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:reference:1","candidateId":"weak","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":159,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:reference:2","candidateId":"weak","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":152,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:target-parity-inversion:1","candidateId":"weak","worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":156,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"weak:target-parity-inversion:2","candidateId":"weak","worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":2,"candidateTestsDiscovered":1,"attributed":true,"durationMs":159,"exitCode":0,"stdoutDigest":"sha256:04f5f819884d24b15642e708b0733ed6024a593cab0332891ce6c5f23391ed90","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"decision":{"status":"VERIFIED","selectedCandidateIds":["strong"],"reasonCodes":["POLICY_SATISFIED"]},"decisionDigest":"sha256:28395039370d3d217191bae9ad37071da44932aab01c1f49a41aef52f311dcc5","artifactDigest":"sha256:754cbd939d6d696971f60894b2380f62e78a209531ca91e6b91585c135e49b5f","adapter":{"kind":"bun-test"},"isolation":{"kind":"trusted-local","level":"UNSANDBOXED","acknowledgedUnsafeExecution":true},"limitations":["UNSANDBOXED trusted-local execution cannot safely contain hostile candidate code.","Process-tree termination is best effort and depends on host operating-system facilities.","Only failures from the AssertLedger-owned assertSame helper are admitted as Bun assertions; native bun:test expect failures are not assertion evidence.","Bun callback instrumentation determines assertion ownership; JUnit counts only check completeness and never classify assertions."]} diff --git a/tests/replay-corpus/documents/evidence-v4.manifest.json b/tests/replay-corpus/documents/evidence-v4.manifest.json new file mode 100644 index 0000000..cd2526f --- /dev/null +++ b/tests/replay-corpus/documents/evidence-v4.manifest.json @@ -0,0 +1 @@ +{"schemaVersion":"4.0.0","repositoryDigest":"sha256:585d44dcdf5abe6ce41d722796a2e0ef241b3f2e29229f1cdd95d066694dbd29","evidenceContext":{"engine":{"name":"testforge","version":"0.1.0"},"adapter":{"name":"bun-test-designated","version":"1.4.2","configuration":{"kind":"bun-test-designated","profile":{"profileId":"bun-test-designated","profileVersion":"1.0.0","official":true,"bunVersion":"1.4.2","bunRevision":"744846f844374847c902b5e7fd59b4342a51ef99","capabilities":{"assertionSource":"bun-builtin-matcher-or-assertSame-issued-error","attributesPerAssertionFailure":true,"controlMode":"load-designated-files-without-running-tests","detectsCollectionFailure":false,"detectsCompileFailure":false,"expectedFailureLine":"first-message-line-exact","reporterTransport":"signed-preload-pipe+junit","supportsContainer":true}},"requestedExecutable":"bun","resolvedExecutable":"C:\\Users\\Hokli\\scoop\\apps\\bun\\1.4.2\\bun.exe","executableDigest":"sha256:15277c59ccd6c6c20f8dc9716c2b59c1776320d606b6a8658f70be8799519ca4","bunVersion":"1.4.2","bunRevision":"744846f844374847c902b5e7fd59b4342a51ef99","driverDigest":"sha256:eb8a4ac86f830b9c8aff5e3ef2e6a5e659c3b6d2d7257a27c966dd787fec347d","helperDigest":"sha256:33cf4653edceb2bde6d6d979d4149b84804c1b5bda12c2b00851d401130ea746","preloadDigest":"sha256:8b8d2cbb11e7c1867e618274cc07cec2afea891343d603b48683fefe72d19a38","runtimePreflight":{"protocolVersion":"1.0.0","probes":[{"name":"expect-assertion","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"expected-line","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"other-line","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"assert-same","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"async-matcher","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"forged-message","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"caught-expect","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"replayed-expect","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"matcher-usage","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"custom-matcher","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"passing","outcome":"PASS","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"absent","outcome":"NO_TEST_DISCOVERED","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"hook-failure","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"late-after-timeout","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"load","outcome":"PASS","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":0},{"name":"load-failure","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0}]},"arguments":["test","--max-concurrency=1","--retry=0","--preload","./node_modules/assertledger/preload.mjs","--reporter=junit","--reporter-outfile=","--test-name-pattern=",""],"testTimeoutMs":null,"designatedTests":[{"candidateId":"designated-parity","file":"tests/parity.test.ts","path":["parity","two is even"],"expectedFailure":null}]}},"execution":{"isolation":"UNSANDBOXED","environmentAllowlist":["PATH","SystemRoot","TEMP","TMP","WINDIR"],"budgets":{"maximumCandidates":1,"maximumWorlds":4,"maximumExecutions":24,"maximumRepositoryFiles":10000,"maximumRepositoryBytes":100000000,"maximumWorldOverlayBytes":1000000,"maximumCandidateBytes":1,"maximumTotalCandidateBytes":1,"timeoutMsPerExecution":10000,"maximumOutputBytes":65536},"candidateRoots":["tests/parity.test.ts"],"backend":{"kind":"trusted-local","level":"UNSANDBOXED"}},"worlds":[{"id":"neutral-bitwise-equivalence","provenance":"corpus:reviewed-equivalent","digest":"sha256:9e927e1f01bf385103eda72ed1bce77d5c344fe835dcd49dcfa20ca5bdf843ac"},{"id":"reference","provenance":"corpus:correct","digest":"sha256:c43382f86fd96a4ae1b9224e9b2297e8d8fe6a887400298d40a8510488cf0f02"},{"id":"target-parity-inversion","provenance":"corpus:known-fault","digest":"sha256:70bc1229564b8d7e41aa306f606dd5c61c2a4c87b728d5bb63b9625166efcd0a"}],"repository":{"includeDependencies":false,"omittedLinks":[],"git":null}},"policy":{"policyVersion":"1.0.0","requiredAttempts":2,"minimumTargetWeightPermille":1000,"maximumSelectedCandidates":1,"acceptedTargetOutcomes":["ASSERTION_FAILURE"]},"worlds":[{"id":"neutral-bitwise-equivalence","kind":"NEUTRAL","required":true,"weight":0},{"id":"reference","kind":"REFERENCE","required":true,"weight":0},{"id":"target-parity-inversion","kind":"TARGET","required":true,"weight":1}],"candidates":[{"id":"designated-parity","digest":"sha256:b376bb5635b113963657cae2c3779c31477e73237e6f931a8e5ad42a9820425c","sizeBytes":198,"status":"ELIGIBLE","killedTargetIds":["target-parity-inversion"],"targetWeightKilled":1,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2","designated-parity:reference:1","designated-parity:reference:2","designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2","designated-parity:reference:1","designated-parity:reference:2","designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2","designated-parity:reference:1","designated-parity:reference:2","designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["designated-parity:reference:1","designated-parity:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"PASSED","evidenceRunIds":["designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]}],"reasonCodes":["POLICY_SATISFIED"]}],"observations":[{"runId":"control:neutral-bitwise-equivalence:1","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":137,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:neutral-bitwise-equivalence:2","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":138,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:1","candidateId":null,"worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":142,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:2","candidateId":null,"worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":139,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:1","candidateId":null,"worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":150,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:2","candidateId":null,"worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":132,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:neutral-bitwise-equivalence:1","candidateId":"designated-parity","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":138,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:neutral-bitwise-equivalence:2","candidateId":"designated-parity","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":143,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:reference:1","candidateId":"designated-parity","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":144,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:reference:2","candidateId":"designated-parity","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":149,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:target-parity-inversion:1","candidateId":"designated-parity","worldId":"target-parity-inversion","attempt":1,"outcome":"ASSERTION_FAILURE","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":147,"exitCode":1,"stdoutDigest":"sha256:bd9f119595e8e221f0ebcbb3b09c49bd229186e081203304312862eddf7ddc25","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:target-parity-inversion:2","candidateId":"designated-parity","worldId":"target-parity-inversion","attempt":2,"outcome":"ASSERTION_FAILURE","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":151,"exitCode":1,"stdoutDigest":"sha256:bd9f119595e8e221f0ebcbb3b09c49bd229186e081203304312862eddf7ddc25","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"decision":{"status":"VERIFIED","selectedCandidateIds":["designated-parity"],"reasonCodes":["POLICY_SATISFIED"]},"decisionDigest":"sha256:67f1f61e2c942377b1ba444294a7c1da0e6a1c17c436b78f31dc6a698e289d6f","artifactDigest":"sha256:a1bfd2732f29c6b07b050dd80fc374890d562ed1ea68474d9e196574d252b944","adapter":{"kind":"bun-test-designated"},"isolation":{"kind":"trusted-local","level":"UNSANDBOXED","acknowledgedUnsafeExecution":true},"limitations":["UNSANDBOXED trusted-local execution cannot safely contain hostile candidate code.","Process-tree termination is best effort and depends on host operating-system facilities.","Only a failure thrown inside the designated test by a bun:test built-in matcher, in Bun's matcher message form, or by assertSame is admitted as an assertion; with a declared expected line, its first message line must equal it. Custom matchers, usage errors and every other error are not assertion evidence.","Controls load every designated test file without running a test: they show that each world evaluates the file and its imports and still registers the designated test, not that other tests pass.","Bun callback instrumentation determines assertion ownership; JUnit counts only check completeness and never classify assertions."]} diff --git a/tests/replay-corpus/documents/evidence-v5.manifest.json b/tests/replay-corpus/documents/evidence-v5.manifest.json new file mode 100644 index 0000000..c7caef7 --- /dev/null +++ b/tests/replay-corpus/documents/evidence-v5.manifest.json @@ -0,0 +1 @@ +{"schemaVersion":"5.0.0","repositoryDigest":"sha256:3abda27f19d7cea6bece7573b362b801f83f0e94c56c31199ceac63826dc9890","evidenceContext":{"engine":{"name":"testforge","version":"0.1.0"},"adapter":{"name":"bun-test-designated","version":"1.4.2","configuration":{"kind":"bun-test-designated","profile":{"profileId":"bun-test-designated","profileVersion":"1.0.0","official":true,"bunVersion":"1.4.2","bunRevision":"744846f844374847c902b5e7fd59b4342a51ef99","capabilities":{"assertionSource":"bun-builtin-matcher-or-assertSame-issued-error","attributesPerAssertionFailure":true,"controlMode":"load-designated-files-without-running-tests","detectsCollectionFailure":false,"detectsCompileFailure":false,"expectedFailureLine":"first-message-line-exact","reporterTransport":"signed-preload-pipe+junit","supportsContainer":true}},"requestedExecutable":"bun","resolvedExecutable":"C:\\Users\\Hokli\\scoop\\apps\\bun\\1.4.2\\bun.exe","executableDigest":"sha256:15277c59ccd6c6c20f8dc9716c2b59c1776320d606b6a8658f70be8799519ca4","bunVersion":"1.4.2","bunRevision":"744846f844374847c902b5e7fd59b4342a51ef99","driverDigest":"sha256:22818b326ea429455b2477300914b27142bac8ecade0b4fbc48e8c6304a8df2a","helperDigest":"sha256:33cf4653edceb2bde6d6d979d4149b84804c1b5bda12c2b00851d401130ea746","preloadDigest":"sha256:9326ccd1f5e56babdd00de5a54b14d5241ca380d3f200c3f8753a2c3d4d1835d","runtimePreflight":{"protocolVersion":"1.0.0","probes":[{"name":"expect-assertion","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"expected-line","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"other-line","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"assert-same","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"async-matcher","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"forged-message","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"caught-expect","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"replayed-expect","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"matcher-usage","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"custom-matcher","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"replaced-built-in","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"preload-replaced-built-in","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"preload-custom-matcher","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"trusted-matcher","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"trusted-matcher-other-name","outcome":"PROCESS_CRASH","attributed":false,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"passing","outcome":"PASS","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1},{"name":"absent","outcome":"NO_TEST_DISCOVERED","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"hook-failure","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"late-after-timeout","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"load","outcome":"PASS","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":0},{"name":"load-failure","outcome":"INFRA_ERROR","attributed":false,"testsDiscovered":0,"candidateTestsDiscovered":0},{"name":"isolated-load","outcome":"PASS","attributed":false,"testsDiscovered":2,"candidateTestsDiscovered":0},{"name":"isolated-run","outcome":"ASSERTION_FAILURE","attributed":true,"testsDiscovered":1,"candidateTestsDiscovered":1}]},"arguments":["test","--config=","--max-concurrency=1","--retry=0","--reporter=junit","--reporter-outfile=","--test-name-pattern=",""],"bunConfiguration":"repository-bunfig-with-assertledger-preload-first","testTimeoutMs":null,"workingDirectory":"repository-root","runDirectory":".","isolate":false,"trustedMatchers":[],"resolvedTrustedMatchers":[],"designatedTests":[{"candidateId":"designated-parity","file":"tests/parity.test.ts","path":["parity","two is even"],"expectedFailure":null}]}},"execution":{"isolation":"UNSANDBOXED","environmentAllowlist":["PATH","SystemRoot","TEMP","TMP","WINDIR"],"budgets":{"maximumCandidates":1,"maximumWorlds":4,"maximumExecutions":24,"maximumRepositoryFiles":10000,"maximumRepositoryBytes":100000000,"maximumWorldOverlayBytes":1000000,"maximumCandidateBytes":1,"maximumTotalCandidateBytes":1,"timeoutMsPerExecution":10000,"maximumOutputBytes":65536},"candidateRoots":["tests/parity.test.ts"],"backend":{"kind":"trusted-local","level":"UNSANDBOXED"}},"worlds":[{"id":"neutral-bitwise-equivalence","provenance":"corpus:reviewed-equivalent","digest":"sha256:9e927e1f01bf385103eda72ed1bce77d5c344fe835dcd49dcfa20ca5bdf843ac"},{"id":"reference","provenance":"corpus:correct","digest":"sha256:c43382f86fd96a4ae1b9224e9b2297e8d8fe6a887400298d40a8510488cf0f02"},{"id":"target-parity-inversion","provenance":"corpus:known-fault","digest":"sha256:70bc1229564b8d7e41aa306f606dd5c61c2a4c87b728d5bb63b9625166efcd0a"}],"repository":{"includeDependencies":false,"omittedLinks":[],"git":null,"snapshot":"repository","include":[],"preservedLinkCount":0,"unboundedClosureEdges":[]}},"policy":{"policyVersion":"1.0.0","requiredAttempts":2,"minimumTargetWeightPermille":1000,"maximumSelectedCandidates":1,"acceptedTargetOutcomes":["ASSERTION_FAILURE"]},"worlds":[{"id":"neutral-bitwise-equivalence","kind":"NEUTRAL","required":true,"weight":0},{"id":"reference","kind":"REFERENCE","required":true,"weight":0},{"id":"target-parity-inversion","kind":"TARGET","required":true,"weight":1}],"candidates":[{"id":"designated-parity","digest":"sha256:b376bb5635b113963657cae2c3779c31477e73237e6f931a8e5ad42a9820425c","sizeBytes":198,"status":"ELIGIBLE","killedTargetIds":["target-parity-inversion"],"targetWeightKilled":1,"gates":[{"name":"COMPLETENESS","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2","designated-parity:reference:1","designated-parity:reference:2","designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]},{"name":"STABILITY","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2","designated-parity:reference:1","designated-parity:reference:2","designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]},{"name":"DISCOVERY","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2","designated-parity:reference:1","designated-parity:reference:2","designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]},{"name":"REFERENCE","status":"PASSED","evidenceRunIds":["designated-parity:reference:1","designated-parity:reference:2"],"reasonCodes":[]},{"name":"NEUTRAL","status":"PASSED","evidenceRunIds":["designated-parity:neutral-bitwise-equivalence:1","designated-parity:neutral-bitwise-equivalence:2"],"reasonCodes":[]},{"name":"TARGET_STRENGTH","status":"PASSED","evidenceRunIds":["designated-parity:target-parity-inversion:1","designated-parity:target-parity-inversion:2"],"reasonCodes":[]}],"reasonCodes":["POLICY_SATISFIED"]}],"observations":[{"runId":"control:neutral-bitwise-equivalence:1","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":143,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:neutral-bitwise-equivalence:2","candidateId":null,"worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":143,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:1","candidateId":null,"worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":150,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:reference:2","candidateId":null,"worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":144,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:1","candidateId":null,"worldId":"target-parity-inversion","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":163,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"control:target-parity-inversion:2","candidateId":null,"worldId":"target-parity-inversion","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":0,"attributed":false,"durationMs":148,"exitCode":0,"stdoutDigest":"sha256:3e1d94ab8d29fe012b4e4e79c0985142470cc0377d7de91e10ed21572ad82a64","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:neutral-bitwise-equivalence:1","candidateId":"designated-parity","worldId":"neutral-bitwise-equivalence","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":144,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:neutral-bitwise-equivalence:2","candidateId":"designated-parity","worldId":"neutral-bitwise-equivalence","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":145,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:reference:1","candidateId":"designated-parity","worldId":"reference","attempt":1,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":153,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:reference:2","candidateId":"designated-parity","worldId":"reference","attempt":2,"outcome":"PASS","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":148,"exitCode":0,"stdoutDigest":"sha256:ea5230a1c58f9802af5fecfae368a860891724194e3457e00f42ba950314964e","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:target-parity-inversion:1","candidateId":"designated-parity","worldId":"target-parity-inversion","attempt":1,"outcome":"ASSERTION_FAILURE","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":145,"exitCode":1,"stdoutDigest":"sha256:bd9f119595e8e221f0ebcbb3b09c49bd229186e081203304312862eddf7ddc25","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},{"runId":"designated-parity:target-parity-inversion:2","candidateId":"designated-parity","worldId":"target-parity-inversion","attempt":2,"outcome":"ASSERTION_FAILURE","testsDiscovered":1,"candidateTestsDiscovered":1,"attributed":true,"durationMs":148,"exitCode":1,"stdoutDigest":"sha256:bd9f119595e8e221f0ebcbb3b09c49bd229186e081203304312862eddf7ddc25","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"decision":{"status":"VERIFIED","selectedCandidateIds":["designated-parity"],"reasonCodes":["POLICY_SATISFIED"]},"decisionDigest":"sha256:a7a0f658f161bd507e3209d49c6f23249a09c8d8425cbbd4cdfb4fc6fb1c6330","artifactDigest":"sha256:f4694b7c59a97b37982366bf068f8949c5c2f1b714515a643db1e0b23cd65b23","adapter":{"kind":"bun-test-designated"},"isolation":{"kind":"trusted-local","level":"UNSANDBOXED","acknowledgedUnsafeExecution":true},"limitations":["UNSANDBOXED trusted-local execution cannot safely contain hostile candidate code.","Process-tree termination is best effort and depends on host operating-system facilities.","Only a failure thrown inside the designated test by a bun:test built-in matcher, in Bun's matcher message form, or by assertSame is admitted as an assertion; with a declared expected line, its first message line must equal it. Custom matchers, usage errors and every other error are not assertion evidence.","Controls load every designated test file without running a test: they show that each world evaluates the file and its imports and still registers the designated test, not that other tests pass.","Bun callback instrumentation determines assertion ownership; JUnit counts only check completeness and never classify assertions.","Bun ran with the run directory's own bunfig, rewritten so that AssertLedger's preload runs before every repository preload; a matcher replaced or added by expect.extend, before or during the test, is not a built-in matcher."]} diff --git a/tests/replay-corpus/documents/qualification-1.5.0-node-test.receipt.json b/tests/replay-corpus/documents/qualification-1.5.0-node-test.receipt.json new file mode 100644 index 0000000..8f905bd --- /dev/null +++ b/tests/replay-corpus/documents/qualification-1.5.0-node-test.receipt.json @@ -0,0 +1 @@ +{"schemaVersion":"1.0.0","plan":{"schemaVersion":"1.0.0","profileId":"corpus-node-test-v1","subject":{"commit":"corpus-commit","baseCommit":"corpus-base","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","inputDigest":"sha256:b82512cb17861d7889b7fd96d35fa764b213a6a743e44f14dd10ffcba28241d1"},"tools":[{"id":"node","version":"22.23.2","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4"}],"suites":[{"id":"public","files":["public.test.mjs"],"extraction":"COMPLETE"}],"obligations":[{"id":"tests","kind":"tests","required":true,"suiteIds":["public"],"checks":[{"id":"outcome","actionId":"test","field":"testOutcome","expected":"PASS"}],"limits":[]}],"actions":[{"id":"test","adapter":"node-test","executable":"C:\\nvm4w\\nodejs\\node.exe","arguments":["public.test.mjs"],"environment":{},"prepareFiles":[],"removePaths":[],"observe":{"trace":false,"outputs":[],"report":null}}],"worlds":[{"id":"reference","kind":"REFERENCE","files":[],"discriminants":[]},{"id":"assertion-fault","kind":"TARGET","files":[{"path":"public.test.mjs","content":"import assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 2));\n"}],"discriminants":[{"obligationId":"tests","checkIds":["outcome"]}]},{"id":"neutral","kind":"NEUTRAL","files":[{"path":"public.test.mjs","content":"// neutral comment\nimport assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 1));\n"}],"discriminants":[]}],"requiredAttempts":2,"timeoutMs":5000,"maximumOutputBytes":65536,"isolation":{"kind":"trusted-local","acknowledgedUnsafeExecution":true},"allowedCandidatePaths":["public.test.mjs"]},"planDigest":"sha256:f4aa7f4ba6636c21a88e8266d9b75bfc38526ecbe0026638d13a95a1f89ea0e2","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","observations":[{"worldId":"reference","attempt":1,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:da71974d353c6b6093eb8b08a7029ee74b1121bb4dc717d53d6f96bc8ea90e25"},{"worldId":"reference","attempt":2,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:437d8f3d37ec6e5aba838e92c7e5f88a9d3e7d38a9a2ab4294a835fcc22a8ea9"},{"worldId":"assertion-fault","attempt":1,"actionId":"test","state":"COMPLETED","facts":{"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":["public.test.mjs"],"testsDiscovered":1,"attributed":true,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"ASSERTION_FAILURE","outputs":{}},"bindingDigest":"sha256:afe9235f48f8404da1b70d8d602be84b347c4e2e9482218333972a2d5022540b"},{"worldId":"assertion-fault","attempt":2,"actionId":"test","state":"COMPLETED","facts":{"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":["public.test.mjs"],"testsDiscovered":1,"attributed":true,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"ASSERTION_FAILURE","outputs":{}},"bindingDigest":"sha256:9a4a29f0323d3c168beffda0d6c32a4a576c7644fd24a21f93ed995b497eaae5"},{"worldId":"neutral","attempt":1,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:e3ea64500e1331c539fa44942e2f11ce9f4f693967e20f4c32d66c345f525410"},{"worldId":"neutral","attempt":2,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:c6da210e321b8100cca6463ce2a2cb095e8de7139a937a8e3527701941d9972f"}],"provenance":{"engineVersion":"1.5.0","adapterVersions":{"command":"1.0.0","turbo":"1.0.0","node-test":"1.0.0","bun-native":"1.0.0","ci-config":"1.0.0"},"runtime":{"mechanismDigest":"sha256:20c77d886b01f77bc1c17047ee8ce0716666dbc101f82ab0e74c44d9d102e970","platform":"win32","architecture":"x64","node":"22.23.2","tools":{"node":{"executable":"C:\\nvm4w\\nodejs\\node.exe","identityPath":"C:\\nvm4w\\nodejs\\node.exe","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4","version":"22.23.2","versionArguments":["--version"]}},"collectorDigest":"sha256:cd294d2e3624b1f8b7a9ea50a145d8d6d2c1f06b415139ead3818d7885d08112","snapshotExcludes":[".git",".testforge",".turbo","coverage","node_modules"],"dependencySnapshot":"EXCLUDED_NODE_MODULES"},"executionTrust":"TRUSTED_LOCAL_UNSANDBOXED"},"externalCi":null,"assessments":[{"obligationId":"tests","kind":"tests","required":true,"status":"COVERED","reasons":[],"mismatchCheckIds":["outcome"]}],"coveredGuaranteeIds":["tests"],"openGuaranteeIds":[],"decision":"QUALIFIED","decisionDigest":"sha256:f78849ed84082ec4da7086ed0e7a852c86eb64a6f250e2a9a302afd41c57490b","artifactDigest":"sha256:e336a92c28fe78831e501b40d521822410f7038fbfeb26740561fc43726e4ea9"} diff --git a/tests/replay-corpus/documents/qualification-1.5.0-propagation.receipt.json b/tests/replay-corpus/documents/qualification-1.5.0-propagation.receipt.json new file mode 100644 index 0000000..d58ae24 --- /dev/null +++ b/tests/replay-corpus/documents/qualification-1.5.0-propagation.receipt.json @@ -0,0 +1 @@ +{"schemaVersion":"1.0.0","plan":{"schemaVersion":"1.0.0","profileId":"corpus-nonzero-command-v1","subject":{"commit":"corpus-commit","baseCommit":"corpus-base","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","inputDigest":"sha256:f1f362d13449b68d9f7b76b964642479fb04326e787bd33ed7d5cc1c8378ec4c"},"tools":[{"id":"node","version":"22.23.2","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4"}],"suites":[],"obligations":[{"id":"propagation","kind":"propagation","required":true,"suiteIds":[],"checks":[{"id":"terminal","actionId":"leaf","field":"exitCode","expected":7}],"limits":[]}],"actions":[{"id":"leaf","adapter":"command","executable":"C:\\nvm4w\\nodejs\\node.exe","arguments":["leaf.mjs"],"environment":{},"prepareFiles":[],"removePaths":[],"observe":{"trace":false,"outputs":[],"report":null}}],"worlds":[{"id":"reference","kind":"REFERENCE","files":[],"discriminants":[]},{"id":"swallowed","kind":"TARGET","files":[{"path":"leaf.mjs","content":"process.exit(0);\n"}],"discriminants":[{"obligationId":"propagation","checkIds":["terminal"]}]},{"id":"neutral","kind":"NEUTRAL","files":[{"path":"leaf.mjs","content":"// harmless\nimport {writeFileSync} from 'node:fs';\nwriteFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify({facts:{commandOutcome:'EXPECTED_FAILURE',exitCode:7},nonce:process.env.ASSERTLEDGER_QUALIFICATION_NONCE,protocolVersion:'1.0.0'})+'\\n');\nprocess.exit(7);\n"}],"discriminants":[]}],"requiredAttempts":2,"timeoutMs":5000,"maximumOutputBytes":65536,"isolation":{"kind":"trusted-local","acknowledgedUnsafeExecution":true},"allowedCandidatePaths":["leaf.mjs"]},"planDigest":"sha256:42f1fa806bd7d111060279b2f45ef68b88be1f6ba3ffe63fbd8fcf22f3ed7194","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","observations":[{"worldId":"reference","attempt":1,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"f0d4e9d5-b6a6-4f30-87d8-d0a3a37c42a5","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:3aae3c3359fbc211133fda737379fc04ee96acf162e7d0edbec3b573a4615660","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"f0d4e9d5-b6a6-4f30-87d8-d0a3a37c42a5","reportDigest":"sha256:0bf28d6804d6249d66d980cfe30dd72c9973ef0246bdd592c021ba6a15944c28","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:01f22bfd7f1ee189e3519fb05061b261896b9d75c9c9bb296221d69ae98f74ec"},{"worldId":"reference","attempt":2,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"18d7537a-beed-4e72-b278-d1ee2b578d61","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:1750d820016fa5cd587ba477eec5fccc4718265b1a8f30e4009e525ff3a39193","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"18d7537a-beed-4e72-b278-d1ee2b578d61","reportDigest":"sha256:a71110ce624e9143fc777da1447d0c7f8ea96d94b29468a966aa28a3f7f7be16","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:59c29006e5e6256330bf2c511b364a82a7eeab4efbbf9dd49185368fd871dc5a"},{"worldId":"swallowed","attempt":1,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","commandOutcome":"PASS","terminal":{"exceptional":false,"exitCode":0,"explicit":true,"natural":false,"nonce":"a31ceee7-fe60-4880-8905-e472c8ca678c","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:acf95206be8771e90ebe186c652921db9719b1a71356936ca087ca22c7ee1327","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","outputs":{}},"bindingDigest":"sha256:b96ed7d597ddbcd0475d8d2d770e8a3bf86ce0764e2f60fa99ab71d8657ca141"},{"worldId":"swallowed","attempt":2,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","commandOutcome":"PASS","terminal":{"exceptional":false,"exitCode":0,"explicit":true,"natural":false,"nonce":"0f8700e0-c68e-4392-8cd9-eef1ec4576c0","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:ab95405314e6ff25f21a66221b25dcd63c6a3a40dedf211cde0bff4b6b7b813d","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","outputs":{}},"bindingDigest":"sha256:c562e9a43bb957456317f2fc674c0d6db5d8fcbf78709b3dbc3c84513407e078"},{"worldId":"neutral","attempt":1,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"3ecf717e-b2f0-461c-8bd5-a0a588abdfa9","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:59c4636d2c37223c7d4c2fe2bd19899e7f8d7066e23563b6bd7030257ead03a7","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"3ecf717e-b2f0-461c-8bd5-a0a588abdfa9","reportDigest":"sha256:f687c2b5f39c433c8c8acdb5a89f4158ccf30e94ea913bfe951073f09142caf9","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:7826f147a99f3977f76e8a2d766d14619d506633f43530f75ffc654b8fe3172a"},{"worldId":"neutral","attempt":2,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"92d63603-0cec-4c62-beac-22758ea31dcf","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:ca904dcf2f4223a83b2423b4b5d0362d30f27fb4b04ffad44aaf78169e5c9d21","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"92d63603-0cec-4c62-beac-22758ea31dcf","reportDigest":"sha256:0e3def7258586acebf4fccae2d04e1749b2578399e1b213fc909000de231c65f","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:fa29b8cb4dc4765d630c9ec5444d3cba12f05a374eb66a15336e897beccac02f"}],"provenance":{"engineVersion":"1.5.0","adapterVersions":{"command":"1.0.0","turbo":"1.0.0","node-test":"1.0.0","bun-native":"1.0.0","ci-config":"1.0.0"},"runtime":{"mechanismDigest":"sha256:20c77d886b01f77bc1c17047ee8ce0716666dbc101f82ab0e74c44d9d102e970","platform":"win32","architecture":"x64","node":"22.23.2","tools":{"node":{"executable":"C:\\nvm4w\\nodejs\\node.exe","identityPath":"C:\\nvm4w\\nodejs\\node.exe","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4","version":"22.23.2","versionArguments":["--version"]}},"collectorDigest":"sha256:cd294d2e3624b1f8b7a9ea50a145d8d6d2c1f06b415139ead3818d7885d08112","snapshotExcludes":[".git",".testforge",".turbo","coverage","node_modules"],"dependencySnapshot":"EXCLUDED_NODE_MODULES"},"executionTrust":"TRUSTED_LOCAL_UNSANDBOXED"},"externalCi":null,"assessments":[{"obligationId":"propagation","kind":"propagation","required":true,"status":"COVERED","reasons":[],"mismatchCheckIds":["terminal"]}],"coveredGuaranteeIds":["propagation"],"openGuaranteeIds":[],"decision":"QUALIFIED","decisionDigest":"sha256:75e0fff422b61aed00775d2b2dfa7b33752e7e84a30b4ca8fb94619d592f5935","artifactDigest":"sha256:7d72a9ea6443ba3a9b809d6fe4628f225dc8b61850dc423bbdf09dd601bbaf02"} diff --git a/tests/replay-corpus/documents/qualification-2.1.0-node-test.receipt.json b/tests/replay-corpus/documents/qualification-2.1.0-node-test.receipt.json new file mode 100644 index 0000000..ecca5da --- /dev/null +++ b/tests/replay-corpus/documents/qualification-2.1.0-node-test.receipt.json @@ -0,0 +1 @@ +{"schemaVersion":"1.0.0","plan":{"schemaVersion":"1.0.0","profileId":"corpus-node-test-v1","subject":{"commit":"corpus-commit","baseCommit":"corpus-base","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","inputDigest":"sha256:b82512cb17861d7889b7fd96d35fa764b213a6a743e44f14dd10ffcba28241d1"},"tools":[{"id":"node","version":"22.23.2","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4"}],"suites":[{"id":"public","files":["public.test.mjs"],"extraction":"COMPLETE"}],"obligations":[{"id":"tests","kind":"tests","required":true,"suiteIds":["public"],"checks":[{"id":"outcome","actionId":"test","field":"testOutcome","expected":"PASS"}],"limits":[]}],"actions":[{"id":"test","adapter":"node-test","executable":"C:\\nvm4w\\nodejs\\node.exe","arguments":["public.test.mjs"],"environment":{},"prepareFiles":[],"removePaths":[],"observe":{"trace":false,"outputs":[],"report":null}}],"worlds":[{"id":"reference","kind":"REFERENCE","files":[],"discriminants":[]},{"id":"assertion-fault","kind":"TARGET","files":[{"path":"public.test.mjs","content":"import assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 2));\n"}],"discriminants":[{"obligationId":"tests","checkIds":["outcome"]}]},{"id":"neutral","kind":"NEUTRAL","files":[{"path":"public.test.mjs","content":"// neutral comment\nimport assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 1));\n"}],"discriminants":[]}],"requiredAttempts":2,"timeoutMs":5000,"maximumOutputBytes":65536,"isolation":{"kind":"trusted-local","acknowledgedUnsafeExecution":true},"allowedCandidatePaths":["public.test.mjs"]},"planDigest":"sha256:f4aa7f4ba6636c21a88e8266d9b75bfc38526ecbe0026638d13a95a1f89ea0e2","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","observations":[{"worldId":"reference","attempt":1,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:da71974d353c6b6093eb8b08a7029ee74b1121bb4dc717d53d6f96bc8ea90e25"},{"worldId":"reference","attempt":2,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:437d8f3d37ec6e5aba838e92c7e5f88a9d3e7d38a9a2ab4294a835fcc22a8ea9"},{"worldId":"assertion-fault","attempt":1,"actionId":"test","state":"COMPLETED","facts":{"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":["public.test.mjs"],"testsDiscovered":1,"attributed":true,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"ASSERTION_FAILURE","outputs":{}},"bindingDigest":"sha256:afe9235f48f8404da1b70d8d602be84b347c4e2e9482218333972a2d5022540b"},{"worldId":"assertion-fault","attempt":2,"actionId":"test","state":"COMPLETED","facts":{"exitCode":1,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":["public.test.mjs"],"testsDiscovered":1,"attributed":true,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"ASSERTION_FAILURE","outputs":{}},"bindingDigest":"sha256:9a4a29f0323d3c168beffda0d6c32a4a576c7644fd24a21f93ed995b497eaae5"},{"worldId":"neutral","attempt":1,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:e3ea64500e1331c539fa44942e2f11ce9f4f693967e20f4c32d66c345f525410"},{"worldId":"neutral","attempt":2,"actionId":"test","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","testFiles":["public.test.mjs"],"assertionFailureFiles":[],"testsDiscovered":1,"attributed":false,"skippedTests":0,"cancelledTests":0,"todoTests":0,"testOutcome":"PASS","outputs":{}},"bindingDigest":"sha256:c6da210e321b8100cca6463ce2a2cb095e8de7139a937a8e3527701941d9972f"}],"provenance":{"engineVersion":"2.1.0","adapterVersions":{"command":"1.0.0","turbo":"1.0.0","node-test":"1.0.0","bun-native":"1.0.0","ci-config":"1.0.0"},"runtime":{"mechanismDigest":"sha256:43514ca4a532d8bb64b6e461a1865123a36b9abd413222dc5d93c64c1c6d98d1","platform":"win32","architecture":"x64","node":"22.23.2","tools":{"node":{"executable":"C:\\nvm4w\\nodejs\\node.exe","identityPath":"C:\\nvm4w\\nodejs\\node.exe","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4","version":"22.23.2","versionArguments":["--version"]}},"collectorDigest":"sha256:cd294d2e3624b1f8b7a9ea50a145d8d6d2c1f06b415139ead3818d7885d08112","snapshotExcludes":[".git",".testforge",".turbo","coverage","node_modules"],"dependencySnapshot":"EXCLUDED_NODE_MODULES"},"executionTrust":"TRUSTED_LOCAL_UNSANDBOXED"},"externalCi":null,"assessments":[{"obligationId":"tests","kind":"tests","required":true,"status":"COVERED","reasons":[],"mismatchCheckIds":["outcome"]}],"coveredGuaranteeIds":["tests"],"openGuaranteeIds":[],"decision":"QUALIFIED","decisionDigest":"sha256:f78849ed84082ec4da7086ed0e7a852c86eb64a6f250e2a9a302afd41c57490b","artifactDigest":"sha256:fce88cdbdd2a97e19ec81f8774d928adfe6e9592cb5ccdf8fcb3906834afdb26"} diff --git a/tests/replay-corpus/documents/qualification-2.1.0-propagation.receipt.json b/tests/replay-corpus/documents/qualification-2.1.0-propagation.receipt.json new file mode 100644 index 0000000..c365976 --- /dev/null +++ b/tests/replay-corpus/documents/qualification-2.1.0-propagation.receipt.json @@ -0,0 +1 @@ +{"schemaVersion":"1.0.0","plan":{"schemaVersion":"1.0.0","profileId":"corpus-nonzero-command-v1","subject":{"commit":"corpus-commit","baseCommit":"corpus-base","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","inputDigest":"sha256:f1f362d13449b68d9f7b76b964642479fb04326e787bd33ed7d5cc1c8378ec4c"},"tools":[{"id":"node","version":"22.23.2","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4"}],"suites":[],"obligations":[{"id":"propagation","kind":"propagation","required":true,"suiteIds":[],"checks":[{"id":"terminal","actionId":"leaf","field":"exitCode","expected":7}],"limits":[]}],"actions":[{"id":"leaf","adapter":"command","executable":"C:\\nvm4w\\nodejs\\node.exe","arguments":["leaf.mjs"],"environment":{},"prepareFiles":[],"removePaths":[],"observe":{"trace":false,"outputs":[],"report":null}}],"worlds":[{"id":"reference","kind":"REFERENCE","files":[],"discriminants":[]},{"id":"swallowed","kind":"TARGET","files":[{"path":"leaf.mjs","content":"process.exit(0);\n"}],"discriminants":[{"obligationId":"propagation","checkIds":["terminal"]}]},{"id":"neutral","kind":"NEUTRAL","files":[{"path":"leaf.mjs","content":"// harmless\nimport {writeFileSync} from 'node:fs';\nwriteFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify({facts:{commandOutcome:'EXPECTED_FAILURE',exitCode:7},nonce:process.env.ASSERTLEDGER_QUALIFICATION_NONCE,protocolVersion:'1.0.0'})+'\\n');\nprocess.exit(7);\n"}],"discriminants":[]}],"requiredAttempts":2,"timeoutMs":5000,"maximumOutputBytes":65536,"isolation":{"kind":"trusted-local","acknowledgedUnsafeExecution":true},"allowedCandidatePaths":["leaf.mjs"]},"planDigest":"sha256:42f1fa806bd7d111060279b2f45ef68b88be1f6ba3ffe63fbd8fcf22f3ed7194","candidateDigest":"sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945","observations":[{"worldId":"reference","attempt":1,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"29ad7590-0a5c-40d7-96d2-8ecc32dceb47","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:6167309bbe60583cc86214de192b728e32744d9c8f997a1c13a4e2b2339828c4","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"29ad7590-0a5c-40d7-96d2-8ecc32dceb47","reportDigest":"sha256:6835cbd03c1014b3bfae120bdbe59753777e7d2d8c8acc6d519b04a7c707e46d","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:01f22bfd7f1ee189e3519fb05061b261896b9d75c9c9bb296221d69ae98f74ec"},{"worldId":"reference","attempt":2,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"599167d3-30cd-4cb1-a8c3-295d9d1259cc","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:4361ee42bbf1341e73b701a7594623de448d623b8a950d2db53287b00cf86320","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"599167d3-30cd-4cb1-a8c3-295d9d1259cc","reportDigest":"sha256:0e6648b8dad44664afe283e0e2a7d0b9297a0978e24854741076ccea82a29e5d","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:59c29006e5e6256330bf2c511b364a82a7eeab4efbbf9dd49185368fd871dc5a"},{"worldId":"swallowed","attempt":1,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","commandOutcome":"PASS","terminal":{"exceptional":false,"exitCode":0,"explicit":true,"natural":false,"nonce":"57113223-39f0-4429-8d2d-8e0e9ff6fb88","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:63900757098f4897e433337f39be0a18fdc1d63ecad91e42619855fc6fe7813e","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","outputs":{}},"bindingDigest":"sha256:b96ed7d597ddbcd0475d8d2d770e8a3bf86ce0764e2f60fa99ab71d8657ca141"},{"worldId":"swallowed","attempt":2,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":0,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","commandOutcome":"PASS","terminal":{"exceptional":false,"exitCode":0,"explicit":true,"natural":false,"nonce":"4f904494-f0f9-48ae-ad2f-c2fa6b6cc1ff","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:fe4eb95a11b6b42f50504733a231be7f2e8c381b3fa201012c4de54d1d7a37d3","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","outputs":{}},"bindingDigest":"sha256:c562e9a43bb957456317f2fc674c0d6db5d8fcbf78709b3dbc3c84513407e078"},{"worldId":"neutral","attempt":1,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"3335ee13-b6c6-4a45-abc5-309b4caabd1c","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:3146e315b7da5f23894e91ca29ddbef90c90ee306496b289000c9fdfd14e0f81","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"3335ee13-b6c6-4a45-abc5-309b4caabd1c","reportDigest":"sha256:f721011d942e65a91b73656f632e916c70129e71bedf9bbe3f827b4adab5240b","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:7826f147a99f3977f76e8a2d766d14619d506633f43530f75ffc654b8fe3172a"},{"worldId":"neutral","attempt":2,"actionId":"leaf","state":"COMPLETED","facts":{"exitCode":7,"stdoutDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","stderrDigest":"sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","terminal":{"exceptional":false,"exitCode":7,"explicit":true,"natural":false,"nonce":"a1029e5b-e6f5-412e-b680-7ba2ae541bf9","protocolVersion":"1.0.0","ready":true,"runtime":"node","runtimeVersion":"22.23.2"},"terminalDigest":"sha256:65273e7a9820ce4298c65dfb02578b62bbe2ea4bf68df5644bc41bab46a305ca","terminalProvenance":"ENGINE_LIFECYCLE_OBSERVED","report":{"commandOutcome":"EXPECTED_FAILURE","exitCode":7},"reportNonce":"a1029e5b-e6f5-412e-b680-7ba2ae541bf9","reportDigest":"sha256:a7299bbdbff00329f80d0be7ae7833c6ca9ddb35a09acf82c1290f57fa9456c3","reportProvenance":"STRUCTURED_ADAPTER_REPORTED","commandOutcome":"EXPECTED_FAILURE","outputs":{}},"bindingDigest":"sha256:fa29b8cb4dc4765d630c9ec5444d3cba12f05a374eb66a15336e897beccac02f"}],"provenance":{"engineVersion":"2.1.0","adapterVersions":{"command":"1.0.0","turbo":"1.0.0","node-test":"1.0.0","bun-native":"1.0.0","ci-config":"1.0.0"},"runtime":{"mechanismDigest":"sha256:43514ca4a532d8bb64b6e461a1865123a36b9abd413222dc5d93c64c1c6d98d1","platform":"win32","architecture":"x64","node":"22.23.2","tools":{"node":{"executable":"C:\\nvm4w\\nodejs\\node.exe","identityPath":"C:\\nvm4w\\nodejs\\node.exe","digest":"sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4","version":"22.23.2","versionArguments":["--version"]}},"collectorDigest":"sha256:cd294d2e3624b1f8b7a9ea50a145d8d6d2c1f06b415139ead3818d7885d08112","snapshotExcludes":[".git",".testforge",".turbo","coverage","node_modules"],"dependencySnapshot":"EXCLUDED_NODE_MODULES"},"executionTrust":"TRUSTED_LOCAL_UNSANDBOXED"},"externalCi":null,"assessments":[{"obligationId":"propagation","kind":"propagation","required":true,"status":"COVERED","reasons":[],"mismatchCheckIds":["terminal"]}],"coveredGuaranteeIds":["propagation"],"openGuaranteeIds":[],"decision":"QUALIFIED","decisionDigest":"sha256:75e0fff422b61aed00775d2b2dfa7b33752e7e84a30b4ca8fb94619d592f5935","artifactDigest":"sha256:277cb7080167b892fecb68501c1bcd8e22af62482dca71d525e52f509bd669bb"} diff --git a/tests/replay-corpus/inputs/evidence-v1/repository/package.json b/tests/replay-corpus/inputs/evidence-v1/repository/package.json new file mode 100644 index 0000000..fcde402 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v1/repository/package.json @@ -0,0 +1,5 @@ +{ + "name": "assertledger-node-example", + "private": true, + "type": "module" +} diff --git a/tests/replay-corpus/inputs/evidence-v1/repository/src/is-even.js b/tests/replay-corpus/inputs/evidence-v1/repository/src/is-even.js new file mode 100644 index 0000000..89e138c --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v1/repository/src/is-even.js @@ -0,0 +1,3 @@ +export function isEven(value) { + return value % 2 === 0; +} diff --git a/tests/replay-corpus/inputs/evidence-v1/repository/tests/base.test.js b/tests/replay-corpus/inputs/evidence-v1/repository/tests/base.test.js new file mode 100644 index 0000000..4509767 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v1/repository/tests/base.test.js @@ -0,0 +1,6 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +test("baseline arithmetic works", () => { + assert.equal(1 + 1, 2); +}); diff --git a/tests/replay-corpus/inputs/evidence-v1/request.json b/tests/replay-corpus/inputs/evidence-v1/request.json new file mode 100644 index 0000000..a370a37 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v1/request.json @@ -0,0 +1,109 @@ +{ + "schemaVersion": "1.0.0", + "repository": { + "root": "./repository", + "exclude": [ + ".git", + ".testforge", + "node_modules" + ] + }, + "adapter": { + "kind": "node-test", + "executable": "node", + "baseTestFiles": [ + "tests/base.test.js" + ] + }, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true, + "environmentAllowlist": [ + "PATH", + "SystemRoot", + "WINDIR", + "TEMP", + "TMP" + ] + }, + "candidateRoots": [ + "tests/candidates" + ], + "budgets": { + "maximumCandidates": 4, + "maximumWorlds": 4, + "maximumExecutions": 24, + "maximumRepositoryFiles": 10000, + "maximumRepositoryBytes": 100000000, + "maximumWorldOverlayBytes": 1000000, + "maximumCandidateBytes": 16384, + "maximumTotalCandidateBytes": 65536, + "timeoutMsPerExecution": 5000, + "maximumOutputBytes": 65536 + }, + "policy": { + "policyVersion": "1.0.0", + "requiredAttempts": 2, + "minimumTargetWeightPermille": 1000, + "maximumSelectedCandidates": 1, + "acceptedTargetOutcomes": [ + "ASSERTION_FAILURE" + ] + }, + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "required": true, + "weight": 0, + "provenance": "example:correct-implementation", + "files": [] + }, + { + "id": "target-parity-inversion", + "kind": "TARGET", + "required": true, + "weight": 1, + "provenance": "example:known-fault", + "files": [ + { + "path": "src/is-even.js", + "content": "export function isEven(value) {\n return value % 2 === 1;\n}\n" + } + ] + }, + { + "id": "neutral-bitwise-equivalence", + "kind": "NEUTRAL", + "required": true, + "weight": 0, + "provenance": "example:reviewed-equivalent", + "files": [ + { + "path": "src/is-even.js", + "content": "export function isEven(value) {\n return (value & 1) === 0;\n}\n" + } + ] + } + ], + "candidates": [ + { + "id": "strong", + "files": [ + { + "path": "tests/candidates/strong.test.js", + "content": "import assert from \"node:assert/strict\";\nimport test from \"node:test\";\nimport { isEven } from \"../../src/is-even.js\";\ntest(\"two is even\", () => assert.equal(isEven(2), true));\n" + } + ] + }, + { + "id": "weak", + "files": [ + { + "path": "tests/candidates/weak.test.js", + "content": "import assert from \"node:assert/strict\";\nimport test from \"node:test\";\nimport { isEven } from \"../../src/is-even.js\";\ntest(\"returns a boolean\", () => assert.equal(typeof isEven(2), \"boolean\"));\n" + } + ] + } + ] +} diff --git a/tests/replay-corpus/inputs/evidence-v2/repository/package.json b/tests/replay-corpus/inputs/evidence-v2/repository/package.json new file mode 100644 index 0000000..fcde402 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v2/repository/package.json @@ -0,0 +1,5 @@ +{ + "name": "assertledger-node-example", + "private": true, + "type": "module" +} diff --git a/tests/replay-corpus/inputs/evidence-v2/repository/src/is-even.js b/tests/replay-corpus/inputs/evidence-v2/repository/src/is-even.js new file mode 100644 index 0000000..89e138c --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v2/repository/src/is-even.js @@ -0,0 +1,3 @@ +export function isEven(value) { + return value % 2 === 0; +} diff --git a/tests/replay-corpus/inputs/evidence-v2/repository/tests/base.test.js b/tests/replay-corpus/inputs/evidence-v2/repository/tests/base.test.js new file mode 100644 index 0000000..4509767 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v2/repository/tests/base.test.js @@ -0,0 +1,6 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +test("baseline arithmetic works", () => { + assert.equal(1 + 1, 2); +}); diff --git a/tests/replay-corpus/inputs/evidence-v2/request.json b/tests/replay-corpus/inputs/evidence-v2/request.json new file mode 100644 index 0000000..92800a7 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v2/request.json @@ -0,0 +1,109 @@ +{ + "schemaVersion": "2.0.0", + "repository": { + "root": "./repository", + "exclude": [ + ".git", + ".testforge", + "node_modules" + ] + }, + "adapter": { + "kind": "node-test", + "executable": "node", + "baseTestFiles": [ + "tests/base.test.js" + ] + }, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true, + "environmentAllowlist": [ + "PATH", + "SystemRoot", + "WINDIR", + "TEMP", + "TMP" + ] + }, + "candidateRoots": [ + "tests/candidates" + ], + "budgets": { + "maximumCandidates": 4, + "maximumWorlds": 4, + "maximumExecutions": 24, + "maximumRepositoryFiles": 10000, + "maximumRepositoryBytes": 100000000, + "maximumWorldOverlayBytes": 1000000, + "maximumCandidateBytes": 16384, + "maximumTotalCandidateBytes": 65536, + "timeoutMsPerExecution": 5000, + "maximumOutputBytes": 65536 + }, + "policy": { + "policyVersion": "1.0.0", + "requiredAttempts": 2, + "minimumTargetWeightPermille": 1000, + "maximumSelectedCandidates": 1, + "acceptedTargetOutcomes": [ + "ASSERTION_FAILURE" + ] + }, + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "required": true, + "weight": 0, + "provenance": "example:correct-implementation", + "files": [] + }, + { + "id": "target-parity-inversion", + "kind": "TARGET", + "required": true, + "weight": 1, + "provenance": "example:known-fault", + "files": [ + { + "path": "src/is-even.js", + "content": "export function isEven(value) {\n return value % 2 === 1;\n}\n" + } + ] + }, + { + "id": "neutral-bitwise-equivalence", + "kind": "NEUTRAL", + "required": true, + "weight": 0, + "provenance": "example:reviewed-equivalent", + "files": [ + { + "path": "src/is-even.js", + "content": "export function isEven(value) {\n return (value & 1) === 0;\n}\n" + } + ] + } + ], + "candidates": [ + { + "id": "strong", + "files": [ + { + "path": "tests/candidates/strong.test.js", + "content": "import assert from \"node:assert/strict\";\nimport test from \"node:test\";\nimport { isEven } from \"../../src/is-even.js\";\ntest(\"two is even\", () => assert.equal(isEven(2), true));\n" + } + ] + }, + { + "id": "weak", + "files": [ + { + "path": "tests/candidates/weak.test.js", + "content": "import assert from \"node:assert/strict\";\nimport test from \"node:test\";\nimport { isEven } from \"../../src/is-even.js\";\ntest(\"returns a boolean\", () => assert.equal(typeof isEven(2), \"boolean\"));\n" + } + ] + } + ] +} diff --git a/tests/replay-corpus/inputs/evidence-v3/repository/package.json b/tests/replay-corpus/inputs/evidence-v3/repository/package.json new file mode 100644 index 0000000..089153b --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v3/repository/package.json @@ -0,0 +1 @@ +{"type":"module"} diff --git a/tests/replay-corpus/inputs/evidence-v3/repository/src/subject.ts b/tests/replay-corpus/inputs/evidence-v3/repository/src/subject.ts new file mode 100644 index 0000000..ef4b95c --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v3/repository/src/subject.ts @@ -0,0 +1 @@ +export const subject = (value: number) => value % 2 === 0; diff --git a/tests/replay-corpus/inputs/evidence-v3/repository/tests/base.test.ts b/tests/replay-corpus/inputs/evidence-v3/repository/tests/base.test.ts new file mode 100644 index 0000000..fcc9692 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v3/repository/tests/base.test.ts @@ -0,0 +1,2 @@ +import { test } from "bun:test"; +test("control", () => {}); diff --git a/tests/replay-corpus/inputs/evidence-v3/request.json b/tests/replay-corpus/inputs/evidence-v3/request.json new file mode 100644 index 0000000..e1a2455 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v3/request.json @@ -0,0 +1,109 @@ +{ + "schemaVersion": "3.0.0", + "repository": { + "root": "./repository", + "exclude": [ + ".git", + ".testforge", + "node_modules" + ] + }, + "adapter": { + "kind": "bun-test", + "executable": "bun", + "baseTestFiles": [ + "tests/base.test.ts" + ] + }, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true, + "environmentAllowlist": [ + "PATH", + "SystemRoot", + "WINDIR", + "TEMP", + "TMP" + ] + }, + "candidateRoots": [ + "tests/candidates" + ], + "budgets": { + "maximumCandidates": 4, + "maximumWorlds": 4, + "maximumExecutions": 24, + "maximumRepositoryFiles": 10000, + "maximumRepositoryBytes": 100000000, + "maximumWorldOverlayBytes": 1000000, + "maximumCandidateBytes": 16384, + "maximumTotalCandidateBytes": 65536, + "timeoutMsPerExecution": 10000, + "maximumOutputBytes": 65536 + }, + "policy": { + "policyVersion": "1.0.0", + "requiredAttempts": 2, + "minimumTargetWeightPermille": 1000, + "maximumSelectedCandidates": 1, + "acceptedTargetOutcomes": [ + "ASSERTION_FAILURE" + ] + }, + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "required": true, + "weight": 0, + "provenance": "corpus:correct", + "files": [] + }, + { + "id": "target-parity-inversion", + "kind": "TARGET", + "required": true, + "weight": 1, + "provenance": "corpus:known-fault", + "files": [ + { + "path": "src/subject.ts", + "content": "export const subject = (value: number) => value % 2 === 1;\n" + } + ] + }, + { + "id": "neutral-bitwise-equivalence", + "kind": "NEUTRAL", + "required": true, + "weight": 0, + "provenance": "corpus:reviewed-equivalent", + "files": [ + { + "path": "src/subject.ts", + "content": "export const subject = (value: number) => (value & 1) === 0;\n" + } + ] + } + ], + "candidates": [ + { + "id": "strong", + "files": [ + { + "path": "tests/candidates/strong.test.ts", + "content": "import { test } from \"bun:test\";\nimport { assertSame } from \"assertledger/bun\";\nimport { subject } from \"../../src/subject.ts\";\ntest(\"two is even\", () => assertSame(subject(2), true));\n" + } + ] + }, + { + "id": "weak", + "files": [ + { + "path": "tests/candidates/weak.test.ts", + "content": "import { test } from \"bun:test\";\nimport { assertSame } from \"assertledger/bun\";\nimport { subject } from \"../../src/subject.ts\";\ntest(\"returns a boolean\", () => assertSame(typeof subject(2), \"boolean\"));\n" + } + ] + } + ] +} diff --git a/tests/replay-corpus/inputs/evidence-v4/repository/package.json b/tests/replay-corpus/inputs/evidence-v4/repository/package.json new file mode 100644 index 0000000..089153b --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v4/repository/package.json @@ -0,0 +1 @@ +{"type":"module"} diff --git a/tests/replay-corpus/inputs/evidence-v4/repository/src/subject.ts b/tests/replay-corpus/inputs/evidence-v4/repository/src/subject.ts new file mode 100644 index 0000000..ef4b95c --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v4/repository/src/subject.ts @@ -0,0 +1 @@ +export const subject = (value: number) => value % 2 === 0; diff --git a/tests/replay-corpus/inputs/evidence-v4/repository/tests/base.test.ts b/tests/replay-corpus/inputs/evidence-v4/repository/tests/base.test.ts new file mode 100644 index 0000000..fcc9692 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v4/repository/tests/base.test.ts @@ -0,0 +1,2 @@ +import { test } from "bun:test"; +test("control", () => {}); diff --git a/tests/replay-corpus/inputs/evidence-v4/repository/tests/parity.test.ts b/tests/replay-corpus/inputs/evidence-v4/repository/tests/parity.test.ts new file mode 100644 index 0000000..dc86dc5 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v4/repository/tests/parity.test.ts @@ -0,0 +1,7 @@ +import { describe, expect, test } from "bun:test"; +import { subject } from "../src/subject.ts"; +describe("parity", () => { + test("two is even", () => { + expect(subject(2)).toBe(true); + }); +}); diff --git a/tests/replay-corpus/inputs/evidence-v4/request.json b/tests/replay-corpus/inputs/evidence-v4/request.json new file mode 100644 index 0000000..6292f28 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v4/request.json @@ -0,0 +1,102 @@ +{ + "schemaVersion": "4.0.0", + "repository": { + "root": "./repository", + "exclude": [ + ".git", + ".testforge", + "node_modules" + ], + "includeDependencies": false, + "git": "excluded" + }, + "adapter": { + "kind": "bun-test-designated", + "executable": "bun", + "testTimeoutMs": null + }, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true, + "environmentAllowlist": [ + "PATH", + "SystemRoot", + "WINDIR", + "TEMP", + "TMP" + ] + }, + "candidateRoots": [ + "tests/parity.test.ts" + ], + "budgets": { + "maximumCandidates": 1, + "maximumWorlds": 4, + "maximumExecutions": 24, + "maximumRepositoryFiles": 10000, + "maximumRepositoryBytes": 100000000, + "maximumWorldOverlayBytes": 1000000, + "maximumCandidateBytes": 1, + "maximumTotalCandidateBytes": 1, + "timeoutMsPerExecution": 10000, + "maximumOutputBytes": 65536 + }, + "policy": { + "policyVersion": "1.0.0", + "requiredAttempts": 2, + "minimumTargetWeightPermille": 1000, + "maximumSelectedCandidates": 1, + "acceptedTargetOutcomes": [ + "ASSERTION_FAILURE" + ] + }, + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "required": true, + "weight": 0, + "provenance": "corpus:correct", + "files": [] + }, + { + "id": "target-parity-inversion", + "kind": "TARGET", + "required": true, + "weight": 1, + "provenance": "corpus:known-fault", + "files": [ + { + "path": "src/subject.ts", + "content": "export const subject = (value: number) => value % 2 === 1;\n" + } + ] + }, + { + "id": "neutral-bitwise-equivalence", + "kind": "NEUTRAL", + "required": true, + "weight": 0, + "provenance": "corpus:reviewed-equivalent", + "files": [ + { + "path": "src/subject.ts", + "content": "export const subject = (value: number) => (value & 1) === 0;\n" + } + ] + } + ], + "candidates": [ + { + "id": "designated-parity", + "test": { + "file": "tests/parity.test.ts", + "path": [ + "parity", + "two is even" + ] + }, + "expectedFailure": null + } + ] +} diff --git a/tests/replay-corpus/inputs/evidence-v5/repository/package.json b/tests/replay-corpus/inputs/evidence-v5/repository/package.json new file mode 100644 index 0000000..089153b --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v5/repository/package.json @@ -0,0 +1 @@ +{"type":"module"} diff --git a/tests/replay-corpus/inputs/evidence-v5/repository/src/subject.ts b/tests/replay-corpus/inputs/evidence-v5/repository/src/subject.ts new file mode 100644 index 0000000..ef4b95c --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v5/repository/src/subject.ts @@ -0,0 +1 @@ +export const subject = (value: number) => value % 2 === 0; diff --git a/tests/replay-corpus/inputs/evidence-v5/repository/tests/base.test.ts b/tests/replay-corpus/inputs/evidence-v5/repository/tests/base.test.ts new file mode 100644 index 0000000..fcc9692 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v5/repository/tests/base.test.ts @@ -0,0 +1,2 @@ +import { test } from "bun:test"; +test("control", () => {}); diff --git a/tests/replay-corpus/inputs/evidence-v5/repository/tests/parity.test.ts b/tests/replay-corpus/inputs/evidence-v5/repository/tests/parity.test.ts new file mode 100644 index 0000000..dc86dc5 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v5/repository/tests/parity.test.ts @@ -0,0 +1,7 @@ +import { describe, expect, test } from "bun:test"; +import { subject } from "../src/subject.ts"; +describe("parity", () => { + test("two is even", () => { + expect(subject(2)).toBe(true); + }); +}); diff --git a/tests/replay-corpus/inputs/evidence-v5/request.json b/tests/replay-corpus/inputs/evidence-v5/request.json new file mode 100644 index 0000000..88807d4 --- /dev/null +++ b/tests/replay-corpus/inputs/evidence-v5/request.json @@ -0,0 +1,107 @@ +{ + "schemaVersion": "5.0.0", + "repository": { + "root": "./repository", + "exclude": [ + ".git", + ".testforge", + "node_modules" + ], + "includeDependencies": false, + "git": "excluded", + "snapshot": "repository", + "include": [] + }, + "adapter": { + "kind": "bun-test-designated", + "executable": "bun", + "testTimeoutMs": null, + "workingDirectory": "repository-root", + "isolate": false, + "trustedMatchers": [] + }, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true, + "environmentAllowlist": [ + "PATH", + "SystemRoot", + "WINDIR", + "TEMP", + "TMP" + ] + }, + "candidateRoots": [ + "tests/parity.test.ts" + ], + "budgets": { + "maximumCandidates": 1, + "maximumWorlds": 4, + "maximumExecutions": 24, + "maximumRepositoryFiles": 10000, + "maximumRepositoryBytes": 100000000, + "maximumWorldOverlayBytes": 1000000, + "maximumCandidateBytes": 1, + "maximumTotalCandidateBytes": 1, + "timeoutMsPerExecution": 10000, + "maximumOutputBytes": 65536 + }, + "policy": { + "policyVersion": "1.0.0", + "requiredAttempts": 2, + "minimumTargetWeightPermille": 1000, + "maximumSelectedCandidates": 1, + "acceptedTargetOutcomes": [ + "ASSERTION_FAILURE" + ] + }, + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "required": true, + "weight": 0, + "provenance": "corpus:correct", + "files": [] + }, + { + "id": "target-parity-inversion", + "kind": "TARGET", + "required": true, + "weight": 1, + "provenance": "corpus:known-fault", + "files": [ + { + "path": "src/subject.ts", + "content": "export const subject = (value: number) => value % 2 === 1;\n" + } + ] + }, + { + "id": "neutral-bitwise-equivalence", + "kind": "NEUTRAL", + "required": true, + "weight": 0, + "provenance": "corpus:reviewed-equivalent", + "files": [ + { + "path": "src/subject.ts", + "content": "export const subject = (value: number) => (value & 1) === 0;\n" + } + ] + } + ], + "candidates": [ + { + "id": "designated-parity", + "test": { + "file": "tests/parity.test.ts", + "path": [ + "parity", + "two is even" + ] + }, + "expectedFailure": null + } + ] +} diff --git a/tests/replay-corpus/inputs/qualification-1.5.0-node-test/request.json b/tests/replay-corpus/inputs/qualification-1.5.0-node-test/request.json new file mode 100644 index 0000000..fc0e7db --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-1.5.0-node-test/request.json @@ -0,0 +1,117 @@ +{ + "root": "./source", + "plan": { + "schemaVersion": "1.0.0", + "requiredAttempts": 2, + "timeoutMs": 5000, + "maximumOutputBytes": 65536, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true + }, + "profileId": "corpus-node-test-v1", + "suites": [ + { + "id": "public", + "files": [ + "public.test.mjs" + ], + "extraction": "COMPLETE" + } + ], + "obligations": [ + { + "id": "tests", + "kind": "tests", + "required": true, + "suiteIds": [ + "public" + ], + "checks": [ + { + "id": "outcome", + "actionId": "test", + "field": "testOutcome", + "expected": "PASS" + } + ], + "limits": [] + } + ], + "actions": [ + { + "id": "test", + "adapter": "node-test", + "executable": "C:\\nvm4w\\nodejs\\node.exe", + "arguments": [ + "public.test.mjs" + ], + "environment": {}, + "prepareFiles": [], + "removePaths": [], + "observe": { + "trace": false, + "outputs": [], + "report": null + } + } + ], + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "files": [], + "discriminants": [] + }, + { + "id": "assertion-fault", + "kind": "TARGET", + "files": [ + { + "path": "public.test.mjs", + "content": "import assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 2));\n" + } + ], + "discriminants": [ + { + "obligationId": "tests", + "checkIds": [ + "outcome" + ] + } + ] + }, + { + "id": "neutral", + "kind": "NEUTRAL", + "files": [ + { + "path": "public.test.mjs", + "content": "// neutral comment\nimport assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 1));\n" + } + ], + "discriminants": [] + } + ], + "subject": { + "commit": "corpus-commit", + "baseCommit": "corpus-base", + "candidateDigest": "sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "inputDigest": "sha256:b82512cb17861d7889b7fd96d35fa764b213a6a743e44f14dd10ffcba28241d1" + }, + "tools": [ + { + "id": "node", + "version": "22.23.2", + "digest": "sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4" + } + ], + "allowedCandidatePaths": [ + "public.test.mjs" + ] + }, + "planDigest": "sha256:f4aa7f4ba6636c21a88e8266d9b75bfc38526ecbe0026638d13a95a1f89ea0e2", + "candidate": { + "files": [] + } +} diff --git a/tests/replay-corpus/inputs/qualification-1.5.0-node-test/source/public.test.mjs b/tests/replay-corpus/inputs/qualification-1.5.0-node-test/source/public.test.mjs new file mode 100644 index 0000000..ce194c9 --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-1.5.0-node-test/source/public.test.mjs @@ -0,0 +1,3 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +test("public qualification", () => assert.equal(1, 1)); diff --git a/tests/replay-corpus/inputs/qualification-1.5.0-propagation/request.json b/tests/replay-corpus/inputs/qualification-1.5.0-propagation/request.json new file mode 100644 index 0000000..ffb71f4 --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-1.5.0-propagation/request.json @@ -0,0 +1,107 @@ +{ + "root": "./source", + "plan": { + "schemaVersion": "1.0.0", + "requiredAttempts": 2, + "timeoutMs": 5000, + "maximumOutputBytes": 65536, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true + }, + "profileId": "corpus-nonzero-command-v1", + "suites": [], + "obligations": [ + { + "id": "propagation", + "kind": "propagation", + "required": true, + "suiteIds": [], + "checks": [ + { + "id": "terminal", + "actionId": "leaf", + "field": "exitCode", + "expected": 7 + } + ], + "limits": [] + } + ], + "actions": [ + { + "id": "leaf", + "adapter": "command", + "executable": "C:\\nvm4w\\nodejs\\node.exe", + "arguments": [ + "leaf.mjs" + ], + "environment": {}, + "prepareFiles": [], + "removePaths": [], + "observe": { + "trace": false, + "outputs": [], + "report": null + } + } + ], + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "files": [], + "discriminants": [] + }, + { + "id": "swallowed", + "kind": "TARGET", + "files": [ + { + "path": "leaf.mjs", + "content": "process.exit(0);\n" + } + ], + "discriminants": [ + { + "obligationId": "propagation", + "checkIds": [ + "terminal" + ] + } + ] + }, + { + "id": "neutral", + "kind": "NEUTRAL", + "files": [ + { + "path": "leaf.mjs", + "content": "// harmless\nimport {writeFileSync} from 'node:fs';\nwriteFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify({facts:{commandOutcome:'EXPECTED_FAILURE',exitCode:7},nonce:process.env.ASSERTLEDGER_QUALIFICATION_NONCE,protocolVersion:'1.0.0'})+'\\n');\nprocess.exit(7);\n" + } + ], + "discriminants": [] + } + ], + "subject": { + "commit": "corpus-commit", + "baseCommit": "corpus-base", + "candidateDigest": "sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "inputDigest": "sha256:f1f362d13449b68d9f7b76b964642479fb04326e787bd33ed7d5cc1c8378ec4c" + }, + "tools": [ + { + "id": "node", + "version": "22.23.2", + "digest": "sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4" + } + ], + "allowedCandidatePaths": [ + "leaf.mjs" + ] + }, + "planDigest": "sha256:42f1fa806bd7d111060279b2f45ef68b88be1f6ba3ffe63fbd8fcf22f3ed7194", + "candidate": { + "files": [] + } +} diff --git a/tests/replay-corpus/inputs/qualification-1.5.0-propagation/source/leaf.mjs b/tests/replay-corpus/inputs/qualification-1.5.0-propagation/source/leaf.mjs new file mode 100644 index 0000000..f1e3f53 --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-1.5.0-propagation/source/leaf.mjs @@ -0,0 +1,3 @@ +import {writeFileSync} from 'node:fs'; +writeFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify({facts:{commandOutcome:'EXPECTED_FAILURE',exitCode:7},nonce:process.env.ASSERTLEDGER_QUALIFICATION_NONCE,protocolVersion:'1.0.0'})+'\n'); +process.exit(7); diff --git a/tests/replay-corpus/inputs/qualification-2.1.0-node-test/request.json b/tests/replay-corpus/inputs/qualification-2.1.0-node-test/request.json new file mode 100644 index 0000000..fc0e7db --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-2.1.0-node-test/request.json @@ -0,0 +1,117 @@ +{ + "root": "./source", + "plan": { + "schemaVersion": "1.0.0", + "requiredAttempts": 2, + "timeoutMs": 5000, + "maximumOutputBytes": 65536, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true + }, + "profileId": "corpus-node-test-v1", + "suites": [ + { + "id": "public", + "files": [ + "public.test.mjs" + ], + "extraction": "COMPLETE" + } + ], + "obligations": [ + { + "id": "tests", + "kind": "tests", + "required": true, + "suiteIds": [ + "public" + ], + "checks": [ + { + "id": "outcome", + "actionId": "test", + "field": "testOutcome", + "expected": "PASS" + } + ], + "limits": [] + } + ], + "actions": [ + { + "id": "test", + "adapter": "node-test", + "executable": "C:\\nvm4w\\nodejs\\node.exe", + "arguments": [ + "public.test.mjs" + ], + "environment": {}, + "prepareFiles": [], + "removePaths": [], + "observe": { + "trace": false, + "outputs": [], + "report": null + } + } + ], + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "files": [], + "discriminants": [] + }, + { + "id": "assertion-fault", + "kind": "TARGET", + "files": [ + { + "path": "public.test.mjs", + "content": "import assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 2));\n" + } + ], + "discriminants": [ + { + "obligationId": "tests", + "checkIds": [ + "outcome" + ] + } + ] + }, + { + "id": "neutral", + "kind": "NEUTRAL", + "files": [ + { + "path": "public.test.mjs", + "content": "// neutral comment\nimport assert from \"node:assert/strict\";\nimport test from \"node:test\";\ntest(\"public qualification\", () => assert.equal(1, 1));\n" + } + ], + "discriminants": [] + } + ], + "subject": { + "commit": "corpus-commit", + "baseCommit": "corpus-base", + "candidateDigest": "sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "inputDigest": "sha256:b82512cb17861d7889b7fd96d35fa764b213a6a743e44f14dd10ffcba28241d1" + }, + "tools": [ + { + "id": "node", + "version": "22.23.2", + "digest": "sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4" + } + ], + "allowedCandidatePaths": [ + "public.test.mjs" + ] + }, + "planDigest": "sha256:f4aa7f4ba6636c21a88e8266d9b75bfc38526ecbe0026638d13a95a1f89ea0e2", + "candidate": { + "files": [] + } +} diff --git a/tests/replay-corpus/inputs/qualification-2.1.0-node-test/source/public.test.mjs b/tests/replay-corpus/inputs/qualification-2.1.0-node-test/source/public.test.mjs new file mode 100644 index 0000000..ce194c9 --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-2.1.0-node-test/source/public.test.mjs @@ -0,0 +1,3 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +test("public qualification", () => assert.equal(1, 1)); diff --git a/tests/replay-corpus/inputs/qualification-2.1.0-propagation/request.json b/tests/replay-corpus/inputs/qualification-2.1.0-propagation/request.json new file mode 100644 index 0000000..ffb71f4 --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-2.1.0-propagation/request.json @@ -0,0 +1,107 @@ +{ + "root": "./source", + "plan": { + "schemaVersion": "1.0.0", + "requiredAttempts": 2, + "timeoutMs": 5000, + "maximumOutputBytes": 65536, + "isolation": { + "kind": "trusted-local", + "acknowledgedUnsafeExecution": true + }, + "profileId": "corpus-nonzero-command-v1", + "suites": [], + "obligations": [ + { + "id": "propagation", + "kind": "propagation", + "required": true, + "suiteIds": [], + "checks": [ + { + "id": "terminal", + "actionId": "leaf", + "field": "exitCode", + "expected": 7 + } + ], + "limits": [] + } + ], + "actions": [ + { + "id": "leaf", + "adapter": "command", + "executable": "C:\\nvm4w\\nodejs\\node.exe", + "arguments": [ + "leaf.mjs" + ], + "environment": {}, + "prepareFiles": [], + "removePaths": [], + "observe": { + "trace": false, + "outputs": [], + "report": null + } + } + ], + "worlds": [ + { + "id": "reference", + "kind": "REFERENCE", + "files": [], + "discriminants": [] + }, + { + "id": "swallowed", + "kind": "TARGET", + "files": [ + { + "path": "leaf.mjs", + "content": "process.exit(0);\n" + } + ], + "discriminants": [ + { + "obligationId": "propagation", + "checkIds": [ + "terminal" + ] + } + ] + }, + { + "id": "neutral", + "kind": "NEUTRAL", + "files": [ + { + "path": "leaf.mjs", + "content": "// harmless\nimport {writeFileSync} from 'node:fs';\nwriteFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify({facts:{commandOutcome:'EXPECTED_FAILURE',exitCode:7},nonce:process.env.ASSERTLEDGER_QUALIFICATION_NONCE,protocolVersion:'1.0.0'})+'\\n');\nprocess.exit(7);\n" + } + ], + "discriminants": [] + } + ], + "subject": { + "commit": "corpus-commit", + "baseCommit": "corpus-base", + "candidateDigest": "sha256:4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945", + "inputDigest": "sha256:f1f362d13449b68d9f7b76b964642479fb04326e787bd33ed7d5cc1c8378ec4c" + }, + "tools": [ + { + "id": "node", + "version": "22.23.2", + "digest": "sha256:0d0f5e39f9f3d9587bc19f73eab3c2c9c4903fd02d6dbf9c853dd81b3d95fad4" + } + ], + "allowedCandidatePaths": [ + "leaf.mjs" + ] + }, + "planDigest": "sha256:42f1fa806bd7d111060279b2f45ef68b88be1f6ba3ffe63fbd8fcf22f3ed7194", + "candidate": { + "files": [] + } +} diff --git a/tests/replay-corpus/inputs/qualification-2.1.0-propagation/source/leaf.mjs b/tests/replay-corpus/inputs/qualification-2.1.0-propagation/source/leaf.mjs new file mode 100644 index 0000000..f1e3f53 --- /dev/null +++ b/tests/replay-corpus/inputs/qualification-2.1.0-propagation/source/leaf.mjs @@ -0,0 +1,3 @@ +import {writeFileSync} from 'node:fs'; +writeFileSync(process.env.ASSERTLEDGER_QUALIFICATION_RESULT_FILE, JSON.stringify({facts:{commandOutcome:'EXPECTED_FAILURE',exitCode:7},nonce:process.env.ASSERTLEDGER_QUALIFICATION_NONCE,protocolVersion:'1.0.0'})+'\n'); +process.exit(7); diff --git a/tsconfig.json b/tsconfig.json index 8699b33..bad61d5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,5 +17,6 @@ "resolveJsonModule": true, "types": ["node"] }, - "include": ["src/**/*.ts", "tests/**/*.ts"] + "include": ["src/**/*.ts", "tests/**/*.ts"], + "exclude": ["tests/replay-corpus"] }