diff --git a/repository-release-note-claim-guard/README.md b/repository-release-note-claim-guard/README.md new file mode 100644 index 00000000..ba85795a --- /dev/null +++ b/repository-release-note-claim-guard/README.md @@ -0,0 +1,36 @@ +# Repository Release Note Claim Guard + +This module is a focused Project Repository & Version Control slice for SCIBASE issue #10. It reviews tagged repository release packets before public release notes, citation badges, API exports, or archive bundles are published. + +The guard checks that every public release-note claim is backed by: + +- a concrete changed repository artifact +- passed evidence of the right kind for the claim type +- path-specific evidence coverage +- fresh evidence within policy +- export-manifest parity for release-note ids and changed artifact digests +- explicit disclosure for breaking changes + +It is intentionally separate from broader repository ledgers, semantic version-tag governance, release signatures, merge queues, branch protection, component-owner approvals, external reference pinning, notebook diffs, fork provenance, restore rehearsals, compute sandbox policy, and review-decision provenance. This slice focuses only on whether public release notes overstate, omit, or misrepresent the actual release evidence. + +## Reviewer Path + +```bash +npm run check +npm test +npm run demo +npm run verify-video +``` + +Generated reviewer artifacts: + +- `reports/clean-packet.json` +- `reports/risky-packet.json` +- `reports/release-note-claim-report.md` +- `reports/summary.svg` +- `reports/demo-script.txt` +- `reports/demo.mp4` + +## Safety + +All fixtures are synthetic. The module does not call Git providers, CI systems, DOI registries, object stores, private repositories, payment processors, payout accounts, credential stores, or external APIs. diff --git a/repository-release-note-claim-guard/demo.js b/repository-release-note-claim-guard/demo.js new file mode 100644 index 00000000..04b9181e --- /dev/null +++ b/repository-release-note-claim-guard/demo.js @@ -0,0 +1,50 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { evaluateReleasePacket, renderMarkdownReport, renderSvgSummary } = require("./index"); +const { cleanReleasePacket, riskyReleasePacket } = require("./sample-data"); + +const reportsDir = path.join(__dirname, "reports"); +fs.mkdirSync(reportsDir, { recursive: true }); + +const cleanEvaluation = evaluateReleasePacket(cleanReleasePacket); +const riskyEvaluation = evaluateReleasePacket(riskyReleasePacket); + +fs.writeFileSync( + path.join(reportsDir, "clean-packet.json"), + `${JSON.stringify({ input: cleanReleasePacket, evaluation: cleanEvaluation }, null, 2)}\n` +); +fs.writeFileSync( + path.join(reportsDir, "risky-packet.json"), + `${JSON.stringify({ input: riskyReleasePacket, evaluation: riskyEvaluation }, null, 2)}\n` +); +fs.writeFileSync( + path.join(reportsDir, "release-note-claim-report.md"), + renderMarkdownReport(riskyReleasePacket, riskyEvaluation) +); +fs.writeFileSync( + path.join(reportsDir, "summary.svg"), + renderSvgSummary(riskyEvaluation) +); +fs.writeFileSync( + path.join(reportsDir, "demo-script.txt"), + [ + "Repository release-note claim evidence guard demo", + "", + "1. Clean packet: all release-note claims bind to changed artifacts, passed evidence, and export-manifest digests.", + ` Decision: ${cleanEvaluation.summary.decision}`, + ` Digest: ${cleanEvaluation.summary.auditDigest}`, + "", + "2. Risky packet: public release notes overclaim reproducibility, hide a breaking API change, cite a non-existent public dataset path, and omit changed data from the export manifest.", + ` Decision: ${riskyEvaluation.summary.decision}`, + ` Findings: ${riskyEvaluation.summary.findingCount}`, + ` Digest: ${riskyEvaluation.summary.auditDigest}`, + "" + ].join("\n") +); + +console.log(JSON.stringify({ + cleanDecision: cleanEvaluation.summary.decision, + riskyDecision: riskyEvaluation.summary.decision, + riskyFindings: riskyEvaluation.summary.findingCount, + report: "reports/release-note-claim-report.md" +}, null, 2)); diff --git a/repository-release-note-claim-guard/index.js b/repository-release-note-claim-guard/index.js new file mode 100644 index 00000000..91a9ed8a --- /dev/null +++ b/repository-release-note-claim-guard/index.js @@ -0,0 +1,528 @@ +const crypto = require("node:crypto"); + +const DAY_MS = 24 * 60 * 60 * 1000; + +const REQUIRED_EVIDENCE_BY_CLAIM = { + analysis: ["test", "reproduction"], + api: ["test", "export"], + breaking_change: ["review", "export"], + citation: ["citation", "export"], + code: ["test"], + dataset: ["data-diff", "checksum"], + figure: ["reproduction", "checksum"], + metadata: ["metadata", "export"], + performance: ["benchmark"], + reproducibility: ["reproduction"] +}; + +const HIGH_RISK_WORDING = [ + { pattern: /\bfully reproducible\b/i, code: "REPRODUCIBILITY_OVERCLAIM", kind: "reproduction" }, + { pattern: /\ball results\b/i, code: "ALL_RESULTS_OVERCLAIM", kind: "reproduction" }, + { pattern: /\bvalidated\b/i, code: "VALIDATION_OVERCLAIM", kind: "test" }, + { pattern: /\bno breaking changes?\b/i, code: "NO_BREAKING_CHANGE_OVERCLAIM", kind: "review" } +]; + +function normalizePath(value) { + return String(value || "") + .replace(/\\/g, "/") + .replace(/\/+/g, "/") + .replace(/^\.\//, "") + .replace(/\/$/, ""); +} + +function asArray(value) { + return Array.isArray(value) ? value : []; +} + +function stableJson(value) { + if (Array.isArray(value)) { + return `[${value.map(stableJson).join(",")}]`; + } + if (value && typeof value === "object") { + return `{${Object.keys(value) + .sort() + .map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`) + .join(",")}}`; + } + return JSON.stringify(value); +} + +function sha256(value) { + return crypto.createHash("sha256").update(stableJson(value)).digest("hex"); +} + +function dateAgeDays(reviewedAt, producedAt) { + const reviewed = new Date(reviewedAt); + const produced = new Date(producedAt); + if (Number.isNaN(reviewed.valueOf()) || Number.isNaN(produced.valueOf())) { + return Number.POSITIVE_INFINITY; + } + return Math.max(0, Math.ceil((reviewed.valueOf() - produced.valueOf()) / DAY_MS)); +} + +function pathMatches(path, coveredPath) { + const left = normalizePath(path); + const right = normalizePath(coveredPath); + if (!left || !right) { + return false; + } + return ( + left === right || + left.startsWith(`${right}/`) || + right.startsWith(`${left}/`) + ); +} + +function buildMaps(packet) { + const changes = asArray(packet.changeSet).map((change) => ({ + ...change, + path: normalizePath(change.path) + })); + const notes = asArray(packet.releaseNotes).map((note) => ({ + ...note, + componentPaths: asArray(note.componentPaths).map(normalizePath), + evidenceIds: asArray(note.evidenceIds) + })); + const evidence = asArray(packet.evidence).map((item) => ({ + ...item, + coversPaths: asArray(item.coversPaths).map(normalizePath) + })); + + return { + changes, + notes, + evidence, + noteById: new Map(notes.map((note) => [note.id, note])), + evidenceById: new Map(evidence.map((item) => [item.id, item])) + }; +} + +function evidenceCoversPath(evidence, path) { + return asArray(evidence.coversPaths).some((coveredPath) => pathMatches(path, coveredPath)); +} + +function noteMentionsChange(note, change) { + return ( + asArray(change.noteIds).includes(note.id) || + asArray(note.componentPaths).some((path) => pathMatches(change.path, path)) + ); +} + +function hasKind(noteEvidence, kind) { + return noteEvidence.some((item) => item && item.kind === kind && item.status === "passed"); +} + +function addFinding(findings, severity, code, message, refs, action) { + findings.push({ + severity, + code, + message, + refs: asArray(refs), + action + }); +} + +function severityRank(severity) { + return { critical: 4, high: 3, medium: 2, low: 1 }[severity] || 0; +} + +function evaluateReleasePacket(packet, options = {}) { + const reviewedAt = options.reviewedAt || packet.reviewedAt || new Date().toISOString(); + const policy = { + maxEvidenceAgeDays: 45, + requireExportManifest: true, + ...packet.policy, + ...options.policy + }; + const { changes, notes, evidence, noteById, evidenceById } = buildMaps(packet); + const findings = []; + const evidenceCoverage = []; + + for (const note of notes) { + if (!note.id) { + addFinding( + findings, + "high", + "RELEASE_NOTE_MISSING_ID", + "A public release-note claim is missing a stable note id.", + [], + "assign_release_note_id" + ); + continue; + } + + if (!String(note.text || "").trim()) { + addFinding( + findings, + "high", + "RELEASE_NOTE_EMPTY_TEXT", + `Release note ${note.id} has no reviewer-visible claim text.`, + [note.id], + "write_public_release_note_claim" + ); + } + + if (note.componentPaths.length === 0) { + addFinding( + findings, + "medium", + "RELEASE_NOTE_UNSCOPED", + `Release note ${note.id} is not scoped to repository components or paths.`, + [note.id], + "bind_claim_to_changed_components" + ); + } + + const noteChanges = changes.filter((change) => noteMentionsChange(note, change)); + for (const componentPath of note.componentPaths) { + if (!changes.some((change) => pathMatches(change.path, componentPath))) { + addFinding( + findings, + "high", + "CLAIMED_COMPONENT_WITHOUT_CHANGE", + `Release note ${note.id} names ${componentPath}, but the release packet has no matching changed artifact.`, + [note.id, componentPath], + "remove_claim_or_attach_changed_artifact" + ); + } + } + + const noteEvidence = note.evidenceIds.map((id) => evidenceById.get(id)); + for (const evidenceId of note.evidenceIds) { + const item = evidenceById.get(evidenceId); + if (!item) { + addFinding( + findings, + "high", + "CLAIM_EVIDENCE_MISSING", + `Release note ${note.id} references missing evidence ${evidenceId}.`, + [note.id, evidenceId], + "attach_evidence_or_remove_claim" + ); + continue; + } + + const age = dateAgeDays(reviewedAt, item.producedAt); + if (age > policy.maxEvidenceAgeDays) { + addFinding( + findings, + "medium", + "CLAIM_EVIDENCE_STALE", + `Evidence ${item.id} for release note ${note.id} is ${age} days old.`, + [note.id, item.id], + "refresh_release_evidence" + ); + } + + if (item.status !== "passed") { + addFinding( + findings, + "high", + "CLAIM_EVIDENCE_NOT_PASSED", + `Evidence ${item.id} for release note ${note.id} has status ${item.status || "unknown"}.`, + [note.id, item.id], + "replace_or_rerun_failed_evidence" + ); + } + } + + const requiredKinds = REQUIRED_EVIDENCE_BY_CLAIM[note.claimType] || ["review"]; + for (const kind of requiredKinds) { + if (!hasKind(noteEvidence, kind)) { + addFinding( + findings, + "high", + "CLAIM_KIND_EVIDENCE_MISSING", + `Release note ${note.id} is a ${note.claimType || "general"} claim but lacks passed ${kind} evidence.`, + [note.id, kind], + "attach_required_claim_evidence" + ); + } + } + + for (const componentPath of note.componentPaths) { + const covered = noteEvidence.some((item) => item && item.status === "passed" && evidenceCoversPath(item, componentPath)); + if (!covered) { + addFinding( + findings, + "medium", + "CLAIM_PATH_NOT_COVERED_BY_EVIDENCE", + `Release note ${note.id} cites ${componentPath}, but no passed evidence covers that path.`, + [note.id, componentPath], + "add_path_specific_release_evidence" + ); + } + } + + for (const wording of HIGH_RISK_WORDING) { + if (wording.pattern.test(note.text || "") && !hasKind(noteEvidence, wording.kind)) { + addFinding( + findings, + "high", + wording.code, + `Release note ${note.id} uses high-confidence wording without passed ${wording.kind} evidence.`, + [note.id], + "downgrade_claim_or_add_evidence" + ); + } + } + + if (/no breaking changes?/i.test(note.text || "")) { + const breakingChanges = changes.filter((change) => change.breaking === true); + if (breakingChanges.length > 0) { + addFinding( + findings, + "critical", + "NO_BREAKING_CHANGE_CONFLICT", + `Release note ${note.id} says there are no breaking changes, but ${breakingChanges.length} changed artifact(s) are marked breaking.`, + [note.id, ...breakingChanges.map((change) => change.id || change.path)], + "remove_no_breaking_claim_or_document_breaking_change" + ); + } + } + + evidenceCoverage.push({ + noteId: note.id, + claimType: note.claimType || "general", + changedArtifacts: noteChanges.length, + evidenceIds: note.evidenceIds, + passedEvidence: noteEvidence.filter((item) => item && item.status === "passed").length, + requiredKinds + }); + } + + for (const change of changes) { + const mentions = notes.filter((note) => noteMentionsChange(note, change)); + if (mentions.length === 0 && change.publicImpact !== false) { + addFinding( + findings, + change.breaking ? "high" : "medium", + "CHANGED_ARTIFACT_UNMENTIONED", + `Changed artifact ${change.path} is not represented in any public release note.`, + [change.id || change.path], + "add_release_note_or_mark_internal_only" + ); + } + + if (!change.digestAfter || change.digestBefore === change.digestAfter) { + addFinding( + findings, + "medium", + "CHANGE_DIGEST_NOT_UPDATED", + `Changed artifact ${change.path} does not carry a new post-release digest.`, + [change.id || change.path], + "record_post_release_digest" + ); + } + + if (change.breaking && !notes.some((note) => note.claimType === "breaking_change" && noteMentionsChange(note, change))) { + addFinding( + findings, + "high", + "BREAKING_CHANGE_NOT_DISCLOSED", + `Breaking artifact ${change.path} has no explicit breaking-change release note.`, + [change.id || change.path], + "document_breaking_change_in_release_notes" + ); + } + } + + const manifest = packet.exportManifest || {}; + if (policy.requireExportManifest && !packet.exportManifest) { + addFinding( + findings, + "high", + "EXPORT_MANIFEST_MISSING", + "The release packet has no export manifest for release-note, citation, and artifact parity checks.", + [], + "attach_export_manifest" + ); + } else if (packet.exportManifest) { + for (const note of notes) { + if (!asArray(manifest.releaseNoteIds).includes(note.id)) { + addFinding( + findings, + "medium", + "EXPORT_MANIFEST_MISSING_RELEASE_NOTE", + `Export manifest does not list public release note ${note.id}.`, + [note.id], + "add_release_note_to_export_manifest" + ); + } + } + + const manifestDigests = manifest.componentDigests || {}; + for (const change of changes) { + if (change.publicImpact === false) { + continue; + } + const expectedDigest = manifestDigests[change.path]; + if (expectedDigest !== change.digestAfter) { + addFinding( + findings, + "high", + "EXPORT_DIGEST_MISMATCH", + `Export manifest digest for ${change.path} does not match the changed artifact digest.`, + [change.path], + "update_export_manifest_digest" + ); + } + } + + if (manifest.versionTag && packet.versionTag && manifest.versionTag !== packet.versionTag) { + addFinding( + findings, + "medium", + "EXPORT_VERSION_TAG_MISMATCH", + `Export manifest tag ${manifest.versionTag} does not match release tag ${packet.versionTag}.`, + [manifest.versionTag, packet.versionTag], + "align_export_manifest_version_tag" + ); + } + } + + findings.sort((a, b) => severityRank(b.severity) - severityRank(a.severity) || a.code.localeCompare(b.code)); + const topSeverity = findings[0] ? findings[0].severity : "none"; + const decision = findings.some((finding) => severityRank(finding.severity) >= 3) + ? "hold_repository_release" + : findings.some((finding) => finding.severity === "medium") + ? "revise_release_notes" + : "publish_release_notes"; + + const summary = { + repositoryId: packet.repositoryId, + versionTag: packet.versionTag, + reviewedAt, + decision, + topSeverity, + releaseNotesReviewed: notes.length, + changedArtifactsReviewed: changes.length, + evidenceItemsReviewed: evidence.length, + findingCount: findings.length, + highOrCriticalFindings: findings.filter((finding) => severityRank(finding.severity) >= 3).length + }; + + const auditDigest = `sha256:${sha256({ summary, findings, evidenceCoverage }).slice(0, 16)}`; + + return { + summary: { + ...summary, + auditDigest + }, + evidenceCoverage, + findings, + actions: buildActions(findings) + }; +} + +function buildActions(findings) { + const actions = []; + const seen = new Set(); + for (const finding of findings) { + if (!finding.action || seen.has(finding.action)) { + continue; + } + seen.add(finding.action); + actions.push({ + id: finding.action, + severity: finding.severity, + refs: finding.refs + }); + } + return actions; +} + +function renderMarkdownReport(packet, evaluation) { + const lines = []; + lines.push(`# Release Note Claim Evidence Review: ${packet.versionTag}`); + lines.push(""); + lines.push(`Decision: **${evaluation.summary.decision}**`); + lines.push(`Audit digest: \`${evaluation.summary.auditDigest}\``); + lines.push(""); + lines.push("## Summary"); + lines.push(""); + lines.push("| Metric | Value |"); + lines.push("| --- | ---: |"); + lines.push(`| Release notes reviewed | ${evaluation.summary.releaseNotesReviewed} |`); + lines.push(`| Changed artifacts reviewed | ${evaluation.summary.changedArtifactsReviewed} |`); + lines.push(`| Evidence items reviewed | ${evaluation.summary.evidenceItemsReviewed} |`); + lines.push(`| Findings | ${evaluation.summary.findingCount} |`); + lines.push(`| High or critical findings | ${evaluation.summary.highOrCriticalFindings} |`); + lines.push(""); + lines.push("## Findings"); + lines.push(""); + if (evaluation.findings.length === 0) { + lines.push("No release-blocking findings were detected."); + } else { + lines.push("| Severity | Code | Message | Action |"); + lines.push("| --- | --- | --- | --- |"); + for (const finding of evaluation.findings) { + lines.push( + `| ${finding.severity} | \`${finding.code}\` | ${escapeMarkdown(finding.message)} | \`${finding.action}\` |` + ); + } + } + lines.push(""); + lines.push("## Evidence Coverage"); + lines.push(""); + lines.push("| Release note | Claim type | Changed artifacts | Passed evidence | Required evidence |"); + lines.push("| --- | --- | ---: | ---: | --- |"); + for (const item of evaluation.evidenceCoverage) { + lines.push( + `| ${item.noteId} | ${item.claimType} | ${item.changedArtifacts} | ${item.passedEvidence} | ${item.requiredKinds.join(", ")} |` + ); + } + lines.push(""); + lines.push("Synthetic data only. No Git provider, DOI provider, CI system, private repository, payment account, credential, or external API is contacted."); + return `${lines.join("\n")}\n`; +} + +function escapeMarkdown(value) { + return String(value).replace(/\|/g, "\\|").replace(/\n/g, " "); +} + +function renderSvgSummary(evaluation) { + const isHold = evaluation.summary.decision === "hold_repository_release"; + const color = isHold ? "#b91c1c" : evaluation.summary.decision === "revise_release_notes" ? "#b45309" : "#047857"; + const safeDecision = escapeXml(evaluation.summary.decision); + const safeDigest = escapeXml(evaluation.summary.auditDigest); + const findings = evaluation.findings.slice(0, 5); + const rows = findings.map((finding, index) => { + const y = 205 + index * 42; + return `${escapeXml(finding.severity.toUpperCase())} ${escapeXml(finding.code)}`; + }).join("\n"); + + return ` + + + + Repository Release Note Claim Guard + + ${safeDecision} + Findings: ${evaluation.summary.findingCount} + High or critical: ${evaluation.summary.highOrCriticalFindings} + Audit digest: ${safeDigest} + + Top Findings + +${rows || 'No findings detected.'} + + +`; +} + +function escapeXml(value) { + return String(value) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +module.exports = { + REQUIRED_EVIDENCE_BY_CLAIM, + evaluateReleasePacket, + renderMarkdownReport, + renderSvgSummary, + sha256 +}; diff --git a/repository-release-note-claim-guard/make-demo-video.js b/repository-release-note-claim-guard/make-demo-video.js new file mode 100644 index 00000000..a4e24c90 --- /dev/null +++ b/repository-release-note-claim-guard/make-demo-video.js @@ -0,0 +1,91 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); +const { evaluateReleasePacket } = require("./index"); +const { cleanReleasePacket, riskyReleasePacket } = require("./sample-data"); + +const reportsDir = path.join(__dirname, "reports"); +const framesDir = path.join(reportsDir, "frames"); +fs.mkdirSync(framesDir, { recursive: true }); + +const clean = evaluateReleasePacket(cleanReleasePacket); +const risky = evaluateReleasePacket(riskyReleasePacket); +const width = 960; +const height = 540; +const frames = 72; +const fps = 18; + +function setPixel(buffer, x, y, r, g, b) { + if (x < 0 || y < 0 || x >= width || y >= height) { + return; + } + const offset = (y * width + x) * 3; + buffer[offset] = r; + buffer[offset + 1] = g; + buffer[offset + 2] = b; +} + +function fillRect(buffer, x, y, w, h, r, g, b) { + for (let row = y; row < y + h; row += 1) { + for (let col = x; col < x + w; col += 1) { + setPixel(buffer, col, row, r, g, b); + } + } +} + +function writeFrame(index, progress) { + const buffer = Buffer.alloc(width * height * 3, 248); + fillRect(buffer, 0, 0, width, height, 248, 250, 252); + fillRect(buffer, 48, 48, 864, 444, 255, 255, 255); + fillRect(buffer, 48, 48, 864, 8, 17, 24, 39); + + const leftBar = Math.floor(340 * Math.min(1, progress * 1.8)); + const rightBar = Math.floor(340 * Math.max(0, (progress - 0.35) * 1.55)); + + fillRect(buffer, 96, 118, 340, 84, 229, 231, 235); + fillRect(buffer, 96, 118, leftBar, 84, 5, 150, 105); + fillRect(buffer, 524, 118, 340, 84, 229, 231, 235); + fillRect(buffer, 524, 118, rightBar, 84, 185, 28, 28); + + const cleanFindings = Math.max(1, clean.summary.releaseNotesReviewed); + for (let i = 0; i < cleanFindings; i += 1) { + fillRect(buffer, 110 + i * 58, 242, 42, 150, 16, 185, 129); + } + + const riskyBars = Math.min(10, risky.summary.findingCount); + for (let i = 0; i < riskyBars; i += 1) { + const barHeight = 42 + (i % 5) * 24; + fillRect(buffer, 540 + i * 28, 392 - barHeight, 20, barHeight, 220, 38, 38); + } + + fillRect(buffer, 96, 430, Math.floor(768 * progress), 18, 37, 99, 235); + const header = Buffer.from(`P6\n${width} ${height}\n255\n`, "ascii"); + fs.writeFileSync(path.join(framesDir, `frame-${String(index).padStart(3, "0")}.ppm`), Buffer.concat([header, buffer])); +} + +for (let index = 0; index < frames; index += 1) { + writeFrame(index, index / (frames - 1)); +} + +const output = path.join(reportsDir, "demo.mp4"); +const ffmpeg = process.env.FFMPEG_PATH || "ffmpeg"; +const result = spawnSync(ffmpeg, [ + "-y", + "-framerate", + String(fps), + "-i", + path.join(framesDir, "frame-%03d.ppm"), + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + output +], { stdio: "inherit" }); + +fs.rmSync(framesDir, { recursive: true, force: true }); + +if (result.status !== 0) { + process.exit(result.status || 1); +} + +console.log(`Wrote ${output}`); diff --git a/repository-release-note-claim-guard/package.json b/repository-release-note-claim-guard/package.json new file mode 100644 index 00000000..154a792d --- /dev/null +++ b/repository-release-note-claim-guard/package.json @@ -0,0 +1,21 @@ +{ + "name": "repository-release-note-claim-guard", + "version": "1.0.0", + "private": true, + "description": "Synthetic release-note claim evidence guard for SCIBASE project repositories.", + "main": "index.js", + "scripts": { + "check": "node --check index.js && node --check sample-data.js && node --check test.js && node --check demo.js && node --check make-demo-video.js", + "test": "node test.js", + "demo": "node demo.js && node make-demo-video.js", + "verify-video": "ffprobe -v error -select_streams v:0 -show_entries stream=codec_name,width,height,duration,avg_frame_rate -show_entries format=duration,size -of default=noprint_wrappers=1 reports/demo.mp4" + }, + "keywords": [ + "scibase", + "repository", + "version-control", + "release-notes", + "evidence" + ], + "license": "MIT" +} diff --git a/repository-release-note-claim-guard/reports/clean-packet.json b/repository-release-note-claim-guard/reports/clean-packet.json new file mode 100644 index 00000000..c1243abf --- /dev/null +++ b/repository-release-note-claim-guard/reports/clean-packet.json @@ -0,0 +1,206 @@ +{ + "input": { + "repositoryId": "scibase-demo-repo", + "versionTag": "preprint-v2.1.0", + "reviewedAt": "2026-06-01T12:00:00Z", + "policy": { + "maxEvidenceAgeDays": 30 + }, + "changeSet": [ + { + "id": "chg-analysis-script", + "path": "code/analysis/model-fit.py", + "component": "code", + "digestBefore": "sha256:1111", + "digestAfter": "sha256:2222", + "noteIds": [ + "rn-001" + ], + "publicImpact": true + }, + { + "id": "chg-results-figure", + "path": "results/figures/figure-2.svg", + "component": "results", + "digestBefore": "sha256:3333", + "digestAfter": "sha256:4444", + "noteIds": [ + "rn-002" + ], + "publicImpact": true + }, + { + "id": "chg-metadata", + "path": "metadata.json", + "component": "metadata", + "digestBefore": "sha256:5555", + "digestAfter": "sha256:6666", + "noteIds": [ + "rn-003" + ], + "publicImpact": true + } + ], + "releaseNotes": [ + { + "id": "rn-001", + "claimType": "code", + "text": "Updated the mixed-effects model fitting script and validated it with repository tests.", + "componentPaths": [ + "code/analysis/model-fit.py" + ], + "evidenceIds": [ + "ev-tests-model" + ] + }, + { + "id": "rn-002", + "claimType": "figure", + "text": "Regenerated Figure 2 from the refreshed analysis output with matching checksums.", + "componentPaths": [ + "results/figures/figure-2.svg" + ], + "evidenceIds": [ + "ev-repro-figure", + "ev-checksum-figure" + ] + }, + { + "id": "rn-003", + "claimType": "metadata", + "text": "Aligned repository metadata and export manifest with the preprint-v2.1.0 citation package.", + "componentPaths": [ + "metadata.json" + ], + "evidenceIds": [ + "ev-export-manifest", + "ev-metadata-review" + ] + } + ], + "evidence": [ + { + "id": "ev-tests-model", + "kind": "test", + "status": "passed", + "command": "npm run repository-tests", + "producedAt": "2026-05-31T15:00:00Z", + "coversPaths": [ + "code/analysis/model-fit.py" + ] + }, + { + "id": "ev-repro-figure", + "kind": "reproduction", + "status": "passed", + "command": "python code/analysis/model-fit.py --rebuild-figures", + "producedAt": "2026-05-31T15:20:00Z", + "coversPaths": [ + "code/analysis/model-fit.py", + "results/figures/figure-2.svg" + ] + }, + { + "id": "ev-checksum-figure", + "kind": "checksum", + "status": "passed", + "producedAt": "2026-05-31T15:25:00Z", + "coversPaths": [ + "results/figures/figure-2.svg" + ] + }, + { + "id": "ev-export-manifest", + "kind": "export", + "status": "passed", + "producedAt": "2026-05-31T16:00:00Z", + "coversPaths": [ + "metadata.json", + "code/analysis/model-fit.py", + "results/figures/figure-2.svg" + ] + }, + { + "id": "ev-metadata-review", + "kind": "metadata", + "status": "passed", + "producedAt": "2026-05-31T16:05:00Z", + "coversPaths": [ + "metadata.json" + ] + } + ], + "exportManifest": { + "versionTag": "preprint-v2.1.0", + "releaseNoteIds": [ + "rn-001", + "rn-002", + "rn-003" + ], + "componentDigests": { + "code/analysis/model-fit.py": "sha256:2222", + "results/figures/figure-2.svg": "sha256:4444", + "metadata.json": "sha256:6666" + } + } + }, + "evaluation": { + "summary": { + "repositoryId": "scibase-demo-repo", + "versionTag": "preprint-v2.1.0", + "reviewedAt": "2026-06-01T12:00:00Z", + "decision": "publish_release_notes", + "topSeverity": "none", + "releaseNotesReviewed": 3, + "changedArtifactsReviewed": 3, + "evidenceItemsReviewed": 5, + "findingCount": 0, + "highOrCriticalFindings": 0, + "auditDigest": "sha256:d845882abd6daba0" + }, + "evidenceCoverage": [ + { + "noteId": "rn-001", + "claimType": "code", + "changedArtifacts": 1, + "evidenceIds": [ + "ev-tests-model" + ], + "passedEvidence": 1, + "requiredKinds": [ + "test" + ] + }, + { + "noteId": "rn-002", + "claimType": "figure", + "changedArtifacts": 1, + "evidenceIds": [ + "ev-repro-figure", + "ev-checksum-figure" + ], + "passedEvidence": 2, + "requiredKinds": [ + "reproduction", + "checksum" + ] + }, + { + "noteId": "rn-003", + "claimType": "metadata", + "changedArtifacts": 1, + "evidenceIds": [ + "ev-export-manifest", + "ev-metadata-review" + ], + "passedEvidence": 2, + "requiredKinds": [ + "metadata", + "export" + ] + } + ], + "findings": [], + "actions": [] + } +} diff --git a/repository-release-note-claim-guard/reports/demo-script.txt b/repository-release-note-claim-guard/reports/demo-script.txt new file mode 100644 index 00000000..203da611 --- /dev/null +++ b/repository-release-note-claim-guard/reports/demo-script.txt @@ -0,0 +1,10 @@ +Repository release-note claim evidence guard demo + +1. Clean packet: all release-note claims bind to changed artifacts, passed evidence, and export-manifest digests. + Decision: publish_release_notes + Digest: sha256:d845882abd6daba0 + +2. Risky packet: public release notes overclaim reproducibility, hide a breaking API change, cite a non-existent public dataset path, and omit changed data from the export manifest. + Decision: hold_repository_release + Findings: 18 + Digest: sha256:8431cbaec2db5bd8 diff --git a/repository-release-note-claim-guard/reports/demo.mp4 b/repository-release-note-claim-guard/reports/demo.mp4 new file mode 100644 index 00000000..2a69e8c2 Binary files /dev/null and b/repository-release-note-claim-guard/reports/demo.mp4 differ diff --git a/repository-release-note-claim-guard/reports/release-note-claim-report.md b/repository-release-note-claim-guard/reports/release-note-claim-report.md new file mode 100644 index 00000000..b903dde9 --- /dev/null +++ b/repository-release-note-claim-guard/reports/release-note-claim-report.md @@ -0,0 +1,47 @@ +# Release Note Claim Evidence Review: preprint-v2.2.0 + +Decision: **hold_repository_release** +Audit digest: `sha256:8431cbaec2db5bd8` + +## Summary + +| Metric | Value | +| --- | ---: | +| Release notes reviewed | 3 | +| Changed artifacts reviewed | 3 | +| Evidence items reviewed | 3 | +| Findings | 18 | +| High or critical findings | 12 | + +## Findings + +| Severity | Code | Message | Action | +| --- | --- | --- | --- | +| critical | `NO_BREAKING_CHANGE_CONFLICT` | Release note rn-101 says there are no breaking changes, but 1 changed artifact(s) are marked breaking. | `remove_no_breaking_claim_or_document_breaking_change` | +| high | `ALL_RESULTS_OVERCLAIM` | Release note rn-102 uses high-confidence wording without passed reproduction evidence. | `downgrade_claim_or_add_evidence` | +| high | `BREAKING_CHANGE_NOT_DISCLOSED` | Breaking artifact code/export/public-api.js has no explicit breaking-change release note. | `document_breaking_change_in_release_notes` | +| high | `CLAIM_EVIDENCE_NOT_PASSED` | Evidence ev-old-repro for release note rn-102 has status failed. | `replace_or_rerun_failed_evidence` | +| high | `CLAIM_KIND_EVIDENCE_MISSING` | Release note rn-101 is a api claim but lacks passed test evidence. | `attach_required_claim_evidence` | +| high | `CLAIM_KIND_EVIDENCE_MISSING` | Release note rn-101 is a api claim but lacks passed export evidence. | `attach_required_claim_evidence` | +| high | `CLAIM_KIND_EVIDENCE_MISSING` | Release note rn-102 is a reproducibility claim but lacks passed reproduction evidence. | `attach_required_claim_evidence` | +| high | `CLAIM_KIND_EVIDENCE_MISSING` | Release note rn-103 is a dataset claim but lacks passed checksum evidence. | `attach_required_claim_evidence` | +| high | `CLAIMED_COMPONENT_WITHOUT_CHANGE` | Release note rn-103 names data/public/cohort-clean.csv, but the release packet has no matching changed artifact. | `remove_claim_or_attach_changed_artifact` | +| high | `EXPORT_DIGEST_MISMATCH` | Export manifest digest for data/restricted/cohort-clean.csv does not match the changed artifact digest. | `update_export_manifest_digest` | +| high | `REPRODUCIBILITY_OVERCLAIM` | Release note rn-102 uses high-confidence wording without passed reproduction evidence. | `downgrade_claim_or_add_evidence` | +| high | `VALIDATION_OVERCLAIM` | Release note rn-102 uses high-confidence wording without passed test evidence. | `downgrade_claim_or_add_evidence` | +| medium | `CHANGE_DIGEST_NOT_UPDATED` | Changed artifact results/figures/figure-5.svg does not carry a new post-release digest. | `record_post_release_digest` | +| medium | `CHANGED_ARTIFACT_UNMENTIONED` | Changed artifact data/restricted/cohort-clean.csv is not represented in any public release note. | `add_release_note_or_mark_internal_only` | +| medium | `CLAIM_EVIDENCE_STALE` | Evidence ev-old-repro for release note rn-102 is 93 days old. | `refresh_release_evidence` | +| medium | `CLAIM_PATH_NOT_COVERED_BY_EVIDENCE` | Release note rn-102 cites results/figures/figure-5.svg, but no passed evidence covers that path. | `add_path_specific_release_evidence` | +| medium | `CLAIM_PATH_NOT_COVERED_BY_EVIDENCE` | Release note rn-103 cites data/public/cohort-clean.csv, but no passed evidence covers that path. | `add_path_specific_release_evidence` | +| medium | `EXPORT_MANIFEST_MISSING_RELEASE_NOTE` | Export manifest does not list public release note rn-103. | `add_release_note_to_export_manifest` | + +## Evidence Coverage + +| Release note | Claim type | Changed artifacts | Passed evidence | Required evidence | +| --- | --- | ---: | ---: | --- | +| rn-101 | api | 1 | 1 | test, export | +| rn-102 | reproducibility | 1 | 0 | reproduction | +| rn-103 | dataset | 0 | 1 | data-diff, checksum | + +Synthetic data only. No Git provider, DOI provider, CI system, private repository, payment account, credential, or external API is contacted. diff --git a/repository-release-note-claim-guard/reports/risky-packet.json b/repository-release-note-claim-guard/reports/risky-packet.json new file mode 100644 index 00000000..2699c40a --- /dev/null +++ b/repository-release-note-claim-guard/reports/risky-packet.json @@ -0,0 +1,439 @@ +{ + "input": { + "repositoryId": "scibase-demo-repo", + "versionTag": "preprint-v2.2.0", + "reviewedAt": "2026-06-01T12:00:00Z", + "policy": { + "maxEvidenceAgeDays": 30 + }, + "changeSet": [ + { + "id": "chg-api-export", + "path": "code/export/public-api.js", + "component": "code", + "digestBefore": "sha256:aaaa", + "digestAfter": "sha256:bbbb", + "noteIds": [ + "rn-101" + ], + "publicImpact": true, + "breaking": true + }, + { + "id": "chg-sensitive-data", + "path": "data/restricted/cohort-clean.csv", + "component": "data", + "digestBefore": "sha256:cccc", + "digestAfter": "sha256:dddd", + "publicImpact": true + }, + { + "id": "chg-figure", + "path": "results/figures/figure-5.svg", + "component": "results", + "digestBefore": "sha256:eeee", + "digestAfter": "sha256:eeee", + "noteIds": [ + "rn-102" + ], + "publicImpact": true + } + ], + "releaseNotes": [ + { + "id": "rn-101", + "claimType": "api", + "text": "Updated the public API export with no breaking changes.", + "componentPaths": [ + "code/export/public-api.js" + ], + "evidenceIds": [ + "ev-api-review" + ] + }, + { + "id": "rn-102", + "claimType": "reproducibility", + "text": "All results are now fully reproducible and validated.", + "componentPaths": [ + "results/figures/figure-5.svg" + ], + "evidenceIds": [ + "ev-old-repro" + ] + }, + { + "id": "rn-103", + "claimType": "dataset", + "text": "Released a cleaned cohort dataset for reviewers.", + "componentPaths": [ + "data/public/cohort-clean.csv" + ], + "evidenceIds": [ + "ev-dataset-diff" + ] + } + ], + "evidence": [ + { + "id": "ev-api-review", + "kind": "review", + "status": "passed", + "producedAt": "2026-05-31T10:00:00Z", + "coversPaths": [ + "code/export/public-api.js" + ] + }, + { + "id": "ev-old-repro", + "kind": "reproduction", + "status": "failed", + "producedAt": "2026-03-01T10:00:00Z", + "coversPaths": [ + "results/figures/figure-5.svg" + ] + }, + { + "id": "ev-dataset-diff", + "kind": "data-diff", + "status": "passed", + "producedAt": "2026-05-31T12:00:00Z", + "coversPaths": [ + "data/restricted/cohort-clean.csv" + ] + } + ], + "exportManifest": { + "versionTag": "preprint-v2.2.0", + "releaseNoteIds": [ + "rn-101", + "rn-102" + ], + "componentDigests": { + "code/export/public-api.js": "sha256:bbbb", + "results/figures/figure-5.svg": "sha256:eeee" + } + } + }, + "evaluation": { + "summary": { + "repositoryId": "scibase-demo-repo", + "versionTag": "preprint-v2.2.0", + "reviewedAt": "2026-06-01T12:00:00Z", + "decision": "hold_repository_release", + "topSeverity": "critical", + "releaseNotesReviewed": 3, + "changedArtifactsReviewed": 3, + "evidenceItemsReviewed": 3, + "findingCount": 18, + "highOrCriticalFindings": 12, + "auditDigest": "sha256:8431cbaec2db5bd8" + }, + "evidenceCoverage": [ + { + "noteId": "rn-101", + "claimType": "api", + "changedArtifacts": 1, + "evidenceIds": [ + "ev-api-review" + ], + "passedEvidence": 1, + "requiredKinds": [ + "test", + "export" + ] + }, + { + "noteId": "rn-102", + "claimType": "reproducibility", + "changedArtifacts": 1, + "evidenceIds": [ + "ev-old-repro" + ], + "passedEvidence": 0, + "requiredKinds": [ + "reproduction" + ] + }, + { + "noteId": "rn-103", + "claimType": "dataset", + "changedArtifacts": 0, + "evidenceIds": [ + "ev-dataset-diff" + ], + "passedEvidence": 1, + "requiredKinds": [ + "data-diff", + "checksum" + ] + } + ], + "findings": [ + { + "severity": "critical", + "code": "NO_BREAKING_CHANGE_CONFLICT", + "message": "Release note rn-101 says there are no breaking changes, but 1 changed artifact(s) are marked breaking.", + "refs": [ + "rn-101", + "chg-api-export" + ], + "action": "remove_no_breaking_claim_or_document_breaking_change" + }, + { + "severity": "high", + "code": "ALL_RESULTS_OVERCLAIM", + "message": "Release note rn-102 uses high-confidence wording without passed reproduction evidence.", + "refs": [ + "rn-102" + ], + "action": "downgrade_claim_or_add_evidence" + }, + { + "severity": "high", + "code": "BREAKING_CHANGE_NOT_DISCLOSED", + "message": "Breaking artifact code/export/public-api.js has no explicit breaking-change release note.", + "refs": [ + "chg-api-export" + ], + "action": "document_breaking_change_in_release_notes" + }, + { + "severity": "high", + "code": "CLAIM_EVIDENCE_NOT_PASSED", + "message": "Evidence ev-old-repro for release note rn-102 has status failed.", + "refs": [ + "rn-102", + "ev-old-repro" + ], + "action": "replace_or_rerun_failed_evidence" + }, + { + "severity": "high", + "code": "CLAIM_KIND_EVIDENCE_MISSING", + "message": "Release note rn-101 is a api claim but lacks passed test evidence.", + "refs": [ + "rn-101", + "test" + ], + "action": "attach_required_claim_evidence" + }, + { + "severity": "high", + "code": "CLAIM_KIND_EVIDENCE_MISSING", + "message": "Release note rn-101 is a api claim but lacks passed export evidence.", + "refs": [ + "rn-101", + "export" + ], + "action": "attach_required_claim_evidence" + }, + { + "severity": "high", + "code": "CLAIM_KIND_EVIDENCE_MISSING", + "message": "Release note rn-102 is a reproducibility claim but lacks passed reproduction evidence.", + "refs": [ + "rn-102", + "reproduction" + ], + "action": "attach_required_claim_evidence" + }, + { + "severity": "high", + "code": "CLAIM_KIND_EVIDENCE_MISSING", + "message": "Release note rn-103 is a dataset claim but lacks passed checksum evidence.", + "refs": [ + "rn-103", + "checksum" + ], + "action": "attach_required_claim_evidence" + }, + { + "severity": "high", + "code": "CLAIMED_COMPONENT_WITHOUT_CHANGE", + "message": "Release note rn-103 names data/public/cohort-clean.csv, but the release packet has no matching changed artifact.", + "refs": [ + "rn-103", + "data/public/cohort-clean.csv" + ], + "action": "remove_claim_or_attach_changed_artifact" + }, + { + "severity": "high", + "code": "EXPORT_DIGEST_MISMATCH", + "message": "Export manifest digest for data/restricted/cohort-clean.csv does not match the changed artifact digest.", + "refs": [ + "data/restricted/cohort-clean.csv" + ], + "action": "update_export_manifest_digest" + }, + { + "severity": "high", + "code": "REPRODUCIBILITY_OVERCLAIM", + "message": "Release note rn-102 uses high-confidence wording without passed reproduction evidence.", + "refs": [ + "rn-102" + ], + "action": "downgrade_claim_or_add_evidence" + }, + { + "severity": "high", + "code": "VALIDATION_OVERCLAIM", + "message": "Release note rn-102 uses high-confidence wording without passed test evidence.", + "refs": [ + "rn-102" + ], + "action": "downgrade_claim_or_add_evidence" + }, + { + "severity": "medium", + "code": "CHANGE_DIGEST_NOT_UPDATED", + "message": "Changed artifact results/figures/figure-5.svg does not carry a new post-release digest.", + "refs": [ + "chg-figure" + ], + "action": "record_post_release_digest" + }, + { + "severity": "medium", + "code": "CHANGED_ARTIFACT_UNMENTIONED", + "message": "Changed artifact data/restricted/cohort-clean.csv is not represented in any public release note.", + "refs": [ + "chg-sensitive-data" + ], + "action": "add_release_note_or_mark_internal_only" + }, + { + "severity": "medium", + "code": "CLAIM_EVIDENCE_STALE", + "message": "Evidence ev-old-repro for release note rn-102 is 93 days old.", + "refs": [ + "rn-102", + "ev-old-repro" + ], + "action": "refresh_release_evidence" + }, + { + "severity": "medium", + "code": "CLAIM_PATH_NOT_COVERED_BY_EVIDENCE", + "message": "Release note rn-102 cites results/figures/figure-5.svg, but no passed evidence covers that path.", + "refs": [ + "rn-102", + "results/figures/figure-5.svg" + ], + "action": "add_path_specific_release_evidence" + }, + { + "severity": "medium", + "code": "CLAIM_PATH_NOT_COVERED_BY_EVIDENCE", + "message": "Release note rn-103 cites data/public/cohort-clean.csv, but no passed evidence covers that path.", + "refs": [ + "rn-103", + "data/public/cohort-clean.csv" + ], + "action": "add_path_specific_release_evidence" + }, + { + "severity": "medium", + "code": "EXPORT_MANIFEST_MISSING_RELEASE_NOTE", + "message": "Export manifest does not list public release note rn-103.", + "refs": [ + "rn-103" + ], + "action": "add_release_note_to_export_manifest" + } + ], + "actions": [ + { + "id": "remove_no_breaking_claim_or_document_breaking_change", + "severity": "critical", + "refs": [ + "rn-101", + "chg-api-export" + ] + }, + { + "id": "downgrade_claim_or_add_evidence", + "severity": "high", + "refs": [ + "rn-102" + ] + }, + { + "id": "document_breaking_change_in_release_notes", + "severity": "high", + "refs": [ + "chg-api-export" + ] + }, + { + "id": "replace_or_rerun_failed_evidence", + "severity": "high", + "refs": [ + "rn-102", + "ev-old-repro" + ] + }, + { + "id": "attach_required_claim_evidence", + "severity": "high", + "refs": [ + "rn-101", + "test" + ] + }, + { + "id": "remove_claim_or_attach_changed_artifact", + "severity": "high", + "refs": [ + "rn-103", + "data/public/cohort-clean.csv" + ] + }, + { + "id": "update_export_manifest_digest", + "severity": "high", + "refs": [ + "data/restricted/cohort-clean.csv" + ] + }, + { + "id": "record_post_release_digest", + "severity": "medium", + "refs": [ + "chg-figure" + ] + }, + { + "id": "add_release_note_or_mark_internal_only", + "severity": "medium", + "refs": [ + "chg-sensitive-data" + ] + }, + { + "id": "refresh_release_evidence", + "severity": "medium", + "refs": [ + "rn-102", + "ev-old-repro" + ] + }, + { + "id": "add_path_specific_release_evidence", + "severity": "medium", + "refs": [ + "rn-102", + "results/figures/figure-5.svg" + ] + }, + { + "id": "add_release_note_to_export_manifest", + "severity": "medium", + "refs": [ + "rn-103" + ] + } + ] + } +} diff --git a/repository-release-note-claim-guard/reports/summary.svg b/repository-release-note-claim-guard/reports/summary.svg new file mode 100644 index 00000000..8a889f36 --- /dev/null +++ b/repository-release-note-claim-guard/reports/summary.svg @@ -0,0 +1,20 @@ + + + + + Repository Release Note Claim Guard + + hold_repository_release + Findings: 18 + High or critical: 12 + Audit digest: sha256:8431cbaec2db5bd8 + + Top Findings + +CRITICAL NO_BREAKING_CHANGE_CONFLICT +HIGH ALL_RESULTS_OVERCLAIM +HIGH BREAKING_CHANGE_NOT_DISCLOSED +HIGH CLAIM_EVIDENCE_NOT_PASSED +HIGH CLAIM_KIND_EVIDENCE_MISSING + + diff --git a/repository-release-note-claim-guard/sample-data.js b/repository-release-note-claim-guard/sample-data.js new file mode 100644 index 00000000..c2e9fd00 --- /dev/null +++ b/repository-release-note-claim-guard/sample-data.js @@ -0,0 +1,205 @@ +const cleanReleasePacket = { + repositoryId: "scibase-demo-repo", + versionTag: "preprint-v2.1.0", + reviewedAt: "2026-06-01T12:00:00Z", + policy: { + maxEvidenceAgeDays: 30 + }, + changeSet: [ + { + id: "chg-analysis-script", + path: "code/analysis/model-fit.py", + component: "code", + digestBefore: "sha256:1111", + digestAfter: "sha256:2222", + noteIds: ["rn-001"], + publicImpact: true + }, + { + id: "chg-results-figure", + path: "results/figures/figure-2.svg", + component: "results", + digestBefore: "sha256:3333", + digestAfter: "sha256:4444", + noteIds: ["rn-002"], + publicImpact: true + }, + { + id: "chg-metadata", + path: "metadata.json", + component: "metadata", + digestBefore: "sha256:5555", + digestAfter: "sha256:6666", + noteIds: ["rn-003"], + publicImpact: true + } + ], + releaseNotes: [ + { + id: "rn-001", + claimType: "code", + text: "Updated the mixed-effects model fitting script and validated it with repository tests.", + componentPaths: ["code/analysis/model-fit.py"], + evidenceIds: ["ev-tests-model"] + }, + { + id: "rn-002", + claimType: "figure", + text: "Regenerated Figure 2 from the refreshed analysis output with matching checksums.", + componentPaths: ["results/figures/figure-2.svg"], + evidenceIds: ["ev-repro-figure", "ev-checksum-figure"] + }, + { + id: "rn-003", + claimType: "metadata", + text: "Aligned repository metadata and export manifest with the preprint-v2.1.0 citation package.", + componentPaths: ["metadata.json"], + evidenceIds: ["ev-export-manifest", "ev-metadata-review"] + } + ], + evidence: [ + { + id: "ev-tests-model", + kind: "test", + status: "passed", + command: "npm run repository-tests", + producedAt: "2026-05-31T15:00:00Z", + coversPaths: ["code/analysis/model-fit.py"] + }, + { + id: "ev-repro-figure", + kind: "reproduction", + status: "passed", + command: "python code/analysis/model-fit.py --rebuild-figures", + producedAt: "2026-05-31T15:20:00Z", + coversPaths: ["code/analysis/model-fit.py", "results/figures/figure-2.svg"] + }, + { + id: "ev-checksum-figure", + kind: "checksum", + status: "passed", + producedAt: "2026-05-31T15:25:00Z", + coversPaths: ["results/figures/figure-2.svg"] + }, + { + id: "ev-export-manifest", + kind: "export", + status: "passed", + producedAt: "2026-05-31T16:00:00Z", + coversPaths: ["metadata.json", "code/analysis/model-fit.py", "results/figures/figure-2.svg"] + }, + { + id: "ev-metadata-review", + kind: "metadata", + status: "passed", + producedAt: "2026-05-31T16:05:00Z", + coversPaths: ["metadata.json"] + } + ], + exportManifest: { + versionTag: "preprint-v2.1.0", + releaseNoteIds: ["rn-001", "rn-002", "rn-003"], + componentDigests: { + "code/analysis/model-fit.py": "sha256:2222", + "results/figures/figure-2.svg": "sha256:4444", + "metadata.json": "sha256:6666" + } + } +}; + +const riskyReleasePacket = { + repositoryId: "scibase-demo-repo", + versionTag: "preprint-v2.2.0", + reviewedAt: "2026-06-01T12:00:00Z", + policy: { + maxEvidenceAgeDays: 30 + }, + changeSet: [ + { + id: "chg-api-export", + path: "code/export/public-api.js", + component: "code", + digestBefore: "sha256:aaaa", + digestAfter: "sha256:bbbb", + noteIds: ["rn-101"], + publicImpact: true, + breaking: true + }, + { + id: "chg-sensitive-data", + path: "data/restricted/cohort-clean.csv", + component: "data", + digestBefore: "sha256:cccc", + digestAfter: "sha256:dddd", + publicImpact: true + }, + { + id: "chg-figure", + path: "results/figures/figure-5.svg", + component: "results", + digestBefore: "sha256:eeee", + digestAfter: "sha256:eeee", + noteIds: ["rn-102"], + publicImpact: true + } + ], + releaseNotes: [ + { + id: "rn-101", + claimType: "api", + text: "Updated the public API export with no breaking changes.", + componentPaths: ["code/export/public-api.js"], + evidenceIds: ["ev-api-review"] + }, + { + id: "rn-102", + claimType: "reproducibility", + text: "All results are now fully reproducible and validated.", + componentPaths: ["results/figures/figure-5.svg"], + evidenceIds: ["ev-old-repro"] + }, + { + id: "rn-103", + claimType: "dataset", + text: "Released a cleaned cohort dataset for reviewers.", + componentPaths: ["data/public/cohort-clean.csv"], + evidenceIds: ["ev-dataset-diff"] + } + ], + evidence: [ + { + id: "ev-api-review", + kind: "review", + status: "passed", + producedAt: "2026-05-31T10:00:00Z", + coversPaths: ["code/export/public-api.js"] + }, + { + id: "ev-old-repro", + kind: "reproduction", + status: "failed", + producedAt: "2026-03-01T10:00:00Z", + coversPaths: ["results/figures/figure-5.svg"] + }, + { + id: "ev-dataset-diff", + kind: "data-diff", + status: "passed", + producedAt: "2026-05-31T12:00:00Z", + coversPaths: ["data/restricted/cohort-clean.csv"] + } + ], + exportManifest: { + versionTag: "preprint-v2.2.0", + releaseNoteIds: ["rn-101", "rn-102"], + componentDigests: { + "code/export/public-api.js": "sha256:bbbb", + "results/figures/figure-5.svg": "sha256:eeee" + } + } +}; + +module.exports = { + cleanReleasePacket, + riskyReleasePacket +}; diff --git a/repository-release-note-claim-guard/test.js b/repository-release-note-claim-guard/test.js new file mode 100644 index 00000000..bd6473bb --- /dev/null +++ b/repository-release-note-claim-guard/test.js @@ -0,0 +1,54 @@ +const assert = require("node:assert/strict"); +const { evaluateReleasePacket, renderMarkdownReport, renderSvgSummary } = require("./index"); +const { cleanReleasePacket, riskyReleasePacket } = require("./sample-data"); + +function clone(value) { + return JSON.parse(JSON.stringify(value)); +} + +function codes(evaluation) { + return new Set(evaluation.findings.map((finding) => finding.code)); +} + +const clean = evaluateReleasePacket(cleanReleasePacket); +assert.equal(clean.summary.decision, "publish_release_notes"); +assert.equal(clean.findings.length, 0); +assert.equal(clean.summary.highOrCriticalFindings, 0); + +const risky = evaluateReleasePacket(riskyReleasePacket); +assert.equal(risky.summary.decision, "hold_repository_release"); +assert.equal(risky.summary.highOrCriticalFindings > 0, true); +assert.equal(codes(risky).has("NO_BREAKING_CHANGE_CONFLICT"), true); +assert.equal(codes(risky).has("BREAKING_CHANGE_NOT_DISCLOSED"), true); +assert.equal(codes(risky).has("CLAIM_EVIDENCE_NOT_PASSED"), true); +assert.equal(codes(risky).has("CLAIMED_COMPONENT_WITHOUT_CHANGE"), true); +assert.equal(codes(risky).has("CHANGED_ARTIFACT_UNMENTIONED"), true); +assert.equal(codes(risky).has("EXPORT_DIGEST_MISMATCH"), true); + +const stale = clone(cleanReleasePacket); +stale.evidence[0].producedAt = "2025-12-01T00:00:00Z"; +const staleResult = evaluateReleasePacket(stale); +assert.equal(staleResult.summary.decision, "revise_release_notes"); +assert.equal(codes(staleResult).has("CLAIM_EVIDENCE_STALE"), true); + +const missingRequiredKind = clone(cleanReleasePacket); +missingRequiredKind.releaseNotes[0].claimType = "performance"; +const missingRequiredKindResult = evaluateReleasePacket(missingRequiredKind); +assert.equal(missingRequiredKindResult.summary.decision, "hold_repository_release"); +assert.equal(codes(missingRequiredKindResult).has("CLAIM_KIND_EVIDENCE_MISSING"), true); + +const missingExport = clone(cleanReleasePacket); +delete missingExport.exportManifest; +const missingExportResult = evaluateReleasePacket(missingExport); +assert.equal(missingExportResult.summary.decision, "hold_repository_release"); +assert.equal(codes(missingExportResult).has("EXPORT_MANIFEST_MISSING"), true); + +const markdown = renderMarkdownReport(riskyReleasePacket, risky); +assert.match(markdown, /Release Note Claim Evidence Review/); +assert.match(markdown, /hold_repository_release/); + +const svg = renderSvgSummary(risky); +assert.match(svg, /