diff --git a/.gitattributes b/.gitattributes index a679db0..20c2eec 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,4 +1,5 @@ -# PCM ledger evidence logs are append-only sets deduped by content hash on read — -# union merge is safe and makes teammate ledgers conflict-free by construction -# (docs/plans/substrate-v2/02-team-memory.md §1). -.forge/ledger/evidence/*.log merge=union +# PCM ledger logs (evidence/provenance/tombstones) are append-only sets deduped by +# content hash on read — union merge is safe and makes teammate ledgers conflict-free +# by construction (docs/plans/substrate-v2/02-team-memory.md §1). `forge init` emits +# this same rule into consumer repos. +.forge/ledger/*/*.log merge=union diff --git a/CHANGELOG.md b/CHANGELOG.md index fb8be27..b5a90f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,30 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Fixed + +- **PCM ledger hardened after an 8-angle adversarial review of the P1 merge.** The + conflict-free-merge guarantee is now structural: claim file bytes are a pure function of + (kind, body, scope) — byte-identical on every replica — while provenance and tombstones + move into per-claim append-only logs (hash-deduped, union-merged like evidence), so + concurrent mints and concurrent retractions can never produce a git conflict or a + merge-order-dependent state. Forged evidence is now powerless AND detectable: `val()` + takes oracle weights from the ORACLES table (never the stored record) and skips unknown + oracles, while `forge ledger verify` recomputes every record's content hash and flags + mismatches, ghost oracles, and inflated weights. `forge ledger import` is truly + idempotent (claims already tracked live are never re-synthesized — no double counting). + Cortex shadow-writes: distillation now supersedes (evidence carried over, template claim + tombstoned); evidence refs carry the confirmation counter so same-day sessions with + colliding episode ids stay distinct; regex-detected reverts contradict at the + conservative bridge weight instead of the full-weight human oracle. Fact claims: one + CRLF-tolerant parser (`recall.readFact`), trimmed bodies (shadow path and import path + mint one id), same-name updates supersede the stale claim, and `forge recall + consolidate` reconciles deletions into tombstones. `putClaim` repairs corrupt/truncated + claim files instead of trusting `existsSync`. `forge ledger --personal` reaches the + personal ledger (previously write-only); `forge ledger show` resolves by shard instead + of scanning; `forge init` emits the union-merge `.gitattributes` rule into consumer + repos. `SCOPE_WEIGHT` has one home (ledger core; lessons re-exports). + ### Documentation - **Substrate v2 plan: the whitepaper, completed (`docs/plans/substrate-v2/`).** Nine specs diff --git a/src/cli.js b/src/cli.js index 559b34a..f82462d 100755 --- a/src/cli.js +++ b/src/cli.js @@ -157,15 +157,24 @@ async function run(argv) { if (res.ok) { // Shadow the fact into the PERSONAL ledger beside the global store (repo // promotion stays an explicit act — docs/plans/substrate-v2/02-team-memory.md §3). - const { join } = await import("node:path"); - const { recordFactEvent } = await import("./ledger_bridge.js"); - const { epochDay } = await import("./util.js"); - recordFactEvent(join(store, "ledger"), name, body, epochDay()); + // Best-effort INCLUDING the imports: a broken bridge module must never turn an + // already-persisted fact into a CLI failure. + try { + const { join } = await import("node:path"); + const { shadowFact } = await import("./ledger_bridge.js"); + shadowFact(join(store, "ledger"), name, body); + } catch {} } console.log(res.ok ? ` saved: ${res.slug}` : ` ${res.reason}`); if (!res.ok) process.exitCode = 1; } else if (sub === "consolidate") { const { removed, kept } = r.consolidate(store); + try { + // Deleted duplicates must not survive as live claims in the shadow ledger. + const { join } = await import("node:path"); + const { reconcileFacts } = await import("./ledger_bridge.js"); + reconcileFacts(store, join(store, "ledger")); + } catch {} console.log(` consolidated: ${removed} duplicate(s) removed, ${kept} kept`); } else { console.error(`recall: unknown subcommand "${sub}" (list | add | consolidate)`); @@ -177,8 +186,15 @@ async function run(argv) { const ls = await import("./ledger_store.js"); const { epochDay } = await import("./util.js"); const root = process.cwd(); - const dir = ls.repoLedger(root); - const sub = argv[1] || "stats"; + // --personal targets the ledger beside the global recall store (~/.forge/recall/ + // ledger) — otherwise facts shadowed by `forge recall add` would be write-only, + // with no command able to inspect or verify them. + const personal = argv.includes("--personal"); + const args = argv.filter((a) => a !== "--json" && a !== "--personal"); + const dir = personal + ? (await import("node:path")).join((await import("./recall.js")).defaultStore(), "ledger") + : ls.repoLedger(root); + const sub = args[1] || "stats"; const json = argv.includes("--json"); const nowDay = epochDay(); if (sub === "stats") { @@ -202,10 +218,12 @@ async function run(argv) { return; } if (sub === "show") { - const id = argv[2]; - const hit = id && ls.loadClaims(dir).find((c) => c.id.startsWith(id)); + const id = args[2]; + const hit = id && id.length >= 2 ? ls.getClaimByPrefix(dir, id) : null; if (!hit) { - console.error(id ? ` no claim matching ${id}` : "usage: forge ledger show "); + console.error( + id ? ` no claim matching ${id}` : "usage: forge ledger show ", + ); process.exitCode = 1; return; } @@ -213,13 +231,16 @@ async function run(argv) { return console.log(JSON.stringify({ ...hit, val: val(hit, nowDay) }, null, 2)); } if (sub === "import") { - const { importLegacy } = await import("./ledger_bridge.js"); - const { brainStore } = await import("./brain.js"); - const r = importLegacy(root, { - recallStore: brainStore(root), - recallLedger: dir, - nowDay, - }); + const b = await import("./ledger_bridge.js"); + let r; + if (personal) { + // Personal import: facts from the global recall store into the personal ledger. + const { defaultStore } = await import("./recall.js"); + r = { lessons: 0, outcomes: 0, ...b.importFacts(defaultStore(), dir, nowDay) }; + } else { + const { brainStore } = await import("./brain.js"); + r = b.importLegacy(root, { recallStore: brainStore(root), recallLedger: dir, nowDay }); + } if (json) return console.log(JSON.stringify(r, null, 2)); console.log( ` imported: ${r.lessons} lesson(s), ${r.facts} fact(s), ${r.outcomes} outcome(s)`, @@ -227,7 +248,9 @@ async function run(argv) { for (const x of r.refused) console.log(` refused: ${x}`); return; } - console.error(`ledger: unknown subcommand "${sub}" (stats | verify | show | import)`); + console.error( + `ledger: unknown subcommand "${sub}" (stats | verify | show | import) [--personal] [--json]`, + ); process.exitCode = 1; return; } @@ -329,10 +352,11 @@ async function run(argv) { const res = b.remember(b.brainStore(process.cwd()), name, body); if (res.ok) { // Brain is repo-scoped and git-committable → shadow into the REPO ledger. - const { recordFactEvent } = await import("./ledger_bridge.js"); - const { repoLedger } = await import("./ledger_store.js"); - const { epochDay } = await import("./util.js"); - recordFactEvent(repoLedger(process.cwd()), name, body, epochDay()); + try { + const { shadowFact } = await import("./ledger_bridge.js"); + const { repoLedger } = await import("./ledger_store.js"); + shadowFact(repoLedger(process.cwd()), name, body); + } catch {} } console.log( res.ok diff --git a/src/cortex.js b/src/cortex.js index 9331174..4c7ae63 100644 --- a/src/cortex.js +++ b/src/cortex.js @@ -4,7 +4,7 @@ // contradiction. Kept fs-thin and deterministic (day + ids passed in) so it's testable // without any hook wiring. -import { recordLessonEvent } from "./ledger_bridge.js"; +import { recordLessonEvent, supersedeLessonClaim } from "./ledger_bridge.js"; import { confidenceOf, confirm, @@ -86,10 +86,13 @@ export function recordMistake(root, { signals, context, nowDay, episodeId, disti }; if (!save(root, updated).ok) return { action: "refused", p, fires }; // Shadow the confirmation into the PCM ledger (P1 bridge — legacy store stays the - // read path; best-effort by design, never blocks the hook). + // read path; best-effort by design, never blocks the hook). The evidence counter + // rides in the ref because episode ids reset per session (ep_m0_…) — without it, + // two same-day sessions confirming via the same file would hash identically and + // the second real confirmation would be silently deduped away. recordLessonEvent(root, updated, { result: "confirm", - ref: `episode:${episodeId}`, + ref: `episode:${episodeId}#n${updated.evidenceCount}`, t: nowDay, }); return { @@ -145,12 +148,14 @@ export function recordContradiction(root, { context, nowDay, episodeId }) { const results = targets.map((l) => { const updated = contradict(l, nowDay); const saved = save(root, updated).ok; - // An explicit human reversal is the strongest oracle we have — shadow it. + // Shadowed at the conservative bridge weight, NOT human.revert (w=1.0): the hook's + // revert detection is regex-based and matches routine `git restore`s, and a + // full-weight contradiction would permanently anchor the claim near dormancy in an + // append-only log. The counter in the ref keeps distinct same-day events distinct. if (saved) recordLessonEvent(root, updated, { result: "contradict", - oracle: "human.revert", - ref: `episode:${episodeId}`, + ref: `episode:${episodeId}#c${updated.contradictionCount}`, t: nowDay, }); return { id: updated.id, status: updated.status, saved }; @@ -191,11 +196,17 @@ export function applyDistillation(root, lessonId, distilled) { if (!distilled) return false; const lesson = load(root).find((l) => l.id === lessonId); if (!lesson) return false; - return save(root, { + const updated = { ...lesson, whatWentWrong: distilled.whatWentWrong, correctedBehavior: distilled.correctedBehavior, - }).ok; + }; + const ok = save(root, updated).ok; + // A body rewrite changes the content-addressed claim id — supersede in the ledger + // (mint the distilled claim, carry the evidence over, tombstone the template claim) + // or the lesson's history splits across two disjoint claims. + if (ok) supersedeLessonClaim(root, lesson, updated); + return ok; } /** The lessons block to inline into AGENTS.md so non-Claude tools see them (empty if none). */ diff --git a/src/init.js b/src/init.js index 0b5553c..511a484 100644 --- a/src/init.js +++ b/src/init.js @@ -1,14 +1,31 @@ // forge init / catalog — the onboarding surface. init gets a repo to a working // state in one command; catalog is the "Start Here" index of everything active. -import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { appendFileSync, existsSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { BRAND } from "./brand.js"; +import { GITATTRIBUTES_RULE } from "./ledger_store.js"; import { sync } from "./sync.js"; import { list as tasteList } from "./taste.js"; +/** Without the union merge driver, two teammates appending to the same ledger log get + * a git conflict — the exact thing the ledger's design promises can't happen + * (docs/plans/substrate-v2/02-team-memory.md §1). Idempotent append. */ +export function ensureLedgerGitattributes(targetRoot = process.cwd()) { + const path = join(targetRoot, ".gitattributes"); + const existing = existsSync(path) ? readFileSync(path, "utf8") : ""; + if (existing.includes(".forge/ledger/")) return { written: false }; + appendFileSync( + path, + `${existing && !existing.endsWith("\n") ? "\n" : ""}${GITATTRIBUTES_RULE}\n`, + ); + return { written: true }; +} + /** Scaffold this repo's cross-tool config (emit every tool) in one step. */ export function init({ targetRoot = process.cwd() } = {}) { - return sync({ targetRoot }); + const r = sync({ targetRoot }); + ensureLedgerGitattributes(targetRoot); + return r; } function skillDescription(dir) { diff --git a/src/ledger.js b/src/ledger.js index 046ffd6..e7cd962 100644 --- a/src/ledger.js +++ b/src/ledger.js @@ -7,8 +7,12 @@ // Design invariants (shared with lessons.js, now protocol law for every stored thing): // - Confidence is EARNED from independent oracles (tests, CI, human accept/revert), // never from the model's self-assessment; retrieval/injection is never confirmation. -// - Claims are immutable by id; every mutable thing (evidence, tombstones) is an -// append-only record — which is what makes merge = set union a join-semilattice. +// val() takes oracle weights from the ORACLES table, NEVER from the stored record — +// a forged/corrupted evidence line cannot buy confidence it isn't entitled to. +// - A claim's persisted bytes are a pure function of (kind, body, scope): anything +// author- or time-varying (provenance, evidence, tombstones) lives in append-only +// logs. That is what makes every file either byte-identical across teammates or +// union-mergeable — the join-semilattice property is structural, not aspirational. // - Unreviewed claims decay toward the PRIOR (0.5, uncertainty), not toward false. import { contentHash } from "./util.js"; @@ -37,7 +41,9 @@ export const KINDS = [ * `family` powers the cross-family gate (a lone behavioral signal never moves a claim * on its own — same rule as lessons.js scoreMistake). The two `bridge: true` entries * exist only for the P1 migration seam (cortex episodes, legacy imports) and carry a - * deliberately conservative weight. + * deliberately conservative weight — Stop-hook revert detection is regex-based and + * routinely matches innocent `git restore`s, so it must NOT ride the full-weight + * human.revert oracle (that one is reserved for explicit, unambiguous human signals). */ export const ORACLES = { "human.revert": { w: 1.0, family: "human" }, @@ -51,13 +57,15 @@ export const ORACLES = { "legacy.import": { w: 0.5, family: "outcome", bridge: true }, }; +/** One source of truth for scope weighting — lessons.js re-exports this. */ export const SCOPE_WEIGHT = { symbol: 1.0, dir: 0.8, repo: 0.6, global: 0.4 }; /** Retrieval weights for Eq. 3 (a=relevance, b=recency, g=validity) — calibrated in P8. */ export const EQ3_WEIGHTS = { a: 0.55, b: 0.15, g: 0.3 }; export const DEFAULT_HALF_LIFE_DAYS = 45; -/** Below this val a claim is dormant: kept for audit, never retrieved. */ +/** Below this val a claim is dormant: kept for audit, never retrieved. The trusted + * band starts at the mirror threshold (1 − DORMANT_VAL) — stats uses both. */ export const DORMANT_VAL = 0.35; /** @@ -90,18 +98,29 @@ export function claimId(kind, body, scope = {}) { return contentHash(canonicalize({ body, kind, scope })); } +/** Stamp a record with its content hash (the dedupe key in every append-only log). */ +export function sealRecord(record) { + return { ...record, h: contentHash(canonicalize(record)) }; +} + /** * Mint a claim. Refuses secrets and unknown kinds ({ok:false, reason} — same contract - * as recall.add / lessons_store.save so callers keep one error shape). + * as recall.add / lessons_store.save so callers keep one error shape). The body/scope + * are normalized through JSON first (Dates → ISO strings, Maps/Sets → {}), so a + * non-JSON value can never make two different bodies collide on one address. + * `provenance` rides on the in-memory claim but is NEVER part of the id or the claim + * file bytes — the store appends it to a per-claim log instead. * @param {{kind:string, body:object, scope?:object, provenance?:object, t?:number}} f * `t` is the mint day (epoch days) — passed in, never read from the clock here. - * @returns {{ok:true, claim:object}|{ok:false, reason:string}} + * @returns {{ok:true, claim:any}|{ok:false, reason:string}} */ export function mintClaim({ kind, body, scope = {}, provenance = {}, t = 0 }) { if (!KINDS.includes(kind)) return { ok: false, reason: `unknown claim kind: ${kind}` }; if (body === null || typeof body !== "object") return { ok: false, reason: "claim body must be an object" }; - const canon = canonicalize({ body, kind, scope }); + const nBody = JSON.parse(JSON.stringify(body)); + const nScope = JSON.parse(JSON.stringify(scope)); + const canon = canonicalize({ body: nBody, kind, scope: nScope }); if (SECRET_RE.test(canon)) return { ok: false, @@ -111,11 +130,11 @@ export function mintClaim({ kind, body, scope = {}, provenance = {}, t = 0 }) { ok: true, claim: { v: 1, - id: claimId(kind, body, scope), + id: claimId(kind, nBody, nScope), kind, - body, - scope, - provenance: { ...provenance, t }, + body: nBody, + scope: nScope, + provenance: sealRecord({ ...provenance, t }), evidence: [], }, }; @@ -123,9 +142,10 @@ export function mintClaim({ kind, body, scope = {}, provenance = {}, t = 0 }) { /** * Build an evidence outcome. Evidence without a verifiable ref (commit SHA, test-run - * id, episode id, CI URL) is rejected — "the model said so" is not evidence. + * id, episode id, CI URL) is rejected — "the model said so" is not evidence. The + * oracle's table weight is recorded for audit, but val() re-reads the table. * @param {{oracle:string, result:"confirm"|"contradict", ref:string, author?:string, t?:number}} f - * @returns {{ok:true, outcome:object}|{ok:false, reason:string}} + * @returns {{ok:true, outcome:any}|{ok:false, reason:string}} */ export function outcomeRecord({ oracle, result, ref, author = "", t = 0 }) { const o = ORACLES[oracle]; @@ -134,24 +154,34 @@ export function outcomeRecord({ oracle, result, ref, author = "", t = 0 }) { return { ok: false, reason: `result must be confirm|contradict, got: ${result}` }; if (!ref || typeof ref !== "string") return { ok: false, reason: "evidence requires a verifiable ref" }; - const outcome = { author, oracle, ref, result, t, w: o.w }; - return { ok: true, outcome: { ...outcome, h: contentHash(canonicalize(outcome)) } }; + return { ok: true, outcome: sealRecord({ author, oracle, ref, result, t, w: o.w }) }; } +/** An evidence record val() will count: known oracle, valid result, a ref, a hash. */ +export function validOutcome(e) { + return Boolean( + e && ORACLES[e.oracle] && (e.result === "confirm" || e.result === "contradict") && e.ref && e.h, + ); +} + +// Weight comes from the ORACLES table — a stored `w` is audit metadata, never trusted +// (a hand-edited or forged log line must not be able to buy extra confidence). const decayed = (outcome, nowDay, halfLife) => - outcome.w * 0.5 ** (Math.max(0, nowDay - (outcome.t ?? 0)) / halfLife); + ORACLES[outcome.oracle].w * 0.5 ** (Math.max(0, nowDay - (outcome.t ?? 0)) / halfLife); /** * Validity — the paper's `val` term as a time-decayed Beta posterior mean with a * Beta(1,1) prior: (1 + Σ confirms·w·λ^Δt/T) / (2 + Σ all·w·λ^Δt/T). * Fresh claim → 0.5. Unreviewed evidence decays, pulling val back toward 0.5 * (uncertainty), never toward 0 — review (new evidence) is what restores weight. - * Pure function of the evidence set ⇒ identical after any merge order. + * Records that fail validOutcome (unknown oracle, malformed) are IGNORED, not + * trusted. Pure function of the evidence set ⇒ identical after any merge order. */ export function val(claim, nowDay = 0, { halfLife = DEFAULT_HALF_LIFE_DAYS } = {}) { let confirms = 0; let all = 0; for (const e of claim.evidence ?? []) { + if (!validOutcome(e)) continue; const d = decayed(e, nowDay, halfLife); all += d; if (e.result === "confirm") confirms += d; @@ -257,6 +287,11 @@ export function claimText(claim) { } } +// Memoize on the claim object — sketch(claimText) is deterministic per id and claims +// are immutable, so first-use caching is safe and keeps retrieve()/clusters() from +// re-hashing every claim on every call. (noAssignInExpressions is off in biome.json.) +const sketchOf = (claim) => (claim._sketch ??= sketch(claimText(claim))); + /** * Eq. 3 retrieval score (paper §7.1): σ(a·rel + b·rec + g·val) × scope weight. * `query` may be a string or a precomputed sketch. The `g·val` term is the protocol's @@ -264,7 +299,7 @@ export function claimText(claim) { */ export function score(query, claim, { nowDay = 0, weights = EQ3_WEIGHTS } = {}) { const qs = Array.isArray(query) ? query : sketch(query); - const rel = jaccard(qs, claim._sketch ?? sketch(claimText(claim))); + const rel = jaccard(qs, sketchOf(claim)); const x = weights.a * rel + weights.b * rec(claim, nowDay) + weights.g * val(claim, nowDay); const sigma = 1 / (1 + Math.exp(-x)); const scopeW = SCOPE_WEIGHT[claim.scope?.level] ?? 0.5; @@ -287,7 +322,7 @@ export function retrieve(query, claims, { nowDay = 0, budget = 12, weights = EQ3 * the documented scale path. Returns clusters of ≥2 as arrays of claim ids. */ export function clusters(claims, { tau = 0.7 } = {}) { - const items = claims.map((c) => ({ id: c.id, s: c._sketch ?? sketch(claimText(c)) })); + const items = claims.map((c) => ({ id: c.id, s: sketchOf(c) })); const parent = new Map(items.map((i) => [i.id, i.id])); const find = (x) => { while (parent.get(x) !== x) { @@ -309,39 +344,68 @@ export function clusters(claims, { tau = 0.7 } = {}) { } // --------------------------------------------------------------------------- -// The CRDT merge. State = { claims: {id→claim}, evidence: {id→outcome[]}, tombstones: -// {id→tomb} } — three grow-only maps keyed by content. Merge is set union with -// evidence deduped by hash; (S, ⊔) is a join-semilattice (commutative, associative, -// idempotent), so replicas converge under ANY merge order. Property-tested. +// The CRDT merge. State = four grow-only maps: +// claims: id → {v, kind, body, scope} (bytes are pure content — identical +// for the same id on every replica) +// evidence: id → outcome[] (union by content hash) +// provenance: id → record[] (union by content hash — every +// author's mint is kept, attribution +// is a set, not a fight) +// tombstones: id → record[] (union by content hash — concurrent +// retractions both survive) +// Merge is set union throughout; (S, ⊔) is a join-semilattice (commutative, +// associative, idempotent), so replicas converge under ANY merge order. The single- +// record views (claim.provenance, claim.tombstone) are derived deterministically +// (earliest by (t, h)), so they converge too. Property-tested. // --------------------------------------------------------------------------- -const sortOutcomes = (arr) => { +/** Dedupe by content hash and sort by (t, h) — the ONE record order everywhere, so a + * log's on-disk line order (which differs across replicas after a union merge) can + * never leak into views or derived values. */ +export const sortRecords = (arr) => { const byHash = new Map(); - for (const o of arr) if (!byHash.has(o.h)) byHash.set(o.h, o); + for (const o of arr) if (o?.h && !byHash.has(o.h)) byHash.set(o.h, o); return [...byHash.values()].sort( (a, b) => (a.t ?? 0) - (b.t ?? 0) || (a.h < b.h ? -1 : a.h > b.h ? 1 : 0), ); }; +const mergeLogMap = (m1 = {}, m2 = {}) => { + const out = {}; + for (const id of new Set([...Object.keys(m1), ...Object.keys(m2)])) + out[id] = sortRecords([...(m1[id] ?? []), ...(m2[id] ?? [])]); + return out; +}; + +/** An empty ledger state. */ +export function emptyState() { + return { claims: {}, evidence: {}, provenance: {}, tombstones: {} }; +} + /** Semilattice join of two ledger states. Pure; inputs are not mutated. */ export function mergeStates(s1, s2) { const claims = { ...s1.claims }; + // Claim values are pure content keyed by their own hash — identical bytes on every + // replica, so first-in is not a choice, it's a no-op. for (const [id, c] of Object.entries(s2.claims ?? {})) claims[id] ??= c; - const evidence = {}; - for (const id of new Set([...Object.keys(s1.evidence ?? {}), ...Object.keys(s2.evidence ?? {})])) - evidence[id] = sortOutcomes([...(s1.evidence?.[id] ?? []), ...(s2.evidence?.[id] ?? [])]); - const tombstones = { ...(s1.tombstones ?? {}) }; - for (const [id, t] of Object.entries(s2.tombstones ?? {})) tombstones[id] ??= t; - return { claims, evidence, tombstones }; + return { + claims, + evidence: mergeLogMap(s1.evidence, s2.evidence), + provenance: mergeLogMap(s1.provenance, s2.provenance), + tombstones: mergeLogMap(s1.tombstones, s2.tombstones), + }; } -/** Materialize a state into claim objects with evidence + tombstones attached. */ +/** Materialize a state into claim views: evidence attached, provenance = earliest + * mint record, tombstone = earliest retraction (deterministic across replicas). */ export function liveClaims(state) { return Object.values(state.claims) .map((c) => ({ ...c, evidence: state.evidence?.[c.id] ?? [], - tombstone: state.tombstones?.[c.id], + provenance: state.provenance?.[c.id]?.[0] ?? c.provenance ?? {}, + provenanceAll: state.provenance?.[c.id] ?? [], + tombstone: state.tombstones?.[c.id]?.[0], })) .sort((a, b) => (a.id < b.id ? -1 : 1)); } diff --git a/src/ledger_bridge.js b/src/ledger_bridge.js index 0be788f..96a490b 100644 --- a/src/ledger_bridge.js +++ b/src/ledger_bridge.js @@ -1,24 +1,47 @@ // forge ledger bridge — the P1 migration seam between the legacy stores (lessons/*.md, // recall facts) and the PCM ledger. Legacy files stay the READ path (every existing // test, hook, and guard keeps working unchanged); the ledger shadows every write as -// the new canonical, and `forge ledger import` back-fills history. P2 flips the read -// path once merge/verify tooling lands. Spec: docs/plans/substrate-v2/01-pcm-protocol.md §7. -import { readFileSync } from "node:fs"; -import { join } from "node:path"; +// the new canonical, and `forge ledger import` back-fills pre-ledger history. P2 flips +// the read path once merge/verify tooling lands. Spec: docs/plans/substrate-v2/01-pcm-protocol.md §7. +// +// Every entry point here is BEST-EFFORT by design: the legacy store is still +// canonical in P1, so a bridge failure returns {ok:false} and must never break a +// hook or CLI write that already succeeded. import { mintClaim, outcomeRecord } from "./ledger.js"; -import { appendEvidence, putClaim, reindex, repoLedger } from "./ledger_store.js"; +import { + appendEvidence, + loadClaims, + putClaim, + readEvidence, + reindex, + repoLedger, + tombstone, +} from "./ledger_store.js"; import { load as loadLessons } from "./lessons_store.js"; -import { list as listFacts } from "./recall.js"; +import { list as listFacts, readFact } from "./recall.js"; +import { epochDay } from "./util.js"; -/** A lesson's claim: body = the CONTENT (trigger + texts); counts/status are evidence- - * derived in the ledger, not body — so confirm/contradict never changes the id. +/** One best-effort policy for the whole bridge (never throws into a caller). */ +const bestEffort = (fn) => { + try { + return fn(); + } catch (err) { + if (process.env.FORGE_DEBUG === "1") + process.stderr.write(`forge ledger bridge: ${err?.message ?? err}\n`); + return { ok: false, reason: String(err?.message ?? err) }; + } +}; + +/** A lesson's claim: body = the CONTENT (trigger + texts) only. Counts/status are + * evidence-derived and the legacy file id rides in PROVENANCE (excluded from the + * content address) — so teammates who learn the same lesson mint the same id even + * if their legacy filenames differ, and confirm/contradict never re-mints. * @returns {{ok:boolean, reason?:string, claim?:any}} */ export function lessonClaim(lesson, t = 0) { return mintClaim({ kind: "lesson", body: { correctedBehavior: lesson.correctedBehavior ?? "", - legacyId: lesson.id, // join key back to .forge/lessons/.md during the bridge trigger: { action: lesson.trigger?.action ?? "", files: lesson.trigger?.files ?? [], @@ -28,17 +51,18 @@ export function lessonClaim(lesson, t = 0) { whatWentWrong: lesson.whatWentWrong ?? "", }, scope: { level: lesson.scope ?? "repo" }, - provenance: { agent: "cortex", author: "" }, + provenance: { agent: "cortex", author: "", task: lesson.id ?? "" }, t, }); } -/** A recall fact's claim. +/** A recall fact's claim. Name/text are trimmed so the shadow-write path and the + * file-parse import path mint the SAME id for the same fact. * @returns {{ok:boolean, reason?:string, claim?:any}} */ export function factClaim(name, text, t = 0) { return mintClaim({ kind: "fact", - body: { name, text }, + body: { name: String(name).trim(), text: String(text).trim() }, scope: { level: "repo" }, provenance: { agent: "recall", author: "" }, t, @@ -46,9 +70,7 @@ export function factClaim(name, text, t = 0) { } /** - * Shadow-write one lesson event into the repo ledger. Best-effort by design: the - * legacy store is still canonical in P1, so a bridge failure must never break the - * hook path — it returns {ok:false} instead of throwing. + * Shadow-write one lesson event into the repo ledger. * @param {string} root repo root * @param {object} lesson the (already saved) lesson object * @param {{result?:"confirm"|"contradict", oracle?:string, ref?:string, t?:number}} ev @@ -57,10 +79,10 @@ export function factClaim(name, text, t = 0) { * result is given (outcomeRecord enforces it). */ export function recordLessonEvent(root, lesson, ev = {}) { - try { + return bestEffort(() => { const dir = repoLedger(root); const minted = lessonClaim(lesson, ev.t ?? 0); - if (!minted.ok) return minted; + if (!minted.ok) return { ok: false, reason: minted.reason }; const put = putClaim(dir, minted.claim); if (!put.ok) return put; if (ev.result) { @@ -70,37 +92,120 @@ export function recordLessonEvent(root, lesson, ev = {}) { ref: ev.ref, t: ev.t ?? 0, }); - if (!o.ok) return o; + if (!o.ok) return { ok: false, reason: "reason" in o ? o.reason : "invalid outcome" }; const a = appendEvidence(dir, minted.claim.id, o.outcome); if (!a.ok) return a; } return { ok: true, id: minted.claim.id }; - } catch (err) { - if (process.env.FORGE_DEBUG === "1") - process.stderr.write(`forge ledger bridge: ${err?.message ?? err}\n`); - return { ok: false, reason: String(err?.message ?? err) }; - } + }); } -/** Shadow-write one recall fact into the ledger that lives beside the recall store. - * @returns {{ok:boolean, reason?:string, id?:string, existed?:boolean}} */ -export function recordFactEvent(ledgerDir, name, text, t = 0) { - try { +/** + * A lesson's body was rewritten (distillation) — content addressing means a NEW claim + * id, so carry the history across: mint the new claim, copy the old claim's evidence + * to it, and tombstone the old claim as superseded. Without this, evidence splits + * between an orphaned template claim and the distilled one. + */ +export function supersedeLessonClaim(root, before, after, t = epochDay()) { + return bestEffort(() => { + const dir = repoLedger(root); + const oldC = lessonClaim(before, t); + const newC = lessonClaim(after, t); + if (!newC.ok) return { ok: false, reason: newC.reason }; + const put = putClaim(dir, newC.claim); + if (!put.ok) return put; + if (oldC.ok && oldC.claim.id !== newC.claim.id) { + for (const o of readEvidence(dir, oldC.claim.id)) appendEvidence(dir, newC.claim.id, o); + tombstone(dir, oldC.claim.id, { + reason: `superseded-by:${newC.claim.id}`, + t, + }); + } + return { ok: true, id: newC.claim.id }; + }); +} + +/** + * Shadow one fact into a ledger, superseding any live fact claim with the same name + * but different content — so `forge remember api-base ` retires the old value + * instead of leaving a stale phantom the P2 read-flip would resurrect. + * @returns {{ok:boolean, reason?:string, id?:string, existed?:boolean}} + */ +export function shadowFact(ledgerDir, name, text, t = epochDay()) { + return bestEffort(() => { const minted = factClaim(name, text, t); if (!minted.ok) return { ok: false, reason: minted.reason }; - return putClaim(ledgerDir, minted.claim); - } catch (err) { - if (process.env.FORGE_DEBUG === "1") - process.stderr.write(`forge ledger bridge: ${err?.message ?? err}\n`); - return { ok: false, reason: String(err?.message ?? err) }; + const put = putClaim(ledgerDir, minted.claim); + if (!put.ok) return put; + for (const c of loadClaims(ledgerDir)) { + if ( + c.kind === "fact" && + !c.tombstone && + c.id !== minted.claim.id && + c.body?.name === minted.claim.body.name + ) + tombstone(ledgerDir, c.id, { reason: `superseded-by:${minted.claim.id}`, t }); + } + reindex(ledgerDir, t); + return { ...put, id: minted.claim.id }; + }); +} + +/** + * Re-align a ledger's fact claims with the store they shadow: any live fact claim + * whose (name, text) no longer exists as a stored fact is tombstoned. Called after + * `forge recall consolidate` (which rm's duplicate files) so deleted memories don't + * survive as live claims. + */ +export function reconcileFacts(store, ledgerDir, t = epochDay()) { + return bestEffort(() => { + const current = new Set(); + for (const slug of listFacts(store)) { + const f = readFact(store, slug); + if (f) { + const minted = factClaim(f.name, f.text, t); + if (minted.ok) current.add(minted.claim.id); + } + } + let removed = 0; + for (const c of loadClaims(ledgerDir)) { + if (c.kind === "fact" && !c.tombstone && !current.has(c.id)) { + tombstone(ledgerDir, c.id, { reason: "removed-from-store", t }); + removed++; + } + } + if (removed) reindex(ledgerDir, t); + return { ok: true, removed }; + }); +} + +/** + * Import every fact in a recall/brain store into a ledger (idempotent; supersedes + * stale same-name claims via shadowFact). + * @returns {{facts:number, refused:string[]}} + */ +export function importFacts(store, ledgerDir, nowDay = 0) { + const refused = []; + let facts = 0; + for (const slug of listFacts(store)) { + const f = readFact(store, slug); + if (!f) { + refused.push(`fact ${slug}: unreadable`); + continue; + } + const r = shadowFact(ledgerDir, f.name, f.text, nowDay); + if (r.ok && !r.existed) facts++; + else if (!r.ok) refused.push(`fact ${slug}: ${r.reason}`); } + return { facts, refused }; } /** - * One-shot back-fill: import every existing lesson (with its evidence/contradiction - * counts as conservative `legacy.import` outcomes) and every recall fact into the - * ledger. Idempotent — content addressing dedupes claims, outcome hashes dedupe - * evidence — so re-running an import is always safe. + * One-shot back-fill of PRE-LEDGER history. A lesson whose claim already exists in + * the ledger is skipped entirely — its live shadow-writes are already tracking it, + * and re-synthesizing evidence from the (still-moving) legacy counters would double- + * count and break idempotence. Only never-seen claims get their aggregate counts + * expanded into conservative `legacy.import` outcomes. * @param {string} root repo root (lessons + repo ledger) * @param {{recallStore?:string, recallLedger?:string, nowDay?:number}} opts * @returns {{lessons:number, facts:number, outcomes:number, refused:string[]}} @@ -109,7 +214,6 @@ export function importLegacy(root, { recallStore, recallLedger, nowDay = 0 } = { const dir = repoLedger(root); const refused = []; let lessons = 0; - let facts = 0; let outcomes = 0; for (const lesson of loadLessons(root)) { @@ -123,7 +227,8 @@ export function importLegacy(root, { recallStore, recallLedger, nowDay = 0 } = { refused.push(`lesson ${lesson.id}: ${put.reason}`); continue; } - if (!put.existed) lessons++; + if (put.existed) continue; // already tracked live — never re-synthesize + lessons++; // Aggregate counts become individual, decay-dated outcomes: confirms carry the // last-confirmed day (what freshness keyed on), contradictions the created day // (the legacy store kept no per-contradiction date — documented approximation). @@ -146,27 +251,13 @@ export function importLegacy(root, { recallStore, recallLedger, nowDay = 0 } = { } } + let facts = 0; if (recallStore) { - const ledgerDir = recallLedger ?? dir; - for (const slug of listFacts(recallStore)) { - const r = importFact(recallStore, ledgerDir, slug, nowDay); - if (r.ok && !r.existed) facts++; - else if (!r.ok) refused.push(`fact ${slug}: ${r.reason}`); - } + const r = importFacts(recallStore, recallLedger ?? dir, nowDay); + facts = r.facts; + refused.push(...r.refused); } reindex(dir, nowDay); return { lessons, facts, outcomes, refused }; } - -function importFact(store, ledgerDir, slug, t) { - try { - const raw = readFileSync(join(store, "facts", `${slug}.md`), "utf8"); - const m = raw.match(/^# (.*)\n\n([\s\S]*)$/); - const name = m ? m[1] : slug; - const text = (m ? m[2] : raw).trim(); - return recordFactEvent(ledgerDir, name, text, t); - } catch (err) { - return { ok: false, reason: String(err?.message ?? err) }; - } -} diff --git a/src/ledger_store.js b/src/ledger_store.js index a3aff1f..b345257 100644 --- a/src/ledger_store.js +++ b/src/ledger_store.js @@ -1,8 +1,9 @@ // forge ledger storage — the on-disk PCM ledger (docs/plans/substrate-v2/02-team-memory.md): -// one immutable canonical-JSON file per claim (sharded by id prefix), one append-only -// evidence log per claim, tombstones as marker files, and a generated human index. -// Every path is content-addressed, so two teammates' ledgers merge in git with zero -// conflicts (evidence logs use the union merge driver; see .gitattributes note below). +// one immutable canonical-JSON file per claim (sharded by id prefix, bytes = pure +// content so every replica writes the identical file), plus three append-only logs per +// claim — evidence, provenance, tombstones — that git union-merges without conflicts +// (`forge init` emits the .gitattributes rule). Everything author- or time-varying is +// a log line; nothing on disk is ever edited in place. import { appendFileSync, existsSync, @@ -13,14 +14,39 @@ import { writeFileSync, } from "node:fs"; import { join } from "node:path"; -import { canonicalize, claimId, liveClaims, mergeStates, SECRET_RE, val } from "./ledger.js"; +import { + canonicalize, + claimId, + DORMANT_VAL, + emptyState, + liveClaims, + mergeStates, + ORACLES, + SECRET_RE, + sealRecord, + sortRecords, + val, + validOutcome, +} from "./ledger.js"; /** The canonical repo ledger. (recall's global store keeps its own sibling ledger.) */ export const repoLedger = (root = process.cwd()) => join(root, ".forge", "ledger"); +/** The union-merge rule consumer repos need for conflict-free ledger merges — + * emitted into .gitattributes by `forge init` (see init.js). NOTE: .gitattributes + * supports full-line comments only, so the rule ships with a comment line above it. */ +export const GITATTRIBUTES_RULE = [ + "# PCM ledger logs are hash-deduped append-only sets - union merge is conflict-free (forge)", + ".forge/ledger/*/*.log merge=union", +].join("\n"); + +const LOGS = ["evidence", "provenance", "tombstones"]; const claimPath = (dir, id) => join(dir, "claims", id.slice(0, 2), `${id}.json`); -const evidencePath = (dir, id) => join(dir, "evidence", `${id}.log`); -const tombPath = (dir, id) => join(dir, "tombstones", `${id}.json`); +const logPath = (dir, log, id) => join(dir, log, `${id}.log`); + +/** Claim file bytes: pure content only. Identical for the same id on every replica. */ +const claimBytes = (claim) => + `${canonicalize({ body: claim.body, kind: claim.kind, scope: claim.scope ?? {}, v: claim.v ?? 1 })}\n`; const readJson = (path) => { try { @@ -30,133 +56,155 @@ const readJson = (path) => { } }; +/** Parse an append-only log: one canonical-JSON record per line, deduped by content + * hash, corrupt lines skipped. The single reader every log goes through. */ +function readLog(dir, log, id) { + const path = logPath(dir, log, id); + if (!existsSync(path)) return []; + const records = []; + for (const line of readFileSync(path, "utf8").split(/\r?\n/)) { + if (!line.trim()) continue; + try { + records.push(JSON.parse(line)); + } catch {} + } + // sortRecords, not file order: after a git union merge the two replicas' logs hold + // the same set in different line orders — views must not depend on that. + return sortRecords(records); +} + +/** Append one sealed record to a log iff its hash isn't already present. */ +function appendRecord(dir, log, id, record) { + if (!record?.h) return { ok: false, reason: "record missing content hash" }; + if (!existsSync(claimPath(dir, id))) + return { ok: false, reason: `no such claim in ledger: ${id}` }; + if (readLog(dir, log, id).some((e) => e.h === record.h)) return { ok: true, deduped: true }; + mkdirSync(join(dir, log), { recursive: true }); + appendFileSync(logPath(dir, log, id), `${canonicalize(record)}\n`); + return { ok: true, deduped: false }; +} + +/** Walk every claim file: yields {id, path, raw, claim(valid-or-null)}. Shared by + * loadState (keep valid) and verify (report invalid) so the two can never drift. */ +function* walkClaimFiles(dir) { + const claimsRoot = join(dir, "claims"); + if (!existsSync(claimsRoot)) return; + for (const shard of readdirSync(claimsRoot).sort()) { + for (const f of readdirSync(join(claimsRoot, shard)) + .filter((f) => f.endsWith(".json")) + .sort()) { + const path = join(claimsRoot, shard, f); + const id = f.replace(/\.json$/, ""); + const parsed = readJson(path); + // Verify the address: a tampered/corrupt claim is surfaced as claim:null. + const valid = parsed && claimId(parsed.kind, parsed.body, parsed.scope) === id; + yield { id, path, raw: readFileSync(path, "utf8"), claim: valid ? { ...parsed, id } : null }; + } + } +} + /** - * Persist a claim (idempotent — content-addressed, so rewriting the same claim is a - * no-op). Refuses id mismatches and secrets; stores canonical bytes only (evidence and - * tombstones live in their own append-only files, never inside the claim file). + * Persist a claim (idempotent — content-addressed). Bytes contain content only; + * the claim's provenance record (if any) is appended to the provenance log. A + * corrupt/truncated file at the claim's path is REPAIRED by rewriting the canonical + * bytes — a killed process must never leave a claim permanently unloadable. * @returns {{ok:boolean, reason?:string, id?:string, existed?:boolean}} */ export function putClaim(dir, claim) { if (!claim?.id || claim.id !== claimId(claim.kind, claim.body, claim.scope)) return { ok: false, reason: "claim id does not match canonical content hash" }; - const text = canonicalize({ - body: claim.body, - kind: claim.kind, - provenance: claim.provenance ?? {}, - scope: claim.scope ?? {}, - v: claim.v ?? 1, - }); + const text = claimBytes(claim); if (SECRET_RE.test(text)) return { ok: false, reason: "refused: claim looks like it contains a secret/credential" }; const path = claimPath(dir, claim.id); - if (existsSync(path)) return { ok: true, id: claim.id, existed: true }; - mkdirSync(join(dir, "claims", claim.id.slice(0, 2)), { recursive: true }); - writeFileSync(path, `${text}\n`); - return { ok: true, id: claim.id, existed: false }; + const already = existsSync(path); + const healthy = already && readJson(path) !== null && readFileSync(path, "utf8") === text; + if (!healthy) { + mkdirSync(join(dir, "claims", claim.id.slice(0, 2)), { recursive: true }); + writeFileSync(path, text); + } + if (claim.provenance?.h) appendRecord(dir, "provenance", claim.id, claim.provenance); + return { ok: true, id: claim.id, existed: already && healthy }; } /** Append one evidence outcome (deduped by its content hash — append is idempotent). */ export function appendEvidence(dir, id, outcome) { - if (!outcome?.h) return { ok: false, reason: "outcome missing content hash (use outcomeRecord)" }; - if (!existsSync(claimPath(dir, id))) - return { ok: false, reason: `no such claim in ledger: ${id}` }; - const existing = readEvidence(dir, id); - if (existing.some((e) => e.h === outcome.h)) return { ok: true, deduped: true }; - mkdirSync(join(dir, "evidence"), { recursive: true }); - appendFileSync(evidencePath(dir, id), `${canonicalize(outcome)}\n`); - return { ok: true, deduped: false }; + if (!validOutcome(outcome)) return { ok: false, reason: "invalid outcome (use outcomeRecord)" }; + return appendRecord(dir, "evidence", id, outcome); } /** All evidence outcomes for a claim (corrupt lines skipped, duplicates dropped). */ export function readEvidence(dir, id) { - const path = evidencePath(dir, id); - if (!existsSync(path)) return []; - const seen = new Set(); - const out = []; - for (const line of readFileSync(path, "utf8").split("\n")) { - if (!line.trim()) continue; - try { - const o = JSON.parse(line); - if (o?.h && !seen.has(o.h)) { - seen.add(o.h); - out.push(o); - } - } catch {} - } - return out; + return readLog(dir, "evidence", id); } -/** Retract a claim (grow-only marker — the claim file stays, for audit). */ +/** Retract a claim — an append-only record, so two teammates retracting concurrently + * both survive the merge (the view shows the earliest deterministically). */ export function tombstone(dir, id, { author = "", reason = "", t = 0 } = {}) { - if (!existsSync(claimPath(dir, id))) - return { ok: false, reason: `no such claim in ledger: ${id}` }; - mkdirSync(join(dir, "tombstones"), { recursive: true }); - const path = tombPath(dir, id); - if (!existsSync(path)) writeFileSync(path, `${canonicalize({ author, reason, t })}\n`); - return { ok: true }; + return appendRecord(dir, "tombstones", id, sealRecord({ author, reason, t })); } -/** Load the full ledger state {claims, evidence, tombstones} from disk. */ +/** Load the full ledger state {claims, evidence, provenance, tombstones}. */ export function loadState(dir) { - const state = { claims: {}, evidence: {}, tombstones: {} }; - const claimsRoot = join(dir, "claims"); - if (existsSync(claimsRoot)) { - for (const shard of readdirSync(claimsRoot).sort()) { - const shardDir = join(claimsRoot, shard); - for (const f of readdirSync(shardDir) - .filter((f) => f.endsWith(".json")) - .sort()) { - const c = readJson(join(shardDir, f)); - const id = f.replace(/\.json$/, ""); - // Verify the address on read: a tampered/corrupt claim is skipped, not trusted. - if (c && claimId(c.kind, c.body, c.scope) === id) { - state.claims[id] = { ...c, id }; - state.evidence[id] = readEvidence(dir, id); - } - } - } - } - const tombsRoot = join(dir, "tombstones"); - if (existsSync(tombsRoot)) { - for (const f of readdirSync(tombsRoot).filter((f) => f.endsWith(".json"))) { - const t = readJson(join(tombsRoot, f)); - if (t) state.tombstones[f.replace(/\.json$/, "")] = t; - } + const state = emptyState(); + for (const { id, claim } of walkClaimFiles(dir)) { + if (!claim) continue; + state.claims[id] = claim; + for (const log of LOGS) state[log][id] = readLog(dir, log, id); } return state; } -/** All claims with evidence + tombstones attached (the retrieval input). */ +/** All claims with evidence/provenance/tombstone views attached (retrieval input). */ export function loadClaims(dir) { return liveClaims(loadState(dir)); } +/** Find one claim by id prefix without scanning the whole ledger (ids are sharded by + * their first two hex chars, so any prefix ≥ 2 chars pins the shard). */ +export function getClaimByPrefix(dir, prefix) { + if (!prefix || prefix.length < 2) return null; + const shardDir = join(dir, "claims", prefix.slice(0, 2)); + if (!existsSync(shardDir)) return null; + const f = readdirSync(shardDir) + .filter((f) => f.endsWith(".json") && f.startsWith(prefix)) + .sort()[0]; + if (!f) return null; + const id = f.replace(/\.json$/, ""); + const claim = readJson(join(shardDir, f)); + if (!claim || claimId(claim.kind, claim.body, claim.scope) !== id) return null; + const state = emptyState(); + state.claims[id] = { ...claim, id }; + for (const log of LOGS) state[log][id] = readLog(dir, log, id); + return liveClaims(state)[0]; +} + /** Semilattice import: merge another ledger state into this directory (P2's `forge * ledger merge` core). Idempotent; safe to re-run. */ export function importState(dir, other) { const merged = mergeStates(loadState(dir), other); let claims = 0; - let outcomes = 0; + let records = 0; for (const c of Object.values(merged.claims)) { const r = putClaim(dir, c); if (r.ok && !r.existed) claims++; - for (const o of merged.evidence[c.id] ?? []) { - const a = appendEvidence(dir, c.id, o); - if (a.ok && !a.deduped) outcomes++; + for (const log of LOGS) { + for (const rec of merged[log][c.id] ?? []) { + const a = appendRecord(dir, log, c.id, rec); + if (a.ok && !a.deduped) records++; + } } } - for (const [id, t] of Object.entries(merged.tombstones)) tombstone(dir, id, t); reindex(dir); - return { claims, outcomes }; + return { claims, records }; } /** Regenerate LEDGER.md — the human index (like recall's MEMORY.md). */ export function reindex(dir, nowDay = 0) { - const claims = loadClaims(dir); - mkdirSync(dir, { recursive: true }); - const rows = claims + const rows = loadClaims(dir) .filter((c) => !c.tombstone) .map((c) => `- \`${c.id.slice(0, 12)}\` ${c.kind} · val ${val(c, nowDay).toFixed(2)}`); + mkdirSync(dir, { recursive: true }); writeFileSync( join(dir, "LEDGER.md"), ["# Proof-Carrying Memory ledger", "", ...rows, ""].join("\n"), @@ -165,43 +213,51 @@ export function reindex(dir, nowDay = 0) { } /** - * Normal-form check (CI-friendly): every claim parses and matches its address, every - * evidence line parses with a valid shape and a ref, no secrets anywhere. + * Normal-form check (CI-friendly): every claim parses and matches its address; every + * log line parses, carries a TRUE content hash, and (for evidence) names a known + * oracle with the table weight; no secrets anywhere. Everything loadState silently + * skips, verify names. * @returns {{ok:boolean, claims:number, outcomes:number, issues:string[]}} */ export function verify(dir) { const issues = []; let claims = 0; let outcomes = 0; - const claimsRoot = join(dir, "claims"); - if (existsSync(claimsRoot)) { - for (const shard of readdirSync(claimsRoot).sort()) { - for (const f of readdirSync(join(claimsRoot, shard)) - .filter((f) => f.endsWith(".json")) - .sort()) { - const raw = readFileSync(join(claimsRoot, shard, f), "utf8"); - const c = readJson(join(claimsRoot, shard, f)); - const id = f.replace(/\.json$/, ""); - if (!c) issues.push(`claim ${id}: unparseable`); - else if (claimId(c.kind, c.body, c.scope) !== id) issues.push(`claim ${id}: id mismatch`); - else claims++; - if (SECRET_RE.test(raw)) issues.push(`claim ${id}: contains secret-like content`); - } + const ids = []; + for (const { id, raw, claim } of walkClaimFiles(dir)) { + if (!claim) issues.push(`claim ${id}: unparseable or id mismatch`); + else { + claims++; + ids.push(id); } + if (SECRET_RE.test(raw)) issues.push(`claim ${id}: contains secret-like content`); } - const evRoot = join(dir, "evidence"); - if (existsSync(evRoot)) { - for (const f of readdirSync(evRoot).filter((f) => f.endsWith(".log"))) { + for (const log of LOGS) { + const logRoot = join(dir, log); + if (!existsSync(logRoot)) continue; + for (const f of readdirSync(logRoot).filter((f) => f.endsWith(".log"))) { const id = f.replace(/\.log$/, ""); - for (const [n, line] of readFileSync(join(evRoot, f), "utf8").split("\n").entries()) { + for (const [n, line] of readFileSync(join(logRoot, f), "utf8").split(/\r?\n/).entries()) { if (!line.trim()) continue; + const where = `${log} ${id}:${n + 1}`; let o = null; try { o = JSON.parse(line); } catch {} - if (!o?.h || !o.ref || !o.oracle) issues.push(`evidence ${id}:${n + 1}: invalid outcome`); - else outcomes++; - if (SECRET_RE.test(line)) issues.push(`evidence ${id}:${n + 1}: secret-like content`); + if (!o?.h) { + issues.push(`${where}: unparseable or missing hash`); + continue; + } + const { h, ...rest } = o; + if (sealRecord(rest).h !== h) { + issues.push(`${where}: content hash mismatch (forged/corrupt)`); + } else if (log === "evidence") { + if (!validOutcome(o)) issues.push(`${where}: invalid outcome (oracle/result/ref)`); + else if (o.w !== ORACLES[o.oracle].w) + issues.push(`${where}: recorded weight ${o.w} != oracle table ${ORACLES[o.oracle].w}`); + else outcomes++; + } + if (SECRET_RE.test(line)) issues.push(`${where}: secret-like content`); } } } @@ -217,7 +273,8 @@ export function pruneToAttic(dir, id) { return { ok: true }; } -/** Counts + val distribution for `forge ledger stats` and the dashboard. */ +/** Counts + val distribution for `forge ledger stats` and the dashboard. Buckets use + * the protocol's DORMANT_VAL threshold (and its mirror) — never a local literal. */ export function stats(dir, nowDay = 0) { const claims = loadClaims(dir); const byKind = {}; @@ -225,8 +282,8 @@ export function stats(dir, nowDay = 0) { for (const c of claims) { byKind[c.kind] = (byKind[c.kind] ?? 0) + 1; const v = val(c, nowDay); - if (v < 0.35) buckets.dormant++; - else if (v < 0.65) buckets.uncertain++; + if (v < DORMANT_VAL) buckets.dormant++; + else if (v < 1 - DORMANT_VAL) buckets.uncertain++; else buckets.trusted++; } return { diff --git a/src/lessons.js b/src/lessons.js index 966ff9d..8b1e2ce 100644 --- a/src/lessons.js +++ b/src/lessons.js @@ -24,7 +24,12 @@ export const SIGNALS = { S7: { weight: 0.35, family: "outcome" }, // lint/type/build regression introduced }; -export const SCOPE_WEIGHT = { symbol: 1.0, dir: 0.8, repo: 0.6, global: 0.4 }; +// One source of truth for scope weighting — the PCM ledger core owns it now, so the +// legacy injection path and Eq.-3 retrieval can never rank the same memory with +// silently different weights during the bridge window. +import { SCOPE_WEIGHT } from "./ledger.js"; + +export { SCOPE_WEIGHT }; /** * Score whether a cluster of signals reflects a real mistake. diff --git a/src/recall.js b/src/recall.js index 05e2b35..456f06c 100644 --- a/src/recall.js +++ b/src/recall.js @@ -36,6 +36,17 @@ export function add(store, name, body) { return { ok: true, slug }; } +/** Parse one stored fact back into {name, text} — THE parser for the fact format + * (`# name\n\ntext`), CRLF-tolerant so a Windows checkout can't fork the format. + * Everything that reads fact files (bridge import, consolidation) must use this. */ +export function readFact(store, slug) { + const path = join(factsDir(store), `${slug}.md`); + if (!existsSync(path)) return null; + const raw = readFileSync(path, "utf8").replace(/\r\n/g, "\n"); + const m = raw.match(/^# (.*)\n\n([\s\S]*)$/); + return m ? { name: m[1].trim(), text: m[2].trim() } : { name: slug, text: raw.trim() }; +} + export function list(store) { const dir = factsDir(store); if (!existsSync(dir)) return []; diff --git a/test/ledger.test.js b/test/ledger.test.js index 21bd26f..6c9d477 100644 --- a/test/ledger.test.js +++ b/test/ledger.test.js @@ -3,7 +3,9 @@ import { test } from "node:test"; import { canonicalize, claimId, + claimText, clusters, + isDormant, jaccard, liveClaims, mergeStates, @@ -12,6 +14,8 @@ import { rec, retrieve, score, + sealRecord, + shingles, sketch, val, } from "../src/ledger.js"; @@ -54,6 +58,13 @@ test("claimId: provenance and evidence never affect the address (teammates conve assert.equal(a.claim.id, b.claim.id); }); +test("mintClaim: normalizes non-JSON values so different Dates can't collide on one id", () => { + const d1 = mintClaim({ kind: "fact", body: { name: "d", text: new Date(0) } }); + const d2 = mintClaim({ kind: "fact", body: { name: "d", text: new Date(86400000) } }); + assert.ok(d1.ok && d2.ok); + assert.notEqual(d1.claim.id, d2.claim.id, "Dates serialize to ISO strings, not {}"); +}); + test("mintClaim: refuses secrets, unknown kinds, and non-object bodies", () => { const s = mintClaim({ kind: "fact", body: { name: "k", text: fakeAnthropic() } }); assert.equal(s.ok, false); @@ -79,7 +90,7 @@ const mkClaim = (evidence = []) => { return { ...m.claim, evidence }; }; const ev = (result, t, oracle = "test.run") => - outcomeRecord({ oracle, result, ref: `r:${result}:${t}`, t }).outcome; + outcomeRecord({ oracle, result, ref: `r:${result}:${t}:${oracle}`, t }).outcome; test("val: fresh claim sits at the 0.5 prior; confirms raise; contradictions lower", () => { assert.equal(val(mkClaim(), 0), 0.5); @@ -90,14 +101,18 @@ test("val: fresh claim sits at the 0.5 prior; confirms raise; contradictions low test("val: monotone in confirmations (more independent evidence is never worse)", () => { let prev = 0.5; for (let n = 1; n <= 5; n++) { - const v = val(mkClaim(Array.from({ length: n }, () => ev("confirm", 0, "ci.run"))), 0); + const outs = Array.from( + { length: n }, + (_, i) => outcomeRecord({ oracle: "ci.run", result: "confirm", ref: `r:${i}`, t: 0 }).outcome, + ); + const v = val(mkClaim(outs), 0); assert.ok(v > prev, `val(${n} confirms)=${v} must exceed ${prev}`); prev = v; } }); test("val: decays toward the PRIOR (uncertainty), never toward false", () => { - const confirmed = mkClaim([ev("confirm", 0), ev("confirm", 0)]); + const confirmed = mkClaim([ev("confirm", 0), ev("confirm", 0, "ci.run")]); const now = val(confirmed, 0); const later = val(confirmed, 90); // two half-lives const muchLater = val(confirmed, 900); @@ -115,6 +130,27 @@ test("val: oracle weight matters — a human revert outweighs a behavioral signa assert.ok(human < behav, "stronger oracle pulls harder"); }); +test("val: forged evidence buys nothing — weight comes from the ORACLES table, unknown oracles are ignored", () => { + // A hand-edited log line claiming w=50 on the strongest oracle: + const forgedWeight = { ...ev("confirm", 0, "human.revert"), w: 50 }; + const honest = val(mkClaim([ev("confirm", 0, "human.revert")]), 0); + assert.equal( + val(mkClaim([forgedWeight]), 0), + honest, + "stored w is audit metadata, never trusted", + ); + // A record naming an oracle that doesn't exist: + const ghost = sealRecord({ + oracle: "made.up", + result: "confirm", + ref: "x", + t: 0, + w: 1, + author: "", + }); + assert.equal(val(mkClaim([ghost]), 0), 0.5, "unknown oracle contributes nothing"); +}); + test("rec: recency keys on the latest evidence, else the mint day", () => { assert.equal(rec(mkClaim(), 0), 1); assert.ok(rec(mkClaim(), 45) < rec(mkClaim(), 1)); @@ -122,8 +158,25 @@ test("rec: recency keys on the latest evidence, else the mint day", () => { assert.ok(rec(fresh, 45) > rec(mkClaim(), 45), "new evidence refreshes recency"); }); +test("isDormant: repeated strong contradictions sink a claim below the retrieval floor", () => { + assert.equal(isDormant(mkClaim(), 0), false, "the prior is not dormant"); + const sunk = mkClaim([ + ev("contradict", 0, "human.revert"), + ev("contradict", 0, "human.accept"), + ev("contradict", 0, "test.run"), + ev("contradict", 0, "ci.run"), + ]); + assert.equal(isDormant(sunk, 0), true); +}); + // --- similarity ---------------------------------------------------------------------- +test("shingles: 4-token windows over normalized text; short texts fall back to tokens", () => { + assert.equal([...shingles("Check the CALLERS first, always")].length, 2); + assert.deepEqual([...shingles("two words")], ["two words"]); + assert.deepEqual([...shingles("")], []); +}); + test("sketch/jaccard: identical text = 1, disjoint ≈ 0, near-duplicates score high", () => { const a = sketch("always run the impacted tests before editing shared utils"); assert.equal(jaccard(a, sketch("always run the impacted tests before editing shared utils")), 1); @@ -136,6 +189,25 @@ test("sketch: deterministic across calls (no randomness — ids and sketches are assert.deepEqual(sketch("some stable text here"), sketch("some stable text here")); }); +test("claimText: every retrievable kind exposes its human text (not canonical JSON)", () => { + const lesson = mintClaim({ + kind: "lesson", + body: { + whatWentWrong: "w", + correctedBehavior: "c", + trigger: { keywords: ["k"], symbols: ["s"] }, + }, + }).claim; + assert.equal(claimText(lesson), "w c k s"); + const fact = mintClaim({ kind: "fact", body: { name: "n", text: "t" } }).claim; + assert.equal(claimText(fact), "n t"); + const diag = mintClaim({ + kind: "diagnosis", + body: { signature: "sig", note: "root cause" }, + }).claim; + assert.equal(claimText(diag), "sig root cause"); +}); + test("clusters: near-duplicates group, distinct claims stay apart", () => { const long = "before renaming any exported symbol in the shared utilities package always query the " + @@ -177,7 +249,12 @@ test("score: outcome-confirmed claims outrank merely-similar unconfirmed ones", test("retrieve: excludes tombstoned and dormant claims, caps at budget", () => { const alive = mkClaim([ev("confirm", 0)]); const dead = { ...mkClaim(), tombstone: { reason: "retracted", t: 0, author: "" } }; - const dormant = mkClaim(Array.from({ length: 4 }, (_, i) => ev("contradict", 0, "human.revert"))); + const dormant = mkClaim([ + ev("contradict", 0, "human.revert"), + ev("contradict", 0, "human.accept"), + ev("contradict", 0, "test.run"), + ev("contradict", 0, "ci.run"), + ]); const out = retrieve("body", [alive, dead, dormant], { nowDay: 0, budget: 10 }); assert.deepEqual( out.map((r) => r.claim.id), @@ -188,9 +265,12 @@ test("retrieve: excludes tombstoned and dormant claims, caps at budget", () => { // --- the CRDT merge: the semilattice laws property-tested ---------------------------- -const state = (claims, evidence = {}, tombstones = {}) => ({ +const tomb = (reason, t, author = "") => sealRecord({ author, reason, t }); +const prov = (author, t) => sealRecord({ agent: "test", author, t }); +const state = (claims, evidence = {}, tombstones = {}, provenance = {}) => ({ claims: Object.fromEntries(claims.map((c) => [c.id, c])), evidence, + provenance, tombstones, }); @@ -202,9 +282,14 @@ test("mergeStates: commutative, associative, idempotent — replicas converge in body: { whatWentWrong: "w", correctedBehavior: "c", trigger: {} }, t: 3, }).claim; - const sA = state([c1, c2], { [c1.id]: [ev("confirm", 1)] }); - const sB = state([c2, c3], { [c1.id]: [ev("confirm", 1), ev("contradict", 2)] }); - const sC = state([c3], {}, { [c2.id]: { reason: "retracted", t: 4, author: "bob" } }); + const sA = state([c1, c2], { [c1.id]: [ev("confirm", 1)] }, {}, { [c1.id]: [prov("alice", 1)] }); + const sB = state( + [c2, c3], + { [c1.id]: [ev("confirm", 1), ev("contradict", 2)] }, + { [c2.id]: [tomb("retracted", 4, "bob")] }, + { [c1.id]: [prov("bob", 2)] }, + ); + const sC = state([c3], {}, { [c2.id]: [tomb("duplicate", 5, "carol")] }); const canon = (s) => canonicalize(liveClaims(s)); // commutativity @@ -220,6 +305,21 @@ test("mergeStates: commutative, associative, idempotent — replicas converge in assert.equal(canon(mergeStates(m, sA)), canon(m), "absorbing a subset is a no-op"); }); +test("mergeStates: concurrent retractions both survive; the view picks one deterministically", () => { + const c = mintClaim({ kind: "fact", body: { name: "x", text: "y" }, t: 0 }).claim; + const sA = state([c], {}, { [c.id]: [tomb("wrong", 3, "alice")] }); + const sB = state([c], {}, { [c.id]: [tomb("stale", 2, "bob")] }); + const ab = mergeStates(sA, sB); + const ba = mergeStates(sB, sA); + assert.equal(ab.tombstones[c.id].length, 2, "both retraction records kept (grow-only set)"); + assert.equal( + canonicalize(liveClaims(ab)[0].tombstone), + canonicalize(liveClaims(ba)[0].tombstone), + "the single-record view is merge-order independent", + ); + assert.equal(liveClaims(ab)[0].tombstone.author, "bob", "earliest by (t, h) wins the view"); +}); + test("mergeStates: evidence unions dedupe by hash; val is identical after any merge order", () => { const c = mintClaim({ kind: "fact", body: { name: "x", text: "y" }, t: 0 }).claim; const e1 = ev("confirm", 1); diff --git a/test/ledger_bridge.test.js b/test/ledger_bridge.test.js index 383ad40..dfe2251 100644 --- a/test/ledger_bridge.test.js +++ b/test/ledger_bridge.test.js @@ -1,15 +1,22 @@ import assert from "node:assert/strict"; -import { mkdtempSync } from "node:fs"; +import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; -import { recordContradiction, recordMistake } from "../src/cortex.js"; +import { applyDistillation, recordContradiction, recordMistake } from "../src/cortex.js"; import { val } from "../src/ledger.js"; -import { importLegacy, lessonClaim, recordFactEvent } from "../src/ledger_bridge.js"; +import { + factClaim, + importLegacy, + lessonClaim, + reconcileFacts, + recordLessonEvent, + shadowFact, +} from "../src/ledger_bridge.js"; import { loadClaims, repoLedger } from "../src/ledger_store.js"; import { newLesson } from "../src/lessons.js"; import { save } from "../src/lessons_store.js"; -import { add as recallAdd } from "../src/recall.js"; +import { readFact, add as recallAdd } from "../src/recall.js"; const tmp = () => mkdtempSync(join(tmpdir(), "forge-bridge-")); const ctx = { symbols: ["computeTax"], files: ["src/tax.ts"], keywords: [] }; @@ -27,12 +34,13 @@ test("cortex dual-write: created lesson mints a claim with ZERO evidence (val = const claims = loadClaims(repoLedger(root)); assert.equal(claims.length, 1); assert.equal(claims[0].kind, "lesson"); - assert.equal(claims[0].body.legacyId, r.id); + assert.equal(claims[0].provenance.task, r.id, "legacy id rides in provenance, NOT the body"); + assert.equal(claims[0].body.legacyId, undefined, "body is pure content — teammates converge"); assert.equal(claims[0].evidence.length, 0, "creation is not confirmation"); assert.equal(val(claims[0], 1), 0.5); }); -test("cortex dual-write: a recurrence appends confirm evidence; a human reversal contradicts", () => { +test("cortex dual-write: recurrence confirms at bridge weight; reversal contradicts conservatively", () => { const root = tmp(); recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 1, episodeId: "e1" }); const r2 = recordMistake(root, { @@ -51,7 +59,7 @@ test("cortex dual-write: a recurrence appends confirm evidence; a human reversal result: claim.evidence[0].result, ref: claim.evidence[0].ref, }, - { oracle: "cortex.episode", result: "confirm", ref: "episode:e2" }, + { oracle: "cortex.episode", result: "confirm", ref: "episode:e2#n1" }, ); const before = val(claim, 2); assert.ok(before > 0.5); @@ -59,24 +67,125 @@ test("cortex dual-write: a recurrence appends confirm evidence; a human reversal recordContradiction(root, { context: ctx, nowDay: 3, episodeId: "e3" }); claim = loadClaims(dir)[0]; assert.equal(claim.evidence.length, 2); - assert.equal(claim.evidence[1].oracle, "human.revert"); + assert.equal( + claim.evidence[1].oracle, + "cortex.episode", + "regex-detected reverts are NOT the full-weight human oracle", + ); assert.ok(val(claim, 3) < before, "the reversal pulled confidence down"); + assert.ok(val(claim, 3) > 0.35, "one noisy revert cannot instantly bury a confirmed lesson"); +}); + +test("cortex dual-write: same-day sessions with colliding episode ids stay distinct evidence", () => { + const root = tmp(); + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 1, episodeId: "ep_m0_x" }); + // Two later sessions the same day — detectEpisodes resets its counter, so both emit + // the same episode id. The evidence counter in the ref keeps them distinct. + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 5, episodeId: "ep_m0_x" }); + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 5, episodeId: "ep_m0_x" }); + const claim = loadClaims(repoLedger(root))[0]; + assert.equal(claim.evidence.length, 2, "both real confirmations recorded, none deduped away"); }); -test("recordFactEvent: mints a fact claim; refuses secrets end-to-end", () => { +test("applyDistillation supersedes: evidence carries over, template claim is tombstoned", () => { + const root = tmp(); + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 1, episodeId: "e1" }); + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 2, episodeId: "e2" }); + const dir = repoLedger(root); + const beforeClaim = loadClaims(dir)[0]; + assert.equal(beforeClaim.evidence.length, 1); + + const ok = applyDistillation(root, "lsn_computetax", { + whatWentWrong: "computeTax was edited without checking its callers.", + correctedBehavior: "Query the atlas for computeTax dependents before editing.", + }); + assert.equal(ok, true); + const claims = loadClaims(dir); + assert.equal(claims.length, 2); + const old = claims.find((c) => c.id === beforeClaim.id); + const distilled = claims.find((c) => c.id !== beforeClaim.id); + assert.match(old.tombstone.reason, new RegExp(`superseded-by:${distilled.id}`)); + assert.equal(distilled.evidence.length, 1, "history carried across the body rewrite"); + assert.equal( + distilled.body.correctedBehavior, + "Query the atlas for computeTax dependents before editing.", + ); +}); + +test("recordLessonEvent: direct contract — mint-only, then evidence on a later event", () => { + const root = tmp(); + const lesson = newLesson( + { + id: "l1", + trigger: { symbols: ["x"], files: [], keywords: [] }, + scope: "symbol", + whatWentWrong: "w", + correctedBehavior: "c", + }, + 0, + ); + const mintOnly = recordLessonEvent(root, lesson, { t: 0 }); + assert.equal(mintOnly.ok, true); + const withEv = recordLessonEvent(root, lesson, { result: "confirm", ref: "episode:e9#n1", t: 1 }); + assert.equal(withEv.ok, true); + assert.equal(mintOnly.id, withEv.id, "same content → same claim, evidence accumulates"); + const missingRef = recordLessonEvent(root, lesson, { result: "confirm", t: 2 }); + assert.equal(missingRef.ok, false, "evidence without a ref is rejected"); +}); + +test("shadowFact: mints, and supersedes the stale same-name claim on update", () => { const dir = join(tmp(), "ledger"); - assert.equal(recordFactEvent(dir, "deploy", "staging needs FLAG=1 first", 4).ok, true); + const first = shadowFact(dir, "api-base", "https://old.example", 1); + assert.equal(first.ok, true); + const second = shadowFact(dir, "api-base", "https://new.example", 2); + assert.equal(second.ok, true); const claims = loadClaims(dir); - assert.equal(claims.length, 1); - assert.deepEqual(claims[0].body, { name: "deploy", text: "staging needs FLAG=1 first" }); - const refused = recordFactEvent(dir, "creds", "api_key = topsecretvalue", 4); - assert.equal(refused.ok, false); - assert.equal(loadClaims(dir).length, 1); + const old = claims.find((c) => c.id === first.id); + const now = claims.find((c) => c.id === second.id); + assert.match(old.tombstone.reason, new RegExp(`superseded-by:${now.id}`)); + assert.equal(now.tombstone, undefined); + const refused = shadowFact(dir, "creds", "api_key = topsecretvalue", 3); + assert.equal(refused.ok, false, "secrets refused end-to-end"); +}); + +test("factClaim: trims name/text so the shadow path and the file-parse path mint one id", () => { + const a = factClaim("deploy", "run migrations first \n", 0); + const b = factClaim(" deploy ", "run migrations first", 5); + assert.equal(a.claim.id, b.claim.id); +}); + +test("readFact: CRLF files parse identically to LF files (no Windows fork of the format)", () => { + const store = tmp(); + recallAdd(store, "API Style", "REST with cursor pagination"); + const lf = readFact(store, "api-style"); + // Simulate a core.autocrlf checkout of the same fact file: + writeFileSync( + join(store, "facts", "api-style.md"), + "# API Style\r\n\r\nREST with cursor pagination\r\n", + ); + const crlf = readFact(store, "api-style"); + assert.deepEqual(lf, crlf); + assert.equal(factClaim(lf.name, lf.text).claim.id, factClaim(crlf.name, crlf.text).claim.id); +}); + +test("reconcileFacts: a fact deleted from the store is tombstoned in the ledger", () => { + const store = tmp(); + const dir = join(store, "ledger"); + recallAdd(store, "keep", "this fact stays"); + recallAdd(store, "extra", "this fact stays too"); + shadowFact(dir, "keep", "this fact stays", 1); + shadowFact(dir, "gone", "this was deleted from the store", 1); + const r = reconcileFacts(store, dir, 2); + assert.equal(r.ok, true); + assert.equal(r.removed, 1); + const claims = loadClaims(dir); + assert.equal(claims.find((c) => c.body.name === "gone").tombstone.reason, "removed-from-store"); + assert.equal(claims.find((c) => c.body.name === "keep").tombstone, undefined); }); -test("importLegacy: back-fills lessons (counts → dated outcomes) and facts; re-run is a no-op", () => { +test("importLegacy: back-fills pre-ledger history; skips claims already tracked live", () => { const root = tmp(); - // A legacy lesson with history: 3 confirmations, 1 contradiction. + // A legacy lesson with history that predates the ledger: 3 confirmations, 1 contradiction. const lesson = { ...newLesson( { @@ -116,12 +225,27 @@ test("importLegacy: back-fills lessons (counts → dated outcomes) and facts; re const v = val(imported, 20); assert.ok(v > 0.5, `3 confirms vs 1 contradiction should trust the lesson (val=${v})`); + // Re-run — even after the legacy counters move — is a no-op for tracked claims: + save(root, { ...lesson, evidenceCount: 4, lastConfirmedDay: 25 }); const again = importLegacy(root, { recallStore: brainStore, recallLedger: repoLedger(root), - nowDay: 20, + nowDay: 25, }); assert.deepEqual({ l: again.lessons, f: again.facts, o: again.outcomes }, { l: 0, f: 0, o: 0 }); + assert.equal(loadClaims(repoLedger(root)).find((c) => c.kind === "lesson").evidence.length, 4); +}); + +test("importLegacy: a lesson already shadow-written live never gets synthetic double-counts", () => { + const root = tmp(); + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 1, episodeId: "e1" }); + recordMistake(root, { signals: strongSignals, context: ctx, nowDay: 2, episodeId: "e2" }); + const before = loadClaims(repoLedger(root))[0]; + assert.equal(before.evidence.length, 1, "one live confirm"); + const r = importLegacy(root, { nowDay: 3 }); + assert.equal(r.lessons, 0, "claim already tracked"); + assert.equal(r.outcomes, 0, "no synthetic evidence for a live-tracked claim"); + assert.equal(loadClaims(repoLedger(root))[0].evidence.length, 1, "evidence unchanged"); }); test("lessonClaim: id is stable under count/status churn — confirms don't re-mint", () => { @@ -137,4 +261,7 @@ test("lessonClaim: id is stable under count/status churn — confirms don't re-m ); const churned = { ...base, evidenceCount: 9, status: "active", lastConfirmedDay: 99 }; assert.equal(lessonClaim(base).claim.id, lessonClaim(churned).claim.id); + // Different legacy filenames must NOT fork the id (it rides in provenance only): + const renamed = { ...base, id: "lsn_renamed_by_hand" }; + assert.equal(lessonClaim(base).claim.id, lessonClaim(renamed).claim.id); }); diff --git a/test/ledger_store.test.js b/test/ledger_store.test.js index fa3ada8..fbc1f1d 100644 --- a/test/ledger_store.test.js +++ b/test/ledger_store.test.js @@ -3,9 +3,10 @@ import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; -import { mintClaim, outcomeRecord, val } from "../src/ledger.js"; +import { mintClaim, outcomeRecord, sealRecord, val } from "../src/ledger.js"; import { appendEvidence, + getClaimByPrefix, importState, loadClaims, loadState, @@ -19,7 +20,8 @@ import { } from "../src/ledger_store.js"; const tmp = () => mkdtempSync(join(tmpdir(), "forge-ledger-")); -const fact = (name, text, t = 0) => mintClaim({ kind: "fact", body: { name, text }, t }).claim; +const fact = (name, text, t = 0) => + mintClaim({ kind: "fact", body: { name, text }, provenance: { author: "tester" }, t }).claim; const ev = (result, ref, t = 0) => outcomeRecord({ oracle: "test.run", result, ref, t }).outcome; test("putClaim/loadClaims: roundtrip; rewrite of the same claim is a no-op", () => { @@ -33,6 +35,34 @@ test("putClaim/loadClaims: roundtrip; rewrite of the same claim is a no-op", () assert.deepEqual(loaded[0].body, c.body); }); +test("claim file bytes are pure content — two authors' mints are byte-identical, provenance goes to the log", () => { + const dirA = tmp(); + const dirB = tmp(); + const a = mintClaim({ + kind: "fact", + body: { name: "x", text: "y" }, + provenance: { author: "alice" }, + t: 1, + }).claim; + const b = mintClaim({ + kind: "fact", + body: { name: "x", text: "y" }, + provenance: { author: "bob" }, + t: 9, + }).claim; + putClaim(dirA, a); + putClaim(dirB, b); + const path = (d, c) => join(d, "claims", c.id.slice(0, 2), `${c.id}.json`); + assert.equal( + readFileSync(path(dirA, a), "utf8"), + readFileSync(path(dirB, b), "utf8"), + "same id ⇒ same bytes on every replica — git can never conflict on a claim file", + ); + assert.ok(!readFileSync(path(dirA, a), "utf8").includes("alice"), "no provenance in claim bytes"); + const viewA = loadClaims(dirA)[0]; + assert.equal(viewA.provenance.author, "alice", "provenance preserved via its own log"); +}); + test("putClaim: refuses an id that doesn't match the content (no forged addresses)", () => { const dir = tmp(); const c = { ...fact("a", "b"), id: "0".repeat(64) }; @@ -41,7 +71,20 @@ test("putClaim: refuses an id that doesn't match the content (no forged addresse assert.match(r.reason, /id does not match/); }); -test("appendEvidence: appends, dedupes by hash, requires the claim to exist", () => { +test("putClaim: repairs a corrupt/truncated claim file instead of trusting existsSync", () => { + const dir = tmp(); + const c = fact("healme", "important content"); + putClaim(dir, c); + const path = join(dir, "claims", c.id.slice(0, 2), `${c.id}.json`); + writeFileSync(path, '{"kind":"fact","bo'); // killed mid-write + assert.equal(loadClaims(dir).length, 0, "corrupt claim is not trusted"); + const again = putClaim(dir, c); + assert.equal(again.ok, true); + assert.equal(again.existed, false, "repair reported as a fresh write"); + assert.equal(loadClaims(dir).length, 1, "claim is loadable again"); +}); + +test("appendEvidence: appends, dedupes by hash, requires the claim to exist and a valid outcome", () => { const dir = tmp(); const c = fact("f", "text"); putClaim(dir, c); @@ -50,11 +93,16 @@ test("appendEvidence: appends, dedupes by hash, requires the claim to exist", () assert.deepEqual(appendEvidence(dir, c.id, o), { ok: true, deduped: true }); assert.equal(readEvidence(dir, c.id).length, 1); assert.equal(appendEvidence(dir, "f".repeat(64), o).ok, false, "evidence for a ghost claim"); + assert.equal( + appendEvidence(dir, c.id, { oracle: "made.up", result: "confirm", ref: "x", h: "y" }).ok, + false, + "unknown oracle rejected at the store boundary too", + ); const loaded = loadClaims(dir)[0]; assert.ok(val(loaded, 3) > 0.5, "evidence is attached on load"); }); -test("corrupt files are quarantined, not fatal: bad claim skipped, bad evidence line skipped", () => { +test("corrupt files are quarantined, not fatal — and verify names what load skips", () => { const dir = tmp(); const good = fact("good", "content"); putClaim(dir, good); @@ -73,20 +121,58 @@ test("corrupt files are quarantined, not fatal: bad claim skipped, bad evidence assert.equal(loaded[0].evidence.length, 1, "corrupt evidence line skipped"); const v = verify(dir); assert.equal(v.ok, false, "verify reports what load silently skips"); - assert.ok(v.issues.some((i) => /id mismatch/.test(i))); + assert.ok(v.issues.some((i) => /unparseable or id mismatch/.test(i))); + assert.ok(v.issues.some((i) => /unparseable or missing hash/.test(i))); +}); + +test("verify: catches forged evidence — wrong content hash, unknown oracle, inflated weight", () => { + const dir = tmp(); + const c = fact("target", "forgery magnet"); + putClaim(dir, c); + appendEvidence(dir, c.id, ev("confirm", "run:legit")); + const logPath = join(dir, "evidence", `${c.id}.log`); + const forged = [ + // real-looking record whose h doesn't match its content + '{"author":"","h":"deadbeef","oracle":"test.run","ref":"x","result":"confirm","t":0,"w":0.8}', + // correctly sealed record naming an oracle that doesn't exist + JSON.stringify( + sealRecord({ author: "", oracle: "made.up", ref: "x", result: "confirm", t: 0, w: 1 }), + ), + ].join("\n"); + writeFileSync(logPath, `${readFileSync(logPath, "utf8")}${forged}\n`); + const v = verify(dir); + assert.equal(v.ok, false); + assert.ok(v.issues.some((i) => /content hash mismatch/.test(i))); assert.ok(v.issues.some((i) => /invalid outcome/.test(i))); + assert.equal(v.outcomes, 1, "only the legit outcome counts"); + // And the forged lines can't move confidence either (val ignores them): + const honest = val(loadClaims(dir)[0], 0); + assert.ok(honest > 0.5 && honest < 0.75, `val=${honest} reflects one real confirm only`); }); -test("tombstone: retracts from stats' live view but the claim file stays for audit", () => { +test("tombstone: append-only records; concurrent retractions coexist; stats reflects it", () => { const dir = tmp(); const c = fact("wrong", "this was retracted"); putClaim(dir, c); - assert.equal(tombstone(dir, c.id, { reason: "superseded", t: 5 }).ok, true); - const loaded = loadClaims(dir); - assert.equal(loaded[0].tombstone.reason, "superseded"); + assert.equal(tombstone(dir, c.id, { reason: "superseded", t: 5, author: "alice" }).ok, true); + assert.equal(tombstone(dir, c.id, { reason: "duplicate", t: 3, author: "bob" }).ok, true); + const loaded = loadClaims(dir)[0]; + assert.equal(loaded.tombstone.author, "bob", "earliest record is the view"); assert.equal(stats(dir).tombstoned, 1); }); +test("getClaimByPrefix: finds one claim via its shard without scanning the ledger", () => { + const dir = tmp(); + const c = fact("needle", "in a stack of shards"); + putClaim(dir, c); + appendEvidence(dir, c.id, ev("confirm", "run:1")); + const hit = getClaimByPrefix(dir, c.id.slice(0, 8)); + assert.equal(hit.id, c.id); + assert.equal(hit.evidence.length, 1); + assert.equal(getClaimByPrefix(dir, "zz"), null); + assert.equal(getClaimByPrefix(dir, "a"), null, "sub-shard prefixes are refused"); +}); + test("importState: semilattice import is idempotent and merges evidence", () => { const a = tmp(); const b = tmp(); @@ -101,9 +187,9 @@ test("importState: semilattice import is idempotent and merges evidence", () => const first = importState(a, loadState(b)); assert.equal(first.claims, 1, "only bob's new claim is new"); - assert.equal(first.outcomes, 1, "only the unseen outcome lands"); + assert.ok(first.records >= 1, "the unseen outcome lands"); const again = importState(a, loadState(b)); - assert.deepEqual({ c: again.claims, o: again.outcomes }, { c: 0, o: 0 }, "re-import is a no-op"); + assert.deepEqual({ c: again.claims, r: again.records }, { c: 0, r: 0 }, "re-import is a no-op"); assert.equal(loadClaims(a).length, 2); assert.equal(readEvidence(a, shared.id).length, 2); });