diff --git a/CHANGELOG.md b/CHANGELOG.md index de2d767..917c5f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,19 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Proof-carrying reuse cache (P3 of the substrate-v2 plan).** `forge reuse` turns + "reuse already-generated code" from prose into a deterministic system: verified code + becomes an `artifact` claim keyed by a normalized task fingerprint (volatile literals → + typed placeholders; MinHash sketch + 16×8 LSH banding for near-match), looked up through + the exact → near → adapt → miss ladder. An artifact serves ONLY while its proof holds — + confidence above the 0.6 floor (an unverified mint sits at the 0.5 prior and does not + serve) and every declared dependency still resolving in the atlas; a failed revalidation + appends a `graph.reval` contradiction, so stale code demotes itself for the whole team. + `forge reuse query|mint|stats`, a reuse stage in `forge substrate` (read-only on the + ambient hook path), and `src/metrics.js` — the stage-tagged `.forge/metrics.jsonl` the + cost model's measured savings are computed from. The `reuse-first` skill now calls the + cache before advising a repo search. + - **Team memory (P2 of the substrate-v2 plan).** The PCM ledger becomes shared: `forge ledger merge ` performs the conflict-free semilattice merge of any other ledger tree (a teammate's checkout, a worktree, a backup) — identical knowledge minted diff --git a/global/tools/reuse-first/SKILL.md b/global/tools/reuse-first/SKILL.md index ef1e75f..950d6fe 100644 --- a/global/tools/reuse-first/SKILL.md +++ b/global/tools/reuse-first/SKILL.md @@ -9,7 +9,14 @@ Default to reusing and following what exists. New code is a liability; the best change is the smallest one that fits the codebase. ## 1. Reuse before building -- Search the repo first (`grep`/`glob`/serena LSP): is there an existing util, +- **Ask the proof-carrying cache first**: `forge reuse query ""`. A hit is code this team already generated AND verified — its test/accept + evidence travels with it (`forge ledger blame ` shows why to trust it): + - **EXACT / NEAR hit** → use that artifact; do not regenerate. + - **ADAPT hit** → read it, start from it, generate only the delta. + - **miss** → build it, then `forge reuse mint "" --file --ref ` + so the next teammate (or session) gets the hit. +- Search the repo next (`grep`/`glob`/serena LSP): is there an existing util, component, hook, service, or pattern that already does this? Extend it. - If not in-repo, is there a maintained library? Vet it with `tech-selector` (current best from Context7 + web + GitHub health) before adding a dependency. diff --git a/src/cli.js b/src/cli.js index 6c5562d..64fc182 100755 --- a/src/cli.js +++ b/src/cli.js @@ -20,6 +20,7 @@ const COMMANDS = { spec: "spec-as-contract — init (OpenSpec) / lock / check drift", cortex: "self-correcting project memory — status / why ", ledger: "proof-carrying memory — stats / verify / show / blame / query / merge / import", + reuse: "proof-carrying code cache — query / mint --file / stats", preflight: "assumption check — what a task names that the repo doesn't define", route: "recommend the cheapest capable model for a task (+ gateway config)", impact: "predict blast radius for a symbol or file from the atlas graph", @@ -325,6 +326,107 @@ async function run(argv) { process.exitCode = 1; return; } + if (cmd === "reuse") { + const ru = await import("./reuse.js"); + const { load: loadAtlas } = await import("./atlas.js"); + const { epochDay } = await import("./util.js"); + const root = process.cwd(); + const nowDay = epochDay(); + const json = argv.includes("--json"); + const flagVal = (name) => { + const i = argv.indexOf(name); + return i >= 0 ? argv[i + 1] : undefined; + }; + const args = argv.filter( + (a, i) => !a.startsWith("--") && argv[i - 1] !== "--file" && argv[i - 1] !== "--ref", + ); + const sub = args[1] || "stats"; + if (sub === "query") { + const spec = args.slice(2).join(" "); + if (!spec) { + console.error('usage: forge reuse query "" [--json]'); + process.exitCode = 1; + return; + } + const r = ru.reuseQuery(root, spec, { atlas: loadAtlas(root), nowDay }); + if (json) + return console.log( + JSON.stringify( + { tier: r.tier, artifact: r.artifact?.id, jaccard: r.jaccard, reasons: r.reasons }, + null, + 2, + ), + ); + if (r.tier === "miss") { + console.log(" miss — nothing verified matches; generate, then `forge reuse mint` it"); + } else { + const a = r.artifact; + console.log( + ` ${r.tier.toUpperCase()} hit (similarity ${(r.jaccard ?? 1).toFixed(2)}) — ${a.body.form}${a.body.code?.path ? ` at ${a.body.code.path}` : ""}`, + ); + console.log( + ` claim ${a.id.slice(0, 12)} — \`forge ledger blame ${a.id.slice(0, 8)}\` for its proof`, + ); + if (r.tier === "adapt") + console.log( + " adapt tier: inject as a verified starting point, generate only the delta", + ); + } + for (const why of r.reasons) console.log(` note: ${why}`); + return; + } + if (sub === "mint") { + const spec = args.slice(2).join(" "); + const file = flagVal("--file"); + const ref = flagVal("--ref"); + if (!spec || !file) { + console.error( + 'usage: forge reuse mint "" --file [--ref ] [--json]', + ); + process.exitCode = 1; + return; + } + const { repoLedger } = await import("./ledger_store.js"); + const desc = ru.describeFile(root, file); + const r = ru.mintArtifact( + repoLedger(root), + { spec, form: "module", ...desc }, + ref + ? { evidence: { oracle: "test.run", result: "confirm", ref }, t: nowDay } + : { t: nowDay }, + ); + if (json) return console.log(JSON.stringify(r, null, 2)); + if (!r.ok) { + console.error(` ${r.reason}`); + process.exitCode = 1; + return; + } + console.log( + ` minted: ${r.id.slice(0, 12)} (${desc.iface.length} export(s), ${desc.deps.length} dep(s))`, + ); + console.log( + r.serves + ? " serving: yes — verification evidence attached" + : " serving: NOT YET — no evidence; attach a verified test/commit ref (--ref) or it stays at the 0.5 prior", + ); + return; + } + if (sub === "stats") { + const { summarize } = await import("./metrics.js"); + const s = summarize(root).cache ?? { events: 0, byOutcome: {}, savedEstimate: 0 }; + if (json) return console.log(JSON.stringify(s, null, 2)); + console.log(`${BRAND.brand} reuse — proof-carrying code cache\n`); + console.log(` lookups: ${s.events}`); + for (const [o, n] of Object.entries(s.byOutcome)) console.log(` ${o}: ${n}`); + console.log(` est. tokens saved: ${s.savedEstimate}`); + return; + } + console.error( + `reuse: unknown subcommand "${sub}" (query | mint --file | stats)`, + ); + process.exitCode = 1; + return; + } if (cmd === "atlas") { const a = await import("./atlas.js"); const sub = argv[1] || "build"; diff --git a/src/metrics.js b/src/metrics.js new file mode 100644 index 0000000..6ee7964 --- /dev/null +++ b/src/metrics.js @@ -0,0 +1,54 @@ +// forge metrics — the measurement backbone of the cost model +// (docs/plans/substrate-v2/05-cost-model.md): every substrate stage appends one +// JSONL line, and savings are later ARITHMETIC on these lines, never an estimate +// asserted after the fact. Append-only, corrupt-line tolerant, best-effort — a +// metrics failure must never break the stage it measures. +import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; + +export const metricsPath = (root = process.cwd()) => join(root, ".forge", "metrics.jsonl"); + +/** + * Record one stage event: { t?, stage, outcome?, tokensIn?, tokensOut?, savedEstimate?, + * tier?, ref?, ... }. `t` defaults to wall-clock ms (metrics are telemetry, not + * content-addressed protocol state — clock use is fine here, unlike the ledger). + */ +export function record(root, entry) { + try { + const dir = join(root, ".forge"); + mkdirSync(dir, { recursive: true }); + appendFileSync(metricsPath(root), `${JSON.stringify({ t: Date.now(), ...entry })}\n`); + return { ok: true }; + } catch { + return { ok: false }; + } +} + +/** All events, oldest first (corrupt lines skipped). Optional stage filter. + * @param {string} root + * @param {{stage?: string}} [opts] */ +export function read(root, { stage } = {}) { + const path = metricsPath(root); + if (!existsSync(path)) return []; + const out = []; + for (const line of readFileSync(path, "utf8").split("\n")) { + if (!line.trim()) continue; + try { + const e = JSON.parse(line); + if (!stage || e.stage === stage) out.push(e); + } catch {} + } + return out; +} + +/** Counts by stage → outcome, plus summed token-saving estimates per stage. */ +export function summarize(root) { + const stages = {}; + for (const e of read(root)) { + const s = (stages[e.stage] ??= { events: 0, byOutcome: {}, savedEstimate: 0 }); + s.events++; + if (e.outcome) s.byOutcome[e.outcome] = (s.byOutcome[e.outcome] ?? 0) + 1; + if (Number.isFinite(e.savedEstimate)) s.savedEstimate += e.savedEstimate; + } + return stages; +} diff --git a/src/reuse.js b/src/reuse.js new file mode 100644 index 0000000..fabeb9a Binary files /dev/null and b/src/reuse.js differ diff --git a/src/substrate.js b/src/substrate.js index dcf386b..00814ca 100644 --- a/src/substrate.js +++ b/src/substrate.js @@ -12,8 +12,10 @@ import { matchingLessons } from "./cortex.js"; import { leanRepo } from "./lean.js"; import { load as loadLessons } from "./lessons_store.js"; import { clarifyBlock, preflightRepo, referencedEntities } from "./preflight.js"; +import { reusePeek, reuseQuery } from "./reuse.js"; import { routeTask } from "./route.js"; import { decompose } from "./scope.js"; +import { epochDay } from "./util.js"; function loadSubstrateSpec() { const path = join(dirname(dirname(fileURLToPath(import.meta.url))), "source", "substrate.json"); @@ -214,6 +216,24 @@ export function substrateCheck( // impacted files change — the impacted files that ARE tests, plus each impacted source file's // sibling test. Cheap, exact-ish, and surfaced BEFORE the edit (not after, like verify). const predictedTests = predictFailingTests(root, impactedFiles); + // P3 reuse stage: has this team already built (and verified) this? The explicit gate + // meters + writes evidence (reuseQuery); the ambient hook path stays read-only + // (reusePeek) so a per-prompt hook never appends to the ledger or metrics. + const reuse = (() => { + try { + const opts = { atlas, nowDay: epochDay() }; + const r = allowBuild ? reuseQuery(root, text, opts) : reusePeek(root, text, opts); + return { + tier: r.tier, + artifact: r.artifact + ? { id: r.artifact.id, path: r.artifact.body.code?.path, form: r.artifact.body.form } + : undefined, + jaccard: r.jaccard, + }; + } catch { + return { tier: "miss" }; // cache trouble must never block the gate + } + })(); const scopedFiles = [...new Set([...entities.files, ...impactedFiles])]; const scope = scopedFiles.length ? decompose(root, scopedFiles) @@ -229,6 +249,7 @@ export function substrateCheck( clarify: clarifyBlock(preflight), route, entities, + reuse, impact: { targets: impactTargets, reports: impacts, impactedFiles, predictedTests }, scope, memory: { @@ -343,6 +364,13 @@ export function renderSubstrate(result) { ` route: ${result.route.model.name} (${result.route.tier}) · complexity ${result.route.score.toFixed(2)}`, ); if (result.route.reasons.length) lines.push(` driven by: ${result.route.reasons.join(", ")}`); + if (result.reuse && result.reuse.tier !== "miss") { + const a = result.reuse.artifact; + lines.push( + "", + ` reuse: ${result.reuse.tier.toUpperCase()} hit — verified ${a?.form ?? "artifact"}${a?.path ? ` at ${a.path}` : ""} (\`forge ledger show ${a?.id.slice(0, 8)}\`) — start from it, don't regenerate`, + ); + } lines.push("", ` impact: ${result.impact.impactedFiles.length} file(s) predicted`); for (const file of result.impact.impactedFiles.slice(0, 10)) lines.push(` - ${file}`); if (result.impact.impactedFiles.length > 10) diff --git a/test/reuse.test.js b/test/reuse.test.js new file mode 100644 index 0000000..9199f5c --- /dev/null +++ b/test/reuse.test.js @@ -0,0 +1,220 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { val } from "../src/ledger.js"; +import { loadClaims, readEvidence, repoLedger } from "../src/ledger_store.js"; +import { read as readMetrics, record, summarize } from "../src/metrics.js"; +import { + artifactClaim, + bandKeys, + describeFile, + fingerprint, + lookup, + mintArtifact, + normalizeSpec, + reuseQuery, + revalidate, +} from "../src/reuse.js"; + +const tmp = () => mkdtempSync(join(tmpdir(), "forge-reuse-")); + +// --- normalization ------------------------------------------------------------------- + +test("normalizeSpec: volatile literals become typed placeholders; prose lowercases", () => { + assert.equal( + normalizeSpec("Add pagination to listUsers with pageSize 25 in src/api/users.ts"), + "add pagination to ⟨ident⟩ with ⟨ident⟩ ⟨num⟩ in ⟨path⟩", + ); + assert.equal(normalizeSpec('sort by "created_at" DESC'), "sort by ⟨str⟩ desc"); + assert.equal(normalizeSpec("use MAX_RETRIES from config.limits"), "use ⟨ident⟩ from ⟨ident⟩"); +}); + +test("normalizeSpec: the same task worded across teammates fingerprints identically", () => { + const a = normalizeSpec("Add pagination to listUsers with pageSize 25"); + const b = normalizeSpec("add pagination to listOrders with maxItems 100"); + assert.equal(a, b, "identifier/number shape matches — same near-neighborhood"); +}); + +test("fingerprint: exact key is context-sensitive (slice), sketch is stable", () => { + const f1 = fingerprint("build a rate limiter", "slice-a"); + const f2 = fingerprint("build a rate limiter", "slice-b"); + const f3 = fingerprint("build a RATE limiter", "slice-a"); + assert.notEqual(f1.exact, f2.exact, "different graph context → different exact key"); + assert.equal(f1.exact, f3.exact, "whitespace/case never fork the key"); + assert.deepEqual(f1.sketch, f2.sketch); +}); + +test("bandKeys: 16 deterministic bands; near-duplicates share at least one", () => { + const long = + "implement a token bucket rate limiter for the public api gateway with configurable " + + "burst size and a redis backing store for distributed counters across instances"; + const k1 = bandKeys(fingerprint(long).sketch); + const k2 = bandKeys(fingerprint(`${long} please`).sketch); + assert.equal(k1.length, 16); + assert.deepEqual(k1, bandKeys(fingerprint(long).sketch), "deterministic"); + assert.ok( + k1.some((k) => k2.includes(k)), + "near-duplicates collide in some band", + ); +}); + +// --- the lookup ladder (pure) ---------------------------------------------------------- + +const SPEC = + "implement a token bucket rate limiter for the public api gateway with configurable " + + "burst size and sliding window fallback"; +const verified = (spec, { slice = "", deps = [], evidence = 2 } = {}) => { + const c = artifactClaim( + { spec, slice, deps, code: { path: "src/limit.js", sha256: "x".repeat(64) } }, + 0, + ).claim; + c.evidence = Array.from({ length: evidence }, (_, i) => ({ + oracle: "test.run", + result: "confirm", + ref: `run:${i}`, + author: "ci", + t: 0, + w: 0.8, + h: `${i}`.repeat(64).slice(0, 64), + })); + return c; +}; + +test("lookup: exact tier — same normalized spec and slice, proof attached", () => { + const r = lookup([verified(SPEC)], SPEC.replace("implement", "IMPLEMENT"), { nowDay: 0 }); + assert.equal(r.tier, "exact"); + assert.equal(r.jaccard, 1); +}); + +test("lookup: the proof floor — an unverified artifact NEVER serves", () => { + const fresh = { ...verified(SPEC), evidence: [] }; + const r = lookup([fresh], SPEC, { nowDay: 0 }); + assert.equal(r.tier, "miss"); + assert.ok(r.reasons.some((x) => /below proof floor/.test(x))); +}); + +test("lookup: near tier for a reworded spec; adapt tier for a related one; miss for unrelated", () => { + const cache = [verified(SPEC)]; + const near = lookup(cache, SPEC.replace("sliding window fallback", "sliding window backup"), { + nowDay: 0, + }); + assert.equal(near.tier, "near"); + assert.ok(near.jaccard >= 0.8); + const adapt = lookup( + cache, + SPEC.replace("and sliding window fallback", "plus prometheus metrics exporters"), + { nowDay: 0 }, + ); + assert.equal(adapt.tier, "adapt"); + assert.ok(adapt.jaccard >= 0.6 && adapt.jaccard < 0.8); + assert.equal(lookup(cache, "write a css dark mode toggle", { nowDay: 0 }).tier, "miss"); +}); + +test("lookup: exact hits are slice-scoped — a different graph context falls through to near", () => { + const cache = [verified(SPEC, { slice: "ctx-A" })]; + const r = lookup(cache, SPEC, { slice: "ctx-B", nowDay: 0 }); + assert.equal(r.tier, "near", "same text, different context: serve-with-diff, not exact"); +}); + +test("revalidate: a vanished dependency blocks serving (stale cache can't ship)", () => { + const atlas = { symbols: [{ name: "validateInput" }] }; + const okArt = verified(SPEC, { deps: ["validateInput"] }); + const staleArt = verified(SPEC, { deps: ["validateInput", "removedHelper"] }); + assert.equal(revalidate(okArt, atlas).ok, true); + const rv = revalidate(staleArt, atlas); + assert.deepEqual({ ok: rv.ok, missing: rv.missing }, { ok: false, missing: ["removedHelper"] }); + const r = lookup([staleArt], SPEC, { atlas, nowDay: 0 }); + assert.equal(r.tier, "miss"); + assert.ok(r.reasons.some((x) => /failed revalidation: missing removedHelper/.test(x))); +}); + +// --- store level: fill → hit → demote -------------------------------------------------- + +test("mintArtifact + reuseQuery: verified fill serves; the serve refreshes structural evidence", () => { + const root = tmp(); + const dir = repoLedger(root); + const m = mintArtifact( + dir, + { spec: SPEC, code: { path: "src/limit.js", sha256: "a".repeat(64) } }, + { evidence: { oracle: "test.run", result: "confirm", ref: "run:42" }, t: 0 }, + ); + assert.equal(m.ok, true); + assert.equal(m.serves, true); + const atlas = { symbols: [] }; + const r = reuseQuery(root, SPEC, { atlas, nowDay: 1 }); + assert.equal(r.tier, "exact"); + const ev = readEvidence(dir, m.id); + assert.ok( + ev.some((e) => e.oracle === "graph.reval" && e.result === "confirm"), + "serving appended a structural confirmation", + ); + const metric = readMetrics(root, { stage: "cache" }).pop(); + assert.equal(metric.outcome, "hit_exact"); + assert.ok(metric.savedEstimate > 0); +}); + +test("reuseQuery: failed revalidation demotes the artifact in the ledger — for everyone", () => { + const root = tmp(); + const dir = repoLedger(root); + const m = mintArtifact( + dir, + { spec: SPEC, deps: ["goneHelper"], code: { path: "src/limit.js", sha256: "b".repeat(64) } }, + { evidence: { oracle: "test.run", result: "confirm", ref: "run:1" }, t: 0 }, + ); + const before = val(loadClaims(dir)[0], 0); + const r = reuseQuery(root, SPEC, { atlas: { symbols: [] }, nowDay: 0 }); + assert.equal(r.tier, "miss"); + const after = loadClaims(dir).find((c) => c.id === m.id); + assert.ok( + after.evidence.some((e) => e.oracle === "graph.reval" && e.result === "contradict"), + "the missing dep became a contradiction", + ); + assert.ok(val(after, 0) < before, "the cache pruned itself by ground truth"); + assert.equal(readMetrics(root, { stage: "cache" }).pop().outcome, "miss"); +}); + +test("mint without evidence is honest: stored but flagged as not serving", () => { + const root = tmp(); + const m = mintArtifact(repoLedger(root), { spec: SPEC, code: {} }, { t: 0 }); + assert.equal(m.ok, true); + assert.equal(m.serves, false); + assert.equal(reuseQuery(root, SPEC, { nowDay: 0 }).tier, "miss"); +}); + +// --- helpers ---------------------------------------------------------------------------- + +test("describeFile: extracts exports, relative-import deps, and a verifiable content hash", () => { + const root = tmp(); + mkdirSync(join(root, "src"), { recursive: true }); + writeFileSync( + join(root, "src", "mod.js"), + [ + 'import { helperA, helperB as hb } from "./util.js";', + 'import fs from "node:fs";', + "export function main() {}", + "export const CONFIG = 1;", + "function internal() {}", + ].join("\n"), + ); + const d = describeFile(root, "src/mod.js"); + assert.deepEqual(d.iface.sort(), ["CONFIG", "main"]); + assert.deepEqual(d.deps.sort(), ["helperA", "helperB"]); + assert.match(d.code.sha256, /^[0-9a-f]{64}$/); + assert.equal(d.code.path, "src/mod.js"); +}); + +test("metrics: record/read/summarize roundtrip; corrupt lines skipped", () => { + const root = tmp(); + record(root, { stage: "cache", outcome: "hit_exact", savedEstimate: 100 }); + record(root, { stage: "cache", outcome: "miss", savedEstimate: 0 }); + record(root, { stage: "route", outcome: "cheap" }); + writeFileSync(join(root, ".forge", "metrics.jsonl"), "not json\n", { flag: "a" }); + assert.equal(readMetrics(root).length, 3); + assert.equal(readMetrics(root, { stage: "cache" }).length, 2); + const s = summarize(root); + assert.equal(s.cache.events, 2); + assert.equal(s.cache.byOutcome.hit_exact, 1); + assert.equal(s.cache.savedEstimate, 100); +});