From 6f51f440d379bac3dbee30873ac032190ec6f058 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 08:23:40 +0000 Subject: [PATCH] =?UTF-8?q?feat(cli):=20DX,=20evidence=20&=20hardening=20?= =?UTF-8?q?=E2=80=94=20uniform=20--json,=20broader=20doctor,=20eval=20harn?= =?UTF-8?q?ess=20(Tranche=20D)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Uniform --json: doctor/route/preflight/verify/scope now emit machine-readable output (were human-only), so CI/scripts can gate on them. - forge doctor sees more silent misconfiguration: guard scripts present AND executable; jq/git availability (guards degrade without jq); atlas presence + freshness (stale graph misleads impact/verify); model-pricing staleness (>90d). - Evaluation harness (src/eval.js): the deterministic core of the prototype's mutation-testing idea — precision/recall/F1 of the impact oracle vs the edited-file-only baseline, checkable in CI. - model_tiers exports PRICING_CURRENCY + PRICING_VERIFIED (doctor uses the date). - One shared call-site extractor (src/extract.js): atlas.js and verify.js each kept a copy of the call regex + builtins ignore-list; now a single source so they can't drift. Deeper shell-guard fail-open hardening (rewriting protect-paths/secret-redact to avoid the no-jq degrade) is surfaced via the new doctor jq warning; a full jq-free parse rewrite is deferred as too fragile to do safely here. 222 tests pass (+4), typecheck + Biome clean, zero new runtime deps. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01HGZUYFdUb4EViXV5tRid19 --- CHANGELOG.md | 19 +++++++++++ src/atlas.js | 2 +- src/cli.js | 44 ++++++++++++++++++++---- src/doctor.js | 82 ++++++++++++++++++++++++++++++++++++++++++++- src/eval.js | 47 ++++++++++++++++++++++++++ src/extract.js | 82 +++++++++++++++++++++++++++++++++++++++++++++ src/model_tiers.js | 10 ++++-- src/verify.js | 77 ++---------------------------------------- test/doctor.test.js | 7 ++++ test/eval.test.js | 37 ++++++++++++++++++++ test/verify.test.js | 11 ++++++ 11 files changed, 334 insertions(+), 84 deletions(-) create mode 100644 src/eval.js create mode 100644 src/extract.js create mode 100644 test/eval.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 05b18af..4a2a804 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,25 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added +- **Uniform `--json`.** `doctor`, `route`, `preflight`, `verify`, and `scope` now accept `--json` + (previously only `impact`/`substrate`/`anchor` did) — so CI and scripts can gate on the health + check, the routed tier, the assumption gap, and the verification result. +- **`forge doctor` sees more silent misconfiguration.** New checks: guard scripts present **and + executable**, `jq`/`git` availability (several guards degrade without `jq`), atlas + **presence + freshness** (a stale graph misleads impact/verify), and **model-pricing staleness** + (warns when the verified date is >90 days old). +- **Evaluation harness (`src/eval.js`).** The deterministic core of the prototype's mutation-testing + idea: score the impact oracle's precision/recall/F1 over labeled cases and against the + edited-file-only baseline the paper measured against — so the graph-quality claim is checkable in CI. + +### Changed + +- **Model tiers carry a currency + a verified date.** `model_tiers.js` exports `PRICING_CURRENCY` + ("USD") and `PRICING_VERIFIED`, which `forge doctor` uses for the staleness warning. +- **One shared call-site extractor (`src/extract.js`).** `atlas.js` and `verify.js` each kept their + own copy of the call regex + builtins ignore-list; they now share one module so the two can't + drift apart. + - **Opt-in enforcing gate (`FORGE_ENFORCE=1`).** The substrate's assumption gate can now be a real *halt* (the paper's Eq 5 / M2 "block on insufficient input"), not just advice. On the Claude Code ambient path it blocks a prompt with **no concrete anchor at all** ("fix it", "make it better") — diff --git a/src/atlas.js b/src/atlas.js index 2deae1c..9f0afd3 100644 --- a/src/atlas.js +++ b/src/atlas.js @@ -5,6 +5,7 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { extname, join, relative } from "node:path"; import { adjudicate, asText, buildRunner, llmEnabled } from "./adjudicate.js"; +import { CALL_RE } from "./extract.js"; const IGNORE = new Set([ "node_modules", @@ -48,7 +49,6 @@ const RULES = { ".java": [{ re: /\b(?:class|interface|enum)\s+([A-Za-z_]\w*)/g, kind: "type" }], }; -const CALL_RE = /(?:^|[^.\w$])([A-Za-z_$][\w$]*)\s*\(/g; const IMPORT_RE = /(?:import\s+(?:[^"'\n]+\s+from\s+)?["']([^"']+)["']|require\(["']([^"']+)["']\)|^\s*(?:from\s+([\w.]+)\s+)?import\s+([\w*,\s]+))/gm; const BUILTINS = new Set([ diff --git a/src/cli.js b/src/cli.js index ac3afd9..8cf6928 100755 --- a/src/cli.js +++ b/src/cli.js @@ -125,6 +125,11 @@ async function run(argv) { if (cmd === "doctor") { const { doctor } = await import("./doctor.js"); const { results, failed } = doctor({ targetRoot: process.cwd() }); + if (argv.includes("--json")) { + console.log(JSON.stringify({ results, failed }, null, 2)); + if (failed) process.exitCode = 1; + return; + } const icon = { ok: "✓", warn: "!", fail: "✗" }; console.log(`${BRAND.brand} doctor\n`); for (const r of results) console.log(` ${icon[r.status]} ${r.label.padEnd(16)} ${r.note}`); @@ -223,7 +228,13 @@ async function run(argv) { } if (cmd === "verify") { const { verify } = await import("./verify.js"); + const json = argv.includes("--json"); const r = verify({ targetRoot: process.cwd() }); + if (json) { + console.log(JSON.stringify(r, null, 2)); + if (!r.ok) process.exitCode = 1; + return; + } console.log(`${BRAND.brand} verify\n`); console.log(` changed files: ${r.changedFiles.length}`); console.log( @@ -376,13 +387,21 @@ async function run(argv) { } if (cmd === "preflight") { const { preflightRepo, clarifyBlock } = await import("./preflight.js"); - const task = argv.slice(1).join(" "); + const json = argv.includes("--json"); + const task = argv + .slice(1) + .filter((a) => a !== "--json") + .join(" "); if (!task) { - console.error('usage: forge preflight ""'); + console.error('usage: forge preflight "" [--json]'); process.exitCode = 1; return; } const r = preflightRepo(process.cwd(), task); + if (json) { + console.log(JSON.stringify(r, null, 2)); + return; + } console.log(`${BRAND.brand} preflight — assumption check\n`); console.log( ` info-gap: ${r.gap.toFixed(2)} · completeness ${r.assumption.completeness.toFixed(2)} (referenced ${r.entities.symbols.length} symbol(s), ${r.entities.files.length} file(s))`, @@ -444,13 +463,21 @@ async function run(argv) { ); return; } - const task = argv.slice(1).join(" "); + const json = argv.includes("--json"); + const task = argv + .slice(1) + .filter((a) => a !== "--json") + .join(" "); if (!task) { - console.error('usage: forge route "" | forge route gateway'); + console.error('usage: forge route "" [--json] | forge route gateway'); process.exitCode = 1; return; } const rec = r.routeTask(process.cwd(), task); + if (json) { + console.log(JSON.stringify(rec, null, 2)); + return; + } console.log(`${BRAND.brand} route — cheapest capable model\n`); console.log( ` → ${rec.model.name} (${rec.tier}, $${rec.model.inCost}/$${rec.model.outCost} per M tok)`, @@ -501,13 +528,18 @@ async function run(argv) { } if (cmd === "scope") { const { decompose } = await import("./scope.js"); - const files = argv.slice(1); + const json = argv.includes("--json"); + const files = argv.slice(1).filter((a) => a !== "--json"); if (!files.length) { - console.error("usage: forge scope [file...]"); + console.error("usage: forge scope [file...] [--json]"); process.exitCode = 1; return; } const d = decompose(process.cwd(), files); + if (json) { + console.log(JSON.stringify(d, null, 2)); + return; + } console.log(`${BRAND.brand} scope — task decomposition\n`); if (d.independentGroups > 1) { console.log( diff --git a/src/doctor.js b/src/doctor.js index b0c15b2..825ff9e 100644 --- a/src/doctor.js +++ b/src/doctor.js @@ -1,11 +1,14 @@ // forge doctor — turn silent misconfiguration into an actionable pass/fail list // (chezmoi-doctor pattern). Exits non-zero only on hard failures, not warnings. -import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { accessSync, constants, existsSync, readdirSync, readFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; +import { isStale, load as loadAtlas } from "./atlas.js"; import { BRAND } from "./brand.js"; import { summary as cortexSummary } from "./cortex.js"; import { extractHash, hashContent } from "./emit/_shared.js"; +import { PRICING_VERIFIED } from "./model_tiers.js"; import { canonical } from "./sync.js"; const ok = (label, note = "") => ({ status: "ok", label, note }); @@ -14,6 +17,79 @@ const fail = (label, note = "") => ({ status: "fail", label, note }); const readJson = (p) => JSON.parse(readFileSync(p, "utf8")); +const hasBin = (bin) => { + try { + execFileSync(bin, ["--version"], { stdio: "ignore" }); + return true; + } catch { + return false; + } +}; + +// External tools the guards/commands depend on. jq is the important one — several guards +// (secret-redact, protect-paths) degrade to a naive parse or a no-op without it. +function checkTooling(out) { + out.push( + hasBin("jq") + ? ok("jq", "found — guards parse hook JSON safely") + : warn("jq", "not found — secret-redact/protect-paths degrade without it; install jq"), + ); + out.push( + hasBin("git") ? ok("git", "found") : warn("git", "not found — churn/impact/anchor need it"), + ); +} + +// Every guard the manifests reference must exist and be executable, or a hook silently no-ops. +function checkGuardsExecutable(out) { + const dir = join(BRAND.root, "global", "guards"); + if (!existsSync(dir)) return; // absence is already reported by checkLayers + const scripts = readdirSync(dir).filter((f) => f.endsWith(".sh")); + const notExec = scripts.filter((f) => { + try { + accessSync(join(dir, f), constants.X_OK); + return false; + } catch { + return true; + } + }); + out.push( + notExec.length + ? warn( + "guards exec", + `${notExec.length} not executable (chmod +x): ${notExec.slice(0, 3).join(", ")}`, + ) + : ok("guards exec", `${scripts.length} guard(s) executable`), + ); +} + +// Model prices drift; a stale table quietly misinforms the cost/route commands. +function checkPricing(out) { + const days = Math.floor((Date.now() - Date.parse(`${PRICING_VERIFIED}T00:00:00Z`)) / 86400000); + out.push( + Number.isFinite(days) && days > 90 + ? warn( + "model pricing", + `verified ${PRICING_VERIFIED} (${days}d ago) — re-verify via dev-radar`, + ) + : ok("model pricing", `verified ${PRICING_VERIFIED}`), + ); +} + +// The atlas backs impact/verify. A missing or STALE graph gives wrong blast-radius / hallucination +// results silently — surface it so the user rebuilds. +function checkAtlas(out, targetRoot) { + const atlas = loadAtlas(targetRoot); + if (!atlas) { + out.push(ok("atlas", "not built — run `forge atlas build` for impact/verify")); + return; + } + out.push( + isStale(targetRoot, atlas) + ? warn("atlas", "stale (files changed since build) — run `forge atlas build`") + : ok("atlas", `${atlas.symbols?.length ?? 0} symbols, fresh`), + ); +} + function checkNode(out) { const major = Number(process.versions.node.split(".")[0]); out.push( @@ -108,8 +184,12 @@ export function doctor({ targetRoot = process.cwd() } = {}) { checkNode(results); checkBrandConsistency(results); checkLayers(results); + checkGuardsExecutable(results); + checkTooling(results); checkInstall(results); checkDrift(results, targetRoot); + checkAtlas(results, targetRoot); + checkPricing(results); checkMcp(results, targetRoot); checkCortex(results, targetRoot); return { results, failed: results.filter((r) => r.status === "fail").length }; diff --git a/src/eval.js b/src/eval.js new file mode 100644 index 0000000..899c8b5 --- /dev/null +++ b/src/eval.js @@ -0,0 +1,47 @@ +// forge eval — a small, honest evaluation harness for the impact oracle. The Python prototype +// used mutation testing against a real suite; this ships the deterministic core of that idea so +// the atlas/impact quality claim is CHECKABLE in CI: for a set of {target → files that truly +// depend on it} cases, score the oracle's precision/recall/F1 and compare it to the naive +// "edited-file-only" baseline the paper measured against. Pure; no repo/network. +import { impact } from "./atlas.js"; + +/** Pure: precision/recall/F1 of a predicted set against ground truth. */ +export function score(predicted, groundTruth) { + const P = new Set(predicted); + const G = new Set(groundTruth); + let tp = 0; + for (const g of G) if (P.has(g)) tp += 1; + const precision = P.size ? tp / P.size : G.size ? 0 : 1; + const recall = G.size ? tp / G.size : 1; + const f1 = precision + recall ? (2 * precision * recall) / (precision + recall) : 0; + return { precision, recall, f1, tp, predicted: P.size, groundTruth: G.size }; +} + +const mean = (xs) => (xs.length ? xs.reduce((a, b) => a + b, 0) / xs.length : 0); + +/** + * Evaluate the impact oracle over labeled cases and against the edited-file-only baseline. + * @param {object} atlas + * @param {{target:string, expected:string[], editedFile?:string}[]} cases + * @returns {{oracle:{precision:number,recall:number,f1:number}, baseline:{recall:number}, n:number, perCase:object[]}} + */ +export function evalImpact(atlas, cases, opts = {}) { + const perCase = cases.map((c) => { + const predicted = impact(atlas, c.target, opts).impactedFiles; + const oracle = score(predicted, c.expected); + // Baseline: an agent that only "knows" the file it edited breaks nothing else it can see. + const baselineHits = c.editedFile && c.expected.includes(c.editedFile) ? [c.editedFile] : []; + const baseline = score(baselineHits, c.expected); + return { target: c.target, oracle, baseline }; + }); + return { + n: cases.length, + oracle: { + precision: mean(perCase.map((p) => p.oracle.precision)), + recall: mean(perCase.map((p) => p.oracle.recall)), + f1: mean(perCase.map((p) => p.oracle.f1)), + }, + baseline: { recall: mean(perCase.map((p) => p.baseline.recall)) }, + perCase, + }; +} diff --git a/src/extract.js b/src/extract.js new file mode 100644 index 0000000..811ccba --- /dev/null +++ b/src/extract.js @@ -0,0 +1,82 @@ +// forge extract — the ONE call-site extractor, shared so atlas.js and verify.js can't drift. +// `CALL_RE` matches a called identifier while skipping `.method(` member calls; `CALL_IGNORE` is +// the language/runtime builtins that are never project symbols. Kept dependency-free and pure. + +export const CALL_RE = /(?:^|[^.\w$])([A-Za-z_$][\w$]*)\s*\(/g; + +// Builtins/keywords that look like calls but are never a project-defined symbol. Superset of the +// two sets atlas.js and verify.js used to keep separately (so neither under-ignores). +export const CALL_IGNORE = new Set([ + "if", + "for", + "while", + "switch", + "catch", + "function", + "return", + "typeof", + "await", + "new", + "delete", + "void", + "yield", + "import", + "export", + "require", + "super", + "console", + "Math", + "JSON", + "Object", + "Array", + "String", + "Number", + "Boolean", + "Promise", + "Set", + "Map", + "WeakMap", + "Date", + "Error", + "RegExp", + "Symbol", + "Buffer", + "parseInt", + "parseFloat", + "isNaN", + "isFinite", + "setTimeout", + "setInterval", + "clearTimeout", + "clearInterval", + "fetch", + "structuredClone", + "process", + "assert", + // Python-ish builtins seen in mixed repos + "print", + "range", + "len", + "int", + "str", + "float", + "dict", + "list", + "set", + "tuple", + "bool", +]); + +/** Pure: the called identifiers in a block of source (member calls `.foo(` are skipped). */ +export function extractCalledSymbols(text, ignore = CALL_IGNORE) { + const found = new Set(); + for (const line of String(text).split("\n")) { + CALL_RE.lastIndex = 0; + let m; + while ((m = CALL_RE.exec(line))) { + const name = m[1]; + if (!ignore.has(name)) found.add(name); + } + } + return [...found]; +} diff --git a/src/model_tiers.js b/src/model_tiers.js index 6da9a9f..c6e1f0e 100644 --- a/src/model_tiers.js +++ b/src/model_tiers.js @@ -1,6 +1,12 @@ // forge model tiers — the routing target table. Cheapest capable model per complexity tier. -// Costs are $/million tokens (input/output), verified 2026-07-05; re-verify via dev-radar. -// The premise: a prime-number finder does not need Fable 5. Size the model to the task. +// Costs are per-million tokens (input/output) in PRICING_CURRENCY. The premise: a prime-number +// finder does not need Fable 5. Size the model to the task. + +/** Currency for every inCost/outCost below. */ +export const PRICING_CURRENCY = "USD"; +/** Date the prices were last checked. `forge doctor` warns when this goes stale (re-verify via dev-radar). */ +export const PRICING_VERIFIED = "2026-07-05"; + export const MODELS = { haiku: { id: "claude-haiku-4-5-20251001", diff --git a/src/verify.js b/src/verify.js index 1b69d6b..a001724 100644 --- a/src/verify.js +++ b/src/verify.js @@ -8,81 +8,10 @@ import { existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { build as buildAtlas, has, isStale, load as loadAtlas } from "./atlas.js"; -// Called identifiers that are language/runtime built-ins, not project symbols. -const IGNORE = new Set([ - "if", - "for", - "while", - "switch", - "catch", - "return", - "function", - "typeof", - "await", - "super", - "new", - "delete", - "void", - "yield", - "import", - "export", - "require", - "print", - "range", - "len", - "str", - "int", - "float", - "list", - "dict", - "set", - "tuple", - "bool", - "console", - "Math", - "JSON", - "Object", - "Array", - "String", - "Number", - "Boolean", - "Promise", - "Set", - "Map", - "WeakMap", - "Date", - "Error", - "RegExp", - "Symbol", - "Buffer", - "parseInt", - "parseFloat", - "isNaN", - "isFinite", - "setTimeout", - "setInterval", - "clearTimeout", - "clearInterval", - "fetch", - "structuredClone", - "process", - "assert", -]); +// Shared call-site extractor — one source of truth with atlas.js (they used to duplicate this). +export { extractCalledSymbols } from "./extract.js"; -/** Pure: called identifiers in a block of source (skips `.method(` calls). */ -export function extractCalledSymbols(text) { - const found = new Set(); - const re = /(?:^|[^.\w$])([A-Za-z_$][\w$]*)\s*\(/g; - for (const line of String(text).split("\n")) { - re.lastIndex = 0; - let m; - while ((m = re.exec(line))) { - const name = m[1]; - if (!IGNORE.has(name)) found.add(name); - } - } - return [...found]; -} +import { extractCalledSymbols } from "./extract.js"; /** Pure: which called symbols are defined nowhere in the atlas (possible hallucinations). */ export function findUnknownSymbols(atlas, symbols) { diff --git a/test/doctor.test.js b/test/doctor.test.js index 3f66fab..93731e1 100644 --- a/test/doctor.test.js +++ b/test/doctor.test.js @@ -47,3 +47,10 @@ test("doctor reports 'no lessons yet' on a fresh repo, and counts once learning const c1 = doctor({ targetRoot: learned }).results.find((r) => r.label === "cortex"); assert.match(c1.note, /1 active/); }); + +test("doctor surfaces the new tooling / guards-exec / atlas / pricing checks", () => { + const labels = doctor({ targetRoot: fixture() }).results.map((r) => r.label); + for (const l of ["guards exec", "jq", "atlas", "model pricing"]) { + assert.ok(labels.includes(l), `doctor runs the '${l}' check`); + } +}); diff --git a/test/eval.test.js b/test/eval.test.js new file mode 100644 index 0000000..2e2e664 --- /dev/null +++ b/test/eval.test.js @@ -0,0 +1,37 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { build } from "../src/atlas.js"; +import { evalImpact, score } from "../src/eval.js"; + +test("score computes precision/recall/f1", () => { + const s = score(["a", "b", "c"], ["b", "c", "d"]); + assert.equal(s.tp, 2); + assert.ok(Math.abs(s.precision - 2 / 3) < 1e-9); + assert.ok(Math.abs(s.recall - 2 / 3) < 1e-9); + assert.equal(score([], []).recall, 1, "nothing to find, nothing predicted → perfect recall"); +}); + +test("evalImpact: the oracle recalls more of the true blast radius than edited-file-only", () => { + const root = mkdtempSync(join(tmpdir(), "forge-eval-")); + // File and symbol names must differ (a file whose module name equals the symbol makes the + // call target ambiguous — a known atlas limitation, not what we're measuring here). + writeFileSync(join(root, "util.js"), "export function coreFn(){ return 1 }\n"); + writeFileSync( + join(root, "a.js"), + "import { coreFn } from './util.js'\nexport function a(){ return coreFn() }\n", + ); + writeFileSync( + join(root, "b.js"), + "import { coreFn } from './util.js'\nexport function b(){ return coreFn() }\n", + ); + const atlas = build({ root }); + const cases = [ + { target: "coreFn", expected: ["util.js", "a.js", "b.js"], editedFile: "util.js" }, + ]; + const r = evalImpact(atlas, cases); + assert.ok(r.oracle.recall >= r.baseline.recall, "oracle recall ≥ baseline"); + assert.ok(r.oracle.recall > 0.5, `oracle finds most dependents (recall ${r.oracle.recall})`); +}); diff --git a/test/verify.test.js b/test/verify.test.js index 182e37e..d12e32f 100644 --- a/test/verify.test.js +++ b/test/verify.test.js @@ -26,3 +26,14 @@ test("extractCalledSymbols dedupes", () => { const syms = extractCalledSymbols("foo()\nfoo()\nbar()"); assert.equal(syms.filter((s) => s === "foo").length, 1); }); + +test("shared extractor: atlas and verify use the same call-site extraction (no drift)", async () => { + const { extractCalledSymbols, CALL_IGNORE } = await import("../src/extract.js"); + // Calls must be separated — the leading-boundary regex consumes the separator, so adjacent + // calls like foo(bar()) only yield the outer one (shared, pre-existing behaviour). + const syms = extractCalledSymbols("const x = foo(); bar(); baz.method(); JSON.parse(y)"); + assert.ok(syms.includes("foo") && syms.includes("bar"), "top-level calls captured"); + assert.ok(!syms.includes("method"), "member call .method( is skipped"); + assert.ok(!syms.includes("JSON"), "builtins ignored"); + assert.ok(CALL_IGNORE.has("console")); +});