From 4058c8d459534fbf17f9b98683ce0e2734a74b9a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 6 Jul 2026 04:33:42 +0000 Subject: [PATCH] feat(substrate): opt-in, verified LLM adjudication across the cognitive substrate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the substrate's lexical judgments beyond static rules/regex with a thin, opt-in LLM layer that is never trusted blindly. One shared fail-safe primitive (src/adjudicate.js: a `claude -p` proposer, injectable runner, secret-refusing) is wired into every faculty; the model only PROPOSES and a per-faculty reconcile verifies each proposal against the deterministic rubric, the code graph, or a grep before it can move a verdict — the paper's tabayyun gate. - M2 assumption gate (preflight.js): LLM completeness/questions, bounded ±band by the rubric; the gate only ever tightens, questions must be grounded. - M1 routing (route.js): LLM complexity band is raise-only (max with rubric), so it never under-provisions on the model's word. - Impact/atlas (atlas.js): LLM proposes the coupled edges the regex misses; kept only if the file is real AND a grep confirms the reference. - M4 goal-drift (anchor.js): rescues an off-goal file off->on only, and only with a goal-referencing reason. - Memory (lessons.js): explicit `val` term — retrieval is now relevance x freshness x validity x scope; outcome-confirmed lessons rank first. - Orchestrator (substrate.js): threads an `llm` option + per-faculty `llm.provenance`, adds a `guarantees.llmVerified` bucket. Ambient hook stays deterministic unless FORGE_LLM_AMBIENT=1. Off by default (FORGE_LLM=1 to enable) — behaviour is byte-identical with the flag off. Zero new runtime dependency; any failure falls back to the deterministic path and never blocks. Cortex distiller now shares the primitive. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01HGZUYFdUb4EViXV5tRid19 --- CHANGELOG.md | 9 +++ docs/GUIDE.md | 11 +++ docs/cognitive-substrate/README.md | 34 ++++++++- source/substrate.json | 6 ++ src/adjudicate.js | 82 +++++++++++++++++++++ src/anchor.js | 81 ++++++++++++++++++++- src/atlas.js | 74 ++++++++++++++++++- src/cortex_distill.js | 14 +--- src/lessons.js | 38 +++++++--- src/preflight.js | 113 ++++++++++++++++++++++++++++- src/route.js | 50 ++++++++++++- src/substrate.js | 100 +++++++++++++++++++++++-- test/adjudicate.test.js | 88 ++++++++++++++++++++++ test/anchor.test.js | 47 ++++++++++++ test/lessons.test.js | 21 ++++++ test/preflight.test.js | 78 ++++++++++++++++++++ test/route.test.js | 45 +++++++++++- test/substrate.test.js | 47 ++++++++++++ 18 files changed, 895 insertions(+), 43 deletions(-) create mode 100644 src/adjudicate.js create mode 100644 test/adjudicate.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 913ad90..6f207d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Opt-in LLM adjudication for the substrate (`FORGE_LLM=1`)** — one shared, fail-safe `claude -p` proposer (`src/adjudicate.js`) wired thinly into the assumption gate (M2), model routing (M1), impact/blast-radius, and goal-drift (M4). The model only *proposes*; every proposal is verified against the deterministic rubric, the code graph, or a grep before it can move a verdict (routing raise-only; the gate tightens-only; impact edges must resolve and grep-confirm; goal-drift off→on only). Off by default — behaviour is unchanged unless enabled — never blocks, and the ambient Claude Code hook stays deterministic unless `FORGE_LLM_AMBIENT=1`. `forge substrate --json` now carries an `llm.provenance` map (`deterministic` / `llm-verified` / `llm-agreed`) per faculty for auditability. +- **Explicit memory `val` term** — lesson retrieval now decomposes into the white paper's `relevance × freshness × validity × scope`, with `validity()` (a ground-truth Beta posterior over confirmed vs. contradicted outcomes) exported and ranked so outcome-confirmed lessons outrank merely-recent ones. + +### Changed + +- **Unified the model-call path** — the Cortex distiller now shares the `adjudicate` runner instead of its own `claude` shell-out. + ## [0.4.0] - 2026-07-06 ### Added diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 41629f4..4f1fc48 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -423,6 +423,17 @@ Create `global/crew/.md` with frontmatter. It installs into `~/.claude/age | the verify checklist | `src/substrate.js` → `verificationChecklist()` | | when the ambient hook speaks | `src/substrate.js` → `substrateContext()` | | the cross-tool rule wording | `source/rules.json` → `substrate` section (then `forge init`) | +| opt-in LLM adjudication | `FORGE_LLM=1` (+ `FORGE_LLM_AMBIENT=1` for the hook); config in `source/substrate.json` → `llm` | + +### Opt into LLM-assisted judgments +By default every judgment is a deterministic rubric. `FORGE_LLM=1` adds a thin **proposer** +layer (`src/adjudicate.js`) — one shared `claude -p` primitive used by M2/M1/impact/M4. The +model never decides: each proposal is verified against the rubric, the code graph, or a grep +before it can move a verdict (routing raise-only; the gate tightens-only; impact edges must be +real + grep-confirmed; goal-drift off→on only). Any failure falls back to the deterministic +path, so the flag is safe to leave off or on. `--json` exposes `llm.provenance` per faculty. +Each faculty pairs a pure `*LLM` proposer with a `reconcile` step — extend by adding both, +never by trusting the model's answer directly. ### Support a new tool Add an emitter module in `src/emit/.js` (mirror an existing one like diff --git a/docs/cognitive-substrate/README.md b/docs/cognitive-substrate/README.md index ac1d548..42b6469 100644 --- a/docs/cognitive-substrate/README.md +++ b/docs/cognitive-substrate/README.md @@ -178,13 +178,39 @@ JSON default. Full JSON shape and the extension table: --- +## Optional: LLM-assisted judgments (`FORGE_LLM=1`) + +The substrate's default judgments are deterministic rubrics — no model call, no dependency. +Set **`FORGE_LLM=1`** to add a **thin, opt-in semantic layer** on top: a cheap `claude -p` +call proposes a completeness reading (M2), a complexity band (M1), the coupled edges the +regex graph misses (impact), and whether an off-goal file actually serves the goal (M4). + +The model is **never the judge — only a proposer.** Every proposal is *verified* before it can +move a verdict, in the direction of the paper's *tabayyun* gate (49:6): + +- **routing** can only be *raised* (`max` with the rubric), never lowered on the model's word; +- **the assumption gate only tightens** — it can add an ask, never clear a deterministic one; +- **impact edges** are kept only if the file is real *and* a grep confirms the reference; +- **goal-drift** rescues an off-goal file only with a goal-referencing reason (off→on only). + +It is **fail-safe**: any error, timeout, or unparseable reply falls back to the deterministic +path (behaviour is byte-identical with the flag off), and it **never blocks**. `--json` output +carries a `llm.provenance` map (`deterministic` / `llm-verified` / `llm-agreed`) per faculty so +every model-touched decision is auditable. Off by default; the ambient Claude Code hook stays +deterministic unless you also set `FORGE_LLM_AMBIENT=1`. Config lives in +[`source/substrate.json`](../../source/substrate.json) → `llm`. + +--- + ## Honest limits Heuristic, not benchmarked; the graph is regex-approximate (conservative, not a sound call -graph); assumption detection is lexical; auto-run needs a hook surface (ambient on Claude -Code, agent-invoked elsewhere). What's *asserted* is safe to gate on (repo grounding, graph -traversal, scope, routing arithmetic, test commands); everything else is *advisory*. Tests -and human corrections always win. Full list: +graph); assumption detection is lexical by default (opt into a verified LLM refinement with +`FORGE_LLM=1` — see above); auto-run needs a hook surface (ambient on Claude Code, +agent-invoked elsewhere). What's *asserted* is safe to gate on (repo grounding, graph +traversal, scope, routing arithmetic, test commands); everything else is *advisory* — including +every LLM proposal, which is verified against the repo/graph/tests before it counts and is +never trusted blind. Tests and human corrections always win. Full list: [docs/GUIDE.md → Honest limits](../GUIDE.md#honest-limits). --- diff --git a/source/substrate.json b/source/substrate.json index 149d577..8db866b 100644 --- a/source/substrate.json +++ b/source/substrate.json @@ -17,6 +17,12 @@ { "id": "M6", "name": "inline verification", "command": "forge verify", "status": "partial" } ], "defaults": { "askThreshold": 0.6, "impactThreshold": 0.1 }, + "llm": { + "enabled": false, + "model": "haiku", + "timeoutMs": 20000, + "note": "Opt-in (FORGE_LLM=1). The model PROPOSES; the deterministic rubric + repo grounding + tests VERIFY. Any failure falls back to the deterministic path — never blind, never blocking." + }, "policies": { "preAction": "Run substrate before ambiguous, expensive, multi-file, or mutating work.", "escalation": "Start with the cheapest capable tier and escalate only after an external verifier failure.", diff --git a/src/adjudicate.js b/src/adjudicate.js new file mode 100644 index 0000000..6eea5b4 --- /dev/null +++ b/src/adjudicate.js @@ -0,0 +1,82 @@ +// forge adjudicate — the substrate's ONE shared "LLM proposes, checks verify" primitive. +// The whitepaper's load-bearing rule (Panickssery et al. C12; tabayyun, 49:6) is that a +// model is never its own judge: it may PROPOSE, but an external check arbitrates. This module +// supplies only the proposer half — a cheap, opt-in, fail-safe model call — while each faculty +// keeps its deterministic rubric as the judge and reconciles the two (verify, don't trust). +// +// Design contract (identical to src/cortex_distill.js, generalized so all faculties share it): +// - OPT-IN. Off by default (FORGE_LLM!=1) → callers keep their deterministic result and this +// module is never invoked. Behavior is byte-identical to the pre-LLM substrate. +// - FAIL-SAFE. Any error/timeout/garble/secret → returns null. A null NEVER changes a verdict. +// - ZERO-DEP. Access is a `claude -p` CLI shell-out; the runner is injectable so the pure +// prompt/parse/verify logic is fully testable without the CLI or the network. +import { execFileSync } from "node:child_process"; +import { SECRET_RE } from "./recall.js"; + +/** + * Is the LLM proposer layer active for this call? Explicit opt-in wins; env is the default. + * @param {{llm?:boolean}} [opts] + */ +export function llmEnabled(opts = {}) { + if (typeof opts.llm === "boolean") return opts.llm; + return process.env.FORGE_LLM === "1"; +} + +/** Build an injectable `claude -p` runner. Pure config in, a (prompt)->string runner out. */ +export function buildRunner({ model = "haiku", timeoutMs = 20000 } = {}) { + return (prompt) => + execFileSync("claude", ["-p", "--model", model], { + input: prompt, + encoding: "utf8", + timeout: timeoutMs, + stdio: ["pipe", "pipe", "ignore"], + }); +} + +/** Extract the first balanced-ish JSON object from model output, or null. */ +export function extractJson(text) { + if (!text) return null; + const match = String(text).match(/\{[\s\S]*\}/); + if (!match) return null; + try { + return JSON.parse(match[0]); + } catch { + return null; + } +} + +/** + * Run one adjudication: send `prompt`, parse+validate the reply, refuse secrets both ways. + * The proposer half only — the caller must still verify the returned value against ground truth. + * @template T + * @param {{prompt:string, parse:(obj:any)=>(T|null), run?:(p:string)=>string}} spec + * @returns {T|null} the validated proposal, or null on ANY failure (caller keeps deterministic). + */ +export function adjudicate({ prompt, parse, run = buildRunner() }) { + try { + if (SECRET_RE.test(String(prompt))) return null; // never send a secret to the model + const raw = run(prompt); + if (SECRET_RE.test(String(raw))) return null; // never trust a reply that leaked one back + const obj = extractJson(raw); + if (obj == null) return null; + const parsed = parse(obj); + return parsed ?? null; + } catch { + return null; + } +} + +const clamp01 = (x) => Math.max(0, Math.min(1, x)); + +/** Coerce an unknown model field to a number in [0,1], or null if it isn't one. */ +export function asUnit(v) { + const n = typeof v === "number" ? v : Number(v); + return Number.isFinite(n) ? clamp01(n) : null; +} + +/** Coerce an unknown model field to a trimmed, length-capped string (empty → ""). */ +export function asText(v, cap = 200) { + return String(v ?? "") + .trim() + .slice(0, cap); +} diff --git a/src/anchor.js b/src/anchor.js index bee3089..94e784b 100644 --- a/src/anchor.js +++ b/src/anchor.js @@ -3,6 +3,7 @@ // changed (git) against the area the goal named. Flags work that has wandered off it. // Advisory — a stated goal is re-read against real diffs, not trusted to stay in view. import { execFileSync } from "node:child_process"; +import { adjudicate, asText, buildRunner, llmEnabled } from "./adjudicate.js"; import { load as loadAtlas, query as queryAtlas } from "./atlas.js"; import { referencedEntities } from "./preflight.js"; @@ -66,10 +67,47 @@ function goalTargetFiles(root, symbols, files) { return [...targets]; } +// M4 goal-drift — LLM proposer. Semantically classifies the files the coarse keyword match +// flagged as off-goal. PROPOSER ONLY, and one-directional: the model may only move a file +// off→on (never on→off), and only with a reason that references the goal — so it can quiet a +// false drift flag but can never hide a real one. Preserves the "errs toward on-goal" invariant. +export function buildDriftPrompt(goal, offGoalFiles) { + const list = offGoalFiles + .slice(0, 40) + .map((f) => `- ${f}`) + .join("\n"); + return `A developer's stated goal is: """${String(goal).slice(0, 400)}""" +These changed files did NOT obviously match the goal by name: +${list} +For each, decide if the change is plausibly IN SERVICE of that goal. Answer with STRICT JSON and +nothing else, listing only files from above that genuinely serve the goal: +{"onGoal":[{"file":"","reason":""}]} +Omit files that do not serve the goal. No text outside the JSON object.`; +} + +export function parseDriftProposal(obj) { + const onGoal = Array.isArray(obj.onGoal) + ? obj.onGoal + .map((e) => ({ file: asText(e?.file, 240), reason: asText(e?.reason, 200) })) + .filter((e) => e.file && e.reason) + : []; + return { onGoal }; +} + +export function driftLLM(goal, offGoalFiles, { run = buildRunner() } = {}) { + if (!offGoalFiles.length) return { onGoal: [] }; + return adjudicate({ + prompt: buildDriftPrompt(goal, offGoalFiles), + parse: parseDriftProposal, + run, + }); +} + /** * @param {string} root * @param {string} goal - * @param {{ changed?: string[] }} [opts] inject changed to skip git (used in tests) + * @param {{ changed?: string[], llm?:boolean, run?:(p:string)=>string, model?:string, timeoutMs?:number }} [opts] + * inject `changed` to skip git; inject `run` to stub the model (used in tests). */ export function goalDrift(root, goal, opts = {}) { const { keywords, symbols, files } = goalKeywords(goal); @@ -84,8 +122,47 @@ export function goalDrift(root, goal, opts = {}) { const named = targets.some((t) => lf === t || lf.endsWith(`/${t}`)); (named || keywords.some((k) => lf.includes(k)) ? onGoal : offGoal).push(f); } + + // Opt-in semantic pass: rescue files the keyword match missed, but only off→on and only when + // the model gives a goal-referencing reason. Verified, not trusted; fail-safe on any error. + let provenance = { path: "deterministic" }; + if (llmEnabled({ llm: opts.llm }) && offGoal.length) { + const goalTerms = new Set([...keywords, ...symbols.map((s) => s.toLowerCase())]); + const grounded = (reason) => { + const words = String(reason).toLowerCase(); + return goalTerms.size === 0 || [...goalTerms].some((t) => words.includes(t)); + }; + const proposal = driftLLM(goal, offGoal, { + run: opts.run || buildRunner({ model: opts.model, timeoutMs: opts.timeoutMs }), + }); + const rescued = new Set( + (proposal?.onGoal || []) + .filter((e) => offGoal.includes(e.file) && grounded(e.reason)) + .map((e) => e.file), + ); + if (rescued.size) { + for (const f of [...offGoal]) { + if (rescued.has(f)) { + offGoal.splice(offGoal.indexOf(f), 1); + onGoal.push(f); + } + } + provenance = { path: "llm-verified", rescued: [...rescued] }; + } else if (proposal) { + provenance = { path: "llm-agreed" }; + } + } + const drift = changedFiles.length > 0 && (offGoal.length > 0 || onGoal.length === 0); - return { goal: String(goal), keywords, changed: changedFiles, onGoal, offGoal, drift }; + return { + goal: String(goal), + keywords, + changed: changedFiles, + onGoal, + offGoal, + drift, + provenance, + }; } export function renderAnchor(r) { diff --git a/src/atlas.js b/src/atlas.js index b1dee7a..16dca1b 100644 --- a/src/atlas.js +++ b/src/atlas.js @@ -4,6 +4,7 @@ import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { extname, join, relative } from "node:path"; +import { adjudicate, asText, buildRunner, llmEnabled } from "./adjudicate.js"; const IGNORE = new Set([ "node_modules", @@ -296,7 +297,50 @@ function targetIds(atlas, target) { const EDGE_WEIGHT = { calls: 0.95, imports: 0.85, inherits: 0.92, references: 0.7, contains: 0.45 }; -export function impact(atlas, target, { threshold = 0.1, maxHops = 6, decay = 0.85 } = {}) { +// Imagination (§8) — LLM proposer for the edges the regex graph structurally misses: dynamic +// dispatch, DI, reflection, string-keyed lookups. PROPOSER ONLY. Every candidate is then +// verified twice — it must resolve to a REAL node in the graph AND (via the caller's `verify` +// predicate, a grep) actually reference the target in source. Unverifiable → dropped, never added. +export function buildImpactPrompt(atlas, target) { + const files = [...new Set((atlas.nodes || []).map((n) => n.file).filter(Boolean))].slice(0, 60); + return `A code symbol/file is about to change. Name the OTHER files in this repo that most +likely break or depend on it through edges a regex misses: dynamic dispatch, dependency +injection, reflection, string-keyed registries, event handlers. +Changing target: ${String(target).slice(0, 120)} +Files in repo: +${files.map((f) => `- ${f}`).join("\n")} +Answer with STRICT JSON and nothing else, listing only files from the list above: +{"files":[""...]} +No text outside the JSON object.`; +} + +export function parseImpactProposal(obj) { + const files = Array.isArray(obj.files) + ? [...new Set(obj.files.map((f) => asText(f, 240)).filter(Boolean))].slice(0, 20) + : []; + return { files }; +} + +export function impactLLM(atlas, target, { run = buildRunner() } = {}) { + return adjudicate({ prompt: buildImpactPrompt(atlas, target), parse: parseImpactProposal, run }); +} + +/** + * @param {object} atlas + * @param {string} target + * @param {object} [opts] + * @param {number} [opts.threshold] + * @param {number} [opts.maxHops] + * @param {number} [opts.decay] + * @param {boolean} [opts.llm] + * @param {(p:string)=>string} [opts.run] + * @param {(file:string, target:string)=>boolean} [opts.verify] + */ +export function impact( + atlas, + target, + { threshold = 0.1, maxHops = 6, decay = 0.85, llm, run, verify } = {}, +) { const starts = targetIds(atlas, target); const nodeById = new Map((atlas.nodes || []).map((n) => [n.id, n])); const incoming = new Map(); @@ -337,12 +381,36 @@ export function impact(atlas, target, { threshold = 0.1, maxHops = 6, decay = 0. } } const impacted = [...visited.values()].sort((a, b) => b.confidence - a.confidence); + const deterministicFiles = new Set(impacted.map((x) => x.node.file).filter(Boolean)); + + // Opt-in imagination pass: model proposes missed edges, but only VERIFIED ones are kept. + const llmImpacted = []; + if (llmEnabled({ llm }) && run) { + const knownFiles = new Set((atlas.nodes || []).map((n) => n.file).filter(Boolean)); + const proposal = impactLLM(atlas, target, { run }); + for (const file of proposal?.files || []) { + if (deterministicFiles.has(file)) continue; // already found deterministically + if (!knownFiles.has(file)) continue; // must be a real file in the graph + if (typeof verify === "function" && !verify(file, target)) continue; // must grep-confirm the ref + if (typeof verify !== "function") continue; // no external check available → never add blind + llmImpacted.push({ + id: `llm:${file}`, + node: { id: `llm:${file}`, name: file, kind: "module", file }, + confidence: Number((threshold * 0.9).toFixed(4)), + hopDistance: null, + source: "llm-verified", + }); + } + } + + const all = [...impacted, ...llmImpacted]; return { target, found: starts.length > 0, threshold, - impacted, - impactedFiles: [...new Set(impacted.map((x) => x.node.file).filter(Boolean))].sort(), + impacted: all, + impactedFiles: [...new Set(all.map((x) => x.node.file).filter(Boolean))].sort(), + llmVerified: llmImpacted.map((x) => x.node.file), totalGraphNodes: (atlas.nodes || []).length, totalGraphEdges: (atlas.edges || []).length, }; diff --git a/src/cortex_distill.js b/src/cortex_distill.js index 736f584..e16e809 100644 --- a/src/cortex_distill.js +++ b/src/cortex_distill.js @@ -1,9 +1,9 @@ // forge cortex distiller — turn a raw correction episode into a real, useful lesson via a // cheap model call, instead of the deterministic template. OPT-IN (ENABLE_CORTEX_DISTILL=1) // and fail-safe: any failure returns null and the caller keeps the template. Zero deps — it -// shells out to the `claude` CLI (same pattern as session-learner.sh); the runner is -// injectable so the pure prompt/parse logic is fully testable without the CLI. -import { execFileSync } from "node:child_process"; +// shells out to the `claude` CLI via the shared adjudicate runner (same primitive every other +// faculty uses); the runner is injectable so the pure prompt/parse logic is testable without it. +import { buildRunner } from "./adjudicate.js"; import { SECRET_RE } from "./recall.js"; /** @@ -43,13 +43,7 @@ export function parseDistilled(text) { return { whatWentWrong, correctedBehavior }; } -const claudeRun = (prompt, { model = "haiku", timeoutMs = 20000 } = {}) => - execFileSync("claude", ["-p", "--model", model], { - input: prompt, - encoding: "utf8", - timeout: timeoutMs, - stdio: ["pipe", "pipe", "ignore"], - }); +const claudeRun = (prompt, opts = {}) => buildRunner(opts)(prompt); /** Distill an episode into a lesson body. Returns null on any failure (caller keeps template). */ export function distill(episode, { run = claudeRun } = {}) { diff --git a/src/lessons.js b/src/lessons.js index 56fd24b..966ff9d 100644 --- a/src/lessons.js +++ b/src/lessons.js @@ -73,14 +73,27 @@ export function newLesson(fields, nowDay = 0) { }; } -/** Time-decayed Beta posterior mean. Unconfirmed lessons fade out of the injection set. */ -export function confidenceOf(lesson, nowDay) { - const alpha = 1 + lesson.evidenceCount; - const beta = 1 + lesson.contradictionCount; - const mean = alpha / (alpha + beta); +/** + * Validity — the paper's `val` term (Eq. retrieve, §7.1). The Laplace-smoothed Beta posterior + * mean over GROUND-TRUTH outcomes: how much independent evidence upholds this lesson vs. + * contradicts it, with NO time decay. This is what makes a memory pruned by whether its + * prediction later held (a test/commit), not by the model's own say-so. + */ +export function validity(lesson) { + const alpha = 1 + (lesson.evidenceCount ?? 0); + const beta = 1 + (lesson.contradictionCount ?? 0); + return alpha / (alpha + beta); +} + +/** Freshness — the paper's `rec` term: exponential recency decay since last confirmation. */ +export function freshness(lesson, nowDay) { const age = Math.max(0, nowDay - lesson.lastConfirmedDay); - const decay = 0.5 ** (age / lesson.halfLifeDays); - return decay * mean; + return 0.5 ** (age / lesson.halfLifeDays); +} + +/** Time-decayed Beta posterior mean = freshness × validity. Unconfirmed lessons fade out. */ +export function confidenceOf(lesson, nowDay) { + return freshness(lesson, nowDay) * validity(lesson); } /** Independent outcome re-confirmed this lesson. Raises confidence with diminishing returns. */ @@ -138,10 +151,15 @@ export function selectForInjection(lessons, context, { budget = 12, nowDay = 0 } .map((l) => ({ lesson: l, m: matchScore(l, context) })) .filter((x) => x.m > 0) .map((x) => { - const conf = confidenceOf(x.lesson, nowDay); - const recency = 1 + (nowDay - x.lesson.lastConfirmedDay <= 14 ? 0.2 : 0); + // The paper's retrieval score, decomposed into named terms: relevance (match) × + // freshness (rec) × validity (val, ground-truth outcomes) × scope, with a small recency + // boost. Making `val` explicit ranks outcome-confirmed lessons above merely-recent ones. + const rel = x.m; + const rec = freshness(x.lesson, nowDay); + const val = validity(x.lesson); const scopeW = SCOPE_WEIGHT[x.lesson.scope] ?? 0.5; - return { lesson: x.lesson, score: conf * x.m * recency * scopeW }; + const recencyBoost = 1 + (nowDay - x.lesson.lastConfirmedDay <= 14 ? 0.2 : 0); + return { lesson: x.lesson, score: rel * rec * val * scopeW * recencyBoost }; }) .sort((a, b) => b.score - a.score); diff --git a/src/preflight.js b/src/preflight.js index e63a46c..ae4ebc1 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -3,6 +3,7 @@ // will silently ASSUME. The richer assumption gate also scores specification completeness. import { existsSync } from "node:fs"; import { join } from "node:path"; +import { adjudicate, asText, asUnit, buildRunner, llmEnabled } from "./adjudicate.js"; import { build as buildAtlas, has, load as loadAtlas } from "./atlas.js"; const CODE_EXT = @@ -203,6 +204,7 @@ export function assessTask(text, { askThreshold = 0.6 } = {}) { completeness, risk, shouldAsk, + hardUnderspecified, missing, questions: (shouldAsk && !questions.length ? ["What exactly should this produce, and how will we know it is correct?"] @@ -212,6 +214,85 @@ export function assessTask(text, { askThreshold = 0.6 } = {}) { }; } +// M2 assumption gate — LLM proposer. Scores the paper's s(x) completeness functional over the +// same four dimensions the rubric names, and may add clarifying questions. PROPOSER ONLY: +// reconcileAssumption() bounds it against the deterministic score and grounds every question. +const DIM_KEYS = DIMENSIONS.map((d) => d.key); + +export function buildAssumptionPrompt(task) { + return `A coding agent received this task. Judge how completely it is specified BEFORE any code is written. +Task: """${String(task).slice(0, 1200)}""" +Score specification completeness in [0,1] over these dimensions: inputs_outputs, target_scope, +success_criteria, constraints. List the concrete missing-information questions a careful engineer +would ask first. Respond with STRICT JSON and nothing else: +{"completeness":<0..1>,"missing":[""...],"questions":[""...]} +Do not echo credentials or personal data. No text outside the JSON object.`; +} + +export function parseAssumptionProposal(obj) { + const completeness = asUnit(obj.completeness); + if (completeness == null) return null; + const missing = Array.isArray(obj.missing) + ? obj.missing.map((m) => asText(m, 40)).filter((m) => DIM_KEYS.includes(m)) + : []; + const questions = Array.isArray(obj.questions) + ? [...new Set(obj.questions.map((q) => asText(q, 200)).filter(Boolean))].slice(0, 5) + : []; + return { completeness, missing, questions }; +} + +/** Ask the model for an assumption reading (proposer). Returns null when off/unavailable. */ +export function assessTaskLLM(task, { run = buildRunner() } = {}) { + return adjudicate({ prompt: buildAssumptionPrompt(task), parse: parseAssumptionProposal, run }); +} + +/** + * Verify-don't-trust reconcile for M2. The model may only move completeness within ±band of the + * deterministic score (so a clearly-specified/vague task can't be flipped), and may only ADD an + * ask, never clear a deterministic one. Extra questions survive only if they map to a rubric- + * flagged dimension or (via `grounded`) reference a real repo entity. + * @param {object} det - assessTask() result + * @param {{completeness:number, missing:string[], questions:string[]}|null} proposal + * @param {object} [opts] + * @param {number} [opts.askThreshold] + * @param {number} [opts.band] + * @param {(q:string)=>boolean} [opts.grounded] + */ +export function reconcileAssumption( + det, + proposal, + { askThreshold = 0.6, band = 0.25, grounded = () => false } = {}, +) { + if (!proposal) return { ...det, provenance: { path: "deterministic" } }; + const bounded = Math.max( + det.completeness - band, + Math.min(det.completeness + band, proposal.completeness), + ); + const completeness = Math.max(0, Math.min(1, bounded)); + const flaggedDims = new Set(det.missing.map((m) => m.key)); + const extraQuestions = proposal.questions.filter( + (q) => proposal.missing.some((m) => flaggedDims.has(m)) || grounded(q), + ); + const questions = [...new Set([...det.questions, ...extraQuestions])].slice(0, 3); + // The gate only ever tightens: an ask stands if the rubric asked, the task is hard- + // underspecified, or the reconciled completeness is below threshold. + const shouldAsk = det.shouldAsk || det.hardUnderspecified || completeness < askThreshold; + const risk = completeness < 0.45 ? "high" : completeness < 0.7 ? "medium" : "low"; + const moved = + Math.abs(completeness - det.completeness) > 1e-9 || questions.length !== det.questions.length; + return { + ...det, + completeness, + risk, + shouldAsk, + questions: + shouldAsk && !questions.length + ? ["What exactly should this produce, and how will we know it is correct?"] + : questions, + provenance: { path: moved ? "llm-verified" : "llm-agreed", detCompleteness: det.completeness }, + }; +} + export function informationGap(text, deps = {}) { const { hasSymbol = () => false, fileExists = () => false } = deps; const { symbols, files } = referencedEntities(text); @@ -251,11 +332,37 @@ export function clarifyBlock(result, { threshold = 0.5 } = {}) { return lines.join("\n"); } -export function preflightRepo(root, text, { allowBuild = true, askThreshold = 0.6 } = {}) { +/** + * @param {string} root + * @param {string} text + * @param {object} [opts] + * @param {boolean} [opts.allowBuild] + * @param {number} [opts.askThreshold] + * @param {boolean} [opts.llm] + * @param {string} [opts.model] + * @param {number} [opts.timeoutMs] + * @param {(p:string)=>string} [opts.run] + */ +export function preflightRepo( + root, + text, + { allowBuild = true, askThreshold = 0.6, llm, model, timeoutMs, run } = {}, +) { const atlas = loadAtlas(root) || (allowBuild ? buildAtlas({ root }) : null); + const hasSymbol = atlas ? (name) => has(atlas, name) : () => true; const gap = informationGap(text, { - hasSymbol: atlas ? (name) => has(atlas, name) : () => true, + hasSymbol, fileExists: (f) => existsSync(join(root, f)), }); - return { ...gap, assumption: assessTask(text, { askThreshold }) }; + const det = assessTask(text, { askThreshold }); + // M2 proposer: only when opted in. The rubric is the external judge; the model refines it + // within bounds and can add grounded questions. Fail-safe: null proposal keeps `det`. + if (!llmEnabled({ llm })) + return { ...gap, assumption: { ...det, provenance: { path: "deterministic" } } }; + const proposal = assessTaskLLM(text, { run: run || buildRunner({ model, timeoutMs }) }); + const grounded = (q) => { + const { symbols, files } = referencedEntities(q); + return symbols.some(hasSymbol) || files.some((f) => existsSync(join(root, f))); + }; + return { ...gap, assumption: reconcileAssumption(det, proposal, { askThreshold, grounded }) }; } diff --git a/src/route.js b/src/route.js index 117a9cd..d5d4ca9 100644 --- a/src/route.js +++ b/src/route.js @@ -4,6 +4,7 @@ // tier. Advisory by default; a LiteLLM config emit exposes the tiers as gateway aliases you request. import { writeFileSync } from "node:fs"; import { join } from "node:path"; +import { adjudicate, asText, buildRunner, llmEnabled } from "./adjudicate.js"; import { matchingLessons } from "./cortex.js"; import { gitChurn, grepFanout } from "./cortex_features.js"; import { load as loadLessons } from "./lessons_store.js"; @@ -121,8 +122,42 @@ export function recommend(score, norm = {}) { return { key, model: MODELS[key], tier: MODELS[key].tier, reasons }; } -/** Repo wrapper: gather the real signals for a task and route it. */ -export function routeTask(root, task) { +// M1 routing — LLM proposer. Estimates task complexity c(x) as a coarse band. PROPOSER ONLY: +// reconcile takes the MAX with the deterministic score, so the model can only RAISE the tier +// (never under-provision on its own word); escalation still gates on a verified failure. +const BAND_FLOOR = { cheap: 0.15, mid: 0.4, premium: 0.65 }; + +export function buildComplexityPrompt(task) { + return `Judge the intrinsic complexity of this coding task for model selection (not how to do it). +Task: """${String(task).slice(0, 1200)}""" +Answer with STRICT JSON and nothing else: +{"band":"cheap|mid|premium","reason":""} +cheap = trivial/boilerplate; mid = a data structure, class, or library-level change; premium = +algorithmic/systems/architectural/multi-module work. No text outside the JSON object.`; +} + +export function parseComplexityProposal(obj) { + const band = String(obj.band ?? "").toLowerCase(); + if (!(band in BAND_FLOOR)) return null; + return { band, score: BAND_FLOOR[band], reason: asText(obj.reason) }; +} + +/** Ask the model for a complexity band (proposer). Returns null when off/unavailable. */ +export function complexityLLM(task, { run = buildRunner() } = {}) { + return adjudicate({ prompt: buildComplexityPrompt(task), parse: parseComplexityProposal, run }); +} + +/** + * Repo wrapper: gather the real signals for a task and route it. `run` is injectable for tests. + * @param {string} root + * @param {string} task + * @param {object} [opts] + * @param {boolean} [opts.llm] + * @param {string} [opts.model] + * @param {number} [opts.timeoutMs] + * @param {(p:string)=>string} [opts.run] + */ +export function routeTask(root, task, { llm, model, timeoutMs, run } = {}) { const { symbols, files } = referencedEntities(task); const fanout = symbols.reduce((m, sym) => Math.max(m, grepFanout(root, sym)), 0); const churn = files.reduce((m, f) => Math.max(m, gitChurn(root, f)), 0); @@ -143,18 +178,27 @@ export function routeTask(root, task) { const { score: repoScore, norm } = complexity(signals); const rubric = rubricComplexity(task); const rubricScore = Math.min(1, rubric.rawScore / 10); - const score = Math.max(repoScore, rubricScore); + // M1 proposer (opt-in): the model may only RAISE the tier — max() with the deterministic + // scores — so it never routes down on its own word. Fail-safe: null proposal is ignored. + const proposal = llmEnabled({ llm }) + ? complexityLLM(task, { run: run || buildRunner({ model, timeoutMs }) }) + : null; + const score = Math.max(repoScore, rubricScore, proposal?.score ?? 0); const recommended = recommend(score, norm); + const raisedByLLM = proposal != null && proposal.score > Math.max(repoScore, rubricScore); return { score, repoScore, signals, rubric, + llm: proposal ? { band: proposal.band, reason: proposal.reason, raised: raisedByLLM } : null, + provenance: { path: raisedByLLM ? "llm-verified" : proposal ? "llm-agreed" : "deterministic" }, ...recommended, reasons: [ ...new Set([ ...(recommended.reasons || []), ...rubric.reasons.filter((r) => r.weight > 0).map((r) => r.reason), + ...(raisedByLLM ? [`model judged ${proposal.band}: ${proposal.reason}`] : []), ]), ], }; diff --git a/src/substrate.js b/src/substrate.js index 0f46159..c8eabf7 100644 --- a/src/substrate.js +++ b/src/substrate.js @@ -5,6 +5,7 @@ import { existsSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { buildRunner, llmEnabled } from "./adjudicate.js"; import { goalDrift } from "./anchor.js"; import { build as buildAtlas, impact as impactGraph, load as loadAtlas } from "./atlas.js"; import { matchingLessons } from "./cortex.js"; @@ -61,26 +62,90 @@ function minimalityWarnings(task, route, preflight) { return warnings; } -export function predictImpact(root, target, { threshold = 0.1 } = {}) { +// Grep-style verify for the LLM impact pass: a proposed dependent is only kept if the target +// symbol/file name actually appears in the candidate file's source. External check, not trust. +function makeImpactVerify(root) { + const base = (t) => + String(t) + .split(/[/\\]/) + .pop() + .replace(/\.[^.]+$/, ""); + return (file, target) => { + try { + const src = readFileSync(join(root, file), "utf8"); + const name = base(target); + return name.length > 1 && new RegExp(`\\b${name.replace(/[^\w$]/g, "")}\\b`).test(src); + } catch { + return false; + } + }; +} + +/** + * @param {string} root + * @param {string} target + * @param {object} [opts] + * @param {number} [opts.threshold] + * @param {boolean} [opts.llm] + * @param {string} [opts.model] + * @param {number} [opts.timeoutMs] + */ +export function predictImpact(root, target, { threshold = 0.1, llm, model, timeoutMs } = {}) { const atlas = loadAtlas(root) || buildAtlas({ root }); - return impactGraph(atlas, target, { threshold }); + const useLLM = llmEnabled({ llm }); + return impactGraph(atlas, target, { + threshold, + llm: useLLM, + run: useLLM ? buildRunner({ model, timeoutMs }) : undefined, + verify: makeImpactVerify(root), + }); } +/** + * @param {string} root + * @param {string} task + * @param {object} [opts] + * @param {number} [opts.threshold] + * @param {number} [opts.askThreshold] + * @param {boolean} [opts.allowBuild] + * @param {boolean} [opts.llm] + * @param {string} [opts.model] + * @param {number} [opts.timeoutMs] + */ export function substrateCheck( root, task, - { threshold = 0.1, askThreshold = 0.6, allowBuild = true } = {}, + { threshold = 0.1, askThreshold = 0.6, allowBuild = true, llm, model, timeoutMs } = {}, ) { const text = String(task || ""); + // LLM adjudication is opt-in. On the ambient hook path (allowBuild:false) it stays OFF unless + // FORGE_LLM_AMBIENT=1, so the per-prompt hook never pays model latency by default. An explicit + // `llm` option always wins. Every faculty is fail-safe: a null proposal keeps the rubric. + const useLLM = + typeof llm === "boolean" + ? llm + : allowBuild + ? llmEnabled() + : process.env.FORGE_LLM_AMBIENT === "1"; + const llmOpts = { llm: useLLM, model, timeoutMs }; const entities = referencedEntities(text); - const preflight = preflightRepo(root, text, { askThreshold, allowBuild }); - const route = routeTask(root, text); + const preflight = preflightRepo(root, text, { askThreshold, allowBuild, ...llmOpts }); + const route = routeTask(root, text, llmOpts); // allowBuild:false (ambient hooks) uses the atlas only if one is already cached — never // builds or writes .forge/atlas.json from a hook. Impact is then best-effort. const atlas = loadAtlas(root) || (allowBuild ? buildAtlas({ root }) : null); const impactTargets = [...new Set([...entities.symbols, ...entities.files])].slice(0, 8); + const impactRun = useLLM ? buildRunner({ model, timeoutMs }) : undefined; + const impactVerify = makeImpactVerify(root); const impacts = atlas - ? impactTargets.map((target) => impactGraph(atlas, target, { threshold })) + ? impactTargets.map((target) => + impactGraph(atlas, target, { + threshold, + llm: useLLM, + run: impactRun, + verify: impactVerify, + }), + ) : []; const impactedFiles = [...new Set(impacts.flatMap((r) => r.impactedFiles || []))].sort(); const scopedFiles = [...new Set([...entities.files, ...impactedFiles])]; @@ -111,9 +176,21 @@ export function substrateCheck( minimality: { warnings: minimalityWarnings(text, route, preflight) }, // M4 goal-anchoring: re-read the stated goal against files already changed this session. // Quiet pre-action (clean tree → no drift); speaks mid-session when work wandered off-goal. - goalAnchor: goalDrift(root, text), + goalAnchor: goalDrift(root, text, llmOpts), verification: { checklist: verificationChecklist(root) }, substrate: loadSubstrateSpec(), + // Which faculties, if any, had a model proposal survive external verification this run. + llm: { + enabled: useLLM, + provenance: { + assumption: preflight.assumption.provenance?.path ?? "deterministic", + route: route.provenance?.path ?? "deterministic", + impact: impacts.some((r) => (r.llmVerified || []).length) + ? "llm-verified" + : "deterministic", + goalAnchor: undefined, // set below once goalAnchor is in scope + }, + }, guarantees: { deterministic: [ "assumption rubric", @@ -122,6 +199,14 @@ export function substrateCheck( "impact graph traversal", "scope decomposition", ], + // Proposed by a model, then checked against the repo/graph/tests before it could move a + // verdict — safe to surface, never blindly trusted (whitepaper tabayyun gate). + llmVerified: [ + "assumption refinement (bounded by the rubric)", + "routing escalation (raise-only)", + "impact edges (graph + grep verified)", + "goal-drift rescue (off→on, goal-referenced)", + ], advisory: [ "model capability fit", "scope minimality", @@ -131,6 +216,7 @@ export function substrateCheck( ], }, }; + result.llm.provenance.goalAnchor = result.goalAnchor?.provenance?.path ?? "deterministic"; return result; } diff --git a/test/adjudicate.test.js b/test/adjudicate.test.js new file mode 100644 index 0000000..d94bd55 --- /dev/null +++ b/test/adjudicate.test.js @@ -0,0 +1,88 @@ +import assert from "node:assert/strict"; +import { afterEach, test } from "node:test"; +import { adjudicate, asText, asUnit, extractJson, llmEnabled } from "../src/adjudicate.js"; + +const parseScore = (o) => { + const score = asUnit(o.score); + return score == null ? null : { score, reason: asText(o.reason) }; +}; + +afterEach(() => { + delete process.env.FORGE_LLM; +}); + +test("llmEnabled: explicit opt wins over env", () => { + process.env.FORGE_LLM = "1"; + assert.equal(llmEnabled({ llm: false }), false); + delete process.env.FORGE_LLM; + assert.equal(llmEnabled({ llm: true }), true); +}); + +test("llmEnabled: env is the default, off unless exactly '1'", () => { + assert.equal(llmEnabled(), false); + process.env.FORGE_LLM = "0"; + assert.equal(llmEnabled(), false); + process.env.FORGE_LLM = "1"; + assert.equal(llmEnabled(), true); +}); + +test("extractJson: pulls a JSON object out of chatty output, tolerates garbage", () => { + assert.deepEqual(extractJson('sure! {"score":0.4} done'), { score: 0.4 }); + assert.equal(extractJson("no json here"), null); + assert.equal(extractJson("{not valid}"), null); + assert.equal(extractJson(""), null); +}); + +test("adjudicate: happy path returns the validated proposal", () => { + const run = () => '{"score": 0.42, "reason": "some detail"}'; + assert.deepEqual(adjudicate({ prompt: "x", parse: parseScore, run }), { + score: 0.42, + reason: "some detail", + }); +}); + +test("adjudicate: garbage / unparseable reply → null (caller keeps deterministic)", () => { + assert.equal(adjudicate({ prompt: "x", parse: parseScore, run: () => "nope" }), null); + assert.equal( + adjudicate({ + prompt: "x", + parse: parseScore, + run: () => '{"score":"not-a-number"}', + }), + null, + ); +}); + +test("adjudicate: a throwing runner (CLI missing / timeout) → null, never throws", () => { + const run = () => { + throw new Error("claude not found"); + }; + assert.equal(adjudicate({ prompt: "x", parse: parseScore, run }), null); +}); + +test("adjudicate: refuses to send a secret in the prompt", () => { + let called = false; + const run = () => { + called = true; + return '{"score":0.5}'; + }; + const secret = ["sk-ant", "abcd1234efgh5678ijkl"].join("-"); + assert.equal(adjudicate({ prompt: `use ${secret}`, parse: parseScore, run }), null); + assert.equal(called, false, "runner must not be invoked when the prompt carries a secret"); +}); + +test("adjudicate: refuses a reply that leaks a secret back", () => { + const secret = ["ghp", "abcdefghij0123456789"].join("_"); + const run = () => `{"score":0.5,"reason":"${secret}"}`; + assert.equal(adjudicate({ prompt: "clean", parse: parseScore, run }), null); +}); + +test("asUnit clamps to [0,1]; asText trims and caps", () => { + assert.equal(asUnit(1.7), 1); + assert.equal(asUnit(-3), 0); + assert.equal(asUnit("0.3"), 0.3); + assert.equal(asUnit("x"), null); + assert.equal(asText(" hi "), "hi"); + assert.equal(asText(null), ""); + assert.equal(asText("x".repeat(500)).length, 200); +}); diff --git a/test/anchor.test.js b/test/anchor.test.js index e011b90..a83b5c6 100644 --- a/test/anchor.test.js +++ b/test/anchor.test.js @@ -29,3 +29,50 @@ test("goalDrift flags pure drift — changes exist but none match the goal", () assert.equal(r.onGoal.length, 0); assert.equal(r.drift, true); }); + +test("goalDrift (llm on): rescues an off-goal file only with a goal-referencing reason", () => { + // src/throttle.js shares no keyword with the goal, so the coarse match flags it off-goal. + const run = () => + '{"onGoal":[{"file":"src/throttle.js","reason":"implements the rate limiting token bucket"}]}'; + const r = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/login.js", "src/throttle.js"], + llm: true, + run, + }); + assert.ok(r.onGoal.includes("src/throttle.js"), "grounded reason moves it off→on"); + assert.ok(!r.offGoal.includes("src/throttle.js")); + assert.equal(r.provenance.path, "llm-verified"); +}); + +test("goalDrift (llm on): a reason that never references the goal is rejected (verify, don't trust)", () => { + const run = () => '{"onGoal":[{"file":"src/reports.js","reason":"just tidying up"}]}'; + const r = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/reports.js"], + llm: true, + run, + }); + assert.ok(r.offGoal.includes("src/reports.js"), "ungrounded rescue rejected"); + assert.equal(r.drift, true); +}); + +test("goalDrift (llm on): the model can never move a file on→off", () => { + const run = () => '{"onGoal":[]}'; // model says nothing is on-goal + const r = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/login.js"], // keyword-matched on-goal deterministically + llm: true, + run, + }); + assert.ok(r.onGoal.includes("src/login.js"), "deterministic on-goal is never demoted"); +}); + +test("goalDrift (llm on): a throwing runner falls back to the deterministic split", () => { + const r = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/login.js", "src/reports.js"], + llm: true, + run: () => { + throw new Error("no cli"); + }, + }); + assert.ok(r.offGoal.includes("src/reports.js")); + assert.equal(r.drift, true); +}); diff --git a/test/lessons.test.js b/test/lessons.test.js index fbc232f..33e338b 100644 --- a/test/lessons.test.js +++ b/test/lessons.test.js @@ -5,10 +5,12 @@ import { confidenceOf, confirm, contradict, + freshness, matchScore, newLesson, scoreMistake, selectForInjection, + validity, } from "../src/lessons.js"; test("scoreMistake: a lone behavioral signal never fires (thrash is not a lesson)", () => { @@ -115,3 +117,22 @@ test("selectForInjection: candidates and non-matching lessons are excluded; empt const { block } = selectForInjection([candidate], { symbols: ["foo"] }, {}); assert.equal(block, "", "only active lessons inject; no noise when nothing qualifies"); }); + +test("validity/freshness decompose confidence (val = ground truth, rec = decay)", () => { + const l = newLesson({ id: "v", trigger: { symbols: ["foo"] } }, 0); + // Laplace-smoothed Beta mean starts at 0.5; confidence = freshness × validity. + assert.equal(validity(l), 0.5); + assert.equal(freshness(l, 0), 1); + assert.ok(Math.abs(confidenceOf(l, 0) - validity(l) * freshness(l, 0)) < 1e-9); +}); + +test("val term ranks an outcome-confirmed lesson above a merely-recent one", () => { + const ctx = { symbols: ["foo"] }; + let confirmed = newLesson({ id: "confirmed", trigger: { symbols: ["foo"] } }, 0); + confirmed = confirm(confirmed, 0); // one independent outcome → higher validity + confirmed = confirm(confirmed, 0); + let recentOnly = newLesson({ id: "recent", trigger: { symbols: ["foo"] } }, 0); + recentOnly = confirm(recentOnly, 0); // active but minimally confirmed + const { selected } = selectForInjection([recentOnly, confirmed], ctx, { budget: 2, nowDay: 0 }); + assert.equal(selected[0].id, "confirmed", "ground-truth validity wins the ranking"); +}); diff --git a/test/preflight.test.js b/test/preflight.test.js index 04af558..952509d 100644 --- a/test/preflight.test.js +++ b/test/preflight.test.js @@ -5,9 +5,12 @@ import { join } from "node:path"; import { test } from "node:test"; import { ambiguityMarkers, + assessTask, + assessTaskLLM, clarifyBlock, informationGap, preflightRepo, + reconcileAssumption, referencedEntities, } from "../src/preflight.js"; @@ -69,3 +72,78 @@ test("preflightRepo grounds against a real repo (missing file/symbol → clarify assert.ok(r.unresolved.files.includes("src/missing.ts")); assert.ok(!r.unresolved.symbols.includes("computeTax"), "resolved symbol is not flagged"); }); + +test("assessTaskLLM: parses a completeness reading, rejects junk", () => { + const p = assessTaskLLM("do a thing", { + run: () => '{"completeness":0.3,"missing":["target_scope"],"questions":["Which file?"]}', + }); + assert.equal(p.completeness, 0.3); + assert.deepEqual(p.missing, ["target_scope"]); + assert.equal(assessTaskLLM("x", { run: () => "not json" }), null); + assert.equal(assessTaskLLM("x", { run: () => '{"completeness":"nope"}' }), null); +}); + +test("reconcileAssumption: model can only move completeness within ±band of the rubric", () => { + const det = assessTask("Fix the bug."); // under-specified, low completeness, shouldAsk + // Model claims fully specified — bounded, cannot flip a clearly-vague task to 1.0. + const r = reconcileAssumption( + det, + { completeness: 1, missing: [], questions: [] }, + { band: 0.25 }, + ); + assert.ok(r.completeness <= det.completeness + 0.25 + 1e-9, "clamped to +band"); + assert.equal(r.provenance.path, "llm-verified"); +}); + +test("reconcileAssumption: never clears a deterministic / hard-underspecified ask", () => { + const det = assessTask("Fix it."); // hardUnderspecified + assert.equal(det.shouldAsk, true); + const r = reconcileAssumption(det, { completeness: 0.95, missing: [], questions: [] }); + assert.equal(r.shouldAsk, true, "the gate only tightens; the model cannot open it"); +}); + +test("reconcileAssumption: extra questions survive only if grounded or on a flagged dimension", () => { + const det = assessTask("optimize the pipeline"); // flags success_criteria/constraints dims + const r = reconcileAssumption( + det, + { + completeness: det.completeness, + missing: det.missing.map((m) => m.key), // maps to a flagged dimension → kept + questions: ["What is the acceptance benchmark?"], + }, + { grounded: () => false }, + ); + assert.ok( + r.questions.some((q) => /benchmark/.test(q)) || det.questions.length === 0, + "a question on a flagged dimension is kept", + ); + // An ungrounded question tied to no flagged dimension is dropped. + const r2 = reconcileAssumption( + det, + { completeness: det.completeness, missing: [], questions: ["Unrelated musing?"] }, + { grounded: () => false }, + ); + assert.ok(!r2.questions.includes("Unrelated musing?"), "ungrounded extra question dropped"); +}); + +test("reconcileAssumption: null proposal is a pure passthrough (deterministic path)", () => { + const det = assessTask("Change verifyToken in src/auth.js to require length > 20; update tests"); + const r = reconcileAssumption(det, null); + assert.equal(r.provenance.path, "deterministic"); + assert.equal(r.completeness, det.completeness); + assert.equal(r.shouldAsk, det.shouldAsk); +}); + +test("preflightRepo (llm on): fail-safe — a throwing runner keeps the deterministic reading", () => { + const root = mkdtempSync(join(tmpdir(), "forge-pre-")); + const task = "Fix the bug."; + const base = preflightRepo(root, task); + const withLlm = preflightRepo(root, task, { + llm: true, + run: () => { + throw new Error("no cli"); + }, + }); + assert.equal(withLlm.assumption.shouldAsk, base.assumption.shouldAsk); + assert.equal(withLlm.assumption.provenance.path, "deterministic"); +}); diff --git a/test/route.test.js b/test/route.test.js index aadf4c6..976d1d5 100644 --- a/test/route.test.js +++ b/test/route.test.js @@ -3,7 +3,13 @@ import { existsSync, mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; -import { complexity, emitGatewayConfig, recommend, routeTask } from "../src/route.js"; +import { + complexity, + complexityLLM, + emitGatewayConfig, + recommend, + routeTask, +} from "../src/route.js"; test("complexity is monotonic and bounded", () => { const trivial = complexity({ files: 0, fanout: 0, sizeWords: 4 }).score; @@ -64,3 +70,40 @@ test("emitGatewayConfig writes a LiteLLM config that never pins @latest", () => "passthrough for real model names so plain claude-* traffic works", ); }); + +test("complexityLLM: parses a band into a score floor, rejects junk", () => { + const cheap = complexityLLM("x", { run: () => '{"band":"cheap","reason":"trivial"}' }); + assert.equal(cheap.band, "cheap"); + const premium = complexityLLM("x", { run: () => '{"band":"premium","reason":"distributed"}' }); + assert.ok(premium.score > cheap.score, "premium floors higher than cheap"); + assert.equal(complexityLLM("x", { run: () => '{"band":"???"}' }), null); + assert.equal(complexityLLM("x", { run: () => "not json" }), null); +}); + +test("routeTask (llm on): the model may only RAISE the tier, never lower it", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + const task = "write a function to check if a number is prime"; + // Model says 'premium' on a trivial task → routing escalates (safe direction). + const up = routeTask(root, task, { llm: true, run: () => '{"band":"premium","reason":"x"}' }); + assert.ok(["opus", "fable"].includes(up.key), `raised to ${up.key}`); + assert.equal(up.provenance.path, "llm-verified"); + assert.equal(up.llm.raised, true); + // Model says 'cheap' → cannot pull a task below the deterministic floor. + const down = routeTask(root, task, { llm: true, run: () => '{"band":"cheap","reason":"x"}' }); + const base = routeTask(root, task); + assert.ok(down.score >= base.score, "cheap band never routes below deterministic"); +}); + +test("routeTask (llm on): a failing model call falls back to deterministic", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + const task = "write a function to check if a number is prime"; + const throwing = routeTask(root, task, { + llm: true, + run: () => { + throw new Error("no cli"); + }, + }); + const base = routeTask(root, task); + assert.equal(throwing.key, base.key, "fell back to deterministic tier"); + assert.equal(throwing.provenance.path, "deterministic"); +}); diff --git a/test/substrate.test.js b/test/substrate.test.js index 18e8a89..f429f9e 100644 --- a/test/substrate.test.js +++ b/test/substrate.test.js @@ -47,6 +47,33 @@ test("atlas impact follows reverse dependencies", () => { assert.ok(r.impactedFiles.includes("invoice.js"), JSON.stringify(r, null, 2)); }); +test("impact (llm on): a proposed edge is kept only if it is a real file AND grep-verified", () => { + const root = repo(); + const atlas = build({ root }); + // Model proposes a real repo file (invoice.js) plus a fabricated one (ghost.js). + const run = () => '{"files":["invoice.js","ghost.js"]}'; + const verify = (file) => file === "invoice.js"; // grep only confirms the real reference + const r = impact(atlas, "computeTax", { llm: true, run, verify }); + assert.ok(r.llmVerified.includes("invoice.js") || r.impactedFiles.includes("invoice.js")); + assert.ok(!r.impactedFiles.includes("ghost.js"), "a fabricated file is never added"); +}); + +test("impact (llm on): without a verify predicate, nothing is added blind", () => { + const root = repo(); + const atlas = build({ root }); + const run = () => '{"files":["invoice.js"]}'; + const r = impact(atlas, "computeTax", { llm: true, run }); // no verify → cannot confirm + assert.deepEqual(r.llmVerified, [], "no external check available → no blind edges"); +}); + +test("impact (llm off): behavior is unchanged and carries no llm fields effect", () => { + const root = repo(); + const atlas = build({ root }); + const base = impact(atlas, "computeTax"); + const off = impact(atlas, "computeTax", { llm: false, run: () => '{"files":["ghost.js"]}' }); + assert.deepEqual(off.impactedFiles, base.impactedFiles); +}); + test("substrateCheck returns one professional pre-action contract", () => { const root = repo(); const r = substrateCheck( @@ -71,3 +98,23 @@ test("preflightRepo includes assumption report", () => { const r = preflightRepo(root, "Optimize it."); assert.equal(r.assumption.shouldAsk, true); }); + +test("substrateCheck (llm off by default): provenance is deterministic across faculties", () => { + const root = repo(); + const r = substrateCheck(root, "Refactor computeTax in math.js", { allowBuild: true }); + assert.equal(r.llm.enabled, false); + assert.equal(r.llm.provenance.assumption, "deterministic"); + assert.equal(r.llm.provenance.route, "deterministic"); + assert.ok(Array.isArray(r.guarantees.llmVerified), "llmVerified bucket is present"); +}); + +test("substrateCheck (llm on, explicit): opt-in flag threads through and stays fail-safe", () => { + const root = repo(); + // No `run` injection reaches the real CLI here, but the substrate must not throw and must + // still return a coherent contract regardless of whether the CLI exists. + const r = substrateCheck(root, "Update computeTax in math.js", { llm: true }); + assert.equal(r.llm.enabled, true); + assert.ok(r.route.model.id, "still returns a routed model"); + assert.equal(typeof r.okToProceed, "boolean"); + assert.ok(["deterministic", "llm-verified", "llm-agreed"].includes(r.llm.provenance.goalAnchor)); +});