diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 5ac06a5..7561cb1 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1,2 @@ +# Powers the "Sponsor" button on the repo. Add funding platforms here (github, ko_fi, …). github: [CodeWithJuber] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f7b3ba..ab35964 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,5 @@ +# CI gate for every push and PR: test matrix (Node 18/20/22), Biome lint+format, ShellCheck, +# the zero-runtime-dependency assertion, and dependency review. All must pass to merge. name: CI on: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 67f549e..30e4277 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -43,8 +43,8 @@ packages; the production runtime remains Node-only and zero-dependency. ## Architecture — a four-layer config compiler with ONE source ``` - source/AGENTS.md (canonical rules: git · testing · security · style) - │ + source/rules.json (canonical rules: git · testing · security · style) + │ (+ source/substrate.json, source/mcp.json) forge sync (emitter: content-hash + DO-NOT-EDIT headers) ┌───────────────┬───────┴───────┬───────────────┬─────────────┐ Claude CLAUDE.md Codex AGENTS.md Cursor .mdc Gemini settings Aider .aider.conf.yml ... @@ -133,8 +133,9 @@ forgekit/ doctor.js # health checks emit/ # one module per tool (claude, codex, cursor, gemini, aider, copilot, windsurf, zed, continue) + mcp source/ - AGENTS.md # THE canonical source - rules/ # git.md, testing.md, security.md, style.md → assembled + rules.json # THE canonical rules source (git · testing · security · style) + substrate.json # cognitive-substrate defaults (thresholds, routing, llm knobs) + mcp.json # MCP server definitions emitted into each tool global/ # installs into ~/.forge, symlinked into ~/.claude tools/ crew/ guards/ mcp/atlas/ recall/ lean/ statusline.sh settings.template.json templates/project-layer/ # per-repo template (was hostlelo-project-layer) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f207d5..d7aea4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Added -- **Opt-in LLM adjudication for the substrate (`FORGE_LLM=1`)** — one shared, fail-safe `claude -p` proposer (`src/adjudicate.js`) wired thinly into the assumption gate (M2), model routing (M1), impact/blast-radius, and goal-drift (M4). The model only *proposes*; every proposal is verified against the deterministic rubric, the code graph, or a grep before it can move a verdict (routing raise-only; the gate tightens-only; impact edges must resolve and grep-confirm; goal-drift off→on only). Off by default — behaviour is unchanged unless enabled — never blocks, and the ambient Claude Code hook stays deterministic unless `FORGE_LLM_AMBIENT=1`. `forge substrate --json` now carries an `llm.provenance` map (`deterministic` / `llm-verified` / `llm-agreed`) per faculty for auditability. +- **Opt-in LLM adjudication for the substrate (`FORGE_LLM=1`)** — one shared, fail-safe `claude -p` proposer (`src/adjudicate.js`) wired thinly into the assumption gate (M2), model routing (M1), impact/blast-radius, and goal-drift (M4). The model only *proposes*; every proposal is verified against the deterministic rubric, the code graph, or a grep before it can move a verdict. Off by default — behaviour is unchanged unless enabled — never blocks, and the ambient Claude Code hook stays deterministic unless `FORGE_LLM_AMBIENT=1`. `forge substrate --json` carries an `llm.provenance` map per faculty for auditability. +- **Bidirectional verified reconcile (default on when `FORGE_LLM=1`; `llm.bidirectional` in `source/substrate.json` to disable).** A verified reading may now *reduce* caution as well as add it — clear a false "ASK FIRST" (`llm-cleared`) and route a task *down* a tier (`llm-lowered`) — but only within `band` and never past the hard floors: the gate can't clear a task with no concrete anchor or one naming symbols/files the repo lacks, and routing can't drop below a strong-signal (algorithmic/architectural) floor. Set `llm.bidirectional: false` for the conservative tighten-/raise-only mode. Impact edges stay graph-+-grep-verified; goal-drift stays off→on with a goal-referencing reason. - **Explicit memory `val` term** — lesson retrieval now decomposes into the white paper's `relevance × freshness × validity × scope`, with `validity()` (a ground-truth Beta posterior over confirmed vs. contradicted outcomes) exported and ranked so outcome-confirmed lessons outrank merely-recent ones. ### Changed diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 4f1fc48..c483f53 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -429,11 +429,15 @@ Create `global/crew/.md` with frontmatter. It installs into `~/.claude/age By default every judgment is a deterministic rubric. `FORGE_LLM=1` adds a thin **proposer** layer (`src/adjudicate.js`) — one shared `claude -p` primitive used by M2/M1/impact/M4. The model never decides: each proposal is verified against the rubric, the code graph, or a grep -before it can move a verdict (routing raise-only; the gate tightens-only; impact edges must be -real + grep-confirmed; goal-drift off→on only). Any failure falls back to the deterministic -path, so the flag is safe to leave off or on. `--json` exposes `llm.provenance` per faculty. -Each faculty pairs a pure `*LLM` proposer with a `reconcile` step — extend by adding both, -never by trusting the model's answer directly. +before it can move a verdict. The reconcile is **bidirectional but rail-guarded** by default — +a verified reading can *clear* a false ask or route a task *down* a tier, not only add caution, +but never past a hard floor (no concrete anchor, unresolved repo entities, or a strong-signal +routing floor). Impact edges must be real + grep-confirmed; goal-drift moves off→on only. Any +failure falls back to the deterministic path, so the flag is safe to leave off or on. `--json` +exposes `llm.provenance` per faculty (`llm-cleared` / `llm-tightened` / `llm-raised` / +`llm-lowered` / …). Set `llm.bidirectional: false` in `source/substrate.json` for the +conservative tighten-/raise-only mode. Each faculty pairs a pure `*LLM` proposer with a +`reconcile` step — extend by adding both, never by trusting the model's answer directly. ### Support a new tool Add an emitter module in `src/emit/.js` (mirror an existing one like diff --git a/docs/cognitive-substrate/README.md b/docs/cognitive-substrate/README.md index 42b6469..07790c8 100644 --- a/docs/cognitive-substrate/README.md +++ b/docs/cognitive-substrate/README.md @@ -185,19 +185,43 @@ Set **`FORGE_LLM=1`** to add a **thin, opt-in semantic layer** on top: a cheap ` call proposes a completeness reading (M2), a complexity band (M1), the coupled edges the regex graph misses (impact), and whether an off-goal file actually serves the goal (M4). -The model is **never the judge — only a proposer.** Every proposal is *verified* before it can -move a verdict, in the direction of the paper's *tabayyun* gate (49:6): +```mermaid +flowchart LR + T[task / edit] --> R[deterministic rubric] + T --> P[LLM proposer] + R --> C{reconcile} + P --> V[verify: rubric band · repo grounding · grep · tests] + V --> C + C -->|passes checks| M[verdict moves\nllm-cleared / lowered / raised / verified] + C -->|fails / unavailable| D[verdict holds\ndeterministic] +``` -- **routing** can only be *raised* (`max` with the rubric), never lowered on the model's word; -- **the assumption gate only tightens** — it can add an ask, never clear a deterministic one; -- **impact edges** are kept only if the file is real *and* a grep confirms the reference; -- **goal-drift** rescues an off-goal file only with a goal-referencing reason (off→on only). +The model **proposes**; the deterministic rubric, the code graph, and the tests **verify**. +The verdict only moves when the proposal survives that check — otherwise it falls back, unchanged. + +The model is **never the judge — only a proposer.** Every proposal is *verified* before it can +move a verdict, in the direction of the paper's *tabayyun* gate (49:6). By default the reconcile +is **bidirectional but rail-guarded** — a verified reading can lower caution as well as raise it, +but never past a hard floor: + +- **routing** — a *raise* is free (spotting hidden complexity costs at most a bigger model); a + *lower* is bounded to one band and never drops below a strong-signal (algorithmic/architectural) + floor, so a "distributed rate-limiter" can't be talked down to the cheap tier; +- **the assumption gate** — can *clear* a false ask **or** *add* one, but never clears a task + with no concrete anchor, or one naming symbols/files the repo doesn't define; +- **impact edges** — kept only if the file is real *and* a grep confirms the reference; +- **goal-drift** — rescues an off-goal file only with a goal-referencing reason (off→on only). + +> **Note** — set `llm.bidirectional: false` in +> [`source/substrate.json`](../../source/substrate.json) for the conservative tighten-/raise-only +> mode (caution can only ever increase). It is **fail-safe**: any error, timeout, or unparseable reply falls back to the deterministic path (behaviour is byte-identical with the flag off), and it **never blocks**. `--json` output -carries a `llm.provenance` map (`deterministic` / `llm-verified` / `llm-agreed`) per faculty so -every model-touched decision is auditable. Off by default; the ambient Claude Code hook stays -deterministic unless you also set `FORGE_LLM_AMBIENT=1`. Config lives in +carries an `llm.provenance` map (`deterministic` / `llm-cleared` / `llm-tightened` / +`llm-raised` / `llm-lowered` / `llm-verified`) per faculty so every model-touched decision is +auditable. Off by default; the ambient Claude Code hook stays deterministic unless you also set +`FORGE_LLM_AMBIENT=1`. Config lives in [`source/substrate.json`](../../source/substrate.json) → `llm`. --- diff --git a/docs/legacy/PLAYBOOK.md b/docs/legacy/PLAYBOOK.md index 97fdb24..94502a4 100644 --- a/docs/legacy/PLAYBOOK.md +++ b/docs/legacy/PLAYBOOK.md @@ -1,5 +1,10 @@ # Playbook — how to drive your Claude setup +> ⚠️ **Archived — pre-Forge.** Kept for history; **not maintained**. It describes an older +> hand-rolled `~/.claude` setup, not today's `forge` CLI. For the current workflow start at +> **[ONBOARDING.md](../../ONBOARDING.md)** → **[docs/GUIDE.md](../GUIDE.md)**. Any credentials or +> open action items below are historical and superseded — do not act on them. + Your global config auto-loads skills/rules by what you ask; you rarely invoke them by name. Below: the exact prompt per situation, and what fires under the hood. diff --git a/docs/legacy/RECONCILE.md b/docs/legacy/RECONCILE.md index 47fb1c8..dfc8dfc 100644 --- a/docs/legacy/RECONCILE.md +++ b/docs/legacy/RECONCILE.md @@ -1,5 +1,10 @@ # Audit of your live ~/.claude + how this bundle fits +> ⚠️ **Archived — pre-Forge.** Kept for history; **not maintained**. It describes an older +> hand-rolled `~/.claude` setup, not today's `forge` CLI. For the current workflow start at +> **[ONBOARDING.md](../../ONBOARDING.md)** → **[docs/GUIDE.md](../GUIDE.md)**. Any credentials or +> open action items below are historical and superseded — do not act on them. + Read from your actual machine on 2026-07-04. Honest feedback, then what changed. ## 🔴 Fix now diff --git a/docs/legacy/RESEARCH-claude-config.md b/docs/legacy/RESEARCH-claude-config.md index a84583b..4327eba 100644 --- a/docs/legacy/RESEARCH-claude-config.md +++ b/docs/legacy/RESEARCH-claude-config.md @@ -1,5 +1,9 @@ # Deep research — best Claude Code configs vs. yours (2026) +> ⚠️ **Archived — pre-Forge.** Kept for history; **not maintained**. Research notes from the +> pre-Forge era. For the current design see **[docs/cognitive-substrate/README.md](../cognitive-substrate/README.md)** +> and **[ARCHITECTURE.md](../../ARCHITECTURE.md)**. + Multi-source, cross-checked. Verdict up front: **your config already matches or exceeds documented community best practice on ~90% of dimensions.** Two genuine gaps remain. Unbiased finding: the quality move now is *consolidation, not adding diff --git a/docs/legacy/RUN.md b/docs/legacy/RUN.md index 632bccc..c543d51 100644 --- a/docs/legacy/RUN.md +++ b/docs/legacy/RUN.md @@ -1,5 +1,10 @@ # RUN — command sheet (current state) +> ⚠️ **Archived — pre-Forge.** Kept for history; **not maintained**. It describes an older +> hand-rolled `~/.claude` setup, not today's `forge` CLI. For the current workflow start at +> **[ONBOARDING.md](../../ONBOARDING.md)** → **[docs/GUIDE.md](../GUIDE.md)**. Any credentials or +> open action items below are historical and superseded — do not act on them. + Bundle: `~/Downloads/claude-global-config/`. Most of this is **already applied live** on your machine. This is the accurate final command sheet. diff --git a/docs/legacy/reports/2026-07-04.md b/docs/legacy/reports/2026-07-04.md index de97147..6ed4431 100644 --- a/docs/legacy/reports/2026-07-04.md +++ b/docs/legacy/reports/2026-07-04.md @@ -1,5 +1,8 @@ # Daily Config Radar — 2026-07-04 +> ⚠️ **Archived — pre-Forge.** A historical auto-generated report, kept for the record. Not +> maintained and not the current state. See **[docs/GUIDE.md](../../GUIDE.md)** for today's tooling. + **TL;DR** - 🔺 **Claude Code 2.1.200** (Jul 3) changes the *default permission mode to "Manual"* and stops AskUserQuestion auto-continuing — your `settings.json` uses `defaultMode: "default"`, worth reviewing. Subagents now run in the background by default (helps your `scout`/`verifier` flow). - 🎨 **shadcn/ui** made **Base UI the default** (over Radix) as of July 2026 — your `ui-workflow` skill + shadcn MCP should expect Base UI in new projects. diff --git a/source/substrate.json b/source/substrate.json index 8db866b..fee67d8 100644 --- a/source/substrate.json +++ b/source/substrate.json @@ -21,7 +21,11 @@ "enabled": false, "model": "haiku", "timeoutMs": 20000, - "note": "Opt-in (FORGE_LLM=1). The model PROPOSES; the deterministic rubric + repo grounding + tests VERIFY. Any failure falls back to the deterministic path — never blind, never blocking." + "bidirectional": true, + "band": 0.25, + "routingBand": 0.2, + "signalFloor": 0.4, + "note": "Opt-in (FORGE_LLM=1). The model PROPOSES; the deterministic rubric + repo grounding + tests VERIFY. Any failure falls back to the deterministic path — never blind, never blocking. `bidirectional` (default true) lets a verified reading also clear a false ask / route a tier down, within `band`/`routingBand` and never past the hard floors (`signalFloor`, repo grounding, no-anchor). Set false for the conservative tighten/raise-only mode." }, "policies": { "preAction": "Run substrate before ambiguous, expensive, multi-file, or mutating work.", diff --git a/src/preflight.js b/src/preflight.js index ae4ebc1..a6d428d 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -248,20 +248,33 @@ export function assessTaskLLM(task, { run = buildRunner() } = {}) { /** * Verify-don't-trust reconcile for M2. The model may only move completeness within ±band of the - * deterministic score (so a clearly-specified/vague task can't be flipped), and may only ADD an - * ask, never clear a deterministic one. Extra questions survive only if they map to a rubric- - * flagged dimension or (via `grounded`) reference a real repo entity. + * deterministic score, so a clearly-specified or clearly-vague task can never be flipped — only a + * borderline reading shifts. In `bidirectional` mode (default) the ask is recomputed purely from + * that bounded completeness, so a verified reading can also CLEAR a false ask — but two hard + * floors the model can never override still force the ask: a task with no concrete anchor + * (`hardUnderspecified`), or one naming symbols/files the repo doesn't define (`hasUnresolved`). + * With `bidirectional:false` the gate only ever tightens (the conservative pre-bidirectional + * behaviour). Extra questions survive only if they map to a rubric-flagged dimension or (via + * `grounded`) reference a real repo entity. * @param {object} det - assessTask() result * @param {{completeness:number, missing:string[], questions:string[]}|null} proposal * @param {object} [opts] * @param {number} [opts.askThreshold] * @param {number} [opts.band] * @param {(q:string)=>boolean} [opts.grounded] + * @param {boolean} [opts.bidirectional] + * @param {boolean} [opts.hasUnresolved] */ export function reconcileAssumption( det, proposal, - { askThreshold = 0.6, band = 0.25, grounded = () => false } = {}, + { + askThreshold = 0.6, + band = 0.25, + grounded = () => false, + bidirectional = true, + hasUnresolved = false, + } = {}, ) { if (!proposal) return { ...det, provenance: { path: "deterministic" } }; const bounded = Math.max( @@ -274,12 +287,18 @@ export function reconcileAssumption( (q) => proposal.missing.some((m) => flaggedDims.has(m)) || grounded(q), ); const questions = [...new Set([...det.questions, ...extraQuestions])].slice(0, 3); - // The gate only ever tightens: an ask stands if the rubric asked, the task is hard- - // underspecified, or the reconciled completeness is below threshold. - const shouldAsk = det.shouldAsk || det.hardUnderspecified || completeness < askThreshold; + // Bidirectional (default): the ask follows the bounded completeness, guarded by two floors the + // model can't override. Tighten-only: the rubric's ask always stands, the model can only add one. + const shouldAsk = bidirectional + ? det.hardUnderspecified || hasUnresolved || completeness < askThreshold + : det.shouldAsk || det.hardUnderspecified || completeness < askThreshold; const risk = completeness < 0.45 ? "high" : completeness < 0.7 ? "medium" : "low"; const moved = Math.abs(completeness - det.completeness) > 1e-9 || questions.length !== det.questions.length; + let path; + if (shouldAsk && !det.shouldAsk) path = "llm-tightened"; + else if (!shouldAsk && det.shouldAsk) path = "llm-cleared"; + else path = moved ? "llm-verified" : "llm-agreed"; return { ...det, completeness, @@ -289,7 +308,7 @@ export function reconcileAssumption( shouldAsk && !questions.length ? ["What exactly should this produce, and how will we know it is correct?"] : questions, - provenance: { path: moved ? "llm-verified" : "llm-agreed", detCompleteness: det.completeness }, + provenance: { path, detCompleteness: det.completeness }, }; } @@ -342,11 +361,22 @@ export function clarifyBlock(result, { threshold = 0.5 } = {}) { * @param {string} [opts.model] * @param {number} [opts.timeoutMs] * @param {(p:string)=>string} [opts.run] + * @param {boolean} [opts.bidirectional] + * @param {number} [opts.band] */ export function preflightRepo( root, text, - { allowBuild = true, askThreshold = 0.6, llm, model, timeoutMs, run } = {}, + { + allowBuild = true, + askThreshold = 0.6, + llm, + model, + timeoutMs, + run, + bidirectional = true, + band, + } = {}, ) { const atlas = loadAtlas(root) || (allowBuild ? buildAtlas({ root }) : null); const hasSymbol = atlas ? (name) => has(atlas, name) : () => true; @@ -364,5 +394,17 @@ export function preflightRepo( const { symbols, files } = referencedEntities(q); return symbols.some(hasSymbol) || files.some((f) => existsSync(join(root, f))); }; - return { ...gap, assumption: reconcileAssumption(det, proposal, { askThreshold, grounded }) }; + // Repo grounding is a hard floor on clearing: if the task names entities the repo lacks, the + // model can never wave the gate through no matter how "complete" it judges the prose. + const hasUnresolved = gap.unresolved.symbols.length + gap.unresolved.files.length > 0; + return { + ...gap, + assumption: reconcileAssumption(det, proposal, { + askThreshold, + grounded, + bidirectional, + hasUnresolved, + ...(typeof band === "number" ? { band } : {}), + }), + }; } diff --git a/src/route.js b/src/route.js index d5d4ca9..42769d8 100644 --- a/src/route.js +++ b/src/route.js @@ -123,8 +123,9 @@ export function recommend(score, norm = {}) { } // M1 routing — LLM proposer. Estimates task complexity c(x) as a coarse band. PROPOSER ONLY: -// reconcile takes the MAX with the deterministic score, so the model can only RAISE the tier -// (never under-provision on its own word); escalation still gates on a verified failure. +// the reconcile in routeTask() lets a RAISE through freely but bounds any LOWER (within one band +// and never below a strong-signal floor), so the model can escalate on hidden complexity yet can +// never under-provision a genuinely hard task; escalation still gates on a verified failure. const BAND_FLOOR = { cheap: 0.15, mid: 0.4, premium: 0.65 }; export function buildComplexityPrompt(task) { @@ -156,8 +157,15 @@ export function complexityLLM(task, { run = buildRunner() } = {}) { * @param {string} [opts.model] * @param {number} [opts.timeoutMs] * @param {(p:string)=>string} [opts.run] + * @param {boolean} [opts.bidirectional] + * @param {number} [opts.routingBand] + * @param {number} [opts.signalFloor] */ -export function routeTask(root, task, { llm, model, timeoutMs, run } = {}) { +export function routeTask( + root, + task, + { llm, model, timeoutMs, run, bidirectional = true, routingBand = 0.2, signalFloor = 0.4 } = {}, +) { const { symbols, files } = referencedEntities(task); const fanout = symbols.reduce((m, sym) => Math.max(m, grepFanout(root, sym)), 0); const churn = files.reduce((m, f) => Math.max(m, gitChurn(root, f)), 0); @@ -178,27 +186,49 @@ export function routeTask(root, task, { llm, model, timeoutMs, run } = {}) { const { score: repoScore, norm } = complexity(signals); const rubric = rubricComplexity(task); const rubricScore = Math.min(1, rubric.rawScore / 10); - // M1 proposer (opt-in): the model may only RAISE the tier — max() with the deterministic - // scores — so it never routes down on its own word. Fail-safe: null proposal is ignored. + const detScore = Math.max(repoScore, rubricScore); + // M1 proposer (opt-in): the model PROPOSES a complexity band. A RAISE is free (spotting hidden + // complexity costs at most a bigger model). A LOWER is bounded — never more than one `band` + // below the rubric, and never below `signalFloor` when the rubric detected algorithmic or + // architectural terms, so a "distributed rate-limiter" can't be talked down to the cheap tier. + // With `bidirectional:false` it stays raise-only. Fail-safe: a null proposal is ignored. const proposal = llmEnabled({ llm }) ? complexityLLM(task, { run: run || buildRunner({ model, timeoutMs }) }) : null; - const score = Math.max(repoScore, rubricScore, proposal?.score ?? 0); + const strongSignal = rubric.signals.hasAlgorithmicTerms || rubric.signals.hasArchitecturalTerms; + let score = detScore; + let path = proposal ? "llm-agreed" : "deterministic"; + if (proposal) { + if (proposal.score > detScore) { + score = proposal.score; // free raise + path = "llm-raised"; + } else if (bidirectional && proposal.score < detScore) { + const floor = Math.max(detScore - routingBand, strongSignal ? signalFloor : 0); + const lowered = Math.max(floor, proposal.score); + if (lowered < detScore) { + score = lowered; // bounded lower + path = "llm-lowered"; + } + } + } const recommended = recommend(score, norm); - const raisedByLLM = proposal != null && proposal.score > Math.max(repoScore, rubricScore); return { score, repoScore, signals, rubric, - llm: proposal ? { band: proposal.band, reason: proposal.reason, raised: raisedByLLM } : null, - provenance: { path: raisedByLLM ? "llm-verified" : proposal ? "llm-agreed" : "deterministic" }, + llm: proposal + ? { band: proposal.band, reason: proposal.reason, direction: path.replace("llm-", "") } + : null, + provenance: { path }, ...recommended, reasons: [ ...new Set([ ...(recommended.reasons || []), ...rubric.reasons.filter((r) => r.weight > 0).map((r) => r.reason), - ...(raisedByLLM ? [`model judged ${proposal.band}: ${proposal.reason}`] : []), + ...(path === "llm-raised" || path === "llm-lowered" + ? [`model judged ${proposal.band} (${path.replace("llm-", "")}): ${proposal.reason}`] + : []), ]), ], }; diff --git a/src/substrate.js b/src/substrate.js index c8eabf7..bc928bb 100644 --- a/src/substrate.js +++ b/src/substrate.js @@ -111,13 +111,23 @@ export function predictImpact(root, target, { threshold = 0.1, llm, model, timeo * @param {boolean} [opts.llm] * @param {string} [opts.model] * @param {number} [opts.timeoutMs] + * @param {boolean} [opts.bidirectional] */ export function substrateCheck( root, task, - { threshold = 0.1, askThreshold = 0.6, allowBuild = true, llm, model, timeoutMs } = {}, + { + threshold = 0.1, + askThreshold = 0.6, + allowBuild = true, + llm, + model, + timeoutMs, + bidirectional, + } = {}, ) { const text = String(task || ""); + const spec = loadSubstrateSpec(); // LLM adjudication is opt-in. On the ambient hook path (allowBuild:false) it stays OFF unless // FORGE_LLM_AMBIENT=1, so the per-prompt hook never pays model latency by default. An explicit // `llm` option always wins. Every faculty is fail-safe: a null proposal keeps the rubric. @@ -127,7 +137,19 @@ export function substrateCheck( : allowBuild ? llmEnabled() : process.env.FORGE_LLM_AMBIENT === "1"; - const llmOpts = { llm: useLLM, model, timeoutMs }; + // Bidirectional (clear-a-false-ask / route-down, within rails) follows the JSON default unless + // the caller overrides it. The numeric bands/floor come from the same config block. + const bi = + typeof bidirectional === "boolean" ? bidirectional : (spec?.llm?.bidirectional ?? true); + const llmOpts = { + llm: useLLM, + model, + timeoutMs, + bidirectional: bi, + band: spec?.llm?.band, + routingBand: spec?.llm?.routingBand, + signalFloor: spec?.llm?.signalFloor, + }; const entities = referencedEntities(text); const preflight = preflightRepo(root, text, { askThreshold, allowBuild, ...llmOpts }); const route = routeTask(root, text, llmOpts); @@ -179,9 +201,12 @@ export function substrateCheck( goalAnchor: goalDrift(root, text, llmOpts), verification: { checklist: verificationChecklist(root) }, substrate: loadSubstrateSpec(), - // Which faculties, if any, had a model proposal survive external verification this run. + // Which faculties, if any, had a model proposal survive external verification this run, and + // which direction it moved (…-cleared / …-tightened for the gate, …-raised / …-lowered for + // routing). Every non-deterministic value was checked before it counted. llm: { enabled: useLLM, + bidirectional: bi, provenance: { assumption: preflight.assumption.provenance?.path ?? "deterministic", route: route.provenance?.path ?? "deterministic", @@ -202,8 +227,8 @@ export function substrateCheck( // Proposed by a model, then checked against the repo/graph/tests before it could move a // verdict — safe to surface, never blindly trusted (whitepaper tabayyun gate). llmVerified: [ - "assumption refinement (bounded by the rubric)", - "routing escalation (raise-only)", + "assumption refinement (bounded ±band; clears a false ask only past the no-anchor + repo-grounding floors)", + "routing (free raise; bounded lower, never below strong-signal floor)", "impact edges (graph + grep verified)", "goal-drift rescue (off→on, goal-referenced)", ], diff --git a/test/preflight.test.js b/test/preflight.test.js index 952509d..483d887 100644 --- a/test/preflight.test.js +++ b/test/preflight.test.js @@ -147,3 +147,85 @@ test("preflightRepo (llm on): fail-safe — a throwing runner keeps the determin assert.equal(withLlm.assumption.shouldAsk, base.assumption.shouldAsk); assert.equal(withLlm.assumption.provenance.path, "deterministic"); }); + +// --- Bidirectional M2 reconcile: clearing a false ask, guarded by hard floors --- +const detStub = (over = {}) => ({ + completeness: 0.5, + risk: "medium", + shouldAsk: true, + hardUnderspecified: false, + missing: [], + questions: ["What exactly should this produce?"], + reasons: [], + ...over, +}); + +test("bidirectional: a verified raise clears a borderline false ask", () => { + const det = detStub({ completeness: 0.5, shouldAsk: true }); + const r = reconcileAssumption( + det, + { completeness: 0.85, missing: [], questions: [] }, + { bidirectional: true, hasUnresolved: false }, + ); + assert.equal(r.shouldAsk, false, "bounded raise crosses the threshold → gate clears"); + assert.equal(r.provenance.path, "llm-cleared"); +}); + +test("bidirectional: a hard-underspecified task is NEVER cleared", () => { + const det = detStub({ completeness: 0.5, shouldAsk: true, hardUnderspecified: true }); + const r = reconcileAssumption(det, { completeness: 1, missing: [], questions: [] }, {}); + assert.equal(r.shouldAsk, true, "no concrete anchor → the model can't wave it through"); +}); + +test("bidirectional: an unresolved-entity task is NEVER cleared (repo grounding floor)", () => { + const det = detStub({ completeness: 0.55, shouldAsk: true }); + const r = reconcileAssumption( + det, + { completeness: 0.95, missing: [], questions: [] }, + { hasUnresolved: true }, + ); + assert.equal(r.shouldAsk, true, "names symbols/files the repo lacks → still asks"); +}); + +test("bidirectional: a genuinely vague task can't be lifted over the line (band clamp)", () => { + const det = detStub({ completeness: 0.2, shouldAsk: true }); + const r = reconcileAssumption( + det, + { completeness: 1, missing: [], questions: [] }, + { band: 0.25 }, + ); + assert.ok(r.completeness <= 0.45 + 1e-9, "clamped to det+band"); + assert.equal(r.shouldAsk, true, "0.45 < 0.6 threshold → still asks"); +}); + +test("bidirectional: the model can still TIGHTEN a rubric-proceed task into an ask", () => { + const det = detStub({ completeness: 0.7, shouldAsk: false, questions: [] }); + const r = reconcileAssumption( + det, + { completeness: 0.4, missing: [], questions: [] }, + { band: 0.25 }, + ); + assert.equal(r.shouldAsk, true, "lowered below threshold → now asks"); + assert.equal(r.provenance.path, "llm-tightened"); +}); + +test("bidirectional:false — the model can never clear a deterministic ask", () => { + const det = detStub({ completeness: 0.5, shouldAsk: true }); + const r = reconcileAssumption( + det, + { completeness: 0.95, missing: [], questions: [] }, + { bidirectional: false, hasUnresolved: false }, + ); + assert.equal(r.shouldAsk, true, "raise-only/tighten-only mode keeps the rubric's ask"); +}); + +test("preflightRepo (bidirectional, integration): unresolved symbol floor blocks a clear", () => { + const root = mkdtempSync(join(tmpdir(), "forge-pre-")); + // Task names `ghostSymbol`, which the (empty) repo doesn't define → the grounding floor holds + // even though the model votes fully specified. + const r = preflightRepo(root, "refactor `ghostSymbol` to be faster", { + llm: true, + run: () => '{"completeness":0.95,"missing":[],"questions":[]}', + }); + assert.equal(r.assumption.shouldAsk, true, "unresolved entity keeps the gate closed"); +}); diff --git a/test/route.test.js b/test/route.test.js index 976d1d5..9508e9d 100644 --- a/test/route.test.js +++ b/test/route.test.js @@ -80,18 +80,52 @@ test("complexityLLM: parses a band into a score floor, rejects junk", () => { assert.equal(complexityLLM("x", { run: () => "not json" }), null); }); -test("routeTask (llm on): the model may only RAISE the tier, never lower it", () => { +test("routeTask (llm on): a RAISE is free — the model can escalate a trivial-looking task", () => { const root = mkdtempSync(join(tmpdir(), "forge-route-")); const task = "write a function to check if a number is prime"; - // Model says 'premium' on a trivial task → routing escalates (safe direction). const up = routeTask(root, task, { llm: true, run: () => '{"band":"premium","reason":"x"}' }); assert.ok(["opus", "fable"].includes(up.key), `raised to ${up.key}`); - assert.equal(up.provenance.path, "llm-verified"); - assert.equal(up.llm.raised, true); - // Model says 'cheap' → cannot pull a task below the deterministic floor. - const down = routeTask(root, task, { llm: true, run: () => '{"band":"cheap","reason":"x"}' }); + assert.equal(up.provenance.path, "llm-raised"); + assert.equal(up.llm.direction, "raised"); +}); + +test("routeTask (bidirectional): the model can LOWER an over-provisioned generic task, bounded", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + // A moderate task with no strong algorithmic/architectural signal — safe to route down. + const task = "add a small in-memory cache with get and set"; + const base = routeTask(root, task); + const down = routeTask(root, task, { + llm: true, + run: () => '{"band":"cheap","reason":"trivial"}', + }); + assert.ok(down.score <= base.score, "cheap band pulls the score down"); + assert.ok(down.score >= base.score - 0.2 - 1e-9, "but never more than one routing band"); + assert.ok(["llm-lowered", "llm-agreed"].includes(down.provenance.path)); +}); + +test("routeTask (bidirectional): a strong-signal task holds the floor even on a 'cheap' vote", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + const task = + "Design and implement a thread-safe distributed rate limiter with a token-bucket algorithm"; const base = routeTask(root, task); - assert.ok(down.score >= base.score, "cheap band never routes below deterministic"); + const down = routeTask(root, task, { + llm: true, + run: () => '{"band":"cheap","reason":"looks easy"}', + }); + assert.ok(down.score >= 0.4, "algorithmic/architectural floor keeps it off the cheap tier"); + assert.ok(["opus", "fable"].includes(down.key) || down.score >= base.score - 0.2); +}); + +test("routeTask (bidirectional:false): reverts to raise-only — a 'cheap' vote can't lower", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + const task = "add a small in-memory cache with get and set"; + const base = routeTask(root, task); + const down = routeTask(root, task, { + llm: true, + bidirectional: false, + run: () => '{"band":"cheap","reason":"trivial"}', + }); + assert.ok(down.score >= base.score, "raise-only mode never routes below deterministic"); }); test("routeTask (llm on): a failing model call falls back to deterministic", () => { diff --git a/test/substrate.test.js b/test/substrate.test.js index f429f9e..b7aef13 100644 --- a/test/substrate.test.js +++ b/test/substrate.test.js @@ -118,3 +118,16 @@ test("substrateCheck (llm on, explicit): opt-in flag threads through and stays f assert.equal(typeof r.okToProceed, "boolean"); assert.ok(["deterministic", "llm-verified", "llm-agreed"].includes(r.llm.provenance.goalAnchor)); }); + +test("substrateCheck: surfaces llm.bidirectional (defaults true from config)", () => { + const root = repo(); + const r = substrateCheck(root, "Refactor computeTax in math.js"); + assert.equal(r.llm.bidirectional, true, "config default is bidirectional"); + assert.equal(typeof r.llm.provenance.assumption, "string"); +}); + +test("substrateCheck: an explicit bidirectional:false threads through", () => { + const root = repo(); + const r = substrateCheck(root, "Refactor computeTax in math.js", { bidirectional: false }); + assert.equal(r.llm.bidirectional, false, "explicit override wins over the config default"); +});