From 46beb55be6599b314b6c2e54e7097b2774475ece Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Sun, 5 Jul 2026 20:26:22 +0400 Subject: [PATCH 01/11] fix(security): pin all package versions, never @latest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Executing a package via npx/uvx @latest auto-runs whatever the moving tag resolves to at run time — the LiteLLM PyPI-malware supply-chain class. Contradicted our own security rule ('PIN a known-good version; never latest'). The context7 one was worst: ambient (MCP startup in every repo), not user-invoked. Pinned to exact verified-good versions (2026-07-05): - @upstash/context7-mcp @latest -> 3.2.2 (source/mcp.json — ships to every user) - ccusage @latest -> 20.0.14 - @fission-ai/openspec @latest -> 1.5.0 - snyk-agent-scan (uvx) @latest -> ==0.5.12 Exact pins (a range still auto-pulls a malicious future release). Re-verify via dev-radar. --- source/mcp.json | 2 +- src/cli.js | 6 ++++-- src/skillgate.js | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/mcp.json b/source/mcp.json index dfd2664..0f69bb4 100644 --- a/source/mcp.json +++ b/source/mcp.json @@ -1,7 +1,7 @@ { "context7": { "command": "npx", - "args": ["-y", "@upstash/context7-mcp@latest"] + "args": ["-y", "@upstash/context7-mcp@3.2.2"] }, "forge-cortex": { "command": "forge", diff --git a/src/cli.js b/src/cli.js index 1080cf5..4e54724 100755 --- a/src/cli.js +++ b/src/cli.js @@ -273,7 +273,8 @@ async function run(argv) { try { out = run("ccusage", ["daily"]); } catch { - out = run("npx", ["-y", "ccusage@latest", "daily"]); + // Pinned (verified 2026-07-05) — never @latest for code we execute; re-verify via dev-radar. + out = run("npx", ["-y", "ccusage@20.0.14", "daily"]); } console.log(out.trim()); } catch { @@ -292,7 +293,8 @@ async function run(argv) { if (sub === "init") { const { execFileSync } = await import("node:child_process"); try { - execFileSync("npx", ["-y", "@fission-ai/openspec@latest", "init"], { + // Pinned (verified 2026-07-05) — never @latest for code we execute; re-verify via dev-radar. + execFileSync("npx", ["-y", "@fission-ai/openspec@1.5.0", "init"], { stdio: "inherit", }); } catch { diff --git a/src/skillgate.js b/src/skillgate.js index 8f65b6b..b453014 100644 --- a/src/skillgate.js +++ b/src/skillgate.js @@ -61,7 +61,8 @@ export function scan(target) { if (isPath && process.env.FORGE_SKILLGATE_NOEXTERNAL !== "1") { try { - const out = execFileSync("uvx", ["snyk-agent-scan@latest", target], { + // Pinned (verified 2026-07-05) — never @latest for code we execute; re-verify via dev-radar. + const out = execFileSync("uvx", ["snyk-agent-scan==0.5.12", target], { encoding: "utf8", stdio: "pipe", timeout: 90000, From a512ebdb3eecc05fd580c3f27f05615e312509bb Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:02:48 +0400 Subject: [PATCH 02/11] =?UTF-8?q?feat(preflight):=20assumption=20detector?= =?UTF-8?q?=20=E2=80=94=20surface=20known-unknowns=20before=20the=20agent?= =?UTF-8?q?=20assumes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The flagship of the pre-flight layer, and the research whitespace: no shipping tool pre-scans the repo for unresolved references before acting. - src/preflight.js (pure): referencedEntities (backtick/camel/snake/file tokens, no English false-positives), ambiguityMarkers, informationGap (unresolved refs + ambiguity → [0,1]), clarifyBlock (silent unless a concrete unresolved ref or gap≥threshold). preflightRepo grounds against the real atlas + fs (load-cached in hooks; build on the explicit CLI). - Wired: UserPromptSubmit hook (mode 'preflight') emits a clarify block as additionalContext so the agent ASKS instead of confabulating; forge preflight "" prints the report. - 7 tests; fail-safe/advisory (never blocks). 125 total green. --- hooks/hooks.json | 4 + src/cli.js | 20 +++++ src/cortex_hook_main.js | 7 ++ src/preflight.js | 163 ++++++++++++++++++++++++++++++++++++++++ test/preflight.test.js | 71 +++++++++++++++++ 5 files changed, 265 insertions(+) create mode 100644 src/preflight.js create mode 100644 test/preflight.test.js diff --git a/hooks/hooks.json b/hooks/hooks.json index 7e01fc4..4d20020 100644 --- a/hooks/hooks.json +++ b/hooks/hooks.json @@ -21,6 +21,10 @@ { "type": "command", "command": "\"${CLAUDE_PLUGIN_ROOT}\"/global/guards/cortex.sh prompt" + }, + { + "type": "command", + "command": "\"${CLAUDE_PLUGIN_ROOT}\"/global/guards/cortex.sh preflight" } ] } diff --git a/src/cli.js b/src/cli.js index 4e54724..76b044f 100755 --- a/src/cli.js +++ b/src/cli.js @@ -19,6 +19,7 @@ const COMMANDS = { cost: "real per-day spend via ccusage + the cost ceiling", spec: "spec-as-contract — init (OpenSpec) / lock / check drift", cortex: "self-correcting project memory — status / why ", + preflight: "assumption check — what a task names that the repo doesn't define", brand: "print the active brand token map", }; @@ -365,6 +366,25 @@ async function run(argv) { console.log("\n stored in .forge/lessons/ (git-committable, auditable)"); return; } + if (cmd === "preflight") { + const { preflightRepo, clarifyBlock } = await import("./preflight.js"); + const task = argv.slice(1).join(" "); + if (!task) { + console.error('usage: forge preflight ""'); + process.exitCode = 1; + return; + } + const r = preflightRepo(process.cwd(), task); + console.log(`${BRAND.brand} preflight — assumption check\n`); + console.log( + ` info-gap: ${r.gap.toFixed(2)} (referenced ${r.entities.symbols.length} symbol(s), ${r.entities.files.length} file(s))`, + ); + const block = clarifyBlock(r); + console.log( + block ? `\n${block}` : "\n ✓ everything this task names is grounded in the codebase.", + ); + return; + } if (!(cmd in COMMANDS)) { console.error(`Unknown command: ${cmd}\nRun \`${BRAND.cli} --help\` to see commands.`); process.exitCode = 1; diff --git a/src/cortex_hook_main.js b/src/cortex_hook_main.js index 2fa378d..da08ae8 100644 --- a/src/cortex_hook_main.js +++ b/src/cortex_hook_main.js @@ -16,6 +16,7 @@ import { readSession, } from "./cortex_hook.js"; import { load } from "./lessons_store.js"; +import { clarifyBlock, preflightRepo } from "./preflight.js"; // Opt-in: distill newly-created lessons into real prose via a cheap model call. Off by // default (deterministic template is used); fail-safe (any error → keep the template). @@ -68,6 +69,12 @@ async function main() { } else if (mode === "pre-edit") { const advice = await preEditAdvisory(root, hook.tool_input?.file_path, today); if (advice) emit("PreToolUse", advice); + } else if (mode === "preflight") { + // Assumption detector: does the task name things the repo doesn't define? + if (typeof hook.prompt === "string" && hook.prompt.trim()) { + const block = clarifyBlock(preflightRepo(root, hook.prompt, { allowBuild: false })); + if (block) emit("UserPromptSubmit", block); + } } } diff --git a/src/preflight.js b/src/preflight.js new file mode 100644 index 0000000..d935803 --- /dev/null +++ b/src/preflight.js @@ -0,0 +1,163 @@ +// forge preflight — the assumption detector. Before the agent spends a token, scan the task +// for code identifiers/files it NAMES but the repo doesn't DEFINE — those are the things it +// will silently ASSUME (the user's "biggest problem"). Plus vague wording with no acceptance +// criteria. Surface the known-unknowns so the agent ASKS instead of confabulating. +// PURE logic here (no fs) so it's fully testable; the repo wrapper (bottom) adds atlas + file checks. +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { build as buildAtlas, has, load as loadAtlas } from "./atlas.js"; + +const CODE_EXT = + /\.(js|ts|jsx|tsx|mjs|cjs|py|go|rs|java|rb|php|c|cc|cpp|h|hpp|cs|json|ya?ml|toml|md|css|scss|html|vue|svelte)$/i; + +// Very common words that look like identifiers but aren't worth grounding. +const STOP = new Set([ + "the", + "this", + "that", + "add", + "fix", + "make", + "use", + "code", + "file", + "test", + "error", + "errors", + "function", + "class", + "value", + "data", + "type", + "name", + "todo", + "note", +]); + +const isCodeIdent = (p, backticked) => { + if (!p || p.length < 2) return false; + if (STOP.has(p.toLowerCase())) return false; + if (backticked) return /^[A-Za-z_$][\w$]*$/.test(p); // trust anything quoted as code + // bare tokens must LOOK like code: camelCase, snake_case, or Pascal+camel + return ( + /[a-z][A-Z]/.test(p) || (p.includes("_") && /[a-z]/i.test(p)) || /^[A-Z][a-z]+[A-Z]/.test(p) + ); +}; + +/** Pure: pull the code identifiers + file paths a task references. */ +export function referencedEntities(text) { + const s = String(text); + const symbols = new Set(); + const files = new Set(); + const consider = (raw, backticked) => { + const tok = raw + .trim() + .replace(/\(\)$/, "") + .replace(/[.,;:]+$/, ""); + if (!tok) return; + if (tok.includes("/") || CODE_EXT.test(tok)) { + files.add(tok); + return; + } + for (const part of tok.split(".").filter(Boolean)) { + if (isCodeIdent(part, backticked)) symbols.add(part); + } + }; + for (const m of s.matchAll(/`([^`]+)`/g)) { + for (const t of m[1].split(/\s+/)) consider(t, true); + } + for (const m of s.matchAll(/[A-Za-z_$][\w$./-]*/g)) { + const t = m[0]; + if (t.includes("/") || CODE_EXT.test(t) || /[a-z][A-Z]/.test(t) || t.includes("_")) { + consider(t, false); + } + } + return { symbols: [...symbols], files: [...files] }; +} + +const AMBIGUITY = [ + /\bsome(how|thing)?\b/i, + /\betc\.?\b/i, + /\band so on\b/i, + /\bas needed\b/i, + /\bappropriate(ly)?\b/i, + /\bproper(ly)?\b/i, + /\ba few\b/i, + /\bseveral\b/i, + /\bmake it work\b/i, + /\bhandle (the )?errors?\b/i, + /\bvarious\b/i, + /\band more\b/i, +]; + +/** Pure: vague phrases that signal missing acceptance criteria. */ +export function ambiguityMarkers(text) { + const out = []; + for (const re of AMBIGUITY) { + const m = String(text).match(re); + if (m) out.push(m[0].trim().toLowerCase()); + } + return [...new Set(out)]; +} + +/** + * Pure: the information gap — unresolved references + ambiguity, normalized to [0,1]. + * @param {string} text + * @param {{hasSymbol?:(name:string)=>boolean, fileExists?:(path:string)=>boolean}} [deps] + */ +export function informationGap(text, deps = {}) { + const { hasSymbol = () => false, fileExists = () => false } = deps; + const { symbols, files } = referencedEntities(text); + const ambiguous = ambiguityMarkers(text); + const unresolvedSymbols = symbols.filter((s) => !hasSymbol(s)); + const unresolvedFiles = files.filter((f) => !fileExists(f)); + const problems = unresolvedSymbols.length + unresolvedFiles.length + ambiguous.length; + const denom = symbols.length + files.length + ambiguous.length; + const gap = denom === 0 ? 0 : Math.min(1, problems / denom); + return { + gap, + unresolved: { symbols: unresolvedSymbols, files: unresolvedFiles }, + ambiguous, + entities: { symbols, files }, + }; +} + +/** + * Pure: render the clarify prompt — or "" when nothing needs clarifying. A single unresolved + * concrete reference always clarifies (high value); pure ambiguity needs to clear the threshold. + */ +export function clarifyBlock(result, { threshold = 0.5 } = {}) { + const nRef = result.unresolved.symbols.length + result.unresolved.files.length; + if (nRef === 0 && result.gap < threshold) return ""; + const lines = [ + "## Before starting — clarify (Forge Preflight)", + "This task names things that aren't grounded in the codebase. Confirm before assuming:", + "", + ]; + for (const s of result.unresolved.symbols) { + lines.push(`- \`${s}\` — not found in the code. Different name, or should it be created?`); + } + for (const f of result.unresolved.files) { + lines.push(`- \`${f}\` — file not found. Confirm the path, or that it's new.`); + } + if (result.ambiguous.length) { + lines.push( + `- Ambiguous: ${result.ambiguous.map((a) => `"${a}"`).join(", ")} — state the concrete acceptance criteria.`, + ); + } + lines.push("", "_Advisory: ask rather than assume._"); + return lines.join("\n"); +} + +/** + * Repo wrapper: gap against the real atlas + filesystem. In hooks pass `allowBuild:false` so we + * only use a CACHED atlas (fast, and if none exists we skip symbol-flagging rather than + * false-alarm on an unindexed repo); the explicit CLI builds it. + */ +export function preflightRepo(root, text, { allowBuild = true } = {}) { + const atlas = loadAtlas(root) || (allowBuild ? buildAtlas({ root }) : null); + return informationGap(text, { + hasSymbol: atlas ? (name) => has(atlas, name) : () => true, + fileExists: (f) => existsSync(join(root, f)), + }); +} diff --git a/test/preflight.test.js b/test/preflight.test.js new file mode 100644 index 0000000..04af558 --- /dev/null +++ b/test/preflight.test.js @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { + ambiguityMarkers, + clarifyBlock, + informationGap, + preflightRepo, + referencedEntities, +} from "../src/preflight.js"; + +test("referencedEntities pulls backtick symbols, file paths, and bare camelCase", () => { + const r = referencedEntities( + "refactor `validateToken` and update `src/auth.ts`, call computeTax", + ); + assert.ok(r.symbols.includes("validateToken")); + assert.ok(r.symbols.includes("computeTax")); + assert.ok(r.files.includes("src/auth.ts")); +}); + +test("referencedEntities ignores plain English (no false identifiers)", () => { + const r = referencedEntities("add a dark mode toggle to the settings page"); + assert.deepEqual(r.symbols, []); + assert.deepEqual(r.files, []); +}); + +test("ambiguityMarkers catches vague wording", () => { + const m = ambiguityMarkers("handle errors somehow and add several validations, etc."); + assert.ok(m.includes("somehow") || m.includes("handle errors")); + assert.ok(m.includes("several")); + assert.ok(m.includes("etc")); +}); + +test("informationGap: all references resolve → gap 0 (silent)", () => { + const has = (n) => n === "computeTax"; + const r = informationGap("refactor `computeTax`", { hasSymbol: has }); + assert.equal(r.gap, 0); + assert.equal(clarifyBlock(r), "", "no clarify block when everything is grounded"); +}); + +test("informationGap: an unresolved symbol drives the gap up and always clarifies", () => { + const r = informationGap("wire `DatabasePool` into the handler", { + hasSymbol: () => false, + }); + assert.ok(r.gap > 0.9); + assert.deepEqual(r.unresolved.symbols, ["DatabasePool"]); + const block = clarifyBlock(r); + assert.match(block, /DatabasePool/); + assert.match(block, /not found/); +}); + +test("clarifyBlock stays silent for a fully-grounded task even with mild wording", () => { + // one resolved symbol, no ambiguity → gap 0 + const r = informationGap("update `computeTax` to round half-up", { + hasSymbol: () => true, + }); + assert.equal(clarifyBlock(r), ""); +}); + +test("preflightRepo grounds against a real repo (missing file/symbol → clarify)", () => { + const root = mkdtempSync(join(tmpdir(), "forge-preflight-")); + writeFileSync(join(root, "tax.js"), "export function computeTax(x){ return x }\n"); + // computeTax exists; `ledgerSync` and src/missing.ts do not + const r = preflightRepo(root, "call `computeTax` then `ledgerSync` in `src/missing.ts`"); + assert.ok(r.entities.symbols.includes("computeTax")); + assert.ok(r.unresolved.symbols.includes("ledgerSync")); + assert.ok(r.unresolved.files.includes("src/missing.ts")); + assert.ok(!r.unresolved.symbols.includes("computeTax"), "resolved symbol is not flagged"); +}); From 401cad70ff0a34bec1ea92cd88a0e79f1f4431c5 Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:07:26 +0400 Subject: [PATCH 03/11] feat(preflight): complexity-based model routing (P2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Route a task to the cheapest CAPABLE model by code-task complexity (the gap generic routers leave open — they score prompt difficulty, not repo signals). - src/model_tiers.js: Haiku 4.5 ($1) → Sonnet 5 ($3) → Opus 4.8 ($5) → Fable 5 ($10), with verified IDs + per-tier use-cases. - src/route.js: complexity() (pure) over files-in-scope, fan-out, churn, past-mistake density, ambiguity, task size → tier; recommend() names the drivers; routeTask() gathers real signals (reuses grepFanout/gitChurn/matchingLessons/preflight). A prime-finder → Haiku; a cross-module high-fanout buggy task → Opus/Fable. - 'both' choice: emitGatewayConfig() writes a LiteLLM config for real auto-routing (gateway traffic only); advisory 'forge route' works with zero deps. LiteLLM wired, not built; config pins exact versions (no floating tag). - forge route "" / forge route gateway. 5 tests. 130 total green. --- src/cli.js | 32 ++++++++++++++++ src/model_tiers.js | 40 +++++++++++++++++++ src/route.js | 96 ++++++++++++++++++++++++++++++++++++++++++++++ test/route.test.js | 60 +++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+) create mode 100644 src/model_tiers.js create mode 100644 src/route.js create mode 100644 test/route.test.js diff --git a/src/cli.js b/src/cli.js index 76b044f..3448c32 100755 --- a/src/cli.js +++ b/src/cli.js @@ -20,6 +20,7 @@ const COMMANDS = { spec: "spec-as-contract — init (OpenSpec) / lock / check drift", cortex: "self-correcting project memory — status / why ", preflight: "assumption check — what a task names that the repo doesn't define", + route: "recommend the cheapest capable model for a task (+ gateway config)", brand: "print the active brand token map", }; @@ -385,6 +386,37 @@ async function run(argv) { ); return; } + if (cmd === "route") { + const r = await import("./route.js"); + if (argv[1] === "gateway") { + const path = r.emitGatewayConfig(process.cwd()); + console.log( + ` wrote ${path} — LiteLLM auto-routes simple tasks to Haiku (gateway traffic only).`, + ); + console.log(" next: pin+install litellm, run it, point ANTHROPIC_BASE_URL at it."); + return; + } + const task = argv.slice(1).join(" "); + if (!task) { + console.error('usage: forge route "" | forge route gateway'); + process.exitCode = 1; + return; + } + const rec = r.routeTask(process.cwd(), task); + console.log(`${BRAND.brand} route — cheapest capable model\n`); + console.log( + ` → ${rec.model.name} (${rec.tier}, $${rec.model.inCost}/$${rec.model.outCost} per M tok)`, + ); + console.log(` ${rec.model.use}`); + console.log( + ` complexity ${rec.score.toFixed(2)}${rec.reasons.length ? ` · driven by: ${rec.reasons.join(", ")}` : ""}`, + ); + console.log( + ` signals: ${rec.signals.files} file(s), fan-out ${rec.signals.fanout}, churn ${rec.signals.churn}, past-mistakes ${rec.signals.pastMistakes}, ambiguity ${rec.signals.ambiguity.toFixed(2)}`, + ); + console.log("\n advisory · auto-routing: `forge route gateway`"); + return; + } if (!(cmd in COMMANDS)) { console.error(`Unknown command: ${cmd}\nRun \`${BRAND.cli} --help\` to see commands.`); process.exitCode = 1; diff --git a/src/model_tiers.js b/src/model_tiers.js new file mode 100644 index 0000000..6da9a9f --- /dev/null +++ b/src/model_tiers.js @@ -0,0 +1,40 @@ +// forge model tiers — the routing target table. Cheapest capable model per complexity tier. +// Costs are $/million tokens (input/output), verified 2026-07-05; re-verify via dev-radar. +// The premise: a prime-number finder does not need Fable 5. Size the model to the task. +export const MODELS = { + haiku: { + id: "claude-haiku-4-5-20251001", + name: "Haiku 4.5", + tier: "simple", + inCost: 1, + outCost: 5, + use: "lint, formatting, docs, stubs, trivial well-defined edits", + }, + sonnet: { + id: "claude-sonnet-5", + name: "Sonnet 5", + tier: "medium", + inCost: 3, + outCost: 15, + use: "refactoring, feature work, tests, code review (the default)", + }, + opus: { + id: "claude-opus-4-8", + name: "Opus 4.8", + tier: "complex", + inCost: 5, + outCost: 25, + use: "architecture, cross-module refactor, novel algorithms, multi-layer debugging", + }, + fable: { + id: "claude-fable-5", + name: "Fable 5", + tier: "extreme", + inCost: 10, + outCost: 50, + use: "only the hardest research-grade reasoning — rarely worth it", + }, +}; + +/** Cheap → expensive. */ +export const TIER_ORDER = ["haiku", "sonnet", "opus", "fable"]; diff --git a/src/route.js b/src/route.js new file mode 100644 index 0000000..5398999 --- /dev/null +++ b/src/route.js @@ -0,0 +1,96 @@ +// forge route — complexity-based model routing. Generic routers score prompt difficulty; +// this scores CODE-TASK complexity from signals Forge already computes (files in scope, impact +// fan-out, churn/fragility, past-mistake density here, ambiguity, task size) → cheapest capable +// tier. Advisory by default; a LiteLLM config emit gives real auto-routing for gateway traffic. +import { writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { matchingLessons } from "./cortex.js"; +import { gitChurn, grepFanout } from "./cortex_features.js"; +import { load as loadLessons } from "./lessons_store.js"; +import { MODELS } from "./model_tiers.js"; +import { preflightRepo, referencedEntities } from "./preflight.js"; + +const clamp01 = (x) => Math.max(0, Math.min(1, x)); + +// Weights sum to 1. Each raw signal is normalized by the point where it reads as "complex". +const WEIGHTS = { + files: 0.22, + fanout: 0.22, + churn: 0.12, + mistakes: 0.18, + ambiguity: 0.12, + size: 0.14, +}; + +/** + * Pure: raw task signals → complexity in [0,1]. + * @param {{files?:number, fanout?:number, churn?:number, pastMistakes?:number, ambiguity?:number, sizeWords?:number}} s + */ +export function complexity(s = {}) { + const norm = { + files: clamp01((s.files ?? 0) / 5), // >5 files touched = complex + fanout: clamp01((s.fanout ?? 0) / 15), // >15 call sites = complex + churn: clamp01((s.churn ?? 0) / 12), // fragile, frequently-changed area + mistakes: clamp01((s.pastMistakes ?? 0) / 3), // repeated pain here + ambiguity: clamp01(s.ambiguity ?? 0), // already 0..1 (from preflight) + size: clamp01((s.sizeWords ?? 0) / 60), // long ask = more moving parts + }; + let score = 0; + for (const k of Object.keys(WEIGHTS)) score += WEIGHTS[k] * norm[k]; + return { score: clamp01(score), norm }; +} + +/** Pure: score → recommended model + the reasons that drove it. */ +export function recommend(score, norm = {}) { + const key = score < 0.25 ? "haiku" : score < 0.55 ? "sonnet" : score < 0.8 ? "opus" : "fable"; + const reasons = Object.entries(norm) + .filter(([, v]) => v >= 0.5) + .map(([k]) => k) + .sort(); + return { key, model: MODELS[key], tier: MODELS[key].tier, reasons }; +} + +/** Repo wrapper: gather the real signals for a task and route it. */ +export function routeTask(root, task) { + const { symbols, files } = referencedEntities(task); + const fanout = symbols.reduce((m, sym) => Math.max(m, grepFanout(root, sym)), 0); + const churn = files.reduce((m, f) => Math.max(m, gitChurn(root, f)), 0); + const pastMistakes = matchingLessons(loadLessons(root), { + files, + symbols, + }).length; + const ambiguity = preflightRepo(root, task, { allowBuild: false }).gap; + const sizeWords = task.trim().split(/\s+/).filter(Boolean).length; + const signals = { + files: files.length, + fanout, + churn, + pastMistakes, + ambiguity, + sizeWords, + }; + const { score, norm } = complexity(signals); + return { score, signals, ...recommend(score, norm) }; +} + +/** Emit a LiteLLM gateway config so simple tasks can auto-route to Haiku (opt-in, gateway-only). */ +export function emitGatewayConfig(root = process.cwd()) { + const path = join(root, "litellm.config.yaml"); + const body = `# Forge Preflight — LiteLLM routing config (complexity tier -> model). +# Auto-routes ONLY traffic sent through the gateway. Advisory 'forge route' works with no gateway. +# pip install "litellm[proxy]==" # supply-chain: pin exact, no floating tag +# litellm --config litellm.config.yaml # then export ANTHROPIC_BASE_URL=http://localhost:4000 +# Models verified 2026-07-05; re-verify via dev-radar. +model_list: + - model_name: forge-simple # ${MODELS.haiku.name} — ${MODELS.haiku.use} + litellm_params: { model: anthropic/${MODELS.haiku.id} } + - model_name: forge-medium # ${MODELS.sonnet.name} — default + litellm_params: { model: anthropic/${MODELS.sonnet.id} } + - model_name: forge-complex # ${MODELS.opus.name} + litellm_params: { model: anthropic/${MODELS.opus.id} } +router_settings: + routing_strategy: simple-shuffle +`; + writeFileSync(path, body); + return path; +} diff --git a/test/route.test.js b/test/route.test.js new file mode 100644 index 0000000..c3b4c6a --- /dev/null +++ b/test/route.test.js @@ -0,0 +1,60 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { complexity, emitGatewayConfig, recommend, routeTask } from "../src/route.js"; + +test("complexity is monotonic and bounded", () => { + const trivial = complexity({ files: 0, fanout: 0, sizeWords: 4 }).score; + const heavy = complexity({ + files: 8, + fanout: 20, + churn: 15, + pastMistakes: 4, + ambiguity: 0.8, + sizeWords: 80, + }).score; + assert.ok(trivial >= 0 && trivial < 0.25, "a trivial task is 'simple'"); + assert.ok(heavy > 0.8 && heavy <= 1, "a heavy task is near the top"); + assert.ok(heavy > trivial); +}); + +test("recommend: a prime-finder gets Haiku, not Fable", () => { + const { score } = complexity({ files: 0, fanout: 0, sizeWords: 5 }); + assert.equal(recommend(score).key, "haiku"); +}); + +test("recommend: a cross-module, high-fanout, buggy task escalates to Opus/Fable", () => { + const { score, norm } = complexity({ + files: 8, + fanout: 20, + churn: 15, + pastMistakes: 4, + ambiguity: 0.9, + sizeWords: 90, + }); + const r = recommend(score, norm); + assert.ok(["opus", "fable"].includes(r.key), `escalated (${r.key})`); + assert.ok( + r.reasons.includes("fanout") && r.reasons.includes("files"), + "reasons name the drivers", + ); +}); + +test("routeTask runs against a real repo and returns a model + reasons", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + const r = routeTask(root, "write a function to check if a number is prime"); + assert.ok(r.model?.id, "picked a concrete model"); + assert.equal(r.key, "haiku", "trivial task → cheapest tier"); + assert.ok(typeof r.score === "number"); +}); + +test("emitGatewayConfig writes a LiteLLM config that never pins @latest", () => { + const root = mkdtempSync(join(tmpdir(), "forge-route-")); + const path = emitGatewayConfig(root); + assert.ok(existsSync(path)); + const yaml = readFileSync(path, "utf8"); + assert.match(yaml, /model_list/); + assert.doesNotMatch(yaml, /@latest|:latest/); +}); From 910635297412586a8ab47823527ec7699a888eb2 Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:12:23 +0400 Subject: [PATCH 04/11] feat(preflight): deterministic task decomposition (P3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split a task into INDEPENDENT clusters so independent work runs in separate sessions (no context pollution) and coupled work stays together — with the 'forgot the related module' guard. - src/scope.js (zero-dep): importGraph (regex import/require/from → undirected file graph), components (connected components), decompose (touched files → clusters + the coupled files you didn't name). Regex imports are approximate (dynamic/DI missed) → graph-MCP seam. - forge scope : N independent groups → 'separate session per group', else lists the coupled files. 3 tests. 133 total green. --- src/cli.js | 28 +++++++++++ src/scope.js | 118 +++++++++++++++++++++++++++++++++++++++++++++ test/scope.test.js | 41 ++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 src/scope.js create mode 100644 test/scope.test.js diff --git a/src/cli.js b/src/cli.js index 3448c32..74ff717 100755 --- a/src/cli.js +++ b/src/cli.js @@ -21,6 +21,7 @@ const COMMANDS = { cortex: "self-correcting project memory — status / why ", preflight: "assumption check — what a task names that the repo doesn't define", route: "recommend the cheapest capable model for a task (+ gateway config)", + scope: "decompose files into independent clusters (+ coupled files you didn't name)", brand: "print the active brand token map", }; @@ -417,6 +418,33 @@ async function run(argv) { console.log("\n advisory · auto-routing: `forge route gateway`"); return; } + if (cmd === "scope") { + const { decompose } = await import("./scope.js"); + const files = argv.slice(1); + if (!files.length) { + console.error("usage: forge scope [file...]"); + process.exitCode = 1; + return; + } + const d = decompose(process.cwd(), files); + console.log(`${BRAND.brand} scope — task decomposition\n`); + if (d.independentGroups > 1) { + console.log( + ` ${d.independentGroups} independent groups → consider a separate session per group:\n`, + ); + } + d.clusters.forEach((c, i) => { + console.log(` [${i + 1}] ${c.touched.join(", ")}`); + if (c.coupled.length) { + const shown = c.coupled.slice(0, 8).join(", "); + console.log( + ` ! also coupled (you didn't name): ${shown}${c.coupled.length > 8 ? " …" : ""}`, + ); + } + }); + if (d.independentGroups === 1) console.log("\n all coupled — keep as one change."); + return; + } if (!(cmd in COMMANDS)) { console.error(`Unknown command: ${cmd}\nRun \`${BRAND.cli} --help\` to see commands.`); process.exitCode = 1; diff --git a/src/scope.js b/src/scope.js new file mode 100644 index 0000000..11285ce --- /dev/null +++ b/src/scope.js @@ -0,0 +1,118 @@ +// forge scope — deterministic task decomposition. Build a cheap import graph (no LLM), find +// connected components, and tell the developer which touched files are INDEPENDENT (→ run in +// separate sessions, so the context window isn't polluted) vs. COUPLED — and which coupled +// files they didn't mention (the "forgot the related module" guard). Regex imports are +// approximate (dynamic/DI edges missed) — a real call-graph MCP is the upgrade seam. +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; + +const SRC = /\.(js|jsx|ts|tsx|mjs|cjs|py)$/; +const IMPORT_RES = [ + /import\s+[^'"]*from\s+['"]([^'"]+)['"]/g, // import x from "y" + /import\s+['"]([^'"]+)['"]/g, // import "y" + /require\(\s*['"]([^'"]+)['"]\s*\)/g, // require("y") + /export\s+[^'"]*from\s+['"]([^'"]+)['"]/g, // export … from "y" + /import\(\s*['"]([^'"]+)['"]\s*\)/g, // dynamic import("y") + /^\s*from\s+(\.[.\w/]*)\s+import/gm, // python: from .y import +]; + +function walk(dir, root, out) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.name === "node_modules" || entry.name.startsWith(".")) continue; + const p = join(dir, entry.name); + if (entry.isDirectory()) walk(p, root, out); + else if (SRC.test(entry.name)) out.push(relative(root, p)); + } +} + +function resolveSpec(fromRel, spec, root, fileSet) { + if (!spec.startsWith(".")) return null; // external / stdlib — not a local edge + const raw = resolve(root, dirname(fromRel), spec); + const cands = [ + raw, + ...[".js", ".ts", ".tsx", ".jsx", ".mjs", ".cjs", ".py"].map((ext) => raw + ext), + ...["index.js", "index.ts"].map((idx) => join(raw, idx)), + ]; + for (const c of cands) { + const rel = relative(root, c); + if (fileSet.has(rel)) return rel; + } + return null; +} + +/** Build an UNDIRECTED file→file import graph (coupling is symmetric for decomposition). */ +export function importGraph(root) { + const files = []; + walk(root, root, files); + const fileSet = new Set(files); + const edges = new Map(files.map((f) => [f, new Set()])); + for (const f of files) { + let text = ""; + try { + text = readFileSync(join(root, f), "utf8"); + } catch { + continue; + } + for (const re of IMPORT_RES) { + for (const m of text.matchAll(re)) { + const target = resolveSpec(f, m[1], root, fileSet); + if (target && target !== f) { + edges.get(f).add(target); + edges.get(target)?.add(f); + } + } + } + } + return { nodes: files, edges }; +} + +/** Connected components (iterative DFS). Each = a set of mutually-coupled files. */ +export function components(graph) { + const seen = new Set(); + const comps = []; + for (const start of graph.nodes) { + if (seen.has(start)) continue; + const stack = [start]; + const comp = []; + seen.add(start); + while (stack.length) { + const cur = stack.pop(); + comp.push(cur); + for (const nb of graph.edges.get(cur) ?? []) { + if (!seen.has(nb)) { + seen.add(nb); + stack.push(nb); + } + } + } + comps.push(comp); + } + return comps; +} + +/** + * Decompose a set of touched files into independent clusters + the coupled files not mentioned. + * @returns {{clusters:{touched:string[], coupled:string[]}[], independentGroups:number}} + */ +export function decompose(root, touched) { + const comps = components(importGraph(root)); + const compOf = new Map(); + comps.forEach((comp, i) => { + for (const f of comp) compOf.set(f, i); + }); + const buckets = new Map(); + let singleton = 0; + for (const t of touched) { + const id = compOf.has(t) ? compOf.get(t) : `solo:${singleton++}`; + if (!buckets.has(id)) buckets.set(id, { touched: [], coupled: new Set() }); + buckets.get(id).touched.push(t); + if (typeof id === "number") { + for (const f of comps[id]) if (!touched.includes(f)) buckets.get(id).coupled.add(f); + } + } + const clusters = [...buckets.values()].map((b) => ({ + touched: b.touched, + coupled: [...b.coupled], + })); + return { clusters, independentGroups: clusters.length }; +} diff --git a/test/scope.test.js b/test/scope.test.js new file mode 100644 index 0000000..37eecc3 --- /dev/null +++ b/test/scope.test.js @@ -0,0 +1,41 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { components, decompose, importGraph } from "../src/scope.js"; + +function repo() { + const root = mkdtempSync(join(tmpdir(), "forge-scope-")); + mkdirSync(join(root, "src"), { recursive: true }); + // src/a.js imports src/b.js (coupled); src/c.js is standalone (independent) + writeFileSync( + join(root, "src/a.js"), + 'import { b } from "./b.js";\nexport const a = () => b();\n', + ); + writeFileSync(join(root, "src/b.js"), "export const b = () => 1;\n"); + writeFileSync(join(root, "src/c.js"), "export const c = () => 2;\n"); + return root; +} + +test("importGraph links a file to what it imports; components separate the islands", () => { + const g = importGraph(repo()); + assert.ok(g.edges.get("src/a.js").has("src/b.js"), "a→b edge"); + assert.ok(g.edges.get("src/b.js").has("src/a.js"), "undirected"); + const comps = components(g); + const sizes = comps.map((c) => c.length).sort(); + assert.deepEqual(sizes, [1, 2], "{a,b} coupled + {c} alone"); +}); + +test("decompose: two unrelated files → two clusters (run as separate sessions)", () => { + const root = repo(); + const d = decompose(root, ["src/a.js", "src/c.js"]); + assert.equal(d.independentGroups, 2, "a and c are independent"); +}); + +test("decompose: editing a.js surfaces the coupled file you didn't mention (b.js)", () => { + const root = repo(); + const d = decompose(root, ["src/a.js"]); + assert.equal(d.clusters.length, 1); + assert.deepEqual(d.clusters[0].coupled, ["src/b.js"], "the forgot-related-module guard"); +}); From 15204b72bcfdff248a761053dab7cdbc670cebc7 Mon Sep 17 00:00:00 2001 From: Juber Shaikh <40266375+CodeWithJuber@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:17:22 +0400 Subject: [PATCH 05/11] =?UTF-8?q?feat(preflight):=20design-quality=20?= =?UTF-8?q?=E2=80=94=20AI-UX=20rules=20+=20assertable=20uicheck=20(P4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuses the design layer instead of building a UX empire. - source/rules.json: an 'AI interfaces & design quality' section every tool inherits via forge sync (anti-slop fonts, WCAG contrast/focus/tap-target/reduced-motion, functional empty states, specific errors, confidence/transparency for AI UIs, pattern selection, and the assert-deterministic / advise-subjective rule). Canonical stays 4767 B (< budget). - src/uicheck.js (pure): WCAG relativeLuminance/contrastRatio/wcagLevel — exact math, no LLM, no false positives — plus the ASSERTABLE vs ADVISORY_ONLY split as shared data. - frontend-verifier crew calibrated: ASSERT only the deterministic (contrast, focus, alt, tap-target, reduced-motion), keep hierarchy/taste/pattern ADVISORY — the fix for hallucinated UI audits. - forge uicheck . 6 tests. 139 total green. --- global/crew/frontend-verifier.md | 22 +++++++-- source/rules.json | 12 +++++ src/cli.js | 23 +++++++++ src/uicheck.js | 84 ++++++++++++++++++++++++++++++++ test/uicheck.test.js | 34 +++++++++++++ 5 files changed, 170 insertions(+), 5 deletions(-) create mode 100644 src/uicheck.js create mode 100644 test/uicheck.test.js diff --git a/global/crew/frontend-verifier.md b/global/crew/frontend-verifier.md index 8a0b4ec..9da5681 100644 --- a/global/crew/frontend-verifier.md +++ b/global/crew/frontend-verifier.md @@ -8,6 +8,7 @@ model: sonnet You verify UI by looking at it, not by reading the code that produced it. Given a URL (and a reference if provided): + 1. Render and screenshot it at a desktop width (~1440) and a mobile width (~390) using the Playwright or chrome-devtools MCP. 2. Compare against the reference/design. Report concrete, fixable differences: @@ -16,9 +17,20 @@ Given a URL (and a reference if provided): 3. Accessibility pass: missing labels/alt, focus states, contrast ratios, keyboard navigation, hit-target size, heading order. +Split every finding into two buckets — this is how you avoid hallucinating audits: + +**ASSERT (deterministic — state it as fact, may block):** contrast ratio (compute the WCAG +number and give it), a missing `:focus-visible` state, a missing `alt`/label, a tap target +under 24×24px, an animation ≥200ms not wrapped in `prefers-reduced-motion`, an empty state +that renders nothing. These are math or a DOM fact — measure, don't guess. + +**ADVISE (subjective — flag for a human, never assert):** visual hierarchy, type-scale +balance, whether the pattern fits, error-message clarity, empty-state usefulness, palette/ +taste, "does the motion feel right." If your confidence is below ~0.8, it belongs here. + Output: -- **Verdict:** matches / needs-fixes. -- **Visual gaps:** each with where it is and what "fixed" looks like. -- **A11y issues:** each with the WCAG concern and the fix. -Attach the screenshots. Report only real differences from the reference or real -a11y problems — not style opinions the design didn't ask for. + +- **Verdict:** matches / needs-fixes (driven only by ASSERT findings). +- **Asserted (deterministic):** each with the measured value + the fix. +- **Advisory (subjective):** each clearly marked as an opinion for a human to weigh. + Attach the screenshots. Never present an opinion as a defect. diff --git a/source/rules.json b/source/rules.json index fc73e01..35a9e65 100644 --- a/source/rules.json +++ b/source/rules.json @@ -77,6 +77,18 @@ "Build and check mobile AND desktop. Include clear loading/empty/error states; avoid layout shift.", "Follow DESIGN.md if present; keep one visual direction per project." ] + }, + { + "id": "ai-ux", + "title": "AI interfaces & design quality", + "rules": [ + "Commit to an aesthetic before coding; avoid AI-slop defaults (Inter/Roboto/Arial/system fonts, purple-gradient-on-white, generic card grids). Pick 2–3 typefaces and a 4–6 color palette with semantic roles.", + "Meet WCAG: body text ≥4.5:1 contrast (large/UI ≥3:1), visible :focus-visible on every interactive element, tap targets ≥24×24px, alt text + form labels, and wrap animations ≥200ms in prefers-reduced-motion.", + "Empty states are never dead ends: show a plain-language message + a clear next step. Error messages name the field, explain the problem, and suggest a fix — never 'Something went wrong'.", + "For AI-driven UIs: show model confidence/uncertainty, make chain-of-thought and tool actions visible, give live preview/feedback over blind prompting, and provide fallback paths when the model fails.", + "Pick the interaction pattern deliberately — a co-editing canvas or inline controls beat a wall-of-text chatbot when the user is steering structured output.", + "When AI audits a UI, only ASSERT the deterministic (contrast math, missing focus ring/alt/label) — keep hierarchy/taste/pattern-fit advisory so the audit doesn't hallucinate false positives." + ] } ] } diff --git a/src/cli.js b/src/cli.js index 74ff717..22350d9 100755 --- a/src/cli.js +++ b/src/cli.js @@ -22,6 +22,7 @@ const COMMANDS = { preflight: "assumption check — what a task names that the repo doesn't define", route: "recommend the cheapest capable model for a task (+ gateway config)", scope: "decompose files into independent clusters (+ coupled files you didn't name)", + uicheck: "deterministic UI check — WCAG contrast (assertable, no guessing)", brand: "print the active brand token map", }; @@ -445,6 +446,28 @@ async function run(argv) { if (d.independentGroups === 1) console.log("\n all coupled — keep as one change."); return; } + if (cmd === "uicheck") { + const { contrastRatio, wcagLevel, ASSERTABLE_CHECKS, ADVISORY_ONLY } = await import( + "./uicheck.js" + ); + const [fg, bg] = [argv[1], argv[2]]; + console.log(`${BRAND.brand} uicheck — deterministic UI review\n`); + if (fg && bg) { + try { + const g = wcagLevel(contrastRatio(fg, bg)); + console.log( + ` contrast ${fg} on ${bg}: ${g.ratio}:1 → ${g.level}${g.passesAA ? " (passes AA)" : " (FAILS AA)"}`, + ); + } catch (e) { + console.error(` ${e.message}`); + process.exitCode = 1; + return; + } + } + console.log(`\n ASSERT (deterministic): ${ASSERTABLE_CHECKS.map((c) => c.id).join(", ")}`); + console.log(` ADVISE (subjective, human-only): ${ADVISORY_ONLY.slice(0, 4).join(", ")} …`); + return; + } if (!(cmd in COMMANDS)) { console.error(`Unknown command: ${cmd}\nRun \`${BRAND.cli} --help\` to see commands.`); process.exitCode = 1; diff --git a/src/uicheck.js b/src/uicheck.js new file mode 100644 index 0000000..246712b --- /dev/null +++ b/src/uicheck.js @@ -0,0 +1,84 @@ +// forge uicheck — the ASSERTABLE half of UI review. AI UI-audits hallucinate on subjective +// calls (hierarchy, taste); this computes the things that are pure math or a DOM fact, so a +// verifier can state them without guessing. WCAG contrast is exact arithmetic — no LLM, no +// false positives. The subjective calls stay ADVISORY (see the frontend-verifier calibration). + +/** Parse #rgb / #rrggbb → {r,g,b} in 0..255. */ +function parseHex(hex) { + let h = String(hex).trim().replace(/^#/, ""); + if (h.length === 3) + h = h + .split("") + .map((c) => c + c) + .join(""); + if (!/^[0-9a-fA-F]{6}$/.test(h)) throw new Error(`bad hex color: ${hex}`); + return { + r: parseInt(h.slice(0, 2), 16), + g: parseInt(h.slice(2, 4), 16), + b: parseInt(h.slice(4, 6), 16), + }; +} + +// WCAG 2.1 sRGB → linear. +const linear = (c) => { + const s = c / 255; + return s <= 0.03928 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; +}; + +/** WCAG relative luminance of a color. */ +export function relativeLuminance(hex) { + const { r, g, b } = parseHex(hex); + return 0.2126 * linear(r) + 0.7152 * linear(g) + 0.0722 * linear(b); +} + +/** WCAG contrast ratio between two colors (1..21). */ +export function contrastRatio(fg, bg) { + const l1 = relativeLuminance(fg); + const l2 = relativeLuminance(bg); + const [hi, lo] = l1 >= l2 ? [l1, l2] : [l2, l1]; + return (hi + 0.05) / (lo + 0.05); +} + +/** + * Grade a contrast ratio. Normal text needs 4.5 (AA) / 7 (AAA); large text or UI components + * need 3 (AA) / 4.5 (AAA). + */ +export function wcagLevel(ratio, { large = false } = {}) { + const aa = large ? 3 : 4.5; + const aaa = large ? 4.5 : 7; + return { + ratio: Math.round(ratio * 100) / 100, + passesAA: ratio >= aa, + passesAAA: ratio >= aaa, + level: ratio >= aaa ? "AAA" : ratio >= aa ? "AA" : "fail", + }; +} + +// The deterministic checks a verifier may ASSERT (vs. the advisory, subjective ones). Kept as +// data so the frontend-verifier and docs share one source of truth. +export const ASSERTABLE_CHECKS = [ + { id: "contrast", how: "compute WCAG ratio; body ≥4.5:1, large/UI ≥3:1" }, + { + id: "focus-visible", + how: "every interactive element has a visible :focus-visible style", + }, + { id: "alt-text", how: "every has a non-empty alt attribute" }, + { + id: "form-labels", + how: "every input/select/textarea has a