From ab34585f21051c6ea870abf31f173b168515a189 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:23 +0000 Subject: [PATCH 1/9] =?UTF-8?q?feat:=20add=20math=20primitives=20=E2=80=94?= =?UTF-8?q?=20Shannon=20entropy,=20charset=20classes,=20exact=20set=20Jacc?= =?UTF-8?q?ard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one numeric family missing from forge's math inventory (cosine, MinHash, Beta posteriors, CUSUM, logistic regression all exist). Entropy backs the upcoming secret-detection upgrade; setJaccard backs exemplar similarity where MinHash estimation is unnecessary. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- src/math.js | 64 +++++++++++++++++++++++++++++++++++++++++++++++ test/math.test.js | 58 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 src/math.js create mode 100644 test/math.test.js diff --git a/src/math.js b/src/math.js new file mode 100644 index 0000000..36d87f9 --- /dev/null +++ b/src/math.js @@ -0,0 +1,64 @@ +// forge math — dependency-free numeric primitives that back decision code. +// Forge's rule for heuristics: DATA may be a table (exemplars, oracle weights, +// policy lists) but DECISIONS must be a formula — graded, inspectable, testable. +// The similarity family (shingles/sketch/jaccard) lives in ledger.js; the decay/ +// posterior family in lessons.js/ledger.js; this module holds the primitives that +// had no home: entropy (secret detection) and exact set overlap (small-set +// similarity, where MinHash estimation is unnecessary). + +/** + * Shannon entropy of a string in bits per character (code points). Empty → 0. + * Random base64/hex credentials sit near their alphabet ceiling (~6/4 bits); + * English prose sits near 3.5–4.5 with far lower per-token entropy — the gap is + * what makes entropy a usable secret signal where format lists have no entry. + * @param {string} s + * @returns {number} + */ +export function shannonEntropy(s) { + const counts = new Map(); + let n = 0; + for (const ch of String(s)) { + counts.set(ch, (counts.get(ch) ?? 0) + 1); + n++; + } + if (!n) return 0; + let h = 0; + for (const c of counts.values()) { + const p = c / n; + h -= p * Math.log2(p); + } + return h; +} + +/** + * Number of character classes present (lowercase, uppercase, digit, other) — a + * cheap charset-diversity signal: machine-generated tokens mix classes, natural + * words rarely use more than two. + * @param {string} s + * @returns {number} 0..4 + */ +export function charsetClasses(s) { + const str = String(s); + let classes = 0; + if (/[a-z]/.test(str)) classes++; + if (/[A-Z]/.test(str)) classes++; + if (/[0-9]/.test(str)) classes++; + if (/[^a-zA-Z0-9]/.test(str)) classes++; + return classes; +} + +/** + * Exact Jaccard similarity of two Sets: |A∩B| / |A∪B|. Both empty → 0. + * For small sets (task shingles vs an exemplar) this is exact and cheaper than + * the MinHash estimate in ledger.js, which exists for scale. + * @param {Set} a + * @param {Set} b + * @returns {number} 0..1 + */ +export function setJaccard(a, b) { + if (!a.size && !b.size) return 0; + let inter = 0; + const [small, large] = a.size <= b.size ? [a, b] : [b, a]; + for (const x of small) if (large.has(x)) inter++; + return inter / (a.size + b.size - inter); +} diff --git a/test/math.test.js b/test/math.test.js new file mode 100644 index 0000000..cee52a5 --- /dev/null +++ b/test/math.test.js @@ -0,0 +1,58 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { charsetClasses, setJaccard, shannonEntropy } from "../src/math.js"; + +test("shannonEntropy: empty and single-char strings", () => { + assert.equal(shannonEntropy(""), 0); + assert.equal(shannonEntropy("a"), 0); + assert.equal(shannonEntropy("aaaaaaaa"), 0); +}); + +test("shannonEntropy: two equiprobable symbols → exactly 1 bit", () => { + assert.equal(shannonEntropy("abababab"), 1); + assert.equal(shannonEntropy("ab"), 1); +}); + +test("shannonEntropy: uniform 4-symbol alphabet → exactly 2 bits", () => { + assert.equal(shannonEntropy("abcdabcd"), 2); +}); + +test("shannonEntropy: random-looking credential scores far above English", () => { + const credential = ["xK9mQ2vT7pL4", "wN8jR3bZ6cF1", "hD5gY0sA"].join(""); + const english = "the quick brown fox jumps over the lazy dog again"; + assert.ok(shannonEntropy(credential) > 4.5, "credential entropy should exceed 4.5 bits"); + assert.ok(shannonEntropy(english) < 4.5, "prose entropy should stay below 4.5 bits"); + assert.ok(shannonEntropy(credential) > shannonEntropy(english)); +}); + +test("shannonEntropy: counts code points, not UTF-16 units", () => { + // Surrogate pairs: two identical emoji = one distinct symbol = 0 bits. + assert.equal(shannonEntropy("😀😀"), 0); + assert.equal(shannonEntropy("😀😅"), 1); +}); + +test("charsetClasses: counts distinct character classes", () => { + assert.equal(charsetClasses(""), 0); + assert.equal(charsetClasses("hello"), 1); + assert.equal(charsetClasses("Hello"), 2); + assert.equal(charsetClasses("Hello1"), 3); + assert.equal(charsetClasses("Hello1!"), 4); + assert.equal(charsetClasses("12345"), 1); +}); + +test("setJaccard: identity, disjoint, empty, partial overlap", () => { + const abc = new Set(["a", "b", "c"]); + assert.equal(setJaccard(abc, new Set(["a", "b", "c"])), 1); + assert.equal(setJaccard(abc, new Set(["x", "y"])), 0); + assert.equal(setJaccard(new Set(), new Set()), 0); + assert.equal(setJaccard(abc, new Set()), 0); + // |{a,b}∩{b,c}| = 1, |∪| = 3 + assert.equal(setJaccard(new Set(["a", "b"]), new Set(["b", "c"])), 1 / 3); +}); + +test("setJaccard: symmetric regardless of argument order", () => { + const small = new Set(["a"]); + const large = new Set(["a", "b", "c", "d"]); + assert.equal(setJaccard(small, large), setJaccard(large, small)); + assert.equal(setJaccard(small, large), 0.25); +}); From 731bd75be67ae66e4e5ad6f21e4d707c3b76eebb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:23 +0000 Subject: [PATCH 2/9] =?UTF-8?q?feat:=20unify=20secret=20detection=20?= =?UTF-8?q?=E2=80=94=20entropy=20gate=20+=20single=20source=20of=20truth?= =?UTF-8?q?=20for=20JS=20and=20shell?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/secrets.js now owns detection AND redaction: the historical format grammars (unchanged semantics, SECRET_RE still exported) plus a Shannon-entropy gate that catches unknown-vendor credentials the format list can never enumerate. Every refusal site (ledger mint, recall/lesson persist, adjudicate both directions, diagnose traces, ledger verify) now calls hasSecret(). global/guards/secret-redact.sh imports the same module via node instead of keeping a narrower divergent sed regex — the shell guard and the JS refusal sites can no longer disagree (sed remains only as a degraded fallback when the source tree is unreachable). Parity pinned by tests. Precision invariants preserved: bare English mentions ("implement password hashing") still pass; hex digests/UUIDs/identifiers are exempt by construction. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- global/guards/secret-redact.sh | 27 +++++++-- src/adjudicate.js | 6 +- src/cortex_distill.js | 4 +- src/diagnose.js | Bin 7135 -> 7130 bytes src/ledger.js | 16 ++--- src/ledger_store.js | 8 +-- src/lessons_store.js | 4 +- src/recall.js | 15 +++-- src/secrets.js | 98 +++++++++++++++++++++++++++++++ test/secretredact.test.js | 19 ++++++ test/secrets.test.js | 103 +++++++++++++++++++++++++++++++++ 11 files changed, 269 insertions(+), 31 deletions(-) create mode 100644 src/secrets.js create mode 100644 test/secrets.test.js diff --git a/global/guards/secret-redact.sh b/global/guards/secret-redact.sh index 6280580..356265a 100755 --- a/global/guards/secret-redact.sh +++ b/global/guards/secret-redact.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash # PostToolUse guard — redact secrets from a tool's output BEFORE it enters context, -# using the `updatedToolOutput` hook primitive. Defensive + advisory: only emits a -# rewrite when something matched. (updatedToolOutput min version is not pinned in the -# docs — this degrades to a no-op on tools/versions that ignore it.) +# using the `updatedToolOutput` hook primitive. Detection/redaction logic lives in +# src/secrets.js (format grammars + entropy scoring — ONE source of truth), imported +# via node so this script can never disagree with the JS refusal sites. Degrades to a +# narrower sed pass over known credential formats only if node or the module is +# unreachable. Defensive + advisory: only emits a rewrite when something changed. set -uo pipefail command -v jq >/dev/null 2>&1 || exit 0 @@ -10,7 +12,24 @@ INPUT="$(cat)" out="$(printf '%s' "$INPUT" | jq -r '.tool_response // .tool_output // empty' 2>/dev/null)" [ -n "$out" ] || exit 0 -red="$(printf '%s' "$out" | sed -E 's/(sk-ant-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,})/[REDACTED]/g')" +# ~/.forge is a symlink to /global, so pwd -P lands inside the real tree in +# both install modes (install.sh symlink and CLAUDE_PLUGIN_ROOT plugin checkout). +DIR="$(cd "$(dirname "$0")" && pwd -P)" +SECRETS_JS="$DIR/../../src/secrets.js" + +red="" +if command -v node >/dev/null 2>&1 && [ -f "$SECRETS_JS" ]; then + red="$(printf '%s' "$out" | node -e ' + let raw = ""; + process.stdin.on("data", (d) => { raw += d; }); + process.stdin.on("end", async () => { + const { redactSecrets } = await import(process.argv[1]); + process.stdout.write(redactSecrets(raw)); + });' "$SECRETS_JS" 2>/dev/null)" || red="" +fi +if [ -z "$red" ]; then + red="$(printf '%s' "$out" | sed -E 's/(sk-ant-[A-Za-z0-9_-]{16,}|sk-[A-Za-z0-9]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|AKIA[0-9A-Z]{16}|xox[baprs]-[A-Za-z0-9-]{10,}|AIza[0-9A-Za-z_-]{20,}|ya29\.[A-Za-z0-9._-]+|eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})/[REDACTED]/g')" +fi if [ "$red" != "$out" ]; then jq -n --arg o "$red" '{hookSpecificOutput:{hookEventName:"PostToolUse",updatedToolOutput:$o}}' diff --git a/src/adjudicate.js b/src/adjudicate.js index 53e042e..a835624 100644 --- a/src/adjudicate.js +++ b/src/adjudicate.js @@ -14,7 +14,7 @@ import { execFileSync, spawnSync } from "node:child_process"; import { buildHttpRunner as httpRunner } from "./llm.js"; import { MODELS } from "./model_tiers.js"; import { envModelOverride } from "./providers.js"; -import { SECRET_RE } from "./recall.js"; +import { hasSecret } from "./recall.js"; /** * Is the LLM proposer layer active for this call? Explicit opt-in wins; env is the default. @@ -77,9 +77,9 @@ export function extractJson(text) { */ export function adjudicate({ prompt, parse, run = buildRunner() }) { try { - if (SECRET_RE.test(String(prompt))) return null; // never send a secret to the model + if (hasSecret(prompt)) return null; // never send a secret to the model const raw = run(prompt); - if (SECRET_RE.test(String(raw))) return null; // never trust a reply that leaked one back + if (hasSecret(raw)) return null; // never trust a reply that leaked one back const obj = extractJson(raw); if (obj == null) return null; const parsed = parse(obj); diff --git a/src/cortex_distill.js b/src/cortex_distill.js index e16e809..8c1225a 100644 --- a/src/cortex_distill.js +++ b/src/cortex_distill.js @@ -4,7 +4,7 @@ // shells out to the `claude` CLI via the shared adjudicate runner (same primitive every other // faculty uses); the runner is injectable so the pure prompt/parse logic is testable without it. import { buildRunner } from "./adjudicate.js"; -import { SECRET_RE } from "./recall.js"; +import { hasSecret } from "./recall.js"; /** * Pure: build the distillation prompt from an episode. @@ -39,7 +39,7 @@ export function parseDistilled(text) { .trim() .slice(0, 200); if (!whatWentWrong || !correctedBehavior) return null; - if (SECRET_RE.test(`${whatWentWrong} ${correctedBehavior}`)) return null; // never persist a secret + if (hasSecret(`${whatWentWrong} ${correctedBehavior}`)) return null; // never persist a secret return { whatWentWrong, correctedBehavior }; } diff --git a/src/diagnose.js b/src/diagnose.js index b774a45abbd8ef702886dedab070275963124a75..3dd736c0e77889657cc52e5f8192dfc27302cfe7 100644 GIT binary patch delta 47 wcmca_e#?AA4zox`VsUV4a#3oDjzVr`UWs!~VrK5<2IiH#959*9W`ZlZ0J@nGwg3PC delta 52 zcmca*e&2jU4zoyZW?qSNPGV-RjzX}jbC7FDe30wr2IiH#yl}2wNow(CL&23?0Q0sH A*Z=?k diff --git a/src/ledger.js b/src/ledger.js index 3e587c0..da92e95 100644 --- a/src/ledger.js +++ b/src/ledger.js @@ -14,15 +14,15 @@ // logs. That is what makes every file either byte-identical across teammates or // union-mergeable — the join-semilattice property is structural, not aspirational. // - Unreviewed claims decay toward the PRIOR (0.5, uncertainty), not toward false. +import { hasSecret } from "./secrets.js"; import { contentHash } from "./util.js"; -// Anything matching this is refused at mint — store a pointer to where the secret -// lives, never the value. Moved here from recall.js (which re-exports it) so NO claim -// kind can persist a credential. See recall.js history for the precision rationale: -// known credential formats, plus a secret-ish key ASSIGNED to a value (a bare English -// mention like "implement password hashing" must NOT be refused). -export const SECRET_RE = - /(-----BEGIN |\bghp_[A-Za-z0-9]{16,}|\bgithub_pat_[A-Za-z0-9_]{20,}|\bsk-[A-Za-z0-9_-]{16,}|\bxox[baprs]-[A-Za-z0-9-]{10,}|\bAIza[0-9A-Za-z_-]{20,}|\bya29\.[A-Za-z0-9._-]+|\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}|AKIA[0-9A-Z]{16}|\b[\w-]*(?:api[_-]?key|secret|passwd|password|token)[\w-]*["']?\s*[:=]\s*["']?\S)/i; +// Anything secret-shaped is refused at mint — store a pointer to where the secret +// lives, never the value. Detection lives in secrets.js (format grammars + entropy +// gate) so NO claim kind — and no shell guard — can disagree about what a secret is. +// SECRET_RE stays re-exported here because recall.js/lessons_store.js/tests +// historically import it from this module. +export { hasSecret, SECRET_RE } from "./secrets.js"; export const KINDS = [ "lesson", // a corrected behavior (cortex) @@ -121,7 +121,7 @@ export function mintClaim({ kind, body, scope = {}, provenance = {}, t = 0 }) { const nBody = JSON.parse(JSON.stringify(body)); const nScope = JSON.parse(JSON.stringify(scope)); const canon = canonicalize({ body: nBody, kind, scope: nScope }); - if (SECRET_RE.test(canon)) + if (hasSecret(canon)) return { ok: false, reason: "refused: looks like a secret/credential — store a pointer, not the value", diff --git a/src/ledger_store.js b/src/ledger_store.js index 2789c26..96160b3 100644 --- a/src/ledger_store.js +++ b/src/ledger_store.js @@ -20,11 +20,11 @@ import { claimId, DORMANT_VAL, emptyState, + hasSecret, liveClaims, mergeStates, mintClaim, ORACLES, - SECRET_RE, sealRecord, sortRecords, val, @@ -116,7 +116,7 @@ export function putClaim(dir, claim) { if (!claim?.id || claim.id !== claimId(claim.kind, claim.body, claim.scope)) return { ok: false, reason: "claim id does not match canonical content hash" }; const text = claimBytes(claim); - if (SECRET_RE.test(text)) + if (hasSecret(text)) return { ok: false, reason: "refused: claim looks like it contains a secret/credential" }; const path = claimPath(dir, claim.id); const already = existsSync(path); @@ -295,7 +295,7 @@ export function verify(dir) { claims++; ids.push(id); } - if (SECRET_RE.test(raw)) issues.push(`claim ${id}: contains secret-like content`); + if (hasSecret(raw)) issues.push(`claim ${id}: contains secret-like content`); } for (const log of LOGS) { const logRoot = join(dir, log); @@ -322,7 +322,7 @@ export function verify(dir) { issues.push(`${where}: recorded weight ${o.w} != oracle table ${ORACLES[o.oracle].w}`); else outcomes++; } - if (SECRET_RE.test(line)) issues.push(`${where}: secret-like content`); + if (hasSecret(line)) issues.push(`${where}: secret-like content`); } } } diff --git a/src/lessons_store.js b/src/lessons_store.js index e363166..cb9b727 100644 --- a/src/lessons_store.js +++ b/src/lessons_store.js @@ -11,7 +11,7 @@ import { writeFileSync, } from "node:fs"; import { join } from "node:path"; -import { SECRET_RE } from "./recall.js"; +import { hasSecret } from "./recall.js"; export const lessonsDir = (root = process.cwd()) => join(root, ".forge", "lessons"); @@ -84,7 +84,7 @@ export function parse(text) { /** Persist one lesson. Refuses secret-like content (store a pointer, never the value). */ export function save(root, lesson) { const text = serialize(lesson); - if (SECRET_RE.test(text)) { + if (hasSecret(text)) { return { ok: false, reason: "refused: lesson looks like it contains a secret/credential", diff --git a/src/recall.js b/src/recall.js index 816a11c..d21a2ee 100644 --- a/src/recall.js +++ b/src/recall.js @@ -4,18 +4,17 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; - -// Secret-refusal now lives in the PCM ledger core (ledger.js) so NO claim kind can -// persist a credential; re-exported here because recall is where callers historically -// imported it from (lessons_store, guards, tests). See ledger.js for the precision -// rationale (credential formats + key-assigned-to-value, never a bare English mention). -import { SECRET_RE } from "./ledger.js"; // The merged read helper (P2 read flip). Import cycle note: ledger_read → lessons_store // → recall is function-level only (no module-eval use on either side), so ESM resolves // it safely — same pattern as lessons_store's own SECRET_RE import from here. import { mergeFactSlugs } from "./ledger_read.js"; +// Secret-refusal now lives in secrets.js (format grammars + entropy gate) so no +// store — and no shell guard — can disagree; re-exported here because recall is where +// callers historically imported it from (lessons_store, guards, tests). See secrets.js +// for the precision rationale (never refuse a bare English mention like "password"). +import { hasSecret, SECRET_RE } from "./secrets.js"; -export { SECRET_RE }; +export { hasSecret, SECRET_RE }; export function defaultStore() { return join(process.env.FORGE_HOME || join(homedir(), ".forge"), "recall"); @@ -26,7 +25,7 @@ const factsDir = (store) => join(store, "facts"); import { slug as slugify } from "./util.js"; export function add(store, name, body) { - if (SECRET_RE.test(`${name}\n${body}`)) { + if (hasSecret(`${name}\n${body}`)) { return { ok: false, reason: "refused: looks like a secret/credential — store a pointer, not the value", diff --git a/src/secrets.js b/src/secrets.js new file mode 100644 index 0000000..dd154a0 --- /dev/null +++ b/src/secrets.js @@ -0,0 +1,98 @@ +// forge secrets — the ONE source of truth for secret detection and redaction. +// Everything that refuses or masks credentials (ledger mint, recall/lesson persist, +// adjudicate prompt/reply gate, diagnose traces, the secret-redact.sh guard) resolves +// here, so JS and shell can never disagree about what a secret is. +// +// Two complementary detectors: +// (i) FORMAT grammars — regexes over *documented* credential shapes (GitHub PAT, +// Anthropic/OpenAI sk-, Slack xox, Google AIza/ya29, JWT, AWS AKIA, PEM). These +// are parsers of known token grammars, kept as regex deliberately. +// (ii) ENTROPY scoring (src/math.js) — a graded gate for tokens no format list has +// an entry for. A ≥20-char mixed-case-plus-digit token whose Shannon entropy +// reaches random-credential territory is treated as a secret even when its +// vendor prefix is unknown. Hex-only strings (git SHAs, digests) are exempt by +// construction: they lack the mixed-case signal and are indistinguishable from +// content hashes anyway — precision first (see the recall.js history: a bare +// English mention like "implement password hashing" must NOT be refused). + +import { shannonEntropy } from "./math.js"; + +// (i) Known credential grammars. `-----BEGIN ` is the PEM header; the final branch +// is a secret-ish key ASSIGNED to a value (never a bare English mention). +const FORMATS = [ + "-----BEGIN ", + "\\bghp_[A-Za-z0-9]{16,}", + "\\bgithub_pat_[A-Za-z0-9_]{20,}", + "\\bsk-[A-Za-z0-9_-]{16,}", + "\\bxox[baprs]-[A-Za-z0-9-]{10,}", + "\\bAIza[0-9A-Za-z_-]{20,}", + "\\bya29\\.[A-Za-z0-9._-]+", + "\\beyJ[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]{10,}", + "AKIA[0-9A-Z]{16}", +]; +const KEYISH = "(?:api[_-]?key|secret|passwd|password|token)"; +const ASSIGNED = `\\b[\\w-]*${KEYISH}[\\w-]*["']?\\s*[:=]\\s*["']?\\S`; + +/** The historical detection regex (formats + key-assigned-to-value), unchanged + * semantics — kept exported because tests and downstream code match against it. + * New code should call hasSecret(), which adds the entropy gate. */ +export const SECRET_RE = new RegExp(`(${[...FORMATS, ASSIGNED].join("|")})`, "i"); + +// (ii) Entropy gate thresholds. 20 chars is below every real credential length but +// above almost all identifiers; 3.9 bits/char sits between English-like tokens +// (camelCase identifiers ≈ 3.5–3.8 even with digits) and sampled random base64/62 +// tokens (≥ 4.2 at 20+ chars). Both are exported so tests pin the calibration. +export const ENTROPY_MIN_LEN = 20; +export const ENTROPY_MIN_BITS = 3.9; + +// Candidate extraction: contiguous base64-class runs. Parsing, not a decision. +const TOKEN_RE = /[A-Za-z0-9+/=_-]{20,}/g; + +/** + * Is this bare token secret-shaped by math alone? Requires all three of: length, + * mixed charset (lower AND upper AND digit — excludes hex/UUID/camelCase words + * without digits), and near-random Shannon entropy. + * @param {string} tok + */ +export function isHighEntropyToken(tok) { + const s = String(tok); + if (s.length < ENTROPY_MIN_LEN) return false; + if (!(/[a-z]/.test(s) && /[A-Z]/.test(s) && /[0-9]/.test(s))) return false; + return shannonEntropy(s) >= ENTROPY_MIN_BITS; +} + +/** + * Does this text contain a secret? Format grammar OR entropy-detected token. + * This is the detection entry point every refusal site should use. + * @param {string} text + */ +export function hasSecret(text) { + const s = String(text); + if (SECRET_RE.test(s)) return true; + const toks = s.match(TOKEN_RE); + return toks ? toks.some(isHighEntropyToken) : false; +} + +// Redaction machinery — used by the secret-redact guard (via node import) and any +// JS caller that wants to keep surrounding text. PEM blocks are masked whole; +// assigned values keep their key (context stays readable, value is gone). +const PEM_BLOCK_G = /-----BEGIN [A-Z ]*-----[\s\S]*?(?:-----END [A-Z ]*-----|$)/g; +const FORMAT_G = new RegExp(FORMATS.slice(1).join("|"), "gi"); +const ASSIGNED_G = new RegExp( + `(\\b[\\w-]*${KEYISH}[\\w-]*["']?\\s*[:=]\\s*["']?)([^\\s"']+)`, + "gi", +); + +/** + * Replace every detected secret with [REDACTED], preserving surrounding text. + * Same detectors as hasSecret — one truth, two verbs. + * @param {string} text + */ +export function redactSecrets(text) { + let s = String(text); + s = s.replace(PEM_BLOCK_G, "[REDACTED]"); + s = s.replace(FORMAT_G, "[REDACTED]"); + s = s.replace(ASSIGNED_G, "$1[REDACTED]"); + s = s.replace(TOKEN_RE, (t) => (isHighEntropyToken(t) ? "[REDACTED]" : t)); + return s; +} diff --git a/test/secretredact.test.js b/test/secretredact.test.js index 7d410eb..770afa3 100644 --- a/test/secretredact.test.js +++ b/test/secretredact.test.js @@ -36,3 +36,22 @@ test("secret-redact stays silent when nothing matches", () => { assert.equal(r.code, 0); assert.equal(r.out.trim(), ""); }); + +// The guard imports src/secrets.js — the shell path and the JS refusal sites share +// ONE detector, so they can never disagree. These two tests pin that parity. +test("secret-redact matches redactSecrets byte-for-byte (shared implementation)", async () => { + const { redactSecrets } = await import("../src/secrets.js"); + const input = `key=${fakeAnthropic("AAAAbbbbCCCCddddEEEEffff")} then DB_PASSWORD=hunter2-value done`; + const r = run({ tool_response: input }); + assert.equal(r.code, 0); + const emitted = JSON.parse(r.out).hookSpecificOutput.updatedToolOutput; + assert.equal(emitted, redactSecrets(input)); +}); + +test("secret-redact catches an unknown-vendor high-entropy token (beyond the old sed list)", () => { + const tok = ["Zq7Rt2", "Xk9Lp4", "Vm1Nc8", "Yb5Ws3", "Hd6Fg0"].join(""); + const r = run({ tool_response: `issued credential ${tok} ok` }); + assert.equal(r.code, 0); + assert.match(r.out, /REDACTED/); + assert.doesNotMatch(r.out, new RegExp(tok)); +}); diff --git a/test/secrets.test.js b/test/secrets.test.js new file mode 100644 index 0000000..4ab4bec --- /dev/null +++ b/test/secrets.test.js @@ -0,0 +1,103 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { shannonEntropy } from "../src/math.js"; +import { + ENTROPY_MIN_BITS, + hasSecret, + isHighEntropyToken, + redactSecrets, + SECRET_RE, +} from "../src/secrets.js"; +import { fakeAnthropic, fakeGithubPat, fakeGoogle, fakeJwt, fakeSlack } from "./_fixtures.js"; + +// A random-looking mixed-case token with NO known vendor prefix — the exact shape +// the format list can never enumerate. Assembled at runtime like the other fixtures. +const fakeUnknownVendor = () => ["Zq7Rt2", "Xk9Lp4", "Vm1Nc8", "Yb5Ws3", "Hd6Fg0"].join(""); + +test("hasSecret: every known credential format is caught (SECRET_RE parity)", () => { + for (const fixture of [fakeAnthropic(), fakeGithubPat(), fakeSlack(), fakeGoogle(), fakeJwt()]) { + assert.ok(SECRET_RE.test(fixture), `SECRET_RE should match ${fixture.slice(0, 8)}…`); + assert.ok(hasSecret(fixture), `hasSecret should catch ${fixture.slice(0, 8)}…`); + assert.ok(hasSecret(`prefix text ${fixture} suffix`), "…also when embedded in prose"); + } +}); + +test("hasSecret: assigned secret-ish key is caught, bare English mention is not", () => { + assert.ok(hasSecret('api_key = "hunter2-value"')); + assert.ok(hasSecret("password: swordfish")); + // The precision invariant from recall.js history — auth-related PROSE must pass. + assert.equal(hasSecret("implement password hashing in auth.js"), false); + assert.equal(hasSecret("rotate the api key helper and the secret loader"), false); +}); + +test("hasSecret: entropy gate catches an unknown-vendor random token (the regex FN)", () => { + const tok = fakeUnknownVendor(); + assert.ok(shannonEntropy(tok) >= ENTROPY_MIN_BITS, "fixture must sit above the entropy bar"); + assert.equal(SECRET_RE.test(tok), false, "the format list has no entry for this shape"); + assert.ok(hasSecret(tok), "entropy detection must close the format-list gap"); + assert.ok(hasSecret(`deploy log: token ${tok} accepted`)); +}); + +test("isHighEntropyToken: hex digests, UUIDs, identifiers, and prose are NOT secrets", () => { + // git SHA / digest: no uppercase → exempt by construction. + assert.equal(isHighEntropyToken("9f86d081884c7d659a2feaa0c55ad015a3bf4f1b"), false); + // UUID: no uppercase. + assert.equal(isHighEntropyToken("550e8400-e29b-41d4-a716-446655440000"), false); + // camelCase identifier: no digit / low entropy. + assert.equal(isHighEntropyToken("getUserAuthenticationTokenFromEnvironment"), false); + assert.equal(isHighEntropyToken("parseHttpResponseHeaders2"), false); + // Too short even if random. + assert.equal(isHighEntropyToken("Zq7Rt2Xk9Lp4"), false); +}); + +test("redactSecrets: masks formats, keeps surrounding text", () => { + const key = fakeAnthropic("AAAAbbbbCCCCddddEEEEffff"); + const out = redactSecrets(`the key is ${key} and more`); + assert.equal(out.includes("AAAAbbbbCCCCddddEEEE"), false); + assert.match(out, /the key is \[REDACTED\] and more/); +}); + +test("redactSecrets: masks an assigned value but keeps the key name readable", () => { + const out = redactSecrets("DB_PASSWORD=super-secret-value ok"); + assert.match(out, /DB_PASSWORD=\[REDACTED\]/); + assert.equal(out.includes("super-secret-value"), false); + assert.match(out, /ok$/); +}); + +test("redactSecrets: masks a whole PEM block", () => { + const pem = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA\n-----END RSA PRIVATE KEY-----"; + const out = redactSecrets(`before\n${pem}\nafter`); + assert.equal(out.includes("MIIEowIBAAKCAQEA"), false); + assert.match(out, /before\n\[REDACTED\]\nafter/); +}); + +test("redactSecrets: masks high-entropy unknown-vendor tokens, leaves prose alone", () => { + const tok = fakeUnknownVendor(); + const prose = "the quick brown fox jumps over the lazy dog"; + assert.equal(redactSecrets(prose), prose); + const out = redactSecrets(`credential ${tok} issued`); + assert.equal(out.includes(tok), false); + assert.match(out, /credential \[REDACTED\] issued/); +}); + +test("redactSecrets: leaves git SHAs and UUIDs untouched (they are not secrets)", () => { + const line = + "commit 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b id 550e8400-e29b-41d4-a716-446655440000"; + assert.equal(redactSecrets(line), line); +}); + +test("hasSecret and redactSecrets agree: anything redacted is detected", () => { + const samples = [ + `key ${fakeAnthropic()}`, + "token = abc123", + `bare ${fakeUnknownVendor()}`, + "plain prose with nothing sensitive", + ]; + for (const s of samples) { + assert.equal( + redactSecrets(s) !== s, + hasSecret(s), + `detect/redact must agree on: ${s.slice(0, 30)}`, + ); + } +}); From d95b5c54f713af0cf79be88658bc36e6493ddcb0 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:24 +0000 Subject: [PATCH 3/9] feat: route by exemplar k-NN similarity instead of topic keyword regexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The text-complexity rubric is now similarity-weighted k-NN regression over a labeled exemplar bank (50 example tasks with target complexities): overlap- coefficient similarity on stopword-filtered, plural-folded unigram+bigram sets, credibility-shrunk toward a prior, plus a bounded structural term (length, code fences, constraint/step counts). Replaces the four hand-tuned ALGO/ARCH/ MODERATE/TRIVIAL keyword regexes and their additive magic weights. Why: a keyword list needs the literal token — "two threads deadlock when the queue is full" scored zero on the old rubric because neither "race condition" nor "mutex" appears. k-NN scores it by resemblance to labeled neighbors, and every score stays attributable (the neighbors are returned with the estimate). The exemplar bank is data: add rows to improve coverage, no weights move. strongSignal (the LLM lower-bound floor) now means "confidently matched a hard exemplar" rather than "contains an algorithmic keyword". All existing routing behavior contracts hold (same tier cutoffs, same band rails, same free-raise/ bounded-lower reconcile). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- src/math.js | 17 ++++ src/route.js | 236 +++++++++++++++++++++++++++++++++------------ test/route.test.js | 51 ++++++++++ 3 files changed, 243 insertions(+), 61 deletions(-) diff --git a/src/math.js b/src/math.js index 36d87f9..cfb253c 100644 --- a/src/math.js +++ b/src/math.js @@ -62,3 +62,20 @@ export function setJaccard(a, b) { for (const x of small) if (large.has(x)) inter++; return inter / (a.size + b.size - inter); } + +/** + * Overlap coefficient of two Sets: |A∩B| / min(|A|,|B|). Either empty → 0. + * Containment-friendly where Jaccard is size-penalized: a short exemplar fully + * contained in a long task scores 1.0 — the right semantics for "does this text + * exhibit that exemplar's concept" (task↔exemplar matching in route.js). + * @param {Set} a + * @param {Set} b + * @returns {number} 0..1 + */ +export function setOverlap(a, b) { + if (!a.size || !b.size) return 0; + let inter = 0; + const [small, large] = a.size <= b.size ? [a, b] : [b, a]; + for (const x of small) if (large.has(x)) inter++; + return inter / small.size; +} diff --git a/src/route.js b/src/route.js index b2d0db1..cedb96a 100644 --- a/src/route.js +++ b/src/route.js @@ -9,81 +9,191 @@ import { matchingLessons } from "./cortex.js"; import { gitChurn, grepFanout } from "./cortex_features.js"; import { recordRoute } from "./cost_report.js"; import { mergedLessons } from "./ledger_read.js"; +import { setOverlap } from "./math.js"; import { MODELS } from "./model_tiers.js"; import { preflightRepo, referencedEntities } from "./preflight.js"; import { activeProvider, envModelOverride } from "./providers.js"; import { clamp01, contentHash, epochDay } from "./util.js"; -// Weights sum to 1. Each raw signal is normalized by the point where it reads as "complex". +// --------------------------------------------------------------------------- +// Text-complexity rubric: similarity-weighted k-NN regression over a labeled +// exemplar bank. The bank is DATA (example tasks with target complexities — +// tunable, diffable, growable); the decision is MATH (overlap-coefficient +// similarity → confidence-shrunk k-NN estimate). This replaced four hand-tuned +// topic keyword regexes: an unseen phrasing ("stop the two workers clobbering +// each other's writes") scores by resemblance to labeled neighbors, where a +// keyword list needed the literal token ("race condition") to appear. +// --------------------------------------------------------------------------- -const ALGO_TERMS = - /\b(recursion|recursive|recursive-?descent|dynamic programming|dijkstra|a\*|concurren|thread-?safe|mutex|race condition|deadlock|distributed|consensus|parser|compiler|cryptograph|np-hard|state machine|invariant|numerical stability|back-?pressure|token[- ]bucket|rate limiter|idempoten|migration|producer|consumer|blocking queue|condition[- ]variable)\b/i; -const ARCH_TERMS = - /\b(architect|\bdesign\b|trade-?off|refactor a|migrate|scal(e|able|ing)|schema (migration|design)|api design|multi-?module|cross-?module|end-?to-?end|consistency (guarantee|trade)|locking strategy|module boundaries)\b/i; -const MODERATE_TERMS = - /\b(class\b|cache|lru|queue|stack|heap|linked list|binary tree|tree|traversal|graph|decorator|regex|debounce|throttle|merge|sort(ed|ing)?|parse|o\(\s*\d|o\(n|o\(1|thread|async|lock|validate|in-?order|adjacency)\b/i; -const TRIVIAL_TERMS = - /\b(hello world|rename|typo|indent|add a comment|reverse a string|reverse the string|is[_ ]?even|is[_ ]?odd|factorial|fibonacci|is[_ ]?prime|prime\b|sum of|sum_list|capitalize|count vowels|celsius|fahrenheit|lower ?case|upper ?case)\b/i; -const MULTISTEP = - /\b(and then|after that|first.*then|step \d|multiple|several|each of|for every)\b/i; +/** + * Labeled exemplars. `y` = target complexity in [0,1], calibrated to the tier + * cutoffs in recommend(): ~0.08 trivial, ~0.42 data-structure/library level, + * ~0.78 algorithmic/systems, ~0.85 architectural. Add rows freely — coverage + * improves routing without touching any weight. + */ +export const EXEMPLARS = [ + // trivial + { text: "fix a typo", y: 0.08 }, + { text: "rename a variable", y: 0.08 }, + { text: "add a comment", y: 0.08 }, + { text: "fix indentation and whitespace", y: 0.08 }, + { text: "reverse a string", y: 0.08 }, + { text: "check if a number is prime", y: 0.08 }, + { text: "check if a number is even or odd", y: 0.08 }, + { text: "compute the factorial of a number", y: 0.08 }, + { text: "print the fibonacci sequence", y: 0.08 }, + { text: "sum a list of numbers", y: 0.08 }, + { text: "count the vowels in a string", y: 0.08 }, + { text: "capitalize or lowercase a word", y: 0.08 }, + { text: "convert celsius to fahrenheit", y: 0.08 }, + { text: "write a hello world program", y: 0.08 }, + // moderate — data structure / class / library-level work + { text: "implement an lru cache class with get and put", y: 0.42 }, + { text: "add a small in-memory cache with get and set", y: 0.42 }, + { text: "write a debounce or throttle helper", y: 0.42 }, + { text: "parse a csv or json file into objects", y: 0.42 }, + { text: "in-order traversal of a binary tree", y: 0.42 }, + { text: "sort records by multiple keys", y: 0.42 }, + { text: "merge two sorted lists", y: 0.42 }, + { text: "add validation to user input", y: 0.42 }, + { text: "implement a linked list stack or queue class", y: 0.42 }, + { text: "write a regex to extract fields from a line", y: 0.42 }, + { text: "add an async retry wrapper around a request", y: 0.42 }, + { text: "build an adjacency list graph and walk it", y: 0.42 }, + { text: "write a decorator that memoizes a function", y: 0.42 }, + { text: "refactor a function to remove duplication", y: 0.42 }, + // algorithmic / systems + { text: "implement dijkstra shortest path algorithm", y: 0.78 }, + { text: "solve with dynamic programming and memoization", y: 0.78 }, + { text: "fix a race condition with mutex locking", y: 0.78 }, + { text: "thread-safe concurrent queue with condition variable signaling", y: 0.78 }, + { text: "implement a rate limiter with a token bucket", y: 0.78 }, + { text: "write a recursive descent parser for a grammar", y: 0.78 }, + { text: "resolve a deadlock between concurrent threads", y: 0.78 }, + { text: "distributed consensus and replication protocol", y: 0.78 }, + { text: "cryptographic signing and verification flow", y: 0.78 }, + { text: "producer consumer blocking queue with backpressure", y: 0.78 }, + { text: "state machine with invariants and transitions", y: 0.78 }, + { text: "numerical stability of a floating point computation", y: 0.78 }, + { text: "np-hard optimization with a heuristic search", y: 0.78 }, + { text: "compiler pass over an abstract syntax tree", y: 0.78 }, + { text: "idempotent retry with exactly-once delivery semantics", y: 0.78 }, + // architectural / cross-module + { text: "design the architecture of a new service", y: 0.85 }, + { text: "refactor module boundaries across the codebase", y: 0.85 }, + { text: "design a schema migration for the database", y: 0.85 }, + { text: "api design with consistency guarantees and trade-offs", y: 0.85 }, + { text: "migrate a multi-module system end to end", y: 0.85 }, + { text: "design a locking strategy across services", y: 0.85 }, + { text: "plan scalability for a growing distributed system", y: 0.85 }, + { text: "cross-module refactor of shared interfaces", y: 0.85 }, +]; + +// Excluded from the lexical footprint: function words AND generic task verbs +// (write/fix/add/…) — both appear in every request regardless of topic, so any +// overlap through them is spurious ("fix the deadlock" must match the deadlock +// exemplar, not "fix a typo"). +const STOP = new Set( + ( + "a an the in on of to for with and or is are be it its this that as at by from into up out " + + "then after first should must please when if between two new small " + + "write implement add make fix resolve create build use check" + ).split(" "), +); +// Naive plural fold: both sides get the same transform, so "threads"↔"thread" +// overlap without a stemmer dependency (mangled stems like "clas" are harmless — +// they only ever compare against identically-mangled stems). +const stem = (t) => (t.length > 3 && t.endsWith("s") ? t.slice(0, -1) : t); + +/** Stopword-filtered, plural-folded unigram+bigram set — the lexical footprint + * similarity runs on. */ +export function contentGrams(text) { + const toks = String(text) + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((t) => t && !STOP.has(t)) + .map(stem); + const grams = new Set(toks); + for (let i = 0; i + 1 < toks.length; i++) grams.add(`${toks[i]} ${toks[i + 1]}`); + return grams; +} + +/** Every rubric constant in one inspectable table (same transparency rule as WEIGHTS). */ +export const RUBRIC = { + k: 3, // neighbors in the k-NN estimate + prior: 0.15, // no-signal complexity (the old "base cost of any task") + confSat: 0.5, // top similarity at which the estimate earns full weight + strongScore: 0.65, // k-NN estimate marking a confidently-hard task (LLM lower-bound floor) + strongConf: 0.5, + bands: { cheap: 0.3, mid: 0.6 }, // score < cheap → cheap; ≤ mid → mid; else premium + struct: { codeContext: 0.05, length: 0.1, constraints: 0.05, steps: 0.05 }, +}; + +/** Structural (non-topic) features of the task text — countable, graded inputs. */ export function rubricSignals(task = "") { const text = String(task); return { lengthTokens: Math.max(1, Math.floor(text.length / 4)), - hasAlgorithmicTerms: ALGO_TERMS.test(text), - hasArchitecturalTerms: ARCH_TERMS.test(text), - hasModerateTerms: MODERATE_TERMS.test(text), - hasTrivialMarkers: TRIVIAL_TERMS.test(text), - hasMultistep: MULTISTEP.test(text), hasCodeContext: /```/.test(text), + // Explicit requirement markers: bullet/numbered lines and modal verbs. A count + // feeding a saturating weight — feature extraction, not a classification. nConstraints: (text.match(/(^\s*[-*\d.]|\b(must|should|ensure|require|constraint)\b)/gim) || []) .length, + nSteps: (text.match(/^\s*\d+[.)]\s/gm) || []).length, }; } +/** + * Text rubric: k-NN over EXEMPLARS with credibility shrinkage toward the prior, + * plus a bounded structural term. Deterministic, and every score is attributable: + * the neighbors that produced it are returned, not just the number. + */ export function rubricComplexity(task = "") { const sig = rubricSignals(task); - const reasons = [{ weight: 1.5, reason: "base cost of any task" }]; - let score = 1.5; - if (sig.hasAlgorithmicTerms) { - score += 4; - reasons.push({ weight: 4, reason: "algorithmic/systems difficulty" }); - } - if (sig.hasArchitecturalTerms) { - score += 4; - reasons.push({ weight: 4, reason: "architectural/design scope" }); - } - if (sig.hasModerateTerms) { - score += 2; - reasons.push({ weight: 2, reason: "data-structure/class/library-level work" }); - } - if (sig.hasMultistep) { - score += 1; - reasons.push({ weight: 1, reason: "multi-step request" }); - } - if (sig.hasCodeContext) { - score += 1; - reasons.push({ weight: 1, reason: "carries code context" }); - } - if (sig.lengthTokens > 120) { - score += 1.5; - reasons.push({ weight: 1.5, reason: `long spec (~${sig.lengthTokens} tok)` }); - } else if (sig.lengthTokens > 55) { - score += 0.7; - reasons.push({ weight: 0.7, reason: `medium spec (~${sig.lengthTokens} tok)` }); - } - if (sig.nConstraints >= 5) { - score += 1; - reasons.push({ weight: 1, reason: `${sig.nConstraints} explicit constraints` }); - } - if (sig.hasTrivialMarkers && !sig.hasAlgorithmicTerms && !sig.hasArchitecturalTerms) { - score -= 3; - reasons.push({ weight: -3, reason: "trivial-task marker" }); - } - const rawScore = Math.max(0, score); - const band = rawScore < 3 ? "cheap" : rawScore <= 6 ? "mid" : "premium"; - return { rawScore, band, signals: sig, reasons }; + const grams = contentGrams(task); + const neighbors = EXEMPLARS.map((e) => ({ ...e, sim: setOverlap(grams, contentGrams(e.text)) })) + .sort((a, b) => b.sim - a.sim) + .slice(0, RUBRIC.k) + .filter((n) => n.sim > 0); + const simSum = neighbors.reduce((s, n) => s + n.sim, 0); + const knn = simSum ? neighbors.reduce((s, n) => s + n.sim * n.y, 0) / simSum : RUBRIC.prior; + const confidence = neighbors.length ? neighbors[0].sim : 0; + // Credibility shrinkage: a weak best-match barely moves the estimate off the prior. + const topic = RUBRIC.prior + (knn - RUBRIC.prior) * clamp01(confidence / RUBRIC.confSat); + const s = RUBRIC.struct; + const struct = + s.codeContext * (sig.hasCodeContext ? 1 : 0) + + s.length * clamp01(sig.lengthTokens / 150) + + s.constraints * clamp01(sig.nConstraints / 5) + + s.steps * clamp01(sig.nSteps / 3); + // Structure adds complexity on top of topic, saturating — it can never flip a + // trivial topic into premium on its own (struct is bounded by Σ weights = 0.25). + const score = clamp01(topic + struct * (1 - topic)); + const band = score < RUBRIC.bands.cheap ? "cheap" : score <= RUBRIC.bands.mid ? "mid" : "premium"; + const strongTopicSignal = knn >= RUBRIC.strongScore && confidence >= RUBRIC.strongConf; + const reasons = [ + ...neighbors + .filter((n) => n.sim >= 0.2) + .map((n) => ({ + weight: n.sim * n.y, + reason: `similar to "${n.text}" (sim ${n.sim.toFixed(2)}, complexity ${n.y})`, + })), + ...(sig.hasCodeContext ? [{ weight: s.codeContext, reason: "carries code context" }] : []), + ...(sig.lengthTokens > 55 + ? [ + { + weight: s.length * clamp01(sig.lengthTokens / 150), + reason: `long spec (~${sig.lengthTokens} tok)`, + }, + ] + : []), + ...(sig.nConstraints >= 5 + ? [{ weight: s.constraints, reason: `${sig.nConstraints} explicit constraints` }] + : []), + ...(sig.nSteps >= 2 ? [{ weight: s.steps, reason: `${sig.nSteps} numbered steps` }] : []), + ]; + return { score, band, confidence, knn, neighbors, signals: sig, reasons, strongTopicSignal }; } const WEIGHTS = { @@ -205,17 +315,21 @@ export function routeTask( }; const { score: repoScore, norm } = complexity(signals); const rubric = rubricComplexity(task); - const rubricScore = Math.min(1, rubric.rawScore / 10); - const detScore = Math.max(repoScore, rubricScore); + // Upper envelope, not an average: text and repo signals measure DIFFERENT facets + // of complexity, and under-provisioning is the expensive failure (an escalation + // retry costs more than a one-tier overshoot). Whichever facet detects difficulty + // sets the tier — same philosophy as the LLM proposer's "free raise" below. + const detScore = Math.max(repoScore, rubric.score); // M1 proposer (opt-in): the model PROPOSES a complexity band. A RAISE is free (spotting hidden // complexity costs at most a bigger model). A LOWER is bounded — never more than one `band` - // below the rubric, and never below `signalFloor` when the rubric detected algorithmic or - // architectural terms, so a "distributed rate-limiter" can't be talked down to the cheap tier. + // below the rubric, and never below `signalFloor` when the rubric confidently matched an + // algorithmic/architectural exemplar, so a "distributed rate-limiter" can't be talked down + // to the cheap tier. // With `bidirectional:false` it stays raise-only. Fail-safe: a null proposal is ignored. const proposal = llmEnabled({ llm }) ? complexityLLM(task, { run: run || buildRunner({ model, timeoutMs }) }) : null; - const strongSignal = rubric.signals.hasAlgorithmicTerms || rubric.signals.hasArchitecturalTerms; + const strongSignal = rubric.strongTopicSignal; let score = detScore; let path = proposal ? "llm-agreed" : "deterministic"; if (proposal) { diff --git a/test/route.test.js b/test/route.test.js index 4b1842a..bbd7cde 100644 --- a/test/route.test.js +++ b/test/route.test.js @@ -7,12 +7,63 @@ import { read as readMetrics } from "../src/metrics.js"; import { complexity, complexityLLM, + contentGrams, + EXEMPLARS, emitGatewayConfig, meterRoute, + RUBRIC, recommend, routeTask, + rubricComplexity, } from "../src/route.js"; +test("contentGrams: stopwords dropped, unigrams+bigrams kept", () => { + const g = contentGrams("Implement a rate limiter with the token bucket"); + assert.ok(g.has("rate") && g.has("limiter") && g.has("token") && g.has("bucket")); + assert.ok(g.has("rate limiter") && g.has("token bucket"), "bigrams bridge stopwords"); + assert.ok(!g.has("a") && !g.has("the") && !g.has("with"), "stopwords carry no topic"); +}); + +test("rubric: no lexical match at all falls back to the prior (abstains)", () => { + const r = rubricComplexity("zzz qqq xxx"); + assert.equal(r.confidence, 0); + assert.ok(Math.abs(r.score - RUBRIC.prior) < 0.05, "score stays at the no-signal prior"); + assert.equal(r.band, "cheap"); +}); + +test("rubric: an UNSEEN phrasing routes by resemblance, not by literal keywords", () => { + // No old keyword ("race condition", "mutex") appears — the k-NN must still find + // the concurrency neighborhood through shared vocabulary. + const r = rubricComplexity("two threads deadlock when the queue is full, fix the locking"); + assert.equal(r.band, "premium", `expected premium, got ${r.band} (score ${r.score})`); + assert.ok(r.neighbors.length > 0, "neighbors are returned, every score is attributable"); +}); + +test("rubric: strongTopicSignal fires on confident hard matches only", () => { + const hard = rubricComplexity("implement a rate limiter with a token bucket algorithm"); + assert.equal(hard.strongTopicSignal, true); + const easy = rubricComplexity("fix a typo in the readme"); + assert.equal(easy.strongTopicSignal, false); + const moderate = rubricComplexity("add a small in-memory cache with get and set"); + assert.equal(moderate.strongTopicSignal, false, "moderate work is safe to route down"); +}); + +test("rubric: confidence shrinks weak matches toward the prior", () => { + const strong = rubricComplexity("implement dijkstra shortest path algorithm"); + const weak = rubricComplexity("the dijkstra approach discussion notes"); + assert.ok(strong.score > weak.score, "a full exemplar match outranks one shared token"); + assert.ok(strong.confidence > weak.confidence); +}); + +test("EXEMPLARS: labels are valid and every band is represented", () => { + for (const e of EXEMPLARS) { + assert.ok(typeof e.text === "string" && e.text.length > 0); + assert.ok(e.y >= 0 && e.y <= 1, `label in range: ${e.text}`); + } + const ys = new Set(EXEMPLARS.map((e) => (e.y < 0.3 ? "cheap" : e.y <= 0.6 ? "mid" : "premium"))); + assert.deepEqual([...ys].sort(), ["cheap", "mid", "premium"]); +}); + test("complexity is monotonic and bounded", () => { const trivial = complexity({ files: 0, fanout: 0, sizeWords: 4 }).score; const heavy = complexity({ From f12492ae9e77a900b4857fcc8ff239665f0ddf2c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:24 +0000 Subject: [PATCH 4/9] feat: wire lessons/anchor/substrate/skillgate/providers to graded math MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - lessons.matchScore keyword tier is graded (0.3 × token-overlap coefficient): an exact keyword keeps its historical 0.3, a same-module partial match earns partial credit instead of the old all-or-nothing string equality. - anchor.goalDrift returns driftScore in [0,1] (off-goal fraction per checkpoint), the graded magnitude cusum() was designed to accumulate but never received. - substrate.minimalityWarnings no longer keeps its own keyword copies: warnings derive from the preflight DIMENSIONS rubric (missing target_scope/constraints) and the route score already computed in the same gate. - skillgate gains an entropy-based obfuscation detector: a 200+ char base64-class blob at >=4.5 bits/char flags as a possible obfuscated payload — signatures catch known attack shapes, entropy catches the unknown one hiding as a blob. - providers: isLikelyGateway documented as a prior; providerStatus now probes /health on any custom base URL and reports behavioral gateway evidence, so a proxy the URL vocabulary missed still gets classified by how it behaves. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- src/anchor.js | 5 +++++ src/lessons.js | 26 +++++++++++++++++++++++--- src/providers.js | 42 +++++++++++++++++++++++++++++++++--------- src/skillgate.js | 18 +++++++++++++++++- src/substrate.js | 17 +++++++---------- test/lessons.test.js | 10 ++++++++++ 6 files changed, 95 insertions(+), 23 deletions(-) diff --git a/src/anchor.js b/src/anchor.js index 10bd8c6..6978571 100644 --- a/src/anchor.js +++ b/src/anchor.js @@ -154,6 +154,10 @@ export function goalDrift(root, goal, opts = {}) { } const drift = changedFiles.length > 0 && (offGoal.length > 0 || onGoal.length === 0); + // Graded drift magnitude Dₜ ∈ [0,1] — the fraction of this checkpoint's changes that + // wandered off-goal. This is the signal cusum() below expects: the binary `drift` + // flag answers "any drift now?", the score accumulates into "sustained drift?". + const driftScore = changedFiles.length ? offGoal.length / changedFiles.length : 0; return { goal: String(goal), keywords, @@ -161,6 +165,7 @@ export function goalDrift(root, goal, opts = {}) { onGoal, offGoal, drift, + driftScore, provenance, }; } diff --git a/src/lessons.js b/src/lessons.js index 8b1e2ce..ae1437d 100644 --- a/src/lessons.js +++ b/src/lessons.js @@ -28,6 +28,7 @@ export const SIGNALS = { // legacy injection path and Eq.-3 retrieval can never rank the same memory with // silently different weights during the bridge window. import { SCOPE_WEIGHT } from "./ledger.js"; +import { setOverlap } from "./math.js"; export { SCOPE_WEIGHT }; @@ -136,13 +137,32 @@ const globToRe = (glob) => { return new RegExp(`^${body}$`); }; -/** Trigger overlap with the current context, in [0,1]: symbol hit beats file beats keyword. */ +/** Token footprint of a keyword list ("src/auth/login.js" → {src, auth, login, js}) + * so two keywords about the same module overlap even when the strings differ. */ +const keywordGrams = (words) => { + const grams = new Set(); + for (const w of words ?? []) { + for (const t of String(w) + .toLowerCase() + .split(/[^a-z0-9]+/)) { + if (t.length > 1) grams.add(t); + } + } + return grams; +}; + +/** + * Trigger overlap with the current context, in [0,1]: symbol hit beats file-glob + * beats keyword. The keyword tier is GRADED — 0.3 × token-overlap coefficient — + * so an exact keyword match keeps its historical 0.3 while a same-module partial + * match ("src/auth/login.js" vs "src/auth/session.js") earns partial credit + * instead of the old all-or-nothing string equality. + */ export function matchScore(lesson, context) { const t = lesson.trigger; if (t.symbols?.some((s) => context.symbols?.includes(s))) return 1.0; if (t.files?.some((g) => context.files?.some((f) => globToRe(g).test(f)))) return 0.6; - if (t.keywords?.some((k) => context.keywords?.includes(k))) return 0.3; - return 0; + return 0.3 * setOverlap(keywordGrams(t.keywords), keywordGrams(context.keywords)); } /** diff --git a/src/providers.js b/src/providers.js index f7c37f3..3cbb919 100644 --- a/src/providers.js +++ b/src/providers.js @@ -19,10 +19,29 @@ function anthropicKeyEnv() { return "ANTHROPIC_API_KEY"; } +// A PRIOR, not a verdict: hostname vocabulary suggests a gateway, and the /health +// probe in providerStatus() supplies the behavioral evidence (a proxy that answers +// /health IS a LiteLLM-style gateway regardless of what its hostname says — pin the +// corrected classification with `forge config provider add`). Detection itself stays +// pure and network-free because it runs inside per-prompt hooks. function isLikelyGateway(url) { return /\b(gateway|litellm|aigateway|llmproxy|llm-proxy)\b/i.test(url); } +/** curl {baseUrl}/health, 2xx → true. Best-effort behavioral probe (5s cap). */ +function probeHealth(baseUrl) { + try { + const out = execFileSync( + "curl", + ["-sf", "-o", "/dev/null", "-w", "%{http_code}", `${baseUrl}/health`], + { encoding: "utf8", timeout: 5000 }, + ); + return out.startsWith("2"); + } catch { + return false; + } +} + function providersPath(root) { return join(root, PROVIDERS_DIR, PROVIDERS_FILE); } @@ -300,16 +319,10 @@ export function providerStatus(root = process.cwd()) { : `${prov.envKey} is NOT set — get it from the Anthropic Console (console.anthropic.com/settings/keys) or your provider's dashboard`, }); } + const customUrl = + prov.baseUrl && prov.baseUrl.replace(/\/+$/, "").toLowerCase() !== ANTHROPIC_DEFAULT_URL; if (prov.type === "litellm") { - let reachable = false; - try { - const out = execFileSync( - "curl", - ["-sf", "-o", "/dev/null", "-w", "%{http_code}", `${prov.baseUrl}/health`], - { encoding: "utf8", timeout: 5000 }, - ); - reachable = out.startsWith("2"); - } catch {} + const reachable = probeHealth(prov.baseUrl); checks.push({ id: "gateway", ok: reachable, @@ -317,6 +330,17 @@ export function providerStatus(root = process.cwd()) { ? `LiteLLM gateway reachable at ${prov.baseUrl}` : `LiteLLM gateway NOT reachable at ${prov.baseUrl}`, }); + } else if (customUrl) { + // Behavioral evidence beats hostname spelling: an "anthropic-proxy" that answers + // /health is actually a LiteLLM-style gateway the URL prior missed. Advisory. + const behavesLikeGateway = probeHealth(prov.baseUrl); + checks.push({ + id: "gateway-behavior", + ok: true, + detail: behavesLikeGateway + ? `${prov.baseUrl} answers /health like a LiteLLM gateway — pin it with \`forge config provider add\`` + : `${prov.baseUrl} does not answer /health (plain proxy, or unreachable from here)`, + }); } const envScan = [ diff --git a/src/skillgate.js b/src/skillgate.js index d2f9011..13b9e44 100644 --- a/src/skillgate.js +++ b/src/skillgate.js @@ -5,6 +5,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, readFileSync } from "node:fs"; +import { shannonEntropy } from "./math.js"; // Instruction-layer + tool-layer red flags. `critical` blocks install. const RULES = [ @@ -45,13 +46,28 @@ const RULES = [ }, ]; +// Obfuscation detector: a long base64-class blob whose Shannon entropy sits in +// encoded-payload territory. The RULES above are signatures for KNOWN attack shapes; +// entropy catches the unknown one hiding as an opaque blob in the instruction layer +// (nothing in a legitimate SKILL.md needs a 200-char undecodable string). +const BLOB_RE = /[A-Za-z0-9+/=]{200,}/g; +const BLOB_MIN_BITS = 4.5; + /** Pure: scan raw text for red flags. Returns findings [{sev, msg}]. */ export function heuristicScan(text) { const s = String(text); - return RULES.filter((r) => r.re.test(s)).map(({ sev, msg }) => ({ + const findings = RULES.filter((r) => r.re.test(s)).map(({ sev, msg }) => ({ sev, msg, })); + const blobs = (s.match(BLOB_RE) || []).filter((b) => shannonEntropy(b) >= BLOB_MIN_BITS); + if (blobs.length) { + findings.push({ + sev: "high", + msg: `${blobs.length} high-entropy encoded blob(s) — possible obfuscated payload`, + }); + } + return findings; } /** Scan a path (SKILL.md/.mcp.json) or raw text. Real scanner if available, else heuristic. */ diff --git a/src/substrate.js b/src/substrate.js index 2eeb6d5..b7f28f4 100644 --- a/src/substrate.js +++ b/src/substrate.js @@ -42,13 +42,13 @@ function verificationChecklist(root) { return [...new Set(checks)]; } -function minimalityWarnings(task, route, preflight) { +// Every warning derives from signals the gate ALREADY computed — the preflight +// DIMENSIONS rubric owns "which dimensions apply and which are missing", routing owns +// complexity. No second keyword copy here to drift out of sync with those sources. +function minimalityWarnings(_task, route, preflight) { const warnings = []; - const text = String(task).toLowerCase(); - if ( - /\b(refactor|rewrite|clean|redesign|optimi[sz]e|improve)\b/.test(text) && - preflight.entities.files.length === 0 - ) { + const missing = new Set((preflight.assumption?.missing ?? []).map((m) => m.key)); + if (missing.has("target_scope") && preflight.entities.files.length === 0) { warnings.push( "High-risk broad change with no target files named; ask for scope before editing.", ); @@ -58,10 +58,7 @@ function minimalityWarnings(task, route, preflight) { "Complex task with medium/low specification completeness; clarify before spending a premium model.", ); } - if ( - /\b(add authentication|payment|migration|distributed|concurrent|production)\b/.test(text) && - !/\btest|acceptance|rollback|constraint|must|should\b/.test(text) - ) { + if (missing.has("constraints")) { warnings.push("Production-sensitive task lacks explicit constraints or acceptance criteria."); } return warnings; diff --git a/test/lessons.test.js b/test/lessons.test.js index 33e338b..412e06f 100644 --- a/test/lessons.test.js +++ b/test/lessons.test.js @@ -89,6 +89,16 @@ test("matchScore: symbol hit beats file-glob beats keyword", () => { assert.equal(matchScore(l, { files: ["src/db/index.ts"] }), 0); }); +test("matchScore: keyword tier is graded — same-module partial overlap earns partial credit", () => { + const l = newLesson({ id: "g", trigger: { keywords: ["src/auth/login.js"] } }); + const exact = matchScore(l, { keywords: ["src/auth/login.js"] }); + const sibling = matchScore(l, { keywords: ["src/auth/session.js"] }); + const unrelated = matchScore(l, { keywords: ["docs/readme.md"] }); + assert.equal(exact, 0.3, "full overlap keeps the historical tier value"); + assert.ok(sibling > 0 && sibling < exact, "same module scores between 0 and exact"); + assert.equal(unrelated, 0, "no shared tokens → no match"); +}); + test("selectForInjection: relevance-ranked, capped, overflow becomes a pointer (never silent)", () => { const ctx = { symbols: ["foo"], files: [], keywords: [] }; const lessons = Array.from({ length: 5 }, (_, i) => { From 1a23c425b3789e4a15585b514ea7680c623014ff Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:24 +0000 Subject: [PATCH 5/9] =?UTF-8?q?feat:=20docs=20drift=20machinery=20?= =?UTF-8?q?=E2=80=94=20forge=20docs=20check,=20CI=20gate,=20docs=20in=20th?= =?UTF-8?q?e=20impact=20graph?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New src/docs_check.js reconciles documentation CLAIMS against the sources of truth that already exist in code, so a feature can no longer merge with its docs silently missing: - commands: README/GUIDE vs the commands.js table (extracted from cli.js so --help and the docs check render from the same data) - env vars: prose docs vs actual process.env reads in src/*.js AND $VAR reads in shell guards; both directions (undocumented reads + phantom vars) - MCP tools: doc counts and tool names vs the cortex_mcp TOOLS registry (now exported) - CHANGELOG: latest header matches package.json, no empty release sections, [Unreleased] can't stay empty while src commits accumulate Wired in three places: new `forge docs check` command (exit non-zero), a doctor check that runs only inside the forge repo itself, and a docs-drift CI job. scripts/bump.mjs now refuses to rotate an empty [Unreleased] — the exact mechanism that produced two empty release sections. Atlas now parses markdown: each doc becomes a graph node with references edges to the code it names (backticked paths/symbols, link targets), so `forge impact src/foo.js` lists the DOCS that go stale — the missing code→doc half of end-to-end propagation. First run of the checker found 56 real drift issues including a phantom env var (FORGE_BRAND) no audit had caught. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- .github/workflows/ci.yml | 14 +++ scripts/bump.mjs | 7 ++ src/atlas.js | 41 ++++++++ src/cli.js | 80 +++++---------- src/commands.js | 63 ++++++++++++ src/cortex_mcp.js | 5 +- src/docs_check.js | 210 +++++++++++++++++++++++++++++++++++++++ src/doctor.js | 17 ++++ test/atlas.test.js | 22 ++++ test/bump.test.js | 8 ++ test/docs_check.test.js | 137 +++++++++++++++++++++++++ 11 files changed, 547 insertions(+), 57 deletions(-) create mode 100644 src/commands.js create mode 100644 src/docs_check.js create mode 100644 test/docs_check.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd2c215..065e216 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -68,6 +68,20 @@ jobs: # and CITATION.cff must all carry the same version (scripts/bump.mjs keeps them in sync). - run: node scripts/bump.mjs check + docs-drift: + name: Docs match code + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 # the CHANGELOG check compares src vs CHANGELOG.md commit times + - uses: actions/setup-node@v6 + with: + node-version: 22 + # Commands, env vars, MCP tools, and CHANGELOG reconciled against the code — + # a feature can't merge with its docs missing (src/docs_check.js). + - run: node src/cli.js docs check + dependency-review: name: Dependency review runs-on: ubuntu-latest diff --git a/scripts/bump.mjs b/scripts/bump.mjs index d0fcc04..33be37a 100644 --- a/scripts/bump.mjs +++ b/scripts/bump.mjs @@ -109,6 +109,13 @@ export function rotateChangelog(changelog, newVersion, prevVersion, date) { } const body = extractUnreleased(changelog); if (body === null) throw new Error("CHANGELOG.md has no ## [Unreleased] section"); + // An empty [Unreleased] would rotate into an empty release section — a header + // with no record of what shipped (exactly how 0.8.0/0.8.1 went undocumented). + // Write the changelog first; the release notes ARE part of the release. + if (!body.trim()) + throw new Error( + "CHANGELOG.md [Unreleased] is empty — describe the release before bumping (see forge docs check)", + ); const start = changelog.search(/^## \[Unreleased\][^\n]*\n/m); const afterHeading = changelog.indexOf("\n", start) + 1; const head = changelog.slice(0, afterHeading); diff --git a/src/atlas.js b/src/atlas.js index 6b95487..6ff4f24 100644 --- a/src/atlas.js +++ b/src/atlas.js @@ -32,6 +32,10 @@ const RULES = { { re: /\b(?:struct|enum|trait)\s+([A-Za-z_]\w*)/g, kind: "type" }, ], ".java": [{ re: /\b(?:class|interface|enum)\s+([A-Za-z_]\w*)/g, kind: "type" }], + // Markdown carries no symbol definitions ([]), but extractDoc() below turns each + // doc into a graph node with `references` edges to the code it talks about — the + // missing code→doc half of impact: change a symbol, its docs show up as dependents. + ".md": [], }; const IMPORT_RE = @@ -114,6 +118,42 @@ function nearestSource(nodes, line, fallback) { return best; } +// A markdown file becomes ONE doc node whose outgoing `references` edges point at the +// code it names: backticked `src/foo.js` paths and `symbolName` identifiers, plus +// [link](path) targets. In the reverse-BFS those edges make every referencing doc a +// DEPENDENT of the code — `forge impact src/route.js` now lists the docs that go +// stale, so end-to-end propagation includes documentation, not just callers. +function extractDoc(rel, text) { + const doc = { id: `doc:${rel}`, name: rel, kind: "doc", file: rel, line: 1 }; + const edges = []; + const seen = new Set(); + const refEdge = (target, confidence, line) => { + if (seen.has(target)) return; + seen.add(target); + edges.push({ source: doc.id, target, kind: "references", confidence, line }); + }; + for (const m of text.matchAll(/`([^`\n]+)`/g)) { + const line = lineOf(text, m.index); + for (const raw of m[1].trim().split(/\s+/)) { + const tok = raw.replace(/[(),;:]+$/, "").replace(/^\.\//, ""); + if (!tok) continue; + if (/[/\\]/.test(tok) && RULES[extname(tok)] && extname(tok) !== ".md") { + refEdge(`module:${moduleId(tok)}`, 0.8, line); // `src/foo.js` → its module node + } else if (/^[A-Za-z_$][\w$]*(\(\))?$/.test(tok) && tok.length >= 3) { + const name = tok.replace(/\(\)$/, ""); + if (!BUILTINS.has(name)) refEdge(name, 0.6, line); // `symbolName` → resolveEdges links it + } + } + } + for (const m of text.matchAll(/\]\(([^)#\s]+)\)/g)) { + const tok = m[1].replace(/^\.\//, ""); + if (/^[a-z]+:/i.test(tok)) continue; // external URL, not a repo path + if (RULES[extname(tok)] && extname(tok) !== ".md") + refEdge(`module:${moduleId(tok)}`, 0.8, lineOf(text, m.index)); + } + return { symbols: [], nodes: [doc], edges, hash: hash(text) }; +} + function extractFile(path, root, preRead) { const ext = extname(path); const rules = RULES[ext]; @@ -126,6 +166,7 @@ function extractFile(path, root, preRead) { return { symbols: [], nodes: [], edges: [], hash: "" }; } } + if (ext === ".md") return extractDoc(rel, text); const mod = { id: `module:${moduleId(rel)}`, diff --git a/src/cli.js b/src/cli.js index f626c5d..424befa 100755 --- a/src/cli.js +++ b/src/cli.js @@ -2,62 +2,9 @@ // forge — zero-dependency dispatcher. Works identically whether installed via the // npm bin, the hardened install.sh symlink, or the Claude Code plugin. import { BRAND } from "./brand.js"; - -const COMMANDS = { - init: "scaffold this repo's config — emits every tool from one shared source", - sync: "recompile the canonical source into each tool's native config files", - doctor: "health-check installed tools, guards, MCP auth, and config drift", - taste: "enable one UI-taste tool for this repo (no arg = list)", - atlas: "build / query the code-graph (where-is-Y, has-symbol)", - recall: "manage cross-session memory (list / add / consolidate)", - catalog: "Start Here — list every tool, crew, and guard with a one-line why", - scan: "vet a skill/MCP for injection/RCE/exfil before install (skill-gate)", - verify: "independent verification gate — tests + hallucinated-symbol + provenance", - harden: "wire security controls — gitleaks pre-commit + sandbox settings", - remember: "add a durable fact to this repo's portable memory (forge brain)", - brain: "show / rebuild the portable project memory index", - cost: "real per-day spend via ccusage + measured stage factors (--stages)", - spec: "spec-as-contract — init (OpenSpec) / lock / check drift", - cortex: "self-correcting project memory — status / why ", - ledger: - "proof-carrying memory — stats / verify / show / blame / query / ratify / retract / merge / import", - reuse: "proof-carrying code cache — query / mint --file / stats", - context: "budgeted context assembly + completeness gate — what an edit NEEDS known", - preflight: "assumption check — what a task names that the repo doesn't define", - config: "provider setup — show / switch / add providers, set default model", - route: "recommend the cheapest capable model for a task (+ gateway config)", - impact: "predict blast radius for a symbol or file from the atlas graph", - substrate: "one pre-action gate: assumptions, route, impact, scope, memory, verify", - scope: "decompose files into independent clusters (+ coupled files you didn't name)", - anchor: "goal-drift check — are your actual (git) changes still on the stated goal?", - diagnose: - "doom-loop check — record a failure; 3× the same signature mints a diagnosis + escalation", - imagine: "consequence simulation — predicted breaks + the minimal dry-run test suite for a task", - lean: "scope-minimality (M5) — measure the diff's footprint vs what the task asked for", - uicheck: - "deterministic UI checks — contrast · fingerprint · design · visual ", - dash: "local dashboard over the ledger, metrics, and blast radius", - brand: "print the active brand token map", -}; - -const GROUPS = { - Core: ["init", "sync", "doctor", "catalog"], - Memory: ["cortex", "recall", "remember", "brain", "ledger", "reuse"], - Substrate: [ - "substrate", - "preflight", - "route", - "impact", - "scope", - "context", - "anchor", - "diagnose", - "imagine", - "lean", - ], - Quality: ["verify", "scan", "spec", "taste", "uicheck", "harden"], - Config: ["config", "cost", "dash", "brand", "atlas"], -}; +// The command surface lives in commands.js as data — docs_check.js reconciles the +// README/GUIDE tables against the same table this help is rendered from. +import { COMMANDS, GROUPS } from "./commands.js"; const printVersion = () => console.log(`${BRAND.brand} (${BRAND.pkg}) v${BRAND.version}`); @@ -185,6 +132,27 @@ async function run(argv) { if (failed) process.exitCode = 1; return; } + if (cmd === "docs") { + // Self-check of the forge package's own docs against its code (commands table, + // env reads, MCP registry, CHANGELOG). `forge docs check` is the only subcommand. + const { docsCheck } = await import("./docs_check.js"); + const r = docsCheck(); + if (argv.includes("--json")) { + console.log(JSON.stringify(r, null, 2)); + if (!r.ok) process.exitCode = 1; + return; + } + console.log(`${BRAND.brand} docs check — docs↔code drift\n`); + if (!r.issues.length) + console.log(" ✓ docs and code agree (commands, env vars, MCP tools, CHANGELOG)"); + for (const i of r.issues) + console.log(` ${i.severity === "error" ? "✗" : "!"} [${i.check}] ${i.detail}`); + if (!r.ok) { + console.log(`\n${r.issues.filter((i) => i.severity === "error").length} problem(s)`); + process.exitCode = 1; + } + return; + } if (cmd === "recall") { const r = await import("./recall.js"); const store = r.defaultStore(); diff --git a/src/commands.js b/src/commands.js new file mode 100644 index 0000000..c8a82ad --- /dev/null +++ b/src/commands.js @@ -0,0 +1,63 @@ +// forge commands — the CLI's command surface as DATA. cli.js renders --help from +// this table and docs_check.js reconciles README/GUIDE against it, so a command can +// no longer ship (or disappear) without the docs check noticing. One line per command. + +export const COMMANDS = { + init: "scaffold this repo's config — emits every tool from one shared source", + sync: "recompile the canonical source into each tool's native config files", + doctor: "health-check installed tools, guards, MCP auth, and config drift", + taste: "enable one UI-taste tool for this repo (no arg = list)", + atlas: "build / query the code-graph (where-is-Y, has-symbol)", + recall: "manage cross-session memory (list / add / consolidate)", + catalog: "Start Here — list every tool, crew, and guard with a one-line why", + scan: "vet a skill/MCP for injection/RCE/exfil before install (skill-gate)", + verify: "independent verification gate — tests + hallucinated-symbol + provenance", + harden: "wire security controls — gitleaks pre-commit + sandbox settings", + remember: "add a durable fact to this repo's portable memory (forge brain)", + brain: "show / rebuild the portable project memory index", + cost: "real per-day spend via ccusage + measured stage factors (--stages)", + spec: "spec-as-contract — init (OpenSpec) / lock / check drift", + cortex: "self-correcting project memory — status / why ", + ledger: + "proof-carrying memory — stats / verify / show / blame / query / ratify / retract / merge / import", + reuse: "proof-carrying code cache — query / mint --file / stats", + context: "budgeted context assembly + completeness gate — what an edit NEEDS known", + preflight: "assumption check — what a task names that the repo doesn't define", + config: "provider setup — show / switch / add providers, set default model", + route: "recommend the cheapest capable model for a task (+ gateway config)", + impact: "predict blast radius for a symbol or file from the atlas graph", + substrate: "one pre-action gate: assumptions, route, impact, scope, memory, verify", + scope: "decompose files into independent clusters (+ coupled files you didn't name)", + anchor: "goal-drift check — are your actual (git) changes still on the stated goal?", + diagnose: + "doom-loop check — record a failure; 3× the same signature mints a diagnosis + escalation", + imagine: "consequence simulation — predicted breaks + the minimal dry-run test suite for a task", + lean: "scope-minimality (M5) — measure the diff's footprint vs what the task asked for", + uicheck: + "deterministic UI checks — contrast · fingerprint · design · visual ", + dash: "local dashboard over the ledger, metrics, and blast radius", + brand: "print the active brand token map", + docs: "docs↔code drift check — commands, env vars, MCP tools, CHANGELOG vs reality", +}; + +export const GROUPS = { + Core: ["init", "sync", "doctor", "catalog", "docs"], + Memory: ["cortex", "recall", "remember", "brain", "ledger", "reuse"], + Substrate: [ + "substrate", + "preflight", + "route", + "impact", + "scope", + "context", + "anchor", + "diagnose", + "imagine", + "lean", + ], + Quality: ["verify", "scan", "spec", "taste", "uicheck", "harden"], + Config: ["config", "cost", "dash", "brand", "atlas"], +}; + +/** Commands that exist but are deliberately not advertised in --help or docs tables. */ +export const HIDDEN_COMMANDS = ["cortex-mcp"]; diff --git a/src/cortex_mcp.js b/src/cortex_mcp.js index 8893d90..0f8d472 100644 --- a/src/cortex_mcp.js +++ b/src/cortex_mcp.js @@ -28,7 +28,10 @@ const PKG_VERSION = (() => { const root = process.env.FORGE_ROOT || process.cwd(); const today = epochDay; -const TOOLS = [ +// Exported so docs_check.js reconciles the documented MCP tool list against the +// registry that actually serves — a tool can't ship undocumented (or be documented +// after removal) without `forge docs check` flagging it. +export const TOOLS = [ { name: "cortex_lessons", description: diff --git a/src/docs_check.js b/src/docs_check.js new file mode 100644 index 0000000..f9b4969 --- /dev/null +++ b/src/docs_check.js @@ -0,0 +1,210 @@ +// forge docs — docs↔code drift reconcilers. Every check compares a DOCUMENT CLAIM +// against the SOURCE OF TRUTH that already exists in code (the commands.js table, the +// cortex_mcp TOOLS registry, actual process.env reads, package.json + git history), so +// a feature can no longer ship without its docs and the gap silently accumulate — the +// exact failure that let a whole command family go undocumented. Self-check: it runs +// against the forge package root (BRAND.root), not the host repo. +import { execFileSync } from "node:child_process"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { BRAND } from "./brand.js"; +import { COMMANDS, HIDDEN_COMMANDS } from "./commands.js"; +import { TOOLS } from "./cortex_mcp.js"; + +/** The user-facing prose docs every claim is reconciled against. */ +const DOC_FILES = ["README.md", "docs/GUIDE.md", "ARCHITECTURE.md", "ROADMAP.md"]; + +// Env vars read in src that are NOT user-facing contract: child-process plumbing and +// values injected by host tools rather than set by users. +const INTERNAL_ENV = new Set(["_FORGE_LLM_KEY", "FORGE_EMBED_KEY", "CLAUDE_PLUGIN_ROOT"]); + +// Prefixes that mark an env var as OURS to document. A doc may freely mention other +// tools' vars (GITHUB_TOKEN, PATH) — those aren't claims about forge's own surface. +const ENV_PREFIX_RE = /\b((?:FORGE|ANTHROPIC|LITELLM|OPENROUTER|ENABLE_CORTEX)_[A-Z0-9_]+)\b/g; + +function readDoc(root, rel) { + const p = join(root, rel); + return existsSync(p) ? readFileSync(p, "utf8") : ""; +} + +function srcFiles(root) { + const dir = join(root, "src"); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter((f) => f.endsWith(".js")) + .map((f) => join(dir, f)); +} + +/** Every env var name the package actually reads: process.env.X in src/*.js plus + * $X / ${X...} in the shell guards (they are part of the same env contract). */ +export function envVarsRead(root = BRAND.root) { + const vars = new Set(); + for (const file of srcFiles(root)) { + const text = readFileSync(file, "utf8"); + for (const m of text.matchAll(/process\.env\.([A-Z_][A-Z0-9_]*)/g)) vars.add(m[1]); + for (const m of text.matchAll(/process\.env\[["']([A-Z_][A-Z0-9_]*)["']\]/g)) vars.add(m[1]); + } + const guards = join(root, "global", "guards"); + if (existsSync(guards)) { + for (const f of readdirSync(guards).filter((f) => f.endsWith(".sh"))) { + const text = readFileSync(join(guards, f), "utf8"); + for (const m of text.matchAll(/\$\{?([A-Z_][A-Z0-9_]*)/g)) vars.add(m[1]); + } + } + return vars; +} + +/** Commands table vs README/GUIDE: every command documented, nothing phantom. */ +function checkCommands(docs, issues) { + for (const target of ["README.md", "docs/GUIDE.md"]) { + const text = docs[target]; + if (!text) continue; + for (const name of Object.keys(COMMANDS)) { + if (!new RegExp(`\\bforge ${name}\\b`).test(text)) { + issues.push({ + check: "commands", + severity: "error", + detail: `\`forge ${name}\` is implemented but ${target} never mentions it`, + }); + } + } + } + for (const [file, text] of Object.entries(docs)) { + for (const m of text.matchAll(/`forge ([a-z][a-z-]*)\b/g)) { + const name = m[1]; + if (!(name in COMMANDS) && !HIDDEN_COMMANDS.includes(name)) { + issues.push({ + check: "commands", + severity: "error", + detail: `${file} documents \`forge ${name}\` but no such command exists`, + }); + } + } + } +} + +/** Env vars: everything src reads is documented; everything docs name is real. */ +function checkEnvVars(root, docs, issues) { + const read = envVarsRead(root); + const prose = Object.values(docs).join("\n"); + for (const v of read) { + if (INTERNAL_ENV.has(v)) continue; + if (!prose.includes(v)) { + issues.push({ + check: "env-vars", + severity: "error", + detail: `src reads ${v} but no prose doc (${DOC_FILES.join(", ")}) documents it`, + }); + } + } + const documented = new Set(); + for (const [file, text] of Object.entries(docs)) { + for (const m of text.matchAll(ENV_PREFIX_RE)) { + if (!documented.has(`${file}:${m[1]}`) && !read.has(m[1]) && !INTERNAL_ENV.has(m[1])) { + documented.add(`${file}:${m[1]}`); + issues.push({ + check: "env-vars", + severity: "error", + detail: `${file} documents ${m[1]} but nothing in src reads it (phantom var)`, + }); + } + } + } +} + +/** MCP tools: documented counts match the registry; every tool name appears somewhere. */ +function checkMcpTools(docs, issues) { + const actual = TOOLS.length; + for (const [file, text] of Object.entries(docs)) { + for (const m of text.matchAll(/(\d+)\s+MCP tools\b/gi)) { + if (Number(m[1]) !== actual) { + issues.push({ + check: "mcp-tools", + severity: "error", + detail: `${file} claims ${m[1]} MCP tools; the registry serves ${actual}`, + }); + } + } + } + const prose = Object.values(docs).join("\n"); + for (const t of TOOLS) { + if (!prose.includes(t.name)) { + issues.push({ + check: "mcp-tools", + severity: "error", + detail: `MCP tool ${t.name} is served but never documented`, + }); + } + } +} + +const git = (root, args) => { + try { + return execFileSync("git", args, { + cwd: root, + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); + } catch { + return ""; + } +}; + +/** CHANGELOG: latest release header matches package.json; no empty release sections; + * [Unreleased] must not be empty while src has commits the changelog hasn't seen. */ +function checkChangelog(root, issues) { + const text = readDoc(root, "CHANGELOG.md"); + if (!text) return; + const sections = [ + ...text.matchAll(/^## \[([^\]]+)\][^\n]*\n([\s\S]*?)(?=^## \[|\n*$(?![\s\S]))/gm), + ]; + const version = JSON.parse(readFileSync(join(root, "package.json"), "utf8")).version; + const released = sections.filter((s) => s[1].toLowerCase() !== "unreleased"); + if (released.length && released[0][1] !== version) { + issues.push({ + check: "changelog", + severity: "error", + detail: `CHANGELOG's latest release is [${released[0][1]}] but package.json is ${version}`, + }); + } + for (const [, name, body] of released) { + if (!body.trim()) { + issues.push({ + check: "changelog", + severity: "error", + detail: `CHANGELOG section [${name}] is an empty header — released work is unrecorded`, + }); + } + } + const unreleased = sections.find((s) => s[1].toLowerCase() === "unreleased"); + if (unreleased && !unreleased[2].trim()) { + const srcT = Number(git(root, ["log", "-1", "--format=%ct", "--", "src"]) || 0); + const clT = Number(git(root, ["log", "-1", "--format=%ct", "--", "CHANGELOG.md"]) || 0); + if (srcT && clT && srcT > clT) { + issues.push({ + check: "changelog", + severity: "error", + detail: "src changed since CHANGELOG.md was last touched, but [Unreleased] is empty", + }); + } + } +} + +/** + * Run every reconciler against the forge package tree. + * @param {{root?: string}} [opts] + * @returns {{ok: boolean, issues: {check:string, severity:string, detail:string}[], checked: string[]}} + */ +export function docsCheck({ root = BRAND.root } = {}) { + const docs = Object.fromEntries(DOC_FILES.map((f) => [f, readDoc(root, f)])); + const issues = []; + checkCommands(docs, issues); + checkEnvVars(root, docs, issues); + checkMcpTools(docs, issues); + checkChangelog(root, issues); + return { + ok: !issues.some((i) => i.severity === "error"), + issues, + checked: ["commands", "env-vars", "mcp-tools", "changelog"], + }; +} diff --git a/src/doctor.js b/src/doctor.js index 8bb1c86..fc70a86 100644 --- a/src/doctor.js +++ b/src/doctor.js @@ -6,6 +6,7 @@ import { join } from "node:path"; import { isStale, load as loadAtlas } from "./atlas.js"; import { BRAND } from "./brand.js"; import { summary as cortexSummary } from "./cortex.js"; +import { docsCheck } from "./docs_check.js"; import { extractHash, hashContent } from "./emit/_shared.js"; import { verify as ledgerVerify, repoLedger } from "./ledger_store.js"; import { PRICING_VERIFIED } from "./model_tiers.js"; @@ -307,6 +308,21 @@ function checkProvider(out, targetRoot) { } } +// Docs↔code drift — a self-check of the forge package's own docs, so it only runs +// when doctor is pointed at the forge repo itself (contributors + CI), never at a +// host project whose README rightly says nothing about forge commands. +function checkDocs(out, targetRoot) { + try { + if (readJson(join(targetRoot, "package.json")).name !== BRAND.pkg) return; + const r = docsCheck({ root: targetRoot }); + out.push( + r.ok + ? ok("docs↔code", "commands, env vars, MCP tools, CHANGELOG all agree") + : warn("docs↔code", `${r.issues.length} drift issue(s) — run \`forge docs check\``), + ); + } catch {} +} + export function doctor({ targetRoot = process.cwd() } = {}) { const results = []; checkNode(results); @@ -318,6 +334,7 @@ export function doctor({ targetRoot = process.cwd() } = {}) { checkTooling(results); checkInstall(results); checkDrift(results, targetRoot); + checkDocs(results, targetRoot); checkAtlas(results, targetRoot); checkPricing(results); checkMcp(results, targetRoot); diff --git a/test/atlas.test.js b/test/atlas.test.js index f84ef05..fb73ce3 100644 --- a/test/atlas.test.js +++ b/test/atlas.test.js @@ -93,3 +93,25 @@ test("impact (llm on): a proposed edge survives only if real + grep-verified", ( const blind = impact(atlas, "computeTax", { llm: true, run: () => '{"files":["a.js"]}' }); assert.deepEqual(blind.llmVerified, [], "no verify predicate → nothing added blind"); }); + +test("markdown docs become graph nodes whose references make them impact dependents", () => { + const root = mkdtempSync(join(tmpdir(), "forge-atlas-")); + writeFileSync(join(root, "src.js"), "export function computeTax(x){ return x*0.2 }\n"); + writeFileSync( + join(root, "GUIDE.md"), + "# guide\n\nThe `computeTax` helper lives in [the source](src.js).\n", + ); + writeFileSync(join(root, "UNRELATED.md"), "# other\n\nNothing about the code here.\n"); + const atlas = build({ root }); + assert.ok( + atlas.nodes.some((n) => n.kind === "doc" && n.file === "GUIDE.md"), + "the doc is a graph node", + ); + // Change the symbol → the doc that references it is in the blast radius. + const bySymbol = impact(atlas, "computeTax"); + assert.ok(bySymbol.impactedFiles.includes("GUIDE.md"), JSON.stringify(bySymbol.impactedFiles)); + assert.ok(!bySymbol.impactedFiles.includes("UNRELATED.md"), "silent docs stay out"); + // Change the file → same answer through the module-path reference. + const byFile = impact(atlas, "src.js"); + assert.ok(byFile.impactedFiles.includes("GUIDE.md"), JSON.stringify(byFile.impactedFiles)); +}); diff --git a/test/bump.test.js b/test/bump.test.js index 6f6fcaa..8789ee2 100644 --- a/test/bump.test.js +++ b/test/bump.test.js @@ -230,3 +230,11 @@ test("collectVersions reports every field and exposes drift", () => { fs.rmSync(dir, { recursive: true, force: true }); } }); + +test("rotateChangelog refuses an empty [Unreleased] — a release must describe itself", () => { + const empty = "# Changelog\n\n## [Unreleased]\n\n## [0.4.0] - 2026-01-01\n\n- old\n"; + assert.throws( + () => rotateChangelog(empty, "0.5.0", "0.4.0", "2026-07-07"), + /\[Unreleased\] is empty/, + ); +}); diff --git a/test/docs_check.test.js b/test/docs_check.test.js new file mode 100644 index 0000000..313c9b5 --- /dev/null +++ b/test/docs_check.test.js @@ -0,0 +1,137 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { COMMANDS } from "../src/commands.js"; +import { TOOLS } from "../src/cortex_mcp.js"; +import { docsCheck, envVarsRead } from "../src/docs_check.js"; + +// A fixture tree whose docs are generated FROM the real registries, so it passes by +// construction — each test then breaks exactly one claim and asserts the reconciler +// catches it. `mutate` edits the file map before writing. +function fixtureRoot(mutate = (files) => files) { + const root = mkdtempSync(join(tmpdir(), "forge-docs-")); + const allCommands = Object.keys(COMMANDS) + .map((c) => `| \`forge ${c}\` | ... |`) + .join("\n"); + const allTools = TOOLS.map((t) => `- \`${t.name}\``).join("\n"); + const envDocs = "Set FORGE_FIXTURE_VAR to tune the fixture."; + const files = { + "package.json": JSON.stringify({ name: "fixture", version: "1.2.3" }), + "README.md": `# fixture\n${allCommands}\n${allTools}\n${TOOLS.length} MCP tools\n${envDocs}\n`, + "docs/GUIDE.md": `# guide\n${allCommands}\n`, + "ARCHITECTURE.md": "# arch\n", + "ROADMAP.md": "# roadmap\n", + "CHANGELOG.md": + "# Changelog\n\n## [Unreleased]\n\n- pending thing\n\n## [1.2.3] - 2026-01-01\n\n- shipped thing\n", + "src/a.js": "const v = process.env.FORGE_FIXTURE_VAR;\n", + }; + for (const [rel, content] of Object.entries(mutate({ ...files }))) { + const p = join(root, rel); + mkdirSync(join(p, ".."), { recursive: true }); + writeFileSync(p, content); + } + return root; +} + +test("docsCheck: a fully consistent tree passes clean", () => { + const r = docsCheck({ root: fixtureRoot() }); + assert.deepEqual(r.issues, []); + assert.equal(r.ok, true); +}); + +test("docsCheck: an implemented command missing from README is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "README.md": f["README.md"].replace("| `forge config` | ... |\n", ""), + })); + const r = docsCheck({ root }); + assert.equal(r.ok, false); + assert.ok(r.issues.some((i) => i.check === "commands" && /forge config.*README/.test(i.detail))); +}); + +test("docsCheck: a documented command that does not exist is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "docs/GUIDE.md": `${f["docs/GUIDE.md"]}\nUse \`forge frobnicate\` to frob.\n`, + })); + const r = docsCheck({ root }); + assert.ok( + r.issues.some((i) => i.check === "commands" && /frobnicate.*no such command/.test(i.detail)), + ); +}); + +test("docsCheck: an env var src reads but no doc mentions is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "src/a.js": `${f["src/a.js"]}const w = process.env.FORGE_SECRET_KNOB;\n`, + })); + const r = docsCheck({ root }); + assert.ok(r.issues.some((i) => i.check === "env-vars" && i.detail.includes("FORGE_SECRET_KNOB"))); +}); + +test("docsCheck: a phantom env var documented but never read is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "ROADMAP.md": `${f["ROADMAP.md"]}Injects LITELLM_GATEWAY_URL so calls route through the proxy.\n`, + })); + const r = docsCheck({ root }); + assert.ok( + r.issues.some((i) => i.check === "env-vars" && /LITELLM_GATEWAY_URL.*phantom/.test(i.detail)), + ); +}); + +test("docsCheck: env vars read by shell guards count as read (no false phantom)", () => { + const root = fixtureRoot((f) => ({ + ...f, + "README.md": `${f["README.md"]}Tune FORGE_GUARD_KNOB for the guard.\n`, + "global/guards/g.sh": 'x="${FORGE_GUARD_KNOB:-10}"\n', + })); + const r = docsCheck({ root }); + assert.equal( + r.issues.filter((i) => i.detail.includes("FORGE_GUARD_KNOB")).length, + 0, + "a guard-read var documented in README is fully consistent", + ); + assert.ok(envVarsRead(root).has("FORGE_GUARD_KNOB")); +}); + +test("docsCheck: an MCP tool-count claim that disagrees with the registry is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "ARCHITECTURE.md": `${f["ARCHITECTURE.md"]}\nExposes 3 MCP tools.\n`, + })); + const r = docsCheck({ root }); + assert.ok(r.issues.some((i) => i.check === "mcp-tools" && /claims 3.*serves \d+/.test(i.detail))); +}); + +test("docsCheck: an undocumented MCP tool is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "README.md": f["README.md"].replace(`- \`${TOOLS[0].name}\`\n`, ""), + })); + const r = docsCheck({ root }); + assert.ok(r.issues.some((i) => i.check === "mcp-tools" && i.detail.includes(TOOLS[0].name))); +}); + +test("docsCheck: empty release sections and version mismatch are flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "CHANGELOG.md": + "# Changelog\n\n## [Unreleased]\n\n## [9.9.9] - 2026-01-01\n\n## [1.2.3] - 2025-12-01\n\n- old\n", + })); + const r = docsCheck({ root }); + assert.ok( + r.issues.some( + (i) => i.check === "changelog" && /\[9\.9\.9\].*package\.json is 1\.2\.3/.test(i.detail), + ), + "latest header must match package.json", + ); + assert.ok( + r.issues.some( + (i) => i.check === "changelog" && /\[9\.9\.9\] is an empty header/.test(i.detail), + ), + "empty released section flagged", + ); +}); From 9467297ab445879fd4b74d58c579efa3a128cd9f Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:24 +0000 Subject: [PATCH 6/9] feat: persistent goal, auto-sync on Stop, stale-docs advisory on edit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes for cross-session amnesia and silent drift: - src/goal.js + `forge anchor set/show/clear`: the active goal now persists in .forge/goal.md. SessionStart re-injects it, and a bare `forge anchor` checks drift against the stored goal — it used to live in one command's argv and vanish with the session, so every new session re-assumed the objective. - sync.autoSyncIfDrifted(): the Stop hook now REPAIRS AGENTS.md drift instead of leaving it for doctor to report — lessons/facts learned in a session reach every AGENTS.md-reading tool immediately. Never adopts an unmanaged repo; kill switch FORGE_AUTOSYNC=0. - pre-edit hook: when the cached atlas knows docs that reference the file being edited (the new doc-reference edges), it says so before the edit — code changes carry their documentation with them instead of leaving it behind. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- src/cli.js | 36 +++++++++++++--- src/cortex_hook_main.js | 33 ++++++++++++++- src/goal.js | 59 ++++++++++++++++++++++++++ src/sync.js | 20 +++++++++ test/goal.test.js | 92 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 src/goal.js create mode 100644 test/goal.test.js diff --git a/src/cli.js b/src/cli.js index 424befa..7bf4fe1 100755 --- a/src/cli.js +++ b/src/cli.js @@ -1083,13 +1083,39 @@ async function run(argv) { } if (cmd === "anchor") { const { goalDrift, renderAnchor } = await import("./anchor.js"); + const { clearGoal, getGoal, setGoal } = await import("./goal.js"); const json = argv.includes("--json"); - const goal = argv - .slice(1) - .filter((a) => a !== "--json") - .join(" "); + const args = argv.slice(1).filter((a) => a !== "--json"); + const sub = args[0]; + // Persistent goal management: `set` stores it, SessionStart re-injects it, and a + // bare `forge anchor` checks against it — the goal survives the session that set it. + if (sub === "set") { + const r = setGoal(process.cwd(), args.slice(1).join(" ")); + if (!r.ok) { + console.error(`forge anchor set: ${r.reason}`); + process.exitCode = 1; + return; + } + console.log( + `goal set: ${r.goal}\n(injected each session start; \`forge anchor\` checks against it)`, + ); + return; + } + if (sub === "show") { + const g = getGoal(process.cwd()); + console.log(g ? `active goal: ${g}` : 'no active goal — set one: forge anchor set ""'); + return; + } + if (sub === "clear") { + clearGoal(process.cwd()); + console.log("goal cleared."); + return; + } + const goal = args.join(" ") || getGoal(process.cwd()); if (!goal) { - console.error('usage: forge anchor "" [--json]'); + console.error( + 'usage: forge anchor "" [--json]\n forge anchor set|show|clear — persist the goal across sessions', + ); process.exitCode = 1; return; } diff --git a/src/cortex_hook_main.js b/src/cortex_hook_main.js index 9bba9c7..b16f5da 100644 --- a/src/cortex_hook_main.js +++ b/src/cortex_hook_main.js @@ -67,15 +67,27 @@ async function main() { clearSession(root, sid); await enrichCreated(root, results); } + // Auto-sync: anything this session taught the memory (lessons, facts) reaches + // every AGENTS.md-reading tool NOW — drift used to be detected by doctor but + // repaired by nobody. Only touches an already-Forge-managed AGENTS.md; + // kill switch FORGE_AUTOSYNC=0. Fail-safe like everything else here. + try { + const { autoSyncIfDrifted } = await import("./sync.js"); + autoSyncIfDrifted(root); + } catch {} } else if (mode === "session-start") { - const block = startupBlock(root, today); + // Learned lessons + the persistent goal — the two things a fresh session forgets. + const { goalBlock } = await import("./goal.js"); + const block = [startupBlock(root, today), goalBlock(root)].filter(Boolean).join("\n"); if (block) emit("SessionStart", block); } else if (mode === "pre-edit") { // A doom loop (the same failure recurring) is the loudest thing to say — it means "stop", // so it takes precedence over lesson/risk advice. const loop = doomLoopAdvisory(readSession(root, sid)); const advice = loop || (await preEditAdvisory(root, hook.tool_input?.file_path, today)); - if (advice) emit("PreToolUse", advice); + const docs = await staleDocsAdvisory(root, hook.tool_input?.file_path); + const combined = [advice, docs].filter(Boolean).join("\n\n"); + if (combined) emit("PreToolUse", combined); } else if (mode === "preflight") { // Ambient cognitive substrate: assumption gate + (when an atlas is already cached) // model routing, blast-radius, memory, and minimality — surfaced before the agent acts. @@ -131,6 +143,23 @@ async function preEditAdvisory(root, file, today) { : ""; } +// Docs that reference the file about to change (atlas doc edges, CACHED graph only — +// a hook never builds). The end-to-end nudge: a code edit carries its docs with it. +async function staleDocsAdvisory(root, file) { + if (!file || file.endsWith(".md")) return ""; + try { + const { load, impact } = await import("./atlas.js"); + const atlas = load(root); + if (!atlas) return ""; + const rel = file.startsWith(root) ? file.slice(root.length).replace(/^[/\\]/, "") : file; + const docs = impact(atlas, rel, { maxHops: 2 }).impactedFiles.filter((f) => f.endsWith(".md")); + if (!docs.length) return ""; + return `Forge impact — docs that reference ${rel}: ${docs.slice(0, 5).join(", ")}. If this change alters behavior, update them in the same pass (\`forge impact ${rel}\` for the full list).`; + } catch { + return ""; + } +} + main() .catch((err) => { if (process.env.FORGE_DEBUG === "1") diff --git a/src/goal.js b/src/goal.js new file mode 100644 index 0000000..6e25bed --- /dev/null +++ b/src/goal.js @@ -0,0 +1,59 @@ +// forge goal — the persistent active goal for a repo. `forge anchor` was per-invocation +// only: the goal lived in one command's argv and vanished with the session, so the next +// session re-assumed. Storing it in .forge/goal.md lets SessionStart re-inject it, lets +// goalDrift default to it, and makes "what are we actually doing" survive context loss. +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { hasSecret } from "./secrets.js"; + +const goalPath = (root) => join(root, ".forge", "goal.md"); + +/** Persist the active goal. Refuses secrets (same rule as every forge store). */ +export function setGoal(root, text, { t = Date.now() } = {}) { + const goal = String(text ?? "").trim(); + if (!goal) return { ok: false, reason: "empty goal — state what you are working toward" }; + if (hasSecret(goal)) + return { ok: false, reason: "refused: goal looks like it contains a secret/credential" }; + mkdirSync(join(root, ".forge"), { recursive: true }); + writeFileSync( + goalPath(root), + `# Goal\n\n${goal}\n\n\n`, + ); + return { ok: true, goal }; +} + +/** The active goal text, or null when none is set. */ +export function getGoal(root) { + const p = goalPath(root); + if (!existsSync(p)) return null; + try { + const goal = readFileSync(p, "utf8") + .replace(/^# Goal\s*/, "") + .replace(//g, "") + .trim(); + return goal || null; + } catch { + return null; + } +} + +export function clearGoal(root) { + try { + rmSync(goalPath(root), { force: true }); + return { ok: true }; + } catch { + return { ok: false }; + } +} + +/** SessionStart injection block — empty string when no goal is set (low-nag). */ +export function goalBlock(root) { + const goal = getGoal(root); + if (!goal) return ""; + return [ + "## Active goal (Forge Anchor)", + `The stated goal for current work in this repo: **${goal}**`, + "Stay on it. `forge anchor` checks your changes against it; `forge anchor clear` when done.", + "", + ].join("\n"); +} diff --git a/src/sync.js b/src/sync.js index cff218a..a2dae0e 100644 --- a/src/sync.js +++ b/src/sync.js @@ -130,3 +130,23 @@ export function sync({ targetRoot = process.cwd() } = {}) { export function canonical(targetRoot = process.cwd()) { return buildCanonical(targetRoot); } + +/** + * Re-run sync ONLY when the managed AGENTS.md no longer matches the canonical source. + * The Stop hook calls this, so lessons/facts learned in a session reach every + * AGENTS.md-reading tool immediately — not whenever someone remembers `forge sync` + * (nothing did before: doctor DETECTED drift but nothing repaired it). + * Never adopts a repo: AGENTS.md must already exist AND be Forge-managed. + * Kill switch: FORGE_AUTOSYNC=0. + * @returns {{synced: boolean, reason: string}} + */ +export function autoSyncIfDrifted(targetRoot = process.cwd()) { + if (process.env.FORGE_AUTOSYNC === "0") return { synced: false, reason: "disabled" }; + const existing = shared.readIfExists(join(targetRoot, "AGENTS.md")); + if (existing === null || !shared.isManaged(existing)) + return { synced: false, reason: "no managed AGENTS.md here" }; + if (shared.extractHash(existing) === shared.hashContent(buildCanonical(targetRoot))) + return { synced: false, reason: "in sync" }; + sync({ targetRoot }); + return { synced: true, reason: "drifted — resynced" }; +} diff --git a/test/goal.test.js b/test/goal.test.js new file mode 100644 index 0000000..f9f3e8c --- /dev/null +++ b/test/goal.test.js @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import { clearGoal, getGoal, goalBlock, setGoal } from "../src/goal.js"; +import { autoSyncIfDrifted, sync } from "../src/sync.js"; +import { fakeAnthropic } from "./_fixtures.js"; + +const repo = () => mkdtempSync(join(tmpdir(), "forge-goal-")); + +test("setGoal/getGoal/clearGoal round-trip, empty and missing cases", () => { + const root = repo(); + assert.equal(getGoal(root), null); + assert.equal(setGoal(root, " ").ok, false); + const r = setGoal(root, "ship the provider auto-detection fix"); + assert.equal(r.ok, true); + assert.equal(getGoal(root), "ship the provider auto-detection fix"); + clearGoal(root); + assert.equal(getGoal(root), null); +}); + +test("setGoal refuses a goal that carries a secret", () => { + const root = repo(); + const r = setGoal(root, `deploy with key ${fakeAnthropic()}`); + assert.equal(r.ok, false); + assert.match(r.reason, /secret/); + assert.equal(getGoal(root), null, "nothing persisted"); +}); + +test("goalBlock renders the goal for SessionStart, empty when unset", () => { + const root = repo(); + assert.equal(goalBlock(root), ""); + setGoal(root, "remove every keyword heuristic"); + const block = goalBlock(root); + assert.match(block, /Active goal/); + assert.match(block, /remove every keyword heuristic/); +}); + +test("session-start hook injects the persistent goal", () => { + const root = repo(); + setGoal(root, "finish the docs drift machinery"); + const entry = join(dirname(fileURLToPath(import.meta.url)), "..", "src", "cortex_hook_main.js"); + const r = spawnSync("node", [entry, "session-start"], { + input: JSON.stringify({ session_id: "s", cwd: root }), + encoding: "utf8", + }); + assert.equal(r.status, 0); + assert.match(r.stdout, /finish the docs drift machinery/); +}); + +test("autoSyncIfDrifted: repairs a drifted managed AGENTS.md, never adopts an unmanaged repo", () => { + const root = repo(); + // Unmanaged repo (no AGENTS.md): must not adopt. + assert.equal(autoSyncIfDrifted(root).synced, false); + assert.equal(existsSync(join(root, "AGENTS.md")), false, "auto-sync never creates from nothing"); + + // Managed repo, in sync: no-op. + sync({ targetRoot: root }); + assert.equal(autoSyncIfDrifted(root).synced, false); + + // Drift the canonical inputs (a per-repo rules override) → auto-sync repairs. + mkdirSync(join(root, ".forge"), { recursive: true }); + writeFileSync( + join(root, ".forge", "rules.json"), + JSON.stringify({ sections: [{ title: "Extra", rules: ["always frob the widget"] }] }), + ); + const r = autoSyncIfDrifted(root); + assert.equal(r.synced, true); + assert.match(readFileSync(join(root, "AGENTS.md"), "utf8"), /always frob the widget/); + assert.equal(autoSyncIfDrifted(root).synced, false, "second pass: already in sync"); +}); + +test("autoSyncIfDrifted: FORGE_AUTOSYNC=0 disables the repair", () => { + const root = repo(); + sync({ targetRoot: root }); + mkdirSync(join(root, ".forge"), { recursive: true }); + writeFileSync( + join(root, ".forge", "rules.json"), + JSON.stringify({ sections: [{ title: "X", rules: ["y"] }] }), + ); + const saved = process.env.FORGE_AUTOSYNC; + process.env.FORGE_AUTOSYNC = "0"; + try { + assert.equal(autoSyncIfDrifted(root).synced, false); + } finally { + if (saved === undefined) delete process.env.FORGE_AUTOSYNC; + else process.env.FORGE_AUTOSYNC = saved; + } +}); From 82e555e91dd4a871087edfe9f47aa016252e67f1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:21:25 +0000 Subject: [PATCH 7/9] =?UTF-8?q?docs:=20full=20docs-code=20alignment=20?= =?UTF-8?q?=E2=80=94=20forge=20docs=20check=20now=20passes=20clean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes every drift issue the new reconciler found (56 on first run): - README: forge config + forge docs rows in the Commands table; accurate 19-MCP-tools claim with a link to the full list; anchor set/show/clear noted. - GUIDE: new forge config section (auto-detection priority order, corporate gateway walkthrough); forge docs section; routing section rewritten for the exemplar k-NN rubric with honest sample output; persistent-goal docs; full 19-tool MCP table; complete environment-variable reference covering every var the code actually reads (kept honest by forge docs check in CI). - ARCHITECTURE: brand-token wording no longer reads as a phantom FORGE_BRAND env var; MCP layer described as the real 19-tool server. - ROADMAP: phantom LITELLM_GATEWAY_URL replaced with the real mechanism (litellm.config.yaml + ANTHROPIC_BASE_URL); OpenAI/Gemini detection moved from Shipped to Next (it is not implemented); version header current. - CHANGELOG: backfilled the empty [0.8.0] and [0.8.1] sections from git history and wrote [Unreleased] covering the gateway work and this change. - CLAUDE.md: docs check added to the pre-commit gate; benchmarks wording updated for the exemplar rubric. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- ARCHITECTURE.md | 10 ++-- CHANGELOG.md | 60 ++++++++++++++++++++++++ CLAUDE.md | 4 +- README.md | 10 ++-- ROADMAP.md | 30 ++++++++---- docs/GUIDE.md | 106 +++++++++++++++++++++++++++++++++++++----- reports/benchmarks.md | 2 +- src/docs_check.js | 7 ++- 8 files changed, 198 insertions(+), 31 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b063c92..4199386 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -25,7 +25,7 @@ for the full list. ## Locked decisions - **Brand = `Forge`** — CLI `forge`; layer names: skills→**tools**, agents→**crew**, hooks→**guards**, code-graph→**atlas**, minimalism→**lean**, memory→**recall**. - Brand stored as **one token** (`brand.json` → `FORGE_BRAND`); rebrand = 1 edit. + Brand stored as **one token** (the `brand` key in `brand.json`); rebrand = 1 edit. - **Distributable id = `forgekit`** (npm package + marketplace id) — fixed even if the brand token changes, so a rename never breaks install. - **Scope = full multi-tool day 1** — nine tools plus MCP, from one canonical source. @@ -68,8 +68,10 @@ The four layers, brand-named and emitted cross-tool: *enforces* rather than suggests.** A guard is a deterministic hook the model cannot drift from. Prose rules in CLAUDE.md get acknowledged and then forgotten after compaction; a guard does not. Every enforceable invariant belongs here. -- **mcp** — the protocol layer. Forge ships the `atlas` code-graph server and the - substrate tools (`substrate_check` / `predict_impact` / `assumption_gate`). +- **mcp** — the protocol layer. Forge ships one stdio server (`src/cortex_mcp.js`) + exposing 19 MCP tools: the substrate checks (`substrate_check` / `predict_impact` / + `assumption_gate` / …), memory reads AND writes (`forge_remember`, ledger + ratify/retract), and ops/health — the full table is in docs/GUIDE.md. Cross-cutting concerns thread through all four: **atlas** (the code graph), **lean** (minimalism — shipped as *both* a tool and a Stop-guard, so it applies whether or not @@ -243,7 +245,7 @@ Roo Code and VS Code receive the Forge MCP server via `forge init` ``` forgekit/ package.json # npm CLI: bin `forge` → src/cli.js - brand.json # single FORGE_BRAND token + layer-name map + brand.json # single brand token + layer-name map README.md # Start-Here index + one bootstrap command src/ cli.js # init | sync | doctor | substrate | ledger | reuse | … (`forge --help` for all) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d02576..b1c179f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,72 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- **Gateway environments work end to end** — `ANTHROPIC_AUTH_TOKEN` is recognized + everywhere `ANTHROPIC_API_KEY` is; `ANTHROPIC_MODEL` / `FORGE_MODEL` pin one model + (bypassing tier routing); a gateway-looking `ANTHROPIC_BASE_URL` auto-classifies as + LiteLLM; and the LLM proposer falls back to **direct HTTP** (`src/llm.js`, Anthropic + Messages API) when the `claude` CLI is absent — or on `FORGE_LLM_HTTP=1`. +- **`forge docs check`** (+ CI job + doctor check) — reconciles README/GUIDE/ + ARCHITECTURE/ROADMAP against the code: every CLI command documented, every env var + read is documented and every documented var is real, MCP tool counts/names match the + registry, CHANGELOG sections non-empty. First run found 56 real drift issues, + including a phantom env var. `scripts/bump.mjs` now refuses to rotate an empty + `[Unreleased]`. +- **Docs are in the impact graph** — the atlas parses markdown into doc nodes with + `references` edges to the code they name, so `forge impact src/foo.js` lists the + docs that go stale, and the pre-edit hook says so before the edit. +- **Persistent goal** — `forge anchor set/show/clear` stores the active goal in + `.forge/goal.md`; SessionStart re-injects it and a bare `forge anchor` checks + against it. `goalDrift` also returns a graded `driftScore` for the CUSUM detector. +- **AGENTS.md auto-repair** — the Stop hook re-runs sync when the managed AGENTS.md + drifts from its canonical inputs (disable: `FORGE_AUTOSYNC=0`). +- **Entropy secret detection** — `src/secrets.js` is the single source of truth + (format grammars + Shannon-entropy gate for unknown-vendor tokens); the + `secret-redact` guard now imports it, ending the JS/shell regex divergence. +- **`src/math.js`** — Shannon entropy, charset classes, exact set Jaccard/overlap. + +### Changed +- **Routing scores by exemplar similarity, not keyword lists** — the text rubric is + similarity-weighted k-NN over a labeled `EXEMPLARS` bank (overlap-coefficient on + stopword-filtered unigram+bigram sets, credibility-shrunk); the four topic keyword + regexes and their additive magic weights are gone. Tune routing by adding labeled + rows, not by editing weights. +- **Lesson matching is graded** — the keyword tier of `matchScore` scores by token + overlap (same-module partial credit) instead of all-or-nothing string equality. +- **Substrate minimality warnings derive from computed signals** (preflight missing + dimensions + route score) instead of a second keyword copy. +- **`forge scan` detects obfuscated payloads** — long high-entropy base64 blobs flag + as findings alongside the signature rules. +- **`providerStatus` probes `/health` on any custom base URL** and reports behavioral + gateway evidence (a proxy that answers /health is a gateway, whatever its hostname). + ## [0.8.1] - 2026-07-08 +### Added +- **MCP write tools** — `forge_remember`, `forge_ledger_ratify`, + `forge_ledger_retract` join the read tools (19 tools total). + +### Changed +- Simplified CLI surface and improved dashboard UX empty states. +### Fixed +- Stale documentation across command references. ## [0.8.0] - 2026-07-08 +### Added +- **Forge work system** — auto-install flow, multi-provider routing, the cost + dashboard (`forge dash`), and the cortex MCP server's read-path tools. +- **Zero-config provider auto-detection** — `autoDetectProvider()` resolves the + provider from the environment (LiteLLM local/hosted, OpenRouter, Anthropic); + `forge init` reports what it found. +- **Hosted LiteLLM gateway support** — `emitGatewayConfig()` writes a + `litellm.config.yaml` exposing complexity tiers as model aliases. + +### Fixed +- TypeScript errors and Biome 2.5.2 lint warnings across source and tests. + ## [0.7.0] - 2026-07-08 diff --git a/CLAUDE.md b/CLAUDE.md index b223809..bd12758 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,5 +18,7 @@ - Match existing patterns: dynamic `await import()` for optional modules, brand tokens from `src/brand.js` (never hardcode "Forge"/"forge"), `BRAND.root` for package root paths. -- Run `npm test && npx biome check && npm run typecheck` before committing. +- Run `npm test && npx biome check && npm run typecheck && node src/cli.js docs check` + before committing — the docs check fails CI when commands/env vars/MCP tools/CHANGELOG + drift from the code, so update docs IN THE SAME CHANGE, not later. - Version lives in `package.json` — `scripts/bump.mjs` keeps all manifests in sync. diff --git a/README.md b/README.md index 1e242c2..4eefbd1 100644 --- a/README.md +++ b/README.md @@ -140,8 +140,10 @@ git pull && forge ledger merge On Claude Code the substrate then runs on **every prompt automatically** via a `UserPromptSubmit` hook — advisory only, silent on clean tasks. Every other tool gets a -native config rule plus MCP tools (`substrate_check`, `predict_impact`, `assumption_gate`, -`route_task`, `scope_files`) it can call itself. +native config rule plus **19 MCP tools** it can call itself — pre-action checks +(`substrate_check`, `predict_impact`, `assumption_gate`, `route_task`, `scope_files`), +memory reads and writes, and ops/health — the full list with schemas is in +[`docs/GUIDE.md`](docs/GUIDE.md#mcp-tools). ## Commands @@ -154,6 +156,8 @@ default 25-file threshold). | **Config layer** | `forge init` | emit every tool's native config from one source | | | `forge sync` | recompile canonical source → each tool's native files (idempotent) | | | `forge doctor` | pass/fail health check: tools, guards, MCP, drift | +| | `forge docs` | docs↔code drift check — commands, env vars, MCP tools, CHANGELOG vs reality | +| | `forge config` | provider setup — show / switch / add providers, set the default model | | | `forge harden` | wire gitleaks pre-commit + sandbox settings | | | `forge catalog` | Start-Here index of every tool / crew / guard | | | `forge brand` | print the brand token map | @@ -171,7 +175,7 @@ default 25-file threshold). | | `forge imagine` | consequence sim + minimal dry-run suite (`--run` executes it sandboxed) | | | `forge context` | budgeted context assembly + completeness gate | | | `forge atlas` | build / query / has (hallucinated-symbol check) the code graph | -| | `forge anchor` | goal-drift check (advisory) | +| | `forge anchor` | goal-drift check (advisory) — `set`/`show`/`clear` persists the goal across sessions | | | `forge diagnose` | doom-loop: same failure 3× → diagnosis + escalation | | | `forge lean` | scope-minimality footprint (advisory) | | | `forge cost` | real per-day spend · measured stage factors (`--stages`) | diff --git a/ROADMAP.md b/ROADMAP.md index ac4bb0b..772982f 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,10 +7,14 @@ every tool. This is where that brain is headed. Direction, not promises — shaped by the two field reports this project is grounded in (the SDLC pain-point map and the ecosystem landscape). Open a Discussion to weigh in. -## Now (`master`, v0.8.0) -Everything from 0.7.0 plus branding layer (`forge brand`), model-tier exports -(`forgekit/model-tiers`), and the Codex plugin manifest — same toolkit, two plugin -surfaces (Claude Code + Codex). See [CHANGELOG.md](./CHANGELOG.md). +## Now (`master`, v0.8.1+) +Everything from 0.8.0 plus gateway-environment support end to end — `ANTHROPIC_AUTH_TOKEN` +recognized everywhere, `ANTHROPIC_MODEL`/`FORGE_MODEL` model override, LiteLLM-gateway +auto-classification of `ANTHROPIC_BASE_URL`, direct-HTTP LLM calls when the `claude` CLI +is absent (`src/llm.js`) — plus decision math replacing keyword heuristics (exemplar k-NN +routing, entropy secret detection), docs↔code drift gating (`forge docs check`, in CI), +docs in the impact graph, and a persistent goal (`forge anchor set`). +See [CHANGELOG.md](./CHANGELOG.md). ## Shipped — Substrate v2 (all phases P0–P8, v0.5.0) The plan lives in [docs/plans/substrate-v2/](./docs/plans/substrate-v2/00-overview.md) @@ -22,12 +26,15 @@ confidence only from independent oracles, and merges across teammates conflict-f ## Shipped — 0.7.0 - **Zero-config provider auto-detection** — `autoDetectProvider()` probes env vars - for OpenRouter, LiteLLM (local + hosted gateway), OpenAI, Anthropic, and Gemini; - `forge init` reports what it found, no manual config needed. -- **Hosted LiteLLM gateway** — `emitGatewayConfig()` writes a guard that injects - `LITELLM_GATEWAY_URL` + key so every model call routes through the team proxy. -- **15 MCP tools** — the cortex MCP server (`src/cortex_mcp.js`) now exposes - read-path tools for ledger, brain, atlas, recall, cost, substrate, and dashboard. + for LiteLLM (local + hosted gateway), OpenRouter, and Anthropic (key, auth token, + or custom base URL); `forge init` reports what it found, no manual config needed. + (OpenAI and Gemini detection: see Next.) +- **Hosted LiteLLM gateway** — `emitGatewayConfig()` writes a `litellm.config.yaml` + exposing the complexity tiers as model aliases; point `ANTHROPIC_BASE_URL` at the + proxy and every model call routes through it. +- **MCP server** — the cortex MCP server (`src/cortex_mcp.js`) exposes read-path + tools for ledger, brain, atlas, recall, cost, substrate, and dashboard (19 MCP tools + as of 0.8.x, including the write tools added in 0.8.0). - **Cost dashboard** — `forge dash` serves a local HTML dashboard showing model spend, event timeline, and ledger health from `.forge/` data. @@ -46,6 +53,9 @@ confidence only from independent oracles, and merges across teammates conflict-f `/status/`. ## Next +- **OpenAI + Gemini provider detection** — extend `autoDetectProvider()` beyond + Anthropic/OpenRouter/LiteLLM (`OPENAI_API_KEY`, `GEMINI_API_KEY`) with the same + zero-config contract. - **Legacy store retirement** — the read-path flip has shipped: every read surface (cortex injection/status, the substrate advisory, routing, `recall list`, brain's AGENTS.md index) is now a merged view (legacy ∪ ledger) via `src/ledger_read.js`, diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 21fb8b3..7d1d53e 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -27,7 +27,7 @@ Every command is real and wired. Grouped by what it does: | Group | Commands | | --- | --- | -| **Config / cross-tool sync** | `forge init` · `forge sync` · `forge doctor` · `forge harden` · `forge catalog` · `forge brand` | +| **Config / cross-tool sync** | `forge init` · `forge sync` · `forge doctor` · `forge docs` · `forge config` · `forge harden` · `forge catalog` · `forge brand` | | **Memory & ledger (PCM)** | `forge ledger` · `forge recall` · `forge remember` · `forge brain` · `forge cortex` · `forge reuse` | | **Code graph & retrieval** | `forge atlas` · `forge context` | | **Substrate / pre-action** | `forge substrate` · `forge preflight` · `forge route` · `forge impact` · `forge scope` · `forge imagine` · `forge anchor` · `forge diagnose` · `forge lean` · `forge cost` | @@ -177,22 +177,53 @@ _Advisory: ask rather than assume._ ### `forge route ""` — the cheapest capable model -A transparent additive rubric (never a second LLM call). Every point is attributable -to a named signal you can inspect and override. +A transparent, deterministic rubric (never a second LLM call), and every score is +attributable. The text side is **similarity-weighted k-NN over a labeled exemplar +bank** (`EXEMPLARS` in `src/route.js`): your task is compared to ~50 example tasks +with known complexities, and the nearest neighbors — which the output names — set the +estimate. The repo side scores real signals (files in scope, impact fan-out, churn, +past-mistake density, ambiguity). Whichever facet detects difficulty sets the tier. ```console $ forge route "write an is_prime function" → Haiku 4.5 (simple, $1/$5 per M tok) lint, formatting, docs, stubs, trivial well-defined edits - complexity 0.13 · driven by: ambiguity, base cost of any task + driven by: similar to "check if a number is prime" (sim 1.00, complexity 0.08) $ forge route "design and implement a distributed rate limiter with sliding windows across 3 services" - → Fable 5 (extreme, $10/$50 per M tok) - complexity 0.95 · driven by: algorithmic/systems difficulty, architectural/design scope + → Fable 5 / Opus (premium tier) + driven by: similar to "implement a rate limiter with a token bucket" (sim 0.71, complexity 0.78) ``` +Unseen phrasings route by resemblance — "two threads deadlock when the queue is full" +lands in the concurrency neighborhood without any keyword list needing the literal +token "race condition". To tune routing, add labeled rows to `EXEMPLARS` (data, not +weights). `ANTHROPIC_MODEL` / `FORGE_MODEL` override the tier choice entirely. + Run `forge route gateway` to emit a LiteLLM config so the routing happens automatically. +### `forge config` — provider setup + +Shows, switches, and registers model providers, and sets the default model. Forge +auto-detects the provider from the environment with zero config — the priority order is +`LITELLM_BASE_URL` → `ANTHROPIC_BASE_URL` (a URL that answers `/health` or names a +gateway is classified as one) → `OPENROUTER_API_KEY` → `ANTHROPIC_API_KEY` → +`ANTHROPIC_AUTH_TOKEN`. An explicit `.forge/providers.json` always wins over detection. + +```console +$ forge config show # active provider + how it was resolved +$ forge config providers # everything detected + configured +$ forge config provider # switch +$ forge config provider add --base-url [--env-key VAR] +$ forge config model # default model +$ forge config gateway # emit litellm.config.yaml (same as forge route gateway) +$ forge config setup # guided first-time setup +``` + +Corporate gateway environments work out of the box: with `ANTHROPIC_BASE_URL` + +`ANTHROPIC_AUTH_TOKEN` set (LiteLLM-style gateways), detection classifies the gateway, +auth uses the token as a Bearer credential, and `ANTHROPIC_MODEL` pins the model. + ### `forge impact ` — what will this edit break? Reverse-dependency blast radius from the atlas graph. Run `forge atlas build` first. @@ -243,7 +274,14 @@ Forge anchor — goal-drift check `src/auth.js` maps to the goal (named file + where `verifyToken` lives); `src/report.js` doesn't — so it's surfaced as drift to confirm or undo. Coarse and advisory by design -(path/keyword match, not semantic). `forge substrate` folds this in automatically. +(path/keyword match, not semantic). `forge substrate` folds this in automatically. The +result also carries `driftScore` (the off-goal fraction per checkpoint) — the graded +signal the `cusum` change-point detector accumulates to catch *sustained* small drift. + +**The goal persists.** `forge anchor set ""` stores it in `.forge/goal.md`; every +new session re-injects it at SessionStart, a bare `forge anchor` checks against it, and +`forge anchor show` / `forge anchor clear` manage it. A goal set on Monday still anchors +Thursday's session — no more each-session re-assumption of what you're working toward. ### `forge verify` — did it actually work? @@ -587,6 +625,7 @@ Plain `forge cost` remains the per-day spend view via `ccusage`. | `forge init` | Emit every tool's native config from one source. | | `forge sync` | Recompile `source/` → each tool's files (idempotent). | | `forge doctor` | Health check: layers, install, drift, cortex. | +| `forge docs check` | Docs↔code drift: commands, env vars, MCP tools, CHANGELOG reconciled against the code (CI-gated on the forge repo itself). | | `forge catalog` | Start-Here index of every tool / crew / guard. | | `forge brain` / `forge remember` | Portable project memory inlined into `AGENTS.md`. | | `forge cost` | Real per-day spend (via `ccusage`) + the cost ceiling; `--stages` for the measured report. | @@ -649,15 +688,30 @@ Nothing to wire — the plugin's [`hooks/hooks.json`](../hooks/hooks.json) insta > `forge substrate "" --json` (or the MCP tool `substrate_check`). If > `okToProceed` is false, ask the questions first; read `impact.impactedFiles` before editing. -…and exposes the substrate as MCP tools any MCP-capable agent can call directly: +…and exposes the substrate as **19 MCP tools** any MCP-capable agent can call directly: + + | MCP tool | Does | | --- | --- | | `substrate_check` | full pre-action check | +| `preflight_check` | assumption / info-gap check | | `assumption_gate` | ask/proceed + questions | -| `predict_impact` | blast radius | +| `predict_impact` | blast radius (code **and** the docs that reference it) | | `route_task` | model recommendation | | `scope_files` | independent vs. coupled | +| `cortex_lessons` | learned lessons for given files/symbols | +| `cortex_status` | memory lifecycle summary | +| `forge_brain` | durable project facts | +| `forge_ledger_query` | ranked retrieval over the PCM ledger | +| `forge_remember` | **write**: add a durable project fact | +| `forge_ledger_ratify` | **write**: human-ratify a claim into a decision | +| `forge_ledger_retract` | **write**: tombstone a claim | +| `forge_diagnose` | doom-loop failure check | +| `forge_doctor` | health check | +| `forge_provider_status` | provider detection + gateway reachability | +| `forge_cost` | spend + stage factors | +| `forge_dash_data` / `forge_dash_summary` | dashboard data feeds | Forge never pretends it can force a hook into a tool that has none — **ambient on Claude Code, agent-invoked everywhere else.** @@ -726,7 +780,7 @@ Create `global/crew/.md` with frontmatter. It installs into `~/.claude/age | --- | --- | | how often it asks | `source/substrate.json` → `defaults.askThreshold` (0.6) | | blast-radius sensitivity | `source/substrate.json` → `defaults.impactThreshold` (0.1) | -| a routing signal | `src/route.js` → `rubricComplexity()` | +| a routing outcome | `src/route.js` → add a labeled row to `EXEMPLARS` (data, not weights); constants in `RUBRIC` | | model tiers / prices | `src/model_tiers.js` | | an assumption question | `src/preflight.js` → `DIMENSIONS[]` | | the verify checklist | `src/substrate.js` → `verificationChecklist()` | @@ -756,9 +810,39 @@ Add an emitter module in `src/emit/.js` (mirror an existing one like `test/sync.test.js` keeps it honest. ### Rebrand -Edit `brand.json` (`FORGE_BRAND`), the `bin` key in `package.json`, and `name` in +Edit the brand token in `brand.json`, the `bin` key in `package.json`, and `name` in `.claude-plugin/plugin.json`. The whole CLI, banner, and emitted headers follow. +### Environment variables + +The complete env contract (`forge docs check` keeps this table honest — a variable the +code reads but this table misses fails CI on the forge repo): + +| Variable | Does | +| --- | --- | +| `ANTHROPIC_API_KEY` | direct Anthropic auth (also used by gateways when set) | +| `ANTHROPIC_AUTH_TOKEN` | gateway/proxy Bearer credential — recognized everywhere the API key is | +| `ANTHROPIC_BASE_URL` | custom API endpoint; gateway-looking URLs auto-classify as LiteLLM | +| `ANTHROPIC_MODEL` / `FORGE_MODEL` | pin one model — bypasses tier routing entirely | +| `LITELLM_BASE_URL` / `LITELLM_API_KEY` | hosted LiteLLM gateway endpoint + key (highest detection priority) | +| `OPENROUTER_API_KEY` | OpenRouter provider | +| `FORGE_LLM` | `1` enables the LLM proposer layer (off = fully deterministic) | +| `FORGE_LLM_AMBIENT` | `1` lets the ambient hook use the proposer too | +| `FORGE_LLM_HTTP` | `1` forces direct HTTP (Anthropic Messages API) instead of the `claude` CLI; automatic when the CLI is absent | +| `FORGE_ENFORCE` | `1` turns the substrate advisory into a hard block on the strongest signals | +| `FORGE_AUTOSYNC` | `0` disables the Stop-hook AGENTS.md auto-repair | +| `FORGE_EMBED` / `FORGE_EMBED_MODEL` / `FORGE_EMBED_TIMEOUT_MS` | optional embeddings tier (ADR-0005) | +| `FORGE_HOME` | override `~/.forge` (recall store location) | +| `FORGE_ROOT` | repo root override for the MCP server | +| `FORGE_AUTHOR` | identity stamped on ledger provenance (defaults to git identity) | +| `FORGE_COST_CEILING` | daily spend (USD) the cost-budget guard warns at (default 10) | +| `FORGE_LOOP_THRESHOLD` | identical tool calls before the doom-loop guard speaks (default 4) | +| `FORGE_LEAN_THRESHOLD` | lines-per-task-word ratio the lean guard nudges at | +| `FORGE_VERIFY_TIMEOUT_MS` | verify test-run timeout (default 600000) | +| `FORGE_SKILLGATE_NOEXTERNAL` | `1` skips the external scanner in `forge scan` (heuristic only) | +| `ENABLE_CORTEX_DISTILL` | `1` distills new lessons into prose via a cheap model call | +| `FORGE_DEBUG` | `1` writes fail-safe error details to stderr instead of swallowing them | + --- ## Honest limits diff --git a/reports/benchmarks.md b/reports/benchmarks.md index cc5b013..40e878c 100644 --- a/reports/benchmarks.md +++ b/reports/benchmarks.md @@ -166,7 +166,7 @@ that forgekit structurally does not. |---|---|---| | routing decision visible *before* dispatch | yes — returns band, signals, and per-signal reasons the user can read and override (`src/route.js`) | decision is made inside the proxy at request time | | rubric versioned in the repo | yes — deterministic scoring over `src/model_tiers.json`, diffable in PRs | routing/cost logic lives in gateway config or the provider's service | -| same input ⇒ same route | yes — regex rubric is deterministic; LLM adjudication is opt-in and clamped inside band rails (a proposal can never jump past them) | depends on gateway load/cost/failover state | +| same input ⇒ same route | yes — the exemplar k-NN rubric is deterministic (same text, same neighbors, same score); LLM adjudication is opt-in and clamped inside band rails (a proposal can never jump past them) | depends on gateway load/cost/failover state | | **what the gateways have that forgekit doesn't** | — | they actually *move traffic*: proxying, failover, quotas, key management. `forge route` is advisory and at most **emits** a LiteLLM config exposing its tiers as aliases (`src/route.js`) — it is a transparency layer over gateways, not a replacement | ### Proof-carrying reuse vs plain RAG retrieval diff --git a/src/docs_check.js b/src/docs_check.js index f9b4969..10daf16 100644 --- a/src/docs_check.js +++ b/src/docs_check.js @@ -48,7 +48,12 @@ export function envVarsRead(root = BRAND.root) { if (existsSync(guards)) { for (const f of readdirSync(guards).filter((f) => f.endsWith(".sh"))) { const text = readFileSync(join(guards, f), "utf8"); - for (const m of text.matchAll(/\$\{?([A-Z_][A-Z0-9_]*)/g)) vars.add(m[1]); + // Only forge-contract prefixes: a guard's own ALL-CAPS locals ($INPUT, $DIR) + // are shell internals, not env surface the docs owe anyone. + for (const m of text.matchAll( + /\$\{?((?:FORGE|ANTHROPIC|LITELLM|OPENROUTER|ENABLE_CORTEX|CLAUDE)_[A-Z0-9_]*)/g, + )) + vars.add(m[1]); } } return vars; From fafaafbc4ad3bf482849f0736529cace2b68286b Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 06:57:42 +0000 Subject: [PATCH 8/9] fix: harden the new math/detection layers per adversarial review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An 8-angle review of the branch surfaced verified defects; all fixed with regression tests: Secret detection (the critical class — false positives corrupted tool output): - TOKEN_RE no longer includes '/', so file paths split into segments instead of fusing into one high-entropy token (src/components/UserProfileCard2/index.js was detected as a secret and redacted to '[REDACTED].js'). - The entropy gate additionally requires >=3 scattered digit runs: long camelCase identifiers with a lone digit clear 3.9 bits/char (measured), random keys have scattered digits — the discriminator entropy alone could not provide. - Redaction of assigned values is now narrower than detection: only quoted or opaque-token values mask, so reading 'const token = jwt.sign(payload, key)' is never mangled. Detection stays broad for store refusals. - PEM masking is case-insensitive and tolerates truncated headers. - The guard prefilters with grep before spawning node, and the node reader uses setEncoding('utf8') so multibyte chars split across chunks can't corrupt output. Routing: - strongConf 0.5 -> 0.35 plus short-phrasing exemplars (deadlock in a worker pool, race condition in a queue, back pressure, schema migration): the LLM lower-bound floor holds again on every review counterexample. - Prose sequence markers ('and then', 'after that') count toward nSteps. - Exemplar gram footprints precomputed at module load (hooks route per prompt). Lessons: generic path tokens (src, js, index, ...) are excluded from keyword matching — a lesson keyed to src/auth/login.js no longer matches every .js edit. Preflight/substrate: dead regex alternatives fixed ('scal' never matched 'scalable'); authentication/payment/billing now imply the constraints dimension; the minimality warning is worded to what the signal means; completeness guarded. Structure: MCP tool registry moved to src/mcp_tools.js (pure data), breaking the doctor -> docs_check -> cortex_mcp -> doctor import cycle; docs_check scans src/ recursively and follows BRAND.cli; generated/churning docs (AGENTS, CHANGELOG) excluded from the atlas so auto-sync can't perpetually re-stale it; DOC_EXTS replaces the empty-array sentinel; the pre-edit hook strips the root prefix only at a path-separator boundary; new user-facing strings use brand tokens; dead math exports removed. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- docs/GUIDE.md | 4 +- global/guards/secret-redact.sh | 9 ++ src/atlas.js | 28 +++-- src/cli.js | 4 +- src/cortex_hook_main.js | 6 +- src/cortex_mcp.js | 196 +-------------------------------- src/docs_check.js | 16 ++- src/doctor.js | 2 +- src/goal.js | 7 +- src/lessons.js | 14 ++- src/math.js | 39 +------ src/mcp_tools.js | 196 +++++++++++++++++++++++++++++++++ src/preflight.js | 9 +- src/route.js | 28 ++++- src/secrets.js | 38 +++++-- src/substrate.js | 9 +- test/docs_check.test.js | 2 +- test/lessons.test.js | 9 ++ test/math.test.js | 31 ++---- test/secrets.test.js | 45 ++++++-- 20 files changed, 389 insertions(+), 303 deletions(-) create mode 100644 src/mcp_tools.js diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 7d1d53e..4dcdcc3 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -688,7 +688,9 @@ Nothing to wire — the plugin's [`hooks/hooks.json`](../hooks/hooks.json) insta > `forge substrate "" --json` (or the MCP tool `substrate_check`). If > `okToProceed` is false, ask the questions first; read `impact.impactedFiles` before editing. -…and exposes the substrate as **19 MCP tools** any MCP-capable agent can call directly: +…and exposes the substrate as **19 MCP tools** any MCP-capable agent can call directly +(the stdio server is launched with `forge cortex-mcp`, wired automatically via the +emitted `.mcp.json`): diff --git a/global/guards/secret-redact.sh b/global/guards/secret-redact.sh index 356265a..4f9c8c4 100755 --- a/global/guards/secret-redact.sh +++ b/global/guards/secret-redact.sh @@ -12,6 +12,12 @@ INPUT="$(cat)" out="$(printf '%s' "$INPUT" | jq -r '.tool_response // .tool_output // empty' 2>/dev/null)" [ -n "$out" ] || exit 0 +# Fast prefilter: PostToolUse fires after EVERY tool call, and most outputs contain +# nothing remotely secret-shaped — skip the node spawn entirely unless a candidate +# (known credential prefix, PEM header, key-ish assignment, or a 20+ char token run) +# is present. The node pass then decides precisely. +printf '%s' "$out" | grep -qE -- '-----BEGIN |ghp_|github_pat_|sk-|xox[baprs]-|AIza|ya29\.|eyJ|AKIA|(api[_-]?key|secret|passwd|password|token)[A-Za-z0-9_-]*["'"'"']?[[:space:]]*[:=]|[A-Za-z0-9+=_-]{20,}' || exit 0 + # ~/.forge is a symlink to /global, so pwd -P lands inside the real tree in # both install modes (install.sh symlink and CLAUDE_PLUGIN_ROOT plugin checkout). DIR="$(cd "$(dirname "$0")" && pwd -P)" @@ -19,7 +25,10 @@ SECRETS_JS="$DIR/../../src/secrets.js" red="" if command -v node >/dev/null 2>&1 && [ -f "$SECRETS_JS" ]; then + # setEncoding: string-concatenating raw Buffers corrupts multibyte UTF-8 split + # across chunk boundaries, and the mangled text would be emitted as a rewrite. red="$(printf '%s' "$out" | node -e ' + process.stdin.setEncoding("utf8"); let raw = ""; process.stdin.on("data", (d) => { raw += d; }); process.stdin.on("end", async () => { diff --git a/src/atlas.js b/src/atlas.js index 6ff4f24..6a1418e 100644 --- a/src/atlas.js +++ b/src/atlas.js @@ -32,12 +32,19 @@ const RULES = { { re: /\b(?:struct|enum|trait)\s+([A-Za-z_]\w*)/g, kind: "type" }, ], ".java": [{ re: /\b(?:class|interface|enum)\s+([A-Za-z_]\w*)/g, kind: "type" }], - // Markdown carries no symbol definitions ([]), but extractDoc() below turns each - // doc into a graph node with `references` edges to the code it talks about — the - // missing code→doc half of impact: change a symbol, its docs show up as dependents. - ".md": [], }; +// Documentation extensions — first-class in the walk, extracted by extractDoc() (a +// doc node + `references` edges to the code it names), never by the symbol RULES. +// This is the missing code→doc half of impact: change a symbol, its docs show up +// as dependents. +const DOC_EXTS = new Set([".md"]); + +// Docs excluded from the graph: generated files the Stop hook rewrites (AGENTS.md +// auto-sync would re-stale the atlas after every session) and the changelog, which +// churns on every change and whose references describe HISTORY, not current code. +const DOC_SKIP = /^(AGENTS|CLAUDE|GEMINI|CHANGELOG)\.md$/i; + const IMPORT_RE = /(?:import\s+(?:[^"'\n]+\s+from\s+)?["']([^"']+)["']|require\(["']([^"']+)["']\)|^\s*(?:from\s+([\w.]+)\s+)?import\s+([\w*,\s]+))/gm; const BUILTINS = new Set([ @@ -98,7 +105,11 @@ function walk(dir, files, cap) { continue; } if (st.isDirectory()) walk(path, files, cap); - else if (RULES[extname(name)] && files.length < cap) files.push(path); + else if ( + (RULES[extname(name)] || (DOC_EXTS.has(extname(name)) && !DOC_SKIP.test(name))) && + files.length < cap + ) + files.push(path); } } @@ -137,7 +148,7 @@ function extractDoc(rel, text) { for (const raw of m[1].trim().split(/\s+/)) { const tok = raw.replace(/[(),;:]+$/, "").replace(/^\.\//, ""); if (!tok) continue; - if (/[/\\]/.test(tok) && RULES[extname(tok)] && extname(tok) !== ".md") { + if (/[/\\]/.test(tok) && RULES[extname(tok)]) { refEdge(`module:${moduleId(tok)}`, 0.8, line); // `src/foo.js` → its module node } else if (/^[A-Za-z_$][\w$]*(\(\))?$/.test(tok) && tok.length >= 3) { const name = tok.replace(/\(\)$/, ""); @@ -148,8 +159,7 @@ function extractDoc(rel, text) { for (const m of text.matchAll(/\]\(([^)#\s]+)\)/g)) { const tok = m[1].replace(/^\.\//, ""); if (/^[a-z]+:/i.test(tok)) continue; // external URL, not a repo path - if (RULES[extname(tok)] && extname(tok) !== ".md") - refEdge(`module:${moduleId(tok)}`, 0.8, lineOf(text, m.index)); + if (RULES[extname(tok)]) refEdge(`module:${moduleId(tok)}`, 0.8, lineOf(text, m.index)); } return { symbols: [], nodes: [doc], edges, hash: hash(text) }; } @@ -166,7 +176,7 @@ function extractFile(path, root, preRead) { return { symbols: [], nodes: [], edges: [], hash: "" }; } } - if (ext === ".md") return extractDoc(rel, text); + if (DOC_EXTS.has(ext)) return extractDoc(rel, text); const mod = { id: `module:${moduleId(rel)}`, diff --git a/src/cli.js b/src/cli.js index 7bf4fe1..52af723 100755 --- a/src/cli.js +++ b/src/cli.js @@ -1092,7 +1092,7 @@ async function run(argv) { if (sub === "set") { const r = setGoal(process.cwd(), args.slice(1).join(" ")); if (!r.ok) { - console.error(`forge anchor set: ${r.reason}`); + console.error(`${BRAND.cli} anchor set: ${r.reason}`); process.exitCode = 1; return; } @@ -1114,7 +1114,7 @@ async function run(argv) { const goal = args.join(" ") || getGoal(process.cwd()); if (!goal) { console.error( - 'usage: forge anchor "" [--json]\n forge anchor set|show|clear — persist the goal across sessions', + `usage: ${BRAND.cli} anchor "" [--json]\n ${BRAND.cli} anchor set|show|clear — persist the goal across sessions`, ); process.exitCode = 1; return; diff --git a/src/cortex_hook_main.js b/src/cortex_hook_main.js index b16f5da..54b3145 100644 --- a/src/cortex_hook_main.js +++ b/src/cortex_hook_main.js @@ -151,7 +151,11 @@ async function staleDocsAdvisory(root, file) { const { load, impact } = await import("./atlas.js"); const atlas = load(root); if (!atlas) return ""; - const rel = file.startsWith(root) ? file.slice(root.length).replace(/^[/\\]/, "") : file; + // root + separator, not a bare prefix: '/home/u/repo' must not strip '/home/u/repository'. + const rel = + file.startsWith(`${root}/`) || file.startsWith(`${root}\\`) + ? file.slice(root.length + 1) + : file; const docs = impact(atlas, rel, { maxHops: 2 }).impactedFiles.filter((f) => f.endsWith(".md")); if (!docs.length) return ""; return `Forge impact — docs that reference ${rel}: ${docs.slice(0, 5).join(", ")}. If this change alters behavior, update them in the same pass (\`forge impact ${rel}\` for the full list).`; diff --git a/src/cortex_mcp.js b/src/cortex_mcp.js index 0f8d472..0d1a5e2 100644 --- a/src/cortex_mcp.js +++ b/src/cortex_mcp.js @@ -28,199 +28,11 @@ const PKG_VERSION = (() => { const root = process.env.FORGE_ROOT || process.cwd(); const today = epochDay; -// Exported so docs_check.js reconciles the documented MCP tool list against the -// registry that actually serves — a tool can't ship undocumented (or be documented -// after removal) without `forge docs check` flagging it. -export const TOOLS = [ - { - name: "cortex_lessons", - description: - "Lessons Forge Cortex learned from past mistakes on THIS repo, for the given files/symbols. Background context — verify before acting, don't blindly obey.", - inputSchema: { - type: "object", - properties: { - files: { - type: "array", - items: { type: "string" }, - description: "file paths in play", - }, - symbols: { - type: "array", - items: { type: "string" }, - description: "symbol names in play", - }, - }, - }, - }, - { - name: "cortex_status", - description: "Summary of learned lessons on this repo (counts by state, top by confidence).", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "preflight_check", - description: - "BEFORE starting a task, check what it names that the repo doesn't define — the things you'd otherwise ASSUME. Returns a clarify list or an all-clear. Ask instead of assuming.", - inputSchema: { - type: "object", - properties: { task: { type: "string", description: "the task/prompt" } }, - required: ["task"], - }, - }, - { - name: "route_task", - description: - "Recommend the cheapest CAPABLE model for a task by code-task complexity (files, fan-out, churn, past mistakes, ambiguity). Advisory — don't burn a top model on a trivial task.", - inputSchema: { - type: "object", - properties: { task: { type: "string" } }, - required: ["task"], - }, - }, +// The tool registry lives in mcp_tools.js as pure data — docs_check.js reconciles +// docs against it without importing this server. Re-exported for compatibility. +import { TOOLS } from "./mcp_tools.js"; - { - name: "assumption_gate", - description: - "Score specification completeness before work starts. Returns shouldAsk, risk, missing dimensions, and concrete questions.", - inputSchema: { - type: "object", - properties: { task: { type: "string", description: "the task/prompt" } }, - required: ["task"], - }, - }, - { - name: "predict_impact", - description: - "Predict blast radius for a symbol or file using Forge atlas reverse-dependency traversal.", - inputSchema: { - type: "object", - properties: { - target: { type: "string", description: "symbol name, qualified name, or file" }, - threshold: { type: "number", description: "confidence threshold, default 0.1" }, - }, - required: ["target"], - }, - }, - { - name: "substrate_check", - description: - "Full Forge cognitive-substrate pre-action check: assumption gate, route, impact, scope, memory, minimality, and verification checklist.", - inputSchema: { - type: "object", - properties: { task: { type: "string", description: "the task/prompt" } }, - required: ["task"], - }, - }, - { - name: "scope_files", - description: - "Decompose files into INDEPENDENT clusters (run as separate sessions) vs coupled, and surface coupled files you didn't name (the 'forgot the related module' guard).", - inputSchema: { - type: "object", - properties: { files: { type: "array", items: { type: "string" } } }, - required: ["files"], - }, - }, - { - name: "forge_cost", - description: - "Cost report — measured stage factors (gate, cache, route, context) from .forge/metrics.jsonl with multiplicative composition.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "forge_dash_data", - description: - "Dashboard JSON payload — ledger stats, metrics, atlas info. Same data forge dash serves at /api/data.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "forge_dash_summary", - description: - "Lightweight dashboard health check — just counts (claims, tombstoned, contested, atlas built, metric events). Cheaper than forge_dash_data.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "forge_brain", - description: "Project memory index — list all remembered facts stored in .forge/brain/.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "forge_ledger_query", - description: - "Query the proof-carrying memory ledger with a natural language query. Returns ranked matching claims.", - inputSchema: { - type: "object", - properties: { - query: { type: "string", description: "what you are about to do or looking for" }, - }, - required: ["query"], - }, - }, - { - name: "forge_diagnose", - description: - "Doom-loop check — record a failure and check if the same signature has recurred (3x = escalation). Prevents thrashing.", - inputSchema: { - type: "object", - properties: { - errorText: { type: "string", description: "the error message" }, - file: { type: "string", description: "file where the error occurred" }, - symbol: { type: "string", description: "symbol involved" }, - }, - required: ["errorText"], - }, - }, - { - name: "forge_doctor", - description: - "Health check — verify installed tools, guards, MCP auth, config drift, and system state.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "forge_provider_status", - description: - "Provider detection — which API provider is active (auto-detected or configured), env vars set, and health checks.", - inputSchema: { type: "object", properties: {} }, - }, - { - name: "forge_remember", - description: - "Store a durable fact in this repo's portable memory (.forge/brain/). Use for non-obvious, lasting knowledge (env quirks, decisions, gotchas).", - inputSchema: { - type: "object", - properties: { - name: { type: "string", description: "short slug for the fact (used as filename)" }, - body: { type: "string", description: "the fact content (markdown)" }, - }, - required: ["name", "body"], - }, - }, - { - name: "forge_ledger_ratify", - description: - "Promote a ledger claim's confidence — record an independent oracle ratification (the claim held under test).", - inputSchema: { - type: "object", - properties: { - id: { type: "string", description: "claim ID or unique prefix" }, - }, - required: ["id"], - }, - }, - { - name: "forge_ledger_retract", - description: - "Tombstone a ledger claim with a reason — mark it as no longer valid so it stops influencing routing and memory.", - inputSchema: { - type: "object", - properties: { - id: { type: "string", description: "claim ID or unique prefix" }, - reason: { type: "string", description: "why the claim is being retracted" }, - }, - required: ["id", "reason"], - }, - }, -]; +export { TOOLS }; async function callTool(name, args = {}) { if (name === "cortex_lessons") { diff --git a/src/docs_check.js b/src/docs_check.js index 10daf16..d7acd7b 100644 --- a/src/docs_check.js +++ b/src/docs_check.js @@ -9,7 +9,7 @@ import { existsSync, readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; import { BRAND } from "./brand.js"; import { COMMANDS, HIDDEN_COMMANDS } from "./commands.js"; -import { TOOLS } from "./cortex_mcp.js"; +import { TOOLS } from "./mcp_tools.js"; /** The user-facing prose docs every claim is reconciled against. */ const DOC_FILES = ["README.md", "docs/GUIDE.md", "ARCHITECTURE.md", "ROADMAP.md"]; @@ -30,7 +30,9 @@ function readDoc(root, rel) { function srcFiles(root) { const dir = join(root, "src"); if (!existsSync(dir)) return []; - return readdirSync(dir) + // recursive: src/emit/*.js reads are part of the same env contract. + return readdirSync(dir, { recursive: true }) + .map(String) .filter((f) => f.endsWith(".js")) .map((f) => join(dir, f)); } @@ -65,23 +67,25 @@ function checkCommands(docs, issues) { const text = docs[target]; if (!text) continue; for (const name of Object.keys(COMMANDS)) { - if (!new RegExp(`\\bforge ${name}\\b`).test(text)) { + // BRAND.cli, not a literal: after a rebrand the docs say ` ` and + // this check must follow them or every command reports as undocumented. + if (!new RegExp(`\\b${BRAND.cli} ${name}\\b`).test(text)) { issues.push({ check: "commands", severity: "error", - detail: `\`forge ${name}\` is implemented but ${target} never mentions it`, + detail: `\`${BRAND.cli} ${name}\` is implemented but ${target} never mentions it`, }); } } } for (const [file, text] of Object.entries(docs)) { - for (const m of text.matchAll(/`forge ([a-z][a-z-]*)\b/g)) { + for (const m of text.matchAll(new RegExp(`\`${BRAND.cli} ([a-z][a-z-]*)\\b`, "g"))) { const name = m[1]; if (!(name in COMMANDS) && !HIDDEN_COMMANDS.includes(name)) { issues.push({ check: "commands", severity: "error", - detail: `${file} documents \`forge ${name}\` but no such command exists`, + detail: `${file} documents \`${BRAND.cli} ${name}\` but no such command exists`, }); } } diff --git a/src/doctor.js b/src/doctor.js index fc70a86..0d6ddab 100644 --- a/src/doctor.js +++ b/src/doctor.js @@ -318,7 +318,7 @@ function checkDocs(out, targetRoot) { out.push( r.ok ? ok("docs↔code", "commands, env vars, MCP tools, CHANGELOG all agree") - : warn("docs↔code", `${r.issues.length} drift issue(s) — run \`forge docs check\``), + : warn("docs↔code", `${r.issues.length} drift issue(s) — run \`${BRAND.cli} docs check\``), ); } catch {} } diff --git a/src/goal.js b/src/goal.js index 6e25bed..6535df2 100644 --- a/src/goal.js +++ b/src/goal.js @@ -4,6 +4,7 @@ // goalDrift default to it, and makes "what are we actually doing" survive context loss. import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { BRAND } from "./brand.js"; import { hasSecret } from "./secrets.js"; const goalPath = (root) => join(root, ".forge", "goal.md"); @@ -17,7 +18,7 @@ export function setGoal(root, text, { t = Date.now() } = {}) { mkdirSync(join(root, ".forge"), { recursive: true }); writeFileSync( goalPath(root), - `# Goal\n\n${goal}\n\n\n`, + `# Goal\n\n${goal}\n\n\n`, ); return { ok: true, goal }; } @@ -51,9 +52,9 @@ export function goalBlock(root) { const goal = getGoal(root); if (!goal) return ""; return [ - "## Active goal (Forge Anchor)", + `## Active goal (${BRAND.brand} Anchor)`, `The stated goal for current work in this repo: **${goal}**`, - "Stay on it. `forge anchor` checks your changes against it; `forge anchor clear` when done.", + `Stay on it. \`${BRAND.cli} anchor\` checks your changes against it; \`${BRAND.cli} anchor clear\` when done.`, "", ].join("\n"); } diff --git a/src/lessons.js b/src/lessons.js index ae1437d..7e9c191 100644 --- a/src/lessons.js +++ b/src/lessons.js @@ -137,7 +137,17 @@ const globToRe = (glob) => { return new RegExp(`^${body}$`); }; -/** Token footprint of a keyword list ("src/auth/login.js" → {src, auth, login, js}) +// Tokens every path shares — matching on them would make a lesson keyed to +// src/auth/login.js "relevant" to every .js file under src/ (review-verified bug: +// the shared {src, js} tokens alone scored 0.15, injecting unrelated lessons and +// mis-attributing outcomes). Extensions and layout boilerplate carry no topic. +const GENERIC_PATH_TOKENS = new Set( + "src lib app test tests spec specs index main utils util js ts jsx tsx mjs cjs py go rs java rb php md json yml yaml".split( + " ", + ), +); + +/** Content-token footprint of a keyword list ("src/auth/login.js" → {auth, login}) * so two keywords about the same module overlap even when the strings differ. */ const keywordGrams = (words) => { const grams = new Set(); @@ -145,7 +155,7 @@ const keywordGrams = (words) => { for (const t of String(w) .toLowerCase() .split(/[^a-z0-9]+/)) { - if (t.length > 1) grams.add(t); + if (t.length > 1 && !GENERIC_PATH_TOKENS.has(t)) grams.add(t); } } return grams; diff --git a/src/math.js b/src/math.js index cfb253c..a80ea4b 100644 --- a/src/math.js +++ b/src/math.js @@ -2,9 +2,9 @@ // Forge's rule for heuristics: DATA may be a table (exemplars, oracle weights, // policy lists) but DECISIONS must be a formula — graded, inspectable, testable. // The similarity family (shingles/sketch/jaccard) lives in ledger.js; the decay/ -// posterior family in lessons.js/ledger.js; this module holds the primitives that -// had no home: entropy (secret detection) and exact set overlap (small-set -// similarity, where MinHash estimation is unnecessary). +// posterior family in lessons.js/ledger.js; this module holds the two primitives +// that had no home: entropy (secret detection) and the overlap coefficient +// (task↔exemplar and keyword matching, where MinHash estimation is unnecessary). /** * Shannon entropy of a string in bits per character (code points). Empty → 0. @@ -30,39 +30,6 @@ export function shannonEntropy(s) { return h; } -/** - * Number of character classes present (lowercase, uppercase, digit, other) — a - * cheap charset-diversity signal: machine-generated tokens mix classes, natural - * words rarely use more than two. - * @param {string} s - * @returns {number} 0..4 - */ -export function charsetClasses(s) { - const str = String(s); - let classes = 0; - if (/[a-z]/.test(str)) classes++; - if (/[A-Z]/.test(str)) classes++; - if (/[0-9]/.test(str)) classes++; - if (/[^a-zA-Z0-9]/.test(str)) classes++; - return classes; -} - -/** - * Exact Jaccard similarity of two Sets: |A∩B| / |A∪B|. Both empty → 0. - * For small sets (task shingles vs an exemplar) this is exact and cheaper than - * the MinHash estimate in ledger.js, which exists for scale. - * @param {Set} a - * @param {Set} b - * @returns {number} 0..1 - */ -export function setJaccard(a, b) { - if (!a.size && !b.size) return 0; - let inter = 0; - const [small, large] = a.size <= b.size ? [a, b] : [b, a]; - for (const x of small) if (large.has(x)) inter++; - return inter / (a.size + b.size - inter); -} - /** * Overlap coefficient of two Sets: |A∩B| / min(|A|,|B|). Either empty → 0. * Containment-friendly where Jaccard is size-penalized: a short exemplar fully diff --git a/src/mcp_tools.js b/src/mcp_tools.js new file mode 100644 index 0000000..b5b9bcf --- /dev/null +++ b/src/mcp_tools.js @@ -0,0 +1,196 @@ +// forge mcp tools — the MCP tool REGISTRY as pure data (name/description/schema). +// Handlers stay in cortex_mcp.js (dispatch by name); keeping the registry data-only +// lets docs_check.js reconcile documented tools against it without importing the +// server (which imports doctor.js — that import cycle cost every doctor run the +// whole tool tree's startup and was one require-order bug away from breaking). + +export const TOOLS = [ + { + name: "cortex_lessons", + description: + "Lessons Forge Cortex learned from past mistakes on THIS repo, for the given files/symbols. Background context — verify before acting, don't blindly obey.", + inputSchema: { + type: "object", + properties: { + files: { + type: "array", + items: { type: "string" }, + description: "file paths in play", + }, + symbols: { + type: "array", + items: { type: "string" }, + description: "symbol names in play", + }, + }, + }, + }, + { + name: "cortex_status", + description: "Summary of learned lessons on this repo (counts by state, top by confidence).", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "preflight_check", + description: + "BEFORE starting a task, check what it names that the repo doesn't define — the things you'd otherwise ASSUME. Returns a clarify list or an all-clear. Ask instead of assuming.", + inputSchema: { + type: "object", + properties: { task: { type: "string", description: "the task/prompt" } }, + required: ["task"], + }, + }, + { + name: "route_task", + description: + "Recommend the cheapest CAPABLE model for a task by code-task complexity (files, fan-out, churn, past mistakes, ambiguity). Advisory — don't burn a top model on a trivial task.", + inputSchema: { + type: "object", + properties: { task: { type: "string" } }, + required: ["task"], + }, + }, + + { + name: "assumption_gate", + description: + "Score specification completeness before work starts. Returns shouldAsk, risk, missing dimensions, and concrete questions.", + inputSchema: { + type: "object", + properties: { task: { type: "string", description: "the task/prompt" } }, + required: ["task"], + }, + }, + { + name: "predict_impact", + description: + "Predict blast radius for a symbol or file using Forge atlas reverse-dependency traversal.", + inputSchema: { + type: "object", + properties: { + target: { type: "string", description: "symbol name, qualified name, or file" }, + threshold: { type: "number", description: "confidence threshold, default 0.1" }, + }, + required: ["target"], + }, + }, + { + name: "substrate_check", + description: + "Full Forge cognitive-substrate pre-action check: assumption gate, route, impact, scope, memory, minimality, and verification checklist.", + inputSchema: { + type: "object", + properties: { task: { type: "string", description: "the task/prompt" } }, + required: ["task"], + }, + }, + { + name: "scope_files", + description: + "Decompose files into INDEPENDENT clusters (run as separate sessions) vs coupled, and surface coupled files you didn't name (the 'forgot the related module' guard).", + inputSchema: { + type: "object", + properties: { files: { type: "array", items: { type: "string" } } }, + required: ["files"], + }, + }, + { + name: "forge_cost", + description: + "Cost report — measured stage factors (gate, cache, route, context) from .forge/metrics.jsonl with multiplicative composition.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "forge_dash_data", + description: + "Dashboard JSON payload — ledger stats, metrics, atlas info. Same data forge dash serves at /api/data.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "forge_dash_summary", + description: + "Lightweight dashboard health check — just counts (claims, tombstoned, contested, atlas built, metric events). Cheaper than forge_dash_data.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "forge_brain", + description: "Project memory index — list all remembered facts stored in .forge/brain/.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "forge_ledger_query", + description: + "Query the proof-carrying memory ledger with a natural language query. Returns ranked matching claims.", + inputSchema: { + type: "object", + properties: { + query: { type: "string", description: "what you are about to do or looking for" }, + }, + required: ["query"], + }, + }, + { + name: "forge_diagnose", + description: + "Doom-loop check — record a failure and check if the same signature has recurred (3x = escalation). Prevents thrashing.", + inputSchema: { + type: "object", + properties: { + errorText: { type: "string", description: "the error message" }, + file: { type: "string", description: "file where the error occurred" }, + symbol: { type: "string", description: "symbol involved" }, + }, + required: ["errorText"], + }, + }, + { + name: "forge_doctor", + description: + "Health check — verify installed tools, guards, MCP auth, config drift, and system state.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "forge_provider_status", + description: + "Provider detection — which API provider is active (auto-detected or configured), env vars set, and health checks.", + inputSchema: { type: "object", properties: {} }, + }, + { + name: "forge_remember", + description: + "Store a durable fact in this repo's portable memory (.forge/brain/). Use for non-obvious, lasting knowledge (env quirks, decisions, gotchas).", + inputSchema: { + type: "object", + properties: { + name: { type: "string", description: "short slug for the fact (used as filename)" }, + body: { type: "string", description: "the fact content (markdown)" }, + }, + required: ["name", "body"], + }, + }, + { + name: "forge_ledger_ratify", + description: + "Promote a ledger claim's confidence — record an independent oracle ratification (the claim held under test).", + inputSchema: { + type: "object", + properties: { + id: { type: "string", description: "claim ID or unique prefix" }, + }, + required: ["id"], + }, + }, + { + name: "forge_ledger_retract", + description: + "Tombstone a ledger claim with a reason — mark it as no longer valid so it stops influencing routing and memory.", + inputSchema: { + type: "object", + properties: { + id: { type: "string", description: "claim ID or unique prefix" }, + reason: { type: "string", description: "why the claim is being retracted" }, + }, + required: ["id", "reason"], + }, + }, +]; diff --git a/src/preflight.js b/src/preflight.js index 9a481d3..28fd935 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -49,7 +49,7 @@ const DIMENSIONS = [ description: "target file/module/component scope", question: "Which specific file, module, component, or symbol should this change touch?", applies: rx( - "\\b(fix|change|edit|update|modify|refactor|add to|remove from|integrate|wire|bug|issue|error)\\b", + "\\b(fix|change|edit|update|modify|refactor|rewrite|redesign|clean|optimi[sz]e|improve|add to|remove from|integrate|wire|bug|issue|error)\\b", ), cues: rx( "\\b(file|module|class|function|component|path|directory|service|layer|endpoint)\\b|`[\\w./-]+`|\\w+\\.\\w+", @@ -72,7 +72,12 @@ const DIMENSIONS = [ description: "hard constraints", question: "What constraints must be respected: performance, dependencies, style, or compatibility?", - applies: rx("\\b(design|architect|production|scal|migrate|distributed|concurren|refactor)\\b"), + // \w* because users type "scalable"/"concurrent"/"migration" — a bare "scal\b" + // alternative never matched anything (review-verified dead branch). auth/payment + // are here because money- and identity-touching work implies constraints. + applies: rx( + "\\b(design|architect|production|scal\\w*|migrat\\w*|distribut\\w*|concurren\\w*|refactor|authenticat\\w*|authoriz\\w*|payment|billing)\\b", + ), cues: rx( "\\b(must|should|constraint|limit|no new dependenc|only use|standard library|without|performance|latency|O\\(|backward|compatib|convention|style)\\b", ), diff --git a/src/route.js b/src/route.js index cedb96a..2b37a79 100644 --- a/src/route.js +++ b/src/route.js @@ -70,9 +70,15 @@ export const EXEMPLARS = [ { text: "implement a rate limiter with a token bucket", y: 0.78 }, { text: "write a recursive descent parser for a grammar", y: 0.78 }, { text: "resolve a deadlock between concurrent threads", y: 0.78 }, + { text: "fix a deadlock in a worker pool", y: 0.78 }, + { text: "fix the race condition in a queue", y: 0.78 }, { text: "distributed consensus and replication protocol", y: 0.78 }, { text: "cryptographic signing and verification flow", y: 0.78 }, - { text: "producer consumer blocking queue with backpressure", y: 0.78 }, + // spaced form on purpose: contentGrams splits "back-pressure" into two tokens, + // so the exemplar must carry the split form for the bigram to line up. + { text: "producer consumer blocking queue with back pressure", y: 0.78 }, + { text: "handle back pressure in a stream pipeline", y: 0.78 }, + { text: "run a database schema migration", y: 0.7 }, { text: "state machine with invariants and transitions", y: 0.78 }, { text: "numerical stability of a floating point computation", y: 0.78 }, { text: "np-hard optimization with a heuristic search", y: 0.78 }, @@ -125,11 +131,18 @@ export const RUBRIC = { prior: 0.15, // no-signal complexity (the old "base cost of any task") confSat: 0.5, // top similarity at which the estimate earns full weight strongScore: 0.65, // k-NN estimate marking a confidently-hard task (LLM lower-bound floor) - strongConf: 0.5, + // Calibrated against short phrasings: "fix the race condition in the worker pool" + // overlaps its exemplar at ~0.43 (extra scope words dilute the coefficient), and + // the floor MUST hold there — 0.5 let a bad LLM vote talk concurrency work down. + strongConf: 0.35, bands: { cheap: 0.3, mid: 0.6 }, // score < cheap → cheap; ≤ mid → mid; else premium struct: { codeContext: 0.05, length: 0.1, constraints: 0.05, steps: 0.05 }, }; +// Exemplar footprints are static — compute once, not per routeTask call (the ambient +// hook routes on every prompt; re-tokenizing 50 exemplars each time was pure waste). +const EXEMPLAR_GRAMS = EXEMPLARS.map((e) => ({ ...e, grams: contentGrams(e.text) })); + /** Structural (non-topic) features of the task text — countable, graded inputs. */ export function rubricSignals(task = "") { const text = String(task); @@ -140,7 +153,11 @@ export function rubricSignals(task = "") { // feeding a saturating weight — feature extraction, not a classification. nConstraints: (text.match(/(^\s*[-*\d.]|\b(must|should|ensure|require|constraint)\b)/gim) || []) .length, - nSteps: (text.match(/^\s*\d+[.)]\s/gm) || []).length, + // Sequencing markers: numbered-list lines plus prose connectives ("and then", + // "after that") — multi-step requests carry complexity the topic estimate misses. + nSteps: + (text.match(/^\s*\d+[.)]\s/gm) || []).length + + (text.match(/\b(and then|after that|step \d)\b/gi) || []).length, }; } @@ -152,7 +169,10 @@ export function rubricSignals(task = "") { export function rubricComplexity(task = "") { const sig = rubricSignals(task); const grams = contentGrams(task); - const neighbors = EXEMPLARS.map((e) => ({ ...e, sim: setOverlap(grams, contentGrams(e.text)) })) + const neighbors = EXEMPLAR_GRAMS.map(({ grams: eg, ...e }) => ({ + ...e, + sim: setOverlap(grams, eg), + })) .sort((a, b) => b.sim - a.sim) .slice(0, RUBRIC.k) .filter((n) => n.sim > 0); diff --git a/src/secrets.js b/src/secrets.js index dd154a0..9131120 100644 --- a/src/secrets.js +++ b/src/secrets.js @@ -38,26 +38,34 @@ const ASSIGNED = `\\b[\\w-]*${KEYISH}[\\w-]*["']?\\s*[:=]\\s*["']?\\S`; * New code should call hasSecret(), which adds the entropy gate. */ export const SECRET_RE = new RegExp(`(${[...FORMATS, ASSIGNED].join("|")})`, "i"); -// (ii) Entropy gate thresholds. 20 chars is below every real credential length but -// above almost all identifiers; 3.9 bits/char sits between English-like tokens -// (camelCase identifiers ≈ 3.5–3.8 even with digits) and sampled random base64/62 -// tokens (≥ 4.2 at 20+ chars). Both are exported so tests pin the calibration. +// (ii) Entropy gate thresholds, exported so tests pin the calibration. Entropy alone +// cannot separate long camelCase identifiers from keys (both clear 4 bits/char at +// 25+ chars — measured, not assumed), so the gate also requires SCATTERED digits: +// ≥3 separate digit runs. Random 62-alphabet tokens have ~16% digits spread +// throughout (P(<3 runs at 20+ chars) is small); identifiers put digits in one or +// two lumps (`UserProfileCard2`, `convertBase64ToUtf8`). Precision first — a rare +// low-digit credential slipping past this gate still hits the format grammars. export const ENTROPY_MIN_LEN = 20; export const ENTROPY_MIN_BITS = 3.9; +export const ENTROPY_MIN_DIGIT_RUNS = 3; -// Candidate extraction: contiguous base64-class runs. Parsing, not a decision. -const TOKEN_RE = /[A-Za-z0-9+/=_-]{20,}/g; +// Candidate extraction: contiguous base64-class runs. Deliberately excludes `/` so a +// file path splits into segments instead of scoring as one token — paths were the #1 +// false positive (a redacted path corrupts the very tool output the guard protects). +const TOKEN_RE = /[A-Za-z0-9+=_-]{20,}/g; /** - * Is this bare token secret-shaped by math alone? Requires all three of: length, - * mixed charset (lower AND upper AND digit — excludes hex/UUID/camelCase words - * without digits), and near-random Shannon entropy. + * Is this bare token secret-shaped by math alone? Requires all of: length, mixed + * charset (lower AND upper AND digit — excludes hex/UUID/camelCase-without-digits), + * ≥3 scattered digit runs (excludes identifiers with a lone version/counter digit), + * and near-random Shannon entropy. * @param {string} tok */ export function isHighEntropyToken(tok) { const s = String(tok); if (s.length < ENTROPY_MIN_LEN) return false; if (!(/[a-z]/.test(s) && /[A-Z]/.test(s) && /[0-9]/.test(s))) return false; + if ((s.match(/[0-9]+/g) || []).length < ENTROPY_MIN_DIGIT_RUNS) return false; return shannonEntropy(s) >= ENTROPY_MIN_BITS; } @@ -76,10 +84,18 @@ export function hasSecret(text) { // Redaction machinery — used by the secret-redact guard (via node import) and any // JS caller that wants to keep surrounding text. PEM blocks are masked whole; // assigned values keep their key (context stays readable, value is gone). -const PEM_BLOCK_G = /-----BEGIN [A-Z ]*-----[\s\S]*?(?:-----END [A-Z ]*-----|$)/g; +// Case-insensitive and tolerant of a truncated header/footer — hasSecret's PEM +// branch is case-insensitive too, and a detected-but-unredacted block would leak +// straight through the guard ("one truth, two verbs" means these must agree). +const PEM_BLOCK_G = /-----BEGIN [\s\S]*?(?:-----END [^\n-]*-----|$)/gi; const FORMAT_G = new RegExp(FORMATS.slice(1).join("|"), "gi"); +// Redaction is deliberately NARROWER than detection here: detection (SECRET_RE's +// ASSIGNED branch) refuses on any assigned value — cheap and conservative for a +// store. Redaction rewrites live tool output, so it only masks values that look +// like opaque tokens (quoted, or an 8+ char credential-class run) — never a code +// expression: reading `const token = jwt.sign(payload, key)` must NOT be mangled. const ASSIGNED_G = new RegExp( - `(\\b[\\w-]*${KEYISH}[\\w-]*["']?\\s*[:=]\\s*["']?)([^\\s"']+)`, + `(\\b[\\w-]*${KEYISH}[\\w-]*["']?\\s*[:=]\\s*)("[^"\\n]{4,}"|'[^'\\n]{4,}'|[A-Za-z0-9+=_-]{8,}(?![\\w(]))`, "gi", ); diff --git a/src/substrate.js b/src/substrate.js index b7f28f4..62ab9d6 100644 --- a/src/substrate.js +++ b/src/substrate.js @@ -53,13 +53,18 @@ function minimalityWarnings(_task, route, preflight) { "High-risk broad change with no target files named; ask for scope before editing.", ); } - if (route.score >= 0.55 && preflight.assumption.completeness < 0.7) { + if (route.score >= 0.55 && (preflight.assumption?.completeness ?? 1) < 0.7) { warnings.push( "Complex task with medium/low specification completeness; clarify before spending a premium model.", ); } + // Worded to what the signal actually means: the constraints DIMENSION applies to + // design/refactor work too, not only production systems — an overclaiming + // "production-sensitive!" on a casual refactor teaches users to ignore warnings. if (missing.has("constraints")) { - warnings.push("Production-sensitive task lacks explicit constraints or acceptance criteria."); + warnings.push( + "Task implies constraints (design/production/auth/payment-class work) but states none — name performance, compatibility, or rollback expectations.", + ); } return warnings; } diff --git a/test/docs_check.test.js b/test/docs_check.test.js index 313c9b5..9a94417 100644 --- a/test/docs_check.test.js +++ b/test/docs_check.test.js @@ -4,8 +4,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { COMMANDS } from "../src/commands.js"; -import { TOOLS } from "../src/cortex_mcp.js"; import { docsCheck, envVarsRead } from "../src/docs_check.js"; +import { TOOLS } from "../src/mcp_tools.js"; // A fixture tree whose docs are generated FROM the real registries, so it passes by // construction — each test then breaks exactly one claim and asserts the reconciler diff --git a/test/lessons.test.js b/test/lessons.test.js index 412e06f..b8bacd9 100644 --- a/test/lessons.test.js +++ b/test/lessons.test.js @@ -99,6 +99,15 @@ test("matchScore: keyword tier is graded — same-module partial overlap earns p assert.equal(unrelated, 0, "no shared tokens → no match"); }); +test("matchScore: generic path tokens (src, js, index) never create cross-module matches", () => { + // Regression: {src, js} alone must not make an auth lesson 'relevant' to billing. + const l = newLesson({ id: "n", trigger: { keywords: ["src/auth/login.js"] } }); + assert.equal(matchScore(l, { keywords: ["src/billing/invoice.js"] }), 0); + assert.equal(matchScore(l, { keywords: ["src/cortex_hook_main.js"] }), 0); + const cfg = newLesson({ id: "c", trigger: { keywords: ["config.js"] } }); + assert.equal(matchScore(cfg, { keywords: ["src/anything_else.js"] }), 0); +}); + test("selectForInjection: relevance-ranked, capped, overflow becomes a pointer (never silent)", () => { const ctx = { symbols: ["foo"], files: [], keywords: [] }; const lessons = Array.from({ length: 5 }, (_, i) => { diff --git a/test/math.test.js b/test/math.test.js index cee52a5..06a9acc 100644 --- a/test/math.test.js +++ b/test/math.test.js @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { charsetClasses, setJaccard, shannonEntropy } from "../src/math.js"; +import { setOverlap, shannonEntropy } from "../src/math.js"; test("shannonEntropy: empty and single-char strings", () => { assert.equal(shannonEntropy(""), 0); @@ -31,28 +31,13 @@ test("shannonEntropy: counts code points, not UTF-16 units", () => { assert.equal(shannonEntropy("😀😅"), 1); }); -test("charsetClasses: counts distinct character classes", () => { - assert.equal(charsetClasses(""), 0); - assert.equal(charsetClasses("hello"), 1); - assert.equal(charsetClasses("Hello"), 2); - assert.equal(charsetClasses("Hello1"), 3); - assert.equal(charsetClasses("Hello1!"), 4); - assert.equal(charsetClasses("12345"), 1); -}); - -test("setJaccard: identity, disjoint, empty, partial overlap", () => { - const abc = new Set(["a", "b", "c"]); - assert.equal(setJaccard(abc, new Set(["a", "b", "c"])), 1); - assert.equal(setJaccard(abc, new Set(["x", "y"])), 0); - assert.equal(setJaccard(new Set(), new Set()), 0); - assert.equal(setJaccard(abc, new Set()), 0); - // |{a,b}∩{b,c}| = 1, |∪| = 3 - assert.equal(setJaccard(new Set(["a", "b"]), new Set(["b", "c"])), 1 / 3); -}); - -test("setJaccard: symmetric regardless of argument order", () => { +test("setOverlap: containment, disjoint, empty, symmetric", () => { const small = new Set(["a"]); const large = new Set(["a", "b", "c", "d"]); - assert.equal(setJaccard(small, large), setJaccard(large, small)); - assert.equal(setJaccard(small, large), 0.25); + assert.equal(setOverlap(small, large), 1, "full containment of the smaller set → 1"); + assert.equal(setOverlap(large, small), 1, "symmetric"); + assert.equal(setOverlap(new Set(["x"]), large), 0); + assert.equal(setOverlap(new Set(), large), 0); + // |{a,b}∩{b,c}| = 1, min size 2 + assert.equal(setOverlap(new Set(["a", "b"]), new Set(["b", "c"])), 0.5); }); diff --git a/test/secrets.test.js b/test/secrets.test.js index 4ab4bec..e60457b 100644 --- a/test/secrets.test.js +++ b/test/secrets.test.js @@ -48,6 +48,36 @@ test("isHighEntropyToken: hex digests, UUIDs, identifiers, and prose are NOT sec assert.equal(isHighEntropyToken("parseHttpResponseHeaders2"), false); // Too short even if random. assert.equal(isHighEntropyToken("Zq7Rt2Xk9Lp4"), false); + // LONG camelCase with a lone digit clears 3.9 bits/char — the scattered-digit-runs + // requirement is what keeps these out (regression: review found them redacted). + assert.equal(isHighEntropyToken("TestSecretRedact_HandlesMultilineOutput2"), false); + assert.equal(isHighEntropyToken("AbstractSingletonProxyFactoryBean2"), false); + assert.equal(isHighEntropyToken("getUserAuthenticationTokenFromEnvironment2"), false); + assert.equal(isHighEntropyToken("convertBase64ToUtf8String"), false); +}); + +test("hasSecret/redactSecrets: file paths and source code are never secrets (regression)", () => { + // TOKEN_RE excludes '/', so a path splits into short segments instead of fusing + // into one high-entropy 'token' — the #1 false positive class. + const path = "src/components/UserProfileCard2/index.js"; + assert.equal(hasSecret(path), false); + assert.equal(redactSecrets(path), path); + assert.equal(hasSecret("wire up OAuth2 login in src/auth/OAuth2Provider"), false); + // Reading auth source must not be mangled: an assigned value that is a code + // expression is not an opaque token. + const code = "const token = jwt.sign(payload, key)"; + assert.equal(redactSecrets(code), code); + const ls = "ls: components/AuthFlow2/LoginForm.tsx"; + assert.equal(redactSecrets(ls), ls); +}); + +test("hasSecret/redactSecrets: PEM agrees case-insensitively (detect ⇒ redact)", () => { + const lower = "-----begin rsa private key-----\nMIIEowIBAAKCAQEA\n-----end rsa private key-----"; + assert.ok(hasSecret(lower)); + assert.equal(redactSecrets(lower).includes("MIIEowIBAAKCAQEA"), false); + const truncated = "-----BEGIN RSA PRIVATE KEY\nMIIEowIBAAKCAQEA"; + assert.ok(hasSecret(truncated)); + assert.equal(redactSecrets(truncated).includes("MIIEowIBAAKCAQEA"), false); }); test("redactSecrets: masks formats, keeps surrounding text", () => { @@ -86,18 +116,19 @@ test("redactSecrets: leaves git SHAs and UUIDs untouched (they are not secrets)" assert.equal(redactSecrets(line), line); }); -test("hasSecret and redactSecrets agree: anything redacted is detected", () => { +test("redaction implies detection: anything redactSecrets rewrites, hasSecret catches", () => { + // One-way by design: detection (a store refusal) is broader than redaction (a + // rewrite of live tool output, which must never corrupt code or paths). const samples = [ `key ${fakeAnthropic()}`, - "token = abc123", + "token = abc123-long-value", `bare ${fakeUnknownVendor()}`, "plain prose with nothing sensitive", + "const token = jwt.sign(payload, key)", ]; for (const s of samples) { - assert.equal( - redactSecrets(s) !== s, - hasSecret(s), - `detect/redact must agree on: ${s.slice(0, 30)}`, - ); + if (redactSecrets(s) !== s) { + assert.ok(hasSecret(s), `redacted but not detected: ${s.slice(0, 30)}`); + } } }); From 1693318f888b7ce775490a591458b5d43621dd3a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 07:05:46 +0000 Subject: [PATCH 9/9] fix: satisfy secret-scan and CodeQL gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test/math.test.js assembles its high-entropy fixture at runtime (the repo's _fixtures.js pattern) so no credential-shaped literal exists for gitleaks to flag — applied through the branch history, which stays clean end to end (the repo deliberately carries no scanner allowlist). - goal.js getGoal slices at the first provenance comment instead of regex-stripping comment blocks (CodeQL js/incomplete-multi-character- sanitization: a replace can leave a partial '/g, "") - .trim(); + // Everything before the first provenance comment IS the goal — a slice, not a + // comment-stripping replace (CodeQL: replace can leave a partial `