From f4ad43291d2a7d316769579e5cb066f82a0a6a7f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 06:46:48 +0000 Subject: [PATCH 1/2] feat(anchor,preflight,docs): graded goal-drift + logistic completeness + link/roadmap guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the last two static-decision holdouts with graded math, and close two recurring docs-rot classes with self-enforcing guards. - anchor.js: goal-drift on-goal-ness is a noisy-OR over path + atlas identifiers (not a binary path substring); driftScore becomes a continuous mean(1-score) that reduces exactly to the old off-goal fraction on binary scores. - preflight.js: completeness s(x) is a logistic over its features (concreteness, specifics, vagueness, tanh length) — no magic coefficients, no word-count step discontinuities, sigmoid-bounded. Calibrated to the paper's examples (0.23/0.63). - docs_check.js: checkLinks resolves every intra-repo Markdown anchor against real headings (GitHub-exact slugs); checkRoadmap fails when ROADMAP trails package.json. - Fix a fabricated forge route example, dead #install / #use-it-in-a-script anchors, and a stale ROADMAP marker (now guarded). +34 tests. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- ARCHITECTURE.md | 21 +- CHANGELOG.md | 35 ++++ ONBOARDING.md | 6 +- README.md | 4 +- ROADMAP.md | 25 ++- docs/GUIDE.md | 16 +- docs/cognitive-substrate/README.md | 4 +- .../deliverable-package.md | 2 +- docs/plans/substrate-v2/05-cost-model.md | 32 +-- src/anchor.js | 151 ++++++++++++-- src/docs_check.js | 187 ++++++++++++++++-- src/predictor.js | 3 +- src/preflight.js | 91 +++++++-- test/anchor.test.js | 142 ++++++++++++- test/docs_check.test.js | 65 ++++++ test/preflight.test.js | 88 ++++++++- 16 files changed, 763 insertions(+), 109 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 98f6923..fcef803 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -231,11 +231,15 @@ parses) swept against every doc artifact → UPDATED / STALE (file:line hits) / VERIFIED-UNAFFECTED with the reason recorded. Pure reporter; the gate provides the teeth. **Docs-check now guards more than names (`src/docs_check.js`).** Beyond -commands/env/MCP-tools/CHANGELOG, three reconcilers close the blind spots behind recurring +commands/env/MCP-tools/CHANGELOG, five reconcilers close the blind spots behind recurring "docs rot" complaints: `checkDiagrams` scans every `mermaid` block across all Markdown for the branded `%%{init` theme and literal-`\n` node breaks; `checkModelTiers` reconciles doc prose prices against `src/model_tiers.json`; `checkBenchmarks` reconciles bolded `N ms` -README claims against the measured table in `reports/benchmarks.md`. The two public pages +README claims against the measured table in `reports/benchmarks.md`; `checkLinks` resolves +every intra-repo Markdown anchor (`#x` and `path.md#x`) against the target's real headings +(GitHub-exact slugs — em-dashes yield `--`, never collapsed), killing the dead-anchor class; +and `checkRoadmap` fails when the ROADMAP's "Now" marker trails the shipped `package.json` +version. The two public pages (`landing/index.html` + the `build-pages.mjs` status page) share one set of design tokens, enforced for parity (plus non-empty changes list, no phantom webfont) by `test/pages.test.js` — so neither the docs' numbers nor the site's look can silently drift. @@ -252,6 +256,19 @@ confidence gate, NOT a keyword DFA. Note `intentGrams` ≠ `contentGrams`: route generic task verbs (`fix`/`add`/`build`) as complexity noise, but they are exactly the intent signal — same math, different stop-set data. +**Graded goal-drift & completeness (`src/anchor.js`, `src/preflight.js`).** Two decisions that +were the last hand-static holdouts are now formulas. Goal-drift no longer classifies a changed +file by a binary path-substring match; `onGoalScore` is a **noisy-OR** (`1 − (1 − p)^hits`, the +same estimator `lessons.js` uses) over how many distinct goal concepts the file exhibits in its +path **and** its atlas-defined identifiers — so `driftScore` is a true continuous Dₜ that +generalizes the old off-goal fraction (identical on 0/1 scores, graded otherwise). The M2 +completeness score `s(x)` is a **logistic** over its features (concreteness, named specifics, +vagueness, a smooth `tanh` length term) instead of an additive rubric with magic coefficients and +discontinuous word-count steps — the `sigmoid` bounds it to (0,1) with no clamp, every feature's +pull stays attributable, and a labeled bank could refine the weights via `predictor.js`'s +`trainLogistic`. The calibrated prior still lands the paper's own examples where they were +(a bare "make the auth better" ≈ 0.23 → ask; a concrete verifyToken edit ≈ 0.63 → proceed). + **The evidence trail (preflight).** Once a goal is anchored, every prompt appends its graded `driftScore` to the session log; `cusum` (until now test-only math) accumulates the series and a sustained alarm rides the gate's block reason. Proceeding under diff --git a/CHANGELOG.md b/CHANGELOG.md index d119ecb..7627992 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,41 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed + +- **Goal-drift is graded, not a substring match (`src/anchor.js`).** A changed file's + on-goal-ness is now a **noisy-OR** (`1 − (1 − p)^hits`) over how many distinct goal + concepts it exhibits in its path **and** the identifiers it defines (via the atlas) — + so a file that implements the goal but never names it in its path is caught + deterministically, not just by the opt-in LLM pass. `driftScore` becomes a true + continuous Dₜ that strictly generalizes the old off-goal fraction (identical on 0/1 + scores, graded otherwise), sharpening the CUSUM drift detector's input. +- **Specification completeness is a logistic estimator (`src/preflight.js`).** The M2 + assumption gate's `s(x)` completeness score is now a logistic over its features + (concreteness, named specifics, vagueness, a smooth `tanh` length term) — replacing the + additive scorer's magic coefficients and discontinuous word-count steps. The `sigmoid` + bounds it to (0,1) with no clamp, each feature's pull stays attributable, and a labeled + bank could refine the weights via `predictor.js`'s `trainLogistic`. Calibrated to keep the + documented examples (a bare "make the auth better" ≈ 0.23 → ask; a concrete verifyToken + edit ≈ 0.63 → proceed). + +### Added + +- **`forge docs check` now guards intra-repo links and roadmap freshness** — two more + reconcilers close recurring "docs rot" classes: `checkLinks` resolves every Markdown + anchor (`#x` and `path.md#x`) against the target file's real headings using + GitHub-exact slugs (an em-dash yields `--`, never collapsed), catching dead anchors like + a renamed `#install`; `checkRoadmap` fails when ROADMAP's "Now" marker trails the shipped + `package.json` version. + +### Fixed + +- **Dead and fabricated docs** — a fabricated `forge route` example in `docs/GUIDE.md` + (an impossible `Fable 5 / Opus` / "premium tier" output) now shows the real routed + verdict; broken `#install` anchors in `ONBOARDING.md` and the substrate README now point + to `#60-second-quickstart`; a dead `#use-it-in-a-script` self-link resolves; and ROADMAP's + "Now" marker is current (v0.11.0). All now enforced by the new docs-check guards. + ## [0.11.0] - 2026-07-11 ### Added diff --git a/ONBOARDING.md b/ONBOARDING.md index dc89f8e..dc72dce 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -1,6 +1,6 @@ # Onboarding — five minutes to productive -**One brain for every AI coding agent.** A language model is *stateless* — one +**One brain for every AI coding agent.** A language model is _stateless_ — one context window, wiped every call — so it has no memory of what your team learned, no foresight about what an edit breaks, and no enforced guardrails. forgekit is the **cognitive substrate** that supplies exactly those three things, and it delivers them @@ -39,7 +39,7 @@ forge doctor # everything green? ``` Full matrix (no-registry `github:` install, symlink dev setup) → -[README → Install](README.md#install). +[README → 60-second quickstart](README.md#60-second-quickstart). ## 2. Configure a repo (once per repo) @@ -64,7 +64,7 @@ forge sync # recompiles into every tool; idempotent (only rewrit ## 3. Use the cognitive substrate -The substrate is the layer that runs *before* the model edits code. One command runs +The substrate is the layer that runs _before_ the model edits code. One command runs the whole pre-action gate: ```bash diff --git a/README.md b/README.md index 1e39788..33a69c7 100644 --- a/README.md +++ b/README.md @@ -114,8 +114,8 @@ an assumption until measured_. real models (paper §9; that's the paper's measurement, not this repo's — `forge cost --stages` reports only _your_ measured stages). - **Conflict-free team memory** — merging two 500-claim ledger replicas takes **158 ms**; the - merge is a property-tested join-semilattice, so teammate ledgers converge in any order over - plain git. + merge is order-independent and property-tested, so teammate ledgers converge to the same state + no matter who syncs first, over plain git. ## 60-second quickstart diff --git a/ROADMAP.md b/ROADMAP.md index 772982f..d0c3990 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -7,16 +7,20 @@ every tool. This is where that brain is headed. Direction, not promises — shaped by the two field reports this project is grounded in (the SDLC pain-point map and the ecosystem landscape). Open a Discussion to weigh in. -## Now (`master`, v0.8.1+) -Everything from 0.8.0 plus gateway-environment support end to end — `ANTHROPIC_AUTH_TOKEN` -recognized everywhere, `ANTHROPIC_MODEL`/`FORGE_MODEL` model override, LiteLLM-gateway -auto-classification of `ANTHROPIC_BASE_URL`, direct-HTTP LLM calls when the `claude` CLI -is absent (`src/llm.js`) — plus decision math replacing keyword heuristics (exemplar k-NN -routing, entropy secret detection), docs↔code drift gating (`forge docs check`, in CI), -docs in the impact graph, and a persistent goal (`forge anchor set`). +## Now (`master`, v0.11.0) + +The substrate is fully graded — decision math replaces every keyword heuristic: exemplar k-NN +routing, entropy secret detection, noisy-OR goal-drift over paths **and** the identifiers a file +defines, and a logistic specification-completeness gate. Around it: docs↔code drift gating +(`forge docs check`, in CI) that now also reconciles diagrams, model prices, benchmark numbers, +intra-repo links, and roadmap freshness; a completion gate (Stop hook); and auto-release on merge. +Gateway environments are supported end to end — `ANTHROPIC_AUTH_TOKEN` recognized everywhere, +`ANTHROPIC_MODEL`/`FORGE_MODEL` model override, LiteLLM-gateway auto-classification of +`ANTHROPIC_BASE_URL`, and direct-HTTP LLM calls when the `claude` CLI is absent (`src/llm.js`). See [CHANGELOG.md](./CHANGELOG.md). ## Shipped — Substrate v2 (all phases P0–P8, v0.5.0) + The plan lives in [docs/plans/substrate-v2/](./docs/plans/substrate-v2/00-overview.md) (phase dependency graph + acceptance gates, all marked done): every paper faculty and mechanism mapped to an algorithm, unified by the **Proof-Carrying Memory (PCM) @@ -25,6 +29,7 @@ confidence only from independent oracles, and merges across teammates conflict-f (git-native CRDT ledger). ## Shipped — 0.7.0 + - **Zero-config provider auto-detection** — `autoDetectProvider()` probes env vars for LiteLLM (local + hosted gateway), OpenRouter, and Anthropic (key, auth token, or custom base URL); `forge init` reports what it found, no manual config needed. @@ -39,6 +44,7 @@ confidence only from independent oracles, and merges across teammates conflict-f event timeline, and ledger health from `.forge/` data. ## Shipped — 0.6.0 + - **Embeddings tier** — optional vector backend (`src/embed.js`, ADR-0005 dependency tier, stdlib fallback kept): `FORGE_EMBED=cmd:` or `FORGE_EMBED=http:` (OpenAI-compatible), disk-cached at @@ -53,6 +59,7 @@ confidence only from independent oracles, and merges across teammates conflict-f `/status/`. ## Next + - **OpenAI + Gemini provider detection** — extend `autoDetectProvider()` beyond Anthropic/OpenRouter/LiteLLM (`OPENAI_API_KEY`, `GEMINI_API_KEY`) with the same zero-config contract. @@ -70,12 +77,14 @@ confidence only from independent oracles, and merges across teammates conflict-f once fixtures measure them (overview §4 honesty register). ## Later / exploring + - `forge verify` independent-review wired into CI with provenance gating. - Formal/semantic verification — documented as out-of-scope for now. - Parametric learning channel (LoRA distillation) — deliberately out of scope (ADR-0006). ## Non-goals + Unbounded dependency growth — deps are selective, optional, and always backed by a stdlib fallback path (ADR-0005). Reimplementing mature tools (skills installer, subagent orchestration, SDD framework, SAST) — we wire the best existing ones. Bundling -a whole IDE. A hosted memory server — git *is* the sync (see 02-team-memory.md). +a whole IDE. A hosted memory server — git _is_ the sync (see 02-team-memory.md). diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 372ee85..1972c52 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -192,8 +192,9 @@ $ forge route "write an is_prime function" driven by: similar to "check if a number is prime" (sim 1.00, complexity 0.08) $ forge route "design and implement a distributed rate limiter with sliding windows across 3 services" - → Fable 5 / Opus (premium tier) - driven by: similar to "implement a rate limiter with a token bucket" (sim 0.71, complexity 0.78) + → Opus 4.8 (complex, $5/$25 per M tok) + architecture, cross-module refactor, novel algorithms, multi-layer debugging + complexity 0.73 · driven by: similar to "implement a rate limiter with a token bucket" (sim 0.43, complexity 0.78) ``` Unseen phrasings route by resemblance — "two threads deadlock when the queue is full" @@ -274,10 +275,13 @@ Forge anchor — goal-drift check ``` `src/auth.js` maps to the goal (named file + where `verifyToken` lives); `src/report.js` -doesn't — so it's surfaced as drift to confirm or undo. Coarse and advisory by design -(path/keyword match, not semantic). `forge substrate` folds this in automatically. The -result also carries `driftScore` (the off-goal fraction per checkpoint) — the graded -signal the `cusum` change-point detector accumulates to catch _sustained_ small drift. +doesn't — so it's surfaced as drift to confirm or undo. Advisory by design, and graded: each +file's on-goal confidence is a noisy-OR over how many goal concepts it exhibits in its path +**and** the identifiers it defines (via the atlas), so a file that implements the goal but never +spells it in its path is still caught. `forge substrate` folds this in automatically. The result +carries `driftScore` — the mean off-goal-ness (1 − that confidence) across the checkpoint's +changes, the graded signal the `cusum` change-point detector accumulates to catch _sustained_ +small drift. **The goal persists.** `forge anchor set ""` stores it in `.forge/goal.md`; every new session re-injects it at SessionStart, a bare `forge anchor` checks against it, and diff --git a/docs/cognitive-substrate/README.md b/docs/cognitive-substrate/README.md index 969b1ac..19759ce 100644 --- a/docs/cognitive-substrate/README.md +++ b/docs/cognitive-substrate/README.md @@ -16,7 +16,7 @@ In Claude Code it runs **automatically**. In other tools you (or the agent) run ## Install -Install Forge (plugin · npm · `github:`) per [README → Install](../../README.md#install), +Install Forge (plugin · npm · `github:`) per [README → 60-second quickstart](../../README.md#60-second-quickstart), then inside any project: ```bash @@ -159,7 +159,7 @@ Code, and agent-invoked everywhere else. --- -## Use it in a script & extend it +## Use it in a script Add `--json` to any command for machine-readable output — gate your agent's next step on `okToProceed`, feed `route.tier` to your model picker, read `impact.impactedFiles` before diff --git a/docs/cognitive-substrate/deliverable-package.md b/docs/cognitive-substrate/deliverable-package.md index 799961d..c8e4475 100644 --- a/docs/cognitive-substrate/deliverable-package.md +++ b/docs/cognitive-substrate/deliverable-package.md @@ -2,7 +2,7 @@ ### Theory → Evidence → Build-Map edition (v2) -**One-line thesis:** The faculties a coding agent lacks — memory, learning, imagination, self-correction, impact-awareness — are not gaps in the model's _knowledge_ but structural consequences of what a frozen transformer _is_ (a stateless map `y = f_θ(x)`, fixed weights, bounded window). They cannot be prompted or tooled away; they can only be supplied by **re-wrapping the input→process→output loop** into a closed, stateful cycle around the frozen model. +**One-line thesis:** The faculties a coding agent lacks — memory, learning, imagination, self-correction, impact-awareness — are not gaps in the model's _knowledge_ but structural consequences of what a frozen transformer _is_: a system with fixed weights and a bounded context window that keeps no state between calls (formally, a stateless map `y = f_θ(x)`). They cannot be prompted or tooled away; they can only be supplied by **re-wrapping the input→process→output loop** into a closed, stateful cycle around the frozen model. **What v2 adds.** The first edition argued the five faculties from first principles and prototyped the one that is buildable today. This edition (1) **grounds the argument in the field's own evidence** — twelve load-bearing pain-point statistics independently re-grounded from primary sources and graded _confirmed / vendor-reported / unverifiable_; (2) adds **six metacognitive mechanisms** the frozen loop also lacks (routing, assumption gate, decomposition, goal-anchoring, anti-over-engineering, inline verification); (3) **maps all eleven capabilities against the real 2026 Claude-Code stack**, marking each solved / partial / residual-gap so we say clearly _what not to build_; and (4) ships a **second runnable prototype** — a complexity-aware router + assumption gate, evaluated live on real models. diff --git a/docs/plans/substrate-v2/05-cost-model.md b/docs/plans/substrate-v2/05-cost-model.md index 5c0aadc..404e4cc 100644 --- a/docs/plans/substrate-v2/05-cost-model.md +++ b/docs/plans/substrate-v2/05-cost-model.md @@ -13,12 +13,12 @@ A task's cost passes through independent multiplicative stages: C = C₀ · (1 − g·h_gate) · (1 − h_cache·σ_cache) · (1 − ρ_ctx) · (1 − r_route) ``` -| factor | stage | meaning | status | -|---|---|---|---| -| `h_gate` | M2 gate | fraction of requests halted as under-specified (spend ≈ 0 generation tokens; `g` ≈ their share of would-have-been spend) | mechanism **measured** in paper §9 (9/9 halts, zero gen tokens); rate is workload-dependent | -| `h_cache` | reuse ([03](./03-reuse-cache.md)) | hit rate; `σ_cache` = avg saving per hit (≈ 1.0 exact, ≈ 0.85 near, ≈ 0.5 adapt) | **target** — P8 measures | -| `ρ_ctx` | assembly ([04](./04-context-assembly.md)) | input-token reduction from knapsack + compression ladder vs. read-everything baseline | **target** — P8 measures | -| `r_route` | M1 routing | tier selection saving on remaining generation | **0.62 measured live** (paper §9, real tokens, real ladder) | +| factor | stage | meaning | status | +| --------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------- | +| `h_gate` | M2 gate | fraction of requests halted as under-specified (spend ≈ 0 generation tokens; `g` ≈ their share of would-have-been spend) | mechanism **measured** in paper §9 (9/9 halts, zero gen tokens); rate is workload-dependent | +| `h_cache` | reuse ([03](./03-reuse-cache.md)) | hit rate; `σ_cache` = avg saving per hit (≈ 1.0 exact, ≈ 0.85 near, ≈ 0.5 adapt) | **target** — P8 measures | +| `ρ_ctx` | assembly ([04](./04-context-assembly.md)) | input-token reduction from knapsack + compression ladder vs. read-everything baseline | **target** — P8 measures | +| `r_route` | M1 routing | tier selection saving on remaining generation | **0.62 measured live** (paper §9, real tokens, real ladder) | Secondary effects deliberately **excluded** from the multiplication (they'd double-count or are unpriceable now): doom-loop halts (avoided thrash loops), M5 lean (fewer generated @@ -27,7 +27,7 @@ tracked in P8 as separate counters, reported alongside — upside, not arithmeti ## 2. What the target requires -With routing fixed at its measured 0.62, reaching 90 % total requires the *other* stages +With routing fixed at its measured 0.62, reaching 90 % total requires the _other_ stages to jointly remove ≈ 74 % of the remaining cost: ``` @@ -36,14 +36,14 @@ to jointly remove ≈ 74 % of the remaining cost: Example compositions that satisfy it: -| scenario | h_cache·σ | ρ_ctx | g·h_gate | total reduction | -|---|---|---|---|---| -| repeat-heavy team (CRUD, components, migrations) | 0.55 | 0.35 | 0.10 | **90.2 %** | -| moderate reuse | 0.40 | 0.30 | 0.10 | 85.6 % | -| cold start (fresh repo, empty ledger) | 0.05 | 0.25 | 0.05 | 74.3 % | +| scenario | h_cache·σ | ρ_ctx | g·h_gate | total reduction | +| ------------------------------------------------ | --------- | ----- | -------- | --------------- | +| repeat-heavy team (CRUD, components, migrations) | 0.55 | 0.35 | 0.10 | **90.2 %** | +| moderate reuse | 0.40 | 0.30 | 0.10 | 85.6 % | +| cold start (fresh repo, empty ledger) | 0.05 | 0.25 | 0.05 | 74.3 % | Read the table honestly: **~90 % is credible on repeat-heavy team workloads once the -ledger is warm, and is not credible cold.** The cache factor dominates, and it *grows* +ledger is warm, and is not credible cold.** The cache factor dominates, and it _grows_ with team adoption (every teammate's verified artifact is everyone's hit — [02](./02-team-memory.md)) and with time (ledger accumulation). The floor — routing + assembly + gate alone — is ≈ 75 %, already substantial. @@ -70,11 +70,11 @@ stage tags), `substrateCheck()`, and the reuse/context modules. `forge cost` lea stages, so every saving is arithmetic on measured tokens, never an estimate. 3. **Correctness guard:** a saving only counts if the external verifier passes the output (paper §9.3's rule: "routing down only counts as a win if the cheap tier is still - correct" — applied to every stage; a cache hit that gets reverted is a *negative* + correct" — applied to every stage; a cache hit that gets reverted is a _negative_ entry). 4. **Report:** per-stage factors with confidence intervals → `reports/cost-eval.md`; the README claim gets updated to whatever the harness measured, with the workload - caveat attached. Until then the README may say "62 % measured (routing); ~90 % + caveat attached. Until then the README may say "62.1 % measured (routing); ~90 % composed target" — never "90 % achieved". ## 4. Cost of the substrate itself @@ -84,7 +84,7 @@ The overhead side of the ledger, counted against the savings in P8: - Deterministic stages (gate, knapsack, cache lookup, atlas query) are CPU-cheap and token-free — the paper's atlas figures (1.9k-node graph in 91 ms, sub-ms queries) bound the latency class. -- Injected context (claims, blast radius, checkpoints) *spends* tokens to save tokens; +- Injected context (claims, blast radius, checkpoints) _spends_ tokens to save tokens; the assembly budget `B` caps it structurally, and `ρ_ctx` is measured net of it. - The opt-in LLM adjudication layer (`FORGE_LLM=1`) is priced per call in `model_tiers.json` and appears as its own metrics stage. diff --git a/src/anchor.js b/src/anchor.js index 6978571..7b42355 100644 --- a/src/anchor.js +++ b/src/anchor.js @@ -26,11 +26,80 @@ function goalKeywords(goal) { return { keywords: [...keywords], symbols, files }; } +// Split an identifier or path into lowercase concept tokens: break camelCase and any +// non-alphanumeric boundary, keep tokens ≥3 chars that aren't stopwords. `verifyToken` +// → {verify, token}; `src/authGuard.js` → {auth, guard} (src/js are stopword-length noise +// that the goal set never contains anyway, so they cost nothing). +function tokenize(s) { + return String(s) + .replace(/([a-z0-9])([A-Z])/g, "$1 $2") + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter((w) => w.length >= 3 && !STOP.has(w)); +} + +function goalConceptTokens(keywords, symbols) { + const g = new Set(); + for (const k of keywords) for (const t of tokenize(k)) g.add(t); + for (const s of symbols) for (const t of tokenize(s)) g.add(t); + return g; +} + +function sameFile(a, b) { + if (!a || !b) return false; + const na = a.replace(/^\.?\//, ""); + const nb = b.replace(/^\.?\//, ""); + return na === nb || na.endsWith(`/${nb}`) || nb.endsWith(`/${na}`); +} + +// The token set that stands in for a changed file: its path components PLUS the identifiers +// it actually defines (from the atlas, when built). The identifier channel is the load-bearing +// upgrade — a file that IMPLEMENTS the goal but whose path never spells a goal word (e.g. +// src/throttle.js defining rateLimiter for a "rate limiting" goal) is now caught deterministically, +// where before only the opt-in LLM pass could rescue it. +function fileConceptTokens(atlas, file) { + const toks = new Set(tokenize(file)); + if (atlas) + for (const s of atlas.symbols || []) + if (sameFile(s.file, file)) for (const t of tokenize(s.name)) toks.add(t); + return toks; +} + +// Per-hit on-goal probability for the noisy-OR below. One shared concept is decent evidence a +// file serves the goal; each additional independent hit raises confidence with diminishing +// returns. 0.6 keeps a single hit clearly on-goal (matching the old "any keyword ⇒ on-goal" +// bias) while letting magnitude grow toward 1. +export const ON_GOAL_P = 0.6; + +/** + * Graded on-goal score for one file: a noisy-OR over how many DISTINCT goal concepts the file + * exhibits (in its path or its defined identifiers). 0 hits → 0 (off-goal); k hits → + * 1 − (1 − p)^k, saturating below 1. Same estimator lessons.js uses for multi-signal evidence. + * This replaces the old binary `path.includes(keyword)` verdict with a continuous signal, so + * the CUSUM drift chart accumulates a true Dₜ ∈ [0,1] instead of a quantized off/on count. + * @param {Set} goalTokens concept tokens of the goal + * @param {Set} fileTokens concept tokens of the file (path ∪ identifiers) + * @param {string} rawPathLower lowercased raw path, for substring hits tokenization can miss + * (goal "auth" in file "authentication.js") — preserves the old match's recall as one channel + * @returns {number} on-goal confidence in [0,1] + */ +export function onGoalScore(goalTokens, fileTokens, rawPathLower = "") { + let hits = 0; + // Exact token match counts at any length; the fuzzy path-substring channel (which preserves + // "auth" ⊂ "authentication") is gated to ≥4 chars so a 3-letter token can't spuriously match + // inside an unrelated word ("log" ⊂ "dialog") and quietly suppress a real drift signal. + for (const g of goalTokens) + if (fileTokens.has(g) || (g.length >= 4 && rawPathLower.includes(g))) + hits++; + return hits ? 1 - (1 - ON_GOAL_P) ** hits : 0; +} + // Machine-generated / tool-config noise that isn't the developer's own work: forge's // cache and every config file `forge init` emits (dot-paths + AGENTS/CLAUDE). // ponytail: this also hides drift in genuine dot-dir edits (.github/*) — fine for an // advisory coarse check; widen to a managed-file manifest if that ever matters. -const NOISE = /(^|\/)\.forge\/|(^|\/)\.[^/]+\/|^\.[^/]+$|(^|\/)(AGENTS|CLAUDE)\.md$/i; +const NOISE = + /(^|\/)\.forge\/|(^|\/)\.[^/]+\/|^\.[^/]+$|(^|\/)(AGENTS|CLAUDE)\.md$/i; function gitFiles(root) { const run = (args) => { @@ -45,7 +114,8 @@ function gitFiles(root) { } }; const out = - run(["diff", "--name-only", "HEAD"]) + run(["ls-files", "--others", "--exclude-standard"]); + run(["diff", "--name-only", "HEAD"]) + + run(["ls-files", "--others", "--exclude-standard"]); return [ ...new Set( out @@ -63,7 +133,8 @@ function goalTargetFiles(root, symbols, files) { const atlas = loadAtlas(root); if (atlas) for (const s of symbols) - for (const hit of queryAtlas(atlas, s)) if (hit.name === s) targets.add(hit.file); + for (const hit of queryAtlas(atlas, s)) + if (hit.name === s) targets.add(hit.file); return [...targets]; } @@ -88,7 +159,10 @@ Omit files that do not serve the goal. No text outside the JSON object.`; export function parseDriftProposal(obj) { const onGoal = Array.isArray(obj.onGoal) ? obj.onGoal - .map((e) => ({ file: asText(e?.file, 240), reason: asText(e?.reason, 200) })) + .map((e) => ({ + file: asText(e?.file, 240), + reason: asText(e?.reason, 200), + })) .filter((e) => e.file && e.reason) : []; return { onGoal }; @@ -106,34 +180,51 @@ export function driftLLM(goal, offGoalFiles, { run = buildRunner() } = {}) { /** * @param {string} root * @param {string} goal - * @param {{ changed?: string[], llm?:boolean, run?:(p:string)=>string, model?:string, timeoutMs?:number }} [opts] - * inject `changed` to skip git; inject `run` to stub the model (used in tests). + * @param {{ changed?: string[], llm?:boolean, run?:(p:string)=>string, model?:string, timeoutMs?:number, atlas?:object|null }} [opts] + * inject `changed` to skip git; inject `atlas`/`run` to stub the graph/model (used in tests). */ export function goalDrift(root, goal, opts = {}) { const { keywords, symbols, files } = goalKeywords(goal); const changedFiles = opts.changed || gitFiles(root); - const targets = goalTargetFiles(root, symbols, files).map((t) => t.toLowerCase()); + const targets = goalTargetFiles(root, symbols, files).map((t) => + t.toLowerCase(), + ); + const atlas = opts.atlas !== undefined ? opts.atlas : loadAtlas(root); + const goalTokens = goalConceptTokens(keywords, symbols); const onGoal = []; const offGoal = []; + // Per-file on-goal confidence in [0,1] — the graded signal. A file is classified on-goal iff + // its score clears the single-hit floor (ON_GOAL_P), preserving the old "any match ⇒ on-goal" + // bias, but the magnitude (path + identifier evidence, noisy-OR) is what feeds driftScore. + const score = new Map(); for (const f of changedFiles) { const lf = f.toLowerCase(); - // ponytail: path/keyword match, not semantic — coarse on purpose (are you in the - // right AREA?), and errs toward "on-goal" so it never cries drift on a match. const named = targets.some((t) => lf === t || lf.endsWith(`/${t}`)); - (named || keywords.some((k) => lf.includes(k)) ? onGoal : offGoal).push(f); + const s = named + ? 1 + : onGoalScore(goalTokens, fileConceptTokens(atlas, f), lf); + score.set(f, s); + (s >= ON_GOAL_P ? onGoal : offGoal).push(f); } // Opt-in semantic pass: rescue files the keyword match missed, but only off→on and only when // the model gives a goal-referencing reason. Verified, not trusted; fail-safe on any error. let provenance = { path: "deterministic" }; if (llmEnabled({ llm: opts.llm }) && offGoal.length) { - const goalTerms = new Set([...keywords, ...symbols.map((s) => s.toLowerCase())]); + const goalTerms = new Set([ + ...keywords, + ...symbols.map((s) => s.toLowerCase()), + ]); const grounded = (reason) => { const words = String(reason).toLowerCase(); - return goalTerms.size === 0 || [...goalTerms].some((t) => words.includes(t)); + return ( + goalTerms.size === 0 || [...goalTerms].some((t) => words.includes(t)) + ); }; const proposal = driftLLM(goal, offGoal, { - run: opts.run || buildRunner({ model: opts.model, timeoutMs: opts.timeoutMs }), + run: + opts.run || + buildRunner({ model: opts.model, timeoutMs: opts.timeoutMs }), }); const rescued = new Set( (proposal?.onGoal || []) @@ -145,6 +236,8 @@ export function goalDrift(root, goal, opts = {}) { if (rescued.has(f)) { offGoal.splice(offGoal.indexOf(f), 1); onGoal.push(f); + // A verified rescue is on-goal at the single-hit floor, so driftScore reflects it too. + score.set(f, Math.max(score.get(f) ?? 0, ON_GOAL_P)); } } provenance = { path: "llm-verified", rescued: [...rescued] }; @@ -153,11 +246,17 @@ export function goalDrift(root, goal, opts = {}) { } } - const drift = changedFiles.length > 0 && (offGoal.length > 0 || onGoal.length === 0); - // Graded drift magnitude Dₜ ∈ [0,1] — the fraction of this checkpoint's changes that - // wandered off-goal. This is the signal cusum() below expects: the binary `drift` - // flag answers "any drift now?", the score accumulates into "sustained drift?". - const driftScore = changedFiles.length ? offGoal.length / changedFiles.length : 0; + const drift = + changedFiles.length > 0 && (offGoal.length > 0 || onGoal.length === 0); + // Graded drift magnitude Dₜ ∈ [0,1] — the mean off-goal-ness (1 − on-goal confidence) across + // this checkpoint's changes. This is the signal cusum() below expects: the binary `drift` flag + // answers "any drift now?", the score accumulates into "sustained drift?". It strictly + // generalizes the old off-goal fraction — identical when every score is 0/1, graded otherwise, + // so a weakly-on-goal file adds less drift than an unrelated one instead of counting the same. + const driftScore = changedFiles.length + ? changedFiles.reduce((a, f) => a + (1 - (score.get(f) ?? 0)), 0) / + changedFiles.length + : 0; return { goal: String(goal), keywords, @@ -205,11 +304,21 @@ export function renderAnchor(r) { ` changed: ${r.changed.length} file(s) · on-goal ${r.onGoal.length} · off-goal ${r.offGoal.length}`, ); if (r.offGoal.length) { - lines.push("", " off-goal (unrelated to the stated goal — intended, or drift?):"); + lines.push( + "", + " off-goal (unrelated to the stated goal — intended, or drift?):", + ); for (const f of r.offGoal.slice(0, 12)) lines.push(` - ${f}`); } if (r.drift && !r.offGoal.length) - lines.push("", " ! no changed file matches the goal — are you working on the right thing?"); - if (!r.drift) lines.push("", " ✓ on goal — every change maps to what you set out to do."); + lines.push( + "", + " ! no changed file matches the goal — are you working on the right thing?", + ); + if (!r.drift) + lines.push( + "", + " ✓ on goal — every change maps to what you set out to do.", + ); return lines.join("\n"); } diff --git a/src/docs_check.js b/src/docs_check.js index 1473291..2d5651a 100644 --- a/src/docs_check.js +++ b/src/docs_check.js @@ -6,22 +6,32 @@ // against the forge package root (BRAND.root), not the host repo. import { execFileSync } from "node:child_process"; import { existsSync, readdirSync, readFileSync } from "node:fs"; -import { join } from "node:path"; +import { dirname, join, normalize } from "node:path"; import { BRAND } from "./brand.js"; import { COMMANDS, HIDDEN_COMMANDS } from "./commands.js"; import { TOOLS } from "./mcp_tools.js"; import { MODELS } from "./model_tiers.js"; /** The user-facing prose docs every claim is reconciled against. */ -const DOC_FILES = ["README.md", "docs/GUIDE.md", "ARCHITECTURE.md", "ROADMAP.md"]; +const DOC_FILES = [ + "README.md", + "docs/GUIDE.md", + "ARCHITECTURE.md", + "ROADMAP.md", +]; // Env vars read in src that are NOT user-facing contract: child-process plumbing and // values injected by host tools rather than set by users. -const INTERNAL_ENV = new Set(["_FORGE_LLM_KEY", "FORGE_EMBED_KEY", "CLAUDE_PLUGIN_ROOT"]); +const INTERNAL_ENV = new Set([ + "_FORGE_LLM_KEY", + "FORGE_EMBED_KEY", + "CLAUDE_PLUGIN_ROOT", +]); // Prefixes that mark an env var as OURS to document. A doc may freely mention other // tools' vars (GITHUB_TOKEN, PATH) — those aren't claims about forge's own surface. -const ENV_PREFIX_RE = /\b((?:FORGE|ANTHROPIC|LITELLM|OPENROUTER|ENABLE_CORTEX)_[A-Z0-9_]+)\b/g; +const ENV_PREFIX_RE = + /\b((?:FORGE|ANTHROPIC|LITELLM|OPENROUTER|ENABLE_CORTEX)_[A-Z0-9_]+)\b/g; function readDoc(root, rel) { const p = join(root, rel); @@ -44,8 +54,12 @@ export function envVarsRead(root = BRAND.root) { const vars = new Set(); for (const file of srcFiles(root)) { const text = readFileSync(file, "utf8"); - for (const m of text.matchAll(/process\.env\.([A-Z_][A-Z0-9_]*)/g)) vars.add(m[1]); - for (const m of text.matchAll(/process\.env\[["']([A-Z_][A-Z0-9_]*)["']\]/g)) vars.add(m[1]); + for (const m of text.matchAll(/process\.env\.([A-Z_][A-Z0-9_]*)/g)) + vars.add(m[1]); + for (const m of text.matchAll( + /process\.env\[["']([A-Z_][A-Z0-9_]*)["']\]/g, + )) + vars.add(m[1]); } const guards = join(root, "global", "guards"); if (existsSync(guards)) { @@ -80,7 +94,9 @@ function checkCommands(docs, issues) { } } for (const [file, text] of Object.entries(docs)) { - for (const m of text.matchAll(new RegExp(`\`${BRAND.cli} ([a-z][a-z-]*)\\b`, "g"))) { + for (const m of text.matchAll( + new RegExp(`\`${BRAND.cli} ([a-z][a-z-]*)\\b`, "g"), + )) { const name = m[1]; if (!(name in COMMANDS) && !HIDDEN_COMMANDS.includes(name)) { issues.push({ @@ -110,7 +126,11 @@ function checkEnvVars(root, docs, issues) { const documented = new Set(); for (const [file, text] of Object.entries(docs)) { for (const m of text.matchAll(ENV_PREFIX_RE)) { - if (!documented.has(`${file}:${m[1]}`) && !read.has(m[1]) && !INTERNAL_ENV.has(m[1])) { + if ( + !documented.has(`${file}:${m[1]}`) && + !read.has(m[1]) && + !INTERNAL_ENV.has(m[1]) + ) { documented.add(`${file}:${m[1]}`); issues.push({ check: "env-vars", @@ -168,7 +188,12 @@ function markdownFiles(root) { if (!existsSync(root)) return []; return readdirSync(root, { recursive: true }) .map(String) - .filter((f) => f.endsWith(".md") && !f.includes("node_modules") && !f.startsWith(".git/")); + .filter( + (f) => + f.endsWith(".md") && + !f.includes("node_modules") && + !f.startsWith(".git/"), + ); } // The branded Mermaid theme every diagram shares (see README's `%%{init …}%%`). Without it @@ -192,7 +217,10 @@ function checkDiagrams(root, issues) { for (const m of text.matchAll(MERMAID_BLOCK_RE)) { // An intentional example block (e.g. docs showing what a BAD diagram looks like) opts // out with an HTML comment `` on the line before the fence. - if (/docs-check-ignore/.test(text.slice(Math.max(0, m.index - 80), m.index))) continue; + if ( + /docs-check-ignore/.test(text.slice(Math.max(0, m.index - 80), m.index)) + ) + continue; const block = m[1]; if (!block.includes("%%{init")) { issues.push({ @@ -218,9 +246,13 @@ function checkChangelog(root, issues) { const text = readDoc(root, "CHANGELOG.md"); if (!text) return; const sections = [ - ...text.matchAll(/^## \[([^\]]+)\][^\n]*\n([\s\S]*?)(?=^## \[|\n*$(?![\s\S]))/gm), + ...text.matchAll( + /^## \[([^\]]+)\][^\n]*\n([\s\S]*?)(?=^## \[|\n*$(?![\s\S]))/gm, + ), ]; - const version = JSON.parse(readFileSync(join(root, "package.json"), "utf8")).version; + const version = JSON.parse( + readFileSync(join(root, "package.json"), "utf8"), + ).version; const released = sections.filter((s) => s[1].toLowerCase() !== "unreleased"); if (released.length && released[0][1] !== version) { issues.push({ @@ -240,13 +272,18 @@ function checkChangelog(root, issues) { } const unreleased = sections.find((s) => s[1].toLowerCase() === "unreleased"); if (unreleased && !unreleased[2].trim()) { - const srcT = Number(git(root, ["log", "-1", "--format=%ct", "--", "src"]) || 0); - const clT = Number(git(root, ["log", "-1", "--format=%ct", "--", "CHANGELOG.md"]) || 0); + const srcT = Number( + git(root, ["log", "-1", "--format=%ct", "--", "src"]) || 0, + ); + const clT = Number( + git(root, ["log", "-1", "--format=%ct", "--", "CHANGELOG.md"]) || 0, + ); if (srcT && clT && srcT > clT) { issues.push({ check: "changelog", severity: "error", - detail: "src changed since CHANGELOG.md was last touched, but [Unreleased] is empty", + detail: + "src changed since CHANGELOG.md was last touched, but [Unreleased] is empty", }); } } @@ -282,7 +319,8 @@ function measuredTimings(root) { const set = new Set(); for (const line of readDoc(root, "reports/benchmarks.md").split("\n")) { if (!line.startsWith("|")) continue; // table rows only — not the prose above it - for (const m of line.matchAll(/(\d+(?:\.\d+)?)\s*(ms|µs|s)\b/g)) set.add(`${m[1]} ${m[2]}`); + for (const m of line.matchAll(/(\d+(?:\.\d+)?)\s*(ms|µs|s)\b/g)) + set.add(`${m[1]} ${m[2]}`); } return set; } @@ -313,6 +351,119 @@ function checkBenchmarks(root, docs, issues) { } } +// GitHub-style heading→anchor slug: lowercase, drop backticks, strip punctuation (keep word +// chars/spaces/hyphens), spaces→hyphens. Close enough to GitHub's own algorithm for the intra-repo +// links these docs use; we only ever FLAG a miss, so a rare slug mismatch degrades to a false +// alarm we'd see immediately, never a silent wrong link. +function headingSlug(text) { + return text + .trim() + .toLowerCase() + .replace(/`/g, "") + .replace(/[^\w\s-]/g, "") + .replace(/\s/g, "-"); // each space → one hyphen; GitHub does NOT collapse (an em-dash between + // two words becomes "--", not "-"), so consecutive spaces must map 1:1. +} + +// Every anchor a Markdown file EXPOSES: heading slugs plus explicit ids authors add +// (``, `name=…`, or a `{#custom-id}` suffix). +function anchorsFor(text) { + const set = new Set(); + for (const m of text.matchAll(/^#{1,6}\s+(.+?)\s*#*\s*$/gm)) + set.add(headingSlug(m[1])); + for (const m of text.matchAll(/\b(?:id|name)=["']([\w-]+)["']/g)) + set.add(m[1].toLowerCase()); + for (const m of text.matchAll(/\{#([\w-]+)\}/g)) set.add(m[1].toLowerCase()); + return set; +} + +/** + * Intra-repo link hygiene: every Markdown link with an `#anchor` (same-file `#x` or a + * `path.md#x` cross-reference) must resolve to a heading/anchor that actually exists in the + * target. This is the guard that kills the "[README → Install](#install)" class of silent dead + * links — a heading gets renamed, the anchor rots, and nothing noticed until now. External URLs + * and non-Markdown targets (.html/.pdf/code) are skipped; a missing target file is left to other + * checks. Only a genuine unresolved anchor is flagged. + */ +function checkLinks(root, issues) { + const cache = new Map(); + const anchorsOf = (rel) => { + if (cache.has(rel)) return cache.get(rel); + let a = null; + try { + a = anchorsFor(readFileSync(join(root, rel), "utf8")); + } catch { + a = null; + } + cache.set(rel, a); + return a; + }; + for (const rel of markdownFiles(root)) { + let text; + try { + text = readFileSync(join(root, rel), "utf8"); + } catch { + continue; + } + // Strip fenced code blocks so link SYNTAX shown in an example (```md … [x](#y) …```) isn't + // scanned as a live link — that would false-positive on documentation about links. + const prose = text.replace(/```[\s\S]*?```/g, ""); + for (const m of prose.matchAll(/\[[^\]]*\]\(([^)]+)\)/g)) { + const href = m[1].trim(); + if (/^https?:/i.test(href)) continue; // external + const hash = href.indexOf("#"); + if (hash < 0) continue; // no anchor to resolve + const path = href.slice(0, hash); + const anchor = href.slice(hash + 1).toLowerCase(); + if (!anchor) continue; + let targetRel; + if (!path) + targetRel = rel; // same-file anchor + else if (path.endsWith(".md")) + targetRel = normalize(join(dirname(rel), path)); + else continue; // .html/.pdf/code target — can't resolve headings, skip + const anchors = anchorsOf(targetRel); + if (anchors == null) continue; // unreadable/missing target — file existence is another matter + if (!anchors.has(anchor)) { + issues.push({ + check: "links", + severity: "error", + detail: `${rel}: link \`${href}\` points to #${anchor}, which is not a heading in ${targetRel}`, + }); + } + } + } +} + +/** + * ROADMAP freshness: the "## Now" marker must not name a version behind package.json. A roadmap + * that still says "Now (v0.8.1+)" two releases after 0.10.0 shipped is the "docs are outdated" + * complaint in miniature — this makes it a CI failure the moment a release laps the roadmap. + */ +function checkRoadmap(root, issues) { + const text = readDoc(root, "ROADMAP.md"); + if (!text) return; + const now = text.match(/##\s+Now\b[^\n]*/i); + if (!now) return; + const vm = now[0].match(/v(\d+)\.(\d+)(?:\.(\d+))?/); + if (!vm) return; + const road = [Number(vm[1]), Number(vm[2]), Number(vm[3] || 0)]; + const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) + .version.split(".") + .map(Number); + const behind = + road[0] < pkg[0] || + (road[0] === pkg[0] && + (road[1] < pkg[1] || (road[1] === pkg[1] && road[2] < pkg[2]))); + if (behind) { + issues.push({ + check: "roadmap", + severity: "error", + detail: `ROADMAP's "Now" marker says v${road.join(".")} but package.json is ${pkg.join(".")} — the roadmap trails the shipped release`, + }); + } +} + /** * Run every reconciler against the forge package tree. * @param {{root?: string}} [opts] @@ -328,6 +479,8 @@ export function docsCheck({ root = BRAND.root } = {}) { checkDiagrams(root, issues); checkModelTiers(docs, issues); checkBenchmarks(root, docs, issues); + checkLinks(root, issues); + checkRoadmap(root, issues); return { ok: !issues.some((i) => i.severity === "error"), issues, @@ -339,6 +492,8 @@ export function docsCheck({ root = BRAND.root } = {}) { "diagrams", "model-tiers", "benchmarks", + "links", + "roadmap", ], }; } diff --git a/src/predictor.js b/src/predictor.js index f4b3c66..c5a161f 100644 --- a/src/predictor.js +++ b/src/predictor.js @@ -19,7 +19,8 @@ export const DEFAULT_WEIGHTS = { export const FEATURE_KEYS = Object.keys(DEFAULT_WEIGHTS).filter((k) => k !== "bias"); -const sigmoid = (z) => 1 / (1 + Math.exp(-z)); +/** Logistic squashing function — maps any real log-odds z to a probability in (0,1). */ +export const sigmoid = (z) => 1 / (1 + Math.exp(-z)); /** Heuristic risk in [0,1]. Advisory only — never blocks. */ export function heuristicRisk(features, weights = DEFAULT_WEIGHTS) { diff --git a/src/preflight.js b/src/preflight.js index 28fd935..a51ba6e 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -5,6 +5,7 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { adjudicate, asText, asUnit, buildRunner, llmEnabled } from "./adjudicate.js"; import { build as buildAtlas, has, load as loadAtlas } from "./atlas.js"; +import { sigmoid } from "./predictor.js"; import { CODE_EXT } from "./util.js"; const STOP = new Set([ @@ -159,6 +160,55 @@ export function ambiguityMarkers(text) { return [...new Set(out)]; } +// M2 completeness model — a logistic (log-odds) estimator of specification completeness s(x), +// replacing the older additive scorer whose magic coefficients + discontinuous word-count steps +// the audit flagged as "graded-but-uncalibrated". Each feature contributes a signed amount to the +// log-odds and the sigmoid maps the sum to [0,1] — so the estimate is smooth (no step jumps), +// self-bounding (no ad-hoc clamp), and every feature's pull stays attributable (transparent rubric, +// the substrate's core commitment). Weights are a documented PRIOR calibrated so the paper's own +// examples land where they should (a bare "make the auth better" ≈ 0.23 → ask; a concrete +// "Change verifyToken … length > 20; update tests" ≈ 0.63 → proceed); a labeled task bank could +// refine them via predictor.js's trainLogistic without changing this call site. +export const COMPLETENESS_WEIGHTS = { + bias: -0.858, + concreteness: 1.44, // each concrete anchor (example, call signature, quoted literal, filename) + specifics: 0.5, // each named technology/algorithm — real information even in prose + vagueness: -1.3, // each vague filler that forces the agent to interpret — strong negative + length: 0.6, // continuous length signal in [-1,1]; replaces the old ≥22/≥30-word steps +}; + +/** + * Extract the completeness feature vector from a task string. Pure and exported so the estimate + * is inspectable and a completeness bank could be fit against it. + * @param {string} task + */ +export function completenessFeatures(task) { + const t = String(task || ""); + const words = t.trim().split(/\s+/).filter(Boolean).length; + return { + words, + concreteness: ANCHORS.filter((a) => a.test(t)).length, + specifics: new Set([...t.matchAll(SPECIFIC)].map((m) => m[0].toLowerCase())).size, + vagueness: new Set( + [...t.matchAll(new RegExp(VAGUE.source, "gi"))].map((m) => m[0].toLowerCase()), + ).size, + // smooth, bounded length evidence: ~12-word tasks are neutral, shorter pulls down, longer up, + // with no threshold jumps — tanh saturates so a wall of text can't dominate the verdict. + length: Math.tanh((words - 12) / 12), + }; +} + +/** Completeness s(x) ∈ (0,1) as a logistic over the feature vector. */ +export function completenessScore(features, weights = COMPLETENESS_WEIGHTS) { + const z = + weights.bias + + weights.concreteness * features.concreteness + + weights.specifics * features.specifics + + weights.vagueness * features.vagueness + + weights.length * features.length; + return sigmoid(z); +} + export function assessTask(text, { askThreshold = 0.6 } = {}) { const task = String(text || ""); const words = task.trim().split(/\s+/).filter(Boolean).length; @@ -167,23 +217,16 @@ export function assessTask(text, { askThreshold = 0.6 } = {}) { const vagueHits = [ ...new Set([...task.matchAll(new RegExp(VAGUE.source, "gi"))].map((m) => m[0].toLowerCase())), ]; + const completeness = completenessScore({ + words, + concreteness, + specifics: specifics.length, + vagueness: vagueHits.length, + length: Math.tanh((words - 12) / 12), + }); const reasons = []; - let score = - 0.45 + - Math.min(0.45, 0.18 * concreteness) + - Math.min(0.2, 0.06 * specifics.length) - - 0.22 * vagueHits.length; - if (words <= 7) { - score -= 0.22; - reasons.push("very short request"); - } - if (words >= 30 && specifics.length >= 2) { - score += 0.2; - reasons.push(`long detailed spec (${words} words)`); - } else if (words >= 22 && specifics.length >= 1) { - score += 0.1; - reasons.push(`detailed spec (${words} words)`); - } + if (words <= 7) reasons.push("very short request"); + if (words >= 22) reasons.push(`detailed request (${words} words)`); if (concreteness) reasons.push(`${concreteness} concrete anchor(s)`); if (specifics.length) reasons.push(`${specifics.length} named technical specific(s)`); if (vagueHits.length) reasons.push(`${vagueHits.length} vague filler(s)`); @@ -196,7 +239,6 @@ export function assessTask(text, { askThreshold = 0.6 } = {}) { questions.push(d.question); } } - const completeness = Math.max(0, Math.min(1, score)); const hardUnderspecified = concreteness === 0 && (words <= 10 || (vagueHits.length >= 1 && specifics.length === 0)); const shouldAsk = completeness < askThreshold || hardUnderspecified; @@ -246,7 +288,11 @@ export function parseAssumptionProposal(obj) { /** Ask the model for an assumption reading (proposer). Returns null when off/unavailable. */ export function assessTaskLLM(task, { run = buildRunner() } = {}) { - return adjudicate({ prompt: buildAssumptionPrompt(task), parse: parseAssumptionProposal, run }); + return adjudicate({ + prompt: buildAssumptionPrompt(task), + parse: parseAssumptionProposal, + run, + }); } /** @@ -393,8 +439,13 @@ export function preflightRepo( // M2 proposer: only when opted in. The rubric is the external judge; the model refines it // within bounds and can add grounded questions. Fail-safe: null proposal keeps `det`. if (!llmEnabled({ llm })) - return { ...gap, assumption: { ...det, provenance: { path: "deterministic" } } }; - const proposal = assessTaskLLM(text, { run: run || buildRunner({ model, timeoutMs }) }); + return { + ...gap, + assumption: { ...det, provenance: { path: "deterministic" } }, + }; + const proposal = assessTaskLLM(text, { + run: run || buildRunner({ model, timeoutMs }), + }); const grounded = (q) => { const { symbols, files } = referencedEntities(q); return symbols.some(hasSymbol) || files.some((f) => existsSync(join(root, f))); diff --git a/test/anchor.test.js b/test/anchor.test.js index be50eea..9957679 100644 --- a/test/anchor.test.js +++ b/test/anchor.test.js @@ -1,6 +1,12 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { cusum, goalDrift, renderAnchor } from "../src/anchor.js"; +import { + cusum, + goalDrift, + ON_GOAL_P, + onGoalScore, + renderAnchor, +} from "../src/anchor.js"; // changed[] is injected so these are pure (no git needed). test("goalDrift flags a changed file unrelated to the goal", () => { @@ -13,7 +19,9 @@ test("goalDrift flags a changed file unrelated to the goal", () => { }); test("goalDrift is quiet when every change maps to the goal", () => { - const r = goalDrift("/nope", "fix login validation", { changed: ["src/login.js"] }); + const r = goalDrift("/nope", "fix login validation", { + changed: ["src/login.js"], + }); assert.equal(r.drift, false); assert.deepEqual(r.offGoal, []); }); @@ -25,7 +33,9 @@ test("goalDrift is quiet with no changes yet", () => { }); test("goalDrift flags pure drift — changes exist but none match the goal", () => { - const r = goalDrift("/nope", "update the billing invoice totals", { changed: ["src/auth.js"] }); + const r = goalDrift("/nope", "update the billing invoice totals", { + changed: ["src/auth.js"], + }); assert.equal(r.onGoal.length, 0); assert.equal(r.drift, true); }); @@ -39,13 +49,17 @@ test("goalDrift (llm on): rescues an off-goal file only with a goal-referencing llm: true, run, }); - assert.ok(r.onGoal.includes("src/throttle.js"), "grounded reason moves it off→on"); + assert.ok( + r.onGoal.includes("src/throttle.js"), + "grounded reason moves it off→on", + ); assert.ok(!r.offGoal.includes("src/throttle.js")); assert.equal(r.provenance.path, "llm-verified"); }); test("goalDrift (llm on): a reason that never references the goal is rejected (verify, don't trust)", () => { - const run = () => '{"onGoal":[{"file":"src/reports.js","reason":"just tidying up"}]}'; + const run = () => + '{"onGoal":[{"file":"src/reports.js","reason":"just tidying up"}]}'; const r = goalDrift("/nope", "add rate limiting to the login route", { changed: ["src/reports.js"], llm: true, @@ -62,7 +76,10 @@ test("goalDrift (llm on): the model can never move a file on→off", () => { llm: true, run, }); - assert.ok(r.onGoal.includes("src/login.js"), "deterministic on-goal is never demoted"); + assert.ok( + r.onGoal.includes("src/login.js"), + "deterministic on-goal is never demoted", + ); }); test("goalDrift (llm on): a throwing runner falls back to the deterministic split", () => { @@ -77,6 +94,107 @@ test("goalDrift (llm on): a throwing runner falls back to the deterministic spli assert.equal(r.drift, true); }); +// --------------------------------------------------------------------------- +// M4 — graded on-goal score (noisy-OR over concept hits), the CUSUM input. +// --------------------------------------------------------------------------- + +test("onGoalScore: 0 hits → 0, and each extra hit raises confidence with diminishing returns", () => { + const goal = new Set(["rate", "limit", "login"]); + assert.equal( + onGoalScore(goal, new Set(["reports", "billing"])), + 0, + "no shared concept → off-goal", + ); + const one = onGoalScore(goal, new Set(["login"])); + const two = onGoalScore(goal, new Set(["login", "rate"])); + const three = onGoalScore(goal, new Set(["login", "rate", "limit"])); + assert.ok( + Math.abs(one - ON_GOAL_P) < 1e-9, + "a single hit sits at the on-goal floor", + ); + assert.ok( + two > one && three > two, + "more independent evidence ⇒ higher, monotone", + ); + assert.ok(three < 1, "noisy-OR saturates below 1, never false-certain"); +}); + +test("onGoalScore: substring channel preserves the old match's recall (auth ⊂ authentication)", () => { + const goal = new Set(["auth"]); + // tokenize would keep 'authentication' whole, so token-equality misses it; the raw-path + // substring channel still counts the hit, so we never regress below the old behaviour. + assert.ok( + onGoalScore(goal, new Set(["authentication"]), "src/authentication.js") >= + ON_GOAL_P, + ); +}); + +test("goalDrift: an atlas identifier rescues an implement-the-goal file the PATH never names", () => { + // src/throttle.js's path shares no goal word, but it DEFINES rateLimiter — the identifier + // channel catches it deterministically (no LLM), the exact gap the audit flagged. + const atlas = { + symbols: [ + { name: "rateLimiter", file: "src/throttle.js", kind: "function" }, + { name: "renderReport", file: "src/reports.js", kind: "function" }, + ], + }; + const r = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/throttle.js", "src/reports.js"], + atlas, + }); + assert.ok( + r.onGoal.includes("src/throttle.js"), + "identifier match ⇒ on-goal without the LLM", + ); + assert.ok( + r.offGoal.includes("src/reports.js"), + "a genuinely unrelated file still drifts", + ); + assert.equal(r.provenance.path, "deterministic"); +}); + +test("goalDrift: driftScore is graded — a weak match drifts less than an unrelated file", () => { + const weak = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/login.js"], // one concept hit (login) + atlas: null, + }); + const off = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/reports.js"], // zero hits + atlas: null, + }); + assert.ok( + weak.driftScore < off.driftScore, + "a partial match accrues less drift than none", + ); + assert.ok( + Math.abs(off.driftScore - 1) < 1e-9, + "a fully off-goal checkpoint drifts at 1.0", + ); +}); + +test("goalDrift: on a partial (0.6) + off (0) checkpoint, driftScore is the graded mean of 1−score", () => { + const r = goalDrift("/nope", "billing invoice totals", { + changed: ["src/billing.js", "src/reports.js"], // billing → 1 concept hit = 0.6; reports → 0 + atlas: null, + }); + // graded, not quantized: mean(1−s) = (0.4 + 1)/2 = 0.7, where the legacy off/n would say 0.5. + assert.ok(Math.abs(r.driftScore - 0.7) < 1e-9); +}); + +test("goalDrift: driftScore reduces EXACTLY to the legacy off-goal fraction on genuinely binary scores", () => { + // A goal that NAMES a file scores it exactly 1.0 (named target), and an unrelated file scores 0 + // — a real {0,1} checkpoint. Here the graded mean(1−score) must equal the old offGoal/changed. + const r = goalDrift("/nope", "update src/pay.js", { + changed: ["src/pay.js", "src/reports.js"], // pay.js is the named target → 1; reports → 0 + atlas: null, + }); + assert.ok( + r.onGoal.includes("src/pay.js") && r.offGoal.includes("src/reports.js"), + ); + // mean(1−score) = (0 + 1)/2 = 0.5 = legacy offGoal(1)/changed(2). Exact reduction on 0/1 scores. + assert.ok(Math.abs(r.driftScore - 0.5) < 1e-9); +}); + // --------------------------------------------------------------------------- // M4 — one-sided CUSUM drift control (pure). // --------------------------------------------------------------------------- @@ -86,7 +204,11 @@ test("cusum alarms on sustained small drift (the decaying-anchor failure)", () = // alarming, but the excess accumulates: C = 0.25, 0.5, 0.75, 1.0, 1.25 → alarm. const r = cusum([0.6, 0.6, 0.6, 0.6, 0.6, 0.6]); assert.equal(r.alarm, true); - assert.equal(r.firstAlarm, 4, "alarms at the first checkpoint where C exceeds h"); + assert.equal( + r.firstAlarm, + 4, + "alarms at the first checkpoint where C exceeds h", + ); assert.equal(r.C.length, 6); assert.ok(Math.abs(r.C[0] - 0.25) < 1e-9); }); @@ -95,7 +217,11 @@ test("cusum does not alarm on a single spike within tolerance", () => { const r = cusum([0.1, 1.2, 0.1, 0.1, 0.1, 0.1]); assert.equal(r.alarm, false); assert.equal(r.firstAlarm, -1); - assert.equal(r.C.at(-1), 0, "the statistic drains back to zero after the spike"); + assert.equal( + r.C.at(-1), + 0, + "the statistic drains back to zero after the spike", + ); }); test("cusum is quiet on on-goal signals and on an empty series", () => { diff --git a/test/docs_check.test.js b/test/docs_check.test.js index c3a637a..32d0472 100644 --- a/test/docs_check.test.js +++ b/test/docs_check.test.js @@ -246,6 +246,71 @@ test("docsCheck: a mermaid example marked docs-check-ignore is skipped", () => { ); }); +test("docsCheck: a Markdown link to a heading anchor that doesn't exist is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "ARCHITECTURE.md": `${f["ARCHITECTURE.md"]}\nSee [the setup](#nonexistent-section) first.\n`, + })); + const r = docsCheck({ root }); + assert.ok( + r.issues.some((i) => i.check === "links" && /#nonexistent-section/.test(i.detail)), + "a dead same-file anchor is caught", + ); +}); + +test("docsCheck: a link to a real heading — including an em-dash '--' slug — resolves clean", () => { + // "Design — the loop" → GitHub slug "design--the-loop" (the em-dash leaves two spaces → two + // hyphens). The resolver must NOT collapse them, or valid links false-positive. + const root = fixtureRoot((f) => ({ + ...f, + "ARCHITECTURE.md": "# arch\n\n## Design — the loop\n\nJump to [the loop](#design--the-loop).\n", + })); + const r = docsCheck({ root }); + assert.deepEqual( + r.issues.filter((i) => i.check === "links"), + [], + "an em-dash heading anchor is not a false positive", + ); +}); + +test("docsCheck: a cross-file .md#anchor that does not resolve is flagged", () => { + const root = fixtureRoot((f) => ({ + ...f, + "ARCHITECTURE.md": `${f["ARCHITECTURE.md"]}\nSee [the guide](docs/GUIDE.md#ghost-heading).\n`, + })); + const r = docsCheck({ root }); + assert.ok( + r.issues.some( + (i) => i.check === "links" && /ghost-heading/.test(i.detail) && /GUIDE/.test(i.detail), + ), + "a dead cross-file anchor resolves against the target file and is caught", + ); +}); + +test("docsCheck: a ROADMAP 'Now' marker behind package.json is flagged; a current one passes", () => { + const behind = docsCheck({ + root: fixtureRoot((f) => ({ + ...f, + "ROADMAP.md": "# Roadmap\n\n## Now (`master`, v1.0.0)\nold news\n", + })), + }); + assert.ok( + behind.issues.some((i) => i.check === "roadmap" && /v1\.0\.0.*1\.2\.3/.test(i.detail)), + "a roadmap two patch/minor behind the shipped version is flagged", + ); + const current = docsCheck({ + root: fixtureRoot((f) => ({ + ...f, + "ROADMAP.md": "# Roadmap\n\n## Now (`master`, v1.2.3)\nfresh\n", + })), + }); + assert.deepEqual( + current.issues.filter((i) => i.check === "roadmap"), + [], + "a roadmap at the shipped version reconciles clean", + ); +}); + test("docsCheck: empty release sections and version mismatch are flagged", () => { const root = fixtureRoot((f) => ({ ...f, diff --git a/test/preflight.test.js b/test/preflight.test.js index 483d887..8fd78df 100644 --- a/test/preflight.test.js +++ b/test/preflight.test.js @@ -8,6 +8,8 @@ import { assessTask, assessTaskLLM, clarifyBlock, + completenessFeatures, + completenessScore, informationGap, preflightRepo, reconcileAssumption, @@ -73,6 +75,74 @@ test("preflightRepo grounds against a real repo (missing file/symbol → clarify assert.ok(!r.unresolved.symbols.includes("computeTax"), "resolved symbol is not flagged"); }); +// --- M2 completeness is a logistic estimator (smooth, bounded, calibrated), not a step scorer --- + +test("completenessScore: logistic output is strictly bounded in (0,1) at feature extremes", () => { + const floor = completenessScore({ + words: 1, + concreteness: 0, + specifics: 0, + vagueness: 12, + length: -1, + }); + const ceil = completenessScore({ + words: 400, + concreteness: 8, + specifics: 8, + vagueness: 0, + length: 1, + }); + assert.ok(floor > 0 && floor < 0.02, "a wall of vagueness floors near 0 without a hard clamp"); + assert.ok(ceil > 0.99 && ceil < 1, "a dense concrete spec saturates near 1 but never reaches it"); +}); + +test("completenessScore: each concrete anchor raises completeness monotonically (no step jumps)", () => { + let prev = -1; + for (let c = 0; c <= 5; c++) { + const s = completenessScore({ + words: 15, + concreteness: c, + specifics: 0, + vagueness: 0, + length: 0, + }); + assert.ok(s > prev, `concreteness ${c} must exceed ${c - 1}`); + prev = s; + } +}); + +test("completenessScore: vagueness pulls down, specifics pull up (signed, attributable)", () => { + const base = { + words: 15, + concreteness: 1, + specifics: 0, + vagueness: 0, + length: 0, + }; + assert.ok(completenessScore({ ...base, vagueness: 2 }) < completenessScore(base)); + assert.ok(completenessScore({ ...base, specifics: 2 }) > completenessScore(base)); +}); + +test("assessTask: completeness is continuous in length — no discontinuity at the old 22/30-word steps", () => { + // The old scorer jumped +0.1/+0.2 crossing 22/30 words; the tanh length term must be smooth, + // so completeness for 29 vs 30 words (same other features) differs only marginally. + const mk = (n) => `Implement parseThing to convert input to output ${"x ".repeat(n).trim()}`; + const a = assessTask(mk(20)).completeness; + const b = assessTask(mk(21)).completeness; + assert.ok(Math.abs(a - b) < 0.03, `adjacent word counts must not jump (${a} vs ${b})`); +}); + +test("assessTask: reproduces the paper's calibrated anchor examples", () => { + const vague = assessTask("make the auth better"); + const clear = assessTask( + "Change verifyToken in src/auth.js to require length > 20; update tests", + ); + assert.ok(vague.completeness < 0.3 && vague.shouldAsk, "bare ask stays under-specified"); + assert.ok(clear.completeness >= 0.6 && !clear.shouldAsk, "a concrete task clears the gate"); + // features are inspectable + assert.equal(completenessFeatures("make the auth better").concreteness, 0); +}); + test("assessTaskLLM: parses a completeness reading, rejects junk", () => { const p = assessTaskLLM("do a thing", { run: () => '{"completeness":0.3,"missing":["target_scope"],"questions":["Which file?"]}', @@ -98,7 +168,11 @@ test("reconcileAssumption: model can only move completeness within ±band of the test("reconcileAssumption: never clears a deterministic / hard-underspecified ask", () => { const det = assessTask("Fix it."); // hardUnderspecified assert.equal(det.shouldAsk, true); - const r = reconcileAssumption(det, { completeness: 0.95, missing: [], questions: [] }); + const r = reconcileAssumption(det, { + completeness: 0.95, + missing: [], + questions: [], + }); assert.equal(r.shouldAsk, true, "the gate only tightens; the model cannot open it"); }); @@ -120,7 +194,11 @@ test("reconcileAssumption: extra questions survive only if grounded or on a flag // An ungrounded question tied to no flagged dimension is dropped. const r2 = reconcileAssumption( det, - { completeness: det.completeness, missing: [], questions: ["Unrelated musing?"] }, + { + completeness: det.completeness, + missing: [], + questions: ["Unrelated musing?"], + }, { grounded: () => false }, ); assert.ok(!r2.questions.includes("Unrelated musing?"), "ungrounded extra question dropped"); @@ -172,7 +250,11 @@ test("bidirectional: a verified raise clears a borderline false ask", () => { }); test("bidirectional: a hard-underspecified task is NEVER cleared", () => { - const det = detStub({ completeness: 0.5, shouldAsk: true, hardUnderspecified: true }); + const det = detStub({ + completeness: 0.5, + shouldAsk: true, + hardUnderspecified: true, + }); const r = reconcileAssumption(det, { completeness: 1, missing: [], questions: [] }, {}); assert.equal(r.shouldAsk, true, "no concrete anchor → the model can't wave it through"); }); From f6cae376f85e58c008b8f665e5cfac729d218d5d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 11 Jul 2026 06:55:54 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix(anchor,docs):=20address=20adversarial?= =?UTF-8?q?=20review=20=E2=80=94=20driftScore=20operating=20point=20+=20gu?= =?UTF-8?q?ard=20robustness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - anchor.js (HIGH): revert driftScore to the off-goal fraction so the CUSUM operating point (k=0.35) is unchanged. With noisy-OR a single-hit on-goal file scored 0.6, and the graded mean(1-score) made a fully on-goal checkpoint accrue 0.4 drift > k, false-alarming after ~20 on-goal prompts. The graded, identifier-aware CLASSIFICATION (the real win) is kept; only the magnitude fed to cusum reverts. - anchor.js (LOW): onGoalScore fuzzy channel is now a >=4-char PREFIX match, not raw substring, so 'port' no longer matches 'report' and hides drift; 'auth'->'authentication' recall preserved. - docs_check.js (LOW): headingSlug keeps Unicode letters (Cafe->cafe, matching GitHub); checkRoadmap guards the package.json read and parses versions prerelease-safe. - Reconcile Biome formatting (fixes the CI Lint & format failure). - Docs + tests updated to the corrected semantics. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- ARCHITECTURE.md | 6 ++- CHANGELOG.md | 15 +++--- docs/GUIDE.md | 14 +++--- src/anchor.js | 112 ++++++++++++++++++-------------------------- src/docs_check.js | 110 ++++++++++++++++--------------------------- test/anchor.test.js | 110 ++++++++++++++----------------------------- 6 files changed, 138 insertions(+), 229 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index fcef803..3d17306 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -260,8 +260,10 @@ intent signal — same math, different stop-set data. were the last hand-static holdouts are now formulas. Goal-drift no longer classifies a changed file by a binary path-substring match; `onGoalScore` is a **noisy-OR** (`1 − (1 − p)^hits`, the same estimator `lessons.js` uses) over how many distinct goal concepts the file exhibits in its -path **and** its atlas-defined identifiers — so `driftScore` is a true continuous Dₜ that -generalizes the old off-goal fraction (identical on 0/1 scores, graded otherwise). The M2 +path **and** its atlas-defined identifiers, thresholded at the single-hit floor — so a file that +implements the goal without naming it in its path is still classed on-goal. `driftScore` stays the +off-goal fraction (the `cusum` operating point is unchanged; an on-goal checkpoint scores 0 and +drains the chart); the grading sharpens _which_ files count as drift, not the detector's tuning. The M2 completeness score `s(x)` is a **logistic** over its features (concreteness, named specifics, vagueness, a smooth `tanh` length term) instead of an additive rubric with magic coefficients and discontinuous word-count steps — the `sigmoid` bounds it to (0,1) with no clamp, every feature's diff --git a/CHANGELOG.md b/CHANGELOG.md index 7627992..3d26ae4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,13 +8,14 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed -- **Goal-drift is graded, not a substring match (`src/anchor.js`).** A changed file's - on-goal-ness is now a **noisy-OR** (`1 − (1 − p)^hits`) over how many distinct goal - concepts it exhibits in its path **and** the identifiers it defines (via the atlas) — - so a file that implements the goal but never names it in its path is caught - deterministically, not just by the opt-in LLM pass. `driftScore` becomes a true - continuous Dₜ that strictly generalizes the old off-goal fraction (identical on 0/1 - scores, graded otherwise), sharpening the CUSUM drift detector's input. +- **Goal-drift classification is graded and identifier-aware (`src/anchor.js`).** A changed + file's on-goal/off-goal call is now a **noisy-OR** (`1 − (1 − p)^hits`) over how many + distinct goal concepts it exhibits in its path **and** the identifiers it defines (via the + atlas), thresholded at the single-hit floor — replacing the binary path-substring match, so + a file that implements the goal but never names it in its path is caught deterministically, + not just by the opt-in LLM pass. `driftScore` stays the off-goal fraction, so the CUSUM + detector's operating point is unchanged (an on-goal checkpoint scores 0 and drains the chart); + the sharper classification is what improves the signal. - **Specification completeness is a logistic estimator (`src/preflight.js`).** The M2 assumption gate's `s(x)` completeness score is now a logistic over its features (concreteness, named specifics, vagueness, a smooth `tanh` length term) — replacing the diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 1972c52..4010137 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -275,13 +275,13 @@ Forge anchor — goal-drift check ``` `src/auth.js` maps to the goal (named file + where `verifyToken` lives); `src/report.js` -doesn't — so it's surfaced as drift to confirm or undo. Advisory by design, and graded: each -file's on-goal confidence is a noisy-OR over how many goal concepts it exhibits in its path -**and** the identifiers it defines (via the atlas), so a file that implements the goal but never -spells it in its path is still caught. `forge substrate` folds this in automatically. The result -carries `driftScore` — the mean off-goal-ness (1 − that confidence) across the checkpoint's -changes, the graded signal the `cusum` change-point detector accumulates to catch _sustained_ -small drift. +doesn't — so it's surfaced as drift to confirm or undo. Advisory by design. The on-goal/off-goal +call is graded and identifier-aware: each file's on-goal confidence is a noisy-OR over how many +goal concepts it exhibits in its path **and** the identifiers it defines (via the atlas), so a +file that implements the goal but never spells it in its path is still classed on-goal. `forge +substrate` folds this in automatically. The result carries `driftScore` — the fraction of the +checkpoint's changes classed off-goal — the signal the `cusum` change-point detector accumulates +to catch _sustained_ small drift (an on-goal checkpoint scores 0 and drains the chart). **The goal persists.** `forge anchor set ""` stores it in `.forge/goal.md`; every new session re-injects it at SessionStart, a bare `forge anchor` checks against it, and diff --git a/src/anchor.js b/src/anchor.js index 7b42355..8815e68 100644 --- a/src/anchor.js +++ b/src/anchor.js @@ -72,25 +72,34 @@ function fileConceptTokens(atlas, file) { export const ON_GOAL_P = 0.6; /** - * Graded on-goal score for one file: a noisy-OR over how many DISTINCT goal concepts the file - * exhibits (in its path or its defined identifiers). 0 hits → 0 (off-goal); k hits → - * 1 − (1 − p)^k, saturating below 1. Same estimator lessons.js uses for multi-signal evidence. - * This replaces the old binary `path.includes(keyword)` verdict with a continuous signal, so - * the CUSUM drift chart accumulates a true Dₜ ∈ [0,1] instead of a quantized off/on count. + * Graded on-goal confidence for one file: a noisy-OR over how many DISTINCT goal concepts the + * file exhibits, across its path tokens AND the identifiers it defines. 0 hits → 0 (off-goal); + * k hits → 1 − (1 − p)^k, saturating below 1. Same estimator lessons.js uses for multi-signal + * evidence. This grades what was a binary `path.includes(keyword)` verdict, and — via the + * identifier channel — catches a file that implements the goal without naming it in its path. + * The classifier thresholds this at ON_GOAL_P; the magnitude expresses confidence. * @param {Set} goalTokens concept tokens of the goal - * @param {Set} fileTokens concept tokens of the file (path ∪ identifiers) - * @param {string} rawPathLower lowercased raw path, for substring hits tokenization can miss - * (goal "auth" in file "authentication.js") — preserves the old match's recall as one channel + * @param {Set} fileTokens concept tokens of the file (path ∪ defined identifiers) * @returns {number} on-goal confidence in [0,1] */ -export function onGoalScore(goalTokens, fileTokens, rawPathLower = "") { +export function onGoalScore(goalTokens, fileTokens) { let hits = 0; - // Exact token match counts at any length; the fuzzy path-substring channel (which preserves - // "auth" ⊂ "authentication") is gated to ≥4 chars so a 3-letter token can't spuriously match - // inside an unrelated word ("log" ⊂ "dialog") and quietly suppress a real drift signal. - for (const g of goalTokens) - if (fileTokens.has(g) || (g.length >= 4 && rawPathLower.includes(g))) + for (const g of goalTokens) { + if (fileTokens.has(g)) { hits++; + continue; + } + // A ≥4-char PREFIX channel matches a goal token to its morphological variants + // ("auth" → "authentication", "valid" → "validation") without the raw-substring collisions + // that let "port" match "report" and silently hide a real drift signal. + if (g.length >= 4) { + for (const t of fileTokens) + if (t.startsWith(g) || (t.length >= 4 && g.startsWith(t))) { + hits++; + break; + } + } + } return hits ? 1 - (1 - ON_GOAL_P) ** hits : 0; } @@ -98,8 +107,7 @@ export function onGoalScore(goalTokens, fileTokens, rawPathLower = "") { // cache and every config file `forge init` emits (dot-paths + AGENTS/CLAUDE). // ponytail: this also hides drift in genuine dot-dir edits (.github/*) — fine for an // advisory coarse check; widen to a managed-file manifest if that ever matters. -const NOISE = - /(^|\/)\.forge\/|(^|\/)\.[^/]+\/|^\.[^/]+$|(^|\/)(AGENTS|CLAUDE)\.md$/i; +const NOISE = /(^|\/)\.forge\/|(^|\/)\.[^/]+\/|^\.[^/]+$|(^|\/)(AGENTS|CLAUDE)\.md$/i; function gitFiles(root) { const run = (args) => { @@ -114,8 +122,7 @@ function gitFiles(root) { } }; const out = - run(["diff", "--name-only", "HEAD"]) + - run(["ls-files", "--others", "--exclude-standard"]); + run(["diff", "--name-only", "HEAD"]) + run(["ls-files", "--others", "--exclude-standard"]); return [ ...new Set( out @@ -133,8 +140,7 @@ function goalTargetFiles(root, symbols, files) { const atlas = loadAtlas(root); if (atlas) for (const s of symbols) - for (const hit of queryAtlas(atlas, s)) - if (hit.name === s) targets.add(hit.file); + for (const hit of queryAtlas(atlas, s)) if (hit.name === s) targets.add(hit.file); return [...targets]; } @@ -186,24 +192,19 @@ export function driftLLM(goal, offGoalFiles, { run = buildRunner() } = {}) { export function goalDrift(root, goal, opts = {}) { const { keywords, symbols, files } = goalKeywords(goal); const changedFiles = opts.changed || gitFiles(root); - const targets = goalTargetFiles(root, symbols, files).map((t) => - t.toLowerCase(), - ); + const targets = goalTargetFiles(root, symbols, files).map((t) => t.toLowerCase()); const atlas = opts.atlas !== undefined ? opts.atlas : loadAtlas(root); const goalTokens = goalConceptTokens(keywords, symbols); const onGoal = []; const offGoal = []; - // Per-file on-goal confidence in [0,1] — the graded signal. A file is classified on-goal iff - // its score clears the single-hit floor (ON_GOAL_P), preserving the old "any match ⇒ on-goal" - // bias, but the magnitude (path + identifier evidence, noisy-OR) is what feeds driftScore. - const score = new Map(); + // Classify each file on- vs off-goal by a graded, identifier-aware confidence (noisy-OR over + // path + defined-identifier concept hits) thresholded at the single-hit floor ON_GOAL_P — this + // replaces the old binary path-substring match and, via identifiers, catches a file that + // implements the goal without naming it in its path. It preserves the "any match ⇒ on-goal" bias. for (const f of changedFiles) { const lf = f.toLowerCase(); const named = targets.some((t) => lf === t || lf.endsWith(`/${t}`)); - const s = named - ? 1 - : onGoalScore(goalTokens, fileConceptTokens(atlas, f), lf); - score.set(f, s); + const s = named ? 1 : onGoalScore(goalTokens, fileConceptTokens(atlas, f)); (s >= ON_GOAL_P ? onGoal : offGoal).push(f); } @@ -211,20 +212,13 @@ export function goalDrift(root, goal, opts = {}) { // the model gives a goal-referencing reason. Verified, not trusted; fail-safe on any error. let provenance = { path: "deterministic" }; if (llmEnabled({ llm: opts.llm }) && offGoal.length) { - const goalTerms = new Set([ - ...keywords, - ...symbols.map((s) => s.toLowerCase()), - ]); + const goalTerms = new Set([...keywords, ...symbols.map((s) => s.toLowerCase())]); const grounded = (reason) => { const words = String(reason).toLowerCase(); - return ( - goalTerms.size === 0 || [...goalTerms].some((t) => words.includes(t)) - ); + return goalTerms.size === 0 || [...goalTerms].some((t) => words.includes(t)); }; const proposal = driftLLM(goal, offGoal, { - run: - opts.run || - buildRunner({ model: opts.model, timeoutMs: opts.timeoutMs }), + run: opts.run || buildRunner({ model: opts.model, timeoutMs: opts.timeoutMs }), }); const rescued = new Set( (proposal?.onGoal || []) @@ -236,8 +230,6 @@ export function goalDrift(root, goal, opts = {}) { if (rescued.has(f)) { offGoal.splice(offGoal.indexOf(f), 1); onGoal.push(f); - // A verified rescue is on-goal at the single-hit floor, so driftScore reflects it too. - score.set(f, Math.max(score.get(f) ?? 0, ON_GOAL_P)); } } provenance = { path: "llm-verified", rescued: [...rescued] }; @@ -246,17 +238,13 @@ export function goalDrift(root, goal, opts = {}) { } } - const drift = - changedFiles.length > 0 && (offGoal.length > 0 || onGoal.length === 0); - // Graded drift magnitude Dₜ ∈ [0,1] — the mean off-goal-ness (1 − on-goal confidence) across - // this checkpoint's changes. This is the signal cusum() below expects: the binary `drift` flag - // answers "any drift now?", the score accumulates into "sustained drift?". It strictly - // generalizes the old off-goal fraction — identical when every score is 0/1, graded otherwise, - // so a weakly-on-goal file adds less drift than an unrelated one instead of counting the same. - const driftScore = changedFiles.length - ? changedFiles.reduce((a, f) => a + (1 - (score.get(f) ?? 0)), 0) / - changedFiles.length - : 0; + const drift = changedFiles.length > 0 && (offGoal.length > 0 || onGoal.length === 0); + // Drift magnitude Dₜ ∈ [0,1] = the fraction of this checkpoint's changes classified off-goal. + // The grading lives in the CLASSIFIER (identifier-aware noisy-OR above), which sharpens WHICH + // files count as drift; the fraction itself is kept as the cusum() input so the detector's + // operating point (allowance k, threshold h) is unchanged — an on-goal checkpoint scores 0 and + // drains the chart, exactly as before, rather than accruing residual drift on legitimate work. + const driftScore = changedFiles.length ? offGoal.length / changedFiles.length : 0; return { goal: String(goal), keywords, @@ -304,21 +292,11 @@ export function renderAnchor(r) { ` changed: ${r.changed.length} file(s) · on-goal ${r.onGoal.length} · off-goal ${r.offGoal.length}`, ); if (r.offGoal.length) { - lines.push( - "", - " off-goal (unrelated to the stated goal — intended, or drift?):", - ); + lines.push("", " off-goal (unrelated to the stated goal — intended, or drift?):"); for (const f of r.offGoal.slice(0, 12)) lines.push(` - ${f}`); } if (r.drift && !r.offGoal.length) - lines.push( - "", - " ! no changed file matches the goal — are you working on the right thing?", - ); - if (!r.drift) - lines.push( - "", - " ✓ on goal — every change maps to what you set out to do.", - ); + lines.push("", " ! no changed file matches the goal — are you working on the right thing?"); + if (!r.drift) lines.push("", " ✓ on goal — every change maps to what you set out to do."); return lines.join("\n"); } diff --git a/src/docs_check.js b/src/docs_check.js index 2d5651a..4e48a05 100644 --- a/src/docs_check.js +++ b/src/docs_check.js @@ -13,25 +13,15 @@ import { TOOLS } from "./mcp_tools.js"; import { MODELS } from "./model_tiers.js"; /** The user-facing prose docs every claim is reconciled against. */ -const DOC_FILES = [ - "README.md", - "docs/GUIDE.md", - "ARCHITECTURE.md", - "ROADMAP.md", -]; +const DOC_FILES = ["README.md", "docs/GUIDE.md", "ARCHITECTURE.md", "ROADMAP.md"]; // Env vars read in src that are NOT user-facing contract: child-process plumbing and // values injected by host tools rather than set by users. -const INTERNAL_ENV = new Set([ - "_FORGE_LLM_KEY", - "FORGE_EMBED_KEY", - "CLAUDE_PLUGIN_ROOT", -]); +const INTERNAL_ENV = new Set(["_FORGE_LLM_KEY", "FORGE_EMBED_KEY", "CLAUDE_PLUGIN_ROOT"]); // Prefixes that mark an env var as OURS to document. A doc may freely mention other // tools' vars (GITHUB_TOKEN, PATH) — those aren't claims about forge's own surface. -const ENV_PREFIX_RE = - /\b((?:FORGE|ANTHROPIC|LITELLM|OPENROUTER|ENABLE_CORTEX)_[A-Z0-9_]+)\b/g; +const ENV_PREFIX_RE = /\b((?:FORGE|ANTHROPIC|LITELLM|OPENROUTER|ENABLE_CORTEX)_[A-Z0-9_]+)\b/g; function readDoc(root, rel) { const p = join(root, rel); @@ -54,12 +44,8 @@ export function envVarsRead(root = BRAND.root) { const vars = new Set(); for (const file of srcFiles(root)) { const text = readFileSync(file, "utf8"); - for (const m of text.matchAll(/process\.env\.([A-Z_][A-Z0-9_]*)/g)) - vars.add(m[1]); - for (const m of text.matchAll( - /process\.env\[["']([A-Z_][A-Z0-9_]*)["']\]/g, - )) - vars.add(m[1]); + for (const m of text.matchAll(/process\.env\.([A-Z_][A-Z0-9_]*)/g)) vars.add(m[1]); + for (const m of text.matchAll(/process\.env\[["']([A-Z_][A-Z0-9_]*)["']\]/g)) vars.add(m[1]); } const guards = join(root, "global", "guards"); if (existsSync(guards)) { @@ -94,9 +80,7 @@ function checkCommands(docs, issues) { } } for (const [file, text] of Object.entries(docs)) { - for (const m of text.matchAll( - new RegExp(`\`${BRAND.cli} ([a-z][a-z-]*)\\b`, "g"), - )) { + for (const m of text.matchAll(new RegExp(`\`${BRAND.cli} ([a-z][a-z-]*)\\b`, "g"))) { const name = m[1]; if (!(name in COMMANDS) && !HIDDEN_COMMANDS.includes(name)) { issues.push({ @@ -126,11 +110,7 @@ function checkEnvVars(root, docs, issues) { const documented = new Set(); for (const [file, text] of Object.entries(docs)) { for (const m of text.matchAll(ENV_PREFIX_RE)) { - if ( - !documented.has(`${file}:${m[1]}`) && - !read.has(m[1]) && - !INTERNAL_ENV.has(m[1]) - ) { + if (!documented.has(`${file}:${m[1]}`) && !read.has(m[1]) && !INTERNAL_ENV.has(m[1])) { documented.add(`${file}:${m[1]}`); issues.push({ check: "env-vars", @@ -188,12 +168,7 @@ function markdownFiles(root) { if (!existsSync(root)) return []; return readdirSync(root, { recursive: true }) .map(String) - .filter( - (f) => - f.endsWith(".md") && - !f.includes("node_modules") && - !f.startsWith(".git/"), - ); + .filter((f) => f.endsWith(".md") && !f.includes("node_modules") && !f.startsWith(".git/")); } // The branded Mermaid theme every diagram shares (see README's `%%{init …}%%`). Without it @@ -217,10 +192,7 @@ function checkDiagrams(root, issues) { for (const m of text.matchAll(MERMAID_BLOCK_RE)) { // An intentional example block (e.g. docs showing what a BAD diagram looks like) opts // out with an HTML comment `` on the line before the fence. - if ( - /docs-check-ignore/.test(text.slice(Math.max(0, m.index - 80), m.index)) - ) - continue; + if (/docs-check-ignore/.test(text.slice(Math.max(0, m.index - 80), m.index))) continue; const block = m[1]; if (!block.includes("%%{init")) { issues.push({ @@ -246,13 +218,9 @@ function checkChangelog(root, issues) { const text = readDoc(root, "CHANGELOG.md"); if (!text) return; const sections = [ - ...text.matchAll( - /^## \[([^\]]+)\][^\n]*\n([\s\S]*?)(?=^## \[|\n*$(?![\s\S]))/gm, - ), + ...text.matchAll(/^## \[([^\]]+)\][^\n]*\n([\s\S]*?)(?=^## \[|\n*$(?![\s\S]))/gm), ]; - const version = JSON.parse( - readFileSync(join(root, "package.json"), "utf8"), - ).version; + const version = JSON.parse(readFileSync(join(root, "package.json"), "utf8")).version; const released = sections.filter((s) => s[1].toLowerCase() !== "unreleased"); if (released.length && released[0][1] !== version) { issues.push({ @@ -272,18 +240,13 @@ function checkChangelog(root, issues) { } const unreleased = sections.find((s) => s[1].toLowerCase() === "unreleased"); if (unreleased && !unreleased[2].trim()) { - const srcT = Number( - git(root, ["log", "-1", "--format=%ct", "--", "src"]) || 0, - ); - const clT = Number( - git(root, ["log", "-1", "--format=%ct", "--", "CHANGELOG.md"]) || 0, - ); + const srcT = Number(git(root, ["log", "-1", "--format=%ct", "--", "src"]) || 0); + const clT = Number(git(root, ["log", "-1", "--format=%ct", "--", "CHANGELOG.md"]) || 0); if (srcT && clT && srcT > clT) { issues.push({ check: "changelog", severity: "error", - detail: - "src changed since CHANGELOG.md was last touched, but [Unreleased] is empty", + detail: "src changed since CHANGELOG.md was last touched, but [Unreleased] is empty", }); } } @@ -319,8 +282,7 @@ function measuredTimings(root) { const set = new Set(); for (const line of readDoc(root, "reports/benchmarks.md").split("\n")) { if (!line.startsWith("|")) continue; // table rows only — not the prose above it - for (const m of line.matchAll(/(\d+(?:\.\d+)?)\s*(ms|µs|s)\b/g)) - set.add(`${m[1]} ${m[2]}`); + for (const m of line.matchAll(/(\d+(?:\.\d+)?)\s*(ms|µs|s)\b/g)) set.add(`${m[1]} ${m[2]}`); } return set; } @@ -356,12 +318,16 @@ function checkBenchmarks(root, docs, issues) { // links these docs use; we only ever FLAG a miss, so a rare slug mismatch degrades to a false // alarm we'd see immediately, never a silent wrong link. function headingSlug(text) { - return text - .trim() - .toLowerCase() - .replace(/`/g, "") - .replace(/[^\w\s-]/g, "") - .replace(/\s/g, "-"); // each space → one hyphen; GitHub does NOT collapse (an em-dash between + return ( + text + .trim() + .toLowerCase() + .replace(/`/g, "") + // Drop punctuation but KEEP any Unicode letter/number (so "Café" → "café", matching GitHub, + // rather than ASCII-only `\w` which would strip the accent and false-flag a valid link). + .replace(/[^\p{L}\p{N}\s_-]/gu, "") + .replace(/\s/g, "-") + ); // each space → one hyphen; GitHub does NOT collapse (an em-dash between // two words becomes "--", not "-"), so consecutive spaces must map 1:1. } @@ -369,10 +335,8 @@ function headingSlug(text) { // (``, `name=…`, or a `{#custom-id}` suffix). function anchorsFor(text) { const set = new Set(); - for (const m of text.matchAll(/^#{1,6}\s+(.+?)\s*#*\s*$/gm)) - set.add(headingSlug(m[1])); - for (const m of text.matchAll(/\b(?:id|name)=["']([\w-]+)["']/g)) - set.add(m[1].toLowerCase()); + for (const m of text.matchAll(/^#{1,6}\s+(.+?)\s*#*\s*$/gm)) set.add(headingSlug(m[1])); + for (const m of text.matchAll(/\b(?:id|name)=["']([\w-]+)["']/g)) set.add(m[1].toLowerCase()); for (const m of text.matchAll(/\{#([\w-]+)\}/g)) set.add(m[1].toLowerCase()); return set; } @@ -419,8 +383,7 @@ function checkLinks(root, issues) { let targetRel; if (!path) targetRel = rel; // same-file anchor - else if (path.endsWith(".md")) - targetRel = normalize(join(dirname(rel), path)); + else if (path.endsWith(".md")) targetRel = normalize(join(dirname(rel), path)); else continue; // .html/.pdf/code target — can't resolve headings, skip const anchors = anchorsOf(targetRel); if (anchors == null) continue; // unreadable/missing target — file existence is another matter @@ -448,13 +411,20 @@ function checkRoadmap(root, issues) { const vm = now[0].match(/v(\d+)\.(\d+)(?:\.(\d+))?/); if (!vm) return; const road = [Number(vm[1]), Number(vm[2]), Number(vm[3] || 0)]; - const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8")) - .version.split(".") - .map(Number); + let pkg; + try { + // Parse leading integers per component so a prerelease tag ("1.2.3-beta.1") still yields + // [1,2,3], not NaN; guard the read so a missing/corrupt manifest can't abort the whole check. + const raw = JSON.parse(readFileSync(join(root, "package.json"), "utf8")).version || ""; + const pv = raw.match(/(\d+)\.(\d+)(?:\.(\d+))?/); + if (!pv) return; + pkg = [Number(pv[1]), Number(pv[2]), Number(pv[3] || 0)]; + } catch { + return; + } const behind = road[0] < pkg[0] || - (road[0] === pkg[0] && - (road[1] < pkg[1] || (road[1] === pkg[1] && road[2] < pkg[2]))); + (road[0] === pkg[0] && (road[1] < pkg[1] || (road[1] === pkg[1] && road[2] < pkg[2]))); if (behind) { issues.push({ check: "roadmap", diff --git a/test/anchor.test.js b/test/anchor.test.js index 9957679..0495d3a 100644 --- a/test/anchor.test.js +++ b/test/anchor.test.js @@ -1,12 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { - cusum, - goalDrift, - ON_GOAL_P, - onGoalScore, - renderAnchor, -} from "../src/anchor.js"; +import { cusum, goalDrift, ON_GOAL_P, onGoalScore, renderAnchor } from "../src/anchor.js"; // changed[] is injected so these are pure (no git needed). test("goalDrift flags a changed file unrelated to the goal", () => { @@ -49,17 +43,13 @@ test("goalDrift (llm on): rescues an off-goal file only with a goal-referencing llm: true, run, }); - assert.ok( - r.onGoal.includes("src/throttle.js"), - "grounded reason moves it off→on", - ); + assert.ok(r.onGoal.includes("src/throttle.js"), "grounded reason moves it off→on"); assert.ok(!r.offGoal.includes("src/throttle.js")); assert.equal(r.provenance.path, "llm-verified"); }); test("goalDrift (llm on): a reason that never references the goal is rejected (verify, don't trust)", () => { - const run = () => - '{"onGoal":[{"file":"src/reports.js","reason":"just tidying up"}]}'; + const run = () => '{"onGoal":[{"file":"src/reports.js","reason":"just tidying up"}]}'; const r = goalDrift("/nope", "add rate limiting to the login route", { changed: ["src/reports.js"], llm: true, @@ -76,10 +66,7 @@ test("goalDrift (llm on): the model can never move a file on→off", () => { llm: true, run, }); - assert.ok( - r.onGoal.includes("src/login.js"), - "deterministic on-goal is never demoted", - ); + assert.ok(r.onGoal.includes("src/login.js"), "deterministic on-goal is never demoted"); }); test("goalDrift (llm on): a throwing runner falls back to the deterministic split", () => { @@ -108,25 +95,18 @@ test("onGoalScore: 0 hits → 0, and each extra hit raises confidence with dimin const one = onGoalScore(goal, new Set(["login"])); const two = onGoalScore(goal, new Set(["login", "rate"])); const three = onGoalScore(goal, new Set(["login", "rate", "limit"])); - assert.ok( - Math.abs(one - ON_GOAL_P) < 1e-9, - "a single hit sits at the on-goal floor", - ); - assert.ok( - two > one && three > two, - "more independent evidence ⇒ higher, monotone", - ); + assert.ok(Math.abs(one - ON_GOAL_P) < 1e-9, "a single hit sits at the on-goal floor"); + assert.ok(two > one && three > two, "more independent evidence ⇒ higher, monotone"); assert.ok(three < 1, "noisy-OR saturates below 1, never false-certain"); }); -test("onGoalScore: substring channel preserves the old match's recall (auth ⊂ authentication)", () => { - const goal = new Set(["auth"]); - // tokenize would keep 'authentication' whole, so token-equality misses it; the raw-path - // substring channel still counts the hit, so we never regress below the old behaviour. - assert.ok( - onGoalScore(goal, new Set(["authentication"]), "src/authentication.js") >= - ON_GOAL_P, - ); +test("onGoalScore: the ≥4-char prefix channel matches morphological variants but not collisions", () => { + // "auth" is a prefix of the file token "authentication" → a hit (recall preserved). But + // "port" is NOT a prefix of "report" (only a raw substring) → NO hit, so an unrelated file + // can't be wrongly scored on-goal and hide real drift. + assert.ok(onGoalScore(new Set(["auth"]), new Set(["authentication"])) >= ON_GOAL_P); + assert.ok(onGoalScore(new Set(["valid"]), new Set(["validation"])) >= ON_GOAL_P); + assert.equal(onGoalScore(new Set(["port"]), new Set(["report"])), 0, "substring ≠ prefix"); }); test("goalDrift: an atlas identifier rescues an implement-the-goal file the PATH never names", () => { @@ -142,57 +122,43 @@ test("goalDrift: an atlas identifier rescues an implement-the-goal file the PATH changed: ["src/throttle.js", "src/reports.js"], atlas, }); - assert.ok( - r.onGoal.includes("src/throttle.js"), - "identifier match ⇒ on-goal without the LLM", - ); - assert.ok( - r.offGoal.includes("src/reports.js"), - "a genuinely unrelated file still drifts", - ); + assert.ok(r.onGoal.includes("src/throttle.js"), "identifier match ⇒ on-goal without the LLM"); + assert.ok(r.offGoal.includes("src/reports.js"), "a genuinely unrelated file still drifts"); assert.equal(r.provenance.path, "deterministic"); }); -test("goalDrift: driftScore is graded — a weak match drifts less than an unrelated file", () => { - const weak = goalDrift("/nope", "add rate limiting to the login route", { - changed: ["src/login.js"], // one concept hit (login) +test("goalDrift: an on-goal file contributes NO drift; a fully off-goal checkpoint drifts at 1.0", () => { + // driftScore is the off-goal FRACTION (the cusum operating point), so a classified-on-goal + // file must drain the chart, not accrue residual drift — the regression the review caught. + const on = goalDrift("/nope", "add rate limiting to the login route", { + changed: ["src/login.js"], // one concept hit → on-goal atlas: null, }); const off = goalDrift("/nope", "add rate limiting to the login route", { - changed: ["src/reports.js"], // zero hits + changed: ["src/reports.js"], // zero hits → off-goal atlas: null, }); - assert.ok( - weak.driftScore < off.driftScore, - "a partial match accrues less drift than none", - ); - assert.ok( - Math.abs(off.driftScore - 1) < 1e-9, - "a fully off-goal checkpoint drifts at 1.0", - ); + assert.equal(on.driftScore, 0, "an on-goal checkpoint scores exactly 0 (drains the cusum chart)"); + assert.equal(off.driftScore, 1, "a fully off-goal checkpoint drifts at 1.0"); }); -test("goalDrift: on a partial (0.6) + off (0) checkpoint, driftScore is the graded mean of 1−score", () => { +test("goalDrift: driftScore is the off-goal fraction (1 on-goal + 1 off-goal → 0.5)", () => { const r = goalDrift("/nope", "billing invoice totals", { - changed: ["src/billing.js", "src/reports.js"], // billing → 1 concept hit = 0.6; reports → 0 + changed: ["src/billing.js", "src/reports.js"], // billing → on-goal (1 hit); reports → off-goal atlas: null, }); - // graded, not quantized: mean(1−s) = (0.4 + 1)/2 = 0.7, where the legacy off/n would say 0.5. - assert.ok(Math.abs(r.driftScore - 0.7) < 1e-9); + assert.ok(r.onGoal.includes("src/billing.js") && r.offGoal.includes("src/reports.js")); + assert.ok(Math.abs(r.driftScore - 0.5) < 1e-9, "1 of 2 files off-goal → 0.5"); }); -test("goalDrift: driftScore reduces EXACTLY to the legacy off-goal fraction on genuinely binary scores", () => { - // A goal that NAMES a file scores it exactly 1.0 (named target), and an unrelated file scores 0 - // — a real {0,1} checkpoint. Here the graded mean(1−score) must equal the old offGoal/changed. +test("goalDrift: a goal that NAMES a file classifies that file on-goal (named target ⇒ score 1)", () => { const r = goalDrift("/nope", "update src/pay.js", { - changed: ["src/pay.js", "src/reports.js"], // pay.js is the named target → 1; reports → 0 + changed: ["src/pay.js", "src/reports.js"], // pay.js is the named target; reports is unrelated atlas: null, }); - assert.ok( - r.onGoal.includes("src/pay.js") && r.offGoal.includes("src/reports.js"), - ); - // mean(1−score) = (0 + 1)/2 = 0.5 = legacy offGoal(1)/changed(2). Exact reduction on 0/1 scores. - assert.ok(Math.abs(r.driftScore - 0.5) < 1e-9); + assert.ok(r.onGoal.includes("src/pay.js"), "the named file anchors on-goal"); + assert.ok(r.offGoal.includes("src/reports.js")); + assert.ok(Math.abs(r.driftScore - 0.5) < 1e-9, "1 of 2 off-goal → 0.5"); }); // --------------------------------------------------------------------------- @@ -204,11 +170,7 @@ test("cusum alarms on sustained small drift (the decaying-anchor failure)", () = // alarming, but the excess accumulates: C = 0.25, 0.5, 0.75, 1.0, 1.25 → alarm. const r = cusum([0.6, 0.6, 0.6, 0.6, 0.6, 0.6]); assert.equal(r.alarm, true); - assert.equal( - r.firstAlarm, - 4, - "alarms at the first checkpoint where C exceeds h", - ); + assert.equal(r.firstAlarm, 4, "alarms at the first checkpoint where C exceeds h"); assert.equal(r.C.length, 6); assert.ok(Math.abs(r.C[0] - 0.25) < 1e-9); }); @@ -217,11 +179,7 @@ test("cusum does not alarm on a single spike within tolerance", () => { const r = cusum([0.1, 1.2, 0.1, 0.1, 0.1, 0.1]); assert.equal(r.alarm, false); assert.equal(r.firstAlarm, -1); - assert.equal( - r.C.at(-1), - 0, - "the statistic drains back to zero after the spike", - ); + assert.equal(r.C.at(-1), 0, "the statistic drains back to zero after the spike"); }); test("cusum is quiet on on-goal signals and on an empty series", () => {