diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 643d9ec..40ef460 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -3,12 +3,12 @@ "name": "forgekit", "displayName": "Forge", "version": "0.3.1", - "description": "One config, every AI coding tool — tools, crew, guards, atlas, lean, recall from one source.", + "description": "One config, every AI coding tool — cognitive substrate, tools, crew, guards, atlas, lean, recall from one source.", "author": { "name": "CodeWithJuber" }, "license": "MIT", "repository": "https://github.com/CodeWithJuber/forgekit", "homepage": "https://github.com/CodeWithJuber/forgekit#readme", - "keywords": ["config", "cross-tool", "agents-md", "ai-coding", "claude-code"], + "keywords": ["config", "cross-tool", "agents-md", "ai-coding", "claude-code", "cognitive-substrate"], "skills": "./global/tools", "agents": "./global/crew", "hooks": "./hooks/hooks.json" diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..ff0d99c --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,29 @@ +{ + "name": "forgekit", + "version": "0.3.1", + "description": "One config, every AI coding tool — cognitive substrate, MCP tools, guards, atlas, recall, and routing from one source.", + "author": { "name": "CodeWithJuber", "url": "https://github.com/CodeWithJuber" }, + "homepage": "https://github.com/CodeWithJuber/forgekit#readme", + "repository": "https://github.com/CodeWithJuber/forgekit", + "license": "MIT", + "keywords": ["codex", "mcp", "ai-coding", "agents-md", "cognitive-substrate"], + "skills": "skills", + "mcpServers": ".mcp.json", + "interface": { + "displayName": "Forge", + "shortDescription": "Cognitive substrate and one config for every AI coding tool.", + "longDescription": "Forge adds a pre-action cognitive substrate for coding agents: assumption gating, transparent model routing, impact prediction, scope decomposition, memory/learning context, and verification checklists. It also emits shared rules and MCP config for Codex, Claude Code, Cursor, Gemini, Aider, Copilot, Windsurf/Devin, Zed, Continue, and Roo.", + "developerName": "CodeWithJuber", + "category": "Productivity", + "capabilities": ["MCP", "Skills", "Code Review", "Verification"], + "websiteURL": "https://github.com/CodeWithJuber/forgekit", + "privacyPolicyURL": "https://github.com/CodeWithJuber/forgekit/blob/master/SECURITY.md", + "termsOfServiceURL": "https://github.com/CodeWithJuber/forgekit/blob/master/LICENSE", + "defaultPrompt": [ + "Run Forge substrate before this edit.", + "Predict impact for this symbol.", + "Route this task to the right model tier." + ], + "brandColor": "#137A72" + } +} diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8b702c6..d17586e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,6 +9,7 @@ - [ ] Conventional commit message (`feat:`/`fix:`/`docs:` …) - [ ] `CHANGELOG.md` updated under `## [Unreleased]` - [ ] No new runtime dependency (dev deps ok) +- [ ] Substrate/docs updated if this changes `forge substrate`, `forge impact`, router/gate, or MCP substrate tools ## Risk & rollback - Risk level: low / medium / high diff --git a/.mcp.json b/.mcp.json new file mode 100644 index 0000000..ad111ea --- /dev/null +++ b/.mcp.json @@ -0,0 +1,8 @@ +{ + "mcpServers": { + "forge-cortex": { + "command": "forge", + "args": ["cortex-mcp"] + } + } +} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ebef124..ad0e530 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -9,6 +9,11 @@ Grounded in a verified multi-source research pass (Reddit/HN/dev.to, GitHub issue trackers, official vendor docs). Evidence + config-format verdicts in the appendix. +The cognitive-substrate paper bundle is committed under +[`docs/cognitive-substrate/`](docs/cognitive-substrate/). It includes the full PDF/HTML +paper, deliverable overview, evidence map, ecosystem map, and the original prototype +packages; the production runtime remains Node-only and zero-dependency. + ## Locked decisions - **Brand = `Forge`** — CLI `forge`; layer names: skills→**tools**, agents→**crew**, hooks→**guards**, code-graph→**atlas**, minimalism→**lean**, memory→**recall**. @@ -58,6 +63,10 @@ Layers map onto the Claude Code substrate, brand-named, and are emitted cross-to Cross-cutting: **atlas** (code-graph), **lean** (shipped as *both* a tool and a Stop-guard, so it works whether or not the model invokes it), **recall** (memory). +**cognitive substrate** (`forge substrate`, `forge impact`, and MCP tools +`substrate_check` / `predict_impact` / `assumption_gate`) composes atlas, preflight, +route, scope, cortex, and verify into one pre-action contract before mutating work. + ## Component map — the reuse ledger (30 components) **Reuse (rename + swap brand token, logic unchanged):** diff --git a/CHANGELOG.md b/CHANGELOG.md index aedc8ed..c12496a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Forge Cognitive Substrate** — one pre-action command (`forge substrate`) and MCP surface (`substrate_check`, `predict_impact`, `assumption_gate`) that combines assumption gating, transparent model routing, impact prediction, scope decomposition, Cortex lessons, minimality warnings, and verification planning. +- **Atlas v2 graph** — dependency nodes/edges, file hashes, and reverse-dependency impact traversal while preserving the old symbol query API. +- Codex plugin manifest and `cognitive-substrate` skill so Forge can be installed/used from Codex-style extension surfaces as well as Claude/NPM. +- Cognitive-substrate paper bundle under `docs/cognitive-substrate/`: full PDF/HTML paper, deliverable overview, evidence map, ecosystem map, and original prototype packages. + ## [0.3.1] - 2026-07-05 ### Changed @@ -76,7 +83,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). check; coverage + type-checking (`tsc --checkJs`); 2026 production-standard rules; OWASP-LLM / NIST SSDF / SLSA control mapping. -[Unreleased]: https://github.com/CodeWithJuber/forgekit/compare/v0.3.0...HEAD +[Unreleased]: https://github.com/CodeWithJuber/forgekit/compare/v0.3.1...HEAD +[0.3.1]: https://github.com/CodeWithJuber/forgekit/compare/v0.3.0...v0.3.1 [0.3.0]: https://github.com/CodeWithJuber/forgekit/compare/v0.2.0...v0.3.0 [0.2.0]: https://github.com/CodeWithJuber/forgekit/compare/v0.1.0...v0.2.0 [0.1.0]: https://github.com/CodeWithJuber/forgekit/releases/tag/v0.1.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d35600b..e58c167 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,6 +41,8 @@ npm run check # Biome lint + format check - `source/` — the single rule + MCP source that `forge sync` compiles. - `global/` — what installs into `~/.forge`: `tools/` (skills), `crew/` (agents), `guards/` (hooks). +- `docs/cognitive-substrate/` — the committed paper bundle, evidence maps, and prototype + artifacts that explain `forge substrate` and `forge impact`. - `test/` — `node --test` suites. ## Before you open a PR @@ -55,6 +57,10 @@ diffs with no tests) will be closed. To get merged: header, or files under `.forge/`. Change the source (`source/`) instead. - **Include tests and pass CI.** A small green diff a maintainer can verify beats a large one they can't. +- **Substrate changes need docs.** If you change `forge substrate`, `forge impact`, + router/gate rubrics, or MCP substrate tools, update + [docs/cognitive-substrate/README.md](./docs/cognitive-substrate/README.md) and + `CHANGELOG.md`. We'd rather give a clear "not now" than merge something that adds maintenance burden — see [GOVERNANCE.md](./GOVERNANCE.md). diff --git a/ONBOARDING.md b/ONBOARDING.md index 4527bf3..f0ca1b0 100644 --- a/ONBOARDING.md +++ b/ONBOARDING.md @@ -28,7 +28,19 @@ Edit `source/rules.json` (or drop a per-repo `.forge/rules.json`), then: forge sync # recompiles into every tool; idempotent (only rewrites what changed) ``` -## 4. Use the extras +## 4. Use the cognitive substrate + +```bash +forge substrate "" # ask/route/impact/scope/memory/verify in one pass +forge substrate "" --json +forge impact +``` + +If `forge substrate` says `ASK FIRST`, ask the returned questions before editing. Read predicted impacted files before making mutating changes. + +Paper and evidence package: [docs/cognitive-substrate/](docs/cognitive-substrate/). + +## 5. Use the extras ```bash forge atlas build # index this repo's symbols → .forge/atlas.json diff --git a/README.md b/README.md index 7eaa6f9..367226f 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ Cursor, Gemini CLI, Aider, Copilot, Windsurf/Devin, and Zed**. # clone, then: bash install.sh # symlinks into ~/.forge + ~/.claude, puts `forge` on PATH forge init # in any repo: emit every tool's config from one source +forge substrate "task" # assumption gate + route + impact + scope + verify forge doctor # verify everything is wired ``` @@ -91,6 +92,22 @@ On **Claude Code** it's fully ambient (hooks). Other tools read the lessons from and a zero-dependency MCP server (`forge cortex-mcp`). Everything lives in `.forge/lessons/` — git-committable and auditable. Try it: `node examples/cortex-demo.mjs`. +## Forge Cognitive Substrate — one pre-action gate + +Forge now wraps the agent loop with the paper's cognitive-substrate controls. Run one command before ambiguous, expensive, or mutating work: + +```bash +forge substrate "Fix the checkout bug in `src/payments.ts` and add tests" +forge substrate "Refactor auth" --json +forge impact computeTax +``` + +`substrate` returns the assumption gate, model route, impact radius, scope clusters, relevant Cortex lessons, minimality warnings, and a verification checklist. MCP-capable extensions get the same flow through `substrate_check`, `predict_impact`, and `assumption_gate`. Deterministic checks are asserted; memory relevance, routing fit, and minimality remain advisory where the research is not a hard guarantee. + +Full paper bundle and original artifacts live in +[`docs/cognitive-substrate/`](docs/cognitive-substrate/): PDF, HTML, evidence map, +ecosystem map, and the original router-gate / impact-oracle prototype packages. + ## Forge Preflight — size the work before spending tokens An LLM is a fixed-capacity stochastic predictor. Most of the cost/quality bleed comes from @@ -122,6 +139,8 @@ forge sync recompile source/ → each tool's native files (idempotent) forge doctor pass/fail health check (layers, install, drift, cortex) forge catalog Start-Here index of every tool/crew/guard forge cortex self-correcting memory — status / why +forge substrate full pre-action cognitive-substrate check +forge impact predict blast radius for a symbol or file forge preflight assumption check — what a task names that the repo doesn't define forge route cheapest capable model for a task (+ gateway config) forge scope decompose files into independent clusters diff --git a/ROADMAP.md b/ROADMAP.md index 27e701a..cb7abc8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,10 +5,14 @@ Direction, not promises — shaped by the two field reports this project is grou ## Now (in `main`) Cross-tool config + MCP emit (8 tools), verification layer (`forge verify`), security gate -(`forge scan`), portable memory (`forge brain`), cost governor (`forge cost`). See -[CHANGELOG.md](./CHANGELOG.md) `[Unreleased]`. +(`forge scan`), portable memory (`forge brain`), cost governor (`forge cost`), and the +cognitive-substrate pre-action gate (`forge substrate`, `forge impact`). The full paper +and evidence bundle live in [docs/cognitive-substrate/](./docs/cognitive-substrate/). +See [CHANGELOG.md](./CHANGELOG.md) `[Unreleased]`. ## Next +- **Substrate calibration fixtures** — expand assumption/routing/impact fixtures beyond + the original paper prototypes while keeping research-edge claims advisory. - **spec-lock** — spec-as-contract drift detection, reusing the `atlas` index. - **Testing** — Playwright MCP/agents opt-in + a coverage Stop-hook gate. - **MCP hygiene** — enforce the ~6-server cap + a registry resolver in `forge doctor`. diff --git a/SUPPORT.md b/SUPPORT.md index 663c504..b1d685c 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -6,6 +6,9 @@ Thanks for using forgekit. Here's where to get help. - **Bugs** → open an issue with the bug template (include `forge doctor` output + `node --version`). - **Feature ideas** → the feature-request template; for larger changes, start a Discussion first. - **Security vulnerabilities** → **do not** open a public issue — see [SECURITY.md](./SECURITY.md). +- **Cognitive substrate questions** → start with + [docs/cognitive-substrate/](./docs/cognitive-substrate/) and include + `forge substrate "" --json` output when reporting a bad ask/route/impact decision. Before opening an issue: run `forge doctor`, search existing issues/discussions, and confirm you're on the latest version (`forge --version` vs `npm view forgekit version`). diff --git a/docs/cognitive-substrate/README.md b/docs/cognitive-substrate/README.md new file mode 100644 index 0000000..5e85ea3 --- /dev/null +++ b/docs/cognitive-substrate/README.md @@ -0,0 +1,74 @@ +# Cognitive Substrate Paper Bundle + +This directory is the research-backed product map for Forge's cognitive-substrate system. +It keeps the full paper, evidence maps, and original prototype packages next to the +production Node implementation so users can install once and use the system from any +Forge-supported agent. + +## Fastest Path + +```bash +# one time, from this repo +bash install.sh + +# one time, inside any project +forge init + +# every day, before ambiguous or mutating work +forge substrate "Fix the checkout bug and add tests" +forge substrate "Refactor auth safely" --json +forge impact computeTax +``` + +For MCP-capable tools, call the same flow through: + +- `substrate_check` — full pre-action gate. +- `assumption_gate` — ask/proceed decision and clarifying questions. +- `predict_impact` — blast-radius prediction for a symbol or file. + +## Install Choices + +| Path | Best For | Steps | +| --- | --- | --- | +| Claude/Codex plugin | Extension-style install | Install the plugin, then run `forge init` in each repo. | +| Git clone | Lowest friction local setup | `bash install.sh`, then `forge init`. | +| npm package | CI/devcontainer usage | `npx @codewithjuber/forgekit init`. | + +After `forge init`, Forge emits native config for Claude Code, Codex, Cursor, Gemini, +Aider, Copilot/VS Code, Windsurf/Devin, Zed, Continue, and Roo where supported. Tools +without hooks receive advisory context plus MCP config; Forge does not pretend it can +force hooks into hosts that do not expose them. + +## Included Artifacts + +- [`cognitive_substrate_whitepaper.pdf`](./cognitive_substrate_whitepaper.pdf) — full paper. +- [`cognitive_substrate_whitepaper.html`](./cognitive_substrate_whitepaper.html) — browser-readable paper. +- [`deliverable-package.md`](./deliverable-package.md) — package overview and headline results. +- [`evidence_map.md`](./evidence_map.md) — source/status map for load-bearing evidence. +- [`ecosystem_map.md`](./ecosystem_map.md) — capability-vs-tooling map. +- [`impact_oracle_src.zip`](./impact_oracle_src.zip) — original Prototype I package. +- [`router_gate_src.zip`](./router_gate_src.zip) — original Prototype II package. +- [`../../research/python-prototypes/`](../../research/python-prototypes/) — unzipped prototype source preserved for auditability. + +## Paper-To-Forge Map + +| Paper capability | Production Forge surface | Status | +| --- | --- | --- | +| Memory | `forge recall`, `forge cortex` | File-backed and auditable; relevance is advisory. | +| Learning | `forge cortex` | Outcome-confirmed lessons; model weights do not change. | +| Imagination | `forge impact`, `forge substrate` | Static graph blast-radius simulation. | +| Self-correction | `forge verify`, doom-loop guard | External checks beat model claims. | +| Impact-awareness | `forge atlas`, `forge impact` | Known symbols/files and likely dependents surfaced. | +| M1 routing | `forge route`, `forge substrate` | Transparent cheapest-capable tier recommendation. | +| M2 assumption gate | `forge preflight`, `forge substrate` | Under-specified tasks return questions. | +| M3 decomposition | `forge scope`, `forge substrate` | Independent/coupled file clusters. | +| M4 goal anchoring | `forge substrate` | One pre-action objective/risk/verification summary. | +| M5 anti-over-engineering | `forge substrate`, `lean-guard` | Broad work gets minimality warnings. | +| M6 inline verification | `forge verify` | Checklist and external verification discipline. | + +## Honest Boundary + +Deterministic parts are asserted: repo symbol/file grounding, graph traversal, emitted +config, protected-file guards, and test/build commands. Research-edge judgments remain +advisory: memory relevance, model fit, scope minimality, and whether a verification +checklist is sufficient for a particular production environment. diff --git a/docs/cognitive-substrate/cognitive_substrate_whitepaper.html b/docs/cognitive-substrate/cognitive_substrate_whitepaper.html new file mode 100644 index 0000000..940e8d8 --- /dev/null +++ b/docs/cognitive-substrate/cognitive_substrate_whitepaper.html @@ -0,0 +1,1009 @@ + + + + + +A Cognitive Substrate for Coding Agents + + + +
+ +

A Cognitive Substrate for Coding Agents

+
Theory → Evidence → Build‑Map edition. Why a frozen language model structurally lacks memory, learning, imagination, self‑correction, and impact‑awareness — plus six metacognitive deficits the field's evidence names — and how to re‑wrap its input→process→output loop to supply them, with two runnable prototypes as evidence.
+ +
+A first‑principles design study, grounded in field evidence. The five faculties are argued from the shape of a frozen model and grounded in 32 primary sources (memory‑augmentation, continual‑learning, self‑correction, world‑model, and program‑analysis literatures). This edition adds an evidence layer — twelve load‑bearing pain‑point statistics independently re‑grounded from primary sources and graded confirmed / vendor‑reported / unverifiable — six mechanisms (routing, assumption gate, decomposition, goal‑anchoring, anti‑over‑engineering, inline verification), a build‑opportunity map against the real 2026 Claude‑Code stack, and a second prototype (a complexity‑aware router + assumption gate, evaluated live on real models). An epistemological and ethical lens drawn from Qur'anic sources runs throughout, used as design framing, never as technical authority.  |  Companion artifacts: evidence map, ecosystem map, gap‑map, reference list, Qur'anic‑lens table, and two runnable packages (impact‑oracle, router‑gate). +
+ +
+

Abstract

+

A large language model at inference time is, mathematically, a fixed function y = fθ(x) with frozen parameters θ and a bounded input window. From this single fact, five apparent “cognitive” deficits of a coding agent follow as structural consequences, not incidental weaknesses: it cannot remember across sessions, cannot learn from outcomes, cannot imagine the consequences of an action before taking it, cannot reliably correct itself, and does not know what already exists in a codebase or what an edit will affect. We show that neither better prompting nor additional tools (skills, MCP servers) remove these deficits, because they leave fθ and the open‑loop pipeline intact. We then specify a cognitive substrate: an external architecture that keeps the LLM frozen but re‑wraps its input→process→output loop into a closed, stateful cycle over persistent stores — an episodic/semantic memory, an online‑updatable learning layer, a consequence simulator, a metacognitive verification gate, and a persistent structural model of the codebase — all under an explicit stewardship boundary. For each faculty we identify precisely what the existing literature solves and what residual gap remains for a coding agent. To turn the weakest‑evidenced claim into something testable, we build and evaluate the impact‑awareness faculty as a runnable prototype: a Codebase World‑Model that parses a repository into a persistent dependency graph, and an Impact Oracle that predicts the blast radius of a proposed edit. Against mutation‑derived ground truth, the oracle is the only method that never misses an affected file (recall = 1.00 across five tested edits), where a text‑search baseline misses transitive dependents and an edited‑file‑only baseline misses 47% of impact. Throughout, a Qur'anic epistemic lens supplies the design's vocabulary of obligation — know what exists before acting (2:31–32), verify before you act (49:6), pursue not that of which you have no knowledge (17:36), and hold what you can damage as a trust (33:72).

+
+ +
+ + + + + + + + + + + + + +
—References, evidence & lens appendices
+
+ +

1 The problem, from the ground

+ +

Start with a scene any developer knows. You open an unfamiliar file. Before you change a line, a great deal happens that you barely notice: you recognize what the file is and how it connects to the rest of the system; you recall that you touched something similar last week and how that went; you silently simulate — “if I rename this, the three callers over there break” — and you feel the weight of the fact that this code runs in production. You are, in one glance, exercising memory, a world‑model, imagination, and a sense of consequence. Only then do you type.

+ +

A coding agent built on a language model does none of this by default, and the reason is not that the model is small or under‑trained. It is that the model has the wrong shape for the job. This paper is an attempt to go to the root of that shape — to ask what the input to the model actually is, what happens to it, what comes out, and why that pipeline cannot, on its own, hold the faculties above — and then to design a supporting structure that can, without pretending to rebuild the model or to mimic a human brain. We do not claim to make a machine that thinks like a person. We claim something narrower and buildable: that the specific faculties a coding agent is missing can be given precise mathematical form, and supplied by an external architecture wrapped around a frozen model.

+ +

Two commitments run through the paper. First, honesty about what is known. Some of these faculties — notably persistent memory with a real forgetting policy, and online learning without catastrophic forgetting — are open research problems; we say so, and we say exactly where the open edge is. Others — notably impact‑awareness — turn out to be mostly an engineering gap: the tools have existed since the 1980s but were never wired into a language model's loop. We prototype that one, because it is the one where a single session can produce real evidence rather than a promise. Second, a lens. This project was asked to think with the Qur'an as a source of epistemology and ethics — not as authority for any engineering claim, but as a disciplined vocabulary for naming what an agent that acts on real systems owes: to know before it acts, to verify what it is told, to treat a capability it can misuse as a trust. That vocabulary turns out to map cleanly onto architectural decisions, and we let it guide the design while keeping every technical claim standing on its own merits.

+ +

2 The root cause, formally

+ +

It is tempting to describe an agent's memory or learning problems as things that will improve with scale or a cleverer prompt. That framing is wrong, and being precise about why is the whole foundation of the design. Consider what one turn of an LLM‑based agent actually computes. Let the input be a token sequence x drawn from a context window of bounded size W. The model applies fixed parameters θ and produces a distribution over the next token; sampling and repeating yields the output y. Abstractly:

+ +
(1) +y = fθ(x),  x ∈ 𝕏≤W,  θ fixed for all inferences. +
+ +

Three properties of Equation (1) are the source of everything that follows. They are not bugs; they are what a deployed transformer is.

+ +

(P1) Statelessness. The function has no argument that carries information from one call to the next except x itself. Whatever the agent “knew” during turn t is gone at turn t+1 unless it was serialized back into the token string. There is no hidden variable st that persists; formally, the map is memoryless: yt depends on xt alone, not on the history (x1,…,xt−1).

+ +

(P2) Frozen parameters. θ does not change as a function of what happens at inference. There is no term in Equation (1) of the form θ ← θ − η∇L: the gradient step that constitutes learning happens only in an offline training loop the deployed agent never enters. An outcome in session N — a test that failed, an edit the developer reverted — therefore has no path by which it can alter behavior in session N+1.

+ +

(P3) Bounded, undifferentiated context. x must fit in W tokens, and every fact competes for the same space on equal terms. There is no separate, larger store the model can address; “give it more context” is bounded by W and, even within W, provides no structure, no provenance, and no persistence.

+ +

Now watch the five faculties fall out of P1–P3 by deduction, not observation:

+ + + + + + + + + + +
FacultyWhy it is structurally absentFollows from
Memory (across sessions)By P1, nothing survives a turn but the token string; by P3, the string is bounded and lost at session end. There is no addressable store that outlives x.P1, P3
Learning (from outcomes)By P2, no inference‑time event writes to θ. In‑context “learning” is real optimization14,15 but lives only inside the current x and vanishes with it (P1) — a simulation of learning, not learning.P2, P1
Imagination (simulate before acting)Equation (1) maps tokens to tokens. There is no separate forward model of “what happens to the world (or codebase) if I take action a” distinct from emitting more tokens; the model cannot roll out and score a hypothetical it does not also have to narrate.P1
Self‑correctionAny “check” the model runs is another evaluation of the same fθ with the same blind spots. There is no independent verifier inside Equation (1); the literature confirms intrinsic self‑correction is unreliable without an external signal.21P2
Impact‑awarenessBy P3, the model sees only the tokens in x. A million‑line repository does not fit; therefore it cannot know, unaided, what elsewhere depends on the symbol it is about to change.P3
+ +
+
Why prompting and tools do not close the gap
+

A better prompt changes x. More tools (skills, MCP servers, function calls) let the agent fetch new x or emit richer y. Both operate inside Equation (1) and leave P1–P3 untouched: the composed system is still a stateless map with frozen weights and a bounded window. A tool call retrieves a document into context, but nothing decides what was worth keeping, consolidates it, or updates the agent's priors for next time. The deficits are properties of the loop shape — open, memoryless, one‑directional — not of the model's knowledge. To remove them you must change the shape of the loop, which is precisely what an external substrate can do while θ stays frozen.

+
+ +

This reframing is the paper's pivot. If the deficits came from the loop shape, then the remedy is to re‑wrap the loop: keep fθ exactly as it is, and surround it with state and update so that the composite system is no longer memoryless, no longer open, and no longer blind beyond W. Figure 1 states the whole thesis in one picture.

+ +
+Two-panel schematic: (a) the frozen LLM today as a stateless input-process-output pipe with a broken write-back loop; (b) the same LLM re-wrapped as a closed, stateful cognitive cycle. +
Figure 1. Re‑wrapping the input→process→output loop. (a) Today: input enters a bounded window, the frozen model maps it to output, and at session end all state is discarded — there is no path from an outcome back to weights or memory. (b) The proposed substrate keeps the identical frozen model (same blue block) but closes the loop into a cycle — perceive (read memory + world‑model), reason, imagine/reflect, verify/act, then write back and consolidate — so state persists across turns and sessions. Nothing inside the model changes; the shape around it does.
+
+ +

3 Five faculties and their gaps

+ +

Before designing anything, we asked what the research literature has already built for each faculty, and — precisely — what it does not yet give a coding agent. This section is deliberately not a survey for its own sake: for each faculty we state the structural lack (from §2), the best existing approaches with citations, and the residual gap that the substrate must actually close. The discipline throughout is to separate what is solved from what is merely named. All 32 sources are listed in the references; the full structured gap‑map is a companion artifact.

+ +

Memory Persistent experience across sessions

+

The lack. A transformer's only state is its weights (frozen) and its context window (bounded, discarded at session end); every apparent memory is either baked into pretraining or re‑supplied as tokens.

+

What exists. The idea of an explicit, addressable external memory read and written by a neural controller goes back to Memory Networks1 and End‑to‑End Memory Networks2, and to the Neural Turing Machine3 and Differentiable Neural Computer4, which showed a network can learn to read/write an external matrix via differentiable addressing. But those memories are trained jointly with the model and cannot be bolted onto an already‑frozen LLM. The pattern that can attach to a frozen model is Retrieval‑Augmented Generation5: pair the generator with a retriever over an external index. For deployed agents, MemGPT6 adds an OS‑style paging scheme (the model issues calls to move information between context and external storage), and Generative Agents7 add a timestamped memory stream with periodic reflection that synthesizes higher‑level memories.

+

Residual gap. None of these gives a coding agent a memory that is simultaneously (a) persistent across sessions without retraining, (b) structured enough to answer “have I seen this bug/pattern before and what happened,” (c) governed by a principled forget/consolidate policy rather than unbounded growth, and (d) tied to verifiable software artifacts (commits, test outcomes, diffs) rather than free‑text the model wrote about itself. RAG is stateless per‑query lookup; MemGPT and Generative Agents self‑manage paging and reflection with no ground‑truth check on what is kept or discarded. This is a genuine research gap, not just integration.

+ +

Learning Durable update from outcomes

+

The lack. By P2, nothing at inference writes to θ; in‑session adaptation is prompt conditioning that vanishes when context clears.

+

What exists. Neuroscience offers the shape of an answer: Complementary Learning Systems theory8,9 argues intelligence needs two systems — a fast, instance‑based (hippocampal) learner and a slow, structured (neocortical) one — because a single fast learner catastrophically overwrites old knowledge. On the engineering side, Elastic Weight Consolidation10 directly mitigates catastrophic forgetting by protecting parameters important to earlier tasks; LoRA13 makes cheap, modular weight updates feasible by freezing the base and training small low‑rank adapters; Test‑Time Training11 shows weights can be adapted per‑input via a self‑supervised loss; Fast Weights12 give a genuine intermediate timescale of adaptation. And the theory of in‑context learning14,15 shows prompting can implement implicit gradient descent — real optimization, but session‑local.

+

Residual gap. There is no accepted, low‑cost, always‑on pathway that takes a concrete outcome (a test failure, a human revert, a review comment) and durably changes future behavior — either by writing to persistent non‑parametric state or by cheaply updating parameters — without a full offline retraining cycle and without catastrophic forgetting. Existing mechanisms are either real‑but‑deliberate retraining (EWC, LoRA, decoupled from the interaction loop) or real‑but‑ephemeral in‑context simulation. Bridging them into a fast, safe, incremental loop is unsolved.

+ +

Imagination Simulating consequences before acting

+

The lack. Equation (1) is token→token; there is no forward model of action consequences and no rollout loop.

+

What exists. Model‑based RL has built exactly this — for other domains. World Models22 learn a compressed generative model of an environment and train a controller inside the “dream.” MuZero23 plans with a learned latent model of only the quantities relevant to planning (reward, value, policy). Dreamer/DreamerV324,25 learn behaviors purely from imagined latent rollouts across many domains. The free‑energy principle / active inference26 gives the deepest theoretical grounding for why an agent must predict the sensory consequences of hypothetical actions and act to minimize expected surprise.

+

Residual gap. Every one of these targets a continuous perception‑action domain (pixels, physics, game boards) with a learnable dynamics model and a dense reward. None target the discrete, symbolic domain of source code, where the dynamics to imagine are “which call sites break,” “does this still type‑check,” “do the tests still pass.” A coding agent's imagination faculty needs a fast surrogate simulator over code‑change consequences — static analysis, symbolic execution, or an approximate learned model of compile/test outcomes — and no general, reusable such component exists. This is an open build target.

+ +

Self-correction An independent check on one's own output

+

The lack. Any check the model runs re‑evaluates the same frozen weights that produced the error.

+

What exists. Reflexion18 has the agent verbally reflect on task feedback and store it to condition the next attempt; Self‑Refine19 alternates generator and critic roles on the same model; Self‑Consistency20 marginalizes over many sampled reasoning paths. Crucially, trained verifiers16 and process‑level reward models17 decouple checking from generating by using a separately trained model — escaping the same‑weights blind spot.

+
+
The honest negative result
+

Huang et al.21 evaluated intrinsic self‑correction — no external feedback — and found it frequently makes correct answers worse. The approaches that do work (Reflexion, verifiers, process reward models) all inject something the frozen transformer lacks on its own: an external oracle (unit tests, human labels, a separately trained reward model). The lesson for our design is unambiguous: self‑correction must be built as an external verification signal, not more self‑prompting.

+
+

Residual gap. For code specifically, no existing system supplies a cheap, general, always‑available correctness oracle equivalent to a proof checker: tests exist for some repos but are incomplete, and an LLM‑based verifier trained once cannot track an evolving, project‑specific notion of correctness. Building that external, evolving check is the open problem — and it is partly addressable now, because two concrete external signals already exist for code: the type/compile system and the test suite. Our prototype leans on exactly those.

+ +

Impact-awareness Knowing what exists and what an edit affects

+

The lack. By P3 the model sees only tokens in x; a whole repository does not fit, so it cannot know what depends on a symbol it changes.

+

What exists — and this is the key observation. The tools to compute exact structural impact are mature and decades old. The Program Dependence Graph30 (1987) combines control and data dependence in one structure; interprocedural slicing28 (1990) computes what code affects, or is affected by, a given point across function boundaries; program slicing was surveyed as a mature toolbox by 199429. The Code Property Graph31 merges AST, control‑flow, and dependence into one queryable graph. On the learned side, “Learning to Represent Programs with Graphs”27 and code2vec32 let neural models see code structure, not just token sequences.

+
+
The gap here is engineering, not science
+

None of these exact tools are wired into the loop of an LLM coding agent. Today's agents either re‑read raw file text inside a bounded window or rely on the model's own unverified guess about what a change affects. The residual gap is a missing bridge: (a) maintain a live, incrementally‑updated structural graph of the whole repository, (b) let the agent query “what depends on this symbol” as a fast, sound lookup before proposing an edit, and (c) feed the result back into context in a form the model reliably uses. Because this gap is engineering‑shaped rather than an open research question, it is the faculty we can honestly build and evaluate in one sitting — which is exactly what §8 does.

+
+ +

The pattern across all five is worth stating plainly. Memory, learning, and imagination for code are genuine research gaps; self‑correction is partly solved given an external oracle; impact‑awareness is an integration gap over mature tools. A credible substrate therefore does not claim to solve all five — it provides the architecture into which solved components drop in now and open ones drop in as they mature, and it demonstrates the faculty that is buildable today.

+ +

4 The evidence: what the field actually measures

+ +

The first edition of this paper argued the five deficits from first principles — from the +shape of a frozen model, before looking at a single usage statistic. That argument stands on +its own. But a design study that only reasons from first principles invites a fair question: does the +field's own evidence agree that these are the real problems? This edition answers it. We took the +load‑bearing pain‑point statistics that circulate in the 2026 discourse and +independently re‑grounded each one from its primary source — because the project's +governing discipline is that a probabilistically‑generated number, repeated often enough to sound +settled, is still an assumption until you have seen where it comes from.

+ +
+

How we grade evidence — and why it matters here. Each claim below carries a status: +confirmed (traceable to an independent primary source — a peer‑reviewed paper, the +original survey, an official announcement), vendor‑reported (real and sourced, but only to a +company with a commercial stake in the finding, not independently reproduced), or unverifiable +(we could not corroborate it in primary form). This is the paper's own thesis applied to itself: we do not +ask you to trust a figure because it is widely repeated. The re‑grounding changed what we +are willing to assert — and caught three numbers that turned out to be misattributed.

+
+ +

4.1 What the strongest evidence confirms

+ +

Five findings survived re‑grounding to an independent primary source, and they are enough to +carry the argument. The most important is not a vendor's dashboard metric but a +randomized controlled trial. METR studied sixteen experienced open‑source developers across +246 tasks in mature repositories they knew well; the developers forecast a 24 % speedup, +believed afterward they had worked about 20 % faster, and were in fact +19 % slower with early‑2025 AI tools [C1 confirmed · +arXiv:2507.09089]. That single result is the empirical heart of this paper: the gap between the +felt productivity and the measured productivity is exactly the gap a system opens when it +has no calibrated sense of its own uncertainty (mechanism M2 below), and it is why “the model +felt confident” is not evidence of anything.

+ +

The trend evidence is equally well‑sourced. Stack Overflow's 2025 survey of more than 49,000 +developers records trust in AI accuracy falling from 40 % to 29 % even as adoption rose +to 84 %, with the top‑ranked frustration — cited by 66 % — being code +that is “almost right, but not quite”, which the survey ties directly to the +second‑ranked frustration, that debugging AI code is more time‑consuming (45 %) +[C2 confirmed]. Google's DORA program (~5,000 professionals) frames AI as an +“amplifier” — magnifying the strengths of strong teams and the dysfunctions of +weak ones — and still finds it increasing delivery instability at near‑universal +adoption [C5 confirmed]. On the trustworthiness of the benchmarks +themselves, OpenAI retired SWE‑bench Verified in 2026 after auditing found at least +59.4 % of the hard problems it examined had flawed test cases or training‑data +contamination [C6 confirmed] — a caution that applies squarely to our +own prototype's numbers, and one reason we report them the way we do. And the reason an agent +cannot be its own sole judge is not rhetorical: Panickssery et al. (NeurIPS 2024, +Oral) show LLM evaluators recognize and systematically favor their own generations +[C12 confirmed · arXiv:2404.13076] — the empirical foundation for +insisting that verification be external.

+ +
+

These five are enough. A randomized trial (miscalibration), a 49,000‑person survey +(“almost right”), a 5,000‑person study (instability), a benchmark retirement (don't trust +the metrics), and a peer‑reviewed result (an LLM favors its own output). Each maps onto a specific +deficit this paper's architecture supplies — and none depends on a vendor's self‑report.

+
+ +

4.2 What is real but vendor‑reported — and what did not survive

+ +

Several widely‑cited figures are real and sourced but trace only to a company selling a +remedy for the problem the figure describes; we use them as corroboration, not proof, and we say so. +Veracode reports 45 % of AI‑generated samples introduce an OWASP‑Top‑10 +vulnerability [C3 vendor]; Faros' telemetry across 22,000 developers +reports median PR‑review time up 441.5 % and 31.3 % of PRs merged with no +review [C7 vendor]; Sonar reports the sharpest single number in the +whole discourse — 96 % of developers do not fully trust AI code, yet only 48 % always +verify it, a 48‑point “verification gap” +[C9 vendor]. These are consistent with the confirmed evidence and with each +other, which is why we cite them — but a reader should weigh them as vendor telemetry, not as +independent science.

+ +

The discipline earned its keep on three claims that did not survive, and which this paper +therefore does not repeat as fact:

+
    +
  • The widely‑quoted “2.74× more vulnerabilities” figure could not be +traced to Veracode's own report; it appears to be conflated with a separate study. Only the 45 % +figure is genuinely Veracode's. We drop the multiplier.
    • +
  • The claim that “Anthropic's 400,000‑session study found 17 % lower +comprehension” merges two different studies — the session study contains no comprehension +finding. We do not use the number.
    • +
  • GitClear's code‑duplication rise is quoted as both 4× and 8× in the vendor's +own title versus body, and JetBrains' “77 % correct conventions every session” could not +be located in the primary report at all. We cite the direction (duplication is rising) without the disputed +multiplier.
    • +
+ +

That a re‑grounding pass changed our own claims is the point, not an embarrassment. It is the +same move the architecture in §7 makes structural: a stored fact is provisional until an +independent check confirms it. A full source‑by‑source table — every claim, its +primary citation, and its status — is in the Evidence appendix. The mechanisms that follow are +each motivated by a confirmed finding wherever one exists, and lean on vendor evidence only where +that is the best the field currently offers.

+ +

5 The Qur'anic epistemic lens

+ +
+
How this lens is used — and how it is not
+

The Qur'an is used here as a framing lens and ethics source, never as technical authority for an engineering claim. No verse is cited to prove that an algorithm works or that a data structure is correct — those claims stand on their engineering merits alone (§3, §8). What the lens supplies is threefold: (1) a precise vocabulary of obligation for what an agent that acts on real systems owes — to truthfulness, to verification, to stewardship; (2) a hierarchy of knowledge (‘ilm → fahm → ḥikma: knowledge → understanding → wisdom) that motivates a layered memory architecture rather than a flat vector store; and (3) ethical constraints on autonomy that translate into concrete safeguards. Where a mapping is marked load‑bearing, the concept motivates a specific design decision (e.g. a mandatory, not optional, verification gate); where marked metaphor, it is illustrative. Canonical text below is presented directly and attributed; it is not paraphrased. Arabic and translations were retrieved from quran.ai; the full 14‑row mapping table is in the appendix.

+
+ +

The lens earns its place because the deepest failure modes of an autonomous coding agent are not computational but epistemic and ethical: acting without knowing, trusting a report without checking it, and treating a granted capability as license. The Qur'anic vocabulary names these with unusual precision, and three verses in particular map so directly onto architectural decisions that they shaped the design rather than decorating it.

+ +
+
17:36 — lā taqfu · the root of impact‑awareness  [load‑bearing]
+
وَلَا تَقْفُ مَا لَيْسَ لَكَ بِهِ عِلْمٌ ۚ إِنَّ السَّمْعَ وَالْبَصَرَ وَالْفُؤَادَ كُلُّ أُولَٰئِكَ كَانَ عَنْهُ مَسْئُولًا
+
“Do not follow blindly what you do not know to be true: ears, eyes, and heart, you will be questioned about all these.”
+
→ Design principle. Before any mutation (write, delete, refactor), a pre‑action gate must establish what the agent actually knows: which entities the change affects, whether the relevant files/tests/dependents were actually read, and whether the predicted outcome rests on evidence rather than a pattern‑matched guess. Actions taken without verified knowledge are blocked, not merely flagged. The verse's own structure — “ears, eyes, heart… questioned about all these” — maps to an audit trail: every channel the agent used (what it read, inferred, assumed) is logged so the decision can be reconstructed and questioned. This is precisely the impact oracle of §8.
+
Grounded with quran.ai: fetch_translation(17:36, en-abdel-haleem); fetch_tafsir(17:36, en-ibn-kathir)
+
+ +
+
49:6 — tabayyun · the verification gate  [load‑bearing]
+
يَا أَيُّهَا الَّذِينَ آمَنُوا إِن جَاءَكُمْ فَاسِقٌ بِنَبَإٍ فَتَبَيَّنُوا أَن تُصِيبُوا قَوْمًا بِجَهَالَةٍ فَتُصْبِحُوا عَلَىٰ مَا فَعَلْتُمْ نَادِمِينَ
+
“Believers, if a troublemaker brings you news, check it first, in case you wrong others unwittingly and later regret what you have done.”
+
→ Design principle. The architecture places a verification step between receiving information (from context, tool output, or its own prior reasoning) and acting on it. The operative term tabayyun demands active investigation, not passive acceptance: before applying a fix based on an error report or its own diagnosis, the agent re‑reads the current file state, confirms the issue still exists, and checks the fix introduces no new breakage detectable by types or tests. The gate is architectural and mandatory — the verse's command is categorical, not conditional on the reporter's trustworthiness — which is exactly the design lesson the self‑correction literature reached empirically21: a real external check, not more self‑prompting.
+
Grounded with quran.ai: fetch_translation(49:6, en-abdel-haleem); fetch_tafsir(49:6, en-ibn-kathir)
+
+ +
+
2:31–32 — ta‘līm al‑asmā' · the world‑model  [load‑bearing]
+
وَعَلَّمَ آدَمَ الْأَسْمَاءَ كُلَّهَا ... قَالُوا سُبْحَانَكَ لَا عِلْمَ لَنَا إِلَّا مَا عَلَّمْتَنَا
+
“He taught Adam all the names [of things]… They said, ‘May You be glorified! We have knowledge only of what You have taught us.’”
+
→ Design principle. Knowledge begins with naming — identifying entities and their relations. The agent must hold a structured map of what exists in the codebase (files, functions, classes, dependencies), not a flat listing: knowing that A calls B, that module X depends on Y, that test T covers class C. The angels' admission — “we have knowledge only of what You have taught us” — is a startlingly exact description of the LLM's own situation under P3: it knows only what is in its window. The external world‑model supplies the “names” of entities that exceed context capacity.
+
Grounded with quran.ai: fetch_translation(2:31-32, en-abdel-haleem)
+
+ +
+
33:72 — al‑amāna · stewardship & bounded autonomy  [load‑bearing]
+
إِنَّا عَرَضْنَا الْأَمَانَةَ عَلَى السَّمَاوَاتِ وَالْأَرْضِ وَالْجِبَالِ فَأَبَيْنَ أَن يَحْمِلْنَهَا ... وَحَمَلَهَا الْإِنسَانُ ۖ إِنَّهُ كَانَ ظَلُومًا جَهُولًا
+
“We offered the Trust to the heavens, the earth, and the mountains, yet they refused to undertake it and were afraid of it; mankind undertook it — they have always been inept and foolish.”
+
→ Design principle. An agent that can modify a codebase bears an amāna — accepted responsibility for something it can damage. The verse's structure is the design: the heavens declined the trust, recognizing its weight; the human bore it and is called ẓalūman jahūlā (given to wrong and ignorance). So the architecture (a) operates under least privilege — a granted capability is never blanket permission; and (b) assumes the agent will err and builds in reversibility (sandboxing, staged commits, rollback), audit, and scope bounds as structural safeguards. The trust is not “the agent is trustworthy”; it is “the agent has accepted accountability for a domain it can harm, and the architecture must respect that weight.” This is the governance boundary enclosing the entire system in Figure 2.
+
Grounded with quran.ai: fetch_translation(33:72, en-abdel-haleem); fetch_tafsir(33:72, en-ibn-kathir)
+
+ +

Four further anchors complete the mapping (full text in the appendix). 20:114 — “rabbi zidnī ‘ilmā,” “My Lord, increase me in knowledge” — frames knowledge as perpetually incomplete and growing, motivating the continual‑learning store that accumulates across sessions. 96:1–5 — iqra' and the teaching “by the pen” (al‑qalam) — is the principle of externalized memory: the pen turns ephemeral thought into durable record, exactly the write‑back mechanism a bounded context window requires. 4:82 and 47:24 — tadabbur, deep reflection, whose root d‑b‑r concerns “what comes after” / consequences — ground the metacognitive controller as a structured trace‑forward through consequences (“if I apply this, what breaks?”), not a vague confidence score. And the classical epistemic ladder ‘ilm → fahm → ḥikma motivates the three‑layer memory of §7: raw logs, consolidated patterns, decision support — dumping everything into one flat store collapses the hierarchy and loses the distinction between fact and actionable understanding.

+ +

The remarkable thing is not that these mappings are poetic; it is that they are operational. “Verify before acting” is not a sentiment here — it is a mandatory gate in the action pipeline. “Know the names of things” is not a metaphor — it is a dependency graph. The lens told us which safeguards are non‑negotiable; the engineering told us how to build them.

+ +

6 Six mechanisms the frozen loop also lacks

+ +

The five faculties of the first edition answer “what cognitive capabilities does a stateless model +structurally lack?” This edition adds a second family of deficits that the field's evidence forced +into view — not cognitive faculties but metacognitive and resource‑allocation +ones: knowing how much effort a task deserves, knowing what you have not been told, knowing when to split +work, when you have drifted, when you have over‑built, and when to check. A frozen map +y = fθ(x) lacks these for the same reason it lacks the first five: +each requires state the model does not carry and a decision about the computation itself +that a single forward pass cannot make. We formalize six.

+ +
+

A note on honesty before we start. Two of these six are already largely solved by +existing tooling, and we say which (M1, M3). Naming a mechanism is not claiming to have invented it. The +contribution of this section is the unified account — showing that all six fall out of the +same frozen‑loop shape, and that the genuinely unsolved ones (M2, M5) are unsolved for a structural +reason, not for lack of effort.

+
+ +

5.1 M1 — Complexity‑aware routing

+

The deficit. A frozen model has no notion of its own running cost. The same +fθ is invoked whether the task is is_prime(n) or a +distributed rate‑limiter, and a premium model billed per token spends the same premium rate on both. +The project stated it plainly: “a simple prime‑number finder — if you use [a premium +model] it will not give you extra.” Formally, let a task x have +an intrinsic complexity c(x) and let a tier ladder +T1 < … < Tk have costs +κ1 < … < κk. Routing selects the +cheapest tier whose capability covers the task:

+
+route(x) = min{ Ti : capable(Ti, c(x)) }, then escalate Ti +→ Ti+1 only if an external check on the output fails. +
+

The escalation clause is what makes this safe: the worst case is a cheap attempt plus a premium attempt, +the common case is cheap alone, and the decision to spend more is driven by a verified failure, +never by the model's self‑assessment. Ecosystem status: largely solved. Model tiering and +gateways (LiteLLM, OpenRouter, per‑agent model: fields) already route by cost. What they +do not give is a transparent, per‑task, auditable complexity judgment the user can +see and override before dispatch — that thin transparency layer is the only thing left to build, and +our prototype (§9) builds it. Anchor: 17:36, “do not pursue what you +have no knowledge of” — spend capability in proportion to what the task actually requires.

+ +

5.2 M2 — The assumption / uncertainty gate root failure

+

The deficit. This is the failure the project named as central: “the biggest problem is +Assumption. If it doesn't have enough context it will assume many things.” A model that computes +arg maxy P(y \mid x) always returns some most‑likely +continuation, even when x under‑determines the task — there is no term in +the objective that fires when the input is insufficient. The model cannot distinguish “I +know this” from “this is merely the least‑improbable guess.” This is precisely what +the METR trial measured from the outside: confident forecasts, slower reality +[C1 confirmed]. Formally, define a specification‑completeness +functional s(x) ∈ [0,1] over the dimensions a task needs pinned down (inputs +and outputs, target scope, success criteria). The gate interposes before execution:

+
+if s(x) < τ :  halt and emit the missing‑information questions | else : +proceed to route(x). +
+

The gate spends zero generation tokens on an under‑specified request. Asking one question +is cheaper than the “almost right, but not quite” rework loop that 66 % of developers +report [C2 confirmed]. Ecosystem status: residual gap — genuine +whitespace. Nothing in the 2026 stack supplies calibrated “known‑unknowns”: the +pain‑point survey names it directly — “models rarely signal uncertainty or say +‘I can't do this’” — and spec‑driven‑development tools only help +when the human already wrote a complete spec, which is the assumption the gate is meant to remove. Our +prototype builds this too. Anchor: 49:6, tabayyun — “if a +source brings you news, verify it”: the discipline of not acting on an unverified report is exactly a +gate on insufficient input.

+ +

5.3 M3 — Task / session decomposition

+

The deficit. A bounded, undifferentiated window (property P3) means a long session accretes +unrelated sub‑tasks that compete for attention and degrade retrieval. The project's remedy was exact: +“instead of using multiple sessions for multiple independent tasks… we normally do full +things in a single session.” Formally, given a task set with a dependency relation, the +decomposition problem is to partition into contexts C1, …, +Cm that minimize cross‑context coupling while keeping each context's working set +inside the window — independent components run in isolated sessions, dependent ones share state. +Ecosystem status: solved. Subagents with isolated windows, the experimental Agent‑Teams +pattern, and git‑worktree fan‑out already implement this well; the only residue is that +choosing the partition boundary is still a human heuristic. We flag it and move on — a paper +that claimed to reinvent subagents would be exactly the over‑selling this project forbids. +Anchor: 20:114, “do not hasten…” — ordered, separated +pursuit over one overloaded pass.

+ +

5.4 M4 — Goal‑anchoring against drift

+

The deficit. To a text model, “generation of a long story and code is the same +concept” — both are just high‑probability token sequences — so nothing +intrinsic keeps output tethered to the objective rather than to local fluency. The project: the +agent “diverts from the main goal.” Formally, let g be +the goal representation fixed at t=0 and yt the +work in progress; drift is a rising divergence D(yt, g). Anchoring adds +a periodic check that re‑validates yt against +g and corrects when D exceeds a bound — the goal is +re‑read, not assumed to still be in view. Ecosystem status: partial. CLAUDE.md +and /goal hold a static anchor, but the field documents it decaying within a session +(“context compression wiping scrollback”), and the open gap is that “specs drift out +of sync with code” with no tool continuously re‑verifying the two. Anchor: +the ḥifẓ / murāja’a discipline — retention through active +review, not one‑time loading.

+ +

5.5 M5 — Anti‑over‑engineering residual gap

+

The deficit. The maximum‑likelihood objective favors the elaborate completion: more +abstraction, more defensive scaffolding, more “production‑ready” ceremony than the task +asked for. The project named the cost precisely: “the default version will be +over‑engineering — waste of tokens + time + quality and cost.” Formally, let a +solution have footprint φ(y) (files touched, abstractions introduced, +lines added) and let the task imply a minimal sufficient footprint +φ*(x). Over‑engineering is φ(y) − +φ*(x) > 0; the check flags scope the task did not request. Ecosystem status: residual +gap. The one discipline that exists — the frontend‑design skill's minimalism rule — +is explicitly UI‑only; no general tool measures unnecessary abstraction against the stated task, and +the failure is invisible to the review agents that scan for bugs. Anchor: 7:31, +“…do not be excessive” — sufficiency as a virtue; the least structure that meets +the need.

+ +

5.6 M6 — Inline verification

+

The deficit. A human “interprets logic while writing… its real‑time +verification”; a frozen model emits the whole sequence and only afterward can anything check it. +Verification is deferred, and the field's sharpest statistic is the size of that deferral: the +48‑point gap between the 96 % who distrust AI code and the 48 % who always verify it +[C9 vendor], with the deferred review time itself up 441.5 % +[C7 vendor]. And the verifier cannot be the same model, because an LLM favors +its own output [C12 confirmed]. Formally, interleave an external check +v(·) at each meaningful step j rather than once at the +end: y = (y1, …, yn) with +v(yj) gating yj+1 — the same +external‑check principle the substrate uses at the architecture level (§7, the mandatory +verify‑gate), pushed down to the generation step. Ecosystem status: partial. Streaming and PostToolUse hooks give passive +visibility and post‑edit linting, but no shipped mechanism forces an interpretive checkpoint +during generation. Anchor: 49:6 again, and 17:36 — verification as a +continuous obligation, not a final formality.

+ +
+

The pattern across all six. Each mechanism is a decision about the computation that a +single forward pass cannot make: how much to spend (M1), whether to proceed (M2), how to split (M3), +whether you have drifted (M4), whether you have over‑built (M5), whether to check now (M6). None can +live inside fθ; all live in the stateful wrapper around it +— which is the same conclusion the five faculties reached, arrived at from the direction of +resource and control rather than cognition.

+
+ +

7 The cognitive substrate

+ +

We now specify the architecture. The design rule is strict: the LLM stays frozen. We add no term that modifies θ. Everything is external state and external update, composed so that the system — not the model — acquires the missing faculties. Where §2 wrote the agent as the memoryless map y = fθ(x), the substrate replaces it with a stateful operator over a persistent store M:

+ +
(2) +(yt, Mt+1) = F​( xt, Mt; fθ ) +
+ +

Read Equation (2) against P1–P3: it takes the previous store Mt as an explicit argument (defeating statelessness, P1), it returns an updated store Mt+1 (a write‑back path that P2 forbade for weights, now legal because it targets external state), and M can be arbitrarily larger than the window W (defeating P3). The frozen fθ appears only as a subroutine. The composite F is what Figure 1(b) draws and what the rest of this section defines.

+ +
+Layered architecture: input and output rails; a five-stage faculty pipeline (perceive, reason, imagine, verify/act) with a metacognition collar around the frozen LLM; four persistent stores beneath; a write-back/learn band; all enclosed in a stewardship boundary. Each faculty carries a Quranic anchor. +
Figure 2. The cognitive substrate for a coding agent. A five‑stage faculty pipeline wraps the frozen LLM (“REASON”): ① perceive reads memory and queries the world‑model; ② reason is the unchanged model; ③ imagine dry‑runs consequences; ④ metacognition (the collar) checks self‑consistency; ⑤ verify/act is the impact oracle plus the mandatory gate. Beneath sit four persistent stores (episodic/semantic memory, learned priors, the codebase world‑model graph, the reflection log); the ⑥ write‑back/learn band captures each outcome and consolidates it into those stores. The whole system runs under a stewardship boundary (least privilege, reversibility, audit, scope). Each faculty is tagged with its Qur'anic anchor (§5) and targets a residual gap from §3. The verify faculty (heavy border) is the one prototyped in §8.
+
+ +

7.1 The five faculties, formally

+ +

Perceive — memory as an addressable store. Let the memory be a growing set of records M = {mi}, each an embedding‑keyed entry with provenance (a commit, a test outcome, a past edit) and metadata (timestamp, access count, validation status). Perception retrieves a relevant slice by a score that — following Generative Agents7 but adding a validity term the literature lacks — combines relevance, recency, and grounded importance:

+
(3) +retrieve(x, M) = top‑ki  σ​( α·rel(x,mi) + β·rec(mi) + γ·val(mi) ) +
+

The val term — did this memory's prediction later prove correct against a test or commit? — is the load‑bearing addition: it is what lets memory be pruned by ground truth rather than by the model's own say‑so, the gap named in §3. This is the ‘ilm layer of the epistemic ladder.

+ +

Reason — the frozen model, unchanged. The LLM receives an assembled context: the raw request plus the retrieved memory slice plus the world‑model's answer to “what does this touch?” It contributes exactly what it is good at — synthesis, code generation, natural‑language understanding — and nothing is asked of it that P1–P3 forbid.

+ +

Imagine — a consequence simulator. Define a surrogate dynamics model g that maps a proposed action a and current codebase state C to a predicted outcome without touching the real repository:

+
(4) +ĉ​  = g(a, C)  →  predicted { broken call sites, type errors, failing tests } +
+

Unlike World Models or MuZero22,23, g here need not be learned from pixels or reward: for code it can be partly exact — static analysis and the dependency graph give sound over‑approximations of “what breaks” — and partly cheap simulation (run the affected tests in a sandbox). This is the faculty the literature has not built for the symbolic domain (§3); the impact oracle of §8 is a first, exact‑analysis instance of g.

+ +

Metacognition — a structured self‑consistency pass. Following the honest lesson of §3—§5 (self‑critique on the same weights is unreliable21; a real external check is required), the controller does not ask the model “are you sure?” It runs a tadabbur trace‑forward: it checks the proposed action against (i) the model's own stated plan, (ii) the type system, and (iii) the existing test suite — three signals at least two of which are external to fθ. Inconsistency halts and backtracks rather than proceeds.

+ +

Verify/act — the mandatory gate. The tabayyun gate (49:6) is a hard predicate on the assembled evidence E(a): the action proceeds only if the impacted set has been computed, the relevant state re‑read, and no external check contradicts the prediction.

+
(5) +act(a) allowed  ⇔  gate(E(a)) = ✓  ∧  scope(a) ⊆ granted +
+ +

7.2 Learning without touching θ

+

The write‑back band closes the loop. After an action resolves, its outcome — test pass/fail, a human revert, a review comment — is a labeled training signal, the external oracle §3 said learning requires. Two update channels, both leaving θ frozen, are available and are ordered by cost:

+
    +
  1. Non‑parametric (always‑on): write the outcome to M with its provenance, and update the val scores of the memories that informed the action. This is instant, reversible, and the primary path — the fast, hippocampal side of Complementary Learning Systems8,9.
    2. +
  2. Parametric (deliberate, optional): when a pattern recurs with high validated confidence, distill it into a small LoRA adapter13 trained offline — the slow, neocortical side — with EWC‑style protection10 against forgetting. This never runs inside the interaction loop.
    3. +
+

The consolidation process between them is the ḥifẓ + murāja‘a principle: periodic review that reinforces recurrent high‑value patterns, decays stale entries, and resolves contradictions — the forgetting policy whose absence §3 flagged in RAG and MemGPT. It is what keeps M from growing without bound and drowning signal in noise.

+ +
+
What is genuinely load‑bearing here
+

The architecture's one non‑obvious commitment is that learning and memory are anchored to verifiable software artifacts — commits, type checks, test outcomes — rather than to the model's own summaries of itself. This is the single design choice that separates the substrate from “RAG plus a scratchpad,” and it is possible only in domains that have a cheap external oracle. Code is such a domain. That is why a coding agent, and not a general chat agent, is the right first target — and why the faculty we prototype next is the one where that oracle is exact. Figure 3 draws the complete picture: the five faculties of this section governed by the six mechanisms of §6 as a metacognitive control layer.

+
+ +
+The extended cognitive substrate: a six-mechanism metacognitive control layer (routing, assumption gate, decomposition, goal-anchoring, anti-over-engineering, inline verification) sits above the five-faculty pipeline, which reads and writes four persistent stores, all enclosed in a stewardship boundary. Each mechanism is marked solved, partial, or residual gap. +
Figure 3. The substrate, extended with the six mechanisms of §6. The five‑faculty pipeline of Figure 2 (perceive→reason→imagine→verify/act) is now governed by a metacognitive control layer — the six mechanisms that decide about the computation itself: how much to spend (M1), whether to proceed (M2), how to split (M3), whether it has drifted (M4), whether it has over‑built (M5), when to check (M6). Each is tagged with its ecosystem status — solved, partial, or residual gap — and the two prototyped in this edition (M1 routing, M2 gate) are marked. Beneath sit the same four persistent stores; the whole runs under the stewardship boundary (amāna, 33:72).
+
+ +

8 Prototype: a Codebase World‑Model & Impact Oracle

+ +

Design papers are cheap; the discipline is to build the one faculty where a claim can be tested in a single sitting. That faculty is impact‑awareness (§3: mature tools, missing bridge). We implemented two of the substrate's components as a runnable Python package: the Codebase World‑Model (the “perceive” store, 2:31–32) and the Impact Oracle (an exact instance of the “imagine→verify” simulator g, 17:36). The full source, demo, and tests are the companion artifact impact_oracle_src.zip.

+ +

8.1 What it does

+

World‑model. A parser walks a repository with Python's ast module and builds a directed graph whose nodes are symbols — modules, classes, functions, methods, module‑level names — with stable qualified IDs (pkg.mod.Class.method) and metadata (file, line, kind, signature). Edges capture five structural relations: imports, calls, inherits, references, contains. The graph is persisted to disk keyed by a per‑file content hash, so an unchanged file is never re‑parsed — the “persistent memory of what already exists,” incremental by construction.

+

Impact oracle. Given a proposed change to a symbol X, the oracle traverses the graph along reverse dependency edges (who calls / imports / inherits / references X), transitively, assigning each reached node a confidence that decays with hop distance and edge type. The output is a ranked blast radius — the set of symbols and files predicted to be affected — each with an explanation path back to X, and a tunable confidence threshold. Figure 4 shows one such propagation.

+ +
+Node-link graph of impact propagation from validate_positive to 27 symbols across 7 files, nodes colored by confidence (dark red = changed symbol, salmon = high confidence, pale = lower confidence), edges colored by relation type (calls, references, inherits, imports, contains). +
Figure 4. Predicted blast radius of one edit. Changing utils.validation.validate_positive (dark red, center) propagates along reverse‑dependency edges to 27 symbols across 7 files. Node shade encodes confidence (high → pale by hop distance); edge style encodes relation (calls, references, inherits, imports, contains). This is the “imagine” faculty made concrete: the consequence set is computed before any edit is applied.
+
+ +

8.2 How we evaluated it — against ground truth, not self‑grading

+

The temptation in a prototype is to let the system grade its own homework. We refused that. Ground truth comes from mutation testing, which is independent of the graph the oracle uses: we built a realistic 10‑module demo package (classes, cross‑module imports, inheritance, call chains) with a 36‑test pytest suite; then for each of five target symbols we actually mutated the symbol to break its contract and ran the suite. The set of files whose tests then failed is the true behavioral blast radius — observed, not predicted. We compared three predictors against it:

+
    +
  • Graph Oracle — our reverse‑dependency traversal.
    • +
  • Grep baseline — text‑search for the symbol name (what an LLM agent does today when it “greps around”).
    • +
  • Edited‑file‑only — assume impact stops at the file you edited (the implicit assumption of a narrow‑context agent).
    • +
+ +
+
1.00
Oracle recall
(mean over 5 edits)
+
0.63
Oracle precision
(threshold‑tunable)
+
0.94
Grep recall
(misses transitive)
+
0.53
Edited‑file recall
(misses 47% of impact)
+
+ +
+Two panels. Left: grouped bars of precision, recall, F1 for Graph Oracle, Grep, and Edited-file-only. Oracle has recall 1.00; edited-file-only has precision 1.00 but recall 0.53. Right: precision, recall, F1 vs confidence threshold, with best F1 0.79 at threshold 0.4. +
Figure 5. Honest evaluation against mutation‑derived ground truth. (left) The oracle is the only method with perfect recall (1.00) — it never misses an affected file — at a precision cost (0.63) versus grep (0.73). Edited‑file‑only achieves perfect precision but catastrophic recall (0.53), missing nearly half of all real impact. (right) Precision and recall trade off with the confidence threshold; best F1 = 0.79 at threshold 0.4. The chart title states the result plainly, including where grep leads on F1 — we do not oversell.
+
+ +

8.3 Reading the result honestly

+

The oracle does not dominate every metric, and the paper is stronger for saying so. On F1 at the default threshold, the grep baseline (0.79) slightly edges the oracle (0.75). What the oracle uniquely provides is guaranteed recall: across all five mutations it never once missed a file that actually broke, whereas grep missed transitive dependents (a caller that never mentions the symbol by name) and edited‑file‑only missed 47% of real impact. For the question that matters when you are about to change production code — “show me everything that could break” — a false negative (a silent breakage) is far more costly than a false positive (an extra file to glance at), and only the structural oracle drives false negatives to zero. Its precision, moreover, is tunable: Figure 5(right) shows the operating point can be moved along the curve, where neither text baseline offers such a control.

+ +

It also scales. Beyond the toy package, the same parser handled real standard‑library code: it built a 303‑node graph of the json package in 18 ms and a 1,903‑node graph of the http package in 91 ms, answering impact queries in under a millisecond. The approach is not a toy that only works on toys; it is a mature technique that simply had not been placed in the agent's loop.

+ +
+
What this proves, and what it does not
+

Proves: the impact‑awareness faculty of the substrate is buildable today, runs on a laptop, persists its world‑model, and beats the two strategies a context‑bounded LLM agent actually uses — on the metric (recall) that governs safety — against independent ground truth. Does not prove: that the other four faculties are equally easy (they are not — §3), that this Python‑ast analysis handles dynamic dispatch, reflection, or cross‑language repos (it does not yet — §12), or that the whole substrate has been built end‑to‑end (it has not). This is one faculty, demonstrated; the rest is specified and argued.

+
+ +

9 Prototype II: a complexity‑aware router and an assumption gate

+ +

The impact oracle prototyped one faculty. This second prototype builds the top of the +build‑opportunity map from the other direction — the two mechanisms whose pain the field's +evidence supports most directly: M1 complexity‑aware routing and M2 the assumption gate. +They compose into a single decision loop, and the whole thing runs live on the real model ladder, so the +numbers below are measured, not asserted.

+ +

+The request decision loop +
Figure 6. The composed loop: an incoming request first meets the +assumption gate, which halts and asks if the specification is incomplete (M2); a well‑specified +request is scored by the router and sent to the cheapest capable tier (M1); the output is checked by +an external verifier, and only a verified failure escalates one tier up. Every decision is a +transparent rubric — never another opaque model call.

+ +

9.1 The design commitment: transparent rubrics, external trust

+

The project's discipline — “never trust AI output; it is mathematically‑calculated +probability” — rules out the obvious lazy design. We do not ask a second LLM +“how hard is this task?” or “is this well specified?”, because that merely moves the +untrusted probability up a level and hides it. Instead both the router and the gate are +transparent, additive rubrics over explicit features. The router scores algorithmic and +architectural signals, moderate‑complexity markers, multi‑step structure, and length, and +subtracts for trivial‑task markers — every point attributable to a named feature the +user can inspect and override. The gate scores information content: concreteness anchors (a code +fence, a call signature, a worked input→output example, a named file), specific named +technologies, minus vague fillers and brevity. Neither delegates its judgment to a model. And escalation +— the only path that spends premium tokens — is driven by an external check on the +output, the same principle the substrate uses at the architecture level (the mandatory verify‑gate, +§7) and M6 uses at the generation step.

+ +

9.2 How we evaluated it — live, on real models

+

We hand‑labeled a 30‑task set: eight trivial, seven moderate, six complex (each with a +gold tier), and nine deliberately under‑specified (gold label: should ask). We ran every task +through the full loop live on the actual ladder — cheap = claude‑haiku‑4‑5, +mid = claude‑sonnet‑5, premium = claude‑opus‑4‑8 +— measuring real token counts from each call. Cost is exact arithmetic on those measured +tokens against published per‑tier list prices; the always‑premium baseline reprices the +identical tokens at the top tier, so any saving comes purely from tier selection.

+ +

+Router and gate live evaluation +
Figure 7. Live evaluation. (a) Real cost for the 21 executed tasks: +complexity‑routing spends 62 % less than always‑premium on the same measured tokens, +with most tasks served by the cheap and mid tiers. (b) All 30 decisions: every well‑specified +task routed to its gold tier (green/orange/red by tier), every under‑specified task halted by the gate +(purple ×). No errors on either axis.

+ +
+
30/30
gate accuracy
(P = R = 1.00)
+
21/21
routing exact‑tier
(well‑specified)
+
62.1%
real cost saved
vs always‑premium
+
3/3
routed‑down outputs
passed execution tests
+
+ +

The gate halted all nine under‑specified tasks (“Fix the bug.”, +“Optimize it.”) with concrete clarifying questions and spent zero generation tokens on +them; it passed all 21 well‑specified tasks through. The router placed every one of those 21 on its +gold tier. Across the executed workload the loop spent $0.54 versus $1.42 for +always‑premium — a genuine 62.1 % reduction on real tokens.

+ +

9.3 Reading this result honestly

+
+

What this proves, and what it does not. Routing down only counts as a win if the cheap +tier is still correct. So for three tasks we let the pipeline's verifier actually execute the +generated code against real test cases: is_prime and factorial passed on the cheap +tier, merge_sorted passed on the mid tier — no escalation needed, the saving was real and +the output worked. That is the honest core of the demonstration.

+

What it is not: a benchmark. The 30‑task set is small, hand‑labeled, and the rubric +thresholds were tuned against it — so the perfect separation shows the rubric can +distinguish these cases, not that it will generalize to an arbitrary task distribution. The +“retired SWE‑bench Verified” finding [C6 confirmed] is a +standing warning we apply to ourselves: a clean number on a self‑built set is a proof of +mechanism, not of field accuracy. A production version would calibrate on a large held‑out +distribution and likely learn the rubric weights rather than hand‑set them. We report the +demonstration for what it is: evidence that the two mechanisms are buildable and that they +behave as designed on cases you can inspect one by one.

+
+ +

10 The build‑opportunity map: what to build, what to skip

+ +

Eleven capabilities — five faculties, six mechanisms — and a real 2026 ecosystem to measure +them against. The point of cross‑referencing them is not to claim all eleven as open territory; it is +the opposite. A design study earns trust by saying what not to build as clearly as what to build. +We graded each capability against the actual Claude‑Code stack — skills, hooks, plugins, MCP +servers, memory backends, subagent and Agent‑Teams orchestration, model‑tiering gateways +— as solved, partial, or residual gap. The full item‑by‑item table is +the Ecosystem appendix; here is the decision it implies.

+ +
+

Already solved — do not rebuild. Two of the six mechanisms are covered well by existing +tooling. Complexity‑aware routing (M1) is handled by model tiering and gateways +(LiteLLM, OpenRouter, per‑agent model:); the only residue is transparency, not routing. +Task decomposition (M3) is handled by subagents, Agent‑Teams, and worktree fan‑out. +Building a new router or a new subagent framework would be reinventing shipped infrastructure — +exactly the waste this project set out to avoid.

+
+ +

10.1 The whitespace, ranked

+

Ranking by (evidence strength for the pain) × (how unsolved it is) × (buildability), +the genuine opportunities are:

+ + + + + + + + + + + + + + + + + + + + + + + + +
RankWhat to buildWhy (grounded)Status
1Assumption / uncertainty gate (M2)The project's named root failure and the field's named gap — “models rarely signal +uncertainty”; the METR miscalibration result [C1] is its clearest +evidence. Nothing in the stack supplies calibrated known‑unknowns.residual gap
2Validity‑anchored memory (Memory faculty)Backends store notes (Mem0, claude‑mem, Auto‑Memory) but none tracks whether a stored fact +was later invalidated by a correction; Auto‑Memory's own “memory cliff” is +documented. The write‑back‑only‑confirmed‑lessons idea is unbuilt.partial
3Mandatory pre‑action impact gate (Impact‑awareness)“One change to a shared utility can break dozens of packages with no cross‑package +awareness.” Indexers retrieve; none is a deterministic, hook‑enforced blast‑radius +check before every edit. This paper's first prototype builds exactly this.partial
4Outcome‑validated learning loop (Learning faculty)No tool closes the loop from a task's actual outcome (did the fix hold, did the PR revert) +back into changed future behavior. Frozen weights guarantee it must be external.residual gap
5Root‑cause correction gate (Self‑correction)Hooks retry; the field documents the “doom loop” — thrash that patches +symptoms and can delete its own work while declaring success. Detecting a repeating failure signature and +escalating with a diagnosis is unsolved.partial
6Scope‑minimality check (M5)Over‑engineering is untracked outside UI; measuring footprint against the task's minimal +sufficient footprint is genuine whitespace.residual gap
7Inline verification checkpoint (M6) · goal‑drift check (M4)Passive streaming and static anchors exist; a forced interpretive checkpoint during generation, and a +continuous goal‑vs‑output re‑check, do not.partial
+ +
+

What this paper actually builds, against that map. Two prototypes, targeting the top of the list +where a single session can produce real evidence. The impact oracle (§8) builds opportunity #3. +The router + assumption gate (§6) builds opportunity #1 — and, honestly, the +already‑solved M1, whose only residue is the transparency layer we add. We prototype these two because +they are the ones where ground truth is checkable now: an edit's blast radius against a mutation +oracle, a routing decision against a hand‑labeled tier, an under‑specified request against a +“should‑ask” label. The remaining opportunities — memory validity, outcome learning, +doom‑loop diagnosis — are the harder research gaps, and we mark them as such rather than +gesturing at them with a demo.

+
+ +

11 Genuinely new vs. reinvented

+

Intellectual honesty demands separating what this proposal invents from what it assembles. Almost every component exists somewhere; the contribution is the composition, the anchoring, and the target.

+ + + + + + + + + + +
ElementStatusAssessment
External memory for a frozen modelReinventedRAG, MemGPT, Generative Agents5,6,7 already attach stores to frozen models. We add nothing to the retrieval mechanism itself.
Fast/slow (non‑parametric + LoRA) learning splitReinvented framingThe CLS fast/slow split8,9 and LoRA13 are established; combining them as an agent's two learning channels is a synthesis, not a new mechanism.
Structural code graph & impact analysisReinvented (decades old)PDG, slicing, CPG30,28,31 are mature. Our parser is a modern, persistent re‑implementation, not new theory.
Validity‑anchored memory (the val term, Eq. 3)Novel emphasisScoring and pruning memory by whether its past prediction was confirmed by an external oracle (test/commit), not by the model's own judgment, is the design's sharpest departure from RAG/MemGPT's self‑managed stores.
Wiring exact impact analysis into the agent's pre‑action gateNovel compositionThe bridge — live graph → “what depends on X” → mandatory gate before the edit — is the integration §3 showed nobody has shipped, and the one we prototyped.
Epistemic‑obligation anchoring (the lens)Novel framingDeriving which safeguards are mandatory (verify‑before‑acting, least privilege) from a coherent epistemology, rather than bolting them on ad hoc, is a contribution of stance and organization.
+

In one line: the components are largely borrowed; the loop shape, the validity anchoring, and the coding‑agent target are the contribution. That is a defensible and useful kind of novelty — it is what turns five scattered literatures into one buildable architecture.

+ +

12 Limitations & threats to validity

+
    +
  • One faculty, not five. Only impact‑awareness is prototyped. Memory, learning, and imagination for code remain specified‑but‑unbuilt, and §3 is explicit that three of them are open research problems, not weekend engineering.
    • +
  • Static analysis has known blind spots. The ast‑based world‑model is sound for static Python structure but conservative‑to‑wrong on dynamic dispatch, monkey‑patching, reflection, eval, and dependency injection. It is single‑language. Precision (0.63) reflects partly that it over‑approximates; recall is its strength precisely because it errs toward inclusion.
    • +
  • Small, self‑built evaluation. Five mutations on a 10‑module package plus two stdlib scale checks is a demonstration, not a benchmark. The mutation‑derived ground truth is only as complete as the test suite (an untested consequence is invisible to it) — so true recall could be lower than measured. A convincing evaluation needs many real repositories with rich test suites and real historical edits.
    • +
  • The hard faculties may not yield to this architecture. The substrate assumes that anchoring memory/learning to external oracles is sufficient; for domains or tasks without a cheap oracle, the whole premise weakens. Code is favorable; much agent work is not.
    • +
  • The lens is framing, not proof. As stated throughout, no verse validates an engineering claim. A reader who rejects the lens loses the organizing vocabulary but none of the technical content, which stands on §3 and §8 alone.
    • +
+ +

13 Conclusion

+

The faculties a coding agent seems to lack — memory, learning, imagination, self‑correction, impact‑awareness — are not deficiencies of knowledge that scale will cure. They are structural consequences of what a frozen transformer is: a stateless map with fixed weights and a bounded window (Eq. 1, P1–P3). Because they follow from the shape of the loop, they cannot be prompted or tooled away; they can only be removed by re‑wrapping the loop into a closed, stateful cycle over persistent stores, with the model left frozen inside it (Eq. 2, Fig. 1–2). We specified that substrate faculty by faculty, said honestly which parts are open research and which are engineering, and — for the one faculty that is buildable today — shipped a running impact oracle that, against independent ground truth, never misses an affected file where the strategies a context‑bounded agent actually uses miss up to half. The Qur'anic lens gave the work its spine of obligation: know what exists before you act, verify what you are told, and hold what you can damage as a trust. Those are not just good engineering defaults; here they are the architecture. The next step is to build the memory and learning layers against the same discipline — anchored to what can be verified, not to what the model says of itself — and to evaluate the whole loop on real repositories with real histories.

+ +
+

References

+
    +
  1. Weston, Chopra, Bordes (2014). Memory Networks. arXiv:1410.3916. Proposes an explicit, addressable long-term memory component read/written by a neural controller, separate from model parameters.
    2. +
  2. Sukhbaatar, Szlam, Weston, Fergus (2015). End-To-End Memory Networks. arXiv:1503.08895. Makes memory-network read/write fully differentiable and trainable end-to-end with weaker supervision than the original Memory Networks.
    3. +
  3. Graves, Wayne, Danihelka (2014). Neural Turing Machines. arXiv:1410.5401. Couples a neural controller to an external memory matrix via differentiable attention, enabling learned algorithms like copy and sort.
    4. +
  4. Graves et al. (2016). Hybrid computing using a neural network with dynamic external memory. Nature. The Differentiable Neural Computer adds dynamic memory allocation and temporal link tracking, letting a network solve graph traversal and reasoning tasks that require variable-sized structured memory.
    5. +
  5. Lewis et al. (2020). Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks. arXiv:2005.11401 / NeurIPS. Combines a parametric seq2seq model with a non-parametric retriever over an external corpus so generation is grounded in retrieved documents rather than frozen weights alone.
    6. +
  6. Packer et al. (2023). MemGPT: Towards LLMs as Operating Systems. arXiv:2310.08560. Treats the LLM context window like virtual memory, using an OS-style paging scheme (self-directed function calls) to move information between a bounded context and external storage to give the illusion of unbounded memory.
    7. +
  7. Park et al. (2023). Generative Agents: Interactive Simulacra of Human Behavior. arXiv:2304.03442. Introduces a memory stream of observations plus a reflection mechanism that periodically synthesizes higher-level memories, retrieved and used to condition believable agent behavior.
    8. +
  8. McClelland, McNaughton, O'Reilly (1995). Why there are complementary learning systems in the hippocampus and neocortex. Psychological Review. Argues the brain needs two learning systems — a fast hippocampal one for episodic recall and a slow neocortical one for structured, interleaved consolidation — to avoid catastrophic interference.
    9. +
  9. Kumaran, Hassabis, McClelland (2016). What Learning Systems do Intelligent Agents Need? Complementary Learning Systems Theory Updated. Trends in Cognitive Sciences, 10.1016/j.tics.2016.05.004. Updates complementary learning systems theory in light of deep RL and episodic-memory findings, motivating fast/slow-learning-system architectures as a computational analogue of hippocampal-neocortical interaction.
    10. +
  10. Kirkpatrick et al. (2017). Overcoming catastrophic forgetting in neural networks. PNAS / arXiv:1612.00796. Elastic Weight Consolidation penalizes changes to parameters important for earlier tasks (via a Fisher-information-weighted quadratic penalty) to mitigate catastrophic forgetting during sequential training.
    11. +
  11. Sun et al. (2020). Test-Time Training with Self-Supervision for Generalization under Distribution Shifts. arXiv:1909.13231 / ICML 2020. Updates model parameters at test time via a self-supervised auxiliary task on each new input, adapting to distribution shift without labeled data.
    12. +
  12. Ba, Hinton, Mnih, Leibo, Ionescu (2016). Using Fast Weights to Attend to the Recent Past. arXiv:1610.06258. Introduces a fast-changing weight matrix, updated by a Hebbian-like rule from recent activity, that lets a network attend to its own recent history at a timescale between slow weights and short-term activity.
    13. +
  13. Hu et al. (2021). LoRA: Low-Rank Adaptation of Large Language Models. arXiv:2106.09685. Freezes pretrained weights and injects trainable low-rank update matrices per layer, making targeted, cheap fine-tuning/adaptation of large models feasible without touching the full parameter set.
    14. +
  14. von Oswald et al. (2022). Transformers Learn In-Context by Gradient Descent. arXiv:2212.07677. Shows constructively that a transformer's forward pass over a prompt can implement steps of gradient descent on an implicit in-context loss, giving a mechanistic account of in-context learning.
    15. +
  15. Dai et al. (2022). Why Can GPT Learn In-Context? Language Models Implicitly Perform Gradient Descent as Meta-Optimizers. arXiv:2212.10559. Draws a formal duality between attention and gradient-descent-based finetuning, framing in-context learning as an implicit, weight-unchanging optimization process.
    16. +
  16. Cobbe et al. (2021). Training Verifiers to Solve Math Word Problems. arXiv:2110.14168. Trains a separate verifier model to score candidate solutions and select among generator samples, offloading correctness-checking to a distinct learned model rather than the generator itself.
    17. +
  17. Lightman et al. (2023). Let's Verify Step by Step. arXiv:2305.20050. Shows a process-level reward model, trained on human step-by-step correctness labels, outperforms outcome-level supervision for catching reasoning errors.
    18. +
  18. Shinn et al. (2023). Reflexion: Language Agents with Verbal Reinforcement Learning. arXiv:2303.11366. Has an agent verbally reflect on task feedback/failures and stores that reflection in an episodic memory buffer used to condition subsequent attempts, substituting for gradient-based RL updates.
    19. +
  19. Madaan et al. (2023). Self-Refine: Iterative Refinement with Self-Feedback. arXiv:2303.17651. Uses the same frozen LLM to generate output, critique it, and refine it iteratively in a feedback loop, with no parameter updates or external training signal.
    20. +
  20. Wang et al. (2022). Self-Consistency Improves Chain of Thought Reasoning in Language Models. arXiv:2203.11171. Samples multiple diverse reasoning chains and marginalizes over them by majority vote on the final answer, improving reasoning accuracy without any self-correction step per se.
    21. +
  21. Huang et al. (2023). Large Language Models Cannot Self-Correct Reasoning Yet. arXiv:2310.01798. Finds that without external, ground-truth feedback, LLMs prompted to self-critique and revise their own reasoning frequently make correct answers worse, undercutting claims of intrinsic self-correction.
    22. +
  22. Ha, Schmidhuber (2018). World Models. arXiv:1803.10122. Trains a compressed generative model of an environment (VAE + RNN) and evolves a small controller entirely inside that learned latent 'dream' simulation before transferring to the real environment.
    23. +
  23. Schrittwieser et al. (2020). Mastering Atari, Go, Chess and Shogi by Planning with a Learned Model. arXiv:1911.08265 / Nature. MuZero learns a model of only the quantities relevant to planning (reward, value, policy) without reconstructing observations, and plans with MCTS purely in that learned latent model.
    24. +
  24. Hafner et al. (2023). Mastering Diverse Domains through World Models. arXiv:2301.04104. DreamerV3 learns a world model from experience and trains a policy purely from imagined rollouts inside it, achieving strong performance across many domains with one fixed set of hyperparameters.
    25. +
  25. Hafner et al. (2019). Dream to Control: Learning Behaviors by Latent Imagination. arXiv:1912.01603. Backpropagates value gradients directly through imagined multi-step latent trajectories of a learned world model to learn behaviors efficiently, without interacting with the real environment.
    26. +
  26. Friston (2010). The free-energy principle: a unified brain theory?. Nature Reviews Neuroscience. Proposes that brains (and action) minimize variational free energy, unifying perception, learning, and action under a single generative-model / predictive-coding framework — active inference.
    27. +
  27. Allamanis, Brockschmidt, Khademi (2018). Learning to Represent Programs with Graphs. arXiv:1711.00740 / ICLR 2018. Represents source code as a graph capturing syntax, control flow, and data flow, and applies gated graph neural networks to tasks like variable-misuse detection, giving a structural rather than purely textual code representation.
    28. +
  28. Horwitz, Reps, Binkley (1990). Interprocedural Slicing Using Dependence Graphs. ACM TOPLAS. Extends program dependence graphs across procedure boundaries to compute interprocedural slices, the structural basis for determining what code is affected by / affects a given statement.
    29. +
  29. Tip (1994). A Survey of Program Slicing Techniques. Journal of Programming Languages. Surveys the family of static and dynamic program-slicing techniques used to identify the subset of a program that can influence (or is influenced by) a given point — a core building block of change-impact analysis.
    30. +
  30. Ferrante, Ottenstein, Warren (1987). The Program Dependence Graph and Its Use in Optimization. ACM TOPLAS. Introduces the program dependence graph combining control and data dependence in one structure, the ancestor representation behind most modern static change-impact and slicing tools.
    31. +
  31. Yamaguchi, Golde, Arp, Rieck (2014). Modeling and Discovering Vulnerabilities with Code Property Graphs. IEEE S&P 2014. Introduces the code property graph, merging AST, control-flow graph, and program-dependence graph into one joint representation, queryable to discover vulnerability patterns across a codebase.
    32. +
  32. Alon, Zilberstein, Levy, Yahav (2019). code2vec: Learning Distributed Representations of Code. POPL 2019 / PACMPL. Represents a code snippet as a bag of AST paths and learns a distributed embedding that predicts semantic properties such as method names, giving a learned structural fingerprint of code.
    33. +
+ +
+

Appendix Evidence map — the twelve statistics, re‑grounded

+

Each load‑bearing pain‑point statistic in the 2026 discourse, traced to its +primary source and graded. Confirmed = independent primary source; vendor‑reported = +real but sourced only to a party with a commercial stake; unverifiable = could not corroborate in +primary form. Full source records and paper‑use recommendations are in evidence_map.json.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
IDClaimPrimary sourceSupportsStatus
C1Experienced open-source developers were 19% SLOWER with AI while believing they were ~20% faster (also forecast 24% speedup beforehand).Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer ProductivityM2 (assumption/uncertainty - miscalibration), P3/self-correcconfirmed
C2Trust in AI accuracy fell from 40% to 29% (or 46% actively distrust vs 33% trust per detailed breakdown); favorability 72%->60%; 66% say AI answers are 'almost right, but not quite2025 Stack Overflow Developer Survey (49,000+ respondents, 177 countries)M2 uncertainty/calibration; M6 inline verification; overall confirmed
C345% of AI-generated code samples introduce an OWASP Top-10 vulnerability; a widely-repeated '2.74x more vulnerabilities than human-written code' figure.2025 GenAI Code Security ReportM5 anti-over-engineering / code-quality thesis; security-vervendor‑reported
C4~8x rise (2024 vs prior years) in frequency of duplicated/copy-pasted code blocks; copy-paste overtaking refactored ('moved') code for the first time.AI Copilot Code Quality: 2025 Data Suggests 4x Growth in Code Clones / AI Copilot Code QuaM5 anti-over-engineering; long-term codebase impact-awarenesvendor‑reported
C5AI's primary role is as an 'amplifier' - magnifying high performers' strengths and low performers' dysfunctions; AI continues to increase delivery instability even as adoption becoState of AI-assisted Software Development 2025 (DORA Report)Systemic framing for all six mechanisms (org context mattersconfirmed
C6OpenAI retired SWE-bench Verified (Feb 2026) after finding at least 59.4% of audited (hard/failed) problems had flawed test cases and/or training-data contamination; large score gaWhy SWE-bench Verified no longer measures frontier coding capabilitiesM6 inline verification / benchmark-trust thesis; supports 'Aconfirmed
C7Median time in PR review up 441.5%; incidents-per-PR up 242.7%; bugs per developer up 54%; 31.3% more PRs merged with no review at all.The AI Engineering Report 2026: The Acceleration WhiplashM6 inline verification; review-bottleneck / verification-layvendor‑reported
C8Developers who delegate code generation to AI score 17% lower on comprehension tests, based on 'Anthropic's own research (~400,000 Claude Code sessions)'.TWO DIFFERENT STUDIES ARE BEING CONFLATED: (a) 'How AI Impacts Skill Formation' by Judy HaM6 inline verification / skill-atrophy thesis (comprehensionunverifiable
C996% of developers don't fully trust AI-generated code is functionally correct, yet only 48% always verify it before committing (a 48-point 'verification gap'/'verification debt').State of Code Developer Survey report 2026M6 inline verification - the central named gap the mechanismvendor‑reported
C10JetBrains' 2025 survey found 77% of developers still manually correct AI output for project conventions every session.The State of Developer Ecosystem 2025M4 goal-anchoring / M5 anti-over-engineering (convention driunverifiable
C11A standard MCP setup (few servers) can consume ~72% of a 200K-token context window before work begins; tool-selection accuracy drops from ~43% to below ~14% as tool count scales ('(a) 72%-context-window claim: no formal paper found, only recurring blog anecdotes (Scott M1 complexity-aware routing / M3 task decomposition (contextvendor‑reported
C12LLM evaluators recognize and favor their own generations - self-preference bias correlates with self-recognition ability - motivating why 'AI verifying AI' is structurally weak.LLM Evaluators Recognize and Favor Their Own GenerationsM6 inline verification (why an LLM cannot be its own sole veconfirmed
+ +
+

Appendix Ecosystem map — faculties & mechanisms vs. the real stack

+

Each faculty and mechanism graded against the actual 2026 Claude‑Code stack (skills, hooks, +plugins, MCP servers, memory backends, subagent/Agent‑Teams orchestration, model‑tiering +gateways). Solved = covered well by existing tooling; partial = tooling exists but leaves a +named gap; residual‑gap = genuine whitespace. Full tool lists and citations are in +ecosystem_map.json.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CapabilityExisting tooling (sample)StatusResidual gapWhat we'd add
Memory (across sessions)
faculty		CLAUDE.md; Auto Memory; Mem0; claude-mem; HindsightpartialPain-points report names this directly: 'no standard, tool-agnostic, durable memory layer that reliably persists project knowledge, decisions, and corrections across sessions, tools, and teammates' (Open gap #2), with JetBrains finding 77% Validity-anchored memory: stored facts carry a confirmed/discredited state updated by verified outcomes, not a static note-dump -- addresses the correction dime
Learning (from outcomes)
faculty		Auto Memory 'feedback' memory type; Superpowers subagent-driven-development two-stage review; ccusage / usage trackersresidual-gapNo tool in the stack closes the loop from a task's actual outcome (did the fix hold, did the test stay green, was the PR reverted) back into changed future behavior. Auto Memory's feedback type is note storage, not an evaluated lesson; frozOutcome-validated learning loop: capture a task's actual downstream result, verify it independently, and write back only confirmed lessons (not raw logs) into t
Imagination (simulate consequences before acting)
faculty		Plan mode / built-in Plan subagent; Explore subagent; Superpowers brainstorm -> plan -> TDD -> subagent-dev -> reviewpartialThese are textual/symbolic planning steps within the stack itself, not a simulation of an edit's actual downstream effects -- no hook, skill, or MCP server in the stack landscape traces a dependency graph before a change is made. (The pain-A pre-action impact simulation step -- dependency-graph/call-graph traversal that predicts affected files/tests before code is written, feeding predicted blast
Self-correction
faculty		Stop hooks; PreToolUse blocking hooks; Playwright's healer agent; TDD RED-GREEN-REFACTOR; /security-review and pr-review-toolkitpartialThese are retry/gate mechanisms, not diagnosis. The pain-points report documents the 'doom loop' by name -- an agent that 'makes a mistake, tries to fix it, makes it worse' and can even delete its own changes while declaring success -- and A root-cause-aware correction gate that distinguishes genuine progress from thrash (same failure signature repeating), halts and escalates with a diagnosis inst
Impact-awareness (what exists in the codebase / what an edit affects)
faculty		filesystem and memory; Context7; sequential-thinking MCP server; sandboxing's 'trust verification for new codebases/MCP servers'partialThese stack primitives expose retrieval (docs, a memory graph, filesystem access) that the model may or may not consult -- none is a deterministic, mandatory gate run before every edit, and none computes blast radius. (The pain-points reporA MANDATORY pre-action impact gate -- a hook-enforced (not LLM-judged) dependency-graph query that runs before every edit is applied, not an optional retrieval
M1 Complexity-aware routing
mechanism		Model tiering; Per-agent model: field in .claude/agents/; LLM gateways: LiteLLM, Portkey, OpenRouter; Named orchestration pattern: 'orchestrator-classifies-then-routes to Haiku/Sonnet/Opus by complexity'solvedThe routing decision is set at config time (a developer picks model: haiku for an agent) or by the gateway's own cost logic -- it is not a transparent, per-task, auditable classification the user can see and override before dispatch.A transparent complexity classification surfaced to the user before dispatch, making the routing decision auditable rather than a silent config default.
M2 Assumption / uncertainty
mechanism		Reasoning models acting as an internal review pass; Context7; Spec-driven developmentresidual-gapNamed directly and unsolved in the pain-points report: 'Models rarely signal uncertainty or say I can't do this' -- the report's own build opportunity is 'calibrated-confidence and known-unknowns tooling -- agents that flag low-confidence rThis is the paper's named root failure: an assumption/uncertainty gate that requires the model to enumerate its unstated assumptions and ask before proceeding w
M3 Task / session decomposition
mechanism		Subagents; Agent Teams; git worktrees for parallel-agent isolation; Parallel fan-out orchestration pattern; Superpowers subagent-driven-developmentsolvedMature, well-tooled pattern. The remaining gap is small: deciding the decomposition boundary itself (what counts as independent vs. needs shared context) is still a manual/heuristic judgment call by the developer, not something any listed tAutomatic decomposition-boundary detection -- deciding when to fork a subagent/session vs. keep work in one context, rather than leaving that call to developer
M4 Goal-anchoring
mechanism		CLAUDE.md; Spec-driven development; /goal command; Task lists in Agent TeamspartialThese anchors are loaded once and are static. The report documents the anchor decaying over a session: 'circular reasoning at 20% [context usage], context compression wiping scrollback at 40%,' and Open gap #1 states plainly that 'specs driA continuous goal-drift check that periodically re-validates in-progress output against the original stated objective, rather than loading the goal once and tru
M5 Anti-over-engineering
mechanism		frontend-design skill's minimalism discipline; TDD RED-GREEN-REFACTOR; pr-review-toolkit / CodeRabbit / Greptile review agentsresidual-gapThe frontend-design skill's discipline is explicitly UI-only; no general-purpose backend/architecture tool measures unnecessary abstraction, premature generalization, or scope creep against the stated task. GitClear's tracked defects (8x duA scope-minimality check that compares an implementation's footprint (files touched, abstractions introduced) against the stated task requirement and flags addi
M6 Inline verification
mechanism		Real-time thinking/tool-call streaming; Self-QA pattern; PostToolUse hooks; Plan mode human-in-the-loop gatingpartialStreaming and immediate post-edit hooks give passive visibility but do not require a human interpretive checkpoint DURING generation. The pain-points report's central statistic is the '48-point gap': 'Sonar's 2026 report: 96% of developers A mandatory inline checkpoint that surfaces a human-checkable claim or diff at each meaningful generation step, shifting verification earlier instead of batchin
+ +

Appendix Qur'anic‑lens mapping table

+

The complete 14‑row mapping (12 load‑bearing, 2 metaphor). Canonical Arabic and translations retrieved from quran.ai; full text, tafsir references, and design principles in the companion artifact quran_lens.json. Caveat: this table is a design lens, not technical or theological authority — see §5.

+ + + + + + + + + + + + + + + + + +
Concept / verseRetrieved glossFacultyDesign principle (abbrev.)Type
17:36 — lā taqfu (do not pursue without knowledge)Do not follow blindly what you do not know to be true: ears, eyes, and heart, you will be questioned about all these.IMPACT-AWARENESSBefore any code mutation (file write, delete, refactor), the agent must run a pre-action verification gate that checks: (1) what entities in the codebase wil…load-bearing
49:6 — tabayyun (verify reports before acting)Believers, if a troublemaker brings you news, check it first, in case you wrong others unwittingly and later regret what you have done,SELF-CORRECTIONThe agent architecture must include a verification gate between receiving information (from context, tool output, or its own prior reasoning) and acting on itload-bearing
2:31-32 — taʿlīm al-asmāʾ (He taught Adam the names of all things)(2:31) He taught Adam all the names [of things], then He showed them to the angels and said, ‘Tell me the names of these if you truly [think you can].’ (2:32) They sai…WORLD-MODEL (codebase graph)The agent must maintain a structured representation of what exists in the codebase — a graph of files, functions, classes, dependencies, and their relationsh…load-bearing
20:114 — rabbi zidnī ʿilmā (My Lord, increase me in knowledge)exalted be God, the one who is truly in control. [Prophet], do not rush to recite before the revelation is fully complete but say, ‘Lord, increase me in knowledge!’CONTINUAL LEARNINGThe agent's knowledge must be treated as perpetually incomplete, with an explicit mechanism for incremental growthload-bearing
96:1-5 — iqraʾ / ʿallama bi-l-qalam (Read; taught by the pen)(96:1) Read! In the name of your Lord who created: (96:2) He created manfrom a clinging form. (96:3) Read! Your Lord is the Most Bountiful One (96:4) who taught by [me…PERSISTENT MEMORYKnowledge must be externalized to survive beyond the moment of computationload-bearing
39:9 — hal yastawī lladhīna yaʿlamūna wa-lladhīna lā yaʿlamūn (are those who know equal to those who do not know?)What about someone who worships devoutly during the night, bowing down, standing in prayer, ever mindful of the life to come, hoping for his Lord’s mercy? Say, ‘How ca…MEMORY + LEARNING (epistemic grounding)An agent that retains and learns from experience is categorically more capable and more trustworthy than one that does not — this is not a nice-to-have but a…metaphor
4:82 — tadabbur al-Qurʾān (do they not reflect deeply upon the Quran; inconsistency as a sign of non-divine origin)Will they not think about this Quran? If it had been from anyone other than God, they would have found much inconsistency in it.SELF-CORRECTION (metacognitive controller)The agent must run self-consistency checks on its own output before committing itload-bearing
47:24 — tadabbur / aqfāl ʿalā qulūb (do they not ponder, or are there locks on their hearts?)Will they not contemplate the Quran? Do they have locks on their hearts?SELF-CORRECTION (iterative refinement)Reflection must be a deliberate, repeatable operation, not a one-pass judgmentmetaphor
33:72 — al-amāna (the Trust offered to heavens, earth, mountains; borne by the human)We offered the Trust to the heavens, the earth, and the mountains, yet they refused to undertake it and were afraid of it; mankind undertook it- they have always been …STEWARDSHIP ETHICS (bounded autonomy)An agent that can modify a codebase bears a trust (amāna) — it has accepted responsibility for something it can damageload-bearing
CONCEPT: ʿilm → fahm → ḥikma (knowledge → understanding → wisdom)A classical epistemological hierarchy: ʿilm is raw knowledge (facts, data, observations); fahm is comprehension (grasping the relations between facts, seeing why); ḥik…MEMORY + LEARNING (data architecture)The agent's memory/learning stack must be layered, not flatload-bearing
CONCEPT: ḥifẓ + murājaʿa (preservation + spaced review/revision)The classical Quranic memorization discipline: ḥifẓ is initial encoding and faithful preservation of the text; murājaʿa is the regular, spaced revision that prevents d…PERSISTENT MEMORY (consolidation & maintenance)Memory is not write-onceload-bearing
CONCEPT: tadabbur (deep, structured reflection — returning to examine consequences)Tadabbur is not casual thought; its root d-b-r relates to 'what is behind' or 'what follows' — examining the consequences and deeper implications of something. In Qura…SELF-CORRECTION (metacognitive controller)The metacognitive controller is a tadabbur loop: after the agent generates a plan or action, the controller examines what comes after (d-b-r) — what are the …load-bearing
CONCEPT: tabayyun (verification of reports before acting on them)Tabayyun is the act of seeking clarity and verification before acting on received information. In 49:6, it is commanded as a mandatory step between receiving a report …SELF-CORRECTION (verification gate)The verification gate is the architectural realization of tabayyunload-bearing
CONCEPT: amāna (trust, stewardship, accepted responsibility)Amāna is the trust or responsibility that is accepted voluntarily and carries accountability. In 33:72, it is described as something so weighty that the heavens, earth…STEWARDSHIP ETHICS (safe operation)When an agent is granted access to a codebase, it accepts an amāna — a trust that carries accountability proportional to its capabilityload-bearing
+ +
+

Companion artifacts. This paper is accompanied by: the runnable impact_oracle_src.zip (world‑model + oracle + demo + 36‑test suite); the impact‑graph and evaluation figures; the structured literature gap_map.json and 32‑entry references.json; and the quran_lens.json mapping table. All results are reproducible via the included demo.py.

+

On the lens. Qur'anic text throughout was retrieved from the quran.ai service and presented directly, not paraphrased. It is used as an epistemological and ethical design lens only; every technical claim rests on the cited literature and the prototype evaluation, independent of the lens.

+
+ +
+ + diff --git a/docs/cognitive-substrate/cognitive_substrate_whitepaper.pdf b/docs/cognitive-substrate/cognitive_substrate_whitepaper.pdf new file mode 100644 index 0000000..44ce7bb Binary files /dev/null and b/docs/cognitive-substrate/cognitive_substrate_whitepaper.pdf differ diff --git a/docs/cognitive-substrate/deliverable-package.md b/docs/cognitive-substrate/deliverable-package.md new file mode 100644 index 0000000..dbc8f98 --- /dev/null +++ b/docs/cognitive-substrate/deliverable-package.md @@ -0,0 +1,85 @@ +# A Cognitive Substrate for Coding Agents — Deliverable Package +### Theory → Evidence → Build-Map edition (v2) + +**One-line thesis:** The faculties a coding agent lacks — memory, learning, imagination, self-correction, impact-awareness — are not gaps in the model's *knowledge* but structural consequences of what a frozen transformer *is* (a stateless map `y = f_θ(x)`, fixed weights, bounded window). They cannot be prompted or tooled away; they can only be supplied by **re-wrapping the input→process→output loop** into a closed, stateful cycle around the frozen model. + +**What v2 adds.** The first edition argued the five faculties from first principles and prototyped the one that is buildable today. This edition (1) **grounds the argument in the field's own evidence** — twelve load-bearing pain-point statistics independently re-grounded from primary sources and graded *confirmed / vendor-reported / unverifiable*; (2) adds **six metacognitive mechanisms** the frozen loop also lacks (routing, assumption gate, decomposition, goal-anchoring, anti-over-engineering, inline verification); (3) **maps all eleven capabilities against the real 2026 Claude-Code stack**, marking each solved / partial / residual-gap so we say clearly *what not to build*; and (4) ships a **second runnable prototype** — a complexity-aware router + assumption gate, evaluated live on real models. + +> **Governing discipline (the user's, adopted throughout):** *AI output is mathematically-calculated probability — non-deterministic, and never blindly trusted.* Every claim in this package is graded by how well it is sourced; every prototype decision is a transparent, attributable rule rather than another opaque model call; and trust is always earned by an **external** check, never asserted by the model. + +--- + +## What's in this package + +### 1. The white paper (core deliverable) — 48 pp +- **`cognitive_substrate_whitepaper.pdf`** / **`cognitive_substrate_whitepaper.html`** — the full study, 13 sections + 3 appendices, 7 figures. + - **§1–3** the root cause and the five faculties (from v1): *why* each faculty is structurally absent (P1 statelessness, P2 frozen weights, P3 bounded context), each grounded in the real literature. + - **§4 Evidence** *(new)* — the twelve statistics, re-grounded. 5 confirmed, 5 vendor-reported, 2 unverifiable. + - **§5** the Qur'anic epistemic lens — design framing/ethics, never technical authority. + - **§6 Six mechanisms** *(new)* — M1 routing, M2 assumption gate, M3 decomposition, M4 goal-anchoring, M5 anti-over-engineering, M6 inline verification — each formalized, with ecosystem status and a Qur'anic anchor. + - **§7** the cognitive substrate, now with the six-mechanism metacognitive control layer (Figure 3). + - **§8 Prototype I** the impact oracle (from v1). **§9 Prototype II** *(new)* the router + gate. **§10 Build-map** *(new)* the ranked opportunity list. + - **§11** new-vs-reinvented. **§12** limitations. **§13** conclusion. + +### 2. Prototype I — Codebase World-Model + Impact Oracle +- **`impact_oracle_src.zip`** — parses a codebase (AST) into a **persistent dependency graph**, predicts the **blast radius** of a proposed edit via reverse-dependency traversal with confidence decay. `python demo.py` runs end-to-end; `pytest` → **36 tests pass** with zero setup. Builds opportunity #3. + +### 3. Prototype II — Complexity-aware router + Assumption gate *(new)* +- **`router_gate_src.zip`** — the two mechanisms at the top of the build-map, composed as `gate → route → execute → verify → escalate`. Both are **transparent additive rubrics**, not opaque LLM calls; escalation is driven by an external check. `python demo.py`, `pytest` → **19 tests pass**, `python evaluate.py --live` reproduces the live numbers. +- **`eval_results.json`** — the live evaluation record (real measured tokens). + +### 4. Evidence & ecosystem maps *(new)* +- **`evidence_map.json`** / **`evidence_map.md`** — every load-bearing statistic, its primary source, and its status. +- **`ecosystem_map.json`** / **`ecosystem_map.md`** — every faculty & mechanism vs. the real stack, with residual gap and proposed contribution. + +### 5. Figures & schematics +- **`schematic_loop.png`** (Fig 1) · **`schematic_system.png`** (Fig 2) · **`schematic_extended.png`** (Fig 3, *new* — the six-mechanism control layer) · **`impact_graph.png`** (Fig 4) · **`eval_precision_recall.png`** (Fig 5) · **`schematic_router_loop.png`** (Fig 6, *new*) · **`router_eval.png`** (Fig 7, *new*). + +### 6. Supporting artifacts (from v1) +- **`gap_map.json`/`.md`**, **`references.json`** (32 sources), **`quran_lens.json`/`.md`** (14-row concept→faculty→design-principle mapping). + +--- + +## The honest headline results + +### Prototype I — Impact Oracle (against mutation-derived ground truth, 5 real edits) +| Method | Precision | Recall | F1 | +|---|---|---|---| +| **Graph Oracle** (ours) | 0.63 | **1.00** | 0.75 | +| Grep baseline (what agents do today) | 0.73 | 0.94 | **0.79** | +| Edited-file-only | 1.00 | 0.53 | 0.65 | + +The oracle does **not** dominate F1 — grep edges it at the default threshold, and we say so. What the oracle uniquely provides is **guaranteed recall**: for "show me everything my edit could break," a silent miss costs far more than an extra file to check, and only the structural oracle drives false negatives to zero (precision tunable, best F1 = 0.79 at threshold 0.4). + +### Prototype II — Router + Gate (live, on real models: haiku / sonnet / opus) +| Metric | Result | +|---|---| +| Gate accuracy (should-ask) | 30/30 · precision 1.00 · recall 1.00 | +| Routing accuracy (well-specified tasks) | 21/21 exact tier | +| **Real cost saved vs always-premium** | **62.1%** (same measured tokens) | +| Execution-verified sub-experiment | 3/3 routed-down outputs passed real test cases | + +**Honest caveat (both prototypes):** these are **demonstrations, not benchmarks**. The router's 30-task set is hand-labeled and the rubric thresholds were tuned against it, so perfect separation shows the rubric *can* distinguish these cases — not field accuracy. The oracle's evaluation is 5 mutations + 2 stdlib scale checks. We apply the "retired SWE-bench Verified" caution (§4, confirmed) to our own numbers. + +## What the evidence re-grounding caught + +The independent re-grounding **changed our claims** — three widely-repeated numbers did not survive and are *not* used as fact in this paper: +- **"2.74× more vulnerabilities"** is not traceable to Veracode's own report (only their 45% OWASP figure is); likely conflated with a separate study. +- **"17% lower comprehension / 400K sessions"** merges two different studies — the session study contains no comprehension finding. +- **GitClear 4× vs 8×** internal inconsistency and **JetBrains 77%** could not be located in primary form. + +That a re-grounding pass corrected the paper is the point, not an embarrassment: it is the same discipline the architecture makes structural — *a stored fact is provisional until an external check confirms it.* + +## The build-opportunity map (what to build, what to skip) + +**Already solved — do not rebuild:** M1 routing (model tiering + gateways like LiteLLM/OpenRouter) and M3 decomposition (subagents, Agent-Teams). The router prototype's honest contribution is only the *transparency layer*, and we say so. + +**The genuine whitespace, ranked:** (1) **assumption/uncertainty gate** — the project's named root failure and the field's named gap; nothing supplies calibrated known-unknowns. (2) **validity-anchored memory** — backends store notes, none tracks invalidation-by-correction. (3) **mandatory pre-action impact gate** — indexers retrieve, none is a deterministic blast-radius check. (4) outcome-validated learning. (5) doom-loop / root-cause correction. (6) scope-minimality. This paper prototypes #1 and #3 — the two where a single session can produce checkable ground truth. + +## What is genuinely new vs. reinvented + +Most components are borrowed (external memory, fast/slow learning, code graphs, model tiering — all exist). The contribution is **the composition and the framing**: the closed-loop shape; **validity-anchored memory** (prune by whether a past prediction was confirmed by an *external* oracle, not by the model's own judgment); wiring exact impact analysis into a **mandatory pre-action gate**; a **transparent** router/gate that explains every decision; and deriving *which* safeguards are non-negotiable from a coherent epistemology. That turns scattered literatures and named-but-unsolved gaps into one buildable architecture aimed squarely at coding agents. + +## Scope & limitations (stated honestly) + +Two faculties/mechanisms are prototyped, not eleven. The impact oracle's static analysis is single-language (Python) and conservative on dynamic dispatch. The router/gate rubrics are keyword heuristics tuned on a small hand-labeled set. Memory validity, outcome learning, and doom-loop diagnosis remain *specified but unbuilt* — the harder research gaps, marked as such rather than gestured at with a demo. The lens is framing: reject it and you lose the organizing vocabulary but none of the technical content. diff --git a/docs/cognitive-substrate/ecosystem_map.md b/docs/cognitive-substrate/ecosystem_map.md new file mode 100644 index 0000000..d2e8ab1 --- /dev/null +++ b/docs/cognitive-substrate/ecosystem_map.md @@ -0,0 +1,86 @@ +# Ecosystem Map: Cognitive Substrate vs. the Mid-2026 Claude Code Stack + +Cross-references the white paper's 5 structural faculties (memory, learning, imagination, +self-correction, impact-awareness) and 6 extension mechanisms (M1-M6: complexity routing, +assumption/uncertainty, task decomposition, goal-anchoring, anti-over-engineering, inline +verification) against the tools actually documented in the stack landscape file, mid-2026. +`existing_tooling` names are drawn from the stack landscape document only; where the separate +pain-points report names additional workaround tools or "Open gap" / "Build opportunity" +language, those are cited in the residual-gap column for context, not counted as stack-sourced +existing tooling. Purpose: don't build what already exists; be honest about what does. + +**Summary: 11 items -- 2 solved by existing tooling, 6 partially +addressed (documented gap remains), 3 residual gap (essentially unaddressed).** + +## Faculties (from the base white paper) + +| Item | Status | Existing tooling (from stack file) | Residual gap | Our contribution | +|---|---|---|---|---| +| **Memory (across sessions)** | Partial | CLAUDE.md (always-on project context, survives /compact); Auto Memory (v2.1.59+, self-written notes in ~/.claude/projects//memory/); Mem0 (~58k stars, hosted MCP + lifecycle hooks); claude-mem (SQLite+FTS5/vector, Haiku summaries); Hindsight (94.6% LongMemEval, self-hostable); supermemory (cross-machine sync); Anthropic memory tool (API, client-side file ops) | Pain-points report names this directly: 'no standard, tool-agnostic, durable memory layer that reliably persists project knowledge, decisions, and corrections across sessions, tools, and teammates' (Open gap #2), with JetBrains finding 77% of devs still manually re-correct conventions every session. Auto Memory itself has a documented 'memory cliff' -- 200-line index cap + 5-files-per-turn retrieval with silent truncation. Existing backends store notes; none track whether a stored fact has since been invalidated by a correction. | Validity-anchored memory: stored facts carry a confirmed/discredited state updated by verified outcomes, not a static note-dump -- addresses the correction dimension the named tools don't. | +| **Learning (from outcomes)** | Residual gap | Auto Memory 'feedback' memory type (stores feedback notes); Superpowers subagent-driven-development two-stage review (spec compliance, then code quality); ccusage / usage trackers (cost signal only, not outcome learning) | No tool in the stack closes the loop from a task's actual outcome (did the fix hold, did the test stay green, was the PR reverted) back into changed future behavior. Auto Memory's feedback type is note storage, not an evaluated lesson; frozen weights mean nothing updates the model itself. Neither source names a shipped mechanism for this -- it is unaddressed rather than partially addressed. | Outcome-validated learning loop: capture a task's actual downstream result, verify it independently, and write back only confirmed lessons (not raw logs) into the memory store consulted on the next similar task. | +| **Imagination (simulate consequences before acting)** | Partial | Plan mode / built-in Plan subagent (read-only); Explore subagent; Superpowers brainstorm -> plan -> TDD -> subagent-dev -> review | These are textual/symbolic planning steps within the stack itself, not a simulation of an edit's actual downstream effects -- no hook, skill, or MCP server in the stack landscape traces a dependency graph before a change is made. (The pain-points report separately documents spec-driven-development tools such as Spec Kit, Kiro, and OpenSpec as workarounds, and names the same open gap: 'Cross-service/architectural-impact awareness at monorepo scale remains weak,' with the build opportunity framed as 'architecture-aware agents that reason over dependency graphs.') | A pre-action impact simulation step -- dependency-graph/call-graph traversal that predicts affected files/tests before code is written, feeding predicted blast radius into the plan rather than discovering it after the edit. | +| **Self-correction** | Partial | Stop hooks (e.g. npm test || exit 2, forces continued work); PreToolUse blocking hooks; Playwright's healer agent (auto-repairs failing tests); TDD RED-GREEN-REFACTOR (Superpowers); /security-review and pr-review-toolkit (5 parallel Sonnet review agents); Ralph Wiggum loop / /loop, /goal, /batch | These are retry/gate mechanisms, not diagnosis. The pain-points report documents the 'doom loop' by name -- an agent that 'makes a mistake, tries to fix it, makes it worse' and can even delete its own changes while declaring success -- and states plainly: 'Automatic doom-loop detection and root-cause reasoning (vs. symptom-patching) are largely unsolved,' naming the build opportunity as 'loop-breakers and budget circuit-breakers that detect thrashing, halt, and escalate to a human with a diagnosis.' | A root-cause-aware correction gate that distinguishes genuine progress from thrash (same failure signature repeating), halts and escalates with a diagnosis instead of blindly re-looping. | +| **Impact-awareness (what exists in the codebase / what an edit affects)** | Partial | filesystem and memory (knowledge graph) MCP servers (modelcontextprotocol/servers monorepo); Context7 (version-specific library docs via MCP); sequential-thinking MCP server; sandboxing's 'trust verification for new codebases/MCP servers' (native control) | These stack primitives expose retrieval (docs, a memory graph, filesystem access) that the model may or may not consult -- none is a deterministic, mandatory gate run before every edit, and none computes blast radius. (The pain-points report separately names full-repo indexers such as Sourcegraph Cody/Amp, Augment Code, Greptile, CodeAnt AI, and CodeRabbit as workarounds, but states directly: 'one change to a shared utility can break dozens of packages with no cross-package awareness' and 'Cross-service/architectural-impact awareness at monorepo scale remains weak.') | A MANDATORY pre-action impact gate -- a hook-enforced (not LLM-judged) dependency-graph query that runs before every edit is applied, not an optional retrieval step the model may or may not consult. | + +## Mechanisms (this extension: M1-M6) + +| Item | Status | Existing tooling (from stack file) | Residual gap | Our contribution | +|---|---|---|---|---| +| **M1 Complexity-aware routing** | Solved | Model tiering (Haiku/Sonnet/Opus, /model command); Per-agent model: field in .claude/agents/; LLM gateways: LiteLLM, Portkey, OpenRouter; Named orchestration pattern: 'orchestrator-classifies-then-routes to Haiku/Sonnet/Opus by complexity' | The routing decision is set at config time (a developer picks model: haiku for an agent) or by the gateway's own cost logic -- it is not a transparent, per-task, auditable classification the user can see and override before dispatch. | A transparent complexity classification surfaced to the user before dispatch, making the routing decision auditable rather than a silent config default. | +| **M2 Assumption / uncertainty** | Residual gap | Reasoning models acting as an internal review pass; Context7 (reduces API-hallucination via current docs, not the underlying issue); Spec-driven development (Spec Kit, Kiro, OpenSpec) as a workaround that forces the human to write the spec | Named directly and unsolved in the pain-points report: 'Models rarely signal uncertainty or say I can't do this' -- the report's own build opportunity is 'calibrated-confidence and known-unknowns tooling -- agents that flag low-confidence regions and ask clarifying questions instead of confabulating.' SDD only helps if the human already wrote a complete spec; it does not make the model self-flag what it doesn't know. | This is the paper's named root failure: an assumption/uncertainty gate that requires the model to enumerate its unstated assumptions and ask before proceeding when confidence is low, instead of depending on the user having pre-empted every ambiguity in a spec. | +| **M3 Task / session decomposition** | Solved | Subagents (isolated context windows, only final message returned); Agent Teams (experimental, shared task list, teammate-to-teammate messaging); git worktrees for parallel-agent isolation; Parallel fan-out orchestration pattern (60-80% wall-clock savings); Superpowers subagent-driven-development (fresh subagent per task) | Mature, well-tooled pattern. The remaining gap is small: deciding the decomposition boundary itself (what counts as independent vs. needs shared context) is still a manual/heuristic judgment call by the developer, not something any listed tool decides automatically. | Automatic decomposition-boundary detection -- deciding when to fork a subagent/session vs. keep work in one context, rather than leaving that call to developer heuristics. | +| **M4 Goal-anchoring** | Partial | CLAUDE.md (always-on context, re-read from disk, survives /compact); Spec-driven development (spec as source of truth); /goal command (v2.1.139+); Task lists in Agent Teams | These anchors are loaded once and are static. The report documents the anchor decaying over a session: 'circular reasoning at 20% [context usage], context compression wiping scrollback at 40%,' and Open gap #1 states plainly that 'specs drift out of sync with code (context drift)' with 'no mature, widely-adopted tooling [that] keeps specs, code, and tests continuously verified against each other.' | A continuous goal-drift check that periodically re-validates in-progress output against the original stated objective, rather than loading the goal once and trusting it stays in view. | +| **M5 Anti-over-engineering** | Residual gap | frontend-design skill's minimalism discipline ('remove one accessory') -- UI/design scope only; TDD RED-GREEN-REFACTOR (write only enough code to pass a test); pr-review-toolkit / CodeRabbit / Greptile review agents (not scoped to over-engineering specifically) | The frontend-design skill's discipline is explicitly UI-only; no general-purpose backend/architecture tool measures unnecessary abstraction, premature generalization, or scope creep against the stated task. GitClear's tracked defects (8x duplication, copy-paste) are the opposite failure mode (under-abstraction) -- over-engineering isn't named or tracked by any tool in either source. | A scope-minimality check that compares an implementation's footprint (files touched, abstractions introduced) against the stated task requirement and flags additions the task didn't ask for. | +| **M6 Inline verification** | Partial | Real-time thinking/tool-call streaming (Claude Code); Self-QA pattern (agent opens localhost and checks its own changes); PostToolUse hooks (auto-format/lint immediately after a write); Plan mode human-in-the-loop gating | Streaming and immediate post-edit hooks give passive visibility but do not require a human interpretive checkpoint DURING generation. The pain-points report's central statistic is the '48-point gap': 'Sonar's 2026 report: 96% of developers don't fully trust AI code is correct, yet only 48% always verify' -- verification is deferred to an end-of-task PR review (median review time up 441.5%), not performed inline. | A mandatory inline checkpoint that surfaces a human-checkable claim or diff at each meaningful generation step, shifting verification earlier instead of batching it into a single end-of-task review. | + +## What NOT to build (already solved) + +Two items need no new infrastructure. **M1 (complexity-aware routing)** is a mature, named +pattern in the 2026 stack: model tiering (Haiku/Sonnet/Opus via `/model`), per-agent `model:` +fields in `.claude/agents/`, and LLM gateways (LiteLLM, Portkey, OpenRouter) that already route +by cost/complexity, with "orchestrator-classifies-then-routes to Haiku/Sonnet/Opus by +complexity" documented as a standard orchestration pattern. **M3 (task/session decomposition)** +is equally mature: Subagents (isolated context, final-message-only return), the experimental +Agent Teams model, git worktrees for isolation, and Superpowers' subagent-driven-development +already give parallel fan-out with 60-80% wall-clock savings. Building a new decomposition or +routing *engine* would be reinventing a well-adopted primitive; the honest opportunity in both +cases is a thin transparency/audit layer on top (surfacing *why* a routing or fork decision was +made), not the mechanism itself. + +Three faculties also have substantial existing tooling worth acknowledging even though a gap +remains: **memory** (CLAUDE.md, Auto Memory, Mem0, claude-mem, Hindsight, supermemory, the +Anthropic memory tool API), **imagination** (Plan mode, Explore subagent, Superpowers' +brainstorm-to-review pipeline), and **impact-awareness** (filesystem/memory-knowledge-graph and +Context7 MCP servers, sequential-thinking MCP, sandboxing's codebase trust verification). None +of these should be reinvented from scratch -- the substrate's job is to close the specific, +named gap each one leaves open (below), not to duplicate the retrieval/planning/indexing layer. + +## Genuine whitespace + +Three items are essentially unaddressed by any tool named in the stack landscape file, and each +is independently corroborated by the pain-points report's own "Open gap" / "Build opportunity" +language rather than asserted by us. **Learning from outcomes** has no counterpart at all -- +Auto Memory's "feedback" type stores notes, not verified, evaluated lessons, and nothing in the +stack closes the loop from an actual task outcome back into changed future behavior. **M2 +(assumption/uncertainty)** is named explicitly as unsolved: models "rarely signal uncertainty or +say I can't do this," and the report's own build opportunity calls for "calibrated-confidence and +known-unknowns tooling" -- this is also the paper's named ROOT deficit, so it is the highest- +priority build target. **M5 (anti-over-engineering)** has a UI-only analogue (the frontend-design +skill's minimalism discipline) but no general-purpose backend equivalent; scope creep and +premature abstraction are not tracked by any named quality gate. + +Two further items are partial-with-a-documented-gap rather than untouched: **self-correction** +has retry/gate mechanisms in the stack (Stop hooks, the Ralph Wiggum loop, Playwright's healer, +pr-review-toolkit) but the pain-points report names "doom-loop" thrashing and calls automatic +root-cause detection "largely unsolved"; **M6 (inline verification)** has passive +streaming/self-QA but the defining 2026 statistic -- 96% of developers don't fully trust AI code +yet only 48% always verify it -- shows verification is still batched into an end-of-task review, +not performed while code is written. **M4 (goal-anchoring)** and **impact-awareness** round out +the partial column: both have static, one-time anchors (CLAUDE.md, `/goal`, MCP filesystem/memory +retrieval) but no tooling in the stack that continuously re-checks alignment or mandatorily gates +an edit against blast radius as the session progresses. + +## Sources + +- Stack landscape: `source_stack_landscape.md` (mid-2026 Claude Code / agent tooling survey) -- sole source for `existing_tooling`. +- Pain-points report: `source_painpoints_report.md` (full-SDLC field report and build-opportunity map, mid-2026) -- source for residual-gap quotes and named workaround tools cited for context only. diff --git a/docs/cognitive-substrate/evidence_map.md b/docs/cognitive-substrate/evidence_map.md new file mode 100644 index 0000000..901e58d --- /dev/null +++ b/docs/cognitive-substrate/evidence_map.md @@ -0,0 +1,213 @@ +# Evidence Map: Independent Re-Grounding of Pain-Point Statistics + +*12 load-bearing claims verified against primary sources, 5 confirmed, 5 vendor-only, 2 unverifiable.* + +## Summary Table + +| # | Claim | Primary Source | Status | Key Discrepancy | +|---|---|---|---|---| +| 1 | Experienced open-source developers were 19% SLOWER with AI while believing they were ~20% ... | arXiv preprint / METR blog (2025) | ✅ confirmed | See note | +| 2 | Trust in AI accuracy fell from 40% to 29% (or 46% actively distrust vs 33% trust per detai... | Stack Overflow (2025) | ✅ confirmed | See note | +| 3 | 45% of AI-generated code samples introduce an OWASP Top-10 vulnerability; a widely-repeate... | Veracode (2025) | ⚠️ vendor-only | See note | +| 4 | ~8x rise (2024 vs prior years) in frequency of duplicated/copy-pasted code blocks; copy-pa... | GitClear (Bill Harding et al.) (2025) | ⚠️ vendor-only | GitClear's own title says 4x, body/press say 8x - unresolved | +| 5 | AI's primary role is as an 'amplifier' - magnifying high performers' strengths and low per... | Official DORA/Google Cloud report (2025) | ✅ confirmed | See note | +| 6 | OpenAI retired SWE-bench Verified (Feb 2026) after finding at least 59.4% of audited (hard... | OpenAI official blog (2026) | ✅ confirmed | See note | +| 7 | Median time in PR review up 441.5%; incidents-per-PR up 242.7%; bugs per developer up 54%;... | Faros AI (2026) | ⚠️ vendor-only | See note | +| 8 | Developers who delegate code generation to AI score 17% lower on comprehension tests, base... | (a) arXiv preprint; (b) Anthropic official research page (2026) | ❌ unverifiable | See note | +| 9 | 96% of developers don't fully trust AI-generated code is functionally correct, yet only 48... | Sonar (2026) | ⚠️ vendor-only | See note | +| 10 | JetBrains' 2025 survey found 77% of developers still manually correct AI output for projec... | JetBrains (2025) | ❌ unverifiable | See note | +| 11 | A standard MCP setup (few servers) can consume ~72% of a 200K-token context window before ... | (b) Qiyao Sun et al. (2025) | ⚠️ vendor-only | See note | +| 12 | LLM evaluators recognize and favor their own generations - self-preference bias correlates... | Advances in Neural Information Processing Systems 37 (NeurIPS 2024), Main Conference Track (Oral) (2024) | ✅ confirmed | See note | + +## Detailed Findings + +### 1. C1_METR_slowdown: ✅ confirmed + +**Claim:** Experienced open-source developers were 19% SLOWER with AI while believing they were ~20% faster (also forecast 24% speedup beforehand). + +**Supports:** M2 (assumption/uncertainty - miscalibration), P3/self-correction deficit; core 'trust but verify' motivation + +**Primary source:** *Measuring the Impact of Early-2025 AI on Experienced Open-Source Developer Productivity* — METR (Model Evaluation & Threat Research), arXiv preprint / METR blog, 2025. arXiv:2507.09089; https://metr.org/blog/2025-07-10-early-2025-ai-experienced-os-dev-study/ + +**What the primary source actually states:** 16 experienced developers, 246 tasks in mature repos (avg 22k+ stars, 1M+ lines); AI use INCREASED completion time by 19%; pre-task forecast was 24% speedup; post-task self-estimate was 20% speedup. + +**What the field report claimed:** 19% slowdown for experienced devs; matches primary source exactly. + +**Note:** Directly reachable on arXiv and METR's own site; numbers match field report exactly. Caveat directly stated by METR: small sample (16 devs), specific to mature/familiar open-source repos, and AI-averse developers increasingly decline to participate (self-selection risk noted by METR itself). + +### 2. C2_SO2025_trust: ✅ confirmed + +**Claim:** Trust in AI accuracy fell from 40% to 29% (or 46% actively distrust vs 33% trust per detailed breakdown); favorability 72%->60%; 66% say AI answers are 'almost right, but not quite'; 45% say debugging AI code is more time-consuming. + +**Supports:** M2 uncertainty/calibration; M6 inline verification; overall trust/verification-gap thesis + +**Primary source:** *2025 Stack Overflow Developer Survey (49,000+ respondents, 177 countries)* — Stack Overflow, Official survey site / Stack Overflow company blog + press release, 2025. https://survey.stackoverflow.co/2025 ; https://survey.stackoverflow.co/2025/ai ; https://stackoverflow.blog/2025/12/29/developers-remain-willing-but-reluctant-to-use-ai-the-2025-developer-survey-results-are-here/ + +**What the primary source actually states:** Trust in AI accuracy fell from 40% (prior years) to 29% (2025); positive favorability fell from 72% to 60%; 84% use or plan to use AI tools (up from 76%); 46% actively distrust AI accuracy vs 33% trust it, only 3% 'highly trust'; 66% cite 'almost right, but not quite' as the #1 frustration, which 'often leads to' the #2 frustration, debugging being more time-consuming (45%). + +**What the field report claimed:** Matches primary source essentially exactly on all four sub-figures. + +**Note:** Reached directly on Stack Overflow's own survey site and company blog. Self-reported survey; Stack Overflow's own methodology notes flag respondent self-selection bias (recruited via Stack Overflow's own channels, so more AI-engaged/skeptical developers may be over-represented). + +### 3. C3_Veracode_vuln: ⚠️ vendor-only + +**Claim:** 45% of AI-generated code samples introduce an OWASP Top-10 vulnerability; a widely-repeated '2.74x more vulnerabilities than human-written code' figure. + +**Supports:** M5 anti-over-engineering / code-quality thesis; security-verification gap + +**Primary source:** *2025 GenAI Code Security Report* — Veracode, Veracode company report/blog, 2025. https://www.veracode.com/resources/analyst-reports/2025-genai-code-security-report/ ; https://www.veracode.com/blog/genai-code-security-report/ + +**What the primary source actually states:** Veracode's own report/blog states only the 45% figure (code samples across 100+ LLMs, Java/JS/Python/C# introducing OWASP Top-10 flaws; Java worst at ~72%; XSS failure 86%). Veracode's own public materials located in this search do NOT state a '2.74x' multiplier anywhere. + +**What the field report claimed:** Field report attributes BOTH '45%' and '2.74x more vulnerabilities than human-written code' to Veracode, then separately says the 2.74x figure was 'independently corroborated by CodeRabbit's December 2025 analysis of 470 real-world PRs (2.74x more security vulnerabilities...)'. + +**Note:** The 45% figure is directly traceable to Veracode's own report (vendor self-reported, not independently reproduced). The '2.74x' figure could NOT be located in Veracode's own primary materials during this search - it appears only in secondary/derivative blog posts (e.g. softwareseni.com) that attribute it to Veracode, while the field report itself sources the same 2.74x number to a DIFFERENT study (CodeRabbit's PR analysis). This looks like a citation conflation between two separate vendor studies that happen to share a number. Treat the 2.74x figure as unverified/possibly misattributed pending direct access to Veracode's full PDF report. + +### 4. C4_GitClear_duplication: ⚠️ vendor-only + +**Claim:** ~8x rise (2024 vs prior years) in frequency of duplicated/copy-pasted code blocks; copy-paste overtaking refactored ('moved') code for the first time. + +**Supports:** M5 anti-over-engineering; long-term codebase impact-awareness (P5-related) + +**Primary source:** *AI Copilot Code Quality: 2025 Data Suggests 4x Growth in Code Clones / AI Copilot Code Quality: Evaluating 2024's Increased Defect Rate* — GitClear (Bill Harding et al.), GitClear company research report, 2025. https://www.gitclear.com/ai_assistant_code_quality_2025_research ; https://gitclear-public.s3.us-west-2.amazonaws.com/GitClear-AI-Copilot-Code-Quality-2025.pdf + +**What the primary source actually states:** GitClear's own materials are internally inconsistent on the multiplier. The report's own page TITLE reads 'AI Copilot Code Quality: 2025 Data Suggests 4x Growth in Code Clones' (gitclear.com), but the body text of the same report and GitClear's press-mentions page both state duplicated code blocks (5+ lines) 'rose eightfold' / 'increased eightfold' during 2024 (211M changed lines, 2020-2024, Google/Microsoft/Meta/enterprise repos). Secondary summaries split roughly evenly between citing '4x' and '8x'. Underlying non-disputed figures: copy-pasted lines rose from 8.3% (2020) to 12.3% (2024); moved/refactored lines fell from ~24-25% to <10%; 2024 was the first year copy-paste exceeded moved lines. The 4x-vs-8x gap could not be resolved from available pages - likely reflects two different metrics (duplicated-block frequency vs. some other clone measure) reported inconsistently across GitClear's own title/body/press materials. + +**What the field report claimed:** Field report states '8x rise in duplicated code blocks' - this matches GitClear's report BODY and press-mentions page, but GitClear's own page TITLE says '4x Growth in Code Clones,' an internal inconsistency the field report does not surface. + +**Note:** GitClear is a code-analytics vendor; findings are corroborated by many independent tech-press writeups summarizing the same underlying dataset, but no independent third party has re-run the analysis separately. Correlational, not causal. IMPORTANT ADDITIONAL CAVEAT: GitClear's own materials are internally inconsistent - the report's page title cites '4x' growth in code clones while the body and press page cite an '8x' rise in duplicated blocks. The paper should either cite the specific metric name (duplicated-block frequency, 8x per body text) rather than a bare multiplier, or note both figures and the discrepancy explicitly rather than asserting '8x' as settled. + +### 5. C5_DORA_amplifier: ✅ confirmed + +**Claim:** AI's primary role is as an 'amplifier' - magnifying high performers' strengths and low performers' dysfunctions; AI continues to increase delivery instability even as adoption becomes near-universal. + +**Supports:** Systemic framing for all six mechanisms (org context matters, not just model quality) + +**Primary source:** *State of AI-assisted Software Development 2025 (DORA Report)* — Google Cloud DORA team (Nathen Harvey et al.), in collaboration with research partners, Official DORA/Google Cloud report, 2025. https://dora.dev/dora-report-2025/ ; https://cloud.google.com/blog/products/ai-machine-learning/announcing-the-2025-dora-report + +**What the primary source actually states:** Nearly 5,000 professionals surveyed (June 13-July 21, 2025) plus 100+ hours interviews; 90% AI adoption (14pp increase from 2024); AI's primary role is 'that of an amplifier... magnifying the strengths of high-performing organisations and the dysfunctions of struggling ones'; in 2025 AI's relationship to delivery throughput reversed to positive vs 2024, but AI continues to increase delivery instability; ~30% report little/no trust in AI-generated code. + +**What the field report claimed:** Matches primary source directly ('AI is an amplifier'; high adoption; throughput/stability tension). + +**Note:** DORA is a Google-run but methodologically transparent, widely-cited industry research program (not a single vendor's self-promotional study); full methodology, sample size and survey window are published. Best-supported of the twelve claims alongside METR. + +### 6. C6_SWEbench_retirement: ✅ confirmed + +**Claim:** OpenAI retired SWE-bench Verified (Feb 2026) after finding at least 59.4% of audited (hard/failed) problems had flawed test cases and/or training-data contamination; large score gap vs SWE-bench Pro (e.g. one model ~80.9% Verified vs ~45.9% Pro). + +**Supports:** M6 inline verification / benchmark-trust thesis; supports 'AI output is probabilistic, don't blindly trust metrics' + +**Primary source:** *Why SWE-bench Verified no longer measures frontier coding capabilities* — OpenAI Frontier Evals team (Mia Glaese, Olivia Watkins et al.), OpenAI official blog, 2026. https://openai.com/index/why-we-no-longer-evaluate-swe-bench-verified/ + +**What the primary source actually states:** OpenAI audited 138 problems (27.6% subset of the 500-task set) that its o3 model could not reliably solve across 64 runs; found at least 59.4% of THOSE audited problems had flawed test cases/descriptions (35.5% narrow tests, 18.8% wide tests, 5.1% other); also found all tested frontier models could reproduce gold-patch solutions from training memory, indicating contamination. The specific 80.9%-Verified-vs-45.9%-Pro pairing (Claude Opus 4.5) was NOT found stated in OpenAI's own blog; it is reported by third-party benchmark aggregators (e.g. Scale AI SEAL leaderboard, BenchLM.ai, cited via codeant.ai) as of April 2026. + +**What the field report claimed:** Field report's phrasing ('59.4% of audited problems had flawed test cases') is accurate to primary source. The 80.9%/45.9% pairing is directionally correct (large real gap exists) but its precise sourcing is a third-party leaderboard snapshot, not OpenAI's own blog post. + +**Note:** The 59.4%-of-audited-problems figure is directly confirmed on OpenAI's own site - a strong, well-documented primary source. The specific 80.9/45.9 percentage pair is a real, traceable leaderboard snapshot (Scale AI SEAL/BenchLM) but should be cited as such, not as OpenAI's own number, and will drift as models are re-benchmarked. + +### 7. C7_Faros_PRreview: ⚠️ vendor-only + +**Claim:** Median time in PR review up 441.5%; incidents-per-PR up 242.7%; bugs per developer up 54%; 31.3% more PRs merged with no review at all. + +**Supports:** M6 inline verification; review-bottleneck / verification-layer thesis + +**Primary source:** *The AI Engineering Report 2026: The Acceleration Whiplash* — Faros AI, Faros AI company research report (telemetry analysis), 2026. https://pages.faros.ai/hubfs/AI_Engineering_Report_2026_The_Acceleration_Whiplash_Faros.pdf ; https://www.faros.ai/blog/ai-acceleration-whiplash-takeaways + +**What the primary source actually states:** Two years of telemetry from 22,000 developers / 4,000+ teams, comparing each org's lowest- vs highest-AI-adoption quarters: median time in PR review +441.5% (average time in review +199.6%, first-review wait +156.6%); incidents-to-PR ratio +242.7%; bugs per developer +54%; 31.3% more PRs merged with no review at all; code churn +861%. + +**What the field report claimed:** Matches primary source numbers exactly. + +**Note:** Faros AI is an engineering-intelligence vendor whose commercial product monitors exactly these metrics; the report itself and third-party coverage (ADTmag) note these are cross-sectional correlations across the vendor's own customer telemetry, not a controlled study, and 2025-vs-2026 report editions are independent cross-sections rather than a longitudinal panel. + +### 8. C8_Anthropic_comprehension: ❌ unverifiable + +**Claim:** Developers who delegate code generation to AI score 17% lower on comprehension tests, based on 'Anthropic's own research (~400,000 Claude Code sessions)'. + +**Supports:** M6 inline verification / skill-atrophy thesis (comprehension while writing, not only after) + +**Primary source:** *TWO DIFFERENT STUDIES ARE BEING CONFLATED: (a) 'How AI Impacts Skill Formation' by Judy Hanwen Shen & Alex Tamkin (arXiv, 2026) - the actual source of the '17% lower comprehension' figure; (b) 'How Claude Code is used in practice' (Anthropic, ~400,000-session analysis) - a real Anthropic study, but about planning/execution decision splits and task-success rates, NOT comprehension testing.* — (a) Judy Hanwen Shen, Alex Tamkin (Tamkin at Anthropic); (b) Anthropic, (a) arXiv preprint; (b) Anthropic official research page, 2026. (a) referenced via arxiv.org/pdf/2604.14228 citing 'Shen and Tamkin, 2026'; (b) https://www.anthropic.com/research/claude-code-expertise + +**What the primary source actually states:** (a) Shen & Tamkin: developers who used AI to learn a new async-programming library completed tasks but scored measurably worse on a post-task comprehension test ('17% lower' per a secondary citation in an arXiv survey paper - I could not independently pull Shen & Tamkin's own abstract/number in this search, only a citing paper's paraphrase). (b) The 400,000-session Anthropic study found users make ~70% of planning decisions and Claude makes ~80% of execution decisions; occupation-based success rates were similar across professions (~26-34%); it does NOT report a comprehension-score deficit. + +**What the field report claimed:** Field report merges these into one sentence ('Anthropic's own research (~400,000 Claude Code sessions) found... 17% lower on comprehension'), incorrectly attributing the comprehension finding to the session-count study. + +**Note:** This is a citation-conflation error carried over from the field report (or its own sources). The 400K-session study is real and directly confirmed, but does not contain a comprehension-deficit finding. The '17% lower comprehension' figure traces to a separate, distinct Shen & Tamkin paper that this search could not directly retrieve/confirm in primary form (only via a third paper's citation of it). RECOMMENDATION: if the white paper wants to use the comprehension-deficit claim, cite Shen & Tamkin (2026) directly and verify the 17% figure against their own abstract/paper before use; do not attribute it to the 400K-session study. + +### 9. C9_Sonar_verification_gap: ⚠️ vendor-only + +**Claim:** 96% of developers don't fully trust AI-generated code is functionally correct, yet only 48% always verify it before committing (a 48-point 'verification gap'/'verification debt'). + +**Supports:** M6 inline verification - the central named gap the mechanism targets + +**Primary source:** *State of Code Developer Survey report 2026* — Sonar, Sonar company press release / report PDF, 2026. https://www.sonarsource.com/company/press-releases/sonar-data-reveals-critical-verification-gap-in-ai-coding/ ; https://www.sonarsource.com/state-of-code-developer-survey-report.pdf + +**What the primary source actually states:** Survey of 1,100+ (some sources say 1,149) professional developers, January 2026: 96% do not fully trust AI-generated code is functionally correct; only 48% always check AI-assisted code before committing; AI accounts for 42% of committed code (projected 65% by 2027); 38% say reviewing AI code takes more effort than reviewing human code; the term 'verification debt' is attributed to AWS CTO Werner Vogels. + +**What the field report claimed:** Matches primary source exactly. + +**Note:** Sonar is a code-quality/verification tooling vendor with a direct commercial interest in this narrative; numbers are self-reported survey data, not independently replicated, though the survey size and methodology are transparently disclosed in the primary PDF. + +### 10. C10_JetBrains_manual_correction: ❌ unverifiable + +**Claim:** JetBrains' 2025 survey found 77% of developers still manually correct AI output for project conventions every session. + +**Supports:** M4 goal-anchoring / M5 anti-over-engineering (convention drift) + +**Primary source:** *The State of Developer Ecosystem 2025* — JetBrains, JetBrains official survey report, 2025. https://blog.jetbrains.com/research/2025/10/state-of-developer-ecosystem-2025/ ; https://devecosystem-2025.jetbrains.com/artificial-intelligence + +**What the primary source actually states:** JetBrains' own 2025 report (24,534 developers) confirms 85% regularly use AI tools and 62% rely on at least one AI coding assistant, but this search could not locate any statement of a '77% manually correct AI output for conventions every session' figure anywhere in JetBrains' own materials, blog posts, or press coverage of the 2025 or 2026 editions. + +**What the field report claimed:** 77% manually correct for conventions every session (attributed to JetBrains 2025). + +**Note:** Could not confirm this specific statistic in JetBrains' own primary materials despite multiple targeted searches of the official report, its AI-specific subpage, and secondary coverage. It may be a misremembered/misattributed figure, or drawn from the raw downloadable dataset (500+ questions) rather than the published highlights - the field report should either drop this figure or the paper authors should independently pull it from JetBrains' raw data release before use. + +### 11. C11_MCP_context_bloat: ⚠️ vendor-only + +**Claim:** A standard MCP setup (few servers) can consume ~72% of a 200K-token context window before work begins; tool-selection accuracy drops from ~43% to below ~14% as tool count scales ('context rot'). + +**Supports:** M1 complexity-aware routing / M3 task decomposition (context budget as a resource to manage) + +**Primary source:** *(a) 72%-context-window claim: no formal paper found, only recurring blog anecdotes (Scott Spence, Sam McLeod, apideck.com, agentpmt.com) describing an informal measurement ('three servers - GitHub, Playwright, IDE - consumed 143K of 200K tokens'). (b) 43%->14% tool-selection accuracy: RAG-MCP: Mitigating Prompt Bloat in LLM Tool Selection via Retrieval-Augmented Generation.* — (b) Qiyao Sun et al., (b) arXiv preprint, 2025. (b) arXiv:2505.03275 + +**What the primary source actually states:** (b) RAG-MCP's own 'MCP stress test' (needle-in-a-haystack-style, N candidate MCP schemas with 1 ground truth) found baseline tool-selection accuracy of 13.62% vs 43.13% for their retrieval-augmented method at scale - i.e. the '43% vs 14%' figures are RAG-MCP's OWN method-vs-baseline comparison on a synthetic stress test, not a general real-world degradation curve as tools accumulate. (a) The 72%/143K-token figure is not from any peer-reviewed or vendor-formal study located in this search; it recurs across multiple blogs as an informal, uncredited individual measurement (one specific developer's personal setup: GitHub + Playwright + IDE MCP servers). + +**What the field report claimed:** Field report states these as if they describe general degradation with tool count ('tool-selection accuracy drops from 43% to below 14% as tools accumulate') and cites a 72% context-window consumption figure as an established fact. + +**Note:** The 43.13%-vs-13.62% numbers ARE real and traceable to a genuine arXiv paper (RAG-MCP), but the field report's framing ('as tools accumulate') mischaracterizes what those specific numbers measure (a baseline vs their proposed retrieval method on one synthetic stress test, not a general accumulation curve). The 72%-window figure has no traceable primary/academic source - only recurring, uncredited blog claims. RECOMMENDATION: if used, cite RAG-MCP correctly as 'a stress test showing retrieval-based tool selection outperforms naive selection at scale' rather than a general context-rot statistic, and treat the 72% figure as illustrative anecdote, not a verified finding. + +### 12. C12_Panickssery_selfpreference: ✅ confirmed + +**Claim:** LLM evaluators recognize and favor their own generations - self-preference bias correlates with self-recognition ability - motivating why 'AI verifying AI' is structurally weak. + +**Supports:** M6 inline verification (why an LLM cannot be its own sole verifier); underpins the paper's argument for independent/external verification loops + +**Primary source:** *LLM Evaluators Recognize and Favor Their Own Generations* — Arjun Panickssery, Samuel R. Bowman, Shi Feng, Advances in Neural Information Processing Systems 37 (NeurIPS 2024), Main Conference Track (Oral), 2024. https://proceedings.neurips.cc/paper_files/paper/2024/hash/7f1f0218e45f5414c79c0679633e47bc-Abstract-Conference.html ; arXiv:2404.13076 + +**What the primary source actually states:** GPT-4 and Llama 2, used as evaluators, have 'non-trivial accuracy' at distinguishing their own outputs from other LLMs' and humans' outputs; a linear correlation is found between self-recognition capability and strength of self-preference bias (LLM evaluators score their own outputs higher while human annotators rate them as equal quality); fine-tuning to improve self-recognition further amplifies self-preference. + +**What the field report claimed:** Field report's characterization ('LLMs show self-preference/self-recognition bias when evaluating') matches the paper's core finding faithfully; no specific number is claimed by the field report beyond the qualitative finding. + +**Note:** Directly confirmed via the official NeurIPS 2024 proceedings page and the underlying arXiv preprint; a peer-reviewed, widely-cited paper (an NeurIPS 2024 Oral). This is the strongest-quality citation among all twelve (peer-reviewed venue, not industry survey/vendor report). + +## What This Means for the Paper + + +**Safe to lean on without hedging (peer-reviewed / official primary source, methodology transparent):** +- METR's 19%-slowdown RCT (C1) — the single best-controlled empirical finding in the set; cite with its own caveats (n=16, mature-repo setting). +- Panickssery et al. NeurIPS 2024 self-preference bias (C12) — the only genuinely peer-reviewed academic paper among the twelve; strongest citation for the M6 argument that an LLM cannot be its sole verifier. +- OpenAI's own retirement of SWE-bench Verified and the 59.4%-of-audited-problems figure (C6) — directly stated on OpenAI's blog. The specific 80.9%/45.9% score pairing, however, should be cited as a third-party leaderboard snapshot (Scale AI SEAL/BenchLM), not as OpenAI's own number, since it will drift release-to-release. +- DORA 2025 "AI is an amplifier" finding (C5) — large, transparent, non-vendor-captured methodology (Google Cloud + independent research partners), the most credible of the survey-based claims. +- Stack Overflow 2025 trust figures (C2) — official large-sample survey (49k+ respondents) with disclosed methodology; cite with the self-selection caveat Stack Overflow itself notes. + +**Usable but must be explicitly hedged as vendor/self-reported (real numbers, but commercially interested source, no independent replication):** +- Veracode 45% OWASP-vulnerability figure (C3) — the 45% number is real and vendor-confirmed; the widely-repeated "2.74x" figure could NOT be traced to Veracode's own report in this search and appears to be conflated with a separate CodeRabbit study. Use 45% only, or independently pull Veracode's full PDF before citing 2.74x. +- GitClear's code-duplication rise (C4) — real, vendor-produced, correlational only, AND internally inconsistent: GitClear's own report page TITLE says "4x Growth in Code Clones" while the report BODY and press-mentions page say duplicated blocks "rose eightfold." The paper should cite the specific underlying metric (e.g. "copy-pasted lines rose from 8.3% to 12.3%, moved/refactored lines fell below 10%") rather than asserting a bare "8x" multiplier as settled, or explicitly note both figures. +- Faros AI's "Acceleration Whiplash" PR-review/incident figures (C7) — real, vendor telemetry, cross-sectional not longitudinal. +- Sonar's 96%-don't-trust / 48%-always-verify verification gap (C9) — real, vendor survey, directly supports the M6 (inline verification) mechanism by name. + +**Must be corrected or dropped:** +- The "Anthropic 400,000-session study found 17% lower comprehension" claim (C8) conflates two different Anthropic-adjacent studies. The 400K-session study is real but does not measure comprehension; the comprehension-deficit figure belongs to a separate Shen & Tamkin paper that could not be independently pulled in this search. **Do not cite the 400K-session study for the comprehension-deficit number** — cite Shen & Tamkin (2026) directly once verified, or drop the specific 17% figure. +- The JetBrains "77% manually correct AI output for conventions" claim (C10) could not be found anywhere in JetBrains' own materials after multiple targeted searches. **Recommend dropping this figure** unless it can be independently located in JetBrains' raw downloadable dataset. +- The MCP context-bloat claims (C11): the 43%→14% tool-selection-accuracy figures ARE real (RAG-MCP, arXiv) but describe a synthetic stress test comparing a proposed method against a naive baseline — not a general "accuracy degrades as tools accumulate" curve as the field report implies. The 72%-of-context-window figure has no traceable formal source, only recurring blog anecdotes about one individual's MCP setup. **Reframe or drop.** + +**Overall calibration for the white paper:** of the twelve claims, two rest on genuinely independent, peer-reviewed or transparently-run research (METR, Panickssery/NeurIPS); one is a strong official primary-source admission (OpenAI's SWE-bench retirement) plus one large transparent multi-stakeholder industry study (DORA); one is a large, disclosed-methodology public survey (Stack Overflow). Five rest on vendor self-reported telemetry/surveys that are real but commercially motivated and not independently replicated (Veracode, GitClear, Faros, Sonar, and the RAG-MCP-adjacent MCP-bloat anecdotes) — and of these, GitClear's own report is additionally internally inconsistent about its headline multiplier (4x vs 8x) and should be cited via its underlying percentages, not a bare multiplier. Two claims as stated in the field report are not supported by verifiable primary sources and should be corrected or removed (the Anthropic-comprehension conflation, and the JetBrains 77% figure). diff --git a/docs/cognitive-substrate/impact_oracle_src.zip b/docs/cognitive-substrate/impact_oracle_src.zip new file mode 100644 index 0000000..3354953 Binary files /dev/null and b/docs/cognitive-substrate/impact_oracle_src.zip differ diff --git a/docs/cognitive-substrate/router_gate_src.zip b/docs/cognitive-substrate/router_gate_src.zip new file mode 100644 index 0000000..b64ca05 Binary files /dev/null and b/docs/cognitive-substrate/router_gate_src.zip differ diff --git a/global/settings.template.json b/global/settings.template.json index 95ecfa7..2f923fa 100644 --- a/global/settings.template.json +++ b/global/settings.template.json @@ -76,6 +76,16 @@ ] }, "hooks": { + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "bash ~/.forge/guards/cortex.sh preflight" + } + ] + } + ], "SessionStart": [ { "matcher": "startup|resume|clear", diff --git a/global/tools/cognitive-substrate/SKILL.md b/global/tools/cognitive-substrate/SKILL.md new file mode 100644 index 0000000..0fcfb96 --- /dev/null +++ b/global/tools/cognitive-substrate/SKILL.md @@ -0,0 +1,31 @@ +--- +name: cognitive-substrate +description: >- + Use before ambiguous, expensive, multi-file, or mutating coding work to run + Forge's cognitive substrate: assumption gate, model routing, impact + prediction, scope decomposition, memory/lesson lookup, minimality check, and + verification planning. Trigger when a user asks to edit/refactor/fix/design + production code, integrate features, choose model effort, inspect blast + radius, or avoid agent assumptions. +--- + +# Cognitive Substrate + +Before acting on non-trivial or mutating code tasks, run: + +```bash +forge substrate "" --json +``` + +Use the result this way: + +1. If `okToProceed` is false, ask the returned `assumption.questions` before editing. +2. Use `route.model` as the cheapest capable tier recommendation; do not escalate without a verifier failure. +3. Read `impact.impactedFiles` before editing a named symbol/file. +4. Use `scope.clusters` to split independent work into separate sessions. +5. Treat `memory.advisory` as context, not law; tests and human corrections override it. +6. Run the `verification.checklist` before claiming completion. + +For details, read `references/capability-map.md` only when you need to explain how the paper's faculties map to Forge commands. + +The full paper bundle is in `docs/cognitive-substrate/` when the repository is available: PDF, HTML, evidence map, ecosystem map, and original prototype packages. diff --git a/global/tools/cognitive-substrate/references/capability-map.md b/global/tools/cognitive-substrate/references/capability-map.md new file mode 100644 index 0000000..71c6511 --- /dev/null +++ b/global/tools/cognitive-substrate/references/capability-map.md @@ -0,0 +1,17 @@ +# Cognitive substrate capability map + +| Paper capability | Forge surface | Guarantee | +| --- | --- | --- | +| Memory | `forge recall`, `forge cortex` | File-backed facts/lessons are persisted and auditable. | +| Learning | `forge cortex` | External outcomes update lessons; model weights do not change. | +| Imagination | `forge impact`, `forge substrate` | Static graph simulates possible blast radius. | +| Self-correction | `forge verify`, doom-loop guard | Tests/builds beat model claims. | +| Impact-awareness | `forge atlas`, `forge impact` | Known symbols/files and likely dependents are surfaced. | +| M1 routing | `forge route` | Transparent model-tier recommendation. | +| M2 assumption gate | `forge preflight`, `forge substrate` | Under-specified tasks return questions. | +| M3 decomposition | `forge scope` | Import clusters show independent vs coupled files. | +| M4 goal anchoring | `forge substrate` | One pre-action summary anchors goal, risk, and verification. | +| M5 anti-over-engineering | `forge substrate`, `lean-guard` | Broad/underspecified work gets minimality warnings. | +| M6 inline verification | `forge verify` | External checks are required before done. | + +Limits: static graph edges are conservative; memory relevance and model routing are advisory; non-hook tools cannot be forcibly blocked. diff --git a/install.sh b/install.sh index 0c5b059..d38352d 100644 --- a/install.sh +++ b/install.sh @@ -63,6 +63,8 @@ install_forge() { "statusLine": { "type": "command", "command": "bash $FORGE_HOME/statusline.sh" }, "hooks": { + "UserPromptSubmit": [ { "hooks": [ { "type": "command", + "command": "bash $FORGE_HOME/guards/cortex.sh preflight" } ] } ], "PreToolUse": [ { "matcher": "Edit|Write|MultiEdit|Bash", "hooks": [ { "type": "command", "command": "bash $FORGE_HOME/guards/protect-paths.sh" } ] } ], "PostToolUse": [ { "matcher": "Edit|Write|MultiEdit", diff --git a/package.json b/package.json index 8c2ceb2..28134a8 100644 --- a/package.json +++ b/package.json @@ -30,9 +30,13 @@ "source", "global", "templates", + "docs/cognitive-substrate", "plugin", "hooks", ".claude-plugin", + ".codex-plugin", + "skills", + ".mcp.json", "bin", "brand.json", "install.sh", diff --git a/research/python-prototypes/README.md b/research/python-prototypes/README.md new file mode 100644 index 0000000..2628d23 --- /dev/null +++ b/research/python-prototypes/README.md @@ -0,0 +1,11 @@ +# Python prototypes + +These are the original evidence prototypes from the cognitive-substrate paper. They are preserved for auditability and result reproduction only. + +Production Forge runtime is Node zero-dependency: + +- `forge substrate ""` +- `forge impact ` +- MCP tools: `substrate_check`, `predict_impact`, `assumption_gate` + +The Python packages remain useful as research baselines for the original router/gate and impact-oracle experiments. diff --git a/research/python-prototypes/impact_oracle/README.md b/research/python-prototypes/impact_oracle/README.md new file mode 100644 index 0000000..1c374a5 --- /dev/null +++ b/research/python-prototypes/impact_oracle/README.md @@ -0,0 +1,124 @@ +# Impact Oracle — Codebase World-Model + Blast-Radius Predictor + +A Python package that builds a persistent structural model of a codebase and +predicts the blast radius of proposed symbol changes. Designed as a concrete +demonstrator of the "know what exists and what it affects" faculty — the kind +of external stateful architecture that a frozen LLM structurally lacks. + +## Why + +A transformer at inference time is a stateless function `y = f_θ(x)`. It +cannot maintain a structural model of a codebase across turns, and it cannot +simulate the consequences of a proposed edit before making it. The Impact +Oracle is an external module that fills this gap: it parses the codebase once, +persists the resulting dependency graph, and — given a proposed change — +traverses reverse dependencies to predict what will break. + +## Architecture + +``` +┌───────────────────────────────────────────────────────┐ +│ World Model (world_model.py) │ +│ ┌─────────────┐ ┌──────────────┐ ┌───────────┐ │ +│ │ AST Parser │──▶│ Dependency │──▶│ Persistent │ │ +│ │ (parser.py) │ │ Graph │ │ Cache │ │ +│ └─────────────┘ │ (NetworkX) │ │ (JSON+pkl) │ │ +│ └──────────────┘ └───────────┘ │ +└───────────────┬───────────────────────────────────────┘ + │ + ▼ +┌───────────────────────────────────────────────────────┐ +│ Impact Oracle (oracle.py) │ +│ - Reverse-dependency BFS with confidence decay │ +│ - Per-edge-kind weights (calls > imports > refs) │ +│ - Ranked impact set with explanation paths │ +│ - Baselines: grep + edited-file-only │ +└───────────────────────────────────────────────────────┘ +``` + +### Graph structure + +- **Nodes** = symbols: modules, classes, functions/methods, module-level names +- **Edges** = structural dependencies: + - `imports` — module-level import statements + - `calls` — function/method calls + - `inherits` — class inheritance + - `references` — attribute access, name usage + - `contains` — parent scope → child definition +- Each edge carries a **confidence** score (1.0 = certain, <1.0 = heuristic) + +### Incremental update + +Files are content-hashed (SHA-256). On subsequent builds, only changed files +are re-parsed. The graph is persisted as both JSON (portable) and pickle +(fast reload). + +## Quick start + +```bash +# Install dependencies +pip install networkx matplotlib pytest + +# Run the demo +python demo.py + +# Run the evaluation (mutation testing) +python evaluate.py + +# CLI usage +python -m impact_oracle.cli build demo_package/ +python -m impact_oracle.cli impact demo_package/ utils.validation.validate_positive +python -m impact_oracle.cli summary demo_package/ +``` + +## Evaluation + +The evaluation uses **mutation testing** as ground truth: + +1. For each target symbol, apply a breaking mutation (change behavior/signature) +2. Run pytest — the set of modules whose tests fail = true behavioral blast radius +3. Compare oracle prediction vs. mutation ground truth + +### Results (5 mutations, averaged) + +| Method | Precision | Recall | F1 | +|-------------------|-----------|--------|-------| +| **Graph Oracle** | 0.633 | **1.000** | **0.753** | +| Grep baseline | 0.733 | 0.943 | 0.787 | +| Edited-file only | **1.000** | 0.529 | 0.650 | + +The oracle achieves **perfect recall** (never misses a truly affected module) +with an F1 of 0.83 at the optimal threshold (t=0.4). + +## File structure + +``` +impact_oracle/ # The package + __init__.py + parser.py # AST-based Python source parser + world_model.py # Persistent structural graph (world model) + oracle.py # Blast-radius prediction engine + cli.py # Command-line interface + +demo_package/ # Example multi-module codebase (8 files) + __init__.py + models.py # Product, PremiumProduct classes + inventory.py # Stock management + orders.py # Order processing + reports.py # Reporting aggregation + utils/ + validation.py # Shared validation functions + formatting.py # Display formatting + sub/ + pricing.py # Pricing analytics + +tests/ + test_demo_package.py # 36 tests exercising the demo package + +demo.py # End-to-end demonstration script +evaluate.py # Mutation-based evaluation +``` + +## License + +Research prototype — no license restrictions. diff --git a/research/python-prototypes/impact_oracle/demo.py b/research/python-prototypes/impact_oracle/demo.py new file mode 100644 index 0000000..c7f3599 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python +"""End-to-end demo of the Impact Oracle. + +Builds a world-model graph of the demo_package codebase, then predicts +the blast radius of editing a symbol, printing the predicted impact set +with confidence scores and dependency paths. + +Usage: + python demo.py +""" + +import os +import sys +import json + +# Ensure the workspace root is on PYTHONPATH +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from impact_oracle.world_model import WorldModel +from impact_oracle.oracle import ImpactOracle + + +def main(): + workspace = os.path.dirname(os.path.abspath(__file__)) + demo_root = os.path.join(workspace, "demo_package") + + # --------------------------------------------------------------- + # 1. Build the world model + # --------------------------------------------------------------- + print("=" * 60) + print(" IMPACT ORACLE — Codebase World-Model Demo") + print("=" * 60) + + wm = WorldModel(demo_root, cache_dir="/tmp/impact_oracle_demo_cache") + stats = wm.build(incremental=True) + + print(f"\n[1] World Model Built") + print(f" Files parsed: {stats['files_parsed']}/{stats['files_total']}") + print(f" Graph size: {stats['nodes']} nodes, {stats['edges']} edges") + + summary = wm.summary() + print(f" Node kinds: {summary['node_kinds']}") + print(f" Edge kinds: {summary['edge_kinds']}") + + # --------------------------------------------------------------- + # 2. Show what the model knows + # --------------------------------------------------------------- + print(f"\n[2] Symbols in the Codebase") + for kind in ["module", "class", "function"]: + nodes = wm.all_nodes(kind=kind) + print(f" {kind}s ({len(nodes)}):") + for n in sorted(nodes, key=lambda x: x["qualified_name"])[:8]: + sig = n.get("signature", "") + print(f" {n['qualified_name']}{sig}") + if len(nodes) > 8: + print(f" ... and {len(nodes) - 8} more") + + # --------------------------------------------------------------- + # 3. Predict blast radius + # --------------------------------------------------------------- + oracle = ImpactOracle(wm) + + targets = [ + "utils.validation.validate_positive", + "models.Product.discounted_price", + "utils.formatting.format_currency", + ] + + for symbol in targets: + print(f"\n{'=' * 60}") + print(f"[3] Impact Prediction: what breaks if we change '{symbol}'?") + print(f"{'=' * 60}") + + report = oracle.predict_impact(symbol, threshold=0.1) + + print(f" Impacted symbols: {len(report.impacted)}") + print(f" Impacted files: {report.impacted_files}") + print(f" Threshold used: {report.threshold_used}") + + print(f"\n Ranked impact set:") + for n in report.impacted: + path_str = " → ".join(n.path) + edge_str = " → ".join(n.edge_kinds) + print(f" [{n.confidence:.3f}] {n.qualified_name}") + print(f" kind={n.kind}, hop={n.hop_distance}") + print(f" path: {path_str}") + print(f" via: {edge_str}") + + # Baselines comparison + grep_files = ImpactOracle.grep_baseline(symbol, demo_root) + edited_files = ImpactOracle.edited_file_baseline(symbol, wm.graph) + + print(f"\n Baselines comparison:") + print(f" Oracle files: {sorted(report.impacted_files)}") + print(f" Grep baseline: {sorted(grep_files)}") + print(f" Edited-file only: {sorted(edited_files)}") + + # --------------------------------------------------------------- + # 4. Incremental update demo + # --------------------------------------------------------------- + print(f"\n{'=' * 60}") + print(f"[4] Incremental Update (re-build without changes)") + stats2 = wm.build(incremental=True) + print(f" Files parsed: {stats2['files_parsed']}/{stats2['files_total']} " + f"(skipped {stats2['files_skipped']} unchanged)") + + print(f"\nDone. The world model is cached at /tmp/impact_oracle_demo_cache/") + + +if __name__ == "__main__": + main() diff --git a/research/python-prototypes/impact_oracle/demo_package/__init__.py b/research/python-prototypes/impact_oracle/demo_package/__init__.py new file mode 100644 index 0000000..e2487a5 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/__init__.py @@ -0,0 +1,7 @@ +"""Demo Package: a small multi-module codebase for testing the Impact Oracle. + +Contains a realistic dependency structure with cross-module imports, +inheritance, call chains, and shared utilities. +""" + +__version__ = "1.0.0" diff --git a/research/python-prototypes/impact_oracle/demo_package/inventory.py b/research/python-prototypes/impact_oracle/demo_package/inventory.py new file mode 100644 index 0000000..4ca73fd --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/inventory.py @@ -0,0 +1,49 @@ +"""Inventory management with stock tracking.""" + +from demo_package.models import Product +from demo_package.utils.validation import validate_quantity, validate_positive + + +class Inventory: + """Manages stock levels for products.""" + + def __init__(self): + self._stock: dict[str, tuple[Product, int]] = {} + + def add_product(self, product: Product, quantity: int): + """Add stock for a product.""" + quantity = validate_quantity(quantity) + if product.name in self._stock: + existing_product, existing_qty = self._stock[product.name] + self._stock[product.name] = (existing_product, existing_qty + quantity) + else: + self._stock[product.name] = (product, quantity) + + def remove_stock(self, product_name: str, quantity: int) -> bool: + """Remove stock. Returns True if sufficient stock was available.""" + quantity = validate_quantity(quantity) + if product_name not in self._stock: + return False + product, current = self._stock[product_name] + if current < quantity: + return False + self._stock[product_name] = (product, current - quantity) + return True + + def get_product(self, name: str) -> Product | None: + """Look up a product by name.""" + entry = self._stock.get(name) + return entry[0] if entry else None + + def stock_level(self, name: str) -> int: + """Return current stock for a product.""" + entry = self._stock.get(name) + return entry[1] if entry else 0 + + def total_value(self) -> float: + """Total value of all inventory at current prices.""" + return sum(p.price * qty for p, qty in self._stock.values()) + + def low_stock_items(self, threshold: int = 5) -> list[str]: + """Return names of products below the stock threshold.""" + return [name for name, (_, qty) in self._stock.items() if qty < threshold] diff --git a/research/python-prototypes/impact_oracle/demo_package/models.py b/research/python-prototypes/impact_oracle/demo_package/models.py new file mode 100644 index 0000000..dcb1592 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/models.py @@ -0,0 +1,35 @@ +"""Core domain models.""" + +from demo_package.utils.validation import validate_positive, validate_name + + +class Product: + """A product in the catalog.""" + + def __init__(self, name: str, price: float, category: str = "general"): + self.name = validate_name(name) + self.price = validate_positive(price) + self.category = category + + def discounted_price(self, discount_pct: float) -> float: + """Return price after applying a percentage discount.""" + discount_pct = validate_positive(discount_pct) + return self.price * (1 - discount_pct / 100) + + def __repr__(self): + return f"Product({self.name!r}, ${self.price:.2f})" + + +class PremiumProduct(Product): + """A premium product with loyalty points.""" + + POINTS_PER_DOLLAR = 10 + + def __init__(self, name: str, price: float, category: str = "premium"): + super().__init__(name, price, category) + self.loyalty_points = int(self.price * self.POINTS_PER_DOLLAR) + + def discounted_price(self, discount_pct: float) -> float: + """Premium products cap discount at 30%.""" + capped = min(discount_pct, 30.0) + return super().discounted_price(capped) diff --git a/research/python-prototypes/impact_oracle/demo_package/orders.py b/research/python-prototypes/impact_oracle/demo_package/orders.py new file mode 100644 index 0000000..6ecfd3f --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/orders.py @@ -0,0 +1,68 @@ +"""Order processing — depends on models, inventory, and formatting.""" + +from demo_package.models import Product, PremiumProduct +from demo_package.inventory import Inventory +from demo_package.utils.formatting import format_currency, format_receipt_line +from demo_package.utils.validation import validate_quantity + + +class OrderLine: + """A single line in an order.""" + + def __init__(self, product: Product, quantity: int, discount_pct: float = 0): + self.product = product + self.quantity = validate_quantity(quantity) + self.discount_pct = discount_pct + + def line_total(self) -> float: + """Compute total for this line after discount.""" + unit = self.product.discounted_price(self.discount_pct) + return unit * self.quantity + + def format(self) -> str: + """Format for receipt display.""" + unit = self.product.discounted_price(self.discount_pct) + return format_receipt_line(self.product.name, self.quantity, unit) + + +class Order: + """An order that pulls from inventory.""" + + def __init__(self, inventory: Inventory): + self.inventory = inventory + self.lines: list[OrderLine] = [] + + def add_item(self, product_name: str, quantity: int, discount_pct: float = 0) -> bool: + """Add an item. Returns False if insufficient stock.""" + product = self.inventory.get_product(product_name) + if product is None: + return False + if not self.inventory.remove_stock(product_name, quantity): + return False + self.lines.append(OrderLine(product, quantity, discount_pct)) + return True + + def total(self) -> float: + """Order total after discounts.""" + return sum(line.line_total() for line in self.lines) + + def loyalty_points(self) -> int: + """Total loyalty points earned (premium products only).""" + pts = 0 + for line in self.lines: + if isinstance(line.product, PremiumProduct): + pts += line.product.loyalty_points * line.quantity + return pts + + def receipt(self) -> str: + """Generate a formatted receipt string.""" + parts = ["=" * 60, " ORDER RECEIPT", "=" * 60] + for line in self.lines: + parts.append(line.format()) + parts.append("-" * 60) + parts.append(f" TOTAL: {format_currency(self.total()):>47s}") + pts = self.loyalty_points() + if pts: + parts.append(f" Loyalty points earned: {pts}") + parts.append("=" * 60) + return "\n".join(parts) diff --git a/research/python-prototypes/impact_oracle/demo_package/reports.py b/research/python-prototypes/impact_oracle/demo_package/reports.py new file mode 100644 index 0000000..12b3b74 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/reports.py @@ -0,0 +1,31 @@ +"""Reporting module — aggregates data from inventory and orders.""" + +from demo_package.inventory import Inventory +from demo_package.orders import Order +from demo_package.utils.formatting import format_currency + + +def inventory_report(inventory: Inventory) -> str: + """Generate a summary report of current inventory state.""" + low = inventory.low_stock_items() + total = inventory.total_value() + lines = [ + "=== Inventory Report ===", + f"Total inventory value: {format_currency(total)}", + f"Low-stock items ({len(low)}): {', '.join(low) if low else 'none'}", + ] + return "\n".join(lines) + + +def order_summary(orders: list[Order]) -> dict: + """Aggregate statistics across multiple orders.""" + total_revenue = sum(o.total() for o in orders) + total_items = sum(sum(l.quantity for l in o.lines) for o in orders) + total_points = sum(o.loyalty_points() for o in orders) + return { + "order_count": len(orders), + "total_revenue": total_revenue, + "total_items": total_items, + "total_loyalty_points": total_points, + "average_order_value": total_revenue / len(orders) if orders else 0, + } diff --git a/research/python-prototypes/impact_oracle/demo_package/sub/__init__.py b/research/python-prototypes/impact_oracle/demo_package/sub/__init__.py new file mode 100644 index 0000000..7674c02 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/sub/__init__.py @@ -0,0 +1 @@ +"""Sub-package for analytics.""" diff --git a/research/python-prototypes/impact_oracle/demo_package/sub/pricing.py b/research/python-prototypes/impact_oracle/demo_package/sub/pricing.py new file mode 100644 index 0000000..48a46ea --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/sub/pricing.py @@ -0,0 +1,37 @@ +"""Pricing analytics — deeper analysis of product pricing strategies.""" + +from demo_package.models import Product, PremiumProduct +from demo_package.utils.validation import validate_positive + + +def margin_analysis(product: Product, cost: float) -> dict: + """Compute margin metrics for a product.""" + cost = validate_positive(cost) + margin = product.price - cost + margin_pct = (margin / product.price) * 100 if product.price > 0 else 0 + return { + "product": product.name, + "price": product.price, + "cost": cost, + "margin": margin, + "margin_pct": round(margin_pct, 2), + "is_premium": isinstance(product, PremiumProduct), + } + + +def bulk_discount_curve(product: Product, quantities: list[int]) -> list[dict]: + """Compute effective prices at various quantity tiers. + + Uses a simple rule: 5% additional discount per 10-unit tier. + """ + results = [] + for qty in quantities: + tier_discount = min(50.0, (qty // 10) * 5.0) + effective = product.discounted_price(tier_discount) + results.append({ + "quantity": qty, + "tier_discount_pct": tier_discount, + "unit_price": round(effective, 2), + "total": round(effective * qty, 2), + }) + return results diff --git a/research/python-prototypes/impact_oracle/demo_package/utils/__init__.py b/research/python-prototypes/impact_oracle/demo_package/utils/__init__.py new file mode 100644 index 0000000..3b8174d --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/utils/__init__.py @@ -0,0 +1 @@ +"""Utility sub-package.""" diff --git a/research/python-prototypes/impact_oracle/demo_package/utils/formatting.py b/research/python-prototypes/impact_oracle/demo_package/utils/formatting.py new file mode 100644 index 0000000..dc54380 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/utils/formatting.py @@ -0,0 +1,15 @@ +"""Formatting utilities for display output.""" + +from demo_package.utils.validation import validate_positive + + +def format_currency(amount: float) -> str: + """Format a float as a USD currency string.""" + amount = validate_positive(amount) + return f"${amount:,.2f}" + + +def format_receipt_line(name: str, qty: int, unit_price: float) -> str: + """Format a single line item for a receipt.""" + total = qty * unit_price + return f" {name:<30s} {qty:>3d} x {format_currency(unit_price):>10s} = {format_currency(total):>10s}" diff --git a/research/python-prototypes/impact_oracle/demo_package/utils/validation.py b/research/python-prototypes/impact_oracle/demo_package/utils/validation.py new file mode 100644 index 0000000..ab7d8e6 --- /dev/null +++ b/research/python-prototypes/impact_oracle/demo_package/utils/validation.py @@ -0,0 +1,22 @@ +"""Validation utility functions used across the package.""" + + +def validate_positive(value: float) -> float: + """Ensure a numeric value is positive.""" + if value < 0: + raise ValueError(f"Value must be non-negative, got {value}") + return float(value) + + +def validate_name(name: str) -> str: + """Validate and normalize a name string.""" + if not name or not name.strip(): + raise ValueError("Name cannot be empty") + return name.strip() + + +def validate_quantity(qty: int) -> int: + """Ensure quantity is a positive integer.""" + if not isinstance(qty, int) or qty < 1: + raise ValueError(f"Quantity must be a positive integer, got {qty}") + return qty diff --git a/research/python-prototypes/impact_oracle/evaluate.py b/research/python-prototypes/impact_oracle/evaluate.py new file mode 100644 index 0000000..1fe69cb --- /dev/null +++ b/research/python-prototypes/impact_oracle/evaluate.py @@ -0,0 +1,305 @@ +"""Mutation-based evaluation of the Impact Oracle. + +For each target symbol, we: +1. Back up the original file +2. Introduce a breaking mutation (change behavior/signature) +3. Run pytest and record which tests fail +4. Map failing tests to their source files to get the true blast radius +5. Restore the original +6. Compare oracle prediction vs mutation ground truth vs baselines +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +from pathlib import Path + +# Ensure workspace root on PYTHONPATH +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from impact_oracle.world_model import WorldModel +from impact_oracle.oracle import ImpactOracle + + +# --- Mutation definitions --- +# Each mutation: (symbol, file_rel, old_text, new_text, description) + +MUTATIONS = [ + { + "symbol": "utils.validation.validate_positive", + "file": "demo_package/utils/validation.py", + "old": ' if value < 0:\n raise ValueError(f"Value must be non-negative, got {value}")\n return float(value)', + "new": ' return -abs(float(value)) # MUTATED: always returns negative', + "description": "validate_positive now always returns negative — breaks all consumers expecting validated positive values", + }, + { + "symbol": "models.Product.discounted_price", + "file": "demo_package/models.py", + "old": " return self.price * (1 - discount_pct / 100)", + "new": " return self.price * (1 + discount_pct / 100) # MUTATED: adds instead of subtracts", + "description": "discounted_price adds discount instead of subtracting — affects orders, pricing analytics", + }, + { + "symbol": "utils.formatting.format_currency", + "file": "demo_package/utils/formatting.py", + "old": ' return f"${amount:,.2f}"', + "new": ' return f"EUR {amount:,.2f}" # MUTATED: changed currency symbol', + "description": "format_currency returns EUR prefix — breaks receipt formatting and report assertions", + }, + { + "symbol": "inventory.Inventory.remove_stock", + "file": "demo_package/inventory.py", + "old": " if current < quantity:\n return False\n self._stock[product_name] = (product, current - quantity)\n return True", + "new": " self._stock[product_name] = (product, current + quantity) # MUTATED: adds instead of removes\n return True", + "description": "remove_stock adds stock instead — affects orders that depend on stock depletion", + }, + { + "symbol": "models.PremiumProduct.discounted_price", + "file": "demo_package/models.py", + "old": " capped = min(discount_pct, 30.0)\n return super().discounted_price(capped)", + "new": " return self.price # MUTATED: no discount at all", + "description": "PremiumProduct.discounted_price ignores discounts entirely", + }, +] + + +# --- Test-to-file mapping --- +# Maps test class names to the modules they exercise + +# Maps test classes to the PRIMARY module they test (the module whose +# behavior is directly exercised, not every transitive import). +# This is the right level: a failing TestInventory test means the +# inventory module's behavior changed, regardless of what it imports. +TEST_FILE_MAP = { + "TestValidation": ["demo_package/utils/validation.py"], + "TestFormatting": ["demo_package/utils/formatting.py"], + "TestProduct": ["demo_package/models.py"], + "TestPremiumProduct": ["demo_package/models.py"], + "TestInventory": ["demo_package/inventory.py"], + "TestOrderLine": ["demo_package/orders.py"], + "TestOrder": ["demo_package/orders.py"], + "TestReports": ["demo_package/reports.py"], + "TestPricing": ["demo_package/sub/pricing.py"], +} + + +def run_mutation_test(mutation: dict, workspace: str) -> dict: + """Apply a mutation, run tests, record failures, restore.""" + filepath = os.path.join(workspace, mutation["file"]) + original = Path(filepath).read_text() + + # Apply mutation + mutated = original.replace(mutation["old"], mutation["new"]) + if mutated == original: + return {"error": f"Mutation pattern not found in {mutation['file']}"} + Path(filepath).write_text(mutated) + + # Clear ALL __pycache__ under workspace to ensure fresh imports + subprocess.run( + ["find", workspace, "-type", "d", "-name", "__pycache__", "-exec", "rm", "-rf", "{}", "+"], + capture_output=True + ) + + # Run pytest with cache disabled in a SEPARATE subprocess + env = os.environ.copy() + env["PYTHONPATH"] = workspace + result = subprocess.run( + [sys.executable, "-m", "pytest", + os.path.join(workspace, "tests/test_demo_package.py"), + "-v", "--tb=line", "-p", "no:cacheprovider"], + capture_output=True, text=True, env=env, cwd=workspace + ) + + # Restore original + Path(filepath).write_text(original) + + # Parse failed tests from pytest summary section + # The summary section starts with "FAILED tests/..." lines (no percentage) + failed_tests = [] + failed_modules = set() + seen_ids = set() + for line in result.stdout.split("\n"): + line = line.strip() + # Match both formats: + # "FAILED tests/test_demo_package.py::TestClass::test_method - ..." + # "tests/test_demo_package.py::TestClass::test_method FAILED [ xx%]" + if "FAILED" not in line: + continue + # Extract test ID (the path::class::method part) + if line.startswith("FAILED "): + # Summary line format + test_id = line.split("FAILED ")[1].split(" - ")[0].strip() + elif " FAILED" in line: + # Verbose inline format + test_id = line.split(" FAILED")[0].strip() + else: + continue + + if test_id in seen_ids: + continue + seen_ids.add(test_id) + failed_tests.append(test_id) + + for cls_name, files in TEST_FILE_MAP.items(): + if cls_name in test_id: + for f in files: + rel = f.replace("demo_package/", "") + failed_modules.add(rel) + break + + return { + "symbol": mutation["symbol"], + "description": mutation["description"], + "failed_tests": failed_tests, + "failed_test_count": len(failed_tests), + "ground_truth_files": sorted(failed_modules), + "exit_code": result.returncode, + } + + +def files_from_oracle(report, threshold=0.1): + """Extract file set from an oracle impact report.""" + return set(report.impacted_files) + + +def compute_precision_recall(predicted: set, actual: set) -> dict: + """Compute precision, recall, F1 between predicted and actual sets.""" + if not predicted and not actual: + return {"precision": 1.0, "recall": 1.0, "f1": 1.0, "predicted": 0, "actual": 0} + if not predicted: + return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "predicted": 0, "actual": len(actual)} + if not actual: + return {"precision": 0.0, "recall": 0.0, "f1": 0.0, "predicted": len(predicted), "actual": 0} + + tp = len(predicted & actual) + precision = tp / len(predicted) if predicted else 0 + recall = tp / len(actual) if actual else 0 + f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0 + + return { + "precision": round(precision, 4), + "recall": round(recall, 4), + "f1": round(f1, 4), + "tp": tp, + "fp": len(predicted - actual), + "fn": len(actual - predicted), + "predicted": len(predicted), + "actual": len(actual), + } + + +def main(): + workspace = os.path.dirname(os.path.abspath(__file__)) + + # Build world model + print("Building world model...") + wm = WorldModel( + os.path.join(workspace, "demo_package"), + cache_dir="/tmp/impact_oracle_eval_cache" + ) + stats = wm.build(incremental=False) + print(f"World model: {stats['nodes']} nodes, {stats['edges']} edges") + + oracle = ImpactOracle(wm) + results = [] + + for mut in MUTATIONS: + print(f"\n{'='*60}") + print(f"Mutating: {mut['symbol']}") + print(f" {mut['description']}") + + # 1. Get mutation ground truth + gt = run_mutation_test(mut, workspace) + if "error" in gt: + print(f" ERROR: {gt['error']}") + continue + print(f" Failed tests: {gt['failed_test_count']}") + print(f" Ground truth files: {gt['ground_truth_files']}") + + # 2. Oracle prediction + report = oracle.predict_impact(mut["symbol"], threshold=0.1) + oracle_files = files_from_oracle(report) + print(f" Oracle predicted files: {sorted(oracle_files)}") + + # 3. Grep baseline + grep_files = ImpactOracle.grep_baseline(mut["symbol"], os.path.join(workspace, "demo_package")) + print(f" Grep baseline files: {sorted(grep_files)}") + + # 4. Edited-file-only baseline + edited_files = ImpactOracle.edited_file_baseline(mut["symbol"], wm.graph) + print(f" Edited-file-only baseline: {sorted(edited_files)}") + + # 5. Compare + gt_set = set(gt["ground_truth_files"]) + + oracle_pr = compute_precision_recall(oracle_files, gt_set) + grep_pr = compute_precision_recall(grep_files, gt_set) + edited_pr = compute_precision_recall(edited_files, gt_set) + + print(f"\n Oracle: P={oracle_pr['precision']:.3f} R={oracle_pr['recall']:.3f} F1={oracle_pr['f1']:.3f}") + print(f" Grep: P={grep_pr['precision']:.3f} R={grep_pr['recall']:.3f} F1={grep_pr['f1']:.3f}") + print(f" Edited-only: P={edited_pr['precision']:.3f} R={edited_pr['recall']:.3f} F1={edited_pr['f1']:.3f}") + + results.append({ + "symbol": mut["symbol"], + "description": mut["description"], + "ground_truth": gt, + "oracle": { + "predicted_files": sorted(oracle_files), + "impacted_symbols": len(report.impacted), + **oracle_pr, + }, + "grep_baseline": { + "predicted_files": sorted(grep_files), + **grep_pr, + }, + "edited_file_baseline": { + "predicted_files": sorted(edited_files), + **edited_pr, + }, + }) + + # Compute averages + avg = lambda key, method: sum(r[method][key] for r in results) / len(results) if results else 0 + summary = { + "oracle_avg": { + "precision": round(avg("precision", "oracle"), 4), + "recall": round(avg("recall", "oracle"), 4), + "f1": round(avg("f1", "oracle"), 4), + }, + "grep_avg": { + "precision": round(avg("precision", "grep_baseline"), 4), + "recall": round(avg("recall", "grep_baseline"), 4), + "f1": round(avg("f1", "grep_baseline"), 4), + }, + "edited_file_avg": { + "precision": round(avg("precision", "edited_file_baseline"), 4), + "recall": round(avg("recall", "edited_file_baseline"), 4), + "f1": round(avg("f1", "edited_file_baseline"), 4), + }, + "graph_size": { + "nodes": stats["nodes"], + "edges": stats["edges"], + "files": stats["files_total"], + }, + } + + print(f"\n{'='*60}") + print("AVERAGES:") + print(f" Oracle: P={summary['oracle_avg']['precision']:.3f} R={summary['oracle_avg']['recall']:.3f} F1={summary['oracle_avg']['f1']:.3f}") + print(f" Grep: P={summary['grep_avg']['precision']:.3f} R={summary['grep_avg']['recall']:.3f} F1={summary['grep_avg']['f1']:.3f}") + print(f" Edited-only: P={summary['edited_file_avg']['precision']:.3f} R={summary['edited_file_avg']['recall']:.3f} F1={summary['edited_file_avg']['f1']:.3f}") + + output = {"per_symbol": results, "summary": summary} + with open("eval_results.json", "w") as f: + json.dump(output, f, indent=2) + print(f"\nResults saved to eval_results.json") + return output + + +if __name__ == "__main__": + main() diff --git a/research/python-prototypes/impact_oracle/impact_oracle/__init__.py b/research/python-prototypes/impact_oracle/impact_oracle/__init__.py new file mode 100644 index 0000000..3266284 --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/__init__.py @@ -0,0 +1,12 @@ +"""Impact Oracle: A Codebase World-Model + Impact Prediction Engine. + +Provides persistent structural analysis of Python codebases and +blast-radius prediction for proposed symbol changes. +""" + +__version__ = "0.2.0" + +from impact_oracle.world_model import WorldModel +from impact_oracle.oracle import ImpactOracle + +__all__ = ["WorldModel", "ImpactOracle"] diff --git a/research/python-prototypes/impact_oracle/impact_oracle/cli.py b/research/python-prototypes/impact_oracle/impact_oracle/cli.py new file mode 100644 index 0000000..6d9c685 --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/cli.py @@ -0,0 +1,83 @@ +"""Command-line interface for the Impact Oracle.""" + +from __future__ import annotations + +import argparse +import json +import sys + +from impact_oracle.world_model import WorldModel +from impact_oracle.oracle import ImpactOracle + + +def main(argv: list[str] | None = None): + parser = argparse.ArgumentParser( + prog="impact-oracle", + description="Codebase world-model builder and impact predictor.", + ) + sub = parser.add_subparsers(dest="command") + + # -- build -- + build_p = sub.add_parser("build", help="Parse codebase and build/update the world-model graph.") + build_p.add_argument("root", help="Codebase root directory.") + build_p.add_argument("--no-incremental", action="store_true", help="Full rebuild (ignore cache).") + build_p.add_argument("--cache-dir", help="Override cache directory.") + + # -- impact -- + imp_p = sub.add_parser("impact", help="Predict blast radius of changing a symbol.") + imp_p.add_argument("root", help="Codebase root directory.") + imp_p.add_argument("symbol", help="Qualified name of the symbol to change.") + imp_p.add_argument("--threshold", type=float, default=0.1, help="Confidence threshold (default 0.1).") + imp_p.add_argument("--cache-dir", help="Override cache directory.") + imp_p.add_argument("--json", action="store_true", help="Output as JSON.") + + # -- summary -- + sum_p = sub.add_parser("summary", help="Print world-model summary.") + sum_p.add_argument("root", help="Codebase root directory.") + sum_p.add_argument("--cache-dir", help="Override cache directory.") + + args = parser.parse_args(argv) + if not args.command: + parser.print_help() + return + + if args.command == "build": + wm = WorldModel(args.root, cache_dir=args.cache_dir) + stats = wm.build(incremental=not args.no_incremental) + print(f"Parsed {stats['files_parsed']}/{stats['files_total']} files " + f"(skipped {stats['files_skipped']} unchanged)") + print(f"World model: {stats['nodes']} nodes, {stats['edges']} edges") + + elif args.command == "impact": + wm = WorldModel(args.root, cache_dir=args.cache_dir) + wm.build(incremental=True) + oracle = ImpactOracle(wm) + report = oracle.predict_impact(args.symbol, threshold=args.threshold) + + if getattr(args, "json", False): + print(json.dumps(report.to_dict(), indent=2)) + else: + print(f"\n=== Impact Report for: {report.changed_symbol} ===") + print(f"Threshold: {report.threshold_used}") + print(f"Impacted symbols: {len(report.impacted)}") + print(f"Impacted files: {len(report.impacted_files)}") + for node in report.impacted: + edge_str = " → ".join(node.edge_kinds) + print(f" [{node.confidence:.3f}] {node.qualified_name} " + f"({node.kind}, hop={node.hop_distance}) via {edge_str}") + if report.impacted_files: + print(f"\nFiles affected:") + for f in report.impacted_files: + print(f" {f}") + + elif args.command == "summary": + wm = WorldModel(args.root, cache_dir=args.cache_dir) + wm.build(incremental=True) + s = wm.summary() + print(f"World model: {s['nodes']} nodes, {s['edges']} edges, {s['files']} files") + print(f"Node kinds: {s['node_kinds']}") + print(f"Edge kinds: {s['edge_kinds']}") + + +if __name__ == "__main__": + main() diff --git a/research/python-prototypes/impact_oracle/impact_oracle/mcp_server.py b/research/python-prototypes/impact_oracle/impact_oracle/mcp_server.py new file mode 100644 index 0000000..ffb4499 --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/mcp_server.py @@ -0,0 +1,150 @@ +"""Minimal MCP-compatible stdio server for Impact Oracle.""" +from __future__ import annotations + +import json +import sys +from typing import Any, Optional + +from impact_oracle.oracle import ImpactOracle +from impact_oracle.world_model import WorldModel + +SERVER_NAME = "impact-oracle" +SERVER_VERSION = "0.2.0" + +TOOLS = [ + { + "name": "build_world_model", + "description": "Parse a Python codebase and build/update its structural dependency graph.", + "inputSchema": { + "type": "object", + "properties": { + "root": {"type": "string", "description": "Codebase root directory"}, + "cache_dir": {"type": "string", "description": "Optional cache directory"}, + "incremental": {"type": "boolean", "description": "Reuse unchanged cached files", "default": True}, + }, + "required": ["root"], + "additionalProperties": False, + }, + }, + { + "name": "predict_impact", + "description": "Predict blast radius for changing a qualified Python symbol.", + "inputSchema": { + "type": "object", + "properties": { + "root": {"type": "string", "description": "Codebase root directory"}, + "symbol": {"type": "string", "description": "Qualified symbol name"}, + "threshold": {"type": "number", "minimum": 0, "maximum": 1, "default": 0.1}, + "cache_dir": {"type": "string", "description": "Optional cache directory"}, + }, + "required": ["root", "symbol"], + "additionalProperties": False, + }, + }, + { + "name": "world_model_summary", + "description": "Return graph summary for a Python codebase world model.", + "inputSchema": { + "type": "object", + "properties": { + "root": {"type": "string", "description": "Codebase root directory"}, + "cache_dir": {"type": "string", "description": "Optional cache directory"}, + }, + "required": ["root"], + "additionalProperties": False, + }, + }, +] + + +def _content(payload: Any) -> dict: + return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, sort_keys=True)}]} + + +def _error(request_id: Any, code: int, message: str) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + + +def _result(request_id: Any, result: Any) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def _world_model(root: str, cache_dir: Optional[str] = None) -> WorldModel: + if not isinstance(root, str) or not root.strip(): + raise ValueError("argument 'root' must be a non-empty string") + return WorldModel(root, cache_dir=cache_dir) + + +def _call_tool(name: str, arguments: dict) -> dict: + if not isinstance(arguments, dict): + raise ValueError("arguments must be an object") + root = arguments.get("root") + cache_dir = arguments.get("cache_dir") + wm = _world_model(root, cache_dir=cache_dir if isinstance(cache_dir, str) else None) + + if name == "build_world_model": + stats = wm.build(incremental=bool(arguments.get("incremental", True))) + return _content({"stats": stats}) + if name == "predict_impact": + symbol = arguments.get("symbol") + if not isinstance(symbol, str) or not symbol.strip(): + raise ValueError("argument 'symbol' must be a non-empty string") + threshold = float(arguments.get("threshold", 0.1)) + if not 0 <= threshold <= 1: + raise ValueError("threshold must be between 0 and 1") + wm.build(incremental=True) + report = ImpactOracle(wm).predict_impact(symbol, threshold=threshold) + return _content({"impact": report.to_dict()}) + if name == "world_model_summary": + wm.build(incremental=True) + return _content({"summary": wm.summary()}) + raise ValueError(f"unknown tool: {name}") + + +def handle(message: dict) -> Optional[dict]: + method = message.get("method") + request_id = message.get("id") + if method == "notifications/initialized": + return None + if method == "initialize": + return _result(request_id, { + "protocolVersion": message.get("params", {}).get("protocolVersion", "2024-11-05"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION}, + }) + if method == "ping": + return _result(request_id, {}) + if method == "tools/list": + return _result(request_id, {"tools": TOOLS}) + if method == "tools/call": + params = message.get("params", {}) + if not isinstance(params, dict): + return _error(request_id, -32602, "params must be an object") + try: + return _result(request_id, _call_tool(str(params.get("name", "")), params.get("arguments", {}) or {})) + except Exception as exc: + return _error(request_id, -32000, str(exc)) + if request_id is None: + return None + return _error(request_id, -32601, f"method not found: {method}") + + +def main() -> int: + for line in sys.stdin: + if not line.strip(): + continue + try: + message = json.loads(line) + if not isinstance(message, dict): + raise ValueError("message must be a JSON object") + response = handle(message) + except Exception as exc: + response = _error(None, -32700, str(exc)) + if response is not None: + sys.stdout.write(json.dumps(response, separators=(",", ":")) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/research/python-prototypes/impact_oracle/impact_oracle/oracle.py b/research/python-prototypes/impact_oracle/impact_oracle/oracle.py new file mode 100644 index 0000000..69700df --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/oracle.py @@ -0,0 +1,217 @@ +"""Impact Oracle: blast-radius prediction for symbol changes. + +Given a world-model graph and a proposed change to symbol X, the oracle +traverses *reverse* dependency edges (who imports/calls/inherits/references X) +to predict the set of symbols and files that would be affected. Each +impacted node carries a decaying confidence score and the dependency path +that connects it to X. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +import networkx as nx + +from impact_oracle.world_model import WorldModel + + +# Edge-kind weights: how strongly each dependency kind propagates impact. +EDGE_WEIGHTS: dict[str, float] = { + "calls": 0.95, + "imports": 0.90, + "inherits": 0.92, + "references": 0.70, + "contains": 0.60, # child → parent propagation is weaker +} + +# Default per-hop decay factor +DEFAULT_DECAY = 0.85 + + +@dataclass +class ImpactedNode: + """A single node in the predicted blast radius.""" + qualified_name: str + kind: str # node kind: module/class/function/name + file: str + confidence: float # cumulative confidence [0, 1] + hop_distance: int # number of edges from the changed symbol + path: list[str] # sequence of qualified_names from changed → this + edge_kinds: list[str] # kind of each edge on the path + + def to_dict(self) -> dict[str, Any]: + return self.__dict__.copy() + + +@dataclass +class ImpactReport: + """Full result of an impact prediction.""" + changed_symbol: str + impacted: list[ImpactedNode] + impacted_files: list[str] + threshold_used: float + total_graph_nodes: int + total_graph_edges: int + + def to_dict(self) -> dict[str, Any]: + return { + "changed_symbol": self.changed_symbol, + "impacted": [n.to_dict() for n in self.impacted], + "impacted_files": self.impacted_files, + "threshold_used": self.threshold_used, + "total_graph_nodes": self.total_graph_nodes, + "total_graph_edges": self.total_graph_edges, + } + + def above_threshold(self, threshold: float | None = None) -> list[ImpactedNode]: + """Return impacted nodes above a confidence threshold.""" + t = threshold if threshold is not None else self.threshold_used + return [n for n in self.impacted if n.confidence >= t] + + +class ImpactOracle: + """Predicts the blast radius of a proposed symbol change. + + Works by traversing *reverse* dependency edges in the world-model + graph: starting from the changed symbol, it walks predecessors + (nodes that depend on the changed symbol) with decaying confidence. + """ + + def __init__( + self, + world_model: WorldModel, + decay: float = DEFAULT_DECAY, + edge_weights: dict[str, float] | None = None, + max_hops: int = 10, + ): + self.wm = world_model + self.decay = decay + self.edge_weights = edge_weights or EDGE_WEIGHTS + self.max_hops = max_hops + + def predict_impact( + self, + symbol: str, + threshold: float = 0.1, + ) -> ImpactReport: + """Predict the blast radius of changing *symbol*. + + Parameters + ---------- + symbol : str + Qualified name of the symbol being changed. + threshold : float + Minimum confidence to include a node in the result. + + Returns + ------- + ImpactReport + """ + graph = self.wm.graph + if symbol not in graph: + return ImpactReport( + changed_symbol=symbol, impacted=[], impacted_files=[], + threshold_used=threshold, + total_graph_nodes=graph.number_of_nodes(), + total_graph_edges=graph.number_of_edges(), + ) + + # BFS over reverse edges with confidence decay + visited: dict[str, ImpactedNode] = {} + # Queue: (node, confidence, hop, path, edge_kinds) + queue: list[tuple[str, float, int, list[str], list[str]]] = [ + (symbol, 1.0, 0, [symbol], []) + ] + + while queue: + current, conf, hop, path, ekinds = queue.pop(0) + if hop > self.max_hops: + continue + + # Get reverse dependents: predecessors in the graph + # (edges go source->target where source depends on target) + for pred in graph.predecessors(current): + if pred == symbol: + continue # skip self-loops + + edge_data = graph.edges[pred, current] + ek = edge_data.get("kind", "references") + ew = self.edge_weights.get(ek, 0.5) + edge_conf = edge_data.get("confidence", 1.0) + + new_conf = conf * ew * edge_conf * self.decay + if new_conf < threshold: + continue + + new_path = path + [pred] + new_ekinds = ekinds + [ek] + + # Keep the highest-confidence path to each node + if pred in visited: + if visited[pred].confidence >= new_conf: + continue + + node_data = graph.nodes.get(pred, {}) + impact = ImpactedNode( + qualified_name=pred, + kind=node_data.get("kind", "unknown"), + file=node_data.get("file", ""), + confidence=round(new_conf, 4), + hop_distance=hop + 1, + path=new_path, + edge_kinds=new_ekinds, + ) + visited[pred] = impact + queue.append((pred, new_conf, hop + 1, new_path, new_ekinds)) + + impacted = sorted(visited.values(), key=lambda n: -n.confidence) + files = sorted({n.file for n in impacted if n.file}) + + return ImpactReport( + changed_symbol=symbol, + impacted=impacted, + impacted_files=files, + threshold_used=threshold, + total_graph_nodes=graph.number_of_nodes(), + total_graph_edges=graph.number_of_edges(), + ) + + @staticmethod + def grep_baseline(symbol_name: str, root: str) -> set[str]: + """Baseline 1: grep for the bare symbol name across all .py files. + + Returns the set of files (relative paths) containing the name. + Over-broad on common names (e.g. 'get', 'run'). + """ + import os + from pathlib import Path + + short_name = symbol_name.split(".")[-1] + hits: set[str] = set() + root_path = Path(root) + for dirpath, dirnames, filenames in os.walk(root_path): + dirnames[:] = [d for d in dirnames if not d.startswith(".") and d != "__pycache__"] + for fn in filenames: + if not fn.endswith(".py"): + continue + fpath = os.path.join(dirpath, fn) + try: + text = Path(fpath).read_text(encoding="utf-8", errors="replace") + except OSError: + continue + if short_name in text: + hits.add(str(Path(fpath).relative_to(root_path))) + return hits + + @staticmethod + def edited_file_baseline(symbol: str, graph: nx.DiGraph) -> set[str]: + """Baseline 2: only the file containing the edited symbol. + + Under-broad — misses all cross-file dependents. + """ + if symbol in graph: + f = graph.nodes[symbol].get("file", "") + return {f} if f else set() + return set() diff --git a/research/python-prototypes/impact_oracle/impact_oracle/parser.py b/research/python-prototypes/impact_oracle/impact_oracle/parser.py new file mode 100644 index 0000000..d59209b --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/parser.py @@ -0,0 +1,350 @@ +"""AST-based Python source parser. + +Extracts structural symbols (modules, classes, functions, names) and +dependency edges (imports, calls, inherits, references, contains) from +Python source files. Designed for incremental re-parsing: each file is +content-hashed so unchanged files are skipped on subsequent runs. +""" + +from __future__ import annotations + +import ast +import hashlib +import os +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + + +# --------------------------------------------------------------------------- +# Data containers +# --------------------------------------------------------------------------- + +@dataclass +class SymbolNode: + """A symbol extracted from source: module, class, function, or name.""" + qualified_name: str # e.g. 'pkg.mod.Class.method' + kind: str # 'module' | 'class' | 'function' | 'name' + file: str # relative path inside the codebase root + lineno: int = 0 + end_lineno: int | None = None + signature: str = "" # for functions/methods + docstring: str = "" + parent: str = "" # qualified_name of enclosing scope + + def to_dict(self) -> dict[str, Any]: + return {k: v for k, v in self.__dict__.items() if v or k == "lineno"} + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "SymbolNode": + return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__}) + + +@dataclass +class DependencyEdge: + """A directed dependency from *source* to *target*.""" + source: str # qualified_name of the depending symbol + target: str # qualified_name of the depended-upon symbol + kind: str # 'imports' | 'calls' | 'inherits' | 'references' | 'contains' + confidence: float = 1.0 # 1.0 = certain; <1 = heuristic inference + lineno: int = 0 + + def to_dict(self) -> dict[str, Any]: + return self.__dict__.copy() + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "DependencyEdge": + return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__}) + + +@dataclass +class ParseResult: + """All symbols and edges extracted from one file.""" + file: str + content_hash: str + nodes: list[SymbolNode] = field(default_factory=list) + edges: list[DependencyEdge] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# AST visitor +# --------------------------------------------------------------------------- + +class _SymbolVisitor(ast.NodeVisitor): + """Walk an AST, accumulating SymbolNodes and DependencyEdges.""" + + def __init__(self, module_qname: str, filepath: str): + self.module_qname = module_qname + self.filepath = filepath + self.nodes: list[SymbolNode] = [] + self.edges: list[DependencyEdge] = [] + # Stack of (qualified_name, kind) for scope tracking + self._scope_stack: list[tuple[str, str]] = [(module_qname, "module")] + # Track import aliases: alias -> qualified_name + self._import_map: dict[str, str] = {} + + # -- helpers -- + + @property + def _current_scope(self) -> str: + return self._scope_stack[-1][0] + + def _qname(self, name: str) -> str: + return f"{self._current_scope}.{name}" + + @staticmethod + def _signature_of(node: ast.FunctionDef | ast.AsyncFunctionDef) -> str: + """Build a human-readable signature string.""" + args = node.args + parts: list[str] = [] + # positional + all_args = args.args + defaults_offset = len(all_args) - len(args.defaults) + for i, a in enumerate(all_args): + ann = "" + if a.annotation: + try: + ann = ": " + ast.unparse(a.annotation) + except Exception: + pass + default = "" + di = i - defaults_offset + if di >= 0: + try: + default = "=" + ast.unparse(args.defaults[di]) + except Exception: + pass + parts.append(f"{a.arg}{ann}{default}") + if args.vararg: + parts.append(f"*{args.vararg.arg}") + for i, a in enumerate(args.kwonlyargs): + ann = "" + if a.annotation: + try: + ann = ": " + ast.unparse(a.annotation) + except Exception: + pass + default = "" + if args.kw_defaults[i]: + try: + default = "=" + ast.unparse(args.kw_defaults[i]) + except Exception: + pass + parts.append(f"{a.arg}{ann}{default}") + if args.kwarg: + parts.append(f"**{args.kwarg.arg}") + + ret = "" + if node.returns: + try: + ret = " -> " + ast.unparse(node.returns) + except Exception: + pass + return f"({', '.join(parts)}){ret}" + + @staticmethod + def _docstring_of(node: ast.AST) -> str: + """Extract the leading docstring, if any.""" + if (isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Module)) + and node.body + and isinstance(node.body[0], ast.Expr) + and isinstance(node.body[0].value, (ast.Constant,))): + v = node.body[0].value.value + if isinstance(v, str): + first_line = v.strip().split("\n")[0] + return first_line[:200] + return "" + + # -- visitors -- + + def visit_ClassDef(self, node: ast.ClassDef): + qn = self._qname(node.name) + self.nodes.append(SymbolNode( + qualified_name=qn, kind="class", file=self.filepath, + lineno=node.lineno, end_lineno=node.end_lineno, + docstring=self._docstring_of(node), parent=self._current_scope, + )) + self.edges.append(DependencyEdge( + source=self._current_scope, target=qn, kind="contains", lineno=node.lineno, + )) + # inheritance edges + for base in node.bases: + base_name = self._resolve_name(base) + if base_name: + self.edges.append(DependencyEdge( + source=qn, target=base_name, kind="inherits", + lineno=node.lineno, confidence=0.9, + )) + self._scope_stack.append((qn, "class")) + self.generic_visit(node) + self._scope_stack.pop() + + def visit_FunctionDef(self, node: ast.FunctionDef): + self._handle_funcdef(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef): + self._handle_funcdef(node) + + def _handle_funcdef(self, node): + qn = self._qname(node.name) + self.nodes.append(SymbolNode( + qualified_name=qn, kind="function", file=self.filepath, + lineno=node.lineno, end_lineno=node.end_lineno, + signature=self._signature_of(node), + docstring=self._docstring_of(node), parent=self._current_scope, + )) + self.edges.append(DependencyEdge( + source=self._current_scope, target=qn, kind="contains", lineno=node.lineno, + )) + self._scope_stack.append((qn, "function")) + self.generic_visit(node) + self._scope_stack.pop() + + def visit_Import(self, node: ast.Import): + for alias in node.names: + local = alias.asname or alias.name + self._import_map[local] = alias.name + self.edges.append(DependencyEdge( + source=self._current_scope, target=alias.name, kind="imports", + lineno=node.lineno, + )) + + def visit_ImportFrom(self, node: ast.ImportFrom): + module = node.module or "" + for alias in (node.names or []): + target_qname = f"{module}.{alias.name}" if module else alias.name + local = alias.asname or alias.name + self._import_map[local] = target_qname + self.edges.append(DependencyEdge( + source=self._current_scope, target=target_qname, kind="imports", + lineno=node.lineno, + )) + + def visit_Call(self, node: ast.Call): + callee = self._resolve_name(node.func) + if callee: + self.edges.append(DependencyEdge( + source=self._current_scope, target=callee, kind="calls", + lineno=getattr(node, "lineno", 0), + confidence=0.85 if "." in callee else 0.95, + )) + self.generic_visit(node) + + def visit_Attribute(self, node: ast.Attribute): + """Capture attribute references like obj.attr as 'references' edges.""" + full = self._resolve_name(node) + if full and full != node.attr: + # Only record if we resolved something meaningful + # Skip if we're inside a Call (that's handled by visit_Call) + self.edges.append(DependencyEdge( + source=self._current_scope, target=full, kind="references", + lineno=getattr(node, "lineno", 0), confidence=0.7, + )) + self.generic_visit(node) + + def visit_Name(self, node: ast.Name): + """Capture bare-name references to imported symbols.""" + if node.id in self._import_map: + self.edges.append(DependencyEdge( + source=self._current_scope, target=self._import_map[node.id], + kind="references", lineno=node.lineno, confidence=0.8, + )) + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign): + """Record module-level name assignments.""" + if self._scope_stack[-1][1] == "module": + for target in node.targets: + if isinstance(target, ast.Name): + qn = self._qname(target.id) + self.nodes.append(SymbolNode( + qualified_name=qn, kind="name", file=self.filepath, + lineno=node.lineno, parent=self._current_scope, + )) + self.edges.append(DependencyEdge( + source=self._current_scope, target=qn, kind="contains", + lineno=node.lineno, + )) + self.generic_visit(node) + + # -- name resolution -- + + def _resolve_name(self, node: ast.AST) -> str | None: + """Best-effort resolution of an AST expression to a qualified name.""" + if isinstance(node, ast.Name): + return self._import_map.get(node.id, self._qname(node.id)) + if isinstance(node, ast.Attribute): + prefix = self._resolve_name(node.value) + if prefix: + return f"{prefix}.{node.attr}" + return None + + +# --------------------------------------------------------------------------- +# File-level parser +# --------------------------------------------------------------------------- + +def content_hash(source: str) -> str: + """SHA-256 of the source text, used for incremental caching.""" + return hashlib.sha256(source.encode("utf-8")).hexdigest()[:16] + + +def parse_file(filepath: str, root: str) -> ParseResult: + """Parse a single Python file and return extracted symbols + edges. + + Parameters + ---------- + filepath : str + Absolute or relative path to the .py file. + root : str + The codebase root directory. Used to derive the module's + qualified name from its path. + """ + path = Path(filepath) + root_path = Path(root) + try: + source = path.read_text(encoding="utf-8", errors="replace") + except (OSError, UnicodeDecodeError) as exc: + return ParseResult(file=str(path.relative_to(root_path)), content_hash="", nodes=[], edges=[]) + + chash = content_hash(source) + rel = path.relative_to(root_path) + # Derive module qualified name from path + parts = list(rel.with_suffix("").parts) + if parts[-1] == "__init__": + parts = parts[:-1] + module_qname = ".".join(parts) if parts else rel.stem + + relpath = str(rel) + + try: + tree = ast.parse(source, filename=str(filepath)) + except SyntaxError: + return ParseResult(file=relpath, content_hash=chash, nodes=[], edges=[]) + + # Module-level node + mod_node = SymbolNode( + qualified_name=module_qname, kind="module", file=relpath, + lineno=1, docstring=_SymbolVisitor._docstring_of(tree), + ) + + visitor = _SymbolVisitor(module_qname, relpath) + visitor.visit(tree) + + nodes = [mod_node] + visitor.nodes + edges = visitor.edges + return ParseResult(file=relpath, content_hash=chash, nodes=nodes, edges=edges) + + +def discover_python_files(root: str) -> list[str]: + """Find all .py files under *root*, excluding hidden/venv dirs.""" + result: list[str] = [] + root_path = Path(root) + exclude_prefixes = {".", "__pycache__", "venv", ".venv", "node_modules", ".git"} + for dirpath, dirnames, filenames in os.walk(root_path): + # Prune excluded directories + dirnames[:] = [d for d in dirnames if d not in exclude_prefixes and not d.startswith(".")] + for fn in filenames: + if fn.endswith(".py"): + result.append(os.path.join(dirpath, fn)) + return sorted(result) diff --git a/research/python-prototypes/impact_oracle/impact_oracle/serialization.py b/research/python-prototypes/impact_oracle/impact_oracle/serialization.py new file mode 100644 index 0000000..62db71c --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/serialization.py @@ -0,0 +1,15 @@ +"""JSON-safe serializers for impact_oracle objects.""" +from __future__ import annotations + +from dataclasses import asdict, is_dataclass +from typing import Any + + +def to_jsonable(value: Any) -> Any: + if is_dataclass(value): + return to_jsonable(asdict(value)) + if isinstance(value, dict): + return {str(k): to_jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple, set)): + return [to_jsonable(v) for v in value] + return value diff --git a/research/python-prototypes/impact_oracle/impact_oracle/world_model.py b/research/python-prototypes/impact_oracle/impact_oracle/world_model.py new file mode 100644 index 0000000..936d37e --- /dev/null +++ b/research/python-prototypes/impact_oracle/impact_oracle/world_model.py @@ -0,0 +1,323 @@ +"""Persistent structural world-model of a Python codebase. + +Builds and maintains a directed graph whose nodes are symbols and whose +edges are structural dependencies. The graph is persisted to disk as +JSON (node-link format) plus a NetworkX pickle. On subsequent runs only +files whose content hash has changed are re-parsed (incremental update). +""" + +from __future__ import annotations + +import json +import os +import pickle +from pathlib import Path +from typing import Any + +import networkx as nx + +from impact_oracle.parser import ( + DependencyEdge, + ParseResult, + SymbolNode, + content_hash, + discover_python_files, + parse_file, +) + + +class WorldModel: + """A persistent, incrementally-updated structural graph of a codebase. + + Nodes carry SymbolNode metadata; edges carry DependencyEdge metadata. + The graph is directed: an edge (A -> B) means 'A depends on B'. + """ + + def __init__(self, root: str, cache_dir: str | None = None): + """ + Parameters + ---------- + root : str + Path to the codebase root directory. + cache_dir : str, optional + Directory to persist the graph. Defaults to ``/.impact_oracle_cache``. + """ + self.root = os.path.abspath(root) + self.cache_dir = cache_dir or os.path.join(self.root, ".impact_oracle_cache") + self.graph: nx.DiGraph = nx.DiGraph() + # content-hash cache: filepath (relative) -> hash + self._file_hashes: dict[str, str] = {} + # Track which files contributed which nodes + self._file_nodes: dict[str, list[str]] = {} # filepath -> [qualified_names] + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def build(self, incremental: bool = True) -> dict[str, Any]: + """Parse the codebase and (re)build the graph. + + Parameters + ---------- + incremental : bool + If True (default), only re-parse files whose content hash + differs from the cached version. + + Returns + ------- + dict + Stats: files_total, files_parsed, files_skipped, nodes, edges. + """ + if incremental: + self._load_cache() + + files = discover_python_files(self.root) + parsed = 0 + skipped = 0 + + for fpath in files: + rel = str(Path(fpath).relative_to(self.root)) + # Read source to check hash + try: + source = Path(fpath).read_text(encoding="utf-8", errors="replace") + except OSError: + skipped += 1 + continue + chash = content_hash(source) + + if incremental and self._file_hashes.get(rel) == chash: + skipped += 1 + continue + + # Remove old nodes/edges from this file + self._remove_file_contributions(rel) + + result = parse_file(fpath, self.root) + self._apply_parse_result(result) + self._file_hashes[rel] = chash + parsed += 1 + + # Remove stale files that no longer exist + current_rels = {str(Path(f).relative_to(self.root)) for f in files} + stale = set(self._file_hashes.keys()) - current_rels + for rel in stale: + self._remove_file_contributions(rel) + del self._file_hashes[rel] + + # Resolve cross-module edges + self._resolve_cross_module_edges() + + self._save_cache() + + return { + "files_total": len(files), + "files_parsed": parsed, + "files_skipped": skipped, + "nodes": self.graph.number_of_nodes(), + "edges": self.graph.number_of_edges(), + } + + def get_node(self, qualified_name: str) -> dict[str, Any] | None: + """Return metadata for a node, or None if not found.""" + if qualified_name in self.graph: + return dict(self.graph.nodes[qualified_name]) + return None + + def get_dependencies(self, qualified_name: str) -> list[dict[str, Any]]: + """Return all symbols that *qualified_name* depends on (outgoing).""" + if qualified_name not in self.graph: + return [] + return [ + {"target": t, **self.graph.edges[qualified_name, t]} + for t in self.graph.successors(qualified_name) + ] + + def get_dependents(self, qualified_name: str) -> list[dict[str, Any]]: + """Return all symbols that depend on *qualified_name* (incoming).""" + if qualified_name not in self.graph: + return [] + return [ + {"source": s, **self.graph.edges[s, qualified_name]} + for s in self.graph.predecessors(qualified_name) + ] + + def all_nodes(self, kind: str | None = None) -> list[dict[str, Any]]: + """List all nodes, optionally filtered by kind.""" + results = [] + for n, data in self.graph.nodes(data=True): + if kind and data.get("kind") != kind: + continue + results.append({"qualified_name": n, **data}) + return results + + def summary(self) -> dict[str, Any]: + """Return a summary of the world model.""" + kinds: dict[str, int] = {} + for _, data in self.graph.nodes(data=True): + k = data.get("kind", "unknown") + kinds[k] = kinds.get(k, 0) + 1 + edge_kinds: dict[str, int] = {} + for _, _, data in self.graph.edges(data=True): + k = data.get("kind", "unknown") + edge_kinds[k] = edge_kinds.get(k, 0) + 1 + return { + "nodes": self.graph.number_of_nodes(), + "edges": self.graph.number_of_edges(), + "node_kinds": kinds, + "edge_kinds": edge_kinds, + "files": len(self._file_hashes), + } + + # ------------------------------------------------------------------ + # Persistence + # ------------------------------------------------------------------ + + def _save_cache(self): + os.makedirs(self.cache_dir, exist_ok=True) + # 1. JSON node-link (portable) + data = nx.node_link_data(self.graph) + with open(os.path.join(self.cache_dir, "graph.json"), "w") as f: + json.dump(data, f, indent=1) + # 2. Pickle (fast reload) + with open(os.path.join(self.cache_dir, "graph.pkl"), "wb") as f: + pickle.dump(self.graph, f, protocol=pickle.HIGHEST_PROTOCOL) + # 3. File hashes + with open(os.path.join(self.cache_dir, "file_hashes.json"), "w") as f: + json.dump(self._file_hashes, f) + # 4. File-node mapping + with open(os.path.join(self.cache_dir, "file_nodes.json"), "w") as f: + json.dump(self._file_nodes, f) + + def _load_cache(self): + pkl_path = os.path.join(self.cache_dir, "graph.pkl") + hash_path = os.path.join(self.cache_dir, "file_hashes.json") + fn_path = os.path.join(self.cache_dir, "file_nodes.json") + + if os.path.exists(pkl_path): + try: + with open(pkl_path, "rb") as f: + self.graph = pickle.load(f) + except Exception: + self.graph = nx.DiGraph() + + if os.path.exists(hash_path): + with open(hash_path) as f: + self._file_hashes = json.load(f) + + if os.path.exists(fn_path): + with open(fn_path) as f: + self._file_nodes = json.load(f) + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + def _apply_parse_result(self, result: ParseResult): + """Add nodes and edges from a parse result to the graph.""" + node_names: list[str] = [] + for sn in result.nodes: + self.graph.add_node(sn.qualified_name, **sn.to_dict()) + node_names.append(sn.qualified_name) + + for edge in result.edges: + self.graph.add_edge( + edge.source, edge.target, + kind=edge.kind, confidence=edge.confidence, lineno=edge.lineno, + ) + + self._file_nodes[result.file] = node_names + + def _remove_file_contributions(self, rel_path: str): + """Remove all nodes and their edges that came from a file.""" + names = self._file_nodes.pop(rel_path, []) + for n in names: + if n in self.graph: + self.graph.remove_node(n) # also removes incident edges + + def _resolve_cross_module_edges(self): + """Best-effort resolution of import targets to known graph nodes. + + When an import edge points at 'pkg.mod.func' and we have that + node in the graph, the edge target is already correct. When the + target is a module but the actual use is 'mod.func', we try to + find the function inside the module. + """ + existing = set(self.graph.nodes) + edges_to_add: list[tuple[str, str, dict]] = [] + edges_to_remove: list[tuple[str, str]] = [] + + for u, v, data in list(self.graph.edges(data=True)): + if v in existing: + continue + # Try parent.v as a resolution + # e.g. target 'utils.helper_func' might live as 'demo_package.utils.helper_func' + candidates = [n for n in existing if n.endswith(f".{v}") or n.endswith(f".{v.split('.')[-1]}")] + if len(candidates) == 1: + edges_to_remove.append((u, v)) + edges_to_add.append((u, candidates[0], {**data, "confidence": data.get("confidence", 1.0) * 0.9})) + elif len(candidates) > 1: + # ambiguous: pick the best match (longest common suffix) + best = max(candidates, key=lambda c: len(os.path.commonprefix([c[::-1], v[::-1]]))) + edges_to_remove.append((u, v)) + edges_to_add.append((u, best, {**data, "confidence": data.get("confidence", 1.0) * 0.7})) + + for u, v in edges_to_remove: + if self.graph.has_edge(u, v): + self.graph.remove_edge(u, v) + for u, v, d in edges_to_add: + self.graph.add_edge(u, v, **d) + + # Merge phantom nodes: when an import creates a node like + # 'pkg.mod.func' but the parser already has 'mod.func' (because + # root=pkg), redirect all edges to the real node and remove the phantom. + self._merge_phantom_nodes() + + def _merge_phantom_nodes(self): + """Merge nodes whose qualified_name is a suffix of another existing node. + + Import targets like 'demo_package.utils.validation.validate_positive' + are phantom duplicates of parsed nodes like 'utils.validation.validate_positive' + when the codebase root IS 'demo_package/'. We redirect all edges + from the phantom to the real node. + """ + # "Real" nodes are those explicitly added by the parser with metadata + real_nodes = {n for n, d in self.graph.nodes(data=True) if d.get("kind")} + all_nodes = set(self.graph.nodes) + phantoms_to_merge: dict[str, str] = {} # phantom -> real + + for n in all_nodes: + if n in real_nodes: + continue + # Check if this is a prefixed version of a real node + # e.g. 'demo_package.utils.validation.validate_positive' -> 'utils.validation.validate_positive' + parts = n.split(".") + for i in range(1, len(parts)): + suffix = ".".join(parts[i:]) + if suffix in real_nodes: + phantoms_to_merge[n] = suffix + break + + for phantom, real in phantoms_to_merge.items(): + # Transfer all incoming edges to the real node + for pred in list(self.graph.predecessors(phantom)): + if pred == real: + continue + edge_data = dict(self.graph.edges[pred, phantom]) + if not self.graph.has_edge(pred, real): + self.graph.add_edge(pred, real, **edge_data) + else: + # Keep the higher-confidence edge + existing = self.graph.edges[pred, real] + if edge_data.get("confidence", 0) > existing.get("confidence", 0): + self.graph.edges[pred, real].update(edge_data) + + # Transfer all outgoing edges from the phantom + for succ in list(self.graph.successors(phantom)): + if succ == real: + continue + edge_data = dict(self.graph.edges[phantom, succ]) + if not self.graph.has_edge(real, succ): + self.graph.add_edge(real, succ, **edge_data) + + # Remove the phantom + self.graph.remove_node(phantom) diff --git a/research/python-prototypes/impact_oracle/pyproject.toml b/research/python-prototypes/impact_oracle/pyproject.toml new file mode 100644 index 0000000..bbd83a9 --- /dev/null +++ b/research/python-prototypes/impact_oracle/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "impact-oracle" +version = "0.2.0" +description = "Codebase world-model and impact blast-radius oracle for Python repositories" +readme = "README.md" +requires-python = ">=3.9" +authors = [{ name = "impact_oracle authors" }] +license = { text = "MIT" } +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Quality Assurance", +] +dependencies = ["networkx>=3.0"] + +[project.optional-dependencies] +dev = ["pytest>=7.0", "matplotlib>=3.7"] + +[project.scripts] +impact-oracle = "impact_oracle.cli:main" +impact-oracle-mcp = "impact_oracle.mcp_server:main" + +[tool.setuptools.packages.find] +include = ["impact_oracle*"] diff --git a/research/python-prototypes/impact_oracle/requirements.txt b/research/python-prototypes/impact_oracle/requirements.txt new file mode 100644 index 0000000..4948daa --- /dev/null +++ b/research/python-prototypes/impact_oracle/requirements.txt @@ -0,0 +1,3 @@ +networkx>=3.0 +pytest>=7.0 +matplotlib>=3.7 diff --git a/research/python-prototypes/impact_oracle/tests/test_demo_package.py b/research/python-prototypes/impact_oracle/tests/test_demo_package.py new file mode 100644 index 0000000..a43ac6c --- /dev/null +++ b/research/python-prototypes/impact_oracle/tests/test_demo_package.py @@ -0,0 +1,255 @@ +"""Comprehensive tests for demo_package — used as ground truth for mutation testing.""" + +import pytest +from demo_package.models import Product, PremiumProduct +from demo_package.inventory import Inventory +from demo_package.orders import Order, OrderLine +from demo_package.reports import inventory_report, order_summary +from demo_package.sub.pricing import margin_analysis, bulk_discount_curve +from demo_package.utils.validation import validate_positive, validate_name, validate_quantity +from demo_package.utils.formatting import format_currency, format_receipt_line + + +# --- Validation tests --- + +class TestValidation: + def test_validate_positive_ok(self): + assert validate_positive(5.0) == 5.0 + assert validate_positive(0) == 0.0 + + def test_validate_positive_negative(self): + with pytest.raises(ValueError): + validate_positive(-1) + + def test_validate_name_ok(self): + assert validate_name(" Widget ") == "Widget" + + def test_validate_name_empty(self): + with pytest.raises(ValueError): + validate_name("") + + def test_validate_quantity_ok(self): + assert validate_quantity(3) == 3 + + def test_validate_quantity_bad(self): + with pytest.raises(ValueError): + validate_quantity(0) + with pytest.raises(ValueError): + validate_quantity(-1) + + +# --- Formatting tests --- + +class TestFormatting: + def test_format_currency(self): + assert format_currency(1234.5) == "$1,234.50" + assert format_currency(0) == "$0.00" + + def test_format_receipt_line(self): + line = format_receipt_line("Widget", 3, 10.0) + assert "Widget" in line + assert "$10.00" in line + assert "$30.00" in line + + +# --- Model tests --- + +class TestProduct: + def test_create_product(self): + p = Product("Widget", 29.99) + assert p.name == "Widget" + assert p.price == 29.99 + assert p.category == "general" + + def test_discounted_price(self): + p = Product("Widget", 100.0) + assert p.discounted_price(10) == 90.0 + assert p.discounted_price(0) == 100.0 + + def test_discounted_price_validates(self): + p = Product("Widget", 100.0) + with pytest.raises(ValueError): + p.discounted_price(-5) + + def test_repr(self): + p = Product("Widget", 29.99) + assert "Widget" in repr(p) + + +class TestPremiumProduct: + def test_inherits_product(self): + pp = PremiumProduct("Deluxe Widget", 199.99) + assert isinstance(pp, Product) + assert pp.category == "premium" + + def test_loyalty_points(self): + pp = PremiumProduct("Deluxe", 100.0) + assert pp.loyalty_points == 1000 # 100 * 10 + + def test_discount_cap(self): + pp = PremiumProduct("Deluxe", 100.0) + # 50% discount should be capped at 30% + assert pp.discounted_price(50) == 70.0 + # 20% discount passes through + assert pp.discounted_price(20) == 80.0 + + +# --- Inventory tests --- + +class TestInventory: + def test_add_and_get(self): + inv = Inventory() + p = Product("Widget", 10.0) + inv.add_product(p, 5) + assert inv.get_product("Widget") is p + assert inv.stock_level("Widget") == 5 + + def test_add_existing_increases_stock(self): + inv = Inventory() + p = Product("Widget", 10.0) + inv.add_product(p, 5) + inv.add_product(p, 3) + assert inv.stock_level("Widget") == 8 + + def test_remove_stock(self): + inv = Inventory() + inv.add_product(Product("Widget", 10.0), 10) + assert inv.remove_stock("Widget", 3) is True + assert inv.stock_level("Widget") == 7 + + def test_remove_stock_insufficient(self): + inv = Inventory() + inv.add_product(Product("Widget", 10.0), 2) + assert inv.remove_stock("Widget", 5) is False + assert inv.stock_level("Widget") == 2 # unchanged + + def test_remove_stock_nonexistent(self): + inv = Inventory() + assert inv.remove_stock("Ghost", 1) is False + + def test_total_value(self): + inv = Inventory() + inv.add_product(Product("A", 10.0), 5) + inv.add_product(Product("B", 20.0), 3) + assert inv.total_value() == 10 * 5 + 20 * 3 + + def test_low_stock_items(self): + inv = Inventory() + inv.add_product(Product("Low", 10.0), 2) + inv.add_product(Product("High", 10.0), 100) + assert inv.low_stock_items() == ["Low"] + + +# --- Order tests --- + +class TestOrderLine: + def test_line_total_no_discount(self): + p = Product("Widget", 50.0) + line = OrderLine(p, 3) + assert line.line_total() == 150.0 + + def test_line_total_with_discount(self): + p = Product("Widget", 100.0) + line = OrderLine(p, 2, discount_pct=10) + assert line.line_total() == 180.0 + + +class TestOrder: + def _make_inventory(self): + inv = Inventory() + inv.add_product(Product("Widget", 50.0), 100) + inv.add_product(PremiumProduct("Deluxe", 200.0), 50) + return inv + + def test_add_item_success(self): + inv = self._make_inventory() + order = Order(inv) + assert order.add_item("Widget", 3) is True + assert len(order.lines) == 1 + assert inv.stock_level("Widget") == 97 + + def test_add_item_insufficient_stock(self): + inv = self._make_inventory() + order = Order(inv) + assert order.add_item("Widget", 999) is False + + def test_add_item_nonexistent(self): + inv = self._make_inventory() + order = Order(inv) + assert order.add_item("Nonexistent", 1) is False + + def test_total(self): + inv = self._make_inventory() + order = Order(inv) + order.add_item("Widget", 2) + order.add_item("Deluxe", 1) + assert order.total() == 2 * 50 + 200 + + def test_loyalty_points(self): + inv = self._make_inventory() + order = Order(inv) + order.add_item("Widget", 1) # no points + order.add_item("Deluxe", 2) # 200*10 * 2 = 4000 + assert order.loyalty_points() == 4000 + + def test_receipt(self): + inv = self._make_inventory() + order = Order(inv) + order.add_item("Widget", 1) + r = order.receipt() + assert "RECEIPT" in r + assert "TOTAL" in r + + +# --- Reports tests --- + +class TestReports: + def test_inventory_report(self): + inv = Inventory() + inv.add_product(Product("Low", 10.0), 2) + inv.add_product(Product("High", 50.0), 100) + report = inventory_report(inv) + assert "Low" in report + assert "$" in report + + def test_order_summary(self): + inv = Inventory() + inv.add_product(Product("W", 10.0), 100) + o1 = Order(inv) + o1.add_item("W", 5) + o2 = Order(inv) + o2.add_item("W", 3) + summary = order_summary([o1, o2]) + assert summary["order_count"] == 2 + assert summary["total_items"] == 8 + assert summary["total_revenue"] == 80.0 + + +# --- Pricing analytics tests --- + +class TestPricing: + def test_margin_analysis(self): + p = Product("W", 100.0) + result = margin_analysis(p, 60.0) + assert result["margin"] == 40.0 + assert result["margin_pct"] == 40.0 + assert result["is_premium"] is False + + def test_margin_analysis_premium(self): + pp = PremiumProduct("D", 200.0) + result = margin_analysis(pp, 80.0) + assert result["is_premium"] is True + + def test_bulk_discount_curve(self): + p = Product("W", 100.0) + curve = bulk_discount_curve(p, [1, 10, 20, 100]) + assert len(curve) == 4 + assert curve[0]["tier_discount_pct"] == 0.0 # qty 1 + assert curve[1]["tier_discount_pct"] == 5.0 # qty 10 + assert curve[2]["tier_discount_pct"] == 10.0 # qty 20 + assert curve[3]["tier_discount_pct"] == 50.0 # qty 100, capped + + def test_margin_analysis_validates(self): + p = Product("W", 100.0) + with pytest.raises(ValueError): + margin_analysis(p, -10) diff --git a/research/python-prototypes/router_gate/README.md b/research/python-prototypes/router_gate/README.md new file mode 100644 index 0000000..888a601 --- /dev/null +++ b/research/python-prototypes/router_gate/README.md @@ -0,0 +1,173 @@ +# router_gate + +Production-ready **assumption gate + complexity-aware model router** for coding agents. + +It is designed to sit in front of Claude Code, MCP clients, custom agent CLIs, internal LLM gateways, or any workflow where you want two safety checks before spending premium model tokens: + +1. **Assumption gate** — if the task is under-specified, halt and ask clarifying questions instead of guessing. +2. **Complexity router** — if the task is clear, route it to `cheap`, `mid`, or `premium` with an auditable explanation. + +The core package is dependency-free. Tests only need `pytest`. + +## Install + +From this source directory: + +```bash +python -m pip install -e . +``` + +For tests: + +```bash +python -m pip install -e '.[dev]' +python -m pytest +``` + +## CLI usage + +Pass task text as an argument, via `--file`, or on stdin. + +```bash +router-gate assess "Fix the bug." --pretty +router-gate route "Implement an LRU cache class with O(1) get/put. Python." --pretty +router-gate decide "Write is_even(n). is_even(4) -> True. Python." --pretty +``` + +`decide` is the safest integration point for agents: it gates first, and only returns a route when the task is specified enough to act on. + +## Execute through any model gateway + +`router-gate run` can call any external command. The task is sent on stdin and the selected model id is exposed as `ROUTER_GATE_MODEL_ID`. You can also use `{model}` in the command string. + +Preferred executor stdout is JSON: + +```json +{"text":"model output", "input_tokens":123, "output_tokens":456} +``` + +Plain text stdout also works; token counts are estimated for cost accounting. + +Example: + +```bash +router-gate run \ + --executor-command './my-agent-call --model {model}' \ + "Write is_prime(n). is_prime(7) -> True, is_prime(8) -> False. Python." \ + --pretty +``` + +If the selected tier fails, the pipeline records the error and escalates to the next tier unless `--no-escalation` is set. + +## Claude Code / MCP usage + +Install the package, then register the stdio MCP server with your MCP-capable client. + +Example MCP server command: + +```bash +router-gate-mcp +``` + +Claude Code-style config shape: + +```json +{ + "mcpServers": { + "router-gate": { + "command": "router-gate-mcp", + "args": [] + } + } +} +``` + +Exposed tools: + +- `assess_task` — returns completeness, risk, missing dimensions, and questions. +- `route_task` — returns `cheap` / `mid` / `premium` plus scoring reasons. +- `decide_task` — runs the assumption gate first; if safe, also routes. + +Use `decide_task` before expensive or ambiguous work. If it returns `halted_for_questions: true`, ask the returned questions before continuing. + +## Configuration + +Config can come from a TOML file or environment variables. + +```toml +ask_threshold = 0.6 +allow_escalation = true +executor_command = "./my-agent-call --model {model}" +executor_timeout_seconds = 120 + +[tiers.cheap] +model_id = "claude-haiku-4-5-20251001" +usd_in_per_mtok = 1.0 +usd_out_per_mtok = 5.0 + +[tiers.mid] +model_id = "claude-sonnet-5" +usd_in_per_mtok = 3.0 +usd_out_per_mtok = 15.0 + +[tiers.premium] +model_id = "claude-opus-4-8" +usd_in_per_mtok = 15.0 +usd_out_per_mtok = 75.0 +``` + +Environment overrides: + +```bash +export ROUTER_GATE_CONFIG=/path/to/router-gate.toml +export ROUTER_GATE_ASK_THRESHOLD=0.65 +export ROUTER_GATE_ALLOW_ESCALATION=true +export ROUTER_GATE_EXECUTOR_COMMAND='./my-agent-call --model {model}' +export ROUTER_GATE_EXECUTOR_TIMEOUT_SECONDS=180 +``` + +Inspect the effective config: + +```bash +router-gate config --pretty +``` + +## Python API + +```python +from router_gate import assess, route, run, external_command_executor + +report = assess("Fix the bug.") +if report.should_ask: + print(report.questions) +else: + decision = route("Write is_even(n). is_even(4) -> True. Python.") + print(decision.tier, decision.explain()) + +executor = external_command_executor("./my-agent-call --model {model}") +result = run( + "Write is_prime(n). is_prime(7) -> True, is_prime(8) -> False. Python.", + executor, +) +print(result.success, result.final_tier, result.total_cost) +``` + +## Architecture + +```text +request -> assumption gate -> halt & ask OR route -> execute -> verify -> escalate +``` + +- Router decisions are transparent rubric scores, not another opaque model call. +- Verification is caller-owned: tests, compilers, regex checks, or review gates can be plugged into `run(..., verifier=...)`. +- Cost accounting uses measured executor token counts when available and conservative estimates otherwise. + +## Development + +```bash +python demo.py +python evaluate.py +python -m pytest +``` + +The included evaluation task set is a demonstration set, not a field benchmark. Calibrate thresholds on your own workload before enforcing automated model selection in high-stakes production paths. diff --git a/research/python-prototypes/router_gate/conftest.py b/research/python-prototypes/router_gate/conftest.py new file mode 100644 index 0000000..ce9030a --- /dev/null +++ b/research/python-prototypes/router_gate/conftest.py @@ -0,0 +1,3 @@ +"""Ensures the package root is importable during test collection (bare `pytest` works).""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) diff --git a/research/python-prototypes/router_gate/demo.py b/research/python-prototypes/router_gate/demo.py new file mode 100644 index 0000000..1d1eb3c --- /dev/null +++ b/research/python-prototypes/router_gate/demo.py @@ -0,0 +1,67 @@ +"""Demo: run a handful of tasks through the gate->route->execute->verify loop. + +Offline by default (deterministic stub executor, no tokens spent), so it runs +anywhere. Prints, for each task, the gate decision, the routed tier, and the cost +versus an always-premium baseline. + + python demo.py + +The point of the demo is to make the two mechanisms legible in a few lines: + - under-specified tasks HALT with concrete questions (M2: don't confabulate); + - well-specified tasks ROUTE to the cheapest capable tier (M1: don't overpay). +""" +import os, sys +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from router_gate import route, assess, run, always_premium_cost +from router_gate.pricing import DEFAULT_LADDER + + +def stub_executor(task, model_id): + itok = max(20, len(task) // 3) + otok = max(30, len(task) // 2) + return (f"[stub answer from {model_id}]", itok, otok) + + +EXAMPLES = [ + "Write a Python function is_prime(n). is_prime(7) -> True, is_prime(8) -> False.", + "Implement an LRU cache class with get and put, O(1), capacity fixed at construction. Python.", + "Design and implement a thread-safe bounded blocking queue in Python supporting put() and " + "get() from multiple producer and consumer threads, with proper condition-variable signaling " + "and no busy-waiting. Include tests for the empty and full boundary conditions.", + "Fix the bug.", + "Optimize it.", +] + + +def main(): + print("=" * 74) + print("Complexity-aware router + assumption gate -- offline demo") + print("=" * 74) + total_routed = total_premium = 0.0 + for task in EXAMPLES: + res = run(task, stub_executor, ladder=DEFAULT_LADDER) + print(f"\nTASK: {task}") + if res.halted_for_questions: + print(f" GATE: HALT (completeness={res.assumption.completeness:.2f}, " + f"risk={res.assumption.risk}) -- asking instead of guessing:") + for q in res.assumption.questions: + print(f" ? {q}") + else: + print(f" GATE: proceed (completeness={res.assumption.completeness:.2f})") + print(f" ROUTE: {res.routing.tier} (score={res.routing.score:.1f})") + print(f" {'; '.join(w for _, w in res.routing.reasons[1:4])}") + print(f" RAN on: {res.final_tier} cost=${res.total_cost:.6f} " + f"(always-premium would be ${always_premium_cost(res):.6f})") + total_routed += res.total_cost + total_premium += always_premium_cost(res) + + if total_premium: + saved = 100 * (total_premium - total_routed) / total_premium + print("\n" + "-" * 74) + print(f"Executed tasks: routed ${total_routed:.6f} vs premium ${total_premium:.6f} " + f"-> {saved:.0f}% saved") + print("\n(For live numbers on real models: python evaluate.py --live)") + + +if __name__ == "__main__": + main() diff --git a/research/python-prototypes/router_gate/eval_results.json b/research/python-prototypes/router_gate/eval_results.json new file mode 100644 index 0000000..f24a9c8 --- /dev/null +++ b/research/python-prototypes/router_gate/eval_results.json @@ -0,0 +1,504 @@ +{ + "summary": { + "mode": "live", + "models": { + "cheap": "claude-haiku-4-5", + "mid": "claude-sonnet-5", + "premium": "claude-opus-4-8" + }, + "n_tasks": 30, + "gate": { + "accuracy": 1.0, + "precision": 1.0, + "recall": 1.0, + "f1": 1.0, + "confusion": { + "tp": 9, + "fp": 0, + "tn": 21, + "fn": 0 + }, + "n_halted": 9 + }, + "routing": { + "n_scored": 21, + "exact_accuracy": 1.0, + "within_1_tier_accuracy": 1.0 + }, + "cost": { + "total_input_tokens": 28954, + "total_output_tokens": 13122, + "routed_cost_usd": 0.537756, + "always_premium_cost_usd": 1.41846, + "cost_saved_usd": 0.880704, + "cost_saved_pct": 62.1 + }, + "verification_subexperiment": { + "description": "For 3 tasks, the pipeline verifier EXECUTED the generated code against real test cases.", + "results": [ + { + "task": "is_prime", + "routed": "cheap", + "verified": true, + "final_tier": "cheap" + }, + { + "task": "factorial", + "routed": "cheap", + "verified": true, + "final_tier": "cheap" + }, + { + "task": "merge_sorted", + "routed": "mid", + "verified": true, + "final_tier": "mid" + } + ], + "finding": "Routing down still produced code that passed execution-based verification; no escalation needed." + }, + "honesty": "30-task hand-labeled DEMONSTRATION set; rubric thresholds were tuned against it. Perfect accuracy shows the rubric can SEPARATE the cases, not a field benchmark. Cost figures are exact arithmetic on REAL measured token counts (live host.llm calls) and approximate public list prices for the 4.x tier ladder. Same tokens repriced at premium = the counterfactual baseline; savings come purely from tier selection." + }, + "rows": [ + { + "task": "Write a Python function is_even(n) that returns True if n is even. Example: is_even(4) -> True.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.96, + "in_tok": 3520, + "out_tok": 156, + "routed_cost": 0.0043, + "premium_cost": 0.0645, + "output": "Here's a simple Python function that checks if a number is even:\n\n```python\ndef is_even(n):\n return n % 2 == 0\n```\n\n**How it works:**\n- The modulo operator `%` returns the remainder after division\n- If `n % 2 == 0`, the number is even (no remainder when divided by 2)\n- Otherwise, it's odd\n\n**Examples:**\n```python\nis_even(4) # True\nis_even(7) # False\nis_even(0) # True\nis_even(-2) # True\n`", + "questions": [] + }, + { + "task": "Reverse a string in Python. Input: 'abc' -> Output: 'cba'. Provide the function.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.87, + "in_tok": 3517, + "out_tok": 336, + "routed_cost": 0.005197, + "premium_cost": 0.077955, + "output": "# Reverse a String in Python\n\nHere are several ways to reverse a string:\n\n## 1. **Slicing (Most Pythonic)**\n```python\ndef reverse_string(s):\n return s[::-1]\n\n# Usage\nprint(reverse_string('abc')) # Output: 'cba'\n```\n\n## 2. **Using `reversed()` Function**\n```python\ndef reverse_string(s):\n return ''.join(reversed(s))\n\n# Usage\nprint(reverse_string('abc')) # Output: 'cba'\n```\n\n## 3. **Loop-Base", + "questions": [] + }, + { + "task": "Write factorial(n) in Python returning n!. factorial(5) -> 120.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.96, + "in_tok": 3510, + "out_tok": 261, + "routed_cost": 0.004815, + "premium_cost": 0.072225, + "output": "# Factorial Function in Python\n\nHere's a simple recursive implementation:\n\n```python\ndef factorial(n):\n if n < 0:\n raise ValueError(\"Factorial is not defined for negative numbers\")\n if n == 0 or n == 1:\n return 1\n return n * factorial(n - 1)\n```\n\n**Usage:**\n```python\nfactorial(5) # Returns 120\n```\n\n---\n\n## Alternative: Iterative Approach\n\nIf you prefer to avoid recursion (u", + "questions": [] + }, + { + "task": "Capitalize the first letter of each word in a string. Input 'hello world' -> 'Hello World'.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.81, + "in_tok": 3515, + "out_tok": 259, + "routed_cost": 0.00481, + "premium_cost": 0.07215, + "output": "I'll create a function that capitalizes the first letter of each word in a string.\n\n```python\ndef capitalize_words(text):\n \"\"\"Capitalize the first letter of each word in a string.\"\"\"\n return ' '.join(word.capitalize() for word in text.split())\n\n# Test with your example\nresult = capitalize_words('hello world')\nprint(result)\n```\n\n**Output:**\n```\nHello World\n```\n\nThis function works by:\n1. **`.", + "questions": [] + }, + { + "task": "Return the sum of a list of integers. sum_list([1,2,3]) -> 6. Python.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.96, + "in_tok": 3516, + "out_tok": 192, + "routed_cost": 0.004476, + "premium_cost": 0.06714, + "output": "# Sum of a List\n\nHere's a Python function to sum a list of integers:\n\n```python\ndef sum_list(numbers):\n return sum(numbers)\n```\n\n**Usage:**\n```python\nsum_list([1, 2, 3]) # Returns 6\n```\n\n**Alternative implementations:**\n\nIf you want to avoid the built-in `sum()` function:\n\n```python\ndef sum_list(numbers):\n total = 0\n for num in numbers:\n total += num\n return total\n```\n\nOr using", + "questions": [] + }, + { + "task": "Write a function that checks if a number is prime. is_prime(7) -> True, is_prime(8) -> False. Python.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.96, + "in_tok": 3522, + "out_tok": 363, + "routed_cost": 0.005337, + "premium_cost": 0.080055, + "output": "```python\ndef is_prime(n):\n \"\"\"\n Check if a number is prime.\n \n Args:\n n: An integer to check for primality\n \n Returns:\n True if n is prime, False otherwise\n \"\"\"\n if n < 2:\n return False\n if n == 2:\n return True\n if n % 2 == 0:\n return False\n \n # Check odd divisors up to sqrt(n)\n for i in range(3, int(n**0.5) + 1, 2):\n ", + "questions": [] + }, + { + "task": "Convert a temperature in Celsius to Fahrenheit. c_to_f(0) -> 32.0. Python function.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.96, + "in_tok": 3519, + "out_tok": 121, + "routed_cost": 0.004124, + "premium_cost": 0.06186, + "output": "```python\ndef c_to_f(celsius):\n \"\"\"Convert temperature from Celsius to Fahrenheit.\"\"\"\n return (celsius * 9/5) + 32\n```\n\n**Usage:**\n```python\nc_to_f(0) # \u2192 32.0\nc_to_f(100) # \u2192 212.0\nc_to_f(-40) # \u2192 -40.0\n```\n\nThe formula is: **F = (C \u00d7 9/5) + 32**", + "questions": [] + }, + { + "task": "Count vowels in a string. count_vowels('apple') -> 2. Python.", + "gold_tier": "cheap", + "gold_ask": false, + "asked": false, + "routed_tier": "cheap", + "completeness": 0.96, + "in_tok": 3511, + "out_tok": 258, + "routed_cost": 0.004801, + "premium_cost": 0.072015, + "output": "I'll write a Python function to count vowels in a string:\n\n```python\ndef count_vowels(s):\n \"\"\"Count the number of vowels in a string.\"\"\"\n vowels = 'aeiouAEIOU'\n return sum(1 for char in s if char in vowels)\n\n# Test with your example\nprint(count_vowels('apple')) # Output: 2\n```\n\nThis function:\n- Defines vowels as both lowercase and uppercase letters\n- Iterates through each character in th", + "questions": [] + }, + { + "task": "Implement an LRU cache class in Python with get(key) and put(key, value), capacity fixed at construction, O(1) operations. Include a docstring.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.99, + "in_tok": 53, + "out_tok": 900, + "routed_cost": 0.013659, + "premium_cost": 0.06829500000000001, + "output": "```python\nfrom collections import OrderedDict\nfrom typing import Any, Optional\n\n\nclass LRUCache:\n \"\"\"\n A fixed-capacity Least Recently Used (LRU) cache.\n\n Both `get` and `put` run in O(1) average time. When the cache is at\n capacity and a new key is inserted, the least recently used entry\n (the one that hasn't been accessed or inserted for the longest time)\n is evicted to make ro", + "questions": [] + }, + { + "task": "Parse a CSV string into a list of dicts using the header row as keys. Handle quoted fields containing commas. Python, standard library only.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.67, + "in_tok": 48, + "out_tok": 900, + "routed_cost": 0.013644, + "premium_cost": 0.06822, + "output": "Here's a straightforward implementation using the standard library `csv` module, which already handles quoted fields containing commas (per RFC 4180 rules):\n\n```python\nimport csv\nimport io\n\n\ndef parse_csv_to_dicts(csv_string: str) -> list[dict]:\n \"\"\"\n Parse a CSV string into a list of dicts using the header row as keys.\n\n Handles quoted fields containing commas, embedded newlines, and esc", + "questions": [] + }, + { + "task": "Write a function that merges two sorted integer lists into one sorted list without using sorted(). Return the merged list. Python.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.69, + "in_tok": 42, + "out_tok": 775, + "routed_cost": 0.011751, + "premium_cost": 0.058754999999999995, + "output": "## Merge Two Sorted Lists\n\nHere's a standard two-pointer merge implementation that runs in O(n + m) time without relying on `sorted()`:\n\n```python\ndef merge_sorted_lists(list1, list2):\n \"\"\"\n Merge two sorted integer lists into a single sorted list.\n\n Args:\n list1: A sorted list of integers.\n list2: A sorted list of integers.\n\n Returns:\n A new sorted list containing", + "questions": [] + }, + { + "task": "Implement debounce(fn, wait_ms) in JavaScript that delays calling fn until wait_ms has elapsed since the last call. Return the debounced function.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.69, + "in_tok": 57, + "out_tok": 813, + "routed_cost": 0.012366, + "premium_cost": 0.06183, + "output": "Here's a standard implementation of `debounce`:\n\n```javascript\nfunction debounce(fn, wait_ms) {\n let timeoutId = null;\n\n function debounced(...args) {\n const context = this;\n\n // Clear any pending call \u2014 restarts the timer\n clearTimeout(timeoutId);\n\n timeoutId = setTimeout(() => {\n timeoutId = null;\n fn.apply(context, args);\n }, wait_ms);\n }\n\n // Optional: allow manual", + "questions": [] + }, + { + "task": "Given a binary tree node class with .left/.right/.val, write an in-order traversal returning a list of values. Python.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.63, + "in_tok": 41, + "out_tok": 588, + "routed_cost": 0.008943, + "premium_cost": 0.044715, + "output": "## In-order traversal of a binary tree\n\nAssuming a node class like:\n\n```python\nclass TreeNode:\n def __init__(self, val=0, left=None, right=None):\n self.val = val\n self.left = left\n self.right = right\n```\n\n### Recursive approach\n\n```python\ndef inorder_traversal(root):\n result = []\n\n def _visit(node):\n if node is None:\n return\n _visit(node.left)", + "questions": [] + }, + { + "task": "Validate an email address with a regex in Python and return True/False. Must reject 'a@b' and accept 'a@b.com'.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.93, + "in_tok": 45, + "out_tok": 900, + "routed_cost": 0.013635, + "premium_cost": 0.068175, + "output": "Here's a Python function that validates email addresses with a regex, requiring at least one dot in the domain part (which is why it rejects `a@b` but accepts `a@b.com`):\n\n```python\nimport re\n\nEMAIL_REGEX = re.compile(\n r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9-]+(\\.[a-zA-Z0-9-]+)*\\.[a-zA-Z]{2,}$'\n)\n\ndef is_valid_email(email: str) -> bool:\n return bool(EMAIL_REGEX.match(email))\n\n\n# Quick tests\nif __na", + "questions": [] + }, + { + "task": "Write a retry decorator in Python: retry(times=3, delay=0.1) that re-runs the wrapped function on exception up to `times`, then re-raises.", + "gold_tier": "mid", + "gold_ask": false, + "asked": false, + "routed_tier": "mid", + "completeness": 0.96, + "in_tok": 56, + "out_tok": 900, + "routed_cost": 0.013668, + "premium_cost": 0.06834, + "output": "Here's a straightforward implementation using `functools.wraps` to preserve the wrapped function's metadata:\n\n```python\nimport functools\nimport time\n\n\ndef retry(times=3, delay=0.1):\n \"\"\"\n Decorator that re-runs the wrapped function on exception.\n\n Args:\n times: Maximum number of attempts (including the first call).\n delay: Seconds to wait between attempts.\n\n Re-raises the", + "questions": [] + }, + { + "task": "Design and implement a thread-safe bounded blocking queue in Python supporting put() and get() from multiple producer and consumer threads, with proper condition-variable signaling and no busy-waiting. Include tests for the empty and full boundary conditions.", + "gold_tier": "premium", + "gold_ask": false, + "asked": false, + "routed_tier": "premium", + "completeness": 0.95, + "in_tok": 84, + "out_tok": 900, + "routed_cost": 0.06876, + "premium_cost": 0.06876, + "output": "Here's a thread-safe bounded blocking queue with condition-variable signaling.\n\n```python\nimport threading\nfrom collections import deque\n\n\nclass BoundedBlockingQueue:\n \"\"\"A thread-safe, bounded FIFO queue that blocks producers when full\n and consumers when empty. Uses two condition variables over a single\n lock so producers and consumers are woken selectively (no busy-waiting,\n no thun", + "questions": [] + }, + { + "task": "Implement Dijkstra's shortest-path algorithm on a weighted directed graph given as an adjacency dict, returning the distance map and the predecessor map. Handle unreachable nodes. Optimize with a binary heap. Python.", + "gold_tier": "premium", + "gold_ask": false, + "asked": false, + "routed_tier": "premium", + "completeness": 0.85, + "in_tok": 81, + "out_tok": 900, + "routed_cost": 0.068715, + "premium_cost": 0.068715, + "output": "Here's a clean, efficient implementation using `heapq` (Python's binary heap):\n\n```python\nimport heapq\nfrom math import inf\n\n\ndef dijkstra(graph, source):\n \"\"\"\n Dijkstra's shortest-path algorithm on a weighted directed graph.\n\n Parameters\n ----------\n graph : dict[node, dict[node, float]]\n Adjacency dict. graph[u][v] = weight of edge u -> v.\n Edge weights must be non-n", + "questions": [] + }, + { + "task": "Refactor a synchronous data pipeline into an async architecture: design the module boundaries, define back-pressure between stages, ensure at-least-once processing under failure, and document the consistency trade-offs. Target Python asyncio.", + "gold_tier": "premium", + "gold_ask": false, + "asked": false, + "routed_tier": "premium", + "completeness": 0.77, + "in_tok": 78, + "out_tok": 900, + "routed_cost": 0.06867000000000001, + "premium_cost": 0.06867000000000001, + "output": "Here's a complete design for refactoring a synchronous pipeline into an async architecture with explicit back-pressure, at-least-once semantics, and documented consistency trade-offs.\n\n## Module boundaries\n\nSplit along failure and scaling boundaries, not just logical steps. Each stage is independently testable and can fail without corrupting neighbors.\n\n```\npipeline/\n sources/ # ingest: pu", + "questions": [] + }, + { + "task": "Write a recursive-descent parser for a small arithmetic grammar (numbers, + - * /, parentheses, correct precedence and associativity) that returns an AST, plus an evaluator. Include error handling for malformed input. Python.", + "gold_tier": "premium", + "gold_ask": false, + "asked": false, + "routed_tier": "premium", + "completeness": 0.77, + "in_tok": 80, + "out_tok": 900, + "routed_cost": 0.06870000000000001, + "premium_cost": 0.06870000000000001, + "output": "Here's a complete recursive-descent parser and evaluator for arithmetic expressions.\n\n```python\n\"\"\"\nRecursive-descent parser + evaluator for arithmetic expressions.\n\nGrammar (standard precedence, left-associative binary operators):\n\n expr := term (('+' | '-') term)*\n term := factor (('*' | '/') factor)*\n factor := ('+' | '-') factor # unary plus/minus\n | primar", + "questions": [] + }, + { + "task": "Implement a distributed rate limiter using a token-bucket algorithm backed by Redis, correct under concurrent access from multiple app instances, with configurable rate and burst. Explain the atomicity guarantees. Python.", + "gold_tier": "premium", + "gold_ask": false, + "asked": false, + "routed_tier": "premium", + "completeness": 0.83, + "in_tok": 76, + "out_tok": 900, + "routed_cost": 0.06864, + "premium_cost": 0.06864, + "output": "Here's a production-grade distributed rate limiter using the token-bucket algorithm backed by Redis, with correctness under concurrent access as the central design goal.\n\n## The core problem\n\nA token bucket has two state variables: the current token count and the timestamp of the last refill. Under concurrent access, a naive implementation does read \u2192 compute \u2192 write in three round-trips, which is", + "questions": [] + }, + { + "task": "Design a schema migration system that applies and rolls back ordered migrations, records applied state, is idempotent on partial failure, and is safe to run concurrently from multiple deployers. Describe the locking strategy and implement the core. Python + SQL.", + "gold_tier": "premium", + "gold_ask": false, + "asked": false, + "routed_tier": "premium", + "completeness": 0.77, + "in_tok": 83, + "out_tok": 900, + "routed_cost": 0.068745, + "premium_cost": 0.068745, + "output": "# Schema Migration System\n\nA migration runner that's ordered, idempotent, rollback-capable, and concurrency-safe. Let me walk through the design, then the implementation.\n\n## Design Overview\n\n### State tracking\nA `schema_migrations` table records what's been applied. Each row is a completed migration. The presence of a row is the source of truth \u2014 never file mtimes or a \"current version\" integer.\n", + "questions": [] + }, + { + "task": "Fix the bug.", + "gold_tier": "cheap", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.23, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "Which specific file, module, or component should this change touch?", + "How will we know it's correct -- a test, an acceptance criterion, or a reference behavior?" + ] + }, + { + "task": "Make the API faster.", + "gold_tier": "mid", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.23, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "What are the exact input and output (types, an example, or the expected return value)?" + ] + }, + { + "task": "Add authentication to the app.", + "gold_tier": "premium", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.23, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "What exactly should this produce, and how will we know it's correct?" + ] + }, + { + "task": "Write a function to process the data and handle everything appropriately.", + "gold_tier": "mid", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.01, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "What are the exact input and output (types, an example, or the expected return value)?" + ] + }, + { + "task": "Refactor this module to be cleaner.", + "gold_tier": "mid", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.01, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "Are there constraints I must respect (performance, allowed dependencies, style, backward-compat)?" + ] + }, + { + "task": "Build the reporting feature the way we discussed.", + "gold_tier": "premium", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.23, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "How will we know it's correct -- a test, an acceptance criterion, or a reference behavior?" + ] + }, + { + "task": "Optimize it.", + "gold_tier": "mid", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.23, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "How will we know it's correct -- a test, an acceptance criterion, or a reference behavior?" + ] + }, + { + "task": "Integrate the payment thing and make it work properly.", + "gold_tier": "premium", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.0, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "Which specific file, module, or component should this change touch?", + "How will we know it's correct -- a test, an acceptance criterion, or a reference behavior?" + ] + }, + { + "task": "Update the config as needed and clean it up.", + "gold_tier": "cheap", + "gold_ask": true, + "asked": true, + "routed_tier": null, + "completeness": 0.01, + "in_tok": 0, + "out_tok": 0, + "routed_cost": 0, + "premium_cost": 0, + "output": "", + "questions": [ + "Which specific file, module, or component should this change touch?" + ] + } + ] +} \ No newline at end of file diff --git a/research/python-prototypes/router_gate/evaluate.py b/research/python-prototypes/router_gate/evaluate.py new file mode 100644 index 0000000..c1f6ae7 --- /dev/null +++ b/research/python-prototypes/router_gate/evaluate.py @@ -0,0 +1,161 @@ +"""Evaluation harness for the router + assumption gate. + +Measures three things, honestly: + + 1. ROUTING ACCURACY - on the well-specified tasks the router actually sees in the + pipeline (under-specified tasks are halted by the gate first), + does it pick the gold tier? Also within-1-tier accuracy. + 2. GATE ACCURACY - across ALL tasks, does should_ask match the gold_ask label? + Reported as precision/recall/F1 for "should ask". + 3. TOKENS / COST SAVED - versus an always-premium baseline, using REAL measured token + counts from host.llm (or the offline stub). Cost is arithmetic + on measured tokens and the pricing ladder -- never guessed. + +Two modes: + --offline (default): deterministic stub executor, no tokens spent. Token counts are + synthetic-but-fixed so the cost arithmetic is exercised end-to-end. + --live : uses host.llm through the tier ladder. Only run where `host` is available. + +Honesty note baked into the output: the task set is a 30-item, hand-labeled DEMONSTRATION +set that the rubric thresholds were tuned against. High accuracy here shows the rubric +*can separate* the cases; it is not a benchmark and does not estimate field accuracy. +""" +from __future__ import annotations +import argparse, json, sys, os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from router_gate import route, assess, run, always_premium_cost +from router_gate.pricing import DEFAULT_LADDER, ladder_from_available +from router_gate import taskset + + +def stub_executor(task, model_id): + """Deterministic offline executor. Token counts scale with task length so the + cost accounting is realistic in shape without calling any model.""" + itok = max(20, len(task) // 3) + otok = max(30, len(task) // 2) # pretend the answer is ~1.5x the prompt + return (f"[stub answer from {model_id}]", itok, otok) + + +def make_live_executor(host): + from router_gate import host_llm_executor + return host_llm_executor(host) + + +def evaluate(executor, ladder, ask_threshold=0.6): + tasks = taskset.load() + rows = [] + # gate confusion + tp = fp = tn = fn = 0 + # routing (only on well-specified tasks, exact + within-1) + route_exact = route_within1 = route_total = 0 + TIER_IDX = {"cheap": 0, "mid": 1, "premium": 2} + + # cost accounting across the WHOLE realistic workload: + # under-specified tasks are halted (0 tokens), well-specified tasks are routed+run. + routed_cost = routed_tokens = 0.0 + baseline_cost = baseline_tokens = 0.0 + halted = 0 + + for x in tasks: + t, gold_tier, gold_ask = x["task"], x["gold_tier"], x["gold_ask"] + res = run(t, executor, ladder=ladder, ask_threshold=ask_threshold) + + # gate scoring + asked = res.halted_for_questions + if gold_ask and asked: tp += 1 + elif gold_ask and not asked: fn += 1 + elif (not gold_ask) and asked: fp += 1 + else: tn += 1 + + if asked: + halted += 1 + else: + # routing scoring (well-specified only, i.e. gold_ask == False expected) + if not gold_ask: + route_total += 1 + if res.final_tier == gold_tier: + route_exact += 1 + if abs(TIER_IDX[res.final_tier] - TIER_IDX[gold_tier]) <= 1: + route_within1 += 1 + # cost: routed vs always-premium on the SAME measured tokens + routed_cost += res.total_cost + routed_tokens += res.total_tokens + baseline_cost += always_premium_cost(res, ladder) + baseline_tokens += res.total_tokens # tokens same; premium just costs more per tok + + rows.append({ + "task": t[:70], "gold_tier": gold_tier, "gold_ask": gold_ask, + "asked": asked, "routed_tier": res.final_tier, + "completeness": round(res.assumption.completeness, 2), + "score": round(res.routing.score, 1) if res.routing else None, + "cost_usd": round(res.total_cost, 6), + "questions": res.assumption.questions if asked else [], + }) + + n = len(tasks) + gate_acc = (tp + tn) / n + gate_prec = tp / (tp + fp) if (tp + fp) else 0.0 + gate_rec = tp / (tp + fn) if (tp + fn) else 0.0 + gate_f1 = (2 * gate_prec * gate_rec / (gate_prec + gate_rec)) if (gate_prec + gate_rec) else 0.0 + + summary = { + "n_tasks": n, + "gate": { + "accuracy": round(gate_acc, 3), + "precision": round(gate_prec, 3), + "recall": round(gate_rec, 3), + "f1": round(gate_f1, 3), + "confusion": {"tp": tp, "fp": fp, "tn": tn, "fn": fn}, + "n_halted": halted, + }, + "routing": { + "n_scored": route_total, + "exact_accuracy": round(route_exact / route_total, 3) if route_total else None, + "within_1_tier_accuracy": round(route_within1 / route_total, 3) if route_total else None, + }, + "cost": { + "routed_cost_usd": round(routed_cost, 6), + "always_premium_cost_usd": round(baseline_cost, 6), + "cost_saved_usd": round(baseline_cost - routed_cost, 6), + "cost_saved_pct": round(100 * (baseline_cost - routed_cost) / baseline_cost, 1) if baseline_cost else 0.0, + "note": "Same measured tokens repriced at the premium tier. Savings come purely from routing cheaper-capable tasks to cheaper tiers.", + }, + "honesty": ( + "30-task hand-labeled DEMONSTRATION set; rubric thresholds were tuned against it. " + "Accuracy shows the rubric can separate the cases, not a field benchmark. " + "Cost figures are arithmetic on measured token counts and approximate public list prices." + ), + } + return summary, rows + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--live", action="store_true", help="use host.llm instead of the offline stub") + ap.add_argument("--out", default="eval_results.json") + args = ap.parse_args() + + ladder = DEFAULT_LADDER + if args.live: + # `host` is injected in the Claude Science kernel; import lazily + import builtins + host = getattr(builtins, "host", None) + if host is None: + print("live mode needs host.llm; falling back to offline"); args.live = False + if args.live: + ladder = ladder_from_available(host.list_models()) + executor = make_live_executor(host) + else: + executor = stub_executor + + summary, rows = evaluate(executor, ladder) + out = {"summary": summary, "rows": rows, "mode": "live" if args.live else "offline"} + with open(args.out, "w") as f: + json.dump(out, f, indent=2) + print(json.dumps(summary, indent=2)) + print(f"\nwrote {args.out} ({len(rows)} rows, mode={out['mode']})") + + +if __name__ == "__main__": + main() diff --git a/research/python-prototypes/router_gate/pyproject.toml b/research/python-prototypes/router_gate/pyproject.toml new file mode 100644 index 0000000..3419b22 --- /dev/null +++ b/research/python-prototypes/router_gate/pyproject.toml @@ -0,0 +1,30 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "router-gate" +version = "0.2.0" +description = "Production-ready complexity router and assumption gate for agentic coding workflows" +readme = "README.md" +requires-python = ">=3.9" +authors = [{ name = "router_gate authors" }] +license = { text = "MIT" } +classifiers = [ + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Topic :: Software Development :: Quality Assurance", +] +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=7.0"] + +[project.scripts] +router-gate = "router_gate.cli:main" +router-gate-mcp = "router_gate.mcp_server:main" + +[tool.setuptools.packages.find] +include = ["router_gate*"] diff --git a/research/python-prototypes/router_gate/requirements.txt b/research/python-prototypes/router_gate/requirements.txt new file mode 100644 index 0000000..b197d32 --- /dev/null +++ b/research/python-prototypes/router_gate/requirements.txt @@ -0,0 +1 @@ +pytest>=7.0 diff --git a/research/python-prototypes/router_gate/router-gate.example.toml b/research/python-prototypes/router_gate/router-gate.example.toml new file mode 100644 index 0000000..16a4662 --- /dev/null +++ b/research/python-prototypes/router_gate/router-gate.example.toml @@ -0,0 +1,19 @@ +ask_threshold = 0.6 +allow_escalation = true +executor_command = "./my-agent-call --model {model}" +executor_timeout_seconds = 120 + +[tiers.cheap] +model_id = "claude-haiku-4-5-20251001" +usd_in_per_mtok = 1.0 +usd_out_per_mtok = 5.0 + +[tiers.mid] +model_id = "claude-sonnet-5" +usd_in_per_mtok = 3.0 +usd_out_per_mtok = 15.0 + +[tiers.premium] +model_id = "claude-opus-4-8" +usd_in_per_mtok = 15.0 +usd_out_per_mtok = 75.0 diff --git a/research/python-prototypes/router_gate/router_gate/__init__.py b/research/python-prototypes/router_gate/router_gate/__init__.py new file mode 100644 index 0000000..c07e9f8 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/__init__.py @@ -0,0 +1,25 @@ +"""router_gate -- a complexity-aware model router + assumption/uncertainty gate. + +Two mechanisms from 'A Cognitive Substrate for Coding Agents' (Theory -> Evidence -> +Build-Map edition), made runnable: + + M1 Complexity-aware routing (router.py) -- don't pay premium rates for trivial tasks. + M2 Assumption/uncertainty gate (gate.py) -- halt and ask instead of confabulating. + +Composed by pipeline.run(): gate -> route -> execute -> verify -> escalate. +""" +from .router import route, RoutingDecision, extract_signals, score_complexity, band +from .gate import assess, AssumptionReport +from .pipeline import run, PipelineResult, host_llm_executor, always_premium_cost +from .pricing import DEFAULT_LADDER, ladder_from_available, Tier +from .config import RouterGateConfig, load_config +from .executors import external_command_executor + +__all__ = [ + "route", "RoutingDecision", "extract_signals", "score_complexity", "band", + "assess", "AssumptionReport", + "run", "PipelineResult", "host_llm_executor", "always_premium_cost", + "DEFAULT_LADDER", "ladder_from_available", "Tier", + "RouterGateConfig", "load_config", "external_command_executor", +] +__version__ = "0.2.0" diff --git a/research/python-prototypes/router_gate/router_gate/cli.py b/research/python-prototypes/router_gate/router_gate/cli.py new file mode 100644 index 0000000..b37e4b2 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/cli.py @@ -0,0 +1,141 @@ +"""Command-line interface for production router_gate workflows.""" +from __future__ import annotations + +import argparse +import json +import sys +from typing import Optional + +from .config import load_config, serialize_ladder +from .executors import external_command_executor +from .gate import assess +from .pipeline import run +from .router import route +from .serialization import result_to_dict, to_jsonable + + +def _read_task(args) -> str: + if args.task is not None: + return args.task + if args.file: + with open(args.file, "r", encoding="utf-8") as handle: + return handle.read() + return sys.stdin.read() + + +def _print(data: dict, *, pretty: bool) -> None: + if pretty: + print(json.dumps(data, indent=2, sort_keys=True)) + else: + print(json.dumps(data, separators=(",", ":"), sort_keys=True)) + + +def _add_task_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("task", nargs="?", help="task text; if omitted, stdin is used") + parser.add_argument("--file", "-f", help="read task text from a file") + + +def _add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--config", help="TOML config file") + parser.add_argument("--pretty", action="store_true", help="pretty-print JSON") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="router-gate", description="Assumption gate + complexity router for agentic coding workflows") + _add_common_args(parser) + sub = parser.add_subparsers(dest="command", required=True) + + p_assess = sub.add_parser("assess", help="score specification completeness and clarifying questions") + _add_task_args(p_assess) + + p_route = sub.add_parser("route", help="score task complexity and choose cheap/mid/premium") + _add_task_args(p_route) + + p_decide = sub.add_parser("decide", help="run gate and route without executing a model") + _add_task_args(p_decide) + + p_run = sub.add_parser("run", help="gate, route, execute an external command, verify only by command success") + _add_task_args(p_run) + p_run.add_argument("--executor-command", help="command to call; task is stdin, model in ROUTER_GATE_MODEL_ID; supports {model}") + p_run.add_argument("--timeout", type=float, help="executor timeout in seconds") + p_run.add_argument("--no-escalation", action="store_true", help="disable tier escalation") + + p_config = sub.add_parser("config", help="show effective runtime configuration") + return parser + + +def command_assess(args) -> dict: + config = load_config(args.config) + report = assess(_read_task(args), ask_threshold=config.ask_threshold) + return {"ok": True, "assumption": to_jsonable(report)} + + +def command_route(args) -> dict: + decision = route(_read_task(args)) + return {"ok": True, "routing": to_jsonable(decision)} + + +def command_decide(args) -> dict: + config = load_config(args.config) + task = _read_task(args) + report = assess(task, ask_threshold=config.ask_threshold) + payload = {"ok": True, "assumption": to_jsonable(report), "halted_for_questions": report.should_ask} + if not report.should_ask: + payload["routing"] = to_jsonable(route(task)) + return payload + + +def command_run(args) -> dict: + config = load_config(args.config) + command = args.executor_command or config.executor_command + if not command: + return {"ok": False, "error": "missing executor command; pass --executor-command or ROUTER_GATE_EXECUTOR_COMMAND"} + timeout = args.timeout if args.timeout is not None else config.executor_timeout_seconds + executor = external_command_executor(command, timeout_seconds=timeout) + try: + result = run( + _read_task(args), + executor, + ladder=list(config.ladder), + ask_threshold=config.ask_threshold, + allow_escalation=(not args.no_escalation and config.allow_escalation), + ) + except Exception as exc: # keep CLI contract JSON-safe for agents + return {"ok": False, "error": str(exc), "type": exc.__class__.__name__} + return {"ok": True, "result": result_to_dict(result)} + + +def command_config(args) -> dict: + config = load_config(args.config) + return { + "ok": True, + "config": { + "ask_threshold": config.ask_threshold, + "allow_escalation": config.allow_escalation, + "executor_command": config.executor_command, + "executor_timeout_seconds": config.executor_timeout_seconds, + "ladder": serialize_ladder(config.ladder), + }, + } + + +def main(argv: Optional[list[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + handlers = { + "assess": command_assess, + "route": command_route, + "decide": command_decide, + "run": command_run, + "config": command_config, + } + try: + payload = handlers[args.command](args) + except Exception as exc: + payload = {"ok": False, "error": str(exc), "type": exc.__class__.__name__} + _print(payload, pretty=args.pretty) + return 0 if payload.get("ok") else 1 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/research/python-prototypes/router_gate/router_gate/config.py b/research/python-prototypes/router_gate/router_gate/config.py new file mode 100644 index 0000000..d91dcae --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/config.py @@ -0,0 +1,137 @@ +"""Configuration loading for production router_gate usage. + +The package is intentionally dependency-free. Configuration is accepted from: + 1. explicit CLI flags / caller-provided values, + 2. a TOML file, and + 3. environment variables. + +Python 3.11+ uses stdlib ``tomllib`` for TOML. On older Python versions, callers can +still use environment variables or pass values directly without installing anything. +""" +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +from typing import Iterable, Optional + +from .pricing import DEFAULT_LADDER, Tier + +try: # pragma: no cover - depends on Python minor version + import tomllib # type: ignore[attr-defined] +except ModuleNotFoundError: # pragma: no cover + tomllib = None # type: ignore[assignment] + + +@dataclass(frozen=True) +class RouterGateConfig: + """Runtime configuration for the gate/router.""" + + ask_threshold: float = 0.6 + allow_escalation: bool = True + ladder: tuple[Tier, ...] = tuple(DEFAULT_LADDER) + executor_command: Optional[str] = None + executor_timeout_seconds: float = 120.0 + + +def _env_bool(name: str, default: bool) -> bool: + value = os.getenv(name) + if value is None: + return default + return value.strip().lower() in {"1", "true", "yes", "y", "on"} + + +def _env_float(name: str, default: float) -> float: + value = os.getenv(name) + if value is None or value.strip() == "": + return default + try: + return float(value) + except ValueError as exc: + raise ValueError(f"{name} must be a number, got {value!r}") from exc + + +def _tier_from_mapping(name: str, data: dict, fallback: Tier) -> Tier: + model_id = str(data.get("model_id", fallback.model_id)) + input_price = float(data.get("usd_in_per_mtok", fallback.usd_in_per_mtok)) + output_price = float(data.get("usd_out_per_mtok", fallback.usd_out_per_mtok)) + return Tier(name, model_id, input_price, output_price) + + +def _ladder_from_mapping(data: dict) -> tuple[Tier, ...]: + configured = data.get("tiers", {}) if isinstance(data, dict) else {} + if not isinstance(configured, dict): + raise ValueError("config field 'tiers' must be a table/object") + + fallback_by_name = {tier.name: tier for tier in DEFAULT_LADDER} + tiers = [] + for name in ("cheap", "mid", "premium"): + raw = configured.get(name, {}) + if raw is None: + raw = {} + if not isinstance(raw, dict): + raise ValueError(f"config field 'tiers.{name}' must be a table/object") + tiers.append(_tier_from_mapping(name, raw, fallback_by_name[name])) + return tuple(tiers) + + +def load_config(path: Optional[str | os.PathLike[str]] = None) -> RouterGateConfig: + """Load config from TOML plus environment overrides. + + Environment variables: + ROUTER_GATE_CONFIG + ROUTER_GATE_ASK_THRESHOLD + ROUTER_GATE_ALLOW_ESCALATION + ROUTER_GATE_EXECUTOR_COMMAND + ROUTER_GATE_EXECUTOR_TIMEOUT_SECONDS + """ + raw_path = str(path or os.getenv("ROUTER_GATE_CONFIG", "")).strip() + data: dict = {} + if raw_path: + config_path = Path(raw_path).expanduser() + if tomllib is None: + raise RuntimeError("TOML config files require Python 3.11+; use env vars instead") + with config_path.open("rb") as handle: + loaded = tomllib.load(handle) + if not isinstance(loaded, dict): + raise ValueError("config file must contain a TOML table") + data = loaded + + ask_threshold = float(data.get("ask_threshold", 0.6)) + allow_escalation = bool(data.get("allow_escalation", True)) + executor_command = data.get("executor_command") + if executor_command is not None: + executor_command = str(executor_command) + executor_timeout = float(data.get("executor_timeout_seconds", 120.0)) + ladder = _ladder_from_mapping(data) + + ask_threshold = _env_float("ROUTER_GATE_ASK_THRESHOLD", ask_threshold) + allow_escalation = _env_bool("ROUTER_GATE_ALLOW_ESCALATION", allow_escalation) + executor_command = os.getenv("ROUTER_GATE_EXECUTOR_COMMAND", executor_command) + executor_timeout = _env_float("ROUTER_GATE_EXECUTOR_TIMEOUT_SECONDS", executor_timeout) + + if not 0.0 <= ask_threshold <= 1.0: + raise ValueError("ask_threshold must be between 0 and 1") + if executor_timeout <= 0: + raise ValueError("executor_timeout_seconds must be positive") + + return RouterGateConfig( + ask_threshold=ask_threshold, + allow_escalation=allow_escalation, + ladder=ladder, + executor_command=executor_command, + executor_timeout_seconds=executor_timeout, + ) + + +def serialize_ladder(ladder: Iterable[Tier]) -> list[dict]: + """Return JSON-safe tier configuration.""" + return [ + { + "name": tier.name, + "model_id": tier.model_id, + "usd_in_per_mtok": tier.usd_in_per_mtok, + "usd_out_per_mtok": tier.usd_out_per_mtok, + } + for tier in ladder + ] diff --git a/research/python-prototypes/router_gate/router_gate/executors.py b/research/python-prototypes/router_gate/router_gate/executors.py new file mode 100644 index 0000000..3d2f634 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/executors.py @@ -0,0 +1,121 @@ +"""Production executor adapters. + +The core pipeline only needs a small callable: ``(task, model_id) -> +(text, input_tokens, output_tokens)``. This module provides hardened adapters for +real agent runtimes while keeping the package dependency-free. +""" +from __future__ import annotations + +import json +import os +import shlex +import subprocess +from dataclasses import dataclass +from typing import Optional + + +@dataclass(frozen=True) +class ExecutorError(RuntimeError): + """Raised when an external executor cannot complete a model call.""" + + message: str + returncode: Optional[int] = None + stderr: str = "" + + def __str__(self) -> str: + parts = [self.message] + if self.returncode is not None: + parts.append(f"returncode={self.returncode}") + if self.stderr: + parts.append(f"stderr={self.stderr[:500]}") + return " | ".join(parts) + + +def approximate_tokens(text: str) -> int: + """Small dependency-free token estimate for executors that do not report usage.""" + return max(1, len(text or "") // 4) + + +def parse_executor_output(stdout: str, task: str) -> tuple[str, int, int]: + """Parse executor stdout. + + Preferred stdout is JSON: + {"text": "...", "input_tokens": 12, "output_tokens": 34} + + Plain text is also accepted; token usage is estimated so cost accounting remains + conservative and never crashes the routing loop. + """ + raw = stdout or "" + stripped = raw.strip() + if not stripped: + return "", approximate_tokens(task), 0 + + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + return raw, approximate_tokens(task), approximate_tokens(raw) + + if not isinstance(parsed, dict): + return raw, approximate_tokens(task), approximate_tokens(raw) + + text = parsed.get("text", parsed.get("output", "")) + if text is None: + text = "" + text = str(text) + usage = parsed.get("usage", {}) if isinstance(parsed.get("usage", {}), dict) else {} + input_tokens = parsed.get("input_tokens", usage.get("input_tokens")) + output_tokens = parsed.get("output_tokens", usage.get("output_tokens")) + + try: + input_count = int(input_tokens) if input_tokens is not None else approximate_tokens(task) + output_count = int(output_tokens) if output_tokens is not None else approximate_tokens(text) + except (TypeError, ValueError): + input_count = approximate_tokens(task) + output_count = approximate_tokens(text) + return text, max(0, input_count), max(0, output_count) + + +def external_command_executor(command: str, *, timeout_seconds: float = 120.0, cwd: Optional[str] = None): + """Create an executor backed by an arbitrary command. + + The task is sent on stdin. ``ROUTER_GATE_MODEL_ID`` is set in the environment. + If the command contains ``{model}``, it is replaced with a shell-quoted model id. + + This makes router_gate usable with Claude Code wrappers, custom agent CLIs, + internal gateways, or any local script without linking a provider SDK. + """ + if not command or not command.strip(): + raise ValueError("executor command cannot be empty") + if timeout_seconds <= 0: + raise ValueError("timeout_seconds must be positive") + + def _exec(task: str, model_id: str) -> tuple[str, int, int]: + env = os.environ.copy() + env["ROUTER_GATE_MODEL_ID"] = model_id + rendered = command.replace("{model}", shlex.quote(model_id)) + try: + completed = subprocess.run( + rendered, + input=task or "", + text=True, + capture_output=True, + shell=True, + cwd=cwd, + env=env, + timeout=timeout_seconds, + check=False, + ) + except subprocess.TimeoutExpired as exc: + raise ExecutorError(f"executor timed out after {timeout_seconds}s", stderr=str(exc)) from exc + except OSError as exc: + raise ExecutorError(f"executor failed to start: {exc}") from exc + + if completed.returncode != 0: + raise ExecutorError( + "executor exited with failure", + returncode=completed.returncode, + stderr=completed.stderr or "", + ) + return parse_executor_output(completed.stdout, task or "") + + return _exec diff --git a/research/python-prototypes/router_gate/router_gate/gate.py b/research/python-prototypes/router_gate/router_gate/gate.py new file mode 100644 index 0000000..44d6e25 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/gate.py @@ -0,0 +1,209 @@ +"""Assumption / uncertainty gate (mechanism M2). + +The root cause the user named: "The biggest problem is 'Assumption'. If it doesn't +have enough context it will assume many things." A frozen LLM maximizes P(next token +| context); when the context under-determines the task, the least-improbable +continuation is still *a* continuation, so the model fills the gap with a confident +guess rather than stopping to ask. There is no built-in signal that says "I am now +extrapolating past what you told me." + +This gate supplies that missing signal. Before a task is executed, it scores the +request's SPECIFICATION COMPLETENESS across named dimensions, detects the specific +gaps, and -- when the request is under-specified past a threshold -- HALTS and emits +the concrete clarifying questions to ask. Asking one question is far cheaper than the +"almost right, but not quite" rework loop (field evidence: 66% of developers hit +that, and debugging the wrong-assumption output costs more than writing it). + +Like the router, the gate is TRANSPARENT: every point of missing-spec is attributed +to a named, auditable dimension. It does not outsource the judgment to another opaque +model call. This is the paper's verify-before-acting principle (Qur'an 49:6, tabayyun: +"if a source brings you news, verify it") turned into a control-flow gate. +""" +from __future__ import annotations +from dataclasses import dataclass, field +import re + + +# Each dimension is a facet of "did you actually tell me enough to not guess?" +# A dimension is UNMET if none of its cue patterns appear AND the task plausibly +# needs it. We keep this conservative: only flag gaps that would force a real +# assumption, not stylistic vagueness. + +@dataclass +class GapDimension: + key: str + description: str + question: str # what to ask the human if this is missing + cues: object # compiled regex of phrases that would SATISFY this dimension + applies: object # compiled regex: task looks like it NEEDS this dimension + + +def _c(p): + return re.compile(p, re.I) + + +# Dimensions carry a `blocking` flag: a missing BLOCKING dimension forces a real +# assumption (you literally cannot proceed without guessing). Non-blocking gaps +# lower the completeness score but do not, alone, trigger a halt. +DIMENSIONS = [ + GapDimension( + "inputs_outputs", + "the exact input/output behavior (an example or expected return)", + "What are the exact input and output (types, an example, or the expected return value)?", + _c(r"(->|=>|:\s*\S)|\b(input|output|returns?|given|example|e\.g\.|for example|" + r"such as|format|json|csv|schema|signature|expected)\b|```"), + _c(r"\b(function|api|endpoint|parse|convert|transform|read|write|generate|compute|return|implement)\b"), + ), + GapDimension( + "target_scope", + "which file / module / component to change is identified", + "Which specific file, module, or component should this change touch?", + _c(r"\b(file|module|class|function|component|path|directory|service|layer|endpoint)\b|`[\w./]+`|\w+\.\w+"), + # only applies to CHANGE tasks (fix/edit/refactor existing code), not fresh writes + _c(r"\b(fix|change|edit|update|modify|refactor|add to|remove from|integrate|wire|the (bug|issue|error))\b"), + ), + GapDimension( + "success_criteria", + "how 'done' will be judged (a test, criterion, or reference behavior)", + "How will we know it's correct -- a test, an acceptance criterion, or a reference behavior?", + _c(r"(->|=>)|\b(test|passes|acceptance|criteri|expected|should (return|equal|match|be)|" + r"verif|assert|benchmark|correct when|e\.g\.|example)\b|```"), + _c(r"\b(fix|optimi[sz]e|make it (faster|work|better)|improve|ensure|feature|behavior)\b"), + ), + GapDimension( + "constraints", + "hard constraints (perf, deps, style, compatibility)", + "Are there constraints I must respect (performance, allowed dependencies, style, backward-compat)?", + _c(r"\b(must|should|constraint|limit|no (new )?dependenc|only use|standard library|" + r"without|performance|latency|O\(|backward|compatib|convention|style)\b"), + # only applies to larger build/design tasks + _c(r"\b(design|architect|production|scal|migrate|distributed|concurren|refactor)\b"), + ), +] + +# which dimensions, if missing, actually FORCE an assumption (=> can trigger a halt) +_BLOCKING = {"inputs_outputs", "target_scope", "success_criteria"} + +# Words that themselves SIGNAL under-specification (raise the assumption risk). +_VAGUE = _c(r"\b(some|somehow|etc|and so on|things?|stuff|appropriate(ly)?|as needed|" + r"handle (it|everything)|make it (work|better|nice|good)|clean it up|" + r"cleaner|the usual|standard way|properly|correctly|the way we " + r"(discussed|talked)|like before|as before)\b") +# A concrete example lowers risk sharply. +_HAS_EXAMPLE = _c(r"\b(e\.g\.|for example|such as|like this|input:|output:)\b|```") + +# CONCRETENESS ANCHORS -- any of these means the request carries actionable detail: +# a code fence, an arrow example, a call signature f(x, y), a quoted literal, a +# filename.ext, a numeric example, or an explicit example marker. +_ANCHORS = [ + _c(r"```"), # code block + _c(r"->|=>"), # example / mapping arrow + _c(r"\b\w+\([^)]*\)"), # call signature: is_prime(n), put(key, value) + _c(r"'[^']+'|\"[^\"]+\""), # quoted literal + _c(r"\b\w+\.\w{1,4}\b"), # filename.ext (e.g. utils.py) -- short ext + _c(r"\b\d+\b.*(->|=>|=|:)|\(\d"), # a number tied to an example + _c(r"\b(e\.g\.|for example|such as|example:)\b"), +] +# Named tech/algorithm specifics raise information content (a real spec, even in prose). +_SPECIFIC = _c(r"\b(python|javascript|typescript|java|rust|golang|react|django|flask|node|redis|sql|postgres|asyncio|dijkstra|lru|adjacency|owasp|regex)\b|token-?bucket|binary heap|condition[- ]variable|recursive-?descent|in-?order|standard library|o\(\s*\d|o\(n|o\(1") + + +def _concreteness(t: str) -> int: + return sum(1 for a in _ANCHORS if a.search(t)) + +def _specificity(t: str) -> int: + return len(set(m.group(0).lower() for m in _SPECIFIC.finditer(t))) + + +@dataclass +class AssumptionReport: + completeness: float # 0..1, higher = better specified + risk: str # "low" | "medium" | "high" + should_ask: bool # HALT and ask before executing? + missing: list = field(default_factory=list) # [(key, description)] + questions: list = field(default_factory=list) # concrete questions to ask + reasons: list = field(default_factory=list) # attributions + + def explain(self) -> str: + head = (f"completeness={self.completeness:.2f} risk={self.risk} " + f"should_ask={self.should_ask}") + body = "\n".join(f" - {r}" for r in self.reasons) + qs = "\n".join(f" ? {q}" for q in self.questions) + return head + ("\n" + body if body else "") + (("\nQuestions to ask:\n" + qs) if qs else "") + + +def assess(task: str, *, ask_threshold: float = 0.6) -> AssumptionReport: + """Score specification completeness and decide whether to halt-and-ask. + + ask_threshold: if completeness < threshold, the gate recommends asking rather + than executing. 0.6 is deliberately cautious for coding tasks, where a wrong + assumption is expensive; callers can loosen it for low-stakes work. + """ + t = task or "" + reasons, missing, questions = [], [], [] + words = len(t.split()) + + # --- information-content measures (the decision driver) --- + conc = _concreteness(t) + spec = _specificity(t) + vague_hits = len(set(m.group(0).lower() for m in _VAGUE.finditer(t))) + + # completeness in [0,1]: earned by concrete anchors and named specifics, + # eroded by vague fillers and extreme brevity. + base = 0.45 + base += min(0.45, 0.18 * conc) + base += min(0.20, 0.06 * spec) + base -= 0.22 * vague_hits + if words <= 7: + base -= 0.22 + reasons.append("very short request (little information)") + # A long, detailed request carries information in its prose even without a code + # example: many words + several named specifics is a real spec, not a guess-magnet. + if words >= 30 and spec >= 2: + base += 0.20 + reasons.append(f"long detailed spec ({words} words, {spec} named specifics)") + elif words >= 22 and spec >= 1: + base += 0.10 + reasons.append(f"detailed spec ({words} words)") + if conc: + reasons.append(f"{conc} concrete anchor(s) (example/signature/literal)") + if spec: + reasons.append(f"{spec} named technical specific(s)") + if vague_hits: + reasons.append(f"{vague_hits} vague filler(s) that force interpretation") + + completeness = max(0.0, min(1.0, base)) + + # --- identify WHICH dimensions are missing, to generate good questions --- + blocking_missing = 0 + for d in DIMENSIONS: + if not d.applies.search(t): + continue + if not d.cues.search(t): + missing.append((d.key, d.description)) + is_blocking = d.key in _BLOCKING + if is_blocking: + blocking_missing += 1 + questions.append((0 if is_blocking else 1, d.question)) + + # --- the halt rule: proceed only if the request carries enough information --- + # Under-specified iff completeness is below threshold. Two decisive patterns: + # (a) short & no concrete anchor -> "Fix the bug." / "Optimize it." + # (b) vague filler & no anchor -> "handle everything appropriately" + hard_underspecified = (conc == 0) and (words <= 10 or (vague_hits >= 1 and spec == 0)) + should_ask = (completeness < ask_threshold) or hard_underspecified + risk = "high" if completeness < 0.45 else ("medium" if completeness < 0.7 else "low") + if hard_underspecified and not any("short" in r or "vague" in r for r in reasons): + reasons.append("no concrete anchor to act on") + + # order questions: blocking first, then the rest; cap at 3 + questions = [q for _, q in sorted(questions, key=lambda x: x[0])] + # if nothing dimension-specific surfaced but we're asking, give a general prompt + if should_ask and not questions: + questions = ["What exactly should this produce, and how will we know it's correct?"] + + # cap questions to the 3 most load-bearing to avoid an interrogation + return AssumptionReport( + completeness=completeness, risk=risk, should_ask=should_ask, + missing=missing, questions=questions[:3], reasons=reasons, + ) diff --git a/research/python-prototypes/router_gate/router_gate/mcp_server.py b/research/python-prototypes/router_gate/router_gate/mcp_server.py new file mode 100644 index 0000000..524827b --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/mcp_server.py @@ -0,0 +1,143 @@ +"""Minimal MCP-compatible stdio server for router_gate. + +This implements the JSON-RPC surface needed by MCP clients such as Claude Code: +initialize, tools/list, tools/call, notifications/initialized, and ping. It stays +stdlib-only so the package can be installed anywhere without native deps. +""" +from __future__ import annotations + +import json +import sys +from typing import Any, Optional + +from .config import load_config +from .gate import assess +from .router import route +from .serialization import to_jsonable + +SERVER_NAME = "router-gate" +SERVER_VERSION = "0.2.0" + + +TOOLS = [ + { + "name": "assess_task", + "description": "Score whether a coding task is sufficiently specified; returns risk, missing dimensions, and clarifying questions.", + "inputSchema": { + "type": "object", + "properties": { + "task": {"type": "string", "description": "Task/request text"}, + "ask_threshold": {"type": "number", "minimum": 0, "maximum": 1}, + }, + "required": ["task"], + "additionalProperties": False, + }, + }, + { + "name": "route_task", + "description": "Choose the cheapest capable model tier for a well-specified coding task and explain the decision.", + "inputSchema": { + "type": "object", + "properties": {"task": {"type": "string", "description": "Task/request text"}}, + "required": ["task"], + "additionalProperties": False, + }, + }, + { + "name": "decide_task", + "description": "Run the assumption gate; if safe to proceed, also return the routing tier.", + "inputSchema": { + "type": "object", + "properties": { + "task": {"type": "string", "description": "Task/request text"}, + "ask_threshold": {"type": "number", "minimum": 0, "maximum": 1}, + }, + "required": ["task"], + "additionalProperties": False, + }, + }, +] + + +def _content(payload: Any) -> dict: + return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, sort_keys=True)}]} + + +def _error(request_id: Any, code: int, message: str) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "error": {"code": code, "message": message}} + + +def _result(request_id: Any, result: Any) -> dict: + return {"jsonrpc": "2.0", "id": request_id, "result": result} + + +def _call_tool(name: str, arguments: dict) -> dict: + config = load_config() + task = arguments.get("task") + if not isinstance(task, str) or not task.strip(): + raise ValueError("argument 'task' must be a non-empty string") + threshold = float(arguments.get("ask_threshold", config.ask_threshold)) + if not 0 <= threshold <= 1: + raise ValueError("ask_threshold must be between 0 and 1") + + if name == "assess_task": + return _content({"assumption": to_jsonable(assess(task, ask_threshold=threshold))}) + if name == "route_task": + return _content({"routing": to_jsonable(route(task))}) + if name == "decide_task": + report = assess(task, ask_threshold=threshold) + payload = {"assumption": to_jsonable(report), "halted_for_questions": report.should_ask} + if not report.should_ask: + payload["routing"] = to_jsonable(route(task)) + return _content(payload) + raise ValueError(f"unknown tool: {name}") + + +def handle(message: dict) -> Optional[dict]: + method = message.get("method") + request_id = message.get("id") + + if method == "notifications/initialized": + return None + if method == "initialize": + return _result(request_id, { + "protocolVersion": message.get("params", {}).get("protocolVersion", "2024-11-05"), + "capabilities": {"tools": {}}, + "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION}, + }) + if method == "ping": + return _result(request_id, {}) + if method == "tools/list": + return _result(request_id, {"tools": TOOLS}) + if method == "tools/call": + params = message.get("params", {}) + if not isinstance(params, dict): + return _error(request_id, -32602, "params must be an object") + try: + return _result(request_id, _call_tool(str(params.get("name", "")), params.get("arguments", {}) or {})) + except Exception as exc: + return _error(request_id, -32000, str(exc)) + if request_id is None: + return None + return _error(request_id, -32601, f"method not found: {method}") + + +def main() -> int: + for line in sys.stdin: + if not line.strip(): + continue + try: + message = json.loads(line) + if not isinstance(message, dict): + raise ValueError("message must be a JSON object") + response = handle(message) + except Exception as exc: + response = _error(None, -32700, str(exc)) + if response is not None: + sys.stdout.write(json.dumps(response, separators=(",", ":")) + "\n") + sys.stdout.flush() + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/research/python-prototypes/router_gate/router_gate/pipeline.py b/research/python-prototypes/router_gate/router_gate/pipeline.py new file mode 100644 index 0000000..9f9a9a4 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/pipeline.py @@ -0,0 +1,133 @@ +"""The decision loop: gate -> route -> execute -> verify -> escalate. + +This composes the two mechanisms into one control flow that embodies the paper's +thesis at the level of a single request: + + 1. ASSUMPTION GATE (M2) - is the request specified enough to act on? If not, + HALT and return questions. (Do not guess.) + 2. ROUTE (M1) - score complexity, pick the cheapest capable tier. + 3. EXECUTE - run on that tier (pluggable executor; host.llm in prod, + a deterministic stub in tests). + 4. VERIFY (external) - check the output with a caller-supplied predicate + (a test, a compile, a regex). Trust is earned here, + not asserted by the model. + 5. ESCALATE - only if verification fails, retry one tier up. Bounded + by the tier ladder, so worst case is a small constant. + +Every call's measured token usage and computed cost is recorded, so the harness can +report REAL tokens/cost saved versus an always-premium baseline -- not an estimate. +""" +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Callable, Optional +from .router import route, next_tier, RoutingDecision +from .gate import assess, AssumptionReport +from .pricing import DEFAULT_LADDER, tier_by_name + + +@dataclass +class CallRecord: + tier: str + model_id: str + input_tokens: int + output_tokens: int + cost_usd: float + verified: bool + error: Optional[str] = None + + +@dataclass +class PipelineResult: + task: str + halted_for_questions: bool + assumption: AssumptionReport + routing: Optional[RoutingDecision] = None + output: Optional[str] = None + calls: list = field(default_factory=list) # list[CallRecord] + final_tier: Optional[str] = None + success: bool = False + + @property + def total_cost(self) -> float: + return sum(c.cost_usd for c in self.calls) + + @property + def total_tokens(self) -> int: + return sum(c.input_tokens + c.output_tokens for c in self.calls) + + +# An executor takes (task, model_id) and returns (text, input_tokens, output_tokens). +Executor = Callable[[str, str], tuple] +# A verifier takes (task, output_text) and returns True if the output passes. +Verifier = Callable[[str, str], bool] + + +def host_llm_executor(host): + """Production executor backed by host.llm, returning real measured token usage.""" + def _exec(task: str, model_id: str): + r = host.llm(task, model=model_id) + u = r.get("usage", {}) or {} + return (r.get("text", ""), + int(u.get("input_tokens", 0)), + int(u.get("output_tokens", 0))) + return _exec + + +def run(task: str, + executor: Executor, + *, + verifier: Optional[Verifier] = None, + ladder=None, + ask_threshold: float = 0.6, + allow_escalation: bool = True) -> PipelineResult: + """Run one task through the full gate->route->execute->verify->escalate loop.""" + ladder = ladder or DEFAULT_LADDER + + # --- 1. assumption gate --- + ar = assess(task, ask_threshold=ask_threshold) + if ar.should_ask: + # HALT: do not spend a single token confabulating past the missing spec. + return PipelineResult(task=task, halted_for_questions=True, assumption=ar, + success=False) + + # --- 2. route --- + rd = route(task) + res = PipelineResult(task=task, halted_for_questions=False, assumption=ar, routing=rd) + + # --- 3-5. execute, verify, escalate --- + tier_name = rd.tier + while tier_name is not None: + tier = tier_by_name(ladder, tier_name) + text = "" + itok = otok = 0 + error = None + try: + text, itok, otok = executor(task, tier.model_id) + ok = True if verifier is None else bool(verifier(task, text)) + except Exception as exc: + ok = False + error = f"{exc.__class__.__name__}: {exc}" + res.calls.append(CallRecord(tier.name, tier.model_id, itok, otok, + tier.cost(itok, otok), ok, error)) + res.output = text + res.final_tier = tier.name + if ok: + res.success = True + break + if not allow_escalation: + break + tier_name = next_tier(tier_name) # escalate one tier up, or None -> stop + + return res + + +def always_premium_cost(records_or_result, ladder=None) -> float: + """Counterfactual: what would this task have cost if every call went to premium? + + Uses the SAME measured token counts, repriced at the premium tier -- an honest + apples-to-apples baseline for 'tokens/cost saved by routing'. + """ + ladder = ladder or DEFAULT_LADDER + premium = ladder[-1] + calls = records_or_result.calls if isinstance(records_or_result, PipelineResult) else records_or_result + return sum(premium.cost(c.input_tokens, c.output_tokens) for c in calls) diff --git a/research/python-prototypes/router_gate/router_gate/pricing.py b/research/python-prototypes/router_gate/router_gate/pricing.py new file mode 100644 index 0000000..2349fb0 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/pricing.py @@ -0,0 +1,68 @@ +"""Model tiers and cost accounting. + +A frozen LLM is a probability engine priced by the token. The whole point of +complexity-aware routing is economic: do not pay premium-tier rates for a task a +cheap tier handles correctly. To reason about that honestly we need (a) a tier +ladder and (b) real per-token prices. + +Prices below are APPROXIMATE public list prices (USD per million tokens) for the +Anthropic 4.x generation, as of mid-2026. They are deliberately overridable: the +mechanism does not depend on the exact numbers, only on the ratio between tiers. +Every cost figure this package reports is arithmetic on measured token counts and +these constants -- never a guess. +""" +from __future__ import annotations +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Tier: + """One model tier: a name, the concrete model id, and its price.""" + name: str # "cheap" | "mid" | "premium" + model_id: str # concrete id passed to host.llm(model=...) + usd_in_per_mtok: float + usd_out_per_mtok: float + + def cost(self, input_tokens: int, output_tokens: int) -> float: + """USD cost of a call given measured token counts.""" + return (input_tokens / 1e6) * self.usd_in_per_mtok + \ + (output_tokens / 1e6) * self.usd_out_per_mtok + + +# The tier ladder. model_id values are resolved at runtime against host.list_models() +# (see ladder_from_available); these defaults match the 4.x generation seen in 2026. +DEFAULT_LADDER = [ + Tier("cheap", "claude-haiku-4-5-20251001", usd_in_per_mtok=1.0, usd_out_per_mtok=5.0), + Tier("mid", "claude-sonnet-5", usd_in_per_mtok=3.0, usd_out_per_mtok=15.0), + Tier("premium", "claude-opus-4-8", usd_in_per_mtok=15.0, usd_out_per_mtok=75.0), +] + +# Order of complexity bands, lowest first. The router emits one of these. +TIER_ORDER = ["cheap", "mid", "premium"] + + +def ladder_from_available(available_model_ids, ladder=None): + """Adapt the default ladder to whatever models actually exist in this session. + + Keeps each tier's price but swaps in a concrete model_id that is present in + `available_model_ids`, matching by family prefix (haiku/sonnet/opus). This makes + the prototype portable across sessions where exact ids differ, without pretending + a model exists that does not. + """ + ladder = ladder or DEFAULT_LADDER + fam = {"cheap": "haiku", "mid": "sonnet", "premium": "opus"} + ids = list(available_model_ids or []) + out = [] + for t in ladder: + want = fam[t.name] + # exact id first, else first available id whose name contains the family + pick = t.model_id if t.model_id in ids else next((m for m in ids if want in m), t.model_id) + out.append(Tier(t.name, pick, t.usd_in_per_mtok, t.usd_out_per_mtok)) + return out + + +def tier_by_name(ladder, name: str) -> Tier: + for t in ladder: + if t.name == name: + return t + raise KeyError(name) diff --git a/research/python-prototypes/router_gate/router_gate/router.py b/research/python-prototypes/router_gate/router_gate/router.py new file mode 100644 index 0000000..739d476 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/router.py @@ -0,0 +1,164 @@ +"""Complexity-aware model router (mechanism M1). + +The failure this attacks, in the user's words: "a simple prime number finder ... +if you use [a premium model] it will not give you extra [value]." A frozen LLM is a +probability engine billed per token; sending a trivial task to the premium tier is +pure waste of tokens, time, and money. The router scores a task's intrinsic +complexity and sends it to the CHEAPEST tier likely to succeed, escalating only on +evidence of failure. + +Design commitment (the user's core value: never blindly trust a probability engine): +the routing decision is a TRANSPARENT, feature-based rubric that explains itself. It +does not ask another opaque LLM "how hard is this?" -- that would just move the +untrusted probability one level up. Every point of the score is attributable to a +named feature, so a human can audit and override it. + +Escalation is the safety valve. Cheap-tier output is checked by an external verifier +(a test, a compile, a regex, or a caller-supplied predicate). Only if that check +FAILS do we spend a higher tier -- so the worst case is 'cheap attempt + premium +attempt', and the common case is 'cheap only'. This mirrors the paper's central +move: trust is earned by an EXTERNAL check, not asserted by the model. +""" +from __future__ import annotations +from dataclasses import dataclass, field +import re +from .pricing import DEFAULT_LADDER, TIER_ORDER, tier_by_name + + +# ---- complexity signals ----------------------------------------------------- +# Each signal contributes points to a 0..~12 raw score, then banded to a tier. +# Signals are intentionally simple and inspectable; the weights are the rubric. + +# HARD signals: genuine algorithmic / systems difficulty (premium-worthy). +_ALGO_TERMS = re.compile(r"\b(recursion|recursive|recursive-?descent|dynamic " + r"programming|dijkstra|a\*|concurren|thread-?safe|mutex|" + r"race condition|deadlock|distributed|consensus|parser|" + r"compiler|cryptograph|np-hard|state machine|invariant|" + r"numerical stability|back-?pressure|token[- ]bucket|" + r"rate limiter|idempoten|migration|producer|consumer|" + r"blocking queue|condition[- ]variable)", re.I) +# ARCHITECTURAL signals: design/whole-system scope (premium-worthy). +_ARCH_TERMS = re.compile(r"\b(architect|\bdesign\b|trade-?off|refactor a|migrate|" + r"scal(e|able|ing)|schema (migration|design)|api design|" + r"multi-?module|cross-?module|end-?to-?end|consistency " + r"(guarantee|trade)|locking strategy|module boundaries)", re.I) +# MODERATE signals: data-structure / class / library-level work (mid-worthy). +_MODERATE_TERMS = re.compile(r"\b(class\b|cache|lru|queue|stack|heap|linked list|" + r"binary tree|tree|traversal|graph|decorator|regex|" + r"debounce|throttle|merge|sort(ed|ing)?|parse|" + r"o\(\s*\d|o\(n|o\(1|thread|async|lock|validate|" + r"in-?order|adjacency)", re.I) +# TRIVIAL markers: one-liner tells (pull the score DOWN). +_TRIVIAL_TERMS = re.compile(r"\b(hello world|rename|typo|indent|add a comment|" + r"reverse a string|reverse the string|is[_ ]?even|" + r"is[_ ]?odd|factorial|fibonacci|is[_ ]?prime|prime\b|" + r"sum of|sum_list|capitalize|count vowels|celsius|" + r"fahrenheit|lower ?case|upper ?case)", re.I) +_MULTISTEP = re.compile(r"\b(and then|after that|first.*then|step \d|" + r"multiple|several|each of|for every)\b", re.I) +_CODEFENCE = re.compile(r"```") + + +@dataclass +class ComplexitySignals: + length_tokens: int + has_algorithmic_terms: bool + has_architectural_terms: bool + has_moderate_terms: bool + has_trivial_markers: bool + has_multistep: bool + has_code_context: bool + n_constraints: int # counted "must"/"should"/"ensure"/bullet lines + + +@dataclass +class RoutingDecision: + tier: str # "cheap" | "mid" | "premium" + score: float # raw complexity score + signals: ComplexitySignals + reasons: list = field(default_factory=list) # human-readable attributions + + def explain(self) -> str: + head = f"tier={self.tier} score={self.score:.1f}" + return head + "\n" + "\n".join(f" +{w:.1f} {why}" for w, why in self.reasons) + + +def _approx_tokens(text: str) -> int: + # cheap, dependency-free approximation (~4 chars/token) + return max(1, len(text) // 4) + + +def extract_signals(task: str) -> ComplexitySignals: + t = task or "" + n_constraints = len(re.findall(r"(?im)^\s*[-*\d.]|\b(must|should|ensure|require|constraint)\b", t)) + return ComplexitySignals( + length_tokens=_approx_tokens(t), + has_algorithmic_terms=bool(_ALGO_TERMS.search(t)), + has_architectural_terms=bool(_ARCH_TERMS.search(t)), + has_moderate_terms=bool(_MODERATE_TERMS.search(t)), + has_trivial_markers=bool(_TRIVIAL_TERMS.search(t)), + has_multistep=bool(_MULTISTEP.search(t)), + has_code_context=bool(_CODEFENCE.search(t)), + n_constraints=n_constraints, + ) + + +def score_complexity(sig: ComplexitySignals): + """Return (raw_score, reasons[]). Transparent additive rubric. + + Bands (see `band`): score < 3 -> cheap, 3..6 -> mid, > 6 -> premium. + Weights are the policy; every one is attributed in `reasons` for audit. + """ + reasons = [] + score = 1.5 + reasons.append((1.5, "base cost of any task")) + + # HARD signals dominate -> push to premium. + if sig.has_algorithmic_terms: + score += 4.0; reasons.append((4.0, "algorithmic/systems difficulty (concurrency, parsing, distributed, ...)")) + if sig.has_architectural_terms: + score += 4.0; reasons.append((4.0, "architectural/design scope (whole-system, migration, trade-offs)")) + + # MODERATE signals -> lift into the mid band. + if sig.has_moderate_terms: + score += 2.0; reasons.append((2.0, "data-structure/class/library-level work")) + + # secondary difficulty signals + if sig.has_multistep: + score += 1.0; reasons.append((1.0, "multi-step / sequenced request")) + if sig.has_code_context: + score += 1.0; reasons.append((1.0, "carries code context to reason over")) + if sig.length_tokens > 120: + score += 1.5; reasons.append((1.5, f"long spec (~{sig.length_tokens} tok)")) + elif sig.length_tokens > 55: + score += 0.7; reasons.append((0.7, f"medium spec (~{sig.length_tokens} tok)")) + if sig.n_constraints >= 5: + score += 1.0; reasons.append((1.0, f"{sig.n_constraints} explicit constraints to satisfy")) + + # trivial markers pull DOWN hard: a strong tell the task is a one-liner. + # (Suppressed when hard signals fire, so 'prime-factorization consensus' isn't miscounted.) + if sig.has_trivial_markers and not sig.has_algorithmic_terms and not sig.has_architectural_terms: + score -= 3.0; reasons.append((-3.0, "trivial-task marker (prime/factorial/reverse/rename)")) + + return max(0.0, score), reasons + + +def band(score: float) -> str: + """Map a raw complexity score to a tier band. Thresholds are the policy knob.""" + if score < 3.0: + return "cheap" + if score <= 6.0: + return "mid" + return "premium" + + +def route(task: str) -> RoutingDecision: + sig = extract_signals(task) + score, reasons = score_complexity(sig) + return RoutingDecision(tier=band(score), score=score, signals=sig, reasons=reasons) + + +def next_tier(tier: str): + """The next tier up for escalation, or None if already at premium.""" + i = TIER_ORDER.index(tier) + return TIER_ORDER[i + 1] if i + 1 < len(TIER_ORDER) else None diff --git a/research/python-prototypes/router_gate/router_gate/serialization.py b/research/python-prototypes/router_gate/router_gate/serialization.py new file mode 100644 index 0000000..9c2683a --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/serialization.py @@ -0,0 +1,22 @@ +"""JSON-safe serializers for router_gate public objects.""" +from __future__ import annotations + +from dataclasses import asdict, is_dataclass +from typing import Any + + +def to_jsonable(value: Any) -> Any: + if is_dataclass(value): + return to_jsonable(asdict(value)) + if isinstance(value, dict): + return {str(k): to_jsonable(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [to_jsonable(v) for v in value] + return value + + +def result_to_dict(result) -> dict: + data = to_jsonable(result) + data["total_cost"] = result.total_cost + data["total_tokens"] = result.total_tokens + return data diff --git a/research/python-prototypes/router_gate/router_gate/taskset.py b/research/python-prototypes/router_gate/router_gate/taskset.py new file mode 100644 index 0000000..ef6fd44 --- /dev/null +++ b/research/python-prototypes/router_gate/router_gate/taskset.py @@ -0,0 +1,59 @@ +"""A labeled task set for evaluating the router and the gate. + +Each task carries GOLD labels assigned by hand: + gold_tier : the cheapest tier a competent engineer would say is *sufficient* + ("cheap"|"mid"|"premium"). This is the routing target. + gold_ask : True if the request is under-specified enough that asking a + clarifying question is the correct action (the gate target). + +This is a DEMONSTRATION set (30 tasks), not a benchmark. It is deliberately small, +hand-labeled, and self-built -- exactly the honesty the paper demands. The gold +labels encode a defensible engineering judgment, not ground truth; they are stated +so a reader can disagree item by item. +""" + +# (task_text, gold_tier, gold_ask) +TASKS = [ + # ---- trivial, well-specified -> cheap, no ask ---- + ("Write a Python function is_even(n) that returns True if n is even. Example: is_even(4) -> True.", "cheap", False), + ("Reverse a string in Python. Input: 'abc' -> Output: 'cba'. Provide the function.", "cheap", False), + ("Write factorial(n) in Python returning n!. factorial(5) -> 120.", "cheap", False), + ("Capitalize the first letter of each word in a string. Input 'hello world' -> 'Hello World'.", "cheap", False), + ("Return the sum of a list of integers. sum_list([1,2,3]) -> 6. Python.", "cheap", False), + ("Write a function that checks if a number is prime. is_prime(7) -> True, is_prime(8) -> False. Python.", "cheap", False), + ("Convert a temperature in Celsius to Fahrenheit. c_to_f(0) -> 32.0. Python function.", "cheap", False), + ("Count vowels in a string. count_vowels('apple') -> 2. Python.", "cheap", False), + + # ---- moderate, well-specified -> mid, no ask ---- + ("Implement an LRU cache class in Python with get(key) and put(key, value), capacity fixed at construction, O(1) operations. Include a docstring.", "mid", False), + ("Parse a CSV string into a list of dicts using the header row as keys. Handle quoted fields containing commas. Python, standard library only.", "mid", False), + ("Write a function that merges two sorted integer lists into one sorted list without using sorted(). Return the merged list. Python.", "mid", False), + ("Implement debounce(fn, wait_ms) in JavaScript that delays calling fn until wait_ms has elapsed since the last call. Return the debounced function.", "mid", False), + ("Given a binary tree node class with .left/.right/.val, write an in-order traversal returning a list of values. Python.", "mid", False), + ("Validate an email address with a regex in Python and return True/False. Must reject 'a@b' and accept 'a@b.com'.", "mid", False), + ("Write a retry decorator in Python: retry(times=3, delay=0.1) that re-runs the wrapped function on exception up to `times`, then re-raises.", "mid", False), + + # ---- complex, well-specified -> premium, no ask ---- + ("Design and implement a thread-safe bounded blocking queue in Python supporting put() and get() from multiple producer and consumer threads, with proper condition-variable signaling and no busy-waiting. Include tests for the empty and full boundary conditions.", "premium", False), + ("Implement Dijkstra's shortest-path algorithm on a weighted directed graph given as an adjacency dict, returning the distance map and the predecessor map. Handle unreachable nodes. Optimize with a binary heap. Python.", "premium", False), + ("Refactor a synchronous data pipeline into an async architecture: design the module boundaries, define back-pressure between stages, ensure at-least-once processing under failure, and document the consistency trade-offs. Target Python asyncio.", "premium", False), + ("Write a recursive-descent parser for a small arithmetic grammar (numbers, + - * /, parentheses, correct precedence and associativity) that returns an AST, plus an evaluator. Include error handling for malformed input. Python.", "premium", False), + ("Implement a distributed rate limiter using a token-bucket algorithm backed by Redis, correct under concurrent access from multiple app instances, with configurable rate and burst. Explain the atomicity guarantees. Python.", "premium", False), + ("Design a schema migration system that applies and rolls back ordered migrations, records applied state, is idempotent on partial failure, and is safe to run concurrently from multiple deployers. Describe the locking strategy and implement the core. Python + SQL.", "premium", False), + + # ---- under-specified (any complexity) -> gold_ask = True ---- + ("Fix the bug.", "cheap", True), + ("Make the API faster.", "mid", True), + ("Add authentication to the app.", "premium", True), + ("Write a function to process the data and handle everything appropriately.", "mid", True), + ("Refactor this module to be cleaner.", "mid", True), + ("Build the reporting feature the way we discussed.", "premium", True), + ("Optimize it.", "mid", True), + ("Integrate the payment thing and make it work properly.", "premium", True), + ("Update the config as needed and clean it up.", "cheap", True), +] + + +def load(): + """Return the task set as a list of dicts.""" + return [{"task": t, "gold_tier": gt, "gold_ask": ga} for (t, gt, ga) in TASKS] diff --git a/research/python-prototypes/router_gate/tests/test_router_gate.py b/research/python-prototypes/router_gate/tests/test_router_gate.py new file mode 100644 index 0000000..09a34d5 --- /dev/null +++ b/research/python-prototypes/router_gate/tests/test_router_gate.py @@ -0,0 +1,207 @@ +"""Unit tests for the router, the gate, and the composed pipeline. + +Offline and deterministic: the executor is a stub, so no tokens are spent and the +suite runs in milliseconds. These check the MECHANISM's logic, not model quality. +""" +import pytest +from router_gate import route, assess, run, always_premium_cost +from router_gate.router import band, score_complexity, extract_signals, next_tier +from router_gate.gate import assess as gate_assess +from router_gate.pricing import DEFAULT_LADDER, ladder_from_available +from router_gate import taskset + + +# ---------------- router ---------------- + +def test_trivial_task_routes_cheap(): + d = route("Write a Python function is_prime(n). is_prime(7) -> True.") + assert d.tier == "cheap", d.explain() + +def test_complex_task_routes_premium(): + d = route("Design and implement a thread-safe bounded blocking queue with " + "condition-variable signaling, multiple producers and consumers, and " + "no busy-waiting. Include tests for empty and full boundaries.") + assert d.tier == "premium", d.explain() + +def test_moderate_task_routes_mid(): + d = route("Implement an LRU cache class with get and put, capacity fixed at " + "construction, O(1) operations, with a docstring.") + assert d.tier in ("mid", "premium"), d.explain() + +def test_routing_is_explainable(): + d = route("Reverse a string. 'abc' -> 'cba'.") + assert d.reasons and all(len(r) == 2 for r in d.reasons) + assert "tier=" in d.explain() + +def test_band_thresholds_monotone(): + assert band(0.0) == "cheap" + assert band(4.0) == "mid" + assert band(10.0) == "premium" + +def test_next_tier_escalation_path(): + assert next_tier("cheap") == "mid" + assert next_tier("mid") == "premium" + assert next_tier("premium") is None + + +# ---------------- gate ---------------- + +def test_underspecified_task_triggers_ask(): + r = assess("Fix the bug.") + assert r.should_ask is True + assert r.questions, "expected clarifying questions" + assert r.risk in ("high", "medium") + +def test_wellspecified_task_does_not_ask(): + r = assess("Write a Python function is_even(n) that returns True if n is even. " + "Example: is_even(4) -> True.") + assert r.should_ask is False, r.explain() + +def test_vague_words_lower_completeness(): + concrete = assess("Write add(a, b) in Python returning a+b. add(2,3) -> 5.") + vague = assess("Handle the data appropriately and make it work properly.") + assert vague.completeness < concrete.completeness + +def test_questions_are_capped(): + r = assess("Do the thing.") + assert len(r.questions) <= 3 + +def test_gate_is_explainable(): + r = assess("Optimize it.") + assert "completeness=" in r.explain() + + +# ---------------- pipeline ---------------- + +def _stub_executor(fixed_tokens=(100, 50)): + """Deterministic executor: returns canned text and fixed token counts.""" + def _exec(task, model_id): + return (f"[output for {model_id[:12]}]", fixed_tokens[0], fixed_tokens[1]) + return _exec + +def test_pipeline_halts_on_underspecified(): + res = run("Fix the bug.", _stub_executor()) + assert res.halted_for_questions is True + assert res.success is False + assert res.calls == [] # spent zero calls -- did not confabulate + assert res.assumption.questions + +def test_pipeline_runs_wellspecified_on_cheap(): + res = run("Write is_prime(n) in Python. is_prime(7) -> True, is_prime(8) -> False.", + _stub_executor()) + assert res.halted_for_questions is False + assert res.final_tier == "cheap" + assert res.success is True + assert len(res.calls) == 1 + +def test_pipeline_escalates_on_failed_verification(): + # verifier always fails on cheap+mid output, passes only at premium + def verifier(task, out): + return "opus" in out + res = run("Write a Python function is_even(n) that returns True for even n. " + "is_even(4) -> True, is_even(3) -> False.", _stub_executor(), verifier=verifier) + # should have climbed the ladder to premium + assert res.final_tier == "premium" + assert [c.tier for c in res.calls] == ["cheap", "mid", "premium"] + assert res.success is True + +def test_pipeline_no_escalation_when_disabled(): + def verifier(task, out): + return False + res = run("Write a Python function reverse(s) that reverses a string. " + "reverse('ab') -> 'ba'.", _stub_executor(), + verifier=verifier, allow_escalation=False) + assert len(res.calls) == 1 + assert res.success is False + +def test_cost_savings_vs_premium_baseline(): + res = run("Write factorial(n). factorial(5) -> 120.", _stub_executor()) + routed = res.total_cost + baseline = always_premium_cost(res) + # routed to cheap tier, so it must cost strictly less than the premium counterfactual + assert routed < baseline + assert routed > 0 and baseline > 0 + + +# ---------------- pricing ---------------- + +def test_ladder_adapts_to_available_models(): + avail = ["claude-haiku-4-5-20251001", "claude-sonnet-5", "claude-opus-4-8"] + lad = ladder_from_available(avail) + assert lad[0].model_id == "claude-haiku-4-5-20251001" + assert "sonnet" in lad[1].model_id + assert "opus" in lad[2].model_id + +def test_cost_is_arithmetic_on_tokens(): + t = DEFAULT_LADDER[0] + assert t.cost(1_000_000, 0) == pytest.approx(t.usd_in_per_mtok) + assert t.cost(0, 1_000_000) == pytest.approx(t.usd_out_per_mtok) + + +# ---------------- taskset sanity ---------------- + +def test_taskset_wellformed(): + ts = taskset.load() + assert len(ts) >= 25 + assert all(x["gold_tier"] in ("cheap", "mid", "premium") for x in ts) + assert any(x["gold_ask"] for x in ts) and any(not x["gold_ask"] for x in ts) + +# ---------------- production surfaces ---------------- + +def test_pipeline_records_executor_failure_and_escalates(): + attempts = [] + def flaky_executor(task, model_id): + attempts.append(model_id) + if len(attempts) == 1: + raise RuntimeError("temporary provider failure") + return ("ok", 10, 5) + + res = run("Write is_even(n) in Python. is_even(4) -> True.", flaky_executor) + assert res.success is True + assert len(res.calls) == 2 + assert res.calls[0].verified is False + assert "temporary provider failure" in res.calls[0].error + assert res.calls[1].verified is True + + +def test_external_command_executor_parses_json(tmp_path): + from router_gate.executors import external_command_executor + + script = tmp_path / "executor.py" + script.write_text( + "import json, os, sys\n" + "task = sys.stdin.read()\n" + "print(json.dumps({'text': os.environ['ROUTER_GATE_MODEL_ID'] + ':' + task, 'input_tokens': 3, 'output_tokens': 4}))\n" + ) + executor = external_command_executor(f"python {script}", timeout_seconds=5) + text, input_tokens, output_tokens = executor("hello", "test-model") + assert text == "test-model:hello" + assert input_tokens == 3 + assert output_tokens == 4 + + +def test_cli_decide_outputs_json(capsys): + from router_gate.cli import main + + code = main(["--pretty", "decide", "Fix the bug."]) + out = capsys.readouterr().out + payload = __import__("json").loads(out) + assert code == 0 + assert payload["ok"] is True + assert payload["halted_for_questions"] is True + assert payload["assumption"]["questions"] + + +def test_mcp_tools_call_decide_task(): + from router_gate.mcp_server import handle + + response = handle({ + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "decide_task", "arguments": {"task": "Fix the bug."}}, + }) + assert response["id"] == 1 + text = response["result"]["content"][0]["text"] + assert "halted_for_questions" in text + assert "questions" in text diff --git a/skills/cognitive-substrate/SKILL.md b/skills/cognitive-substrate/SKILL.md new file mode 100644 index 0000000..0fcfb96 --- /dev/null +++ b/skills/cognitive-substrate/SKILL.md @@ -0,0 +1,31 @@ +--- +name: cognitive-substrate +description: >- + Use before ambiguous, expensive, multi-file, or mutating coding work to run + Forge's cognitive substrate: assumption gate, model routing, impact + prediction, scope decomposition, memory/lesson lookup, minimality check, and + verification planning. Trigger when a user asks to edit/refactor/fix/design + production code, integrate features, choose model effort, inspect blast + radius, or avoid agent assumptions. +--- + +# Cognitive Substrate + +Before acting on non-trivial or mutating code tasks, run: + +```bash +forge substrate "" --json +``` + +Use the result this way: + +1. If `okToProceed` is false, ask the returned `assumption.questions` before editing. +2. Use `route.model` as the cheapest capable tier recommendation; do not escalate without a verifier failure. +3. Read `impact.impactedFiles` before editing a named symbol/file. +4. Use `scope.clusters` to split independent work into separate sessions. +5. Treat `memory.advisory` as context, not law; tests and human corrections override it. +6. Run the `verification.checklist` before claiming completion. + +For details, read `references/capability-map.md` only when you need to explain how the paper's faculties map to Forge commands. + +The full paper bundle is in `docs/cognitive-substrate/` when the repository is available: PDF, HTML, evidence map, ecosystem map, and original prototype packages. diff --git a/skills/cognitive-substrate/references/capability-map.md b/skills/cognitive-substrate/references/capability-map.md new file mode 100644 index 0000000..71c6511 --- /dev/null +++ b/skills/cognitive-substrate/references/capability-map.md @@ -0,0 +1,17 @@ +# Cognitive substrate capability map + +| Paper capability | Forge surface | Guarantee | +| --- | --- | --- | +| Memory | `forge recall`, `forge cortex` | File-backed facts/lessons are persisted and auditable. | +| Learning | `forge cortex` | External outcomes update lessons; model weights do not change. | +| Imagination | `forge impact`, `forge substrate` | Static graph simulates possible blast radius. | +| Self-correction | `forge verify`, doom-loop guard | Tests/builds beat model claims. | +| Impact-awareness | `forge atlas`, `forge impact` | Known symbols/files and likely dependents are surfaced. | +| M1 routing | `forge route` | Transparent model-tier recommendation. | +| M2 assumption gate | `forge preflight`, `forge substrate` | Under-specified tasks return questions. | +| M3 decomposition | `forge scope` | Import clusters show independent vs coupled files. | +| M4 goal anchoring | `forge substrate` | One pre-action summary anchors goal, risk, and verification. | +| M5 anti-over-engineering | `forge substrate`, `lean-guard` | Broad/underspecified work gets minimality warnings. | +| M6 inline verification | `forge verify` | External checks are required before done. | + +Limits: static graph edges are conservative; memory relevance and model routing are advisory; non-hook tools cannot be forcibly blocked. diff --git a/source/substrate.json b/source/substrate.json new file mode 100644 index 0000000..149d577 --- /dev/null +++ b/source/substrate.json @@ -0,0 +1,31 @@ +{ + "version": 1, + "summary": "Forge cognitive substrate wraps a frozen model with deterministic pre-action gates, repo state, memory, routing, impact simulation, and verification discipline.", + "faculties": [ + { "id": "memory", "forge": "recall + cortex", "status": "partial", "guarantee": "facts and lessons are persisted as auditable files; relevance is advisory" }, + { "id": "learning", "forge": "cortex lessons", "status": "partial", "guarantee": "external outcomes update lesson confidence; no model weights are changed" }, + { "id": "imagination", "forge": "impact graph", "status": "operational-v1", "guarantee": "reverse dependency traversal predicts possible blast radius" }, + { "id": "self-correction", "forge": "verify + doom-loop guard", "status": "partial", "guarantee": "tests/builds are trusted over model claims" }, + { "id": "impact-awareness", "forge": "atlas + impact", "status": "operational-v1", "guarantee": "known symbols/files and likely dependents are surfaced before edits" } + ], + "mechanisms": [ + { "id": "M1", "name": "complexity-aware routing", "command": "forge route", "status": "solved-with-transparency-layer" }, + { "id": "M2", "name": "assumption gate", "command": "forge preflight", "status": "operational-v1" }, + { "id": "M3", "name": "task/session decomposition", "command": "forge scope", "status": "solved-with-boundary-helper" }, + { "id": "M4", "name": "goal anchoring", "command": "forge substrate", "status": "partial" }, + { "id": "M5", "name": "scope minimality", "command": "forge substrate", "status": "operational-v1-advisory" }, + { "id": "M6", "name": "inline verification", "command": "forge verify", "status": "partial" } + ], + "defaults": { "askThreshold": 0.6, "impactThreshold": 0.1 }, + "policies": { + "preAction": "Run substrate before ambiguous, expensive, multi-file, or mutating work.", + "escalation": "Start with the cheapest capable tier and escalate only after an external verifier failure.", + "hookEnforcement": "Block or warn only on deterministic risks; keep research-edge judgments advisory.", + "honesty": "Label memory relevance, routing fit, and minimality as advisory unless verified by tests, graph membership, or explicit user scope." + }, + "limits": [ + "Graph edges are static and conservative; dynamic dispatch and generated code may be missed.", + "Memory and learning are external file-backed lessons, not weight-level learning.", + "Non-hook editors receive advisory MCP/context only; Forge cannot enforce hooks where the host has no hook surface." + ] +} diff --git a/src/atlas.js b/src/atlas.js index 518b675..b1dee7a 100644 --- a/src/atlas.js +++ b/src/atlas.js @@ -1,10 +1,7 @@ -// forge atlas — a portable, precomputed symbol index. `build` scans the repo ONCE -// and writes .forge/atlas.json; queries read that artifact (token-cheap, and any -// tool can read it via `forge atlas` or plain jq — no MCP needed to consume). -// -// lean: v1 indexes symbol DEFINITIONS (where-is-X) + membership (hallucination -// check) via per-language regex. The richer "what-calls-Z" call graph is the -// documented upgrade — back it with an LSP/serena export when that's needed. +// forge atlas — a portable code graph. Build once, then query definitions, membership, +// reverse dependents, and impact radius without asking a model to rediscover the repo. + +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; import { extname, join, relative } from "node:path"; @@ -23,39 +20,77 @@ const IGNORE = new Set([ "vendor", ]); -const JS = [ - { - re: /(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)/, - kind: "function", - }, - { re: /(?:export\s+)?class\s+([A-Za-z_$][\w$]*)/, kind: "class" }, - { - re: /(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=/, - kind: "const", - }, +const JS_RULES = [ + { re: /(?:export\s+)?(?:async\s+)?function\s+([A-Za-z_$][\w$]*)/g, kind: "function" }, + { re: /(?:export\s+)?class\s+([A-Za-z_$][\w$]*)/g, kind: "class" }, + { re: /(?:export\s+)?(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=/g, kind: "const" }, ]; const RULES = { - ".js": JS, - ".jsx": JS, - ".ts": JS, - ".tsx": JS, - ".mjs": JS, - ".cjs": JS, + ".js": JS_RULES, + ".jsx": JS_RULES, + ".ts": JS_RULES, + ".tsx": JS_RULES, + ".mjs": JS_RULES, + ".cjs": JS_RULES, ".py": [ - { re: /^\s*def\s+([A-Za-z_]\w*)/, kind: "function" }, - { re: /^\s*class\s+([A-Za-z_]\w*)/, kind: "class" }, + { re: /^\s*def\s+([A-Za-z_]\w*)/gm, kind: "function" }, + { re: /^\s*class\s+([A-Za-z_]\w*)/gm, kind: "class" }, ], ".go": [ - { re: /^func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)/, kind: "function" }, - { re: /^type\s+([A-Za-z_]\w*)/, kind: "type" }, + { re: /^func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)/gm, kind: "function" }, + { re: /^type\s+([A-Za-z_]\w*)/gm, kind: "type" }, ], ".rs": [ - { re: /\bfn\s+([A-Za-z_]\w*)/, kind: "function" }, - { re: /\b(?:struct|enum|trait)\s+([A-Za-z_]\w*)/, kind: "type" }, + { re: /\bfn\s+([A-Za-z_]\w*)/g, kind: "function" }, + { re: /\b(?:struct|enum|trait)\s+([A-Za-z_]\w*)/g, kind: "type" }, ], - ".java": [{ re: /\b(?:class|interface|enum)\s+([A-Za-z_]\w*)/, kind: "type" }], + ".java": [{ re: /\b(?:class|interface|enum)\s+([A-Za-z_]\w*)/g, kind: "type" }], }; +const CALL_RE = /(?:^|[^.\w$])([A-Za-z_$][\w$]*)\s*\(/g; +const IMPORT_RE = + /(?:import\s+(?:[^"'\n]+\s+from\s+)?["']([^"']+)["']|require\(["']([^"']+)["']\)|^\s*(?:from\s+([\w.]+)\s+)?import\s+([\w*,\s]+))/gm; +const BUILTINS = new Set([ + "if", + "for", + "while", + "switch", + "catch", + "function", + "return", + "console", + "String", + "Number", + "Boolean", + "Array", + "Object", + "Promise", + "Set", + "Map", + "Date", + "Error", + "RegExp", + "parseInt", + "parseFloat", + "setTimeout", + "clearTimeout", + "fetch", + "print", + "len", + "range", + "int", + "str", + "float", + "dict", + "list", + "set", + "super", +]); + +function hash(text) { + return createHash("sha256").update(text).digest("hex"); +} + function walk(dir, files, cap) { let entries; try { @@ -77,34 +112,152 @@ function walk(dir, files, cap) { } } +function moduleId(rel) { + return rel.replace(/\.[^.]+$/, "").replace(/[/\\]/g, "."); +} + +function lineOf(text, index) { + return text.slice(0, index).split("\n").length; +} + +function nearestSource(nodes, line, fallback) { + let best = fallback; + for (const n of nodes) { + if (n.line <= line && (!best.line || n.line >= best.line)) best = n; + } + return best; +} + function extractFile(path, root) { - const rules = RULES[extname(path)]; - const out = []; + const ext = extname(path); + const rules = RULES[ext]; + const rel = relative(root, path); let text; try { text = readFileSync(path, "utf8"); } catch { - return out; + return { symbols: [], nodes: [], edges: [], hash: "" }; } - const rel = relative(root, path); - text.split("\n").forEach((line, i) => { - for (const { re, kind } of rules) { - const m = re.exec(line); - if (m) out.push({ name: m[1], kind, file: rel, line: i + 1 }); + + const mod = { + id: `module:${moduleId(rel)}`, + name: moduleId(rel), + kind: "module", + file: rel, + line: 1, + }; + const symbols = []; + const nodes = [mod]; + const edges = []; + + for (const { re, kind } of rules) { + re.lastIndex = 0; + let m; + while ((m = re.exec(text))) { + const name = m[1]; + const line = lineOf(text, m.index); + const node = { + id: `${rel}:${name}:${line}`, + qname: `${moduleId(rel)}.${name}`, + name, + kind, + file: rel, + line, + }; + symbols.push({ name, kind, file: rel, line, id: node.id, qname: node.qname }); + nodes.push(node); + edges.push({ source: mod.id, target: node.id, kind: "contains", confidence: 1, line }); + } + } + + IMPORT_RE.lastIndex = 0; + let im; + while ((im = IMPORT_RE.exec(text))) { + const target = im[1] || im[2] || im[3] || ""; + const names = im[4] + ? im[4] + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : []; + const line = lineOf(text, im.index); + const source = nearestSource(nodes.slice(1), line, mod); + if (target) edges.push({ source: source.id, target, kind: "imports", confidence: 0.85, line }); + for (const name of names) { + if (name !== "*") + edges.push({ + source: source.id, + target: target ? `${target}.${name}` : name, + kind: "imports", + confidence: 0.85, + line, + }); + } + } + + const lines = text.split("\n"); + for (let index = 0; index < lines.length; index += 1) { + const lineText = lines[index]; + CALL_RE.lastIndex = 0; + const line = index + 1; + let cm; + while ((cm = CALL_RE.exec(lineText))) { + const callee = cm[1]; + if (BUILTINS.has(callee)) continue; + const source = nearestSource(nodes.slice(1), line, mod); + if (source.name === callee) continue; + edges.push({ source: source.id, target: callee, kind: "calls", confidence: 0.75, line }); + } + } + + return { symbols, nodes, edges, hash: hash(text) }; +} + +function resolveEdges(nodes, edges) { + const byName = new Map(); + const byQname = new Map(); + for (const n of nodes) { + if (n.name) { + const arr = byName.get(n.name) || []; + arr.push(n); + byName.set(n.name, arr); } + if (n.qname) byQname.set(n.qname, n); + } + return edges.map((edge) => { + if (nodes.some((n) => n.id === edge.target)) return edge; + const direct = byQname.get(edge.target); + if (direct) return { ...edge, target: direct.id, resolved: true }; + const short = String(edge.target).split(".").pop(); + const matches = byName.get(short) || []; + if (matches.length === 1) + return { ...edge, target: matches[0].id, resolved: true, confidence: edge.confidence * 0.9 }; + return { ...edge, unresolved: true }; }); - return out; } export function build({ root = process.cwd(), cap = 20000 } = {}) { const files = []; walk(root, files, cap); const symbols = []; - for (const f of files) symbols.push(...extractFile(f, root)); + const nodes = []; + const rawEdges = []; + const fileHashes = {}; + for (const f of files) { + const parsed = extractFile(f, root); + symbols.push(...parsed.symbols); + nodes.push(...parsed.nodes); + rawEdges.push(...parsed.edges); + fileHashes[relative(root, f)] = parsed.hash; + } + const edges = resolveEdges(nodes, rawEdges); const atlas = { - version: 1, + version: 2, files: files.length, symbols, + nodes, + edges, + fileHashes, capped: files.length >= cap, }; mkdirSync(join(root, ".forge"), { recursive: true }); @@ -119,9 +272,78 @@ export function load(root = process.cwd()) { export function query(atlas, term) { const t = String(term).toLowerCase(); - return atlas.symbols.filter((s) => s.name.toLowerCase().includes(t)); + return (atlas.symbols || []).filter( + (s) => + s.name.toLowerCase().includes(t) || + String(s.qname || "") + .toLowerCase() + .includes(t), + ); } export function has(atlas, name) { - return atlas.symbols.some((s) => s.name === name); + return (atlas.symbols || []).some((s) => s.name === name || s.qname === name || s.id === name); +} + +function targetIds(atlas, target) { + const t = String(target); + const nodes = atlas.nodes || []; + const matches = nodes.filter( + (n) => n.id === t || n.name === t || n.qname === t || n.file === t || n.file?.endsWith(`/${t}`), + ); + return matches.map((n) => n.id); +} + +const EDGE_WEIGHT = { calls: 0.95, imports: 0.85, inherits: 0.92, references: 0.7, contains: 0.45 }; + +export function impact(atlas, target, { threshold = 0.1, maxHops = 6, decay = 0.85 } = {}) { + const starts = targetIds(atlas, target); + const nodeById = new Map((atlas.nodes || []).map((n) => [n.id, n])); + const incoming = new Map(); + for (const e of atlas.edges || []) { + if (e.unresolved) continue; + const arr = incoming.get(e.target) || []; + arr.push(e); + incoming.set(e.target, arr); + } + const visited = new Map(); + const queue = starts.map((id) => ({ id, confidence: 1, hop: 0, path: [id], edgeKinds: [] })); + while (queue.length) { + const current = queue.shift(); + if (!current || current.hop >= maxHops) continue; + for (const edge of incoming.get(current.id) || []) { + if (starts.includes(edge.source)) continue; + const nextConfidence = + current.confidence * (EDGE_WEIGHT[edge.kind] || 0.5) * (edge.confidence ?? 1) * decay; + if (nextConfidence < threshold) continue; + const prev = visited.get(edge.source); + if (prev && prev.confidence >= nextConfidence) continue; + const item = { + id: edge.source, + node: nodeById.get(edge.source) || { id: edge.source, name: edge.source, kind: "unknown" }, + confidence: Number(nextConfidence.toFixed(4)), + hopDistance: current.hop + 1, + path: [...current.path, edge.source], + edgeKinds: [...current.edgeKinds, edge.kind], + }; + visited.set(edge.source, item); + queue.push({ + id: edge.source, + confidence: nextConfidence, + hop: current.hop + 1, + path: item.path, + edgeKinds: item.edgeKinds, + }); + } + } + const impacted = [...visited.values()].sort((a, b) => b.confidence - a.confidence); + return { + target, + found: starts.length > 0, + threshold, + impacted, + impactedFiles: [...new Set(impacted.map((x) => x.node.file).filter(Boolean))].sort(), + totalGraphNodes: (atlas.nodes || []).length, + totalGraphEdges: (atlas.edges || []).length, + }; } diff --git a/src/cli.js b/src/cli.js index c251294..4c9c515 100755 --- a/src/cli.js +++ b/src/cli.js @@ -21,6 +21,8 @@ const COMMANDS = { cortex: "self-correcting project memory — status / why ", preflight: "assumption check — what a task names that the repo doesn't define", route: "recommend the cheapest capable model for a task (+ gateway config)", + impact: "predict blast radius for a symbol or file from the atlas graph", + substrate: "one pre-action gate: assumptions, route, impact, scope, memory, verify", scope: "decompose files into independent clusters (+ coupled files you didn't name)", uicheck: "deterministic UI check — WCAG contrast (assertable, no guessing)", brand: "print the active brand token map", @@ -381,7 +383,7 @@ async function run(argv) { const r = preflightRepo(process.cwd(), task); console.log(`${BRAND.brand} preflight — assumption check\n`); console.log( - ` info-gap: ${r.gap.toFixed(2)} (referenced ${r.entities.symbols.length} symbol(s), ${r.entities.files.length} file(s))`, + ` info-gap: ${r.gap.toFixed(2)} · completeness ${r.assumption.completeness.toFixed(2)} (referenced ${r.entities.symbols.length} symbol(s), ${r.entities.files.length} file(s))`, ); const block = clarifyBlock(r); console.log( @@ -389,6 +391,46 @@ async function run(argv) { ); return; } + if (cmd === "impact") { + const { predictImpact } = await import("./substrate.js"); + const json = argv.includes("--json"); + const target = argv + .slice(1) + .filter((a) => a !== "--json") + .join(" "); + if (!target) { + console.error("usage: forge impact [--json]"); + process.exitCode = 1; + return; + } + const r = predictImpact(process.cwd(), target); + if (json) { + console.log(JSON.stringify(r, null, 2)); + return; + } + console.log(`${BRAND.brand} impact — blast radius\n`); + console.log(` target: ${target} ${r.found ? "✓ found" : "not found"}`); + console.log(` impacted files: ${r.impactedFiles.length}`); + for (const file of r.impactedFiles.slice(0, 20)) console.log(` - ${file}`); + if (r.impactedFiles.length > 20) console.log(` … ${r.impactedFiles.length - 20} more`); + return; + } + if (cmd === "substrate") { + const { renderSubstrate, substrateCheck } = await import("./substrate.js"); + const json = argv.includes("--json"); + const task = argv + .slice(1) + .filter((a) => a !== "--json") + .join(" "); + if (!task) { + console.error('usage: forge substrate "" [--json]'); + process.exitCode = 1; + return; + } + const r = substrateCheck(process.cwd(), task); + console.log(json ? JSON.stringify(r, null, 2) : renderSubstrate(r)); + return; + } if (cmd === "route") { const r = await import("./route.js"); if (argv[1] === "gateway") { diff --git a/src/cortex_mcp.js b/src/cortex_mcp.js index e95248e..3da22bf 100644 --- a/src/cortex_mcp.js +++ b/src/cortex_mcp.js @@ -6,9 +6,10 @@ import { argv } from "node:process"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; import { lessonsForContext, summary } from "./cortex.js"; -import { clarifyBlock, preflightRepo } from "./preflight.js"; +import { assessTask, clarifyBlock, preflightRepo } from "./preflight.js"; import { routeTask } from "./route.js"; import { decompose } from "./scope.js"; +import { predictImpact, substrateCheck } from "./substrate.js"; const root = process.env.FORGE_ROOT || process.cwd(); const today = () => Math.floor(Date.now() / 86400000); @@ -59,6 +60,40 @@ const TOOLS = [ required: ["task"], }, }, + + { + name: "assumption_gate", + description: + "Score specification completeness before work starts. Returns shouldAsk, risk, missing dimensions, and concrete questions.", + inputSchema: { + type: "object", + properties: { task: { type: "string", description: "the task/prompt" } }, + required: ["task"], + }, + }, + { + name: "predict_impact", + description: + "Predict blast radius for a symbol or file using Forge atlas reverse-dependency traversal.", + inputSchema: { + type: "object", + properties: { + target: { type: "string", description: "symbol name, qualified name, or file" }, + threshold: { type: "number", description: "confidence threshold, default 0.1" }, + }, + required: ["target"], + }, + }, + { + name: "substrate_check", + description: + "Full Forge cognitive-substrate pre-action check: assumption gate, route, impact, scope, memory, minimality, and verification checklist.", + inputSchema: { + type: "object", + properties: { task: { type: "string", description: "the task/prompt" } }, + required: ["task"], + }, + }, { name: "scope_files", description: @@ -91,6 +126,17 @@ function callTool(name, args = {}) { const rec = routeTask(root, String(args.task ?? "")); return `Recommended: ${rec.model.name} (${rec.tier}). complexity ${rec.score.toFixed(2)}${rec.reasons.length ? ` — ${rec.reasons.join(", ")}` : ""}.`; } + + if (name === "assumption_gate") + return JSON.stringify(assessTask(String(args.task ?? "")), null, 2); + if (name === "predict_impact") + return JSON.stringify( + predictImpact(root, String(args.target ?? ""), { threshold: Number(args.threshold ?? 0.1) }), + null, + 2, + ); + if (name === "substrate_check") + return JSON.stringify(substrateCheck(root, String(args.task ?? "")), null, 2); if (name === "scope_files") { const d = decompose(root, args.files ?? []); return JSON.stringify(d, null, 2); diff --git a/src/preflight.js b/src/preflight.js index d935803..e63a46c 100644 --- a/src/preflight.js +++ b/src/preflight.js @@ -1,8 +1,6 @@ // forge preflight — the assumption detector. Before the agent spends a token, scan the task // for code identifiers/files it NAMES but the repo doesn't DEFINE — those are the things it -// will silently ASSUME (the user's "biggest problem"). Plus vague wording with no acceptance -// criteria. Surface the known-unknowns so the agent ASKS instead of confabulating. -// PURE logic here (no fs) so it's fully testable; the repo wrapper (bottom) adds atlas + file checks. +// will silently ASSUME. The richer assumption gate also scores specification completeness. import { existsSync } from "node:fs"; import { join } from "node:path"; import { build as buildAtlas, has, load as loadAtlas } from "./atlas.js"; @@ -10,7 +8,6 @@ import { build as buildAtlas, has, load as loadAtlas } from "./atlas.js"; const CODE_EXT = /\.(js|ts|jsx|tsx|mjs|cjs|py|go|rs|java|rb|php|c|cc|cpp|h|hpp|cs|json|ya?ml|toml|md|css|scss|html|vue|svelte)$/i; -// Very common words that look like identifiers but aren't worth grounding. const STOP = new Set([ "the", "this", @@ -34,17 +31,79 @@ const STOP = new Set([ "note", ]); +const rx = (pattern) => new RegExp(pattern, "i"); + +const DIMENSIONS = [ + { + key: "inputs_outputs", + description: "exact input/output behavior", + question: "What exact inputs, outputs, examples, or return values should this satisfy?", + applies: rx( + "\\b(function|api|endpoint|parse|convert|transform|read|write|generate|compute|return|implement)\\b", + ), + cues: rx( + "(->|=>|input|output|returns?|given|example|e\\.g\\.|for example|format|json|csv|schema|signature|expected|```)", + ), + }, + { + key: "target_scope", + description: "target file/module/component scope", + question: "Which specific file, module, component, or symbol should this change touch?", + applies: rx( + "\\b(fix|change|edit|update|modify|refactor|add to|remove from|integrate|wire|bug|issue|error)\\b", + ), + cues: rx( + "\\b(file|module|class|function|component|path|directory|service|layer|endpoint)\\b|`[\\w./-]+`|\\w+\\.\\w+", + ), + }, + { + key: "success_criteria", + description: "external success criteria", + question: + "How will we verify it: tests, acceptance criteria, benchmark, or reference behavior?", + applies: rx( + "\\b(fix|optimi[sz]e|make it (faster|work|better)|improve|ensure|feature|behavior)\\b", + ), + cues: rx( + "(->|=>|test|passes|acceptance|criteria|expected|should return|should equal|should match|verify|assert|benchmark|correct when|e\\.g\\.|example|```)", + ), + }, + { + key: "constraints", + description: "hard constraints", + question: + "What constraints must be respected: performance, dependencies, style, or compatibility?", + applies: rx("\\b(design|architect|production|scal|migrate|distributed|concurren|refactor)\\b"), + cues: rx( + "\\b(must|should|constraint|limit|no new dependenc|only use|standard library|without|performance|latency|O\\(|backward|compatib|convention|style)\\b", + ), + }, +]; + +const VAGUE = rx( + "\\b(some|somehow|etc|and so on|things?|stuff|appropriate(ly)?|as needed|handle (it|everything)|make it (work|better|nice|good)|clean it up|cleaner|the usual|standard way|properly|correctly|the way we (discussed|talked)|like before|as before)\\b", +); +const ANCHORS = [ + /```/, + /->|=>/, + /\b\w+\([^)]*\)/, + /'[^']+'|"[^"]+"/, + /\b\w+\.\w{1,5}\b/, + /\b\d+\b.*(->|=>|=|:)|\(\d/, + /\b(e\.g\.|for example|such as|example:)\b/i, +]; +const SPECIFIC = + /\b(python|javascript|typescript|java|rust|golang|react|django|flask|node|redis|sql|postgres|asyncio|dijkstra|lru|adjacency|owasp|regex)\b|token-?bucket|binary heap|condition[- ]variable|recursive-?descent|in-?order|standard library|o\(\s*\d|o\(n|o\(1/gi; + const isCodeIdent = (p, backticked) => { if (!p || p.length < 2) return false; if (STOP.has(p.toLowerCase())) return false; - if (backticked) return /^[A-Za-z_$][\w$]*$/.test(p); // trust anything quoted as code - // bare tokens must LOOK like code: camelCase, snake_case, or Pascal+camel + if (backticked) return /^[A-Za-z_$][\w$]*$/.test(p); return ( /[a-z][A-Z]/.test(p) || (p.includes("_") && /[a-z]/i.test(p)) || /^[A-Z][a-z]+[A-Z]/.test(p) ); }; -/** Pure: pull the code identifiers + file paths a task references. */ export function referencedEntities(text) { const s = String(text); const symbols = new Set(); @@ -63,14 +122,11 @@ export function referencedEntities(text) { if (isCodeIdent(part, backticked)) symbols.add(part); } }; - for (const m of s.matchAll(/`([^`]+)`/g)) { - for (const t of m[1].split(/\s+/)) consider(t, true); - } + for (const m of s.matchAll(/`([^`]+)`/g)) for (const t of m[1].split(/\s+/)) consider(t, true); for (const m of s.matchAll(/[A-Za-z_$][\w$./-]*/g)) { const t = m[0]; - if (t.includes("/") || CODE_EXT.test(t) || /[a-z][A-Z]/.test(t) || t.includes("_")) { + if (t.includes("/") || CODE_EXT.test(t) || /[a-z][A-Z]/.test(t) || t.includes("_")) consider(t, false); - } } return { symbols: [...symbols], files: [...files] }; } @@ -90,7 +146,6 @@ const AMBIGUITY = [ /\band more\b/i, ]; -/** Pure: vague phrases that signal missing acceptance criteria. */ export function ambiguityMarkers(text) { const out = []; for (const re of AMBIGUITY) { @@ -100,11 +155,63 @@ export function ambiguityMarkers(text) { return [...new Set(out)]; } -/** - * Pure: the information gap — unresolved references + ambiguity, normalized to [0,1]. - * @param {string} text - * @param {{hasSymbol?:(name:string)=>boolean, fileExists?:(path:string)=>boolean}} [deps] - */ +export function assessTask(text, { askThreshold = 0.6 } = {}) { + const task = String(text || ""); + const words = task.trim().split(/\s+/).filter(Boolean).length; + const concreteness = ANCHORS.filter((a) => a.test(task)).length; + const specifics = [...new Set([...task.matchAll(SPECIFIC)].map((m) => m[0].toLowerCase()))]; + const vagueHits = [ + ...new Set([...task.matchAll(new RegExp(VAGUE.source, "gi"))].map((m) => m[0].toLowerCase())), + ]; + const reasons = []; + let score = + 0.45 + + Math.min(0.45, 0.18 * concreteness) + + Math.min(0.2, 0.06 * specifics.length) - + 0.22 * vagueHits.length; + if (words <= 7) { + score -= 0.22; + reasons.push("very short request"); + } + if (words >= 30 && specifics.length >= 2) { + score += 0.2; + reasons.push(`long detailed spec (${words} words)`); + } else if (words >= 22 && specifics.length >= 1) { + score += 0.1; + reasons.push(`detailed spec (${words} words)`); + } + if (concreteness) reasons.push(`${concreteness} concrete anchor(s)`); + if (specifics.length) reasons.push(`${specifics.length} named technical specific(s)`); + if (vagueHits.length) reasons.push(`${vagueHits.length} vague filler(s)`); + + const missing = []; + const questions = []; + for (const d of DIMENSIONS) { + if (d.applies.test(task) && !d.cues.test(task)) { + missing.push({ key: d.key, description: d.description }); + questions.push(d.question); + } + } + const completeness = Math.max(0, Math.min(1, score)); + const hardUnderspecified = + concreteness === 0 && (words <= 10 || (vagueHits.length >= 1 && specifics.length === 0)); + const shouldAsk = completeness < askThreshold || hardUnderspecified; + if (hardUnderspecified && !reasons.includes("very short request")) + reasons.push("no concrete anchor to act on"); + const risk = completeness < 0.45 ? "high" : completeness < 0.7 ? "medium" : "low"; + return { + completeness, + risk, + shouldAsk, + missing, + questions: (shouldAsk && !questions.length + ? ["What exactly should this produce, and how will we know it is correct?"] + : questions + ).slice(0, 3), + reasons, + }; +} + export function informationGap(text, deps = {}) { const { hasSymbol = () => false, fileExists = () => false } = deps; const { symbols, files } = referencedEntities(text); @@ -122,42 +229,33 @@ export function informationGap(text, deps = {}) { }; } -/** - * Pure: render the clarify prompt — or "" when nothing needs clarifying. A single unresolved - * concrete reference always clarifies (high value); pure ambiguity needs to clear the threshold. - */ export function clarifyBlock(result, { threshold = 0.5 } = {}) { const nRef = result.unresolved.symbols.length + result.unresolved.files.length; - if (nRef === 0 && result.gap < threshold) return ""; + const assumption = result.assumption; + if (nRef === 0 && result.gap < threshold && !assumption?.shouldAsk) return ""; const lines = [ "## Before starting — clarify (Forge Preflight)", - "This task names things that aren't grounded in the codebase. Confirm before assuming:", + "This task has unknowns that would otherwise become assumptions:", "", ]; - for (const s of result.unresolved.symbols) { + for (const s of result.unresolved.symbols) lines.push(`- \`${s}\` — not found in the code. Different name, or should it be created?`); - } - for (const f of result.unresolved.files) { - lines.push(`- \`${f}\` — file not found. Confirm the path, or that it's new.`); - } - if (result.ambiguous.length) { + for (const f of result.unresolved.files) + lines.push(`- \`${f}\` — file not found. Confirm the path, or that it is new.`); + if (result.ambiguous.length) lines.push( - `- Ambiguous: ${result.ambiguous.map((a) => `"${a}"`).join(", ")} — state the concrete acceptance criteria.`, + `- Ambiguous: ${result.ambiguous.map((a) => `"${a}"`).join(", ")} — state concrete acceptance criteria.`, ); - } + if (assumption?.shouldAsk) for (const q of assumption.questions) lines.push(`- ${q}`); lines.push("", "_Advisory: ask rather than assume._"); return lines.join("\n"); } -/** - * Repo wrapper: gap against the real atlas + filesystem. In hooks pass `allowBuild:false` so we - * only use a CACHED atlas (fast, and if none exists we skip symbol-flagging rather than - * false-alarm on an unindexed repo); the explicit CLI builds it. - */ -export function preflightRepo(root, text, { allowBuild = true } = {}) { +export function preflightRepo(root, text, { allowBuild = true, askThreshold = 0.6 } = {}) { const atlas = loadAtlas(root) || (allowBuild ? buildAtlas({ root }) : null); - return informationGap(text, { + const gap = informationGap(text, { hasSymbol: atlas ? (name) => has(atlas, name) : () => true, fileExists: (f) => existsSync(join(root, f)), }); + return { ...gap, assumption: assessTask(text, { askThreshold }) }; } diff --git a/src/route.js b/src/route.js index eb8e017..117a9cd 100644 --- a/src/route.js +++ b/src/route.js @@ -13,6 +13,77 @@ import { preflightRepo, referencedEntities } from "./preflight.js"; const clamp01 = (x) => Math.max(0, Math.min(1, x)); // Weights sum to 1. Each raw signal is normalized by the point where it reads as "complex". + +const ALGO_TERMS = + /\b(recursion|recursive|recursive-?descent|dynamic programming|dijkstra|a\*|concurren|thread-?safe|mutex|race condition|deadlock|distributed|consensus|parser|compiler|cryptograph|np-hard|state machine|invariant|numerical stability|back-?pressure|token[- ]bucket|rate limiter|idempoten|migration|producer|consumer|blocking queue|condition[- ]variable)\b/i; +const ARCH_TERMS = + /\b(architect|\bdesign\b|trade-?off|refactor a|migrate|scal(e|able|ing)|schema (migration|design)|api design|multi-?module|cross-?module|end-?to-?end|consistency (guarantee|trade)|locking strategy|module boundaries)\b/i; +const MODERATE_TERMS = + /\b(class\b|cache|lru|queue|stack|heap|linked list|binary tree|tree|traversal|graph|decorator|regex|debounce|throttle|merge|sort(ed|ing)?|parse|o\(\s*\d|o\(n|o\(1|thread|async|lock|validate|in-?order|adjacency)\b/i; +const TRIVIAL_TERMS = + /\b(hello world|rename|typo|indent|add a comment|reverse a string|reverse the string|is[_ ]?even|is[_ ]?odd|factorial|fibonacci|is[_ ]?prime|prime\b|sum of|sum_list|capitalize|count vowels|celsius|fahrenheit|lower ?case|upper ?case)\b/i; +const MULTISTEP = + /\b(and then|after that|first.*then|step \d|multiple|several|each of|for every)\b/i; + +export function rubricSignals(task = "") { + const text = String(task); + return { + lengthTokens: Math.max(1, Math.floor(text.length / 4)), + hasAlgorithmicTerms: ALGO_TERMS.test(text), + hasArchitecturalTerms: ARCH_TERMS.test(text), + hasModerateTerms: MODERATE_TERMS.test(text), + hasTrivialMarkers: TRIVIAL_TERMS.test(text), + hasMultistep: MULTISTEP.test(text), + hasCodeContext: /```/.test(text), + nConstraints: (text.match(/(^\s*[-*\d.]|\b(must|should|ensure|require|constraint)\b)/gim) || []) + .length, + }; +} + +export function rubricComplexity(task = "") { + const sig = rubricSignals(task); + const reasons = [{ weight: 1.5, reason: "base cost of any task" }]; + let score = 1.5; + if (sig.hasAlgorithmicTerms) { + score += 4; + reasons.push({ weight: 4, reason: "algorithmic/systems difficulty" }); + } + if (sig.hasArchitecturalTerms) { + score += 4; + reasons.push({ weight: 4, reason: "architectural/design scope" }); + } + if (sig.hasModerateTerms) { + score += 2; + reasons.push({ weight: 2, reason: "data-structure/class/library-level work" }); + } + if (sig.hasMultistep) { + score += 1; + reasons.push({ weight: 1, reason: "multi-step request" }); + } + if (sig.hasCodeContext) { + score += 1; + reasons.push({ weight: 1, reason: "carries code context" }); + } + if (sig.lengthTokens > 120) { + score += 1.5; + reasons.push({ weight: 1.5, reason: `long spec (~${sig.lengthTokens} tok)` }); + } else if (sig.lengthTokens > 55) { + score += 0.7; + reasons.push({ weight: 0.7, reason: `medium spec (~${sig.lengthTokens} tok)` }); + } + if (sig.nConstraints >= 5) { + score += 1; + reasons.push({ weight: 1, reason: `${sig.nConstraints} explicit constraints` }); + } + if (sig.hasTrivialMarkers && !sig.hasAlgorithmicTerms && !sig.hasArchitecturalTerms) { + score -= 3; + reasons.push({ weight: -3, reason: "trivial-task marker" }); + } + const rawScore = Math.max(0, score); + const band = rawScore < 3 ? "cheap" : rawScore <= 6 ? "mid" : "premium"; + return { rawScore, band, signals: sig, reasons }; +} + const WEIGHTS = { files: 0.22, fanout: 0.22, @@ -69,8 +140,24 @@ export function routeTask(root, task) { ambiguity, sizeWords, }; - const { score, norm } = complexity(signals); - return { score, signals, ...recommend(score, norm) }; + const { score: repoScore, norm } = complexity(signals); + const rubric = rubricComplexity(task); + const rubricScore = Math.min(1, rubric.rawScore / 10); + const score = Math.max(repoScore, rubricScore); + const recommended = recommend(score, norm); + return { + score, + repoScore, + signals, + rubric, + ...recommended, + reasons: [ + ...new Set([ + ...(recommended.reasons || []), + ...rubric.reasons.filter((r) => r.weight > 0).map((r) => r.reason), + ]), + ], + }; } /** Emit a LiteLLM config exposing the complexity tiers as aliases (request the one `forge route` picks). */ diff --git a/src/substrate.js b/src/substrate.js new file mode 100644 index 0000000..75c6481 --- /dev/null +++ b/src/substrate.js @@ -0,0 +1,149 @@ +// forge substrate — one pre-action surface for the cognitive substrate described in +// the paper: gate assumptions, route model effort, inspect scope/impact, surface memory, +// and produce an external verification checklist. Deterministic where possible; +// advisory where the paper marks the research edge. +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { build as buildAtlas, impact as impactGraph, load as loadAtlas } from "./atlas.js"; +import { matchingLessons } from "./cortex.js"; +import { load as loadLessons } from "./lessons_store.js"; +import { assessTask, clarifyBlock, preflightRepo, referencedEntities } from "./preflight.js"; +import { routeTask } from "./route.js"; +import { decompose } from "./scope.js"; + +function loadSubstrateSpec() { + const path = join(dirname(dirname(fileURLToPath(import.meta.url))), "source", "substrate.json"); + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +function verificationChecklist(root) { + const checks = []; + if (existsSync(join(root, "package.json"))) { + checks.push("npm test"); + checks.push("npm run typecheck"); + checks.push("npm run lint"); + } + if (existsSync(join(root, "pyproject.toml")) || existsSync(join(root, "pytest.ini"))) + checks.push("pytest -q"); + checks.push("review impacted files before editing"); + checks.push("run the narrowest affected test first, then the broader suite"); + return [...new Set(checks)]; +} + +function minimalityWarnings(task, route, preflight) { + const warnings = []; + const text = String(task).toLowerCase(); + if ( + /\b(refactor|rewrite|clean|redesign|optimi[sz]e|improve)\b/.test(text) && + preflight.entities.files.length === 0 + ) { + warnings.push( + "High-risk broad change with no target files named; ask for scope before editing.", + ); + } + if (route.score >= 0.55 && preflight.assumption.completeness < 0.7) { + warnings.push( + "Complex task with medium/low specification completeness; clarify before spending a premium model.", + ); + } + if ( + /\b(add authentication|payment|migration|distributed|concurrent|production)\b/.test(text) && + !/\btest|acceptance|rollback|constraint|must|should\b/.test(text) + ) { + warnings.push("Production-sensitive task lacks explicit constraints or acceptance criteria."); + } + return warnings; +} + +export function predictImpact(root, target, { threshold = 0.1 } = {}) { + const atlas = loadAtlas(root) || buildAtlas({ root }); + return impactGraph(atlas, target, { threshold }); +} + +export function substrateCheck(root, task, { threshold = 0.1, askThreshold = 0.6 } = {}) { + const text = String(task || ""); + const entities = referencedEntities(text); + const preflight = preflightRepo(root, text, { askThreshold }); + const assumption = assessTask(text, { askThreshold }); + const route = routeTask(root, text); + const atlas = loadAtlas(root) || buildAtlas({ root }); + const impactTargets = [...new Set([...entities.symbols, ...entities.files])].slice(0, 8); + const impacts = impactTargets.map((target) => impactGraph(atlas, target, { threshold })); + const impactedFiles = [...new Set(impacts.flatMap((r) => r.impactedFiles || []))].sort(); + const scopedFiles = [...new Set([...entities.files, ...impactedFiles])]; + const scope = scopedFiles.length + ? decompose(root, scopedFiles) + : { clusters: [], independentGroups: 0 }; + const lessons = matchingLessons(loadLessons(root), { + files: scopedFiles, + symbols: entities.symbols, + }); + const result = { + okToProceed: !preflight.assumption.shouldAsk && !assumption.shouldAsk, + task: text, + assumption: preflight.assumption.shouldAsk ? preflight.assumption : assumption, + clarify: clarifyBlock(preflight), + route, + entities, + impact: { targets: impactTargets, reports: impacts, impactedFiles }, + scope, + memory: { + matchingLessons: lessons.length, + advisory: lessons + .slice(0, 5) + .map((lesson) => ({ id: lesson.id, status: lesson.status, scope: lesson.scope })), + }, + minimality: { warnings: minimalityWarnings(text, route, preflight) }, + verification: { checklist: verificationChecklist(root) }, + substrate: loadSubstrateSpec(), + guarantees: { + deterministic: [ + "assumption rubric", + "repo symbol/file grounding", + "model routing rubric", + "impact graph traversal", + "scope decomposition", + ], + advisory: [ + "model capability fit", + "scope minimality", + "memory/learning relevance", + "verification completeness", + ], + }, + }; + return result; +} + +export function renderSubstrate(result) { + const lines = ["Forge substrate — pre-action check", ""]; + lines.push(` proceed: ${result.okToProceed ? "yes" : "ASK FIRST"}`); + lines.push( + ` assumption: ${result.assumption.risk} risk · completeness ${result.assumption.completeness.toFixed(2)}`, + ); + if (result.assumption.questions.length) { + lines.push("", " clarify:"); + for (const q of result.assumption.questions) lines.push(` - ${q}`); + } + lines.push( + "", + ` route: ${result.route.model.name} (${result.route.tier}) · complexity ${result.route.score.toFixed(2)}`, + ); + if (result.route.reasons.length) lines.push(` driven by: ${result.route.reasons.join(", ")}`); + lines.push("", ` impact: ${result.impact.impactedFiles.length} file(s) predicted`); + for (const file of result.impact.impactedFiles.slice(0, 10)) lines.push(` - ${file}`); + if (result.impact.impactedFiles.length > 10) + lines.push(` … ${result.impact.impactedFiles.length - 10} more`); + if (result.minimality.warnings.length) { + lines.push("", " minimality warnings:"); + for (const w of result.minimality.warnings) lines.push(` - ${w}`); + } + lines.push("", " verify:"); + for (const c of result.verification.checklist) lines.push(` - ${c}`); + return lines.join("\n"); +} diff --git a/test/mcp.test.js b/test/mcp.test.js index f91df98..d4767a0 100644 --- a/test/mcp.test.js +++ b/test/mcp.test.js @@ -55,3 +55,19 @@ test("MCP merge preserves a user's own server", () => { assert.ok(j.mcpServers.mine, "user server preserved"); assert.ok(j.mcpServers.context7, "forge server added"); }); + +test("cortex MCP exposes substrate tools", async () => { + const { handle } = await import("../src/cortex_mcp.js"); + const listed = handle({ jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }); + const names = listed.result.tools.map((tool) => tool.name); + assert.ok(names.includes("substrate_check")); + assert.ok(names.includes("predict_impact")); + assert.ok(names.includes("assumption_gate")); + const called = handle({ + jsonrpc: "2.0", + id: 2, + method: "tools/call", + params: { name: "assumption_gate", arguments: { task: "Fix the bug." } }, + }); + assert.match(called.result.content[0].text, /shouldAsk/); +}); diff --git a/test/substrate.test.js b/test/substrate.test.js new file mode 100644 index 0000000..18e8a89 --- /dev/null +++ b/test/substrate.test.js @@ -0,0 +1,73 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { build, impact } from "../src/atlas.js"; +import { assessTask, preflightRepo } from "../src/preflight.js"; +import { routeTask, rubricComplexity } from "../src/route.js"; +import { substrateCheck } from "../src/substrate.js"; + +function repo() { + const root = mkdtempSync(join(tmpdir(), "forge-substrate-")); + writeFileSync(join(root, "math.js"), "export function computeTax(x){ return x * 0.2 }\n"); + writeFileSync( + join(root, "invoice.js"), + "import { computeTax } from './math.js'\nexport function invoiceTotal(x){ return x + computeTax(x) }\n", + ); + return root; +} + +test("assumption gate asks on under-specified work", () => { + const r = assessTask("Fix the bug."); + assert.equal(r.shouldAsk, true); + assert.equal(r.risk, "high"); + assert.ok(r.questions.length > 0); +}); + +test("routing rubric separates trivial, moderate, and premium tasks", () => { + assert.equal(rubricComplexity("Write is_prime(n). is_prime(7) -> True. Python.").band, "cheap"); + assert.equal( + rubricComplexity("Implement an LRU cache class with O(1) get/put. Python.").band, + "mid", + ); + assert.equal( + rubricComplexity( + "Design and implement a thread-safe bounded blocking queue with condition-variable signaling.", + ).band, + "premium", + ); +}); + +test("atlas impact follows reverse dependencies", () => { + const root = repo(); + const atlas = build({ root }); + const r = impact(atlas, "computeTax"); + assert.equal(r.found, true); + assert.ok(r.impactedFiles.includes("invoice.js"), JSON.stringify(r, null, 2)); +}); + +test("substrateCheck returns one professional pre-action contract", () => { + const root = repo(); + const r = substrateCheck( + root, + "Update `computeTax` in `math.js` so computeTax(100) -> 25 and tests pass.", + ); + assert.equal(typeof r.okToProceed, "boolean"); + assert.ok(r.assumption); + assert.ok(r.route.model.id); + assert.ok(Array.isArray(r.impact.impactedFiles)); + assert.ok(Array.isArray(r.verification.checklist)); +}); + +test("routeTask uses text rubric to avoid under-routing moderate standalone tasks", () => { + const root = repo(); + const r = routeTask(root, "Implement an LRU cache class with O(1) get and put. JavaScript."); + assert.ok(["sonnet", "opus", "fable"].includes(r.key), `got ${r.key}`); +}); + +test("preflightRepo includes assumption report", () => { + const root = repo(); + const r = preflightRepo(root, "Optimize it."); + assert.equal(r.assumption.shouldAsk, true); +});