diff --git a/.env.template b/.env.template index 7265954e0..0f7ea3ea2 100644 --- a/.env.template +++ b/.env.template @@ -33,6 +33,18 @@ PERFORMANCE_API_KEY= # MODEL_ALIASES='{"requested-name": "served-name"}' # audited equivalences only CLAUDE_CODE_ATTREBUTION_HEADER=0 +# Strict execution backend (doc/RFC-provider-registry.md). REQUIRED for Strict +# runs — they refuse to start while this is unset (CLI runs default to api). +# api = the raw-API keys above +# cursor = Cursor subscription via the cursor-agent CLI (no API key; +# the CLI must be installed and logged in) +# claude-code / codex = declared, ship in M2/M3 +STRICT_BACKEND=api +# STRICT_BACKEND_MODEL= # model id inside the harness (optional) +# STRICT_BACKEND_CONCURRENCY=2 # concurrent harness sessions +# STRICT_BACKEND_CLI= # binary path override (else PATH lookup) +# STRICT_BACKEND_TIMEOUT_S=1800 # per-session wall-clock ceiling + # Repos: JSON map of repo name -> local path (machine-specific; .env is gitignored, # which is why absolute paths belong HERE and never in a committed file). REPO_PATHS={"vllm-omni": "/path/to/vllm-omni", "afd-plugin": "/path/to/afd-plugin"} diff --git a/doc/RFC-provider-registry.md b/doc/RFC-provider-registry.md new file mode 100644 index 000000000..69d88367d --- /dev/null +++ b/doc/RFC-provider-registry.md @@ -0,0 +1,343 @@ +# RFC — Provider registry: subscription-auth harness backends for Strict + +- Status: accepted (grilling session 2026-08-14); M1 — registry, api + parity, tool bridge, cursor transport — implemented on this branch; + claude-code (M2) and codex (M3) declared but unshipped +- Owner: LLM/backend layer (`llm.py`, `config.py`), agent runtime + (`engine/agent_runtime/runner.py`), new `src/infermatrix_copilot/providers/` +- Prior art studied: Hermes Agent `api_mode` transports + (hermes-agent.nousresearch.com/docs/developer-guide/adding-providers) · + opencode provider registry (opencode.ai/v2/docs/providers) + +## Motivation + +Strict mode runs the full execution spine, and every model call goes through +`llm.py::LLM` — a raw Anthropic/OpenAI-compatible completions client that +requires `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. Users whose only model +access is a coding-agent subscription (Claude Code Pro/Max, ChatGPT plan for +Codex CLI, Cursor) cannot run Strict at all. + +Claude Code, Codex CLI and cursor-agent expose no raw completions API on +subscription auth — they are *agent harnesses* that own their tool loop. So +the plug-in point cannot be `LLM.create()` (stateless +`system+messages+tools → tool_use` round trips); it has to sit one level up, +where a whole agent step is delegated. + +Hermes solves the adjacent problem with an `api_mode` transport abstraction +(canonical internal message shape; per-mode adapters for request building, +response normalization, usage extraction; one standardized runtime-resolution +record). This codebase is already half-way there: the Anthropic block +protocol is our canonical shape, `_openai_messages()` is a transport adapter, +and `Settings.tier_target()` → `ResolvedTarget` is runtime resolution. This +RFC formalizes that into a provider registry and adds the axis Hermes only +needed for its out-of-process Codex path: **transport kind** (`api` vs +`harness`). + +## Decisions (locked with the maintainer, 2026-08-14) + +1. **Purpose: product reach.** Strict on subscription auth for users without + raw API keys. Fidelity is best-effort per harness and labeled per run. + Eval/campaign tables are unaffected — a harness backend measures + loop+model and is *never* pooled into generator-ablation tables (standing + rule: generator arms ride our pipeline with only the model swapped). +2. **Explicit selection, hard error.** A `.env` key selects the backend. For + a Strict run with no selected backend the server errors upfront with the + exact missing item (same philosophy as `TierNotConfiguredError` / + `strict_readiness`). No auto-detection, no silent fallback. +3. **Tools bridged, preventive-first.** Harness sessions get the copilot's + own tools over MCP, every call passing `tools.dispatch` + (ToolScope/PathScope choke point preserved). Vendor built-ins are + disabled where the harness supports it; cursor-agent additionally gets + the productized post-run audit (defense in depth), with the control class + disclosed in RUN_REPORT. +4. **Unified registry.** The existing API path becomes provider `api` under + the same registry — one resolution path, one trace vocabulary. Parity + ratchet: with provider `api`, behavior is byte-identical and the existing + test suite is the gate. +5. **Phasing: cursor-agent first**, then Claude Code, then Codex. + +## Design + +### Vocabulary + +- **provider id** — registry key and the value of the selection config: + `api`, `cursor`, `claude-code`, `codex`. +- **kind** — `api` (stateless completions; implements `complete`) or + `harness` (owns a tool loop; implements `run_session` + one-shot + `complete`). +- **api_mode** — the wire protocol *within* the `api` provider: + `anthropic_messages` | `chat_completions`. This is exactly today's + `Settings.resolved_llm_provider`; the term "provider" in existing code + (`llm_provider`, `ResolvedTarget.provider`) keeps meaning the API vendor + protocol and maps onto api_mode unchanged. + +### Package layout + +``` +src/infermatrix_copilot/providers/ + __init__.py re-exports; register_builtin_providers() + base.py ProviderSpec + Transport protocol + session dataclasses + api.py provider "api": wraps the existing LLM class (both + api_modes). Wrapper, not rewrite — LLM internals, + tier_target, served-model guard, tracing all unchanged. + cursor.py M1 harness transport (cursor-agent CLI) + claude_code.py M2 harness transport (claude -p) + codex.py M3 harness transport (codex exec) + audit.py post-run session audit (productized from + eval/dataset/run_cursor_arm.py) + registry.py PROVIDER_REGISTRY + resolve_provider(settings) +src/infermatrix_copilot/tool_bridge.py + stdio MCP server exposing the run's scoped tools + (entry: python -m infermatrix_copilot.tool_bridge) +``` + +### base.py — the contracts + +```python +@dataclass(frozen=True) +class ProviderSpec: + id: str # "api" | "cursor" | "claude-code" | "codex" + kind: Literal["api", "harness"] + display: str + cli_names: tuple[str, ...] = () # binaries to probe (harness) + capabilities: frozenset[str] = frozenset() + # capability flags: "mcp_tools", "builtin_tools_off", "max_turns", + # "system_prompt", "usage_reporting", "cost_reporting" + +class Transport(Protocol): + # api kind + tool-less harness calls; signature mirrors LLM.create + def complete(self, *, system, messages, tools=None, model=None, + max_tokens=None, on_text=None, role="") -> Reply: ... + # harness kind only: one whole agent step + def run_session(self, req: AgentSessionRequest) -> AgentOutcome: ... + +@dataclass +class AgentSessionRequest: + system: str # contract preamble (harnesses without a system- + prompt: str # prompt channel prepend it to the prompt) + scope: ToolScope + bridge_spec_path: Path # written by the runner; see tool bridge + model: str + max_iters: int # mapped to --max-turns where supported, else + timeout_s: float # wall-clock timeout + budget-discipline prompt + run_dir: Path +``` + +`run_session` returns the existing `agent_loop.AgentOutcome` (text, +iterations, tool_calls, truncated, refusals, token usage, tools_used) so +everything downstream of the runner branch — `_coerce_output`, the +`agent_output` trace event, `_to_step_result` — is untouched. Fields a +harness cannot know (iterations) are best-effort; `tools_used` comes from +the bridge trace. + +### Resolution — one place, extended + +`ResolvedTarget` (config.py) gains two fields with defaults that keep every +existing construction site valid: + +```python +provider_id: str = "api" +kind: Literal["api", "harness"] = "api" +``` + +`Settings.tier_target()` stays the only place model×endpoint×credential pair +up; when `strict_backend` selects a harness provider it returns a target +with that provider id/kind and the harness model (`STRICT_BACKEND_MODEL` or +the provider default) — base_url/api_key empty (subscription auth lives in +the harness CLI, never in our config). `LLM.for_target()` is only consulted +for `kind == "api"` targets. + +Selection scope: `STRICT_BACKEND` is **required for Strict runs** — checked +in `CopilotMCP.strict_readiness()` (mcp_server.py:175, alongside the current +`shared_api_key` check, which becomes the `api`-provider readiness item) and +re-checked at `--execute-strict-reserved` startup in the child. The CLI path +(`run_task`) treats empty as `api` — no breaking change for maintainers — +and may grow a `--backend` flag later. + +### Runner branch — the single integration point + +In `run_agent_step` (engine/agent_runtime/runner.py:170), the current call + +```python +outcome = await asyncio.to_thread(run_agent, step_llm, system=..., ...) +``` + +becomes a two-way branch on the resolved target's kind. `harness` targets +write the bridge spec (below) into the run dir and call +`provider.run_session(...)` in the same worker thread. Everything before the +branch (dispatch context, evidence pack, briefing, scope binding to the +PR-time worktree root, `agent_dispatch` trace) and everything after +(`_coerce_output` with its one repair round and escalation salvage, +`agent_output` trace, skill touch) is shared — the harness rides the exact +v13 prompt bundle and output contract. + +Tool-less roles (review planner gray-zone call, ensemble reducer/merge, +coverage promotion, `_coerce_output` repair) go through a `HarnessLLM` +adapter implementing the `LLM.create()` signature: it raises if `tools` is +non-empty, runs a one-shot CLI invocation, and returns a normalized `Reply` +with usage from the CLI's JSON. Its `available` property mirrors doctor's +CLI probe so `run_agent_step`'s `ctx.llm.available` gate keeps working. For +cursor, one-shot completions run in an **empty scratch cwd** so native tools +have nothing to read. + +MoA (`for_member`) rejects harness providers as mixture members in v1 — the +reservation ledger requires per-request pricing that subscription CLIs +cannot provide. + +### The tool bridge — choke-point preservation + +`python -m infermatrix_copilot.tool_bridge --spec /bridge/.json` +is a stdio MCP server (FastMCP, already in the `[mcp]` extra) exposing +exactly the tools the step's scope allows. The spec file serializes: + +- the `ToolScope` (frozen dataclass → JSON: name, allowed_tools, read_only, + root, path_scope patterns) — trivially round-trippable; +- repo name + repo_path (the PR-time worktree), run_dir; +- which extra tool families to reconstruct (knowledge/doc/repo_map) — these + are rebuilt in the bridge process from `Settings()` + the adapter, the + same factories the runner uses (`_knowledge_tools`, `_repo_map_tool`, + `_repo_docs_tool`). + +Every MCP tool handler is a thin shim over `tools.dispatch(name, args, +scope=scope, trace=bridge_trace, extra=extra)` — refusals, out-of-scope +records, path resolution against the worktree root, and result bounding all +behave identically to the in-process loop. The bridge appends to +`bridge_trace.jsonl` in the run dir (its own file: append-only jsonl from a +second process must not interleave with the parent's `run_trace.jsonl`); +after the session the runner folds a summary (tool_calls, tools_used, +refusals) into the normal trace events. + +Guard test (the exfiltration case): a bridged `read_file` of +`~/.infermatrix-copilot/.env` — outside the scope root — is refused and +traced. + +### Environment sanitization — load-bearing + +Harness subprocesses get a constructed allowlist env (PATH, HOME, TERM, +LANG/LC_*, PYTHON* as needed, plus provider-specific vars), **never** +`os.environ` passthrough. Concrete hazards on this very machine: + +- our `.env` points `ANTHROPIC_BASE_URL` at a DeepSeek-compatible gateway — + leaked into `claude`, subscription calls would route to the wrong endpoint + and bill the wrong account; +- `ANTHROPIC_API_KEY`/`OPENAI_API_KEY` present in env would silently switch + the CLIs from subscription auth to API billing; +- `CLAUDECODE`/host markers confuse nested-CC when the Strict host *is* + Claude Code. + +The Strict child itself already receives a curated env from the server +worker (mcp_server.py:100–115); the same discipline extends one level down. + +### Guards and accounting + +- **Served-model guard**: harness CLIs report the model used in their JSON + output; recorded via the existing `llm.served` event with the same + verdict logic (mismatch → fail per `MODEL_MISMATCH_POLICY`; absent → + `unverified`, warn). +- **Usage/cost**: token counts parsed from CLI JSON (claude `-p + --output-format json` includes usage + cost; codex `--json` emits token + events; cursor stream-json includes usage). `metrics.py` records cost + source `subscription` and does not consult `MODEL_PRICES` for harness + providers — CATQ's C term must not fabricate USD. +- **Ensemble stagger** (`ensemble_stagger_seconds`) is disabled for harness + targets: there is no shared prompt-cache prefix across CLI sessions to + warm. Each delegated pass is one harness session, so context caching + within the 32-iter deep pass is the harness's own concern. +- **Concurrency**: `STRICT_BACKEND_CONCURRENCY` (default 2) is a semaphore + over concurrent sessions (deep passes + verify fan-out) — subscription + rate windows are real; auth/limit errors surface as BLOCKED, never + retried into a lockout. + +### Config schema (.env) + +``` +STRICT_BACKEND= # REQUIRED for Strict: api | cursor | claude-code | codex +STRICT_BACKEND_MODEL= # model id inside the harness (optional) +STRICT_BACKEND_CONCURRENCY=2 # concurrent harness sessions +STRICT_BACKEND_CLI= # binary path override (else PATH lookup) +STRICT_BACKEND_TIMEOUT_S=1800 # per-session wall-clock ceiling +``` + +Installer and `.env.template` write `STRICT_BACKEND=api` so existing setups +migrate with the file refresh; `strict_readiness` names the exact fix line +when it is missing. `doctor` gains per-provider checks: binary found + +version, auth status, bridge self-test (spawn, list tools, one scoped read); +`doctor --probe` performs one cheap round trip through the selected +provider. + +### Cursor transport (M1 specifics) + +- Invocation: `cursor-agent --print --output-format stream-json --force`, + prompt on stdin, `--model` from config, MCP config file pointing at the + tool bridge, cwd = the PR-time worktree. +- Parsing, session accounting, and the boundary prompt line reuse what the + Composer eval arm already learned (`eval/dataset/run_cursor_arm.py`) — + including the incident where a smoke run read `~/.claude/skills/`. +- **Open question to spike first**: whether cursor-agent can disable its + built-in tools when MCP tools are configured. If yes → preventive; if no → + the bridge is additive and `providers/audit.py` is the enforcement layer: + realpath-normalized worktree bound on every file access, write ban, + discussion-access regex; verdict + flags traced and rendered in + RUN_REPORT ("backend: cursor — native tools possible, audit: clean/N + flags"). +- No `--max-turns` equivalent → wall-clock timeout + the existing + budget-discipline prompt lines (reserve final rounds for the contract). + +### Claude Code (M2) and Codex (M3) + +- `claude -p --output-format json --mcp-config --allowedTools + --max-turns --model `; cleanest harness + citizen (built-ins fully off, native max-turns, system prompt supported, + usage+cost in output). +- `codex exec --json --sandbox read-only` + MCP servers in config; no + system-prompt channel (contract prepended to prompt); budget via timeout. + +## Milestones and acceptance + +**M1 — provider layer + cursor-agent** +1. `providers/` + registry + `ResolvedTarget` extension + `api` provider + wrapping `LLM`. Acceptance: full existing suite green with no config + change; `STRICT_BACKEND=api` byte-identical (parity ratchet). +2. Tool bridge + bridge tests. Acceptance: dispatch parity (same refusals/ + bounds as in-process), `.env` exfiltration read refused + traced. +3. Cursor transport + audit + doctor checks + readiness wiring. Acceptance: + fake-CLI offline tests green; one live Strict smoke on a small PR with + backend label, usage, and audit verdict in RUN_REPORT; unset + `STRICT_BACKEND` on a Strict start yields the exact fix-line error. + +**M2 — claude-code** · **M3 — codex**: same acceptance shape (offline +fake-CLI tests + one live smoke each); M2 additionally proves nested-CC +(Strict host = Claude Code) works under the sanitized env. + +## Test plan (offline-first, house discipline) + +- `test_providers.py` — registry resolution; strict-vs-CLI empty-selection + behavior; `HarnessLLM` raising on tools; MoA member rejection; tier + interplay (`tier_target` returning harness targets). +- `test_tool_bridge.py` — spec round-trip; scope enforcement through the + bridge; bridge trace contents. +- `test_provider_cursor.py` — fixture script emitting canned stream-json: + final-text extraction, usage parsing, timeout → truncated outcome, audit + flag surfacing. +- Doctor: per-provider check rendering + fix lines. +- Ratchets untouched: `test_repo_neutral_core`, `test_llm_providers.py`, + `test_tier_split.py`, `test_thin_mcp_server.py`, `test_mcp.py`. +- New SPEC pages under `doc/SPEC/` for `providers/` and `tool_bridge.py` + once implemented (house rule: file-level constraints live there). + +## Risks and open questions + +1. cursor-agent built-in tool restriction unknown → M1 spike; governance + fallback already decided (audit as enforcement). +2. Contract compliance without a system-prompt channel (cursor/codex) — + absorbed by the existing repair + escalation-salvage path; watched in + smoke before trusting. +3. Harness inner reasoning is invisible to our trace; only bridged tool + calls are audited. Disclosed per run — never presented as full-fidelity + Strict tracing. +4. Subscription ToS/rate limits: concurrency capped, limit errors BLOCK + loudly. Users choose their own account exposure by selecting the + backend explicitly. +5. Model naming inside harnesses (composer ids, codex model slugs) — + `doctor --probe` reports what the CLI actually serves; served-model + guard records it per call. diff --git a/src/infermatrix_copilot/cli/copilot.py b/src/infermatrix_copilot/cli/copilot.py index 6306ecf45..5565fc730 100644 --- a/src/infermatrix_copilot/cli/copilot.py +++ b/src/infermatrix_copilot/cli/copilot.py @@ -68,7 +68,11 @@ def __init__(self, settings: Settings | None = None): configured dir, and the planner over both. `last_run_dir` starts unset and is filled by the first execution.""" self.settings = settings or Settings() - self.llm = LLM(self.settings) + # provider-registry seam: a real llm.LLM under backend "api" + # (byte-identical), a HarnessLLM adapter under a harness backend + from ..providers import llm_for + + self.llm = llm_for(self.settings) self.registry = register_builtin_steps(StepRegistry()) self.store = PlaybookStore(self.settings.playbooks_dir, self.registry) self.planner = Planner(self.store, self.registry) diff --git a/src/infermatrix_copilot/cli/doctor.py b/src/infermatrix_copilot/cli/doctor.py index 0dde98ed8..aa8e26614 100644 --- a/src/infermatrix_copilot/cli/doctor.py +++ b/src/infermatrix_copilot/cli/doctor.py @@ -28,6 +28,12 @@ def _check_deps() -> tuple[bool, str]: def _check_env(settings) -> tuple[bool, str]: if not settings.shared_api_key: + if settings.strict_backend not in ("", "api"): + # harness backends carry subscription auth in the vendor CLI — + # a missing API key is a valid state, not a failure + return True, (f"no API key (valid: STRICT_BACKEND=" + f"{settings.strict_backend} uses the vendor CLI's " + "subscription auth)") return False, ( "LLM credential is empty — set ANTHROPIC_API_KEY or " "OPENAI_API_KEY in ~/.infermatrix-copilot/.env " @@ -35,6 +41,32 @@ def _check_env(settings) -> tuple[bool, str]: return True, f"{settings.resolved_llm_provider} credentials configured" +def _check_strict_backend(settings) -> tuple[bool, str]: + """Provider-registry selection (doc/RFC-provider-registry.md). Unset is + reported but not failed here — CLI runs default to api; Strict runs + refuse at strict_readiness with the same fix line.""" + backend = settings.strict_backend + if not backend: + return True, ("STRICT_BACKEND unset — CLI runs default to api; " + "Strict runs REFUSE until it is set (fix: add " + "STRICT_BACKEND=api or cursor to " + "~/.infermatrix-copilot/.env)") + if backend == "api": + return True, "backend api (raw-API path; credentials checked above)" + from ..providers import transport_for + + try: + transport = transport_for(settings) + except NotImplementedError as exc: + return False, str(exc) + cli = transport.cli_path() + if not cli: + return False, (f"backend {backend} selected but its CLI is missing — " + "fix: install it or set STRICT_BACKEND_CLI=/path/to/" + "cli in ~/.infermatrix-copilot/.env") + return True, f"backend {backend} via {cli}" + + def _check_gh() -> tuple[bool, str]: if shutil.which("gh") is None: return False, ("GitHub CLI not installed — fix: " @@ -218,6 +250,7 @@ def run_doctor(as_json: bool = False, probe: bool = False) -> int: checks = [ ("deps", _check_deps()), ("env", _check_env(settings)), + ("backend", _check_strict_backend(settings)), ("gh", _check_gh()), ("repos", _check_repos(settings)), ("backends", _check_model_backends(settings)), diff --git a/src/infermatrix_copilot/config.py b/src/infermatrix_copilot/config.py index c424793eb..df2cc2425 100644 --- a/src/infermatrix_copilot/config.py +++ b/src/infermatrix_copilot/config.py @@ -62,10 +62,19 @@ class ResolvedTarget: api_key: str source: str provider: Literal["anthropic", "openai"] = "anthropic" + # Provider registry (doc/RFC-provider-registry.md): which registry entry + # serves this target and how. `api` keeps today's exact semantics; + # `harness` targets carry NO endpoint/credential here — subscription auth + # lives inside the vendor CLI and is never resolved by Settings. + provider_id: str = "api" + kind: Literal["api", "harness"] = "api" @property def host(self) -> str: - """Endpoint hostname for traces/echo lines.""" + """Endpoint hostname for traces/echo lines (`cli:` for + harness targets, which have no HTTP endpoint of ours).""" + if self.kind == "harness": + return f"cli:{self.provider_id}" if self.base_url: return urlparse(self.base_url).netloc return ("api.openai.com" if self.provider == "openai" @@ -141,6 +150,18 @@ class Settings(BaseSettings): mcp_repo_allowlist: list[str] = [] mcp_report_max_bytes: int = 65536 + # Strict execution backend (doc/RFC-provider-registry.md): which provider + # powers runs. REQUIRED for Strict — `strict_readiness` names the exact + # fix when empty (decision: explicit selection, never a silent fallback); + # the CLI path treats empty as "api" so maintainer setups keep working. + # Harness ids (cursor / claude-code / codex) need no API key: the vendor + # CLI holds the subscription auth. + strict_backend: str = "" # "" | api | cursor | claude-code | codex + strict_backend_model: str = "" # model id INSIDE the harness (optional) + strict_backend_concurrency: int = 2 # concurrent harness sessions + strict_backend_cli: str = "" # binary path override (else PATH) + strict_backend_timeout_s: float = 1800.0 # per-session wall-clock ceiling + # Shared, human-curated knowledge base — vendored from the community docs # (see doc/KNOWLEDGE.md), organized as general/ (cross-repo experience) + # repos// (repo-specific). Adapters reference only their repos// @@ -278,6 +299,18 @@ def _parse_aliases(cls, v): "string->string equivalences") return obj + @field_validator("strict_backend") + @classmethod + def _validate_strict_backend(cls, v): + """Unknown backend ids fail at startup, not mid-run: the selection is + a routing decision and a typo must not silently mean 'api'.""" + allowed = {"", "api", "cursor", "claude-code", "codex"} + if v not in allowed: + raise ValueError( + f"STRICT_BACKEND must be one of {sorted(allowed - {''})} " + f"(or unset), got {v!r}") + return v + @model_validator(mode="after") def _validate_tier_atomicity(self): """Reject partial tier backends at startup: URL or key alone could pair @@ -386,6 +419,16 @@ def tier_target(self, mode: str, role: str = "agent") -> ResolvedTarget: shared one); `performance` with no `performance_model` raises `TierNotConfiguredError` — failing upfront replaced the silent agent_model fallback that once mislabeled a whole run.""" + backend = self.strict_backend + if backend not in ("", "api"): + # Harness target (doc/RFC-provider-registry.md): the vendor CLI + # holds the subscription credential, so base_url/api_key stay + # empty and per-tier API backends do not apply — the harness + # serves both modes (its model comes from STRICT_BACKEND_MODEL). + return ResolvedTarget(role, self.strict_backend_model, "", "", + f"backend:{backend}", + self.resolved_llm_provider, + provider_id=backend, kind="harness") if mode == "performance": if not self.performance_model: raise TierNotConfiguredError( diff --git a/src/infermatrix_copilot/engine/agent_runtime/runner.py b/src/infermatrix_copilot/engine/agent_runtime/runner.py index 63982a108..9e7354f91 100644 --- a/src/infermatrix_copilot/engine/agent_runtime/runner.py +++ b/src/infermatrix_copilot/engine/agent_runtime/runner.py @@ -167,12 +167,24 @@ async def run_agent_step( prompt = dispatch_ctx.render() if guidance: prompt += f"\n\n## STEP GUIDANCE\n{guidance}" - outcome = await asyncio.to_thread( - run_agent, - step_llm, system=system, prompt=prompt, scope=scope, - trace=ctx.trace, extra_tools=all_extra, max_iters=budget, - model=tier_model, - ) + if target.kind == "harness" and llm_override is None: + # Harness backend (doc/RFC-provider-registry.md): the vendor CLI owns + # the tool loop for this whole step — same prompt bundle and output + # contract, builtin tools served back through the MCP bridge under + # this scope. MoA members (llm_override) are raw-API endpoints and + # keep the in-process loop even under a harness backend. + from ...providers import run_harness_step + + outcome = await asyncio.to_thread( + run_harness_step, ctx, target, step_name=step_name, + system=system, prompt=prompt, scope=scope, max_iters=budget) + else: + outcome = await asyncio.to_thread( + run_agent, + step_llm, system=system, prompt=prompt, scope=scope, + trace=ctx.trace, extra_tools=all_extra, max_iters=budget, + model=tier_model, + ) output = _coerce_output(outcome.text, ctx, contract) if output is not None: output.setdefault("_tools_used", outcome.tools_used[:40]) diff --git a/src/infermatrix_copilot/mcp_server.py b/src/infermatrix_copilot/mcp_server.py index b4750f1f6..d6edb9497 100644 --- a/src/infermatrix_copilot/mcp_server.py +++ b/src/infermatrix_copilot/mcp_server.py @@ -175,12 +175,33 @@ def configure_strict_repo(self, repo: str, repo_path: str = "") -> None: def strict_readiness(self, repo: str) -> list[str]: """Return actionable setup gaps before reserving a Strict run.""" missing = [] - if not self.settings.shared_api_key: + # Backend selection is EXPLICIT for Strict (doc/RFC-provider-registry + # .md): unset refuses with the exact fix, never falls back silently. + backend = self.settings.strict_backend + if not backend: missing.append( - "model credential missing; set ANTHROPIC_API_KEY or " - "OPENAI_API_KEY in " - "~/.infermatrix-copilot/.env" - ) + "STRICT_BACKEND not set; add STRICT_BACKEND=api (or cursor / " + "claude-code / codex) to ~/.infermatrix-copilot/.env") + elif backend == "api": + if not self.settings.shared_api_key: + missing.append( + "model credential missing; set ANTHROPIC_API_KEY or " + "OPENAI_API_KEY in " + "~/.infermatrix-copilot/.env" + ) + else: + from .providers import transport_for + + try: + transport = transport_for(self.settings) + except NotImplementedError as exc: + missing.append(str(exc)) + else: + if not transport.cli_path(): + missing.append( + f"STRICT_BACKEND={backend} selected but its CLI is " + "not installed; install it or set STRICT_BACKEND_CLI " + "in ~/.infermatrix-copilot/.env") repo_path = self.copilot._resolve_repo_path(repo) if not repo_path or not Path(repo_path).is_dir(): missing.append( diff --git a/src/infermatrix_copilot/providers/__init__.py b/src/infermatrix_copilot/providers/__init__.py new file mode 100644 index 000000000..83d01b6f6 --- /dev/null +++ b/src/infermatrix_copilot/providers/__init__.py @@ -0,0 +1,76 @@ +"""Provider registry — one table of ways to reach a model +(doc/RFC-provider-registry.md). + +Public surface: the registry (`PROVIDERS`, `resolve_provider`, +`transport_for`), the contracts (`ProviderSpec`, `AgentSessionRequest`, +`HarnessTransport`), and the two integration factories the core calls: +`llm_for` (LLM construction under the selected backend) and +`run_harness_step` (the agent-runtime branch for harness targets). +""" + +from __future__ import annotations + +from .base import AgentSessionRequest, HarnessTransport, ProviderSpec +from .harness_llm import HarnessLLM +from .registry import PROVIDERS, resolve_provider, transport_for + +__all__ = [ + "PROVIDERS", "AgentSessionRequest", "HarnessLLM", "HarnessTransport", + "ProviderSpec", "llm_for", "resolve_provider", "run_harness_step", + "transport_for", +] + + +class _UnshippedTransport(HarnessTransport): + """Placeholder for declared-but-unshipped backends (claude-code/codex): + construction must degrade — `available` False, the milestone message on + any use — while `strict_readiness`/doctor report the gap with its fix.""" + + def __init__(self, settings, reason: str): + super().__init__(settings) + self._reason = reason + + def cli_path(self): + return None + + def run_session(self, req): + raise RuntimeError(self._reason) + + def complete(self, **kwargs): + raise RuntimeError(self._reason) + + +def llm_for(settings): + """The LLM-shaped client for the selected backend: the real `llm.LLM` + under `api` (parity: byte-identical construction), a `HarnessLLM` + adapter under a harness. Construction never requires the CLI (or the + transport) to exist — absence surfaces as `available == False`, the same + degrade signal an unset API key produces today.""" + if resolve_provider(settings).kind == "harness": + try: + return HarnessLLM(settings, transport_for(settings)) + except NotImplementedError as exc: + return HarnessLLM(settings, _UnshippedTransport(settings, str(exc))) + from ..llm import LLM + + return LLM(settings) + + +def run_harness_step(ctx, target, *, step_name: str, system: str, prompt: str, + scope, max_iters: int): + """Run one agent step through the selected harness: write the bridge spec + for this scope, then delegate the whole step to the transport. Returns + `agent_loop.AgentOutcome` so the runner's downstream (output coercion, + traces, salvage) is shared with the in-process loop.""" + from ..tool_bridge import write_bridge_spec + + transport = transport_for(ctx.settings) + spec = ctx.state.get("task_spec") or {} + bridge_spec = write_bridge_spec( + run_dir=ctx.run_dir, step_name=step_name, scope=scope, + repo=str(spec.get("repo") or ctx.settings.default_repo)) + return transport.run_session(AgentSessionRequest( + system=system, prompt=prompt, scope=scope, model=target.model, + max_iters=max_iters, timeout_s=ctx.settings.strict_backend_timeout_s, + run_dir=ctx.run_dir, step_name=step_name, + bridge_spec_path=bridge_spec, trace=ctx.trace)) diff --git a/src/infermatrix_copilot/providers/audit.py b/src/infermatrix_copilot/providers/audit.py new file mode 100644 index 000000000..3232991ee --- /dev/null +++ b/src/infermatrix_copilot/providers/audit.py @@ -0,0 +1,101 @@ +"""Post-run session audit for harness backends whose native tools cannot be +disabled (cursor-agent today). + +Productized from the Composer eval arm (`eval/dataset/run_cursor_arm.py`), +which caught a live out-of-bounds read (`~/.claude/skills/...`) only through +exactly this check. Product policy audited here: file reads stay inside the +session's containment roots (the PR-time worktree + the run dir), and a +read-only scope means no write/edit tool calls. The eval arm layers its own +extra rule (no PR-discussion access) on top — that is a ground-truth-leakage +concern, not a product one, and deliberately does NOT live here. + +Detective, not preventive — the disclosed fallback of the tool-governance +decision in doc/RFC-provider-registry.md. Findings are surfaced as +violations for the caller to trace and render in RUN_REPORT, never silently +dropped. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field + +# cursor-agent spools MCP tool RESULTS into its per-project state dir and the +# model reads them back with its native read tool — that read-back is how +# bridge output is consumed, not an exfiltration, so it is exempt from root +# containment (live-smoke finding: every bridge-using session was flagged). +_CLI_TOOL_SPOOL = re.compile(r"/\.cursor/projects/[^/]+/agent-tools/[^/]+$") + + +@dataclass +class SessionAudit: + """The audited facts of one harness session: rule `violations` (empty = + clean) and activity counters for the trace/report.""" + + violations: list[str] = field(default_factory=list) + shell_commands: int = 0 + file_reads: int = 0 + writes: int = 0 + other_tool_calls: int = 0 + tools_used: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return not self.violations + + @property + def tool_calls(self) -> int: + return (self.shell_commands + self.file_reads + self.writes + + self.other_tool_calls) + + +def contained_in(path: str, roots: tuple[str, ...]) -> bool: + """True when `path` lies under any of `roots`. realpath both sides: this + machine reaches the same worktree via /home and /data prefixes, and a + symlinked HOME must not read as a violation. Shared with the tool + bridge's preventive read containment.""" + real = os.path.realpath(path) + return any(real == r or real.startswith(r + os.sep) + for r in (os.path.realpath(r) for r in roots if r)) + + +def audit_events(events: list[dict], *, roots: tuple[str, ...], + read_only: bool = True) -> SessionAudit: + """Audit a cursor-agent stream-json event list. Each tool_call event + appears twice (issue + result); only the issue (no "result" key) is + counted so calls are not double-counted. + + Containment is checked on EVERY native tool call that names a path — + read, grep, ls, glob, whatever the CLI grows next — not just reads: the + live smoke showed grepToolCall events sailing past a read-only audit.""" + audit = SessionAudit() + for event in events: + if event.get("type") != "tool_call": + continue + tc = event.get("tool_call") or {} + for key, call in tc.items(): + if not isinstance(call, dict) or "result" in call: + continue + name = key.removesuffix("ToolCall") + audit.tools_used.append(name) + if name == "shell": + audit.shell_commands += 1 + continue + if name in ("write", "edit"): + audit.writes += 1 + if read_only: + audit.violations.append( + "write attempted in a read-only session") + continue + if name == "read": + audit.file_reads += 1 + else: + audit.other_tool_calls += 1 + path = str((call.get("args") or {}).get("path") or "") + if (path and roots and not contained_in(path, roots) + and not _CLI_TOOL_SPOOL.search(os.path.realpath(path))): + audit.violations.append( + f"{name} outside session roots: {path[:160]}") + break + return audit diff --git a/src/infermatrix_copilot/providers/base.py b/src/infermatrix_copilot/providers/base.py new file mode 100644 index 000000000..45396ad2e --- /dev/null +++ b/src/infermatrix_copilot/providers/base.py @@ -0,0 +1,139 @@ +"""Provider-layer contracts (doc/RFC-provider-registry.md). + +A *provider* is one way to reach a model. Two kinds exist: + +- ``api`` — a raw Anthropic/OpenAI-compatible completions endpoint (today's + `llm.LLM`, resolved by `Settings.tier_target`). Stateless + ``system+messages+tools -> tool_use`` round trips; our `agent_loop` owns + the tool loop. +- ``harness`` — a subscription-authed vendor agent CLI (cursor-agent, + claude -p, codex exec). Harnesses own their tool loop, so the integration + seam is a WHOLE agent step (`run_session`), never `LLM.create`; tool + access flows back through the MCP tool bridge so every call still passes + `tools.dispatch`. + +The credential model is deliberately asymmetric: api providers resolve +key+endpoint through Settings; harness providers hold their subscription +auth inside the vendor CLI's own state and this codebase never sees it. +""" + +from __future__ import annotations + +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Literal + +from ..scopes import ToolScope + + +@dataclass(frozen=True) +class ProviderSpec: + """One registry entry: identity, kind, and the capability flags the + integration can rely on (``mcp_tools``, ``builtin_tools_off``, + ``max_turns``, ``system_prompt``, ``usage_reporting``, + ``cost_reporting``). `cli_names` are the binaries probed on PATH for + harness kinds.""" + + id: str + kind: Literal["api", "harness"] + display: str + cli_names: tuple[str, ...] = () + capabilities: frozenset[str] = frozenset() + + +@dataclass +class AgentSessionRequest: + """One delegated agent step: the SAME prompt bundle `run_agent` would + receive (contract `system` + rendered dispatch context `prompt`), the + step's `scope`, and the session bounds. `bridge_spec_path` points at the + serialized tool-bridge spec the harness's MCP config should launch; + `max_iters` maps to a native turn cap where the harness has one + (claude --max-turns) and otherwise rides `timeout_s` plus the prompt's + budget-discipline lines.""" + + system: str + prompt: str + scope: ToolScope + model: str + max_iters: int + timeout_s: float + run_dir: Path + step_name: str = "" + bridge_spec_path: Path | None = None + trace: Any = None # RunTrace-shaped (record(kind, **fields)) or None + + +@dataclass +class SessionUsage: + """Best-effort usage extracted from a harness CLI's output. Harnesses + report unevenly; absent numbers stay 0 and cost stays None (metrics must + record source "subscription", never fabricate USD).""" + + input_tokens: int = 0 + output_tokens: int = 0 + cost_usd: float | None = None + served_model: str = "" + tools_used: list[str] = field(default_factory=list) + + +class HarnessTransport: + """Base for harness-kind transports. Subclasses set `spec` and implement + `run_session` (a whole agent step) and `complete` (one-shot TOOL-LESS + call for `HarnessLLM`). `cli_path` is the shared binary resolution: + explicit `STRICT_BACKEND_CLI` first, then the spec's names on PATH.""" + + spec: ProviderSpec + + def __init__(self, settings): + self.settings = settings + + def cli_path(self) -> str | None: + """The harness binary to execute, or None when not installed.""" + override = getattr(self.settings, "strict_backend_cli", "") + if override: + return override + for name in self.spec.cli_names: + found = shutil.which(name) + if found: + return found + return None + + def require_cli(self) -> str: + """`cli_path` or a RuntimeError naming the exact fix.""" + cli = self.cli_path() + if not cli: + raise RuntimeError( + f"{self.spec.id} backend selected but no CLI found " + f"({' / '.join(self.spec.cli_names)}) — install it or set " + "STRICT_BACKEND_CLI=/path/to/cli in ~/.infermatrix-copilot/.env") + return cli + + # -- contract ------------------------------------------------------------ + def run_session(self, req: AgentSessionRequest): + """Run one delegated agent step; returns `agent_loop.AgentOutcome`.""" + raise NotImplementedError + + def complete(self, *, system: str, messages: list[dict], + model: str = "", max_tokens: int | None = None, + role: str = ""): + """One-shot tool-less completion; returns a normalized `llm.Reply`.""" + raise NotImplementedError + + +def flatten_messages(system: str, messages: list[dict]) -> str: + """Render an internal block-protocol conversation as one prompt string for + harnesses without a messages channel. Tool-less conversations only (the + caller guarantees it): content is either a string or text blocks.""" + parts = [system] if system else [] + for message in messages: + role = str(message.get("role", "user")).upper() + content = message.get("content", "") + if isinstance(content, str): + text = content + else: + text = "\n".join(str(b.get("text", "")) for b in content + if isinstance(b, dict) and b.get("type") == "text") + if text: + parts.append(f"[{role}]\n{text}") + return "\n\n".join(parts) diff --git a/src/infermatrix_copilot/providers/cursor.py b/src/infermatrix_copilot/providers/cursor.py new file mode 100644 index 000000000..a22ee4f1a --- /dev/null +++ b/src/infermatrix_copilot/providers/cursor.py @@ -0,0 +1,198 @@ +"""Cursor harness transport — Strict on a Cursor subscription (M1). + +Invocation shape proven by the Composer eval arm +(`eval/dataset/run_cursor_arm.py`): headless `cursor-agent --print --force +--output-format stream-json`, prompt on STDIN (argv has a 128KiB per-arg +limit on Linux and evidence packs exceed it), events parsed line-wise. + +Governance (decision record in doc/RFC-provider-registry.md): cursor-agent +cannot fully disable its built-in tools, so the copilot's scoped tools are +OFFERED via the MCP tool bridge (preventive where used) and every session is +post-audited (`audit.py`) with the verdict traced — the detective fallback, +disclosed, never silent. The subprocess env is an allowlist: the vendor CLI +must keep its own subscription auth (HOME state) but must never inherit our +model-endpoint variables (this machine's ANTHROPIC_BASE_URL points at a +DeepSeek gateway) or repo credentials. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +from ..agent_loop import AgentOutcome +from ..llm import Block, Reply +from .base import AgentSessionRequest, HarnessTransport, SessionUsage, flatten_messages +from .registry import PROVIDERS + +# Env the CLI subprocess keeps. Everything else — API keys, base URLs, gh +# tokens, host markers — is dropped; subscription auth lives in HOME state. +_ENV_KEEP = {"PATH", "HOME", "TERM", "COLORTERM", "LANG", "USER", "LOGNAME", + "SHELL", "TMPDIR"} +_ENV_KEEP_PREFIXES = ("LC_", "XDG_") + + +class CursorTransport(HarnessTransport): + """cursor-agent CLI as a Strict backend.""" + + spec = PROVIDERS["cursor"] + + # -- process plumbing ---------------------------------------------------- + def _env(self) -> dict[str, str]: + return {k: v for k, v in os.environ.items() + if k in _ENV_KEEP or k.startswith(_ENV_KEEP_PREFIXES)} + + def _run(self, text: str, *, cwd: str, timeout_s: float, + model: str = "") -> tuple[list[dict], bool]: + """One CLI invocation → (parsed events, timed_out). A timeout kills + the process but keeps the partial stream — a half-done investigation + is salvage material, not garbage.""" + # --approve-mcps is load-bearing: headless runs do not auto-approve + # configured MCP servers, and without it the tool bridge is silently + # ignored (found in the live smoke — session ran on native tools only) + cmd = [self.require_cli(), "--print", "--force", "--approve-mcps", + "--output-format", "stream-json"] + selected = model or self.settings.strict_backend_model + if selected: + cmd += ["--model", selected] + timed_out = False + try: + proc = subprocess.run( + cmd, input=text, cwd=cwd, env=self._env(), + capture_output=True, text=True, encoding="utf-8", + errors="replace", timeout=timeout_s, check=False) + stdout = proc.stdout or "" + except subprocess.TimeoutExpired as exc: + timed_out = True + raw = exc.stdout or b"" + stdout = raw.decode("utf-8", "replace") if isinstance(raw, bytes) \ + else str(raw) + events: list[dict] = [] + for line in stdout.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + events.append(json.loads(line)) + except json.JSONDecodeError: + continue + return events, timed_out + + @staticmethod + def _final_text(events: list[dict]) -> str: + for event in reversed(events): + if event.get("type") == "result": + return str(event.get("result") or "").strip() + return "" + + @staticmethod + def _usage(events: list[dict]) -> SessionUsage: + """Best-effort: cursor-agent's usage/model reporting varies by + version; absent fields stay 0/empty and cost stays None.""" + usage = SessionUsage() + for event in events: + model = event.get("model") + if isinstance(model, str) and model: + usage.served_model = model + raw = event.get("usage") + if isinstance(raw, dict): + # live cursor-agent emits camelCase (inputTokens); accept + # snake_case too so a future rename does not zero the counts + usage.input_tokens += int(raw.get("inputTokens") + or raw.get("input_tokens") or 0) + usage.output_tokens += int(raw.get("outputTokens") + or raw.get("output_tokens") or 0) + return usage + + # -- MCP bridge wiring --------------------------------------------------- + def _write_mcp_config(self, cwd: Path, spec_path: Path) -> list[Path]: + """Project-scope `.cursor/mcp.json` in the session cwd pointing at the + tool bridge. Returns the paths WE created (and only those) so the + session can restore the tree afterwards — the cwd is our detached + PR-time worktree and must not accumulate config litter.""" + import sys + + created: list[Path] = [] + cursor_dir = cwd / ".cursor" + if not cursor_dir.exists(): + cursor_dir.mkdir() + created.append(cursor_dir) + config = cursor_dir / "mcp.json" + if config.exists(): # never clobber a repo-committed config + return created + package_root = Path(__file__).resolve().parents[2] + config.write_text(json.dumps({"mcpServers": {"infermatrix-tools": { + "command": sys.executable, + "args": ["-m", "infermatrix_copilot.tool_bridge", + "--spec", str(spec_path)], + "env": {"PYTHONPATH": str(package_root)}, + }}}, indent=2), encoding="utf-8") + created.insert(0, config) + return created + + # -- transport contract -------------------------------------------------- + def run_session(self, req: AgentSessionRequest) -> AgentOutcome: + from .audit import audit_events + + cwd = Path(req.scope.root or req.run_dir) + created: list[Path] = [] + if req.bridge_spec_path is not None: + created = self._write_mcp_config(cwd, req.bridge_spec_path) + try: + events, timed_out = self._run( + f"{req.system}\n\n{req.prompt}", cwd=str(cwd), + timeout_s=req.timeout_s, model=req.model) + finally: + for path in created: + if path.is_dir(): + shutil.rmtree(path, ignore_errors=True) + else: + path.unlink(missing_ok=True) + audit = audit_events(events, roots=(str(cwd), str(req.run_dir)), + read_only=req.scope.read_only) + usage = self._usage(events) + if req.trace is not None: + req.trace.record( + "harness_session", provider=self.spec.id, step=req.step_name, + audit_ok=audit.ok, audit_violations=audit.violations[:10], + shell_commands=audit.shell_commands, + file_reads=audit.file_reads, writes=audit.writes, + other_tool_calls=audit.other_tool_calls, timed_out=timed_out, + served_model=usage.served_model) + return AgentOutcome( + text=self._final_text(events), + iterations=0, # the harness does not expose its round count + tool_calls=audit.tool_calls, + truncated=timed_out, + refusals=[f"audit: {v}" for v in audit.violations], + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + tools_used=audit.tools_used[:40]) + + def complete(self, *, system: str, messages: list[dict], + model: str = "", max_tokens: int | None = None, + role: str = "") -> Reply: + """Tool-less one-shot. Runs in an EMPTY scratch cwd so cursor-agent's + native tools have nothing to read — the containment for calls that + need no repo at all (intent, reducer, repair).""" + scratch = tempfile.mkdtemp(prefix="imc-cursor-oneshot-") + try: + events, timed_out = self._run( + flatten_messages(system, messages), cwd=scratch, + timeout_s=self.settings.strict_backend_timeout_s, model=model) + finally: + shutil.rmtree(scratch, ignore_errors=True) + usage = self._usage(events) + text = self._final_text(events) + return Reply( + blocks=[Block(type="text", text=text)] if text else [], + stop_reason="max_tokens" if timed_out else "end_turn", + usage={"input_tokens": usage.input_tokens, + "output_tokens": usage.output_tokens, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0}, + model=usage.served_model) diff --git a/src/infermatrix_copilot/providers/harness_llm.py b/src/infermatrix_copilot/providers/harness_llm.py new file mode 100644 index 000000000..7c976a5ee --- /dev/null +++ b/src/infermatrix_copilot/providers/harness_llm.py @@ -0,0 +1,66 @@ +"""`HarnessLLM` — LLM-shaped adapter over a harness transport, TOOL-LESS only. + +Callers that today hold an `llm.LLM` (intent classification, ensemble +reducer/merge, output repair, chat) keep their `create()` call sites; under +a harness backend those tool-less calls become one-shot CLI invocations. +Any call that passes tools raises: agent steps must route through +`run_session` (the harness owns its loop), and a loud error here is the +guard against silently running a second, ungoverned tool loop. +""" + +from __future__ import annotations + +from .base import HarnessTransport + + +class HarnessLLM: + """Duck-typed `llm.LLM` replacement for harness backends. `available` + reflects CLI presence so every existing `ctx.llm.available` degrade path + keeps working; `for_member` still returns a REAL api-backed client — MoA + members carry their own endpoint+key and are independent of the run's + backend selection.""" + + def __init__(self, settings, transport: HarnessTransport): + self.settings = settings + self._transport = transport + + @property + def available(self) -> bool: + """True when the harness CLI is installed (auth is probed by doctor, + not here — presence is the cheap honest signal).""" + return self._transport.cli_path() is not None + + def for_target(self, target) -> HarnessLLM: + """Per-target client. Harness targets are served by this adapter; an + api target (possible when MoA/tier config mixes backends) gets a real + `llm.LLM` so it never silently rides the harness.""" + if getattr(target, "kind", "api") == "harness": + return self + from ..llm import LLM + + return LLM(self.settings).for_target(target) + + def for_member(self, member): + """MoA members are raw-API endpoints with their own credentials — + delegate to the real client; the harness never serves members.""" + from ..llm import LLM + + return LLM(self.settings).for_member(member) + + def create(self, *, system: str, messages: list[dict], + tools: list[dict] | None = None, model: str | None = None, + max_tokens: int | None = None, on_text=None, role: str = ""): + """Tool-less one-shot completion via the harness CLI. `tools` raises + by design (see module docstring).""" + if tools: + raise RuntimeError( + "harness backend supports tool-less create() only — agent " + "steps run through the provider's run_session " + "(doc/RFC-provider-registry.md)") + reply = self._transport.complete( + system=system, messages=messages, + model=model or self.settings.strict_backend_model, + max_tokens=max_tokens, role=role) + if on_text is not None and reply.text: + on_text(reply.text) + return reply diff --git a/src/infermatrix_copilot/providers/registry.py b/src/infermatrix_copilot/providers/registry.py new file mode 100644 index 000000000..fa4b8485b --- /dev/null +++ b/src/infermatrix_copilot/providers/registry.py @@ -0,0 +1,72 @@ +"""Provider registry — the one table of ways to reach a model. + +All four ids are declared so the config surface is stable from day one, but +only implemented transports resolve: `transport_for` raises a milestone +pointer for claude-code/codex (M2/M3, doc/RFC-provider-registry.md) so +`strict_readiness`/doctor report "not yet shipped" instead of a run failing +mid-flight. +""" + +from __future__ import annotations + +from .base import HarnessTransport, ProviderSpec + +PROVIDERS: dict[str, ProviderSpec] = { + p.id: p for p in ( + ProviderSpec( + id="api", kind="api", + display="Raw API (Anthropic/OpenAI-compatible, keyed in .env)"), + ProviderSpec( + id="cursor", kind="harness", + display="Cursor subscription via cursor-agent CLI", + cli_names=("cursor-agent",), + capabilities=frozenset({"mcp_tools", "usage_reporting"})), + ProviderSpec( + id="claude-code", kind="harness", + display="Claude subscription via claude CLI (M2)", + cli_names=("claude",), + capabilities=frozenset({ + "mcp_tools", "builtin_tools_off", "max_turns", + "system_prompt", "usage_reporting", "cost_reporting"})), + ProviderSpec( + id="codex", kind="harness", + display="ChatGPT subscription via codex CLI (M3)", + cli_names=("codex",), + capabilities=frozenset({"mcp_tools", "usage_reporting"})), + ) +} + +# Registered but shipping in a later milestone — transport_for names the +# milestone instead of returning a transport that cannot work. +_UNSHIPPED: dict[str, str] = {"claude-code": "M2", "codex": "M3"} + + +def resolve_provider(settings) -> ProviderSpec: + """The selected provider: `STRICT_BACKEND`, with empty meaning `api` for + the CLI path (Strict enforces explicit selection at its own boundary — + `strict_readiness` — not here).""" + selected = getattr(settings, "strict_backend", "") or "api" + spec = PROVIDERS.get(selected) + if spec is None: # Settings validation rejects unknown ids upfront + raise ValueError( + f"unknown STRICT_BACKEND {selected!r} — one of: " + + ", ".join(sorted(PROVIDERS))) + return spec + + +def transport_for(settings) -> HarnessTransport: + """The harness transport for the selected provider. Raises ValueError for + api (it has no harness transport) and NotImplementedError with the + milestone for declared-but-unshipped harnesses.""" + spec = resolve_provider(settings) + if spec.kind != "harness": + raise ValueError("provider 'api' is not a harness — resolution stays " + "with Settings.tier_target/llm.LLM") + milestone = _UNSHIPPED.get(spec.id) + if milestone: + raise NotImplementedError( + f"backend {spec.id!r} is declared but ships in {milestone} " + "(doc/RFC-provider-registry.md) — use cursor or api for now") + from .cursor import CursorTransport + + return CursorTransport(settings) diff --git a/src/infermatrix_copilot/tool_bridge.py b/src/infermatrix_copilot/tool_bridge.py new file mode 100644 index 000000000..fd94fbfa0 --- /dev/null +++ b/src/infermatrix_copilot/tool_bridge.py @@ -0,0 +1,221 @@ +"""Scoped-tool MCP bridge for harness backends (doc/RFC-provider-registry.md). + +Entry: ``python -m infermatrix_copilot.tool_bridge --spec `` +— an stdio MCP server a harness session launches from its MCP config. It +exposes the run's builtin tools plus knowledge doc search/read, and every +builtin call passes ``tools.dispatch`` with the step's deserialized +`ToolScope`, so refusals, relative-path resolution against the PR-time +worktree, and result bounds behave identically to the in-process loop. + +Two things are deliberately STRONGER than the in-process loop: + +- **Read containment.** `ToolScope` path-guards only writes; a harness is a + less-trusted caller holding an untrusted PR diff, so the bridge refuses + read/list/grep targets outside the containment roots (scope root + run + dir) — the `.env`-exfiltration guard. +- **Separate trace file.** Tool events append to ``bridge_trace.jsonl`` + next to the spec; a second process must not interleave with the parent's + ``run_trace.jsonl``. + +Known M1 gap, disclosed: the in-process extra tools built from a live +`StepContext` (skill/memory retrieval, repo_map) are not reconstructed here +— harness sessions get builtins + doc_search/doc_read. +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +from pathlib import Path + +from .run_trace import RunTrace +from .scopes import PathScope, ToolScope +from .tools import _PATH_ARGS, TOOLS, dispatch + +_SPEC_VERSION = 1 + + +# -- spec serialization ------------------------------------------------------ +def write_bridge_spec(*, run_dir: Path, step_name: str, scope: ToolScope, + repo: str) -> Path: + """Serialize one step's tool surface under ``/bridge/``. The + filename is sanitized from the step name (ensemble steps carry ``#``).""" + bridge_dir = Path(run_dir) / "bridge" + bridge_dir.mkdir(parents=True, exist_ok=True) + stem = re.sub(r"[^A-Za-z0-9._-]+", "_", step_name or "step") or "step" + path = bridge_dir / f"{stem}.json" + ps = scope.path_scope + path.write_text(json.dumps({ + "version": _SPEC_VERSION, + "scope": { + "name": scope.name, + "allowed_tools": sorted(scope.allowed_tools), + "read_only": scope.read_only, + "root": scope.root, + "path_scope": {"writable": list(ps.writable), + "primary": list(ps.primary)} if ps else None, + }, + "repo": repo, + "run_dir": str(run_dir), + "trace_path": str(Path(run_dir) / "bridge_trace.jsonl"), + }, indent=2), encoding="utf-8") + return path + + +def load_bridge_spec(path: Path) -> tuple[ToolScope, dict]: + """Deserialize a spec back into a `ToolScope` + the raw dict.""" + data = json.loads(Path(path).read_text(encoding="utf-8")) + if data.get("version") != _SPEC_VERSION: + raise ValueError(f"unsupported bridge spec version: {data.get('version')!r}") + s = data["scope"] + ps = s.get("path_scope") + scope = ToolScope( + name=str(s["name"]), + allowed_tools=frozenset(s["allowed_tools"]), + path_scope=PathScope(writable=tuple(ps["writable"]), + primary=tuple(ps["primary"])) if ps else None, + read_only=bool(s["read_only"]), + root=str(s.get("root") or ""), + ) + return scope, data + + +# -- server ------------------------------------------------------------------ +def make_dispatcher(scope: ToolScope, roots: tuple[str, ...], trace: RunTrace): + """The bridge's call path: read containment (stronger than ToolScope, + which path-guards only writes — see module docstring) and then the same + `tools.dispatch` the in-process loop uses. Errors raise so the MCP layer + marks the tool result as an error. Module-level so the guard is testable + without the MCP SDK installed.""" + from .providers.audit import contained_in + + def _call(name: str, args: dict) -> str: + key = _PATH_ARGS.get(name) + if key is not None and roots: + # mirror dispatch's relative-path resolution BEFORE containment, + # so the checked path is the one that would actually be touched + target = args.get(key) + if isinstance(target, str) and target and not os.path.isabs(target): + target = os.path.join(roots[0], target) + if isinstance(target, str) and target and not contained_in(target, roots): + trace.record("tool_refused", tool=name, + reason=f"path outside session roots: {target}") + raise RuntimeError( + f"refused: {name} target is outside this session's " + "worktree/run dir") + out = dispatch(name, args, scope=scope, trace=trace) + if not out["ok"]: + raise RuntimeError(str(out.get("error") or "tool error")) + return str(out["result"]) + + return _call + + +def build_server(spec_path: Path): + """Build the FastMCP server for one spec. Import of the MCP SDK is local + so the module stays importable (spec read/write) without the extra.""" + from mcp.server.fastmcp import FastMCP + + scope, spec = load_bridge_spec(spec_path) + trace = RunTrace(Path(spec["trace_path"])) + roots = tuple(r for r in (scope.root, spec.get("run_dir", "")) if r) + _call = make_dispatcher(scope, roots, trace) + + mcp = FastMCP("infermatrix-tool-bridge") + allowed = scope.allowed_tools & set(TOOLS) + + if "read_file" in allowed: + @mcp.tool(description=TOOLS["read_file"].description) + def read_file(path: str, offset: int = 0) -> str: + return _call("read_file", {"path": path, "offset": offset}) + + if "list_dir" in allowed: + @mcp.tool(description=TOOLS["list_dir"].description) + def list_dir(path: str) -> str: + return _call("list_dir", {"path": path}) + + if "grep" in allowed: + @mcp.tool(description=TOOLS["grep"].description) + def grep(pattern: str, path: str, regex: bool = False) -> str: + return _call("grep", {"pattern": pattern, "path": path, + "regex": regex}) + + if "write_file" in allowed: + @mcp.tool(description=TOOLS["write_file"].description) + def write_file(path: str, content: str) -> str: + return _call("write_file", {"path": path, "content": content}) + + if "edit_file" in allowed: + @mcp.tool(description=TOOLS["edit_file"].description) + def edit_file(path: str, old: str, new: str) -> str: + return _call("edit_file", {"path": path, "old": old, "new": new}) + + if "run_shell" in allowed: + @mcp.tool(description=TOOLS["run_shell"].description) + def run_shell(cmd: str, cwd: str = "") -> str: + return _call("run_shell", {"cmd": cmd, "cwd": cwd or None}) + + _register_doc_tools(mcp, spec, trace) + return mcp + + +def _register_doc_tools(mcp, spec: dict, trace: RunTrace) -> None: + """Knowledge doc search/read — same read-only surface the thin MCP + exposes, scoped to general/ + this repo's slice. Absent knowledge root + (source checkout moved) degrades to not registering, never to a crash.""" + from .config import Settings + from .knowledge_docs import KnowledgeDocs + + try: + settings = Settings() + docs = KnowledgeDocs(settings.knowledge_dir, + repo_subdir=f"repos/{spec.get('repo', '')}" + if spec.get("repo") else None) + except Exception as exc: # noqa: BLE001 — degrade, never crash the bridge + trace.record("capability_gap", capability="bridge.knowledge_docs", + effect=f"doc tools unavailable: {type(exc).__name__}: {exc}") + return + + @mcp.tool(description="Search the curated knowledge base (general + this " + "repo's slice); returns matching doc paths.") + def doc_search(query: str, limit: int = 20) -> str: + trace.record("tool_call", tool="doc_search", ok=True, + out_of_scope=False, path=None) + hits = docs.search(query, limit=limit) + return "\n".join( + f"{h.get('path')}:{h.get('line')} — {str(h.get('snippet') or '').strip()}" + for h in hits) or "(no matches)" + + @mcp.tool(description="Read a knowledge doc returned by doc_search " + "(paged; pass the previous next_offset).") + def doc_read(path: str, offset: int = 0) -> str: + trace.record("tool_call", tool="doc_read", ok=True, + out_of_scope=False, path=path) + page = docs.read(path, offset=offset) + text = str(page.get("content") or "") + nxt = page.get("next_offset") + return text + (f"\n\n[continues — doc_read offset={nxt}]" if nxt else "") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="infermatrix_copilot.tool_bridge", + description="Scoped-tool MCP bridge for harness backends (stdio).") + parser.add_argument("--spec", required=True, help="bridge spec JSON path") + args = parser.parse_args(argv) + try: + server = build_server(Path(args.spec)) + except ImportError: + import sys + + sys.stderr.write("the tool bridge needs the MCP SDK — " + "pip install 'infermatrix-copilot[mcp]'\n") + return 1 + server.run() # stdio transport by default + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/test_mcp.py b/test/test_mcp.py index ab9c8f69c..a6018623f 100644 --- a/test/test_mcp.py +++ b/test/test_mcp.py @@ -223,6 +223,7 @@ def _core(settings): def test_strict_readiness_reports_setup_gaps(settings): + settings.strict_backend = "api" core = _core(settings) missing = core.strict_readiness("vllm-omni") @@ -232,11 +233,32 @@ def test_strict_readiness_reports_setup_gaps(settings): assert any("pr-review playbook missing" in item for item in missing) +def test_strict_readiness_requires_explicit_backend(settings, monkeypatch): + # doc/RFC-provider-registry.md: selection is explicit — unset refuses + # with the exact fix, never a silent api fallback + missing = _core(settings).strict_readiness("vllm-omni") + assert any("STRICT_BACKEND not set" in item for item in missing) + + # a harness backend without its CLI is a named gap, not a key complaint + monkeypatch.setattr("infermatrix_copilot.providers.base.shutil.which", + lambda name: None) + settings.strict_backend = "cursor" + missing = _core(settings).strict_readiness("vllm-omni") + assert any("CLI is not installed" in item for item in missing) + assert not any("model credential" in item for item in missing) + + # declared-but-unshipped backends point at their milestone + settings.strict_backend = "codex" + missing = _core(settings).strict_readiness("vllm-omni") + assert any("M3" in item for item in missing) + + def test_strict_readiness_accepts_packaged_runtime(settings, tmp_path): repo = tmp_path / "vllm-omni" (repo / ".git").mkdir(parents=True) settings.repo_paths = {"vllm-omni": str(repo)} settings.anthropic_api_key = "test-key" + settings.strict_backend = "api" settings.playbooks_dir.mkdir(parents=True) shutil.copy2( Path(__file__).resolve().parents[1] / "playbooks" / "pr-review.yaml", @@ -252,6 +274,7 @@ def test_strict_readiness_accepts_openai_credential(settings, tmp_path): (repo / ".git").mkdir(parents=True) settings.repo_paths = {"vllm-omni": str(repo)} settings.openai_api_key = "test-key" + settings.strict_backend = "api" settings.playbooks_dir.mkdir(parents=True) shutil.copy2( Path(__file__).resolve().parents[1] / "playbooks" / "pr-review.yaml", diff --git a/test/test_provider_cursor.py b/test/test_provider_cursor.py new file mode 100644 index 000000000..24f16f022 --- /dev/null +++ b/test/test_provider_cursor.py @@ -0,0 +1,195 @@ +"""Cursor transport against a fake cursor-agent CLI — fully offline. + +The fake binary reproduces the real CLI's observable contract (stream-json +on stdout, prompt on stdin, cwd = the session worktree) and captures its +invocation next to itself, so assertions cover exactly the seam the real +CLI would see.""" + +import json +import os +import stat +from pathlib import Path + +from infermatrix_copilot.config import Settings +from infermatrix_copilot.providers.base import AgentSessionRequest +from infermatrix_copilot.providers.cursor import CursorTransport +from infermatrix_copilot.scopes import read_only_scope +from infermatrix_copilot.tool_bridge import write_bridge_spec + +_FAKE_CLI = """#!/usr/bin/env python3 +import json, os, sys, time +here = os.path.dirname(os.path.abspath(__file__)) +text = sys.stdin.read() +with open(os.path.join(here, "capture.json"), "w") as f: + json.dump({"argv": sys.argv[1:], "stdin": text, "cwd": os.getcwd(), + "env_anthropic": os.environ.get("ANTHROPIC_BASE_URL", "")}, f) +if os.path.exists(os.path.join(here, "sleep")): + time.sleep(10) +events = os.path.join(here, "events.jsonl") +if os.path.exists(events): + sys.stdout.write(open(events).read()) +else: + mcp = "yes" if os.path.exists(os.path.join(os.getcwd(), ".cursor", "mcp.json")) else "no" + print(json.dumps({"type": "tool_call", "tool_call": {"readToolCall": { + "args": {"path": os.path.join(os.getcwd(), "somefile.py")}}}})) + print(json.dumps({"type": "result", "result": "REVIEW mcp=" + mcp, + "model": "composer-2.5", + "usage": {"inputTokens": 10, "outputTokens": 5}})) +""" + + +class FakeTrace: + def __init__(self): + self.events = [] + + def record(self, kind, **fields): + self.events.append({"kind": kind, **fields}) + + +def _fake_cli(tmp_path: Path) -> Path: + cli = tmp_path / "bin" / "cursor-agent" + cli.parent.mkdir() + cli.write_text(_FAKE_CLI, encoding="utf-8") + cli.chmod(cli.stat().st_mode | stat.S_IXUSR) + return cli + + +def _transport(tmp_path: Path, **kw) -> CursorTransport: + return CursorTransport(Settings( + _env_file=None, strict_backend="cursor", + strict_backend_cli=str(_fake_cli(tmp_path)), + strict_backend_model="composer-2.5", **kw)) + + +def _request(tmp_path: Path, transport: CursorTransport, + with_bridge: bool = True) -> AgentSessionRequest: + worktree = tmp_path / "worktree" + worktree.mkdir() + run_dir = tmp_path / "run" + run_dir.mkdir() + scope = read_only_scope() + scope = type(scope)(name=scope.name, allowed_tools=scope.allowed_tools, + read_only=True, root=str(worktree)) + bridge = write_bridge_spec(run_dir=run_dir, step_name="agent.review_diff", + scope=scope, repo="vllm-omni") \ + if with_bridge else None + return AgentSessionRequest( + system="SYS", prompt="PROMPT", scope=scope, model="composer-2.5", + max_iters=8, timeout_s=30.0, run_dir=run_dir, + step_name="agent.review_diff", bridge_spec_path=bridge, + trace=FakeTrace()) + + +def test_run_session_end_to_end(tmp_path, monkeypatch): + monkeypatch.setenv("ANTHROPIC_BASE_URL", "https://wrong.gateway/v1") + transport = _transport(tmp_path) + req = _request(tmp_path, transport) + + outcome = transport.run_session(req) + + # the harness saw the MCP bridge config; the worktree was restored after + assert outcome.text == "REVIEW mcp=yes" + assert not (Path(req.scope.root) / ".cursor").exists() + assert outcome.tool_calls == 1 and outcome.truncated is False + assert (outcome.input_tokens, outcome.output_tokens) == (10, 5) + assert outcome.refusals == [] # read stayed inside the worktree + + capture = json.loads( + (tmp_path / "bin" / "capture.json").read_text(encoding="utf-8")) + assert capture["cwd"] == os.path.realpath(req.scope.root) or \ + capture["cwd"] == req.scope.root # session runs IN the worktree + assert capture["stdin"].startswith("SYS\n\nPROMPT") + assert "--model" in capture["argv"] + assert capture["env_anthropic"] == "" # sanitized env: no gateway leak + + session_events = [e for e in req.trace.events + if e["kind"] == "harness_session"] + assert session_events and session_events[0]["audit_ok"] is True + assert session_events[0]["served_model"] == "composer-2.5" + + +def test_run_session_flags_out_of_bounds_read(tmp_path): + transport = _transport(tmp_path) + req = _request(tmp_path, transport, with_bridge=False) + (tmp_path / "bin" / "events.jsonl").write_text("\n".join([ + json.dumps({"type": "tool_call", "tool_call": {"readToolCall": { + "args": {"path": "/etc/passwd"}}}}), + json.dumps({"type": "result", "result": "REVIEW"}), + ]) + "\n", encoding="utf-8") + + outcome = transport.run_session(req) + + assert outcome.text == "REVIEW" + assert any("outside session roots" in r for r in outcome.refusals) + session = [e for e in req.trace.events if e["kind"] == "harness_session"] + assert session[0]["audit_ok"] is False + + +def test_run_session_timeout_is_truncated_not_a_crash(tmp_path): + transport = _transport(tmp_path) + req = _request(tmp_path, transport, with_bridge=False) + req.timeout_s = 0.8 + (tmp_path / "bin" / "sleep").write_text("", encoding="utf-8") + + outcome = transport.run_session(req) + + assert outcome.truncated is True and outcome.text == "" + + +def test_complete_flattens_messages_and_runs_in_scratch_cwd(tmp_path): + transport = _transport(tmp_path) + + reply = transport.complete( + system="CLASSIFY", messages=[{"role": "user", "content": "review pr 5"}]) + + assert reply.text.startswith("REVIEW") + capture = json.loads( + (tmp_path / "bin" / "capture.json").read_text(encoding="utf-8")) + assert "[USER]\nreview pr 5" in capture["stdin"] + assert "CLASSIFY" in capture["stdin"] + # one-shots run in an empty scratch dir, never in a repo + assert "imc-cursor-oneshot-" in capture["cwd"] + assert reply.usage["input_tokens"] == 10 + + +def test_audit_checks_paths_on_all_native_tools(tmp_path): + # live-smoke lesson: grep/ls/glob calls carry paths too — a read-only + # audit that checks only readToolCall lets an out-of-tree grep through + from infermatrix_copilot.providers.audit import audit_events + + events = [ + {"type": "tool_call", "tool_call": {"grepToolCall": { + "args": {"path": "/etc", "pattern": "key"}}}}, + {"type": "tool_call", "tool_call": {"lsToolCall": { + "args": {"path": str(tmp_path)}}}}, + ] + audit = audit_events(events, roots=(str(tmp_path),)) + assert audit.other_tool_calls == 2 + assert audit.violations == ["grep outside session roots: /etc"] + assert audit.tools_used == ["grep", "ls"] + + +def test_audit_exempts_cli_mcp_result_spool(tmp_path): + # cursor-agent buffers MCP tool results under ~/.cursor/projects/

/ + # agent-tools/ and reads them back — that is bridge-output consumption, + # not an out-of-tree read + from infermatrix_copilot.providers.audit import audit_events + + spool = "/home/u/.cursor/projects/some-worktree/agent-tools/abc123.txt" + events = [{"type": "tool_call", "tool_call": {"readToolCall": { + "args": {"path": spool}}}}] + audit = audit_events(events, roots=(str(tmp_path),)) + assert audit.ok and audit.file_reads == 1 + + +def test_cli_absence_is_a_named_error(tmp_path, monkeypatch): + monkeypatch.setattr("infermatrix_copilot.providers.base.shutil.which", + lambda name: None) + transport = CursorTransport(Settings(_env_file=None, + strict_backend="cursor")) + req = _request(tmp_path, transport, with_bridge=False) + try: + transport.run_session(req) + raise AssertionError("expected RuntimeError") + except RuntimeError as exc: + assert "STRICT_BACKEND_CLI" in str(exc) diff --git a/test/test_providers.py b/test/test_providers.py new file mode 100644 index 000000000..1167543fb --- /dev/null +++ b/test/test_providers.py @@ -0,0 +1,160 @@ +"""Provider registry: selection, harness targets, the HarnessLLM adapter, and +the run_harness_step seam (doc/RFC-provider-registry.md).""" + +from types import SimpleNamespace + +import pytest + +from infermatrix_copilot import providers +from infermatrix_copilot.config import Settings +from infermatrix_copilot.llm import LLM, Block, Reply +from infermatrix_copilot.providers import ( + HarnessLLM, + llm_for, + resolve_provider, + transport_for, +) +from infermatrix_copilot.scopes import read_only_scope + + +def _settings(**kw) -> Settings: + return Settings(_env_file=None, **kw) + + +class FakeTransport: + def __init__(self): + self.calls = [] + self.cli = "/fake/cursor-agent" + + def cli_path(self): + return self.cli + + def complete(self, **kwargs): + self.calls.append(kwargs) + return Reply(blocks=[Block(type="text", text="ok")]) + + def run_session(self, req): + from infermatrix_copilot.agent_loop import AgentOutcome + + self.calls.append(req) + return AgentOutcome(text='{"status": "success"}', iterations=1, + tool_calls=0) + + +# -- selection --------------------------------------------------------------- +def test_empty_selection_resolves_api(): + assert resolve_provider(_settings()).id == "api" + assert resolve_provider(_settings()).kind == "api" + + +def test_unknown_backend_rejected_at_startup(): + with pytest.raises(Exception, match="STRICT_BACKEND"): + _settings(strict_backend="not-a-backend") + + +def test_unshipped_backends_name_their_milestone(): + with pytest.raises(NotImplementedError, match="M2"): + transport_for(_settings(strict_backend="claude-code")) + with pytest.raises(NotImplementedError, match="M3"): + transport_for(_settings(strict_backend="codex")) + + +def test_api_provider_has_no_harness_transport(): + with pytest.raises(ValueError, match="not a harness"): + transport_for(_settings(strict_backend="api")) + + +# -- harness targets --------------------------------------------------------- +def test_harness_target_shape_and_host_label(): + settings = _settings(strict_backend="cursor", + strict_backend_model="composer-2.5") + target = settings.tier_target("eco") + assert (target.kind, target.provider_id) == ("harness", "cursor") + assert target.model == "composer-2.5" + assert (target.base_url, target.api_key) == ("", "") # auth lives in the CLI + assert target.host == "cli:cursor" + assert target.source == "backend:cursor" + + +def test_harness_serves_performance_mode_without_tier_config(): + # tier backends do not apply under a harness — performance must not raise + target = _settings(strict_backend="cursor").tier_target("performance") + assert target.kind == "harness" + + +def test_api_target_unchanged_by_default(): + target = _settings(anthropic_api_key="k").tier_target("eco") + assert (target.kind, target.provider_id) == ("api", "api") + assert target.host == "api.anthropic.com" + + +# -- llm_for ----------------------------------------------------------------- +def test_llm_for_api_returns_the_real_client(): + assert isinstance(llm_for(_settings(anthropic_api_key="k")), LLM) + + +def test_llm_for_harness_returns_adapter_and_absence_degrades(monkeypatch): + monkeypatch.setattr("infermatrix_copilot.providers.base.shutil.which", + lambda name: None) + adapter = llm_for(_settings(strict_backend="cursor")) + assert isinstance(adapter, HarnessLLM) + assert adapter.available is False # missing CLI == missing key: degrade + + +# -- HarnessLLM -------------------------------------------------------------- +def test_harness_llm_refuses_tools(): + adapter = HarnessLLM(_settings(strict_backend="cursor"), FakeTransport()) + with pytest.raises(RuntimeError, match="tool-less"): + adapter.create(system="s", messages=[], tools=[{"name": "grep"}]) + + +def test_harness_llm_delegates_toolless_create_and_streams(): + transport = FakeTransport() + adapter = HarnessLLM( + _settings(strict_backend="cursor", strict_backend_model="m1"), + transport) + streamed = [] + reply = adapter.create(system="s", + messages=[{"role": "user", "content": "hi"}], + on_text=streamed.append) + assert reply.text == "ok" and streamed == ["ok"] + assert transport.calls[0]["model"] == "m1" # backend model applied + + +def test_harness_llm_members_get_a_real_api_client(): + adapter = HarnessLLM(_settings(strict_backend="cursor"), FakeTransport()) + member = SimpleNamespace(api_key="k", base_url="") + assert isinstance(adapter.for_member(member), LLM) + + +def test_harness_llm_for_target_routes_api_targets_to_real_client(): + adapter = HarnessLLM(_settings(strict_backend="cursor"), FakeTransport()) + api_target = SimpleNamespace(kind="api", model="m", base_url="", + api_key="k", provider="anthropic") + assert isinstance(adapter.for_target(api_target), LLM) + harness_target = SimpleNamespace(kind="harness") + assert adapter.for_target(harness_target) is adapter + + +# -- run_harness_step seam --------------------------------------------------- +def test_run_harness_step_writes_bridge_spec_and_delegates(tmp_path, monkeypatch): + transport = FakeTransport() + monkeypatch.setattr(providers, "transport_for", lambda s: transport) + settings = _settings(strict_backend="cursor", + strict_backend_timeout_s=42.0) + ctx = SimpleNamespace(settings=settings, + state={"task_spec": {"repo": "vllm-omni"}}, + run_dir=tmp_path, trace=None) + scope = read_only_scope() + target = settings.tier_target("eco") + + outcome = providers.run_harness_step( + ctx, target, step_name="agent.review_diff#lens0", + system="SYS", prompt="PROMPT", scope=scope, max_iters=7) + + assert outcome.text == '{"status": "success"}' + req = transport.calls[0] + assert (req.system, req.prompt, req.max_iters) == ("SYS", "PROMPT", 7) + assert req.timeout_s == 42.0 + assert req.bridge_spec_path is not None and req.bridge_spec_path.is_file() + assert req.bridge_spec_path.parent == tmp_path / "bridge" diff --git a/test/test_tool_bridge.py b/test/test_tool_bridge.py new file mode 100644 index 000000000..992f32847 --- /dev/null +++ b/test/test_tool_bridge.py @@ -0,0 +1,88 @@ +"""Tool-bridge spec round-trip and the read-containment guard +(doc/RFC-provider-registry.md — the `.env`-exfiltration case).""" + +from pathlib import Path + +import pytest + +from infermatrix_copilot.run_trace import RunTrace +from infermatrix_copilot.scopes import ToolScope, read_only_scope +from infermatrix_copilot.tool_bridge import ( + load_bridge_spec, + make_dispatcher, + write_bridge_spec, +) + + +def _scoped(tmp_path: Path) -> tuple[ToolScope, Path, Path]: + worktree = tmp_path / "worktree" + worktree.mkdir() + run_dir = tmp_path / "run" + run_dir.mkdir() + scope = read_only_scope() + scope = type(scope)(name=scope.name, allowed_tools=scope.allowed_tools, + read_only=True, root=str(worktree)) + return scope, worktree, run_dir + + +def test_bridge_spec_round_trips(tmp_path): + scope, _worktree, run_dir = _scoped(tmp_path) + spec_path = write_bridge_spec(run_dir=run_dir, + step_name="agent.review_diff#lens0", + scope=scope, repo="vllm-omni") + loaded, raw = load_bridge_spec(spec_path) + assert loaded == scope + assert raw["repo"] == "vllm-omni" + assert raw["run_dir"] == str(run_dir) + assert "lens0" in spec_path.name and "#" not in spec_path.name + + +def test_bridged_read_inside_root_passes_and_relative_resolves(tmp_path): + scope, worktree, run_dir = _scoped(tmp_path) + (worktree / "mod.py").write_text("x = 1\n", encoding="utf-8") + trace = RunTrace(run_dir / "bridge_trace.jsonl") + call = make_dispatcher(scope, (str(worktree), str(run_dir)), trace) + + assert call("read_file", {"path": str(worktree / "mod.py")}) == "x = 1\n" + # repo-relative paths resolve against the worktree, as in-process + assert call("read_file", {"path": "mod.py"}) == "x = 1\n" + assert any(e["tool"] == "read_file" + for e in trace.events("tool_call")) + + +def test_bridged_read_outside_roots_is_refused_and_traced(tmp_path): + scope, worktree, run_dir = _scoped(tmp_path) + secret = tmp_path / "home" / ".infermatrix-copilot" / ".env" + secret.parent.mkdir(parents=True) + secret.write_text("ANTHROPIC_API_KEY=leak-me\n", encoding="utf-8") + trace = RunTrace(run_dir / "bridge_trace.jsonl") + call = make_dispatcher(scope, (str(worktree), str(run_dir)), trace) + + with pytest.raises(RuntimeError, match="refused"): + call("read_file", {"path": str(secret)}) + refusals = list(trace.events("tool_refused")) + assert refusals and "outside session roots" in refusals[0]["reason"] + + +def test_scope_tool_allowlist_still_enforced(tmp_path): + scope, worktree, run_dir = _scoped(tmp_path) + trace = RunTrace(run_dir / "bridge_trace.jsonl") + call = make_dispatcher(scope, (str(worktree), str(run_dir)), trace) + + # read_only_scope has no run_shell — dispatch refuses by tool name; the + # bridge surfaces it as an error, identically to the in-process loop + with pytest.raises(RuntimeError, match="not allowed"): + call("run_shell", {"cmd": "echo hi"}) + + +def test_bridged_write_refused_in_read_only_scope(tmp_path): + scope, worktree, run_dir = _scoped(tmp_path) + scope = type(scope)(name=scope.name, + allowed_tools=scope.allowed_tools | {"write_file"}, + read_only=True, root=str(worktree)) + trace = RunTrace(run_dir / "bridge_trace.jsonl") + call = make_dispatcher(scope, (str(worktree), str(run_dir)), trace) + + with pytest.raises(RuntimeError, match="read-only"): + call("write_file", {"path": str(worktree / "a.txt"), "content": "x"}) + assert not (worktree / "a.txt").exists()