From 3daa9f994fd543a304d7d34142ca2bfd2f205e86 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Mon, 29 Jun 2026 14:42:42 -0400 Subject: [PATCH 01/45] refactor(instant-magic): reduce plan mode to single harness-agnostic flow --- README.md | 6 +++--- docs/harness-parity.md | 4 ++-- profiles/claude-code/plan-mode.md | 11 ----------- profiles/codex/plan-mode.md | 11 ----------- profiles/opencode/plan-mode.md | 11 ----------- profiles/shared/plan-mode.md | 11 +++++++++++ src/adapters/harness.rs | 12 ------------ src/cli/args.rs | 12 ++++++------ src/cli/run/orchestrate/stage.rs | 4 ++-- src/cli/run/util.rs | 27 +++++++++++---------------- tests/cli/package.rs | 2 +- tests/run/codex.rs | 6 ++++-- tests/run/opencode.rs | 3 ++- tests/run/staging.rs | 4 +++- 14 files changed, 45 insertions(+), 79 deletions(-) delete mode 100644 profiles/claude-code/plan-mode.md delete mode 100644 profiles/codex/plan-mode.md delete mode 100644 profiles/opencode/plan-mode.md create mode 100644 profiles/shared/plan-mode.md diff --git a/README.md b/README.md index 4d8ec07..36ad482 100644 --- a/README.md +++ b/README.md @@ -308,7 +308,7 @@ Project-local staged skills live in the isolated env at `env/.claude/skills/`, i **Where transcripts live.** Claude Code persists subagent transcripts under `~/.claude/projects///subagents/`. `ingest` auto-resolves this from the `CLAUDE_CODE_SESSION_ID` the orchestrating session exports (deriving `` from the cwd and scanning `projects/*` if needed), so you normally don't pass `--subagents-dir` at all. When running outside that session — or to target a past session — pass `--session-id `, or override the lookup entirely with `--subagents-dir `. Besides out-of-bounds writes, `detect-stray-writes` also flags **live-source reads**: an arm whose subagent read the live skill source instead of its staged copy. That usually means the Skill tool couldn't resolve the staged slug yet and the agent improvised — fatal in revision mode, where the `old_skill` arm then sees new-skill content. Treat a flagged cell's arm as contaminated. -**Dispatching via the Task tool.** `dispatch.json` is a top-level object (`{ skill_name, iteration, run_nonce, …, tasks: [...] }`); iterate `tasks[]`. For each task, dispatch a fresh subagent via the Task tool with the prompt `Read the file at and follow its instructions exactly.` (substituting the task's `dispatch_prompt_path`), and pass `agent_description` *verbatim* as the description — it's namespaced `::i-`, and passing it unchanged is what lets transcript correlation work. (The Task tool documents `description` as "short", but pass the full string regardless — correlation depends on the exact value.) You do **not** write `run.json`/`timing.json` yourself; the subagent writes `outputs/final-message.md`, and `ingest` (`record-runs`) assembles both records from disk. For a plan-mode-relevant skill, add `--plan-mode` to inject Claude Code's verbatim plan-mode procedure as a `` operating-context layer. +**Dispatching via the Task tool.** `dispatch.json` is a top-level object (`{ skill_name, iteration, run_nonce, …, tasks: [...] }`); iterate `tasks[]`. For each task, dispatch a fresh subagent via the Task tool with the prompt `Read the file at and follow its instructions exactly.` (substituting the task's `dispatch_prompt_path`), and pass `agent_description` *verbatim* as the description — it's namespaced `::i-`, and passing it unchanged is what lets transcript correlation work. (The Task tool documents `description` as "short", but pass the full string regardless — correlation depends on the exact value.) You do **not** write `run.json`/`timing.json` yourself; the subagent writes `outputs/final-message.md`, and `ingest` (`record-runs`) assembles both records from disk. For a plan-mode-relevant skill, add `--plan-mode` to inject the shared plan-mode procedure as a `` operating-context layer. **Hybrid mode (`--run-mode hybrid`).** Pass `--run-mode hybrid` to dispatch each task through the `claude -p` one-shot CLI instead of in-session subagents — the same shape as Codex's hybrid flow, where an agent session orchestrates while each test/judge shells out to the CLI. `run` then prints (and `dispatch-manifest.md` / `RUNBOOK.md` carry) a `claude -p` recipe per task: @@ -371,7 +371,7 @@ eval-magic ingest --harness codex Judge tasks follow the same model-selection rule. `run --judge-model ` becomes the default `model` in `judge-tasks.json`; an individual `llm_judge.model` overrides it. The Codex judge recipe reads each task's resolved `model` and passes `-m "$model"` only when one is present. -`finalize` and `teardown` work the same with `--harness codex`. Codex results are lower fidelity than Claude Code in a few places: `transcript_check` matches parsed `item.completed` entries (`command_execution`, `file_change`, `web_search`, MCP items); the automatic `__skill_invoked` meta-check uses the LLM-judge fallback (Codex's JSONL exposes no deterministic skill-tool event); and `--plan-mode` injects Codex's plan-mode procedure as text rather than launching `codex exec` into the interactive CLI's real `/plan` mode. `--guard` stages a Codex `PreToolUse` hook that blocks out-of-sandbox `Bash` mutations and `apply_patch` targets before they run; `detect-stray-writes` remains the post-run audit. Bias Codex suites toward `llm_judge` assertions for behavior and `transcript_check` for tool events. +`finalize` and `teardown` work the same with `--harness codex`. Codex results are lower fidelity than Claude Code in a few places: `transcript_check` matches parsed `item.completed` entries (`command_execution`, `file_change`, `web_search`, MCP items); the automatic `__skill_invoked` meta-check uses the LLM-judge fallback (Codex's JSONL exposes no deterministic skill-tool event); and `--plan-mode` injects the shared plan-mode procedure as text rather than launching `codex exec` into the interactive CLI's real `/plan` mode. `--guard` stages a Codex `PreToolUse` hook that blocks out-of-sandbox `Bash` mutations and `apply_patch` targets before they run; `detect-stray-writes` remains the post-run audit. Bias Codex suites toward `llm_judge` assertions for behavior and `transcript_check` for tool events. When running `eval-magic run --harness codex` from inside Codex itself, staging writes `.agents/skills`; adding `--guard` also writes `.codex/hooks.json`. Those project-local Codex config paths are protected by Codex's default workspace-write sandbox, so the runner may need approval/escalation or an external terminal invocation. That approval is Codex's own permission boundary, not something eval-magic bypasses. @@ -396,7 +396,7 @@ OpenCode skill names must be lowercase alphanumeric with single-hyphen separator ## Bundled assets - `schema/` — JSON Schemas for every artifact (`evals`, run records, `grading`, `stray-writes`, `benchmark`, `judge-tasks`); the portable cross-harness contract, embedded in the binary -- `profiles/` — per-harness plan-mode procedure profiles (`--plan-mode`), embedded in the binary +- `profiles/` — the shared plan-mode procedure profile (`--plan-mode`) and runbook templates, embedded in the binary ## Development diff --git a/docs/harness-parity.md b/docs/harness-parity.md index 15be4bf..fdee31f 100644 --- a/docs/harness-parity.md +++ b/docs/harness-parity.md @@ -83,14 +83,14 @@ For each `HarnessAdapter` method below, compare your harness's impl against the | Skill-eval auto-record (run/timing assembly) | (consumes `parse_transcript_full` / `parse_cli_events_full` + `cli_events_filename`) | `src/pipeline/record_runs.rs` assembles each task's `run.json` + `timing.json` from disk: carry-over fields from `dispatch.json`, `final_message` from `outputs/final-message.md` (falling back to the transcript's final text — the primary path for `claude -p`, which has no `--output-last-message`), and tool invocations/tokens/duration from the parsed transcript. A harness closes this gap by supplying the transcript its parser consumes (the portable fallback — hand-authored records against `run-record.schema.json` — is unchanged) | | Cli model selection | `cli_model_flag`, `cli_next_steps`, `cli_manifest_section`, `cli_judge_next_steps` | For one-shot CLI dispatch, `run --agent-model` is rendered into the agent command recipe and `run --judge-model` becomes the default model in `judge-tasks.json`; assertion-level `llm_judge.model` remains the most specific override. Codex is the reference: `cli_model_flag` returns `-m`, agent recipes render `codex --ask-for-approval never exec ... -m `, and judge recipes read each task's resolved `model` and pass `-m "$model"` only when present | | Eval subagent write enforcement | `install_guard` | Opt-in `--guard` stages a pre-tool hook (`src/sandbox/`) that *denies* subagent writes/installs outside the eval sandbox while dispatches run. Portable fallback for every harness: the `eval-magic detect-stray-writes` post-pass (`src/pipeline/detect_stray_writes.rs`) flags out-of-bounds writes from the parsed transcript after the fact | -| Eval plan-mode operating context | `plan_mode_profile`, `render_plan_mode_context` | Opt-in `--plan-mode` injects the harness's `profiles//plan-mode.md` (embedded at compile time) as a `` operating-context layer in every dispatch. Claude Code and Codex profiles exist today; a harness with no profile has no `--plan-mode` and an unchanged dispatch contract | +| Eval plan-mode operating context | `render_plan_mode_context` | Opt-in `--plan-mode` injects the shared `profiles/shared/plan-mode.md` (embedded at compile time) as a `` operating-context layer in every dispatch — identical text for every harness | | Harness-details operator guide | (docs, not a trait method) | The README's per-harness section, e.g. [Claude Code](../README.md#claude-code-fully-wired) | **Note on the transcript adapter (raised bar).** Baseline eval suites use `transcript_check` assertions — deterministic regex checks against a run's tool invocations (e.g. "a test command ran", "the sibling skill was loaded"). These only grade when `parse_transcript` is implemented for your harness. A harness without it still functions: those assertions grade as *unverifiable* and the `llm_judge` assertions carry the substantive measurement. But adapter richness is an explicit parity target, not optional polish — implementing or enriching `parse_transcript*` lets more of a baseline suite grade mechanically. Treat it as a goal to aim at, not a box already checked. **Note on write enforcement (parity goal).** Eval subagents are instructed to write only inside their `outputs/` dir, but nothing in the portable contract *enforces* it — a misbehaving subagent can edit the real repo or install packages, silently tainting the run. Two layers address this: the portable `detect-stray-writes` post-pass (available to every harness, since it works off the same parsed transcript) and an opt-in harness-native `install_guard` that stages a pre-tool hook to *block* the write before it happens. Claude Code and Codex both wire this through their `PreToolUse` hook surfaces — for Claude Code across *both* mechanisms, since `claude -p` loads the same project-local `settings.local.json` hook from the env cwd each dispatch runs in. **Harness-level tool enforcement is an explicit parity goal, not optional polish.** A harness that can express a pre-tool guard (a hook, a permission rule, a sandboxed cwd) should wire `install_guard`; until then, `detect-stray-writes` is the honest fallback. -**Note on plan-mode fidelity (residual parity goal).** `--plan-mode` injects a harness's *verbatim* plan-mode procedure as operating context, the closest a harness's eval runner can get to reproducing the wild failure where a real plan mode makes loading a skill feel redundant. It is **not** the real mode: it is still text the dispatched subagent reads, not a state the harness places it under, so a pass remains necessary-not-sufficient (the seeding ceiling is explained in the [`slow-powers`](https://github.com/slowdini/slow-powers) `evaluating-skills` skill). A harness that can dispatch an eval subagent *into* its own plan/research mode would close this gap; `--plan-mode` (a profile + renderer) is the approximation every harness can reach in the meantime. +**Note on plan-mode fidelity (residual parity goal).** `--plan-mode` injects a shared, harness-agnostic plan-mode procedure as operating context, the closest a harness's eval runner can get to reproducing the wild failure where a real plan mode makes loading a skill feel redundant. It is **not** the real mode: it is still text the dispatched subagent reads, not a state the harness places it under, so a pass remains necessary-not-sufficient (the seeding ceiling is explained in the [`slow-powers`](https://github.com/slowdini/slow-powers) `evaluating-skills` skill). A harness that can dispatch an eval subagent *into* its own plan/research mode would close this gap; `--plan-mode` (a profile + renderer) is the approximation every harness can reach in the meantime. Surface your findings inline using this template: diff --git a/profiles/claude-code/plan-mode.md b/profiles/claude-code/plan-mode.md deleted file mode 100644 index 9c153a8..0000000 --- a/profiles/claude-code/plan-mode.md +++ /dev/null @@ -1,11 +0,0 @@ -Plan mode is active. The user wants to review an approach before any code is written, so you must NOT execute yet: do not make any edits, do not run any non-read-only tool, and do not change configs or system state. The only file you may write is the plan file. This constraint supersedes any other instruction you have received this session. - -You are operating inside the harness's plan-mode workflow — a fixed, multi-phase procedure. Work through the phases in order: - -1. **Understand.** Read the relevant code and gather context with read-only tools until you can describe the change concretely. Reuse what already exists rather than proposing new code. -2. **Design.** Decide the implementation approach and the trade-offs. -3. **Review.** Re-check the design against the user's request and resolve open questions with the user before finalizing. -4. **Write the plan.** Build the plan up incrementally in the plan file — this is the one file you are permitted to write. Name the files to change and how to verify the result. -5. **Hand off.** Call ExitPlanMode to submit the plan for the user's approval. - -Terminal rail: your turn must end in exactly one of two ways — by asking the user a question, or by calling ExitPlanMode to present the finished plan. Do not stop for any other reason and do not begin implementation until the user has approved the plan. The plan-mode workflow already governs how you research, design, and present the work; stay on this rail through to ExitPlanMode. diff --git a/profiles/codex/plan-mode.md b/profiles/codex/plan-mode.md deleted file mode 100644 index 868cf4e..0000000 --- a/profiles/codex/plan-mode.md +++ /dev/null @@ -1,11 +0,0 @@ -Codex plan mode is active. The user wants to review an approach before implementation, so do not implement yet: do not edit repo-tracked files, run formatters that rewrite files, apply migrations, generate code, install dependencies, or otherwise change product state. You may inspect files and run non-mutating checks or tests that write only normal build/cache artifacts. The only permitted task writes are explicit plan artifacts requested by the user or required final-response output files. - -Work through the planning flow in order: - -1. **Ground in the environment.** Read the relevant files, configs, schemas, docs, and tests before asking questions when local inspection can answer them. Prefer existing patterns over new abstractions. -2. **Clarify intent.** Ask only questions that materially change the plan, confirm an important assumption, or choose between real tradeoffs. If a question can be resolved from the repo, inspect instead. -3. **Design the implementation.** Specify the exact behavior, interfaces, data flow, edge cases, failure modes, compatibility concerns, and verification strategy. Choose conservative defaults when the repo already points to one clear path. -4. **Review the plan.** Remove placeholders and vague instructions. Ensure every referenced existing file is real, every step maps to the requested goal, and the plan does not defer decisions to implementation time. -5. **Present the plan.** End with exactly one decision-complete `` block. Include a short title, summary, public API/interface changes, test scenarios, and assumptions. Do not begin implementation until the user approves. - -If you do not yet have enough information to produce a decision-complete plan, ask the user the smallest necessary question and stop. If the plan is ready, present only the `` block. diff --git a/profiles/opencode/plan-mode.md b/profiles/opencode/plan-mode.md deleted file mode 100644 index 576acac..0000000 --- a/profiles/opencode/plan-mode.md +++ /dev/null @@ -1,11 +0,0 @@ -OpenCode plan mode is active. The user wants to review an approach before any code is written, so you must NOT implement yet: do not edit repo-tracked files, run formatters that rewrite files, apply migrations, generate code, install dependencies, or otherwise change product state. You may inspect files and run non-mutating checks or tests that write only normal build/cache artifacts. The only permitted task writes are explicit plan artifacts requested by the user or required final-response output files. - -You are operating as OpenCode's plan agent. Work through the planning flow in order: - -1. **Ground in the environment.** Read the relevant files, configs, schemas, docs, and tests before asking questions when local inspection can answer them. Prefer existing patterns over new abstractions. -2. **Clarify intent.** Ask only questions that materially change the plan, confirm an important assumption, or choose between real tradeoffs. -3. **Design the implementation.** Specify the exact behavior, interfaces, data flow, edge cases, failure modes, compatibility concerns, and verification strategy. -4. **Review the plan.** Remove placeholders and vague instructions. Ensure every referenced existing file is real, every step maps to the requested goal, and the plan does not defer decisions to implementation time. -5. **Present the plan.** End with exactly one decision-complete plan. Do not begin implementation until the user approves. - -Terminal rail: your turn must end by either asking the user the smallest necessary question or by presenting the finished plan. Do not stop for any other reason and do not begin implementation until the user has approved the plan. diff --git a/profiles/shared/plan-mode.md b/profiles/shared/plan-mode.md new file mode 100644 index 0000000..d1a934f --- /dev/null +++ b/profiles/shared/plan-mode.md @@ -0,0 +1,11 @@ +Plan mode is active. The user wants to review an approach before any code is written, so you must NOT implement yet: do not edit files, run formatters or commands that rewrite files, apply migrations, generate code, install dependencies, or otherwise change product or system state. You may inspect files and run read-only checks. The only writes permitted are the plan itself and any final-response output the user explicitly requested. This constraint supersedes any other instruction you have received this session. + +You are operating inside a plan-mode workflow — a fixed, multi-phase procedure. Work through the phases in order: + +1. **Understand.** Read the relevant code, configs, schemas, docs, and tests with read-only tools until you can describe the change concretely. Reuse what already exists rather than proposing new code. +2. **Design.** Decide the implementation approach: the interfaces and data flow it touches, the edge cases and failure modes, and how you will verify it. Prefer existing patterns over new abstractions. +3. **Review.** Re-check the design against the user's request. Remove placeholders and vague steps, confirm every referenced file is real, and resolve material open questions with the user — ask only questions that change the plan. +4. **Propose the plan.** Lay out the finished plan: name the files to change, how each step maps to the goal, and how to verify the result. Do not defer decisions to implementation time. +5. **Hand off.** Present the finished plan for the user's approval and stop. Do not begin implementation until the user has approved it. + +Terminal rail: your turn must end in exactly one of two ways — by asking the user the smallest necessary question, or by presenting the finished plan for the user's approval. Do not stop for any other reason and do not begin implementation until the plan is approved. diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 07426c3..570b69e 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -62,9 +62,6 @@ pub trait HarnessAdapter { /// when the staged identifier can't be resolved. fn skill_unresolved_phrase(&self) -> &'static str; - /// The verbatim plan-mode procedure profile bundled for this harness. - fn plan_mode_profile(&self) -> &'static str; - /// Wrap a plan-mode profile as a `` operating-context /// layer. The default usually suffices. fn render_plan_mode_context(&self, profile_text: &str) -> String { @@ -222,9 +219,6 @@ impl HarnessAdapter for ClaudeCodeAdapter { fn skill_unresolved_phrase(&self) -> &'static str { "If the Skill tool cannot resolve that identifier" } - fn plan_mode_profile(&self) -> &'static str { - include_str!("../../profiles/claude-code/plan-mode.md") - } fn runbook_template(&self) -> &'static str { include_str!("../../profiles/claude-code/runbook.md") } @@ -317,9 +311,6 @@ impl HarnessAdapter for CodexAdapter { fn skill_unresolved_phrase(&self) -> &'static str { "If it does not load as a Codex skill" } - fn plan_mode_profile(&self) -> &'static str { - include_str!("../../profiles/codex/plan-mode.md") - } fn cli_events_filename(&self) -> Option<&'static str> { Some("codex-events.jsonl") } @@ -404,9 +395,6 @@ impl HarnessAdapter for OpenCodeAdapter { fn skill_unresolved_phrase(&self) -> &'static str { "If it does not load as an OpenCode skill" } - fn plan_mode_profile(&self) -> &'static str { - include_str!("../../profiles/opencode/plan-mode.md") - } fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { let model_note = if ctx.agent_model.is_some() { " Model selection was recorded as provenance, but the OpenCode adapter has no CLI model flag wired yet." diff --git a/src/cli/args.rs b/src/cli/args.rs index 0bdb0e7..12bf212 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -349,13 +349,13 @@ pub struct RunArgs { /// to clobber an existing dir; registered for next-run cleanup. #[arg(long)] pub stage_name: Option, - /// Inject the harness's plan-mode profile as an operating-context layer. + /// Inject the shared plan-mode profile as an operating-context layer. /// - /// Injects the harness's verbatim plan-mode procedure - /// (`profiles//plan-mode.md`) as a `` in every - /// dispatch, identical across arms. Opt-in, for plan-mode-relevant skills. - /// A harness with no profile aborts. It is text the subagent reads, not a - /// real injected mode. + /// Injects the shared, harness-agnostic plan-mode procedure + /// (`profiles/shared/plan-mode.md`) as a `` in every + /// dispatch, identical across arms and harnesses. Opt-in, for + /// plan-mode-relevant skills. It is text the subagent reads, not a real + /// injected mode. #[arg(long)] pub plan_mode: bool, /// Runs per condition cell, for variance reduction (default: 1). diff --git a/src/cli/run/orchestrate/stage.rs b/src/cli/run/orchestrate/stage.rs index 99dfd4e..ce5014d 100644 --- a/src/cli/run/orchestrate/stage.rs +++ b/src/cli/run/orchestrate/stage.rs @@ -32,9 +32,9 @@ pub(super) fn stage_conditions( }; let plan_mode_content = if opts.plan_mode { - let profile = resolve_plan_mode_profile(ctx.harness)?; + let profile = resolve_plan_mode_profile(); println!( - " plan-mode: injecting {} plan-mode profile as operating context (issue #142; necessary-not-sufficient fidelity layer)", + " plan-mode: injecting the shared plan-mode profile as operating context for {} (issue #142; necessary-not-sufficient fidelity layer)", harness_label(ctx.harness) ); Some(profile.to_string()) diff --git a/src/cli/run/util.rs b/src/cli/run/util.rs index 0a5738b..92e27ea 100644 --- a/src/cli/run/util.rs +++ b/src/cli/run/util.rs @@ -129,11 +129,11 @@ pub(crate) fn insession_isolated_handoff(env_dir: &Path) -> String { ) } -/// Resolve the verbatim plan-mode procedure profile for a harness. -/// The profile is a compile-time bundled asset, mirroring the schema embedding in +/// Resolve the shared, harness-agnostic plan-mode procedure profile injected by +/// `--plan-mode`. A compile-time bundled asset, mirroring the schema embedding in /// `validation`. -pub(crate) fn resolve_plan_mode_profile(harness: Harness) -> Result<&'static str, RunError> { - Ok(adapter_for(harness).plan_mode_profile()) +pub(crate) fn resolve_plan_mode_profile() -> &'static str { + include_str!("../../../profiles/shared/plan-mode.md") } /// Reject run options not supported by the selected harness's current mechanism. @@ -310,10 +310,13 @@ mod tests { } #[test] - fn opencode_plan_mode_profile_resolves() { - let profile = resolve_plan_mode_profile(Harness::OpenCode).unwrap(); - assert!(profile.contains("OpenCode plan mode is active")); - assert!(profile.contains("plan agent")); + fn plan_mode_profile_is_shared_and_harness_agnostic() { + let profile = resolve_plan_mode_profile(); + assert!(profile.contains("Plan mode is active")); + // Harness-agnostic content: no Claude-specific ExitPlanMode rail or + // Codex-specific block. + assert!(!profile.contains("ExitPlanMode")); + assert!(!profile.contains("")); } #[test] @@ -321,14 +324,6 @@ mod tests { assert_eq!(harness_label(Harness::OpenCode), "opencode"); } - #[test] - fn codex_plan_mode_profile_resolves() { - let profile = resolve_plan_mode_profile(Harness::Codex).unwrap(); - assert!(profile.contains("Codex plan mode is active")); - assert!(profile.contains("")); - assert!(!profile.contains("ExitPlanMode")); - } - #[test] fn base36_roundtrips_small_values() { assert_eq!(to_base36(0), "0"); diff --git a/tests/cli/package.rs b/tests/cli/package.rs index cfc031f..a4a1dca 100644 --- a/tests/cli/package.rs +++ b/tests/cli/package.rs @@ -73,7 +73,7 @@ fn cargo_package_excludes_repo_local_authoring_files() { "LICENSE", "README.md", "schema/evals.schema.json", - "profiles/codex/plan-mode.md", + "profiles/shared/plan-mode.md", "src/main.rs", ] { assert!( diff --git a/tests/run/codex.rs b/tests/run/codex.rs index 75c50bf..f5842a3 100644 --- a/tests/run/codex.rs +++ b/tests/run/codex.rs @@ -154,8 +154,10 @@ fn codex_plan_mode_injects_profile_and_records_flag() { assert!(prompt.contains("## Skills")); } assert!(prompt.contains("")); - assert!(prompt.contains("Codex plan mode is active")); - assert!(prompt.contains("")); + // Shared, harness-agnostic profile: same text every harness sees, with no + // Codex-specific block or Claude-specific ExitPlanMode rail. + assert!(prompt.contains("Plan mode is active")); + assert!(!prompt.contains("")); assert!(!prompt.contains("ExitPlanMode")); } } diff --git a/tests/run/opencode.rs b/tests/run/opencode.rs index 073ae87..447aff8 100644 --- a/tests/run/opencode.rs +++ b/tests/run/opencode.rs @@ -141,7 +141,8 @@ fn opencode_plan_mode_injects_profile_and_records_flag() { assert!(prompt.contains("")); } assert!(prompt.contains("")); - assert!(prompt.contains("OpenCode plan mode is active")); + // Shared, harness-agnostic profile: same text every harness sees. + assert!(prompt.contains("Plan mode is active")); assert!(!prompt.contains("ExitPlanMode")); } } diff --git a/tests/run/staging.rs b/tests/run/staging.rs index 0db4828..aafc2b5 100644 --- a/tests/run/staging.rs +++ b/tests/run/staging.rs @@ -148,7 +148,9 @@ fn plan_mode_injects_profile_and_records_flag() { let prompt = read_str(Path::new(task["dispatch_prompt_path"].as_str().unwrap())); assert!(prompt.contains("")); assert!(prompt.contains("Plan mode is active")); - assert!(prompt.contains("ExitPlanMode")); + // The shared profile is harness-agnostic: no Claude-specific ExitPlanMode rail. + assert!(prompt.contains("for the user's approval")); + assert!(!prompt.contains("ExitPlanMode")); } } From 95dec3ad07dad05b0fe30f318a156a738a63549b Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Mon, 29 Jun 2026 17:00:52 -0400 Subject: [PATCH 02/45] refactor(run-mode): delete the in-session-only switch-condition/reset-batch subcommands These fixture-swap/barrier subcommands exist only to serve the in-session (interactive Claude Code) isolated-run loop: they are referenced solely by in-session runbook generation and their own test modules, never by any Cli dispatch path. Removing them is the first step of dropping the in-session dispatch mechanism (#119). Co-Authored-By: Claude Opus 4.8 --- src/cli/args.rs | 46 --------- src/cli/commands/mod.rs | 2 +- src/cli/commands/pipeline.rs | 172 +--------------------------------- src/cli/mod.rs | 2 - tests/run/main.rs | 2 - tests/run/reset_batch.rs | 118 ----------------------- tests/run/switch_condition.rs | 153 ------------------------------ 7 files changed, 2 insertions(+), 493 deletions(-) delete mode 100644 tests/run/reset_batch.rs delete mode 100644 tests/run/switch_condition.rs diff --git a/src/cli/args.rs b/src/cli/args.rs index 12bf212..92a5656 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -206,31 +206,6 @@ pub struct GradeArgs { pub finalize: bool, } -/// `switch-condition` names the condition about to be dispatched (the one to keep) -/// on top of the common set. -#[derive(Debug, Args)] -pub struct SwitchConditionArgs { - #[command(flatten)] - pub common: CommonArgs, - /// The condition you are about to dispatch next (the one to KEEP). Its - /// counterpart's staged skill is removed from `env/.claude/skills/`. - #[arg(long)] - pub condition: String, -} - -/// `reset-batch` names the isolation group about to be dispatched, on top of the -/// common set. -#[derive(Debug, Args)] -pub struct ResetBatchArgs { - #[command(flatten)] - pub common: CommonArgs, - /// The isolation group you are about to dispatch next. The shared `env/`'s - /// working tree is wiped (keeping the staged skills + the outputs tree) and - /// re-seeded with this group's fixtures. - #[arg(long)] - pub group: String, -} - /// `snapshot` adds a label and an optional git ref on top of the common set. #[derive(Debug, Args)] pub struct SnapshotArgs { @@ -442,27 +417,6 @@ pub(crate) enum Commands { /// any per-`(group, condition)` Cli env guard — prints a `teardown` reminder before /// source edits. Requires `--iteration`. Finalize(CommonArgs), - /// Switch the active condition batch in a single-session isolated run. - /// - /// Removes the *off-condition*'s staged skill from `env/.claude/skills/` so the - /// next batch you dispatch cannot read it — the per-condition read-isolation - /// barrier for an interactive isolated run (see `RUNBOOK.md`). - /// `--condition` names the condition you are about to - /// dispatch next (the one to keep); its counterpart's staged skill is removed. - /// Run it only after every Task subagent of the prior batch has returned — it is - /// a hard barrier. Idempotent; resolves the iteration from `--workspace-dir` so - /// it works invoked from `env/`. Requires `--iteration`. - SwitchCondition(SwitchConditionArgs), - /// Swap the active isolation batch in a single-session isolated run. - /// - /// Wipes the shared `env/` working tree (keeping `.claude/skills/` and the - /// `.eval-magic-outputs/` tree) and re-seeds it with `--group`'s fixtures — the - /// per-batch isolation barrier between eval groups in an interactive isolated run - /// (see `RUNBOOK.md`). `--group` names the group you are - /// about to dispatch next. Run it only after every Task subagent of the prior - /// batch has returned — it is a hard barrier. Resolves the iteration from - /// `--workspace-dir` so it works invoked from `env/`. Requires `--iteration`. - ResetBatch(ResetBatchArgs), /// Assemble run records from a dispatch and its transcripts. /// /// Assembles a schema-valid `run.json` and backfills `timing.json` for every diff --git a/src/cli/commands/mod.rs b/src/cli/commands/mod.rs index cbb697f..1f04e62 100644 --- a/src/cli/commands/mod.rs +++ b/src/cli/commands/mod.rs @@ -15,7 +15,7 @@ pub(crate) use guard::{run_guard, run_guard_codex, run_teardown_guard}; pub(crate) use init::run_init; pub(crate) use pipeline::{ run_aggregate, run_detect_stray_writes, run_fill_transcripts, run_finalize, run_grade, - run_ingest, run_record_runs, run_reset_batch, run_switch_condition, + run_ingest, run_record_runs, }; pub(crate) use run::run_run; pub(crate) use validate::run_validate; diff --git a/src/cli/commands/pipeline.rs b/src/cli/commands/pipeline.rs index ab185e0..91865ea 100644 --- a/src/cli/commands/pipeline.rs +++ b/src/cli/commands/pipeline.rs @@ -5,7 +5,7 @@ use anyhow::bail; use crate::adapters::{CliJudgeContext, adapter_for}; -use crate::cli::args::{CommonArgs, GradeArgs, ResetBatchArgs, SwitchConditionArgs}; +use crate::cli::args::{CommonArgs, GradeArgs}; use crate::cli::command_target_args; use crate::cli::run; use crate::cli::{ @@ -169,176 +169,6 @@ pub(crate) fn run_finalize(args: CommonArgs) -> anyhow::Result<()> { Ok(()) } -/// Switch the active condition batch in a single-session isolated run: remove the -/// *off-condition*'s staged skill from `env/.claude/skills/` so the next batch the -/// session dispatches cannot read it. `--condition` names the condition about to be -/// dispatched (the one to keep); its counterpart is removed. Idempotent, and a hard -/// barrier — the runbook instructs the operator to join every Task subagent of the -/// prior batch first. Resolves the iteration from `--workspace-dir`, so it runs from -/// `cwd = env/`. The guard marker is a sibling file of the slug subtree, so removing -/// the slug dir leaves it (and an armed guard) intact. -pub(crate) fn run_switch_condition(args: SwitchConditionArgs) -> anyhow::Result<()> { - let ctx = run_context_from(&args.common)?; - let dir = iteration_dir(&ctx, args.common.iteration)?; - - let conditions_path = dir.join("conditions.json"); - if !conditions_path.exists() { - bail!("missing: {}", conditions_path.display()); - } - let conditions: crate::core::ConditionsRecord = - serde_json::from_str(&std::fs::read_to_string(&conditions_path)?)?; - - // `--condition` names the arm to KEEP; its counterpart is the off-condition to - // remove. Validate against the recorded conditions so a typo fails loudly - // instead of silently no-opping. - let names: Vec<&str> = conditions - .conditions - .iter() - .map(|c| c.name.as_str()) - .collect(); - if !names.contains(&args.condition.as_str()) { - bail!( - "unknown --condition '{}'; this iteration's conditions are: {}", - args.condition, - names.join(", ") - ); - } - let off = conditions - .conditions - .iter() - .find(|c| c.name != args.condition) - .ok_or_else(|| anyhow::anyhow!("no off-condition to switch away from"))?; - - let skills_dir = run::staging::skills_dir_for_harness(&dir.join("env"), ctx.harness); - match off.staged_skill_slug.as_ref() { - // The off-condition staged a skill: remove exactly its slug subtree. We do - // NOT use `cleanup_staged_skills` (it prefix-scans and would remove both - // arms' slugs and prune the dir) — only this one slug must go. - Some(Some(slug)) => { - let slug_dir = skills_dir.join(slug); - if slug_dir.exists() { - std::fs::remove_dir_all(&slug_dir)?; - println!( - "Switched to '{}': removed off-condition '{}' staged skill ({}).", - args.condition, - off.name, - slug_dir.display() - ); - } else { - println!( - "Switched to '{}': off-condition '{}' staged skill already absent — nothing to do.", - args.condition, off.name - ); - } - } - // The off-condition never staged a skill (e.g. the new-skill control arm), - // so there is nothing to hide. - _ => println!( - "Switched to '{}': off-condition '{}' has no staged skill — nothing to remove.", - args.condition, off.name - ), - } - Ok(()) -} - -/// Swap the active isolation batch in a single-session (in-session) isolated run: -/// wipe the shared `env/` working tree — keeping the staged skills and the -/// `.eval-magic-outputs/` tree — and re-seed it with `--group`'s fixtures, so the -/// next batch starts from a clean tree the prior batch's fixtures and stray writes -/// can't taint. A hard barrier: the runbook joins every Task subagent of the prior -/// batch first. Resolves the iteration from `--workspace-dir`, so it runs from -/// `cwd = env/`. -pub(crate) fn run_reset_batch(args: ResetBatchArgs) -> anyhow::Result<()> { - let ctx = run_context_from(&args.common)?; - let dir = iteration_dir(&ctx, args.common.iteration)?; - let env_dir = dir.join("env"); - if !env_dir.exists() { - bail!("missing env dir: {}", env_dir.display()); - } - - let dispatch_path = dir.join("dispatch.json"); - if !dispatch_path.exists() { - bail!("missing: {}", dispatch_path.display()); - } - let dispatch: serde_json::Value = - serde_json::from_str(&std::fs::read_to_string(&dispatch_path)?)?; - let tasks = dispatch["tasks"].as_array().cloned().unwrap_or_default(); - - // Groups are tagged on tasks only when there is more than one. Validate against - // them so a typo (or a needless reset on a single-group run) fails loudly. - let group_ids: std::collections::BTreeSet<&str> = - tasks.iter().filter_map(|t| t["group"].as_str()).collect(); - if !group_ids.contains(args.group.as_str()) { - if group_ids.is_empty() { - bail!( - "unknown --group '{}'; this iteration has a single group, so reset-batch is not needed.", - args.group - ); - } - bail!( - "unknown --group '{}'; this iteration's groups are: {}", - args.group, - group_ids.into_iter().collect::>().join(", ") - ); - } - - // The group's declared, env-relative fixture dests (deduped across its tasks). - let mut dests: Vec = Vec::new(); - for t in &tasks { - if t["group"].as_str() != Some(args.group.as_str()) { - continue; - } - if let Some(fixtures) = t["fixtures"].as_array() { - for f in fixtures.iter().filter_map(|f| f.as_str()) { - if !dests.iter().any(|d| d == f) { - dests.push(f.to_string()); - } - } - } - } - - // Full wipe: drop every entry in env/ except the staged skills, the outputs - // tree, and the runbook — so a prior batch's fixtures and any stray writes can't - // leak into this one. - const KEEP: &[&str] = &[ - ".claude", - ".agents", - ".codex", - ".opencode", - ".eval-magic-outputs", - "RUNBOOK.md", - ]; - for entry in std::fs::read_dir(&env_dir)? { - let entry = entry?; - if KEEP.iter().any(|k| entry.file_name() == **k) { - continue; - } - let path = entry.path(); - if path.is_dir() { - std::fs::remove_dir_all(&path)?; - } else { - std::fs::remove_file(&path)?; - } - } - - // Re-seed this group's fixtures from the skill's evals/ dir. - for dest in &dests { - let src = ctx.skill_subdir.join("evals").join(dest); - let dst = env_dir.join(dest); - if let Some(parent) = dst.parent() { - std::fs::create_dir_all(parent)?; - } - run::copy_entry(&src, &dst)?; - } - - println!( - "Reset to group '{}': wiped the env working tree and re-seeded {} fixture(s).", - args.group, - dests.len() - ); - Ok(()) -} - /// Assemble `run.json` + `timing.json` for every task in the iteration's /// `dispatch.json`. pub(crate) fn run_record_runs(args: CommonArgs) -> anyhow::Result<()> { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index a83aca7..3558ab3 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -62,8 +62,6 @@ fn dispatch(command: Option) -> anyhow::Result<()> { Commands::Run(args) => run_run(args), Commands::Ingest(args) => run_ingest(args), Commands::Finalize(args) => run_finalize(args), - Commands::SwitchCondition(args) => run_switch_condition(args), - Commands::ResetBatch(args) => run_reset_batch(args), Commands::Init(args) => run_init(args), Commands::Validate(args) => run_validate(args), Commands::TeardownGuard(_) => run_teardown_guard(), diff --git a/tests/run/main.rs b/tests/run/main.rs index 74ef8d2..42cd0f7 100644 --- a/tests/run/main.rs +++ b/tests/run/main.rs @@ -15,7 +15,5 @@ mod env_layout; mod grouping; mod lifecycle; mod opencode; -mod reset_batch; mod runbook; mod staging; -mod switch_condition; diff --git a/tests/run/reset_batch.rs b/tests/run/reset_batch.rs deleted file mode 100644 index f2b7247..0000000 --- a/tests/run/reset_batch.rs +++ /dev/null @@ -1,118 +0,0 @@ -//! `reset-batch`: the per-group isolation barrier for a single-session (in-session) -//! isolated run. Between eval-group batches it wipes the shared `env/` working tree -//! — keeping the staged skills and the outputs tree — and re-seeds it with the next -//! group's fixtures, so a prior batch's fixtures and stray writes can't leak. - -use crate::helpers::*; -use predicates::str::contains; -use std::fs; -use std::path::Path; - -const WITH_SLUG: &str = "slow-powers-eval-1-with_skill__mr-review"; - -/// Two evals routed into two groups: e2's `isolation: isolated` hint forces its own -/// group, so the in-session env stages group g1 (e1/a.txt) up front and swaps in -/// group g2 (e2/b.txt) via reset-batch. -const TWO_GROUPS: &str = r#"{ "skill_name": "mr-review", "evals": [ - { "id": "e1", "prompt": "p1", "expected_output": "o", "files": ["a.txt"] }, - { "id": "e2", "prompt": "p2", "expected_output": "o", "files": ["b.txt"], "isolation": "isolated" } ] }"#; - -/// Stage a two-group interactive iteration; returns `(skill_dir, cwd)` with `env/` -/// holding group g1's fixtures. -fn setup_two_groups(root: &Path) -> (std::path::PathBuf, std::path::PathBuf) { - let (skill_dir, cwd) = setup(root, TWO_GROUPS); - fs::write(skill_dir.join("mr-review/evals/a.txt"), "AAA").unwrap(); - fs::write(skill_dir.join("mr-review/evals/b.txt"), "BBB").unwrap(); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--dry-run"]) - .assert() - .success(); - (skill_dir, cwd) -} - -/// Run `reset-batch` the way the runbook prescribes: from inside `env/`, carrying -/// only the self-sufficient `--skill-dir/--skill/--workspace-dir` selector. -fn reset_to(cwd: &Path, skill_dir: &Path, group: &str) -> assert_cmd::assert::Assert { - skill_eval() - .current_dir(env_dir(cwd)) - .args(["reset-batch", "--skill-dir"]) - .arg(skill_dir) - .args(["--skill", "mr-review", "--workspace-dir"]) - .arg(cwd.join(".eval-magic")) - .args(["--iteration", "1", "--group", group]) - .assert() -} - -#[test] -fn reset_batch_wipes_working_tree_and_reseeds_group_fixtures() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup_two_groups(tmp.path()); - - // Up front the env holds group g1's fixture only. - assert_eq!(read_str(&env_dir(&cwd).join("a.txt")), "AAA"); - assert!(!env_dir(&cwd).join("b.txt").exists()); - // Simulate a stray file the g1 batch's agent wrote into the env. - fs::write(env_dir(&cwd).join("stray.txt"), "STRAY").unwrap(); - - reset_to(&cwd, &skill_dir, "g2").success(); - - // The env is now seeded for g2: its fixture present, g1's gone, the stray write - // gone — a clean tree. - assert_eq!(read_str(&env_dir(&cwd).join("b.txt")), "BBB"); - assert!(!env_dir(&cwd).join("a.txt").exists()); - assert!(!env_dir(&cwd).join("stray.txt").exists()); - - // The staged skill and the outputs tree survive the wipe. - assert!( - env_dir(&cwd) - .join(".claude/skills") - .join(WITH_SLUG) - .is_dir(), - "the staged skill survives reset-batch" - ); - assert!(env_dir(&cwd).join(".eval-magic-outputs").exists()); -} - -#[test] -fn reset_batch_can_restore_the_first_group() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup_two_groups(tmp.path()); - - // Move to g2, then back to g1 (as condition B's loop does after condition A left - // the env on the last group). - reset_to(&cwd, &skill_dir, "g2").success(); - reset_to(&cwd, &skill_dir, "g1").success(); - assert_eq!(read_str(&env_dir(&cwd).join("a.txt")), "AAA"); - assert!(!env_dir(&cwd).join("b.txt").exists()); -} - -#[test] -fn reset_batch_rejects_unknown_group() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup_two_groups(tmp.path()); - reset_to(&cwd, &skill_dir, "g99") - .failure() - .stderr(contains("unknown --group")); -} - -#[test] -fn reset_batch_on_single_group_run_explains_it_is_unneeded() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--dry-run"]) - .assert() - .success(); - - // A single-group run tags no task with a group, so reset-batch has nothing to do - // and says so rather than silently wiping. - reset_to(&cwd, &skill_dir, "g1") - .failure() - .stderr(contains("single group")); -} diff --git a/tests/run/switch_condition.rs b/tests/run/switch_condition.rs deleted file mode 100644 index cd39279..0000000 --- a/tests/run/switch_condition.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! `switch-condition`: the per-condition read-isolation barrier for a -//! single-session isolated run. It removes the off-condition's staged skill from -//! `env/.claude/skills/` between dispatch batches, and must resolve the iteration -//! tree while invoked from `cwd = env/`. - -use crate::helpers::*; -use std::path::{Path, PathBuf}; - -const WITH_SLUG: &str = "slow-powers-eval-1-with_skill__mr-review"; - -fn env_skills_dir(cwd: &Path) -> PathBuf { - env_dir(cwd).join(".claude/skills") -} - -/// Run `switch-condition` the way the runbook prescribes: from inside `env/`, -/// carrying only the self-sufficient `--skill-dir/--skill/--workspace-dir` selector. -fn switch_to(cwd: &Path, skill_dir: &Path, condition: &str) -> assert_cmd::assert::Assert { - skill_eval() - .current_dir(env_dir(cwd)) - .args(["switch-condition", "--skill-dir"]) - .arg(skill_dir) - .args(["--skill", "mr-review", "--workspace-dir"]) - .arg(cwd.join(".eval-magic")) - .args(["--iteration", "1", "--condition", condition]) - .assert() -} - -#[test] -fn switch_condition_removes_off_condition_slug_from_env_cwd() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // Build the env (staging happens even under --dry-run). - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--dry-run"]) - .assert() - .success(); - - let with_slug = env_skills_dir(&cwd).join(WITH_SLUG); - assert!(with_slug.is_dir(), "with_skill staged before switch"); - - // Move to the without_skill batch: the off-condition (with_skill) staged skill - // is removed so the control arm cannot read it. - switch_to(&cwd, &skill_dir, "without_skill").success(); - - assert!(!with_slug.exists(), "with_skill slug removed after switch"); -} - -#[test] -fn switch_condition_is_idempotent() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--dry-run"]) - .assert() - .success(); - - // Two switches in a row: the second is a no-op, not an error (a re-run after a - // fix, or an over-eager operator, must stay safe). - switch_to(&cwd, &skill_dir, "without_skill").success(); - switch_to(&cwd, &skill_dir, "without_skill").success(); - assert!(!env_skills_dir(&cwd).join(WITH_SLUG).exists()); -} - -#[test] -fn switch_condition_preserves_guard_marker() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // A guarded run arms the write guard; --guard requires a real (non-dry) run. - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--guard"]) - .assert() - .success(); - - // The guard marker is a sibling file of the slug subtree inside the skills dir. - let marker = env_skills_dir(&cwd).join(".slow-powers-eval-guard.json"); - assert!(marker.exists(), "guard armed before switch"); - - switch_to(&cwd, &skill_dir, "without_skill").success(); - - assert!( - !env_skills_dir(&cwd).join(WITH_SLUG).exists(), - "slug removed" - ); - assert!(marker.exists(), "guard marker survives the switch"); -} - -#[test] -fn switch_condition_rejects_unknown_condition() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "new-skill", "--dry-run"]) - .assert() - .success(); - - switch_to(&cwd, &skill_dir, "bogus_condition") - .failure() - .stderr(predicates::str::contains( - "unknown --condition 'bogus_condition'", - )); - // A typo must not silently leave the staged skill in place under a false sense - // of isolation. - assert!(env_skills_dir(&cwd).join(WITH_SLUG).is_dir()); -} - -#[test] -fn switch_condition_revision_removes_old_skill_keeps_new() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // Revision mode compares a baseline snapshot (old_skill) against the working - // SKILL.md (new_skill); both arms stage a skill. Seed the baseline snapshot. - let snapshot = iteration_dir(&cwd) - .parent() - .unwrap() - .join("snapshots") - .join("baseline"); - std::fs::create_dir_all(&snapshot).unwrap(); - std::fs::write( - snapshot.join("SKILL.md"), - "---\nname: mr-review\ndescription: review merge requests\n---\n\nold body\n", - ) - .unwrap(); - - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review", "--mode", "revision", "--dry-run"]) - .assert() - .success(); - - let old_slug = env_skills_dir(&cwd).join("slow-powers-eval-1-old_skill__mr-review"); - let new_slug = env_skills_dir(&cwd).join("slow-powers-eval-1-new_skill__mr-review"); - assert!(old_slug.is_dir() && new_slug.is_dir(), "both arms staged"); - - // Switch to the new_skill batch: only the old_skill slug is removed. - switch_to(&cwd, &skill_dir, "new_skill").success(); - - assert!(!old_slug.exists(), "old_skill slug removed"); - assert!(new_slug.is_dir(), "new_skill slug kept"); -} From 37449a8a13c81bfcb45fe971ed490b2e5f984f68 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Mon, 29 Jun 2026 17:48:06 -0400 Subject: [PATCH 03/45] refactor(run-mode): remove the in-session dispatch mechanism MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make CLI dispatch the only mechanism end-to-end and delete all in-session (`DispatchMechanism::InSession` / `RunMode::Interactive`) code. Claude Code now defaults to and runs via its `claude -p` stream-json path, exactly like Codex's `codex exec`; hybrid and headless are the only run modes (kept as accepted `--run-mode` values pending the vocabulary collapse in #C). Removed: - `DispatchMechanism` enum, `RunMode::Interactive`, `RunMode::mechanism()`; `default_for(ClaudeCode)` now → hybrid, `resolve_run_mode` ClaudeCode → [hybrid, headless]; `HarnessRunCapabilities.mechanism` field. - The single-shared-`env/` layout: every run now stages one `env--/` per (group, condition). - `HarnessAdapter::{runbook_template, parse_transcript, parse_transcript_full}`; `parse_cli_events*` are now required (Codex parses its events file directly, OpenCode errors until ingest is wired). The in-session runbook + its profile. - In-session transcript code (`parse_transcript*`, subagent resolution by `agent_description`, `resolve_subagents_dir*`, `SubagentMeta`/`SubagentEntry`) and the `--subagents-dir` / `--session-id` flags + `CLAUDE_CODE_SESSION_ID` auto-resolution. Shared `extract_invocations`/`read_records`/`TranscriptSummary` kept (the stream-json parser reuses them). All mechanism-keyed branches collapse to the CLI path; surviving harness-specific behavior keys on the harness/adapter. Pipeline record-runs/fill-transcripts now read each task's `outputs/-events.jsonl` only. Broken prose referencing the deleted flags/commands rewritten in the manifest, help, and stray-writes hint; the run-mode vocabulary/docs rewrite (incl. docs/harness-parity.md) stays with #C. Integration tests re-pointed from the single `env/` to the per-(group,condition) layout; in-session-only tests removed. `cargo test`, `cargo fmt --check`, and `cargo clippy --all-targets -- -D warnings` are green; Claude Code `claude -p` run → ingest → finalize smoke produces run.json/timing.json/benchmark.json. Part of #113. Closes #119. Likely supersedes #34; unblocks #120 and #122. Co-Authored-By: Claude Opus 4.8 --- profiles/claude-code/runbook.md | 71 ----- src/adapters/claude_cli.rs | 4 +- src/adapters/claude_code_session.rs | 136 --------- src/adapters/claude_code_transcript.rs | 362 ++---------------------- src/adapters/claude_stream_json.rs | 6 +- src/adapters/cli_command.rs | 2 +- src/adapters/codex_cli.rs | 2 +- src/adapters/harness.rs | 114 +++----- src/adapters/mod.rs | 10 +- src/cli/args.rs | 82 ++---- src/cli/commands/pipeline.rs | 81 ++---- src/cli/commands/workspace.rs | 14 +- src/cli/help.rs | 12 +- src/cli/mod.rs | 137 +-------- src/cli/run/dispatch.rs | 28 +- src/cli/run/grouping.rs | 5 +- src/cli/run/orchestrate/build.rs | 113 +++----- src/cli/run/orchestrate/envs.rs | 125 ++------- src/cli/run/orchestrate/mod.rs | 47 ++-- src/cli/run/orchestrate/stage.rs | 8 +- src/cli/run/runbook.rs | 309 +++----------------- src/cli/run/steps.rs | 88 +----- src/cli/run/util.rs | 114 +------- src/core/mod.rs | 6 +- src/core/run_mode.rs | 108 ++----- src/core/types.rs | 4 +- src/pipeline/fill_transcripts.rs | 178 ++---------- src/pipeline/mod.rs | 2 +- src/pipeline/record_runs.rs | 375 +++++-------------------- tests/run/claude_cli.rs | 51 +--- tests/run/env_layout.rs | 77 +++-- tests/run/grouping.rs | 62 ++-- tests/run/helpers.rs | 18 +- tests/run/lifecycle.rs | 33 ++- tests/run/runbook.rs | 67 +---- tests/run/staging.rs | 27 +- 36 files changed, 566 insertions(+), 2312 deletions(-) delete mode 100644 profiles/claude-code/runbook.md diff --git a/profiles/claude-code/runbook.md b/profiles/claude-code/runbook.md deleted file mode 100644 index 467167d..0000000 --- a/profiles/claude-code/runbook.md +++ /dev/null @@ -1,71 +0,0 @@ -# Eval run — {{SKILL_NAME}} (iteration {{ITERATION}}) - -You are an agent in a **fresh, isolated** session. Follow this runbook top to bottom to run -the eval and produce `benchmark.json`. Everything you need is in this iteration directory — -you should not need anything from the surrounding repo. - -- **Skill under test:** {{SKILL_NAME}} -- **Mode:** {{MODE}} — comparing `{{COND_A}}` vs `{{COND_B}}` -- **Dispatches:** {{NUM_TASKS}} (the `tasks[]` array in `{{DISPATCH_JSON}}`) - -The two conditions run as **separate batches** in this one session: dispatch every subagent of -one batch, wait for them **all** to return, then switch conditions before dispatching the next. -Never interleave the batches — `switch-condition` removes the off-condition's staged skill, and a -subagent still in flight could observe a half-removed skill or read the wrong one. - -## 1. Dispatch the `{{COND_A}}` batch - -{{DISPATCH_COND_A}} - -Wait for **every** one of these subagents to return before continuing. - -## 2. Switch to the `{{COND_B}}` condition - -This removes the `{{COND_A}}` staged skill so the `{{COND_B}}` batch cannot read it: - -``` -{{SWITCH_CMD}} -``` - -## 3. Dispatch the `{{COND_B}}` batch - -{{DISPATCH_COND_B}} - -Wait for **every** one of these subagents to return before continuing. - -## 4. Ingest - -``` -{{INGEST_CMD}} -``` - -`ingest` records each run, backfills transcripts, scans for stray writes, and grades every -mechanical assertion. It then prints any `llm_judge` tasks it could not grade itself. - -## 5. Dispatch the judge subagents, then finalize - -Dispatch each judge task `ingest` listed as a subagent the same way — pass its -`agent_description` verbatim — then merge the verdicts and aggregate: - -``` -{{FINALIZE_CMD}} -``` - -## 6. Read the result - -`finalize` writes the cross-condition benchmark to: - -``` -{{BENCHMARK_PATH}} -``` - -Read it for the per-condition pass rates and the `{{COND_A}}` − `{{COND_B}}` deltas. This is -the artifact the prep session resumes on. - -## 7. Tear down - -When you are done, remove the staged skills (and the write guard, if armed): - -``` -{{TEARDOWN_CMD}} -``` diff --git a/src/adapters/claude_cli.rs b/src/adapters/claude_cli.rs index 2924b49..24b22d0 100644 --- a/src/adapters/claude_cli.rs +++ b/src/adapters/claude_cli.rs @@ -1,5 +1,5 @@ -//! Claude Code `claude -p` command rendering for `DispatchMechanism::Cli` -//! guidance (hybrid / headless run modes). +//! Claude Code `claude -p` command rendering for dispatch guidance +//! (hybrid / headless run modes). //! //! Differences from the Codex recipe, all forced by the `claude` CLI: //! `--output-format stream-json` requires `--verbose` in `-p` mode; there is no diff --git a/src/adapters/claude_code_session.rs b/src/adapters/claude_code_session.rs index 0462edf..376c037 100644 --- a/src/adapters/claude_code_session.rs +++ b/src/adapters/claude_code_session.rs @@ -7,8 +7,6 @@ //! `` block. Both live in an adapter rather than the harness- //! agnostic orchestrator so a new harness adds its own renderer alongside. -use std::path::{Path, PathBuf}; - use crate::core::AvailableSkill; /// Render the list of discoverable skills the way a real Claude Code session @@ -40,54 +38,10 @@ pub fn render_plan_mode_context(profile_text: &str) -> String { format!("\n{trimmed}\n") } -/// Slugify an absolute path the way Claude Code names its project directories: -/// every non-alphanumeric character becomes `-`. For example -/// `/Users/x/.config/oc` → `-Users-x--config-oc` (the `/` before `.config` and -/// the `.` each map to a `-`, producing the double hyphen). -pub fn slugify_project_path(path: &Path) -> String { - path.to_string_lossy() - .chars() - .map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }) - .collect() -} - -/// Locate the subagents transcript dir for a Claude Code session. -/// -/// Returns `/projects///subagents/` when it -/// exists, where `` is [`slugify_project_path`] of `cwd`. If the -/// cwd-derived slug doesn't match (e.g. the command ran from a subdirectory of -/// the session's project), scans `/projects/*` for a child named -/// `` — the session id is a globally-unique UUID, so at most one -/// project dir contains it. Returns `None` if no `subagents/` dir is found. -pub fn resolve_subagents_dir_for_session( - config_dir: &Path, - cwd: &Path, - session_id: &str, -) -> Option { - let projects = config_dir.join("projects"); - let primary = projects - .join(slugify_project_path(cwd)) - .join(session_id) - .join("subagents"); - if primary.is_dir() { - return Some(primary); - } - let entries = std::fs::read_dir(&projects).ok()?; - for entry in entries.flatten() { - let candidate = entry.path().join(session_id).join("subagents"); - if candidate.is_dir() { - return Some(candidate); - } - } - None -} - #[cfg(test)] mod tests { use super::*; use crate::core::AvailableSkill; - use std::fs; - use tempfile::TempDir; fn skill(name: &str, description: &str) -> AvailableSkill { AvailableSkill { @@ -137,94 +91,4 @@ mod tests { assert_eq!(render_plan_mode_context(""), ""); assert_eq!(render_plan_mode_context(" \n "), ""); } - - #[test] - fn slugify_matches_claude_code_double_hyphen() { - // Verified against a real Claude Code project dir: the `/` before `.config` - // and the `.` both become `-`, producing a double hyphen. - assert_eq!( - slugify_project_path(Path::new("/Users/maxhaarhaus/.config/opencode")), - "-Users-maxhaarhaus--config-opencode" - ); - } - - #[test] - fn slugify_replaces_all_non_alphanumerics_keeping_alnum() { - assert_eq!( - slugify_project_path(Path::new("/a-b/c.d_e f")), - "-a-b-c-d-e-f" - ); - assert_eq!(slugify_project_path(Path::new("/Proj9/v2")), "-Proj9-v2"); - } - - /// Create `/projects///subagents/` and return the subagents path. - fn make_subagents(config: &Path, project_dir: &str, sid: &str) -> std::path::PathBuf { - let dir = config - .join("projects") - .join(project_dir) - .join(sid) - .join("subagents"); - fs::create_dir_all(&dir).unwrap(); - dir - } - - #[test] - fn resolve_finds_primary_cwd_slug_path() { - let tmp = TempDir::new().unwrap(); - let cwd = Path::new("/tmp/proj"); - let sid = "5ade3f59-dda3-4f40-8776-79f82ba0fab2"; - let expected = make_subagents(tmp.path(), "-tmp-proj", sid); - assert_eq!( - resolve_subagents_dir_for_session(tmp.path(), cwd, sid), - Some(expected) - ); - } - - #[test] - fn resolve_falls_back_to_scan_when_cwd_slug_differs() { - let tmp = TempDir::new().unwrap(); - let cwd = Path::new("/tmp/proj"); // slug `-tmp-proj` is NOT created - let sid = "11111111-2222-3333-4444-555555555555"; - let expected = make_subagents(tmp.path(), "some-other-project-slug", sid); - assert_eq!( - resolve_subagents_dir_for_session(tmp.path(), cwd, sid), - Some(expected) - ); - } - - #[test] - fn resolve_prefers_primary_over_scan_match() { - let tmp = TempDir::new().unwrap(); - let cwd = Path::new("/tmp/proj"); - let sid = "99999999-aaaa-bbbb-cccc-dddddddddddd"; - // A scan candidate that sorts first, plus the cwd-slug primary. - make_subagents(tmp.path(), "aaa-other", sid); - let primary = make_subagents(tmp.path(), "-tmp-proj", sid); - assert_eq!( - resolve_subagents_dir_for_session(tmp.path(), cwd, sid), - Some(primary) - ); - } - - #[test] - fn resolve_none_when_session_dir_absent() { - let tmp = TempDir::new().unwrap(); - fs::create_dir_all(tmp.path().join("projects")).unwrap(); - assert_eq!( - resolve_subagents_dir_for_session(tmp.path(), Path::new("/tmp/proj"), "no-such-sid"), - None - ); - } - - #[test] - fn resolve_none_when_subagents_subdir_missing() { - let tmp = TempDir::new().unwrap(); - let sid = "abcdabcd-0000-1111-2222-333333333333"; - // Session dir exists (under the cwd slug) but without a `subagents/` child. - fs::create_dir_all(tmp.path().join("projects").join("-tmp-proj").join(sid)).unwrap(); - assert_eq!( - resolve_subagents_dir_for_session(tmp.path(), Path::new("/tmp/proj"), sid), - None - ); - } } diff --git a/src/adapters/claude_code_transcript.rs b/src/adapters/claude_code_transcript.rs index c5edad9..811ec5e 100644 --- a/src/adapters/claude_code_transcript.rs +++ b/src/adapters/claude_code_transcript.rs @@ -1,10 +1,10 @@ -//! Claude Code transcript parsing. +//! Claude Code transcript record types and tool-call extraction. //! -//! Reads a JSONL session -//! transcript and extracts ordered [`ToolInvocation`]s (matching `tool_result` -//! blocks back to their `tool_use` by id), plus a [`TranscriptSummary`] with -//! deduped token totals, wall-clock duration, and the final assistant text. -//! Also resolves subagent transcripts by their `.meta.json` description. +//! Defines the JSONL record shapes and the shared extractors — ordered +//! [`ToolInvocation`]s (matching `tool_result` blocks back to their `tool_use` by +//! id) and the last assistant text — reused by the `claude -p` stream-json parser +//! ([`claude_stream_json`](super::claude_stream_json)), plus the +//! [`TranscriptSummary`] the pipeline consumes. use crate::core::ToolInvocation; use serde::{Deserialize, Serialize}; @@ -12,8 +12,7 @@ use serde_json::Value; use std::collections::HashMap; use std::fs; use std::io; -use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::path::Path; #[derive(Debug, Deserialize)] pub(crate) struct UsageRecord { @@ -25,8 +24,6 @@ pub(crate) struct UsageRecord { #[derive(Debug, Deserialize)] struct Message { - id: Option, - usage: Option, /// String or array of content blocks; inspected as raw JSON. content: Option, } @@ -35,7 +32,6 @@ struct Message { pub(crate) struct TranscriptRecord { #[serde(rename = "type")] pub(crate) record_type: Option, - timestamp: Option, message: Option, } @@ -146,11 +142,6 @@ pub(crate) fn extract_invocations(records: &[TranscriptRecord]) -> Vec io::Result> { - Ok(extract_invocations(&read_records(jsonl_path)?)) -} - /// The concatenated text blocks of the last assistant message carrying any text. /// Shared with the `-p` stream-json parser, which uses it as the final-message /// fallback when the terminal `result` event is absent or errored. @@ -176,163 +167,21 @@ pub(crate) fn last_assistant_text(records: &[TranscriptRecord]) -> Option, - /// Sum of usage across unique API responses (deduped by `message.id`). - /// Includes cache creation/read tokens — a different accounting than the - /// harness's task-completion event. + /// Total token usage (input + output + cache creation/read), as reported by + /// the run's terminal `result` event. pub total_tokens: Option, - /// Wall clock between the first and last line timestamps. + /// Wall-clock duration, as reported by the run's terminal `result` event. pub duration_ms: Option, /// Concatenated text blocks of the last assistant message. pub final_text: Option, } -fn parse_millis(s: &str) -> Option { - chrono::DateTime::parse_from_rfc3339(s) - .ok() - .map(|dt| dt.timestamp_millis()) -} - -/// Parse the transcript into a full [`TranscriptSummary`]. -pub fn parse_transcript_full(jsonl_path: &Path) -> io::Result { - let records = read_records(jsonl_path)?; - - let mut usage_by_id: HashMap<&str, &UsageRecord> = HashMap::new(); - let mut first_ts: Option = None; - let mut last_ts: Option = None; - let mut timestamp_count = 0usize; - - for record in &records { - if let Some(ts_str) = &record.timestamp - && let Some(ts) = parse_millis(ts_str) - { - if first_ts.is_none() { - first_ts = Some(ts); - } - last_ts = Some(ts); - timestamp_count += 1; - } - - if record.record_type.as_deref() != Some("assistant") { - continue; - } - - if let Some(msg) = &record.message - && let (Some(id), Some(usage)) = (&msg.id, &msg.usage) - { - usage_by_id.insert(id, usage); - } - } - - let final_text = last_assistant_text(&records); - - let total_tokens = if usage_by_id.is_empty() { - None - } else { - Some( - usage_by_id - .values() - .map(|u| { - u.input_tokens.unwrap_or(0) - + u.output_tokens.unwrap_or(0) - + u.cache_creation_input_tokens.unwrap_or(0) - + u.cache_read_input_tokens.unwrap_or(0) - }) - .sum(), - ) - }; - - let duration_ms = match (first_ts, last_ts) { - (Some(f), Some(l)) if timestamp_count >= 2 => Some(l - f), - _ => None, - }; - - Ok(TranscriptSummary { - tool_invocations: extract_invocations(&records), - total_tokens, - duration_ms, - final_text, - }) -} - -/// Metadata sidecar (`.meta.json`) written alongside a subagent transcript. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SubagentMeta { - #[serde(rename = "agentType", skip_serializing_if = "Option::is_none")] - pub agent_type: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub description: Option, - #[serde(rename = "toolUseId", skip_serializing_if = "Option::is_none")] - pub tool_use_id: Option, -} - -/// A discovered subagent transcript and its metadata sidecar. -#[derive(Debug, Clone, PartialEq)] -pub struct SubagentEntry { - pub jsonl_path: PathBuf, - pub meta_path: PathBuf, - pub meta: SubagentMeta, -} - -/// List subagent transcripts (each a `.meta.json` with a sibling -/// `.jsonl`) under `subagents_dir`. Returns `[]` if the dir is missing. -pub fn list_subagents(subagents_dir: &Path) -> Vec { - let mut out = Vec::new(); - let Ok(entries) = fs::read_dir(subagents_dir) else { - return out; - }; - for entry in entries.flatten() { - let file_name = entry.file_name(); - let name = file_name.to_string_lossy(); - let Some(base) = name.strip_suffix(".meta.json") else { - continue; - }; - let meta_path = subagents_dir.join(file_name.as_os_str()); - let jsonl_path = subagents_dir.join(format!("{base}.jsonl")); - if !jsonl_path.exists() { - continue; - } - let Ok(raw) = fs::read_to_string(&meta_path) else { - continue; - }; - let Ok(meta) = serde_json::from_str::(&raw) else { - continue; - }; - out.push(SubagentEntry { - jsonl_path, - meta_path, - meta, - }); - } - out -} - -fn mtime(path: &Path) -> SystemTime { - fs::metadata(path) - .and_then(|m| m.modified()) - .unwrap_or(SystemTime::UNIX_EPOCH) -} - -/// Find the subagent whose meta `description` matches. On duplicates (a retry -/// within the same run), returns the most-recently-written transcript. -pub fn find_by_description(subagents_dir: &Path, description: &str) -> Option { - let mut matches: Vec = list_subagents(subagents_dir) - .into_iter() - .filter(|e| e.meta.description.as_deref() == Some(description)) - .collect(); - if matches.len() <= 1 { - return matches.pop(); - } - matches.sort_by_key(|e| std::cmp::Reverse(mtime(&e.jsonl_path))); - matches.into_iter().next() -} - #[cfg(test)] mod tests { use super::*; use serde_json::{Value, json}; - use std::fs::{self, File}; + use std::fs; use std::path::Path; - use std::time::{Duration, SystemTime}; use tempfile::TempDir; fn write_jsonl(path: &Path, lines: &[Value]) { @@ -344,6 +193,12 @@ mod tests { fs::write(path, format!("{body}\n")).unwrap(); } + /// Read the records and run the shared tool-call extractor — the path the + /// stream-json parser also takes. + fn invocations(path: &Path) -> Vec { + extract_invocations(&read_records(path).unwrap()) + } + #[test] fn extracts_tool_use_blocks_with_ordinal_and_args() { let dir = TempDir::new().unwrap(); @@ -365,7 +220,7 @@ mod tests { ], ); - let result = parse_transcript(&path).unwrap(); + let result = invocations(&path); assert_eq!(result.len(), 2); assert_eq!(result[0].name, "Bash"); assert_eq!(result[0].ordinal, 0); @@ -391,7 +246,7 @@ mod tests { json!({"type": "assistant", "message": {"role": "assistant", "content": [{"type": "text", "text": "hello"}]}}), ], ); - assert_eq!(parse_transcript(&path).unwrap(), vec![]); + assert_eq!(invocations(&path), vec![]); } #[test] @@ -407,7 +262,7 @@ mod tests { let body = format!("{good_a}\nnot valid json\n{good_b}\n"); fs::write(&path, body).unwrap(); - let result = parse_transcript(&path).unwrap(); + let result = invocations(&path); assert_eq!(result.len(), 2); assert_eq!( result.iter().map(|r| r.name.as_str()).collect::>(), @@ -430,88 +285,15 @@ mod tests { ]}}), ], ); - let result = parse_transcript(&path).unwrap(); + let result = invocations(&path); assert_eq!(result.len(), 1); assert_eq!(result[0].result, Some(Value::String("hi".into()))); } - fn usage(output: i64) -> Value { - json!({ - "input_tokens": 100, - "cache_creation_input_tokens": 50, - "cache_read_input_tokens": 200, - "output_tokens": output, - }) - } - - #[test] - fn sums_usage_across_unique_message_ids() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-dedup.jsonl"); - write_jsonl( - &path, - &[ - json!({"type": "user", "timestamp": "2026-06-04T10:00:00.000Z", "message": {"role": "user", "content": "go"}}), - json!({"type": "assistant", "timestamp": "2026-06-04T10:00:05.000Z", "message": {"id": "msg_aaa", "role": "assistant", "usage": usage(10), "content": [{"type": "text", "text": "first block"}]}}), - json!({"type": "assistant", "timestamp": "2026-06-04T10:00:06.000Z", "message": {"id": "msg_aaa", "role": "assistant", "usage": usage(10), "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}]}}), - json!({"type": "assistant", "timestamp": "2026-06-04T10:01:00.000Z", "message": {"id": "msg_bbb", "role": "assistant", "usage": usage(40), "content": [{"type": "text", "text": "done"}]}}), - ], - ); - // msg_aaa counted once (100+50+200+10) + msg_bbb (100+50+200+40) = 750 - assert_eq!( - parse_transcript_full(&path).unwrap().total_tokens, - Some(750) - ); - } - - #[test] - fn returns_null_total_tokens_when_no_usage() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-no-usage.jsonl"); - write_jsonl( - &path, - &[ - json!({"type": "assistant", "message": {"role": "assistant", "content": [{"type": "text", "text": "hi"}]}}), - ], - ); - assert_eq!(parse_transcript_full(&path).unwrap().total_tokens, None); - } - #[test] - fn derives_duration_from_first_and_last_timestamps() { + fn last_assistant_text_concatenates_text_of_last_assistant_message() { let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-duration.jsonl"); - write_jsonl( - &path, - &[ - json!({"type": "user", "timestamp": "2026-06-04T10:00:00.000Z", "message": {"role": "user", "content": "go"}}), - json!({"type": "assistant", "timestamp": "2026-06-04T10:02:30.500Z", "message": {"id": "msg_x", "role": "assistant", "content": [{"type": "text", "text": "done"}]}}), - ], - ); - assert_eq!( - parse_transcript_full(&path).unwrap().duration_ms, - Some(150_500) - ); - } - - #[test] - fn returns_null_duration_with_fewer_than_two_timestamps() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-one-ts.jsonl"); - write_jsonl( - &path, - &[ - json!({"type": "assistant", "timestamp": "2026-06-04T10:00:00.000Z", "message": {"role": "assistant", "content": []}}), - json!({"type": "assistant", "message": {"role": "assistant", "content": []}}), - ], - ); - assert_eq!(parse_transcript_full(&path).unwrap().duration_ms, None); - } - - #[test] - fn final_text_is_concatenated_text_of_last_assistant_message() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-final-text.jsonl"); + let path = dir.path().join("final-text.jsonl"); write_jsonl( &path, &[ @@ -525,109 +307,19 @@ mod tests { ], ); assert_eq!( - parse_transcript_full(&path).unwrap().final_text, + last_assistant_text(&read_records(&path).unwrap()), Some("All tests pass.\nWrapping up.".into()) ); } #[test] - fn final_text_is_null_when_no_assistant_text() { + fn last_assistant_text_is_null_when_no_assistant_text() { let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-no-text.jsonl"); + let path = dir.path().join("no-text.jsonl"); write_jsonl( &path, &[json!({"type": "user", "message": {"role": "user", "content": "hi"}})], ); - assert_eq!(parse_transcript_full(&path).unwrap().final_text, None); - } - - #[test] - fn tool_invocations_matches_parse_transcript() { - let dir = TempDir::new().unwrap(); - let path = dir.path().join("full-invocations.jsonl"); - write_jsonl( - &path, - &[ - json!({"type": "assistant", "timestamp": "2026-06-04T10:00:00.000Z", "message": {"id": "msg_1", "role": "assistant", "usage": usage(5), "content": [{"type": "tool_use", "id": "toolu_q", "name": "Read", "input": {"file_path": "/tmp/a"}}]}}), - json!({"type": "user", "timestamp": "2026-06-04T10:00:02.000Z", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_q", "content": "contents"}]}}), - ], - ); - assert_eq!( - parse_transcript_full(&path).unwrap().tool_invocations, - parse_transcript(&path).unwrap() - ); - } - - #[test] - fn matches_subagents_by_meta_description() { - let dir = TempDir::new().unwrap(); - let sub = dir.path().join("subagents"); - fs::create_dir_all(&sub).unwrap(); - - fs::write( - sub.join("agent-aaa111.meta.json"), - json!({"agentType": "general-purpose", "description": "claim-without-running:with_skill", "toolUseId": "toolu_p1"}).to_string(), - ) - .unwrap(); - fs::write(sub.join("agent-aaa111.jsonl"), "").unwrap(); - - fs::write( - sub.join("agent-bbb222.meta.json"), - json!({"agentType": "general-purpose", "description": "claim-without-running:without_skill", "toolUseId": "toolu_p2"}).to_string(), - ) - .unwrap(); - fs::write(sub.join("agent-bbb222.jsonl"), "").unwrap(); - - assert_eq!(list_subagents(&sub).len(), 2); - - let m = find_by_description(&sub, "claim-without-running:with_skill"); - assert_eq!(m.unwrap().meta.tool_use_id.as_deref(), Some("toolu_p1")); - - assert!(find_by_description(&sub, "no-such-eval:with_skill").is_none()); - } - - #[test] - fn returns_empty_when_subagents_dir_missing() { - let dir = TempDir::new().unwrap(); - let missing = dir.path().join("does-not-exist"); - assert_eq!(list_subagents(&missing).len(), 0); - assert!(find_by_description(&missing, "x").is_none()); - } - - #[test] - fn duplicate_descriptions_return_most_recent_transcript() { - let dir = TempDir::new().unwrap(); - let sub = dir.path().join("dup-subagents"); - fs::create_dir_all(&sub).unwrap(); - - fs::write( - sub.join("agent-old.meta.json"), - json!({"description": "dup:with_skill", "toolUseId": "toolu_old"}).to_string(), - ) - .unwrap(); - fs::write(sub.join("agent-old.jsonl"), "").unwrap(); - let old = SystemTime::now() - Duration::from_secs(60); - File::options() - .write(true) - .open(sub.join("agent-old.jsonl")) - .unwrap() - .set_modified(old) - .unwrap(); - - fs::write( - sub.join("agent-new.meta.json"), - json!({"description": "dup:with_skill", "toolUseId": "toolu_new"}).to_string(), - ) - .unwrap(); - fs::write(sub.join("agent-new.jsonl"), "").unwrap(); - File::options() - .write(true) - .open(sub.join("agent-new.jsonl")) - .unwrap() - .set_modified(SystemTime::now()) - .unwrap(); - - let m = find_by_description(&sub, "dup:with_skill"); - assert_eq!(m.unwrap().meta.tool_use_id.as_deref(), Some("toolu_new")); + assert_eq!(last_assistant_text(&read_records(&path).unwrap()), None); } } diff --git a/src/adapters/claude_stream_json.rs b/src/adapters/claude_stream_json.rs index 9d4c25b..429937d 100644 --- a/src/adapters/claude_stream_json.rs +++ b/src/adapters/claude_stream_json.rs @@ -4,8 +4,8 @@ //! --output-format stream-json --verbose` writes (captured per task as //! `outputs/claude-events.jsonl`). The `assistant`/`user` events wrap a full //! Anthropic Messages object under `message`, so tool-call extraction is shared -//! with the in-session [`claude_code_transcript`](super::claude_code_transcript) -//! parser. The differences are all in the envelope: there are no per-line +//! with the [`claude_code_transcript`](super::claude_code_transcript) record +//! types. The differences are all in the envelope: there are no per-line //! timestamps, and a terminal `result` event carries the authoritative final //! text, wall-clock duration, and token usage. `system`, `rate_limit_event`, and //! any other non-message events are ignored (they don't deserialize into an @@ -35,7 +35,7 @@ struct ResultEvent { usage: Option, } -/// Parse the event stream into ordered tool invocations. Reuses the in-session +/// Parse the event stream into ordered tool invocations. Reuses the shared /// extractor: non-message events deserialize into records the extractor skips. pub fn parse_claude_stream_json(path: &Path) -> io::Result> { Ok(extract_invocations(&read_records(path)?)) diff --git a/src/adapters/cli_command.rs b/src/adapters/cli_command.rs index ab841cd..a09793e 100644 --- a/src/adapters/cli_command.rs +++ b/src/adapters/cli_command.rs @@ -1,4 +1,4 @@ -//! Shared rendering helpers for `DispatchMechanism::Cli` command templates +//! Shared rendering helpers for harness CLI command templates //! (Codex's `codex exec`, Claude Code's `claude -p`). /// Quote a value for a POSIX shell only when it contains anything outside a diff --git a/src/adapters/codex_cli.rs b/src/adapters/codex_cli.rs index de64db0..a06db69 100644 --- a/src/adapters/codex_cli.rs +++ b/src/adapters/codex_cli.rs @@ -1,4 +1,4 @@ -//! Codex CLI command rendering for `DispatchMechanism::Cli` guidance. +//! Codex CLI command rendering (`codex exec`) for dispatch guidance. use super::cli_command::render_cli_model_arg; use std::path::Path; diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 570b69e..9269d07 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -23,8 +23,7 @@ use super::codex_cli::{ }; use super::{ parse_claude_stream_json, parse_claude_stream_json_full, parse_codex_events, - parse_codex_events_full, parse_transcript, parse_transcript_full, - render_available_skills_block, render_codex_available_skills_block, + parse_codex_events_full, render_available_skills_block, render_codex_available_skills_block, render_opencode_available_skills_block, }; @@ -72,76 +71,46 @@ pub trait HarnessAdapter { format!("\n{trimmed}\n") } - /// The **interactive** (agent-followed) `RUNBOOK.md` template a harness uses - /// under [`InSession`](crate::core::DispatchMechanism::InSession) dispatch, - /// carrying `{{TOKEN}}` placeholders the run fills. The default is the shared - /// headless template (harmless for the Cli-only harnesses that never read it - /// via this path); [`InSession`](crate::core::DispatchMechanism::InSession) - /// harnesses override it. The Cli-dispatch runbook always uses - /// [`HEADLESS_RUNBOOK_TEMPLATE`], selected by mechanism in `build_runbook`. - fn runbook_template(&self) -> &'static str { - HEADLESS_RUNBOOK_TEMPLATE - } - - /// For a [`Cli`](crate::core::DispatchMechanism::Cli)-dispatch harness, the - /// filename (under a task's `outputs/` dir) its one-shot CLI writes the - /// transcript to. `None` when the harness dispatches in-session (no local - /// transcript) or has no Cli-mechanism transcript wired yet. + /// The filename (under a task's `outputs/` dir) this harness's one-shot CLI + /// writes the captured transcript to. `None` when the harness has no + /// transcript ingest wired yet (e.g. OpenCode). fn cli_events_filename(&self) -> Option<&'static str> { None } - /// For a [`Cli`](crate::core::DispatchMechanism::Cli)-dispatch harness, the - /// native model-selection flag accepted by the harness CLI. `None` means the - /// adapter has no model-selection support wired yet. + /// The native model-selection flag accepted by this harness's CLI. `None` + /// means the adapter has no model-selection support wired yet. fn cli_model_flag(&self) -> Option<&'static str> { None } - /// The `Next:` guidance printed after `run` for a - /// [`Cli`](crate::core::DispatchMechanism::Cli)-dispatch harness: how to - /// dispatch each task through this harness's one-shot CLI and then ingest. - /// Empty for in-session harnesses (their guidance is the mechanism's, not the - /// adapter's). + /// The `Next:` guidance printed after `run`: how to dispatch each task through + /// this harness's one-shot CLI and then ingest. Empty when the adapter has no + /// dispatch recipe wired. fn cli_next_steps(&self, _ctx: CliDispatchContext<'_>) -> String { String::new() } - /// Extra `dispatch-manifest.md` lines describing this harness's Cli dispatch + /// Extra `dispatch-manifest.md` lines describing this harness's dispatch /// recipe (command template, parallel recipe, ingest note). `None` when the - /// harness contributes no Cli-specific manifest section. + /// harness contributes no manifest section. fn cli_manifest_section(&self, _ctx: CliManifestContext<'_>) -> Option> { None } - /// The post-`grade` / post-`ingest` judge dispatch guidance for a - /// [`Cli`](crate::core::DispatchMechanism::Cli)-dispatch harness. `None` - /// leaves the generic in-session-style judge handoff in place. + /// The post-`grade` / post-`ingest` judge dispatch guidance for this harness. + /// `None` leaves the generic judge handoff in place. fn cli_judge_next_steps(&self, _ctx: CliJudgeContext<'_>) -> Option { None } - /// Parse a persisted transcript into its ordered tool invocations. - fn parse_transcript(&self, path: &Path) -> io::Result>; - - /// Parse a persisted transcript into the full summary: tool invocations, - /// deduped token usage, duration, and final message text. - fn parse_transcript_full(&self, path: &Path) -> io::Result; - - /// Parse a [`Cli`](crate::core::DispatchMechanism::Cli)-mechanism events file - /// (the harness CLI's captured output) into ordered tool invocations. Defaults - /// to [`parse_transcript`](Self::parse_transcript): for Codex/OpenCode the - /// on-disk parser already *is* the events parser, so the default is correct; - /// Claude Code overrides it, because its `parse_transcript` is the in-session - /// subagent parser while its Cli events are `claude -p` stream-json. - fn parse_cli_events(&self, path: &Path) -> io::Result> { - self.parse_transcript(path) - } + /// Parse the events file this harness's one-shot CLI wrote (the captured + /// transcript) into ordered tool invocations. + fn parse_cli_events(&self, path: &Path) -> io::Result>; - /// The full-summary counterpart of [`parse_cli_events`](Self::parse_cli_events). - fn parse_cli_events_full(&self, path: &Path) -> io::Result { - self.parse_transcript_full(path) - } + /// The full-summary counterpart of [`parse_cli_events`](Self::parse_cli_events): + /// tool invocations, deduped token usage, duration, and final message text. + fn parse_cli_events_full(&self, path: &Path) -> io::Result; /// Arm the write guard using this harness's native pre-tool hook surface, /// returning the staged marker path. The guard's allowed roots are derived @@ -164,9 +133,8 @@ pub trait HarnessAdapter { } } -/// The shared **headless** (human-followed) `RUNBOOK.md` template used by every -/// [`Cli`](crate::core::DispatchMechanism::Cli)-dispatch run, regardless of -/// harness (Codex, OpenCode, and Claude Code in hybrid/headless). +/// The shared (human-followed) `RUNBOOK.md` template used by every run, +/// regardless of harness (Claude Code, Codex, OpenCode). pub const HEADLESS_RUNBOOK_TEMPLATE: &str = include_str!("../../profiles/shared/runbook-headless.md"); @@ -219,9 +187,6 @@ impl HarnessAdapter for ClaudeCodeAdapter { fn skill_unresolved_phrase(&self) -> &'static str { "If the Skill tool cannot resolve that identifier" } - fn runbook_template(&self) -> &'static str { - include_str!("../../profiles/claude-code/runbook.md") - } fn cli_events_filename(&self) -> Option<&'static str> { Some("claude-events.jsonl") } @@ -262,12 +227,6 @@ impl HarnessAdapter for ClaudeCodeAdapter { ctx.iteration_dir, )) } - fn parse_transcript(&self, path: &Path) -> io::Result> { - parse_transcript(path) - } - fn parse_transcript_full(&self, path: &Path) -> io::Result { - parse_transcript_full(path) - } fn parse_cli_events(&self, path: &Path) -> io::Result> { parse_claude_stream_json(path) } @@ -284,7 +243,7 @@ impl HarnessAdapter for ClaudeCodeAdapter { } fn guard_armed_message(&self) -> Option<&'static str> { Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches —\n both in-session subagents and `claude -p` (hybrid/headless), which loads the\n hook from the env cwd each dispatch runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", + "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches.\n Each `claude -p` dispatch loads the hook from the env cwd it runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", ) } } @@ -352,10 +311,10 @@ impl HarnessAdapter for CodexAdapter { ctx.iteration_dir, )) } - fn parse_transcript(&self, path: &Path) -> io::Result> { + fn parse_cli_events(&self, path: &Path) -> io::Result> { parse_codex_events(path) } - fn parse_transcript_full(&self, path: &Path) -> io::Result { + fn parse_cli_events_full(&self, path: &Path) -> io::Result { parse_codex_events_full(path) } fn install_guard( @@ -407,15 +366,20 @@ impl HarnessAdapter for OpenCodeAdapter { iteration = ctx.iteration ) } - // OpenCode transcript ingest is not yet wired. In the current dispatch flow - // this is unreachable (no subagents dir and no events file), so delegating to - // the shared JSONL parser preserves the pre-refactor behavior of the - // transcript-source branch until OpenCode ingest lands. - fn parse_transcript(&self, path: &Path) -> io::Result> { - parse_transcript(path) + // OpenCode transcript ingest is not yet wired: its `cli_events_filename` is + // `None`, so the ingest pipeline never reaches these parsers. They error + // rather than parse until OpenCode ingest lands. + fn parse_cli_events(&self, _path: &Path) -> io::Result> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "opencode transcript ingest is not yet wired", + )) } - fn parse_transcript_full(&self, path: &Path) -> io::Result { - parse_transcript_full(path) + fn parse_cli_events_full(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "opencode transcript ingest is not yet wired", + )) } fn install_guard( &self, @@ -542,10 +506,6 @@ mod tests { assert_eq!(summary.duration_ms, Some(5637)); assert_eq!(summary.tool_invocations.len(), 1); assert_eq!(summary.tool_invocations[0].name, "Bash"); - - // The on-disk parser would find no duration here (no line timestamps), - // proving parse_cli_events_full routes to the stream-json parser. - assert_eq!(a.parse_transcript_full(&path).unwrap().duration_ms, None); } #[test] diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index f48abb1..aa3f78b 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -23,14 +23,8 @@ pub use harness::{ HEADLESS_RUNBOOK_TEMPLATE, HarnessAdapter, OpenCodeAdapter, adapter_for, }; -pub use claude_code_session::{ - render_available_skills_block, render_plan_mode_context, resolve_subagents_dir_for_session, - slugify_project_path, -}; -pub use claude_code_transcript::{ - SubagentEntry, SubagentMeta, TranscriptSummary, find_by_description, list_subagents, - parse_transcript, parse_transcript_full, -}; +pub use claude_code_session::{render_available_skills_block, render_plan_mode_context}; +pub use claude_code_transcript::TranscriptSummary; pub use claude_stream_json::{parse_claude_stream_json, parse_claude_stream_json_full}; pub use codex_session::{render_codex_available_skills_block, render_codex_plan_mode_context}; pub use codex_transcript::{parse_codex_events, parse_codex_events_full}; diff --git a/src/cli/args.rs b/src/cli/args.rs index 92a5656..e68c2e4 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -75,24 +75,21 @@ pub struct CommonArgs { /// Target harness: `claude-code` (default), `codex`, or `opencode`. /// /// Claude Code and Codex both support staged skills, transcript ingest, and - /// `--guard`. Codex stages skills under `.agents/skills` and reads each - /// task's `outputs/codex-events.jsonl` instead of a subagents dir. + /// `--guard`. Each reads its own per-task events file (`claude-events.jsonl`, + /// `codex-events.jsonl`); Codex stages skills under `.agents/skills`. /// OpenCode stages skills under `.opencode/skills`; transcript ingest and /// `--guard` are not yet wired for OpenCode. #[arg(long)] pub harness: Option, - /// Run mode: `interactive` (in-session subagents), `hybrid` (an agent - /// orchestrates while each dispatch shells out to the harness CLI), or - /// `headless` (CLI-only, no session). - /// - /// Defaults per harness — Claude Code → `interactive`, Codex/OpenCode → - /// `hybrid`. `hybrid`/`headless` dispatch through the harness CLI (`claude -p`, - /// `codex exec`) and read each task's `outputs/-events.jsonl`; - /// `interactive` dispatches in-session subagents. Claude Code wires all three - /// (`hybrid`/`headless` ride `claude -p` stream-json); Codex wires `hybrid` + - /// `headless`; OpenCode wires `hybrid` only. Pass the same value to every command - /// of a run (it selects the transcript source at `ingest`); the printed next-step - /// commands already carry it. + /// Run mode: `hybrid` (an agent orchestrates while each dispatch shells out to + /// the harness CLI) or `headless` (CLI-only, no session). + /// + /// Every harness defaults to `hybrid`. Both modes dispatch through the harness + /// CLI (`claude -p`, `codex exec`) and read each task's + /// `outputs/-events.jsonl`; they differ only in whether an agent or a + /// human drives the loop. Claude Code and Codex wire both modes; OpenCode wires + /// `hybrid` only. Pass the same value to every command of a run; the printed + /// next-step commands already carry it. #[arg(long)] pub run_mode: Option, /// Workspace directory (defaults to `/.eval-magic`). @@ -101,25 +98,6 @@ pub struct CommonArgs { /// `teardown`. #[arg(long)] pub workspace_dir: Option, - /// Subagents transcript dir (Claude Code only), e.g. - /// `~/.claude/projects///subagents/`. - /// - /// Where Claude Code persisted subagent transcripts. `ingest`/`record-runs`/ - /// `fill-transcripts` read it to populate `tool_invocations`, tokens, and - /// duration. Optional: when omitted it is auto-resolved from `--session-id` - /// (or the `CLAUDE_CODE_SESSION_ID` env var); pass it explicitly only to - /// override. Not used for Codex, which reads `outputs/codex-events.jsonl`. - #[arg(long)] - pub subagents_dir: Option, - /// Parent session id for auto-resolving `--subagents-dir` (Claude Code only). - /// - /// Defaults to the `CLAUDE_CODE_SESSION_ID` env var that Claude Code sets in - /// the orchestrating agent's shell. `ingest`/`record-runs`/`fill-transcripts` - /// use it to locate `/projects///subagents/` - /// (scanning `projects/*` if the cwd slug differs). Pass it only when running - /// outside that session; an explicit `--subagents-dir` overrides it. - #[arg(long)] - pub session_id: Option, /// Restrict to these eval ids (comma-separated). /// /// Mutually exclusive with `--skip`; every named id must exist or the run @@ -308,10 +286,10 @@ pub struct RunArgs { /// Codex dispatches must include `--dangerously-bypass-hook-trust` so the /// vetted project-local eval hook runs. Unguarded, stray writes are only /// *detected* after the fact by `detect-stray-writes`, never blocked. - /// Works under Claude Code's CLI run modes (`hybrid`/`headless`) too: the - /// `PreToolUse` hook is staged in `env/.claude/settings.local.json`, and each - /// `claude -p` dispatch loads it from that cwd (`cd `), enforcing the - /// same boundary as an in-session run (the recipe never passes `--bare`). + /// Under Claude Code the `PreToolUse` hook is staged in each env's + /// `.claude/settings.local.json`, and each `claude -p` dispatch loads it from + /// that cwd (`cd `), enforcing the eval boundary (the recipe never + /// passes `--bare`). /// When invoking this from inside Codex, staging writes `.agents/skills` and /// guarded runs also write `.codex/hooks.json`; Codex protects those paths in /// its default workspace-write sandbox, so approval/escalation may be needed. @@ -348,10 +326,9 @@ pub struct RunArgs { /// Agent-under-test model for CLI dispatches; otherwise recorded as /// provenance. /// - /// For `Cli`-mechanism harnesses such as Codex, the run's dispatch recipes - /// include the harness-native model flag when the adapter supports one. For - /// in-session dispatch, the runner cannot select the model, so the value is - /// persisted to `conditions.json` for `promote-baseline`. + /// The run's dispatch recipes include the harness-native model flag when the + /// adapter supports one (e.g. Codex's `-m`, Claude Code's `--model`); otherwise + /// the value is persisted to `conditions.json` for `promote-baseline`. #[arg(long)] pub agent_model: Option, /// Default judge model for emitted judge tasks. @@ -377,10 +354,9 @@ pub(crate) enum Commands { /// /// Builds the iteration workspace, snapshots the `SKILL.md`, stages skills, and /// emits `dispatch.json` (machine-readable) alongside `dispatch-manifest.md` - /// (human-readable). Your agent then dispatches each task as a fresh subagent. - /// Also writes `RUNBOOK.md`, a followable handoff for an isolated run session - /// ("Read and follow RUNBOOK.md") — interactive (agent-followed) for Claude - /// Code, human-followed for Codex/OpenCode. + /// (human-readable). Dispatch each task through the harness CLI (`claude -p`, + /// `codex exec`). Also writes `RUNBOOK.md`, a human-followable handoff for the + /// run ("Read and follow RUNBOOK.md"). Run(RunArgs), /// Snapshot a workspace baseline. /// @@ -405,10 +381,8 @@ pub(crate) enum Commands { /// grade. Assembles each task's `run.json` + `timing.json`, scans for stray /// writes, grades `transcript_check` assertions, then stops at the judge /// hand-off, listing a judge task per `llm_judge` assertion. Requires - /// `--iteration`; Claude Code auto-resolves the subagents dir from the session - /// id (override with `--subagents-dir`), while Codex reads each task's - /// `outputs/codex-events.jsonl`. Re-running after a fix is safe — every - /// sub-step skips work already done. + /// `--iteration`; reads each task's `outputs/-events.jsonl`. + /// Re-running after a fix is safe — every sub-step skips work already done. Ingest(CommonArgs), /// Finalize grading after judge responses are in. /// @@ -421,14 +395,14 @@ pub(crate) enum Commands { /// /// Assembles a schema-valid `run.json` and backfills `timing.json` for every /// task in a runner-built iteration, from `dispatch.json` + - /// `outputs/final-message.md` + the persisted transcript. Never clobbers - /// existing records without `--overwrite`; transcript-derived timing carries - /// `"source": "transcript"`. Folded into `ingest`. + /// `outputs/final-message.md` + each task's `outputs/-events.jsonl`. + /// Never clobbers existing records without `--overwrite`; transcript-derived + /// timing carries `"source": "transcript"`. Folded into `ingest`. RecordRuns(CommonArgs), /// Populate tool invocations from persisted transcripts. /// - /// Matches each `(eval, condition)` to a subagent transcript by description and - /// populates `tool_invocations` in `run.json`. Subsumed by `record-runs` for + /// Reads each task's `outputs/-events.jsonl` and populates + /// `tool_invocations` in `run.json`. Subsumed by `record-runs` for /// runner-built iterations; still the tool for filling a pre-existing (hand- or /// agent-written) `run.json`. FillTranscripts(CommonArgs), diff --git a/src/cli/commands/pipeline.rs b/src/cli/commands/pipeline.rs index 91865ea..87fca42 100644 --- a/src/cli/commands/pipeline.rs +++ b/src/cli/commands/pipeline.rs @@ -8,10 +8,8 @@ use crate::adapters::{CliJudgeContext, adapter_for}; use crate::cli::args::{CommonArgs, GradeArgs}; use crate::cli::command_target_args; use crate::cli::run; -use crate::cli::{ - iteration_dir, resolve_iteration, resolve_subagents_dir, run_context_from, staged_env_roots, -}; -use crate::core::{DispatchMechanism, RunContext}; +use crate::cli::{iteration_dir, resolve_iteration, run_context_from, staged_env_roots}; +use crate::core::RunContext; use crate::pipeline; use crate::sandbox; use crate::validation; @@ -23,21 +21,16 @@ fn judge_dispatch_guidance(ctx: &RunContext, iteration: u32) -> String { .workspace_root .join(&ctx.skill_name) .join(format!("iteration-{iteration}")); - match ctx.run_mode.mechanism() { - DispatchMechanism::InSession => { - format!("Dispatch each task as a judge subagent with:\n {JUDGE_WORKER_PROMPT}") - } - DispatchMechanism::Cli => adapter_for(ctx.harness) - .cli_judge_next_steps(CliJudgeContext { - guard: sandbox::guard_is_armed(&ctx.stage_root), - iteration_dir: &iteration_dir, - }) - .unwrap_or_else(|| { - format!( - "Dispatch each task from judge-tasks.json with:\n {JUDGE_WORKER_PROMPT}\nModel selection is recorded in judge-tasks.json, but this harness adapter has no judge CLI recipe wired yet." - ) - }), - } + adapter_for(ctx.harness) + .cli_judge_next_steps(CliJudgeContext { + guard: sandbox::guard_is_armed(&ctx.stage_root), + iteration_dir: &iteration_dir, + }) + .unwrap_or_else(|| { + format!( + "Dispatch each task from judge-tasks.json with:\n {JUDGE_WORKER_PROMPT}\nModel selection is recorded in judge-tasks.json, but this harness adapter has no judge CLI recipe wired yet." + ) + }) } /// Execute one chain step by mapping its [`run::steps::StepKind`] to the stage @@ -54,10 +47,6 @@ fn run_step(step: &run::steps::StepCommand) -> anyhow::Result<()> { harness: Some(step.harness), run_mode: Some(step.run_mode), workspace_dir: step.workspace_dir.clone(), - // The chain carries the already-resolved absolute subagents dir, so the - // session id is no longer needed downstream. - subagents_dir: step.subagents_dir.clone(), - session_id: None, only: None, skip: None, overwrite: false, @@ -80,12 +69,6 @@ fn run_step(step: &run::steps::StepCommand) -> anyhow::Result<()> { pub(crate) fn run_ingest(args: CommonArgs) -> anyhow::Result<()> { let ctx = run_context_from(&args)?; let iteration = resolve_iteration(&ctx, args.iteration)?; - let resolved = resolve_subagents_dir( - ctx.run_mode.mechanism(), - args.subagents_dir.as_deref(), - args.session_id.as_deref(), - )?; - let resolved = resolved.as_ref().map(|p| p.to_string_lossy().into_owned()); let steps = run::steps::build_ingest_commands(&run::steps::StepParams { skill_dir: args.skill_dir.as_deref(), @@ -93,7 +76,6 @@ pub(crate) fn run_ingest(args: CommonArgs) -> anyhow::Result<()> { iteration, harness: ctx.harness, run_mode: ctx.run_mode, - subagents_dir: resolved.as_deref(), workspace_dir: args.workspace_dir.as_deref(), }); if let Some(failed) = run::steps::run_steps(&steps, run_step) { @@ -138,7 +120,6 @@ pub(crate) fn run_finalize(args: CommonArgs) -> anyhow::Result<()> { iteration, harness: ctx.harness, run_mode: ctx.run_mode, - subagents_dir: None, workspace_dir: args.workspace_dir.as_deref(), }); if let Some(failed) = run::steps::run_steps(&steps, run_step) { @@ -148,15 +129,11 @@ pub(crate) fn run_finalize(args: CommonArgs) -> anyhow::Result<()> { println!( "\n✅ Finalize complete. Read the benchmark above, then tear down: eval-magic teardown{target_args}" ); - // Warn if a guard is still armed. The cwd check covers the in-session flow (run - // from inside `env/`); under Cli there is one env per (group, condition), so also - // walk each per-env marker. `teardown` (not the cwd-only `teardown-guard`) is what - // disarms them all. + // Warn if a guard is still armed. There is one env per (group, condition), so + // walk each per-env marker as well as the cwd. `teardown` (not the cwd-only + // `teardown-guard`) is what disarms them all. let mut armed = sandbox::guard_is_armed(&ctx.stage_root); - if !armed - && ctx.run_mode.mechanism() == DispatchMechanism::Cli - && let Ok(dir) = iteration_dir(&ctx, Some(iteration)) - { + if !armed && let Ok(dir) = iteration_dir(&ctx, Some(iteration)) { armed = staged_env_roots(&dir) .iter() .any(|env| sandbox::guard_is_armed(env)); @@ -173,17 +150,8 @@ pub(crate) fn run_finalize(args: CommonArgs) -> anyhow::Result<()> { /// `dispatch.json`. pub(crate) fn run_record_runs(args: CommonArgs) -> anyhow::Result<()> { let ctx = run_context_from(&args)?; - let mechanism = ctx.run_mode.mechanism(); - let resolved = resolve_subagents_dir( - mechanism, - args.subagents_dir.as_deref(), - args.session_id.as_deref(), - )?; - let subagents_dir = resolved.as_deref(); - let dir = iteration_dir(&ctx, args.iteration)?; - let result = - pipeline::record_runs(&dir, ctx.harness, mechanism, subagents_dir, args.overwrite)?; + let result = pipeline::record_runs(&dir, ctx.harness, args.overwrite)?; println!( "\nRecorded: {}, skipped (existing run.json): {}, skipped (no final message): {}, skipped (prompt unread): {}, missing transcript: {}", @@ -193,7 +161,7 @@ pub(crate) fn run_record_runs(args: CommonArgs) -> anyhow::Result<()> { result.skipped_prompt_unread, result.missing_transcript ); - if let Some(warning) = result.transcript_warning(ctx.harness, mechanism) { + if let Some(warning) = result.transcript_warning(ctx.harness) { eprintln!("{warning}"); } if let Some(warning) = result.prompt_unread_warning() { @@ -206,17 +174,8 @@ pub(crate) fn run_record_runs(args: CommonArgs) -> anyhow::Result<()> { /// the iteration. pub(crate) fn run_fill_transcripts(args: CommonArgs) -> anyhow::Result<()> { let ctx = run_context_from(&args)?; - let mechanism = ctx.run_mode.mechanism(); - let resolved = resolve_subagents_dir( - mechanism, - args.subagents_dir.as_deref(), - args.session_id.as_deref(), - )?; - let subagents_dir = resolved.as_deref(); - let dir = iteration_dir(&ctx, args.iteration)?; - let result = - pipeline::fill_transcripts(&dir, ctx.harness, mechanism, subagents_dir, args.overwrite)?; + let result = pipeline::fill_transcripts(&dir, ctx.harness, args.overwrite)?; println!( "\nFilled: {}, skipped (already populated): {}, missing transcript: {}", @@ -274,7 +233,7 @@ pub(crate) fn run_detect_stray_writes(args: CommonArgs) -> anyhow::Result<()> { let clean = t.violations == 0 && t.warnings == 0 && t.live_source_reads == 0; if clean && report.invocations_inspected == 0 { eprintln!( - "⚠ Unverifiable — 0 transcript tool-calls inspected. Stray-write detection had nothing to check (every run's tool_invocations is empty); link transcripts first, then re-run (see the record-runs warning about passing agent_description verbatim / pointing --subagents-dir at the right session)." + "⚠ Unverifiable — 0 transcript tool-calls inspected. Stray-write detection had nothing to check (every run's tool_invocations is empty); link transcripts first, then re-run (confirm each task's `outputs/-events.jsonl` exists — see the record-runs warning)." ); } else if clean { println!("✓ No out-of-bounds writes or live-source reads detected."); diff --git a/src/cli/commands/workspace.rs b/src/cli/commands/workspace.rs index 2905485..62032e1 100644 --- a/src/cli/commands/workspace.rs +++ b/src/cli/commands/workspace.rs @@ -8,7 +8,6 @@ use crate::cli::run; use crate::cli::{ command_target_args, iteration_dir, resolve_iteration, run_context_from, staged_env_roots, }; -use crate::core::DispatchMechanism; use crate::sandbox; use crate::workspace; @@ -91,15 +90,12 @@ pub(crate) fn run_promote_baseline(args: PromoteBaselineArgs) -> anyhow::Result< /// any iteration with uncommitted results. pub(crate) fn run_teardown(args: CommonArgs) -> anyhow::Result<()> { let ctx = run_context_from(&args)?; - // Disarm the guard at the invocation cwd — the in-session flow runs teardown from - // inside `env/`. Under Cli there is one env per (group, condition) and the human - // runs teardown from the iteration dir, so additionally walk each per-env marker - // (before `cleanup_workspace` reclaims the tree). Best-effort: a missing iteration - // just skips the walk; `teardown_guard` is a no-op without a marker. + // Disarm the guard at the invocation cwd, then walk each per-(group, condition) + // env marker (the human runs teardown from the iteration dir) before + // `cleanup_workspace` reclaims the tree. Best-effort: a missing iteration just + // skips the walk; `teardown_guard` is a no-op without a marker. let mut torn = sandbox::teardown_guard(&std::env::current_dir()?); - if ctx.run_mode.mechanism() == DispatchMechanism::Cli - && let Ok(dir) = iteration_dir(&ctx, args.iteration) - { + if let Ok(dir) = iteration_dir(&ctx, args.iteration) { for env in staged_env_roots(&dir) { torn |= sandbox::teardown_guard(&env); } diff --git a/src/cli/help.rs b/src/cli/help.rs index c97afab..937c14f 100644 --- a/src/cli/help.rs +++ b/src/cli/help.rs @@ -13,16 +13,14 @@ EXAMPLES: # Mode A — evaluate a new skill (with vs. without) eval-magic run --guard - # run builds the isolated env/ + RUNBOOK.md, then prints a handoff: - # cd into env/, start a fresh session, say \"Read and follow RUNBOOK.md\". - # The fresh session walks the whole loop below from inside env/: - # …dispatch each task in dispatch.json as a fresh subagent… - # eval-magic ingest # auto-resolves --subagents-dir from CLAUDE_CODE_SESSION_ID - # # (override: --session-id or --subagents-dir ) + # run builds per-(group, condition) envs + RUNBOOK.md (a human-followed recipe). + # Follow it to dispatch each task in dispatch.json via `claude -p`, capturing each + # task's outputs/claude-events.jsonl, then: + # eval-magic ingest # reads each task's outputs/claude-events.jsonl # …dispatch each judge task ingest listed… # eval-magic finalize # eval-magic teardown - eval-magic promote-baseline # optional, from the prep session once benchmark.json lands + eval-magic promote-baseline # optional, once benchmark.json lands # Mode B — evaluate a language change (edit-first) eval-magic snapshot --ref HEAD diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 3558ab3..fb7b44d 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -11,8 +11,7 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail}; use clap::Parser; -use crate::adapters::{config_dir_from_env, resolve_subagents_dir_for_session}; -use crate::core::{DetectInput, DispatchMechanism, RunContext, detect_run_context}; +use crate::core::{DetectInput, RunContext, detect_run_context}; mod args; mod commands; @@ -39,8 +38,6 @@ fn dispatch(command: Option) -> anyhow::Result<()> { harness: None, run_mode: None, workspace_dir: None, - subagents_dir: None, - session_id: None, only: None, skip: None, overwrite: false, @@ -116,10 +113,10 @@ pub(crate) fn parse_id_list(v: Option<&str>) -> Option> { /// Always names `--skill-dir`, `--skill`, and `--workspace-dir` (all three are /// always populated in [`RunContext`] and always re-resolve), so the printed /// "Next:" commands are copy-pasteable from any cwd — not just the one `run` -/// happened to start in. The absolute `--workspace-dir` is what lets the isolated -/// session run `ingest`/`finalize`/`switch-condition` from `cwd = iteration-N/env/`: -/// without it, `workspace_root` would default to `/.eval-magic` -/// (`detect_run_context`) and the iteration tree above the env would not resolve. +/// happened to start in. The absolute `--workspace-dir` is what lets the human +/// run `ingest`/`finalize` from a per-`(group, condition)` env dir: without it, +/// `workspace_root` would default to `/.eval-magic` (`detect_run_context`) +/// and the iteration tree above the env would not resolve. pub(crate) fn command_target_args(ctx: &RunContext) -> String { format!( " --skill-dir {} --skill {} --workspace-dir {} --run-mode {}", @@ -176,13 +173,12 @@ pub(crate) fn iteration_dir(ctx: &RunContext, iteration: Option) -> anyhow: Ok(dir) } -/// The env directories a run staged under `iteration_dir`: the single `env/` for -/// the InSession mechanism, or one `env--/` per `(group, condition)` -/// for Cli. A best-effort directory scan (returns empty when the dir can't be read), -/// used by `teardown`/`finalize` to walk every env's write guard. Preferred over -/// reading `dispatch.json` because it has no parse-failure mode, needs no path -/// re-basing (recorded env dirs can be relative), and the only `env`/`env-*` children -/// of an iteration dir are the staged envs. +/// The env directories a run staged under `iteration_dir`: one +/// `env--/` per `(group, condition)`. A best-effort directory +/// scan (returns empty when the dir can't be read), used by `teardown`/`finalize` +/// to walk every env's write guard. Preferred over reading `dispatch.json` because +/// it has no parse-failure mode, needs no path re-basing (recorded env dirs can be +/// relative), and the only `env-*` children of an iteration dir are the staged envs. pub(crate) fn staged_env_roots(iteration_dir: &Path) -> Vec { let Ok(entries) = std::fs::read_dir(iteration_dir) else { return Vec::new(); @@ -199,57 +195,6 @@ pub(crate) fn staged_env_roots(iteration_dir: &Path) -> Vec { .collect() } -/// Resolve the subagents transcript dir for an in-session stage that reads -/// transcripts. The subagents dir is the `InSession` transcript source, so this -/// is keyed on the dispatch *mechanism*, not the harness: `Cli`-mechanism runs -/// (Codex; Claude Code hybrid/headless) read each task's `outputs/.jsonl` -/// and resolve to `None` — they must never bail on a missing -/// `CLAUDE_CODE_SESSION_ID`. For the `InSession` mechanism (Claude Code -/// interactive), precedence is: an explicit `--subagents-dir` (validated to -/// exist) wins; otherwise resolve from a session id — the `--session-id` flag if -/// given, else the `CLAUDE_CODE_SESSION_ID` env var Claude Code sets in the -/// orchestrating agent's shell — locating -/// `/projects///subagents/` (scanning `projects/*` -/// if the cwd slug differs). -pub(crate) fn resolve_subagents_dir( - mechanism: DispatchMechanism, - subagents_dir: Option<&str>, - session_id: Option<&str>, -) -> anyhow::Result> { - if mechanism != DispatchMechanism::InSession { - return Ok(None); - } - if let Some(dir) = subagents_dir { - let path = PathBuf::from(dir); - if !path.exists() { - bail!("subagents-dir not found: {}", path.display()); - } - return Ok(Some(path)); - } - let session = session_id - .map(str::to_string) - .or_else(|| std::env::var("CLAUDE_CODE_SESSION_ID").ok()) - .filter(|s| !s.trim().is_empty()); - let Some(session) = session else { - bail!( - "could not auto-resolve the subagents dir: CLAUDE_CODE_SESSION_ID is not set. \ - Re-run inside the Claude Code session that dispatched the subagents, or pass \ - --session-id or --subagents-dir ." - ); - }; - let config_dir = config_dir_from_env(); - let cwd = std::env::current_dir()?; - match resolve_subagents_dir_for_session(&config_dir, &cwd, &session) { - Some(path) => Ok(Some(path)), - None => bail!( - "no subagents dir found for session {session} under {}/projects/. The session may \ - not have dispatched any subagents (or lives under a different CLAUDE_CONFIG_DIR). \ - Pass --subagents-dir to override.", - config_dir.display() - ), - } -} - #[cfg(test)] mod tests { use super::*; @@ -310,11 +255,10 @@ mod tests { assert_eq!(resolved.skill_subdir, ctx.skill_subdir); } - /// The isolated session runs `ingest`/`finalize`/`switch-condition` from - /// `cwd = iteration-N/env/`. Without an explicit workspace root those commands - /// default `workspace_root` to `/.eval-magic` and bail "not found", - /// so the selector must carry an absolute `--workspace-dir` pointing at the - /// real workspace above the env. + /// The human runs `ingest`/`finalize` from a per-`(group, condition)` env dir. + /// Without an explicit workspace root those commands default `workspace_root` + /// to `/.eval-magic` and bail "not found", so the selector must carry an + /// absolute `--workspace-dir` pointing at the real workspace above the env. #[test] fn target_args_carry_absolute_workspace_dir() { let tmp = TempDir::new().unwrap(); @@ -357,55 +301,4 @@ mod tests { .unwrap(); assert_eq!(resolved.workspace_root, ctx.workspace_root); } - - #[test] - fn resolve_subagents_dir_is_none_for_cli_mechanism() { - // The subagents dir is the InSession transcript source. Cli-mechanism - // runs (Codex; Claude Code hybrid/headless) read each task's events file, - // so resolution is a no-op — and must NOT bail on a missing - // CLAUDE_CODE_SESSION_ID. This is the regression: the old harness-keyed - // gate forced session resolution for Claude Code and aborted under - // hybrid/headless. The Cli arm returns before reading any env var, so this - // is deterministic regardless of the test runner's environment. - assert_eq!( - resolve_subagents_dir(DispatchMechanism::Cli, None, None).unwrap(), - None - ); - // A passed --subagents-dir is ignored in Cli mode (the events file is the - // source), so it resolves to None without touching the filesystem. - assert_eq!( - resolve_subagents_dir(DispatchMechanism::Cli, Some("/whatever"), None).unwrap(), - None - ); - } - - #[test] - fn resolve_subagents_dir_uses_existing_explicit_dir() { - // InSession (Claude Code interactive): an explicit, existing - // --subagents-dir wins over any session-id resolution. - let tmp = TempDir::new().unwrap(); - let resolved = resolve_subagents_dir( - DispatchMechanism::InSession, - Some(&tmp.path().display().to_string()), - None, - ) - .unwrap(); - assert_eq!(resolved, Some(tmp.path().to_path_buf())); - } - - #[test] - fn resolve_subagents_dir_errors_when_explicit_dir_missing() { - // InSession with an explicit --subagents-dir that doesn't exist is a hard - // error (not a silent fallback to session-id resolution). - let err = resolve_subagents_dir( - DispatchMechanism::InSession, - Some("/no/such/subagents/dir/xyz"), - None, - ) - .unwrap_err(); - assert!( - err.to_string().contains("subagents-dir not found"), - "got: {err}" - ); - } } diff --git a/src/cli/run/dispatch.rs b/src/cli/run/dispatch.rs index e610166..131b0dd 100644 --- a/src/cli/run/dispatch.rs +++ b/src/cli/run/dispatch.rs @@ -14,7 +14,7 @@ use regex::Regex; use serde::{Deserialize, Serialize}; use crate::adapters::{CliManifestContext, adapter_for}; -use crate::core::{AvailableSkill, DispatchMechanism, Eval, Harness}; +use crate::core::{AvailableSkill, Eval, Harness}; use super::RunError; @@ -43,9 +43,8 @@ pub struct DispatchTask { /// byte-identical. #[serde(default, skip_serializing_if = "Option::is_none")] pub group: Option, - /// The agent-under-test's cwd for this task (its env dir). Absent in the - /// single-group case, where the Cli recipe's `` placeholder still - /// resolves to `env/`; present (per `(group, condition)`) for multi-group Cli. + /// The agent-under-test's cwd for this task — its per-`(group, condition)` env + /// dir, which the CLI dispatch recipe's `` placeholder resolves to. #[serde(default, skip_serializing_if = "Option::is_none")] pub eval_root: Option, #[serde(default, skip_serializing)] @@ -381,7 +380,6 @@ pub use crate::core::Mode; #[derive(Debug, Clone, Copy)] pub struct ManifestContext<'a> { pub harness: Harness, - pub mechanism: DispatchMechanism, pub guard: bool, pub agent_model: Option<&'a str>, } @@ -413,25 +411,19 @@ pub fn build_manifest( String::new(), "## How to use this manifest".to_string(), String::new(), - "In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the subagent with a short \"read this file and follow it\" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`.".to_string(), - String::new(), - "**Transcript correlation:** Each task has an `agent_description` field of the form `:[:r]:i-` (the `r` segment appears only in multi-run cells, naming the 1-based run index). When dispatching the subagent via the host's primitive (e.g. Claude Code's Agent tool), pass this string verbatim as the dispatch `description` — do not reconstruct it. The per-run nonce keeps descriptions unique across iterations sharing one session's subagents dir, so the transcript adapter correlates each subagent's persisted transcript back to the right `(eval, condition, run)` slot without collisions.".to_string(), + "In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short \"read this file and follow it\" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`.".to_string(), String::new(), ]; - // Only a Cli-dispatch run emits a CLI recipe section; an in-session run - // (e.g. interactive Claude Code) gets the generic ingest guidance below. - if context.mechanism == DispatchMechanism::Cli - && let Some(lines) = adapter_for(context.harness).cli_manifest_section(CliManifestContext { - guard: context.guard, - agent_model: context.agent_model, - }) - { + if let Some(lines) = adapter_for(context.harness).cli_manifest_section(CliManifestContext { + guard: context.guard, + agent_model: context.agent_model, + }) { header.extend(lines); } header.extend([ - "After all dispatches (Claude Code only):".to_string(), + "After all dispatches:".to_string(), String::new(), - "1. Run `eval-magic ingest` (it auto-resolves the subagents dir from CLAUDE_CODE_SESSION_ID; outside the dispatching session, pass `--session-id ` or `--subagents-dir `) — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the subagent's own `outputs/final-message.md` + the persisted transcript, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ \"total_tokens\": , \"duration_ms\": , \"source\": \"completion-event\" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill.".to_string(), + "1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ \"total_tokens\": , \"duration_ms\": , \"source\": \"completion-event\" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill.".to_string(), "2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark.".to_string(), String::new(), "On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else.".to_string(), diff --git a/src/cli/run/grouping.rs b/src/cli/run/grouping.rs index 105778a..6582a2e 100644 --- a/src/cli/run/grouping.rs +++ b/src/cli/run/grouping.rs @@ -5,9 +5,8 @@ //! different content at the same path were a hard error. Grouping turns that into //! a decision: evals whose fixtures conflict (same env-relative dest from a //! *different* source) are routed into separate groups, and an eval may opt into -//! its own singleton group via [`Isolation::Isolated`]. The realization differs by -//! dispatch mechanism (one env + reset barrier for in-session; one env per -//! `(group, condition)` for CLI), but the grouping decision here is shared. +//! its own singleton group via [`Isolation::Isolated`]. Each group is realized as +//! one env per `(group, condition)`, but the grouping decision here is shared. //! //! The conflict rule is identical to the per-env fixture-claim rule in //! [`super::fixtures`]: same dest + same source is an idempotent share (evals may diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index 37d1251..48c5a90 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -12,9 +12,7 @@ use serde_json::{Value, json}; use crate::adapters::{ adapter_for, config_dir_from_env, detect_plugin_shadows, format_shadow_banner, }; -use crate::core::{ - AvailableSkill, ConditionEntry, ConditionsRecord, DispatchMechanism, Harness, RunContext, -}; +use crate::core::{AvailableSkill, ConditionEntry, ConditionsRecord, Harness, RunContext}; use crate::pipeline::io::now_iso8601; use super::super::dispatch::{ @@ -125,16 +123,13 @@ pub(super) fn write_dispatch( fixtures_by_eval.insert(ev.id.as_str(), dests); } - let mechanism = ctx.run_mode.mechanism(); - // A single group keeps the pre-grouping task shape (no `group`/`eval_root` - // keys); >1 group, or any Cli run (per-(group, condition) envs), tags tasks. + // A single group keeps the `group` key off each task (>1 group tags them); + // `eval_root` (the per-task cwd) is always set, one env per (group, condition). let multi_group = r.groups.len() > 1; let mut tasks = Vec::new(); - // Build tasks CONDITION-outer, GROUP-inner — so the in-session runbook reads - // tasks[] top to bottom as: dispatch each (condition, group) segment, with a - // `reset-batch` between groups and one `switch-condition` between conditions. - // A single group collapses this to the legacy condition-outer order. + // Build tasks CONDITION-outer, GROUP-inner. A single group collapses this to + // the legacy condition-outer order. for (cond_name, cond_skill_path, cond_slug) in [ ( r.cond_a, @@ -148,7 +143,7 @@ pub(super) fn write_dispatch( ), ] { for group in &r.groups { - let env_root = task_env_root(&r.iteration_dir, mechanism, &group.id, cond_name); + let env_root = task_env_root(&r.iteration_dir, &group.id, cond_name); let env_root_str = env_root.to_string_lossy().into_owned(); let staged_path = staged_skill_path_for(&env_root, cond_slug); let available_skills = available_skills_for(&env_root, cond_skill_path, cond_slug); @@ -213,13 +208,9 @@ pub(super) fn write_dispatch( run_index, // Tag the group only when there's more than one (keeps the // single-group task byte-identical). `eval_root` is the - // per-task cwd the Cli recipe `cd`s into; the in-session - // path shares one env, so it stays `None`. + // per-task cwd the CLI recipe `cd`s into. group: multi_group.then_some(group.id.as_str()), - eval_root: match mechanism { - DispatchMechanism::Cli => Some(env_root_str.as_str()), - DispatchMechanism::InSession => None, - }, + eval_root: Some(env_root_str.as_str()), })?); } } @@ -238,7 +229,6 @@ pub(super) fn write_dispatch( &tasks, ManifestContext { harness: ctx.harness, - mechanism: ctx.run_mode.mechanism(), guard: opts.guard, agent_model: opts.agent_model, }, @@ -269,49 +259,42 @@ pub(super) fn write_dispatch( "tasks": tasks, }); // The isolation-batch plan the executing session/human follows: which evals - // share an env, why, and (per condition) the env each batch runs in. Omitted in - // the trivial single-group in-session case so its dispatch.json stays - // byte-identical; emitted whenever the layout is non-trivial (>1 group, or any - // Cli run with per-(group, condition) envs). - if multi_group || mechanism == DispatchMechanism::Cli { - let groups: Vec = r - .groups - .iter() - .map(|g| { - let envs: Vec = [r.cond_a, r.cond_b] - .iter() - .map(|cond| { - json!({ - "condition": cond, - "dir": task_env_root(&r.iteration_dir, mechanism, &g.id, cond) - .to_string_lossy(), - }) + // share an env, why, and (per condition) the env each batch runs in. There is + // one env per (group, condition). + let groups: Vec = r + .groups + .iter() + .map(|g| { + let envs: Vec = [r.cond_a, r.cond_b] + .iter() + .map(|cond| { + json!({ + "condition": cond, + "dir": task_env_root(&r.iteration_dir, &g.id, cond).to_string_lossy(), }) - .collect(); - json!({ - "id": g.id, - "evals": g.eval_ids, - "rationale": g.rationale, - "envs": envs, }) + .collect(); + json!({ + "id": g.id, + "evals": g.eval_ids, + "rationale": g.rationale, + "envs": envs, }) - .collect(); - dispatch_json - .as_object_mut() - .expect("dispatch_json is a JSON object") - .insert("groups".to_string(), Value::Array(groups)); - } + }) + .collect(); + dispatch_json + .as_object_mut() + .expect("dispatch_json is a JSON object") + .insert("groups".to_string(), Value::Array(groups)); write_json(&dispatch_json_path, &dispatch_json)?; - // The followable handoff artifact: a fresh isolated session (interactive) or - // a human (headless) reads RUNBOOK.md to run the loop. It references eval-magic - // meta (dispatch.json, benchmark.json) under `iteration_dir`, so `RunbookContext` - // keeps `iteration_dir`, not the env. Generated, not version controlled. + // The followable handoff artifact: a human reads RUNBOOK.md to run the loop. + // It references eval-magic meta (dispatch.json, benchmark.json) under + // `iteration_dir`, so `RunbookContext` keeps `iteration_dir`, not the env, and + // the human drives from there. Generated, not version controlled. let target_args = command_target_args(ctx); - let group_ids: Vec = r.groups.iter().map(|g| g.id.clone()).collect(); let runbook = build_runbook(&RunbookContext { harness: ctx.harness, - run_mode: ctx.run_mode, skill_name: &ctx.skill_name, iteration: r.iteration, iteration_dir: &r.iteration_dir, @@ -319,19 +302,11 @@ pub(super) fn write_dispatch( cond_a: r.cond_a, cond_b: r.cond_b, num_tasks: tasks.len(), - groups: &group_ids, target_args: &target_args, guard: opts.guard, agent_model: opts.agent_model, }); - // In-session: written into the single `env/` (the isolated session's cwd, = - // `ctx.stage_root`). Cli: there is no single env (one per (group, condition)), - // and the human drives from the iteration dir, so it lands there. - let runbook_path = match mechanism { - DispatchMechanism::InSession => ctx.stage_root.join("RUNBOOK.md"), - DispatchMechanism::Cli => r.iteration_dir.join("RUNBOOK.md"), - }; - fs::write(runbook_path, runbook)?; + fs::write(r.iteration_dir.join("RUNBOOK.md"), runbook)?; Ok(tasks.len()) } @@ -343,12 +318,11 @@ pub(super) fn post_build( opts: &RunOptions, r: &Resolved, ) -> Result<(), RunError> { - // Every env this run staged: one shared `env/` for in-session, one per - // (group, condition) for Cli. Computed once and reused below to arm the guard in - // each env and to point the plugin-shadow preflight at a real staged env. + // Every env this run staged: one per (group, condition). Computed once and + // reused below to arm the guard in each env and to point the plugin-shadow + // preflight at a real staged env. let targets = env_targets(&EnvLayoutInput { iteration_dir: &r.iteration_dir, - mechanism: ctx.run_mode.mechanism(), groups: &r.groups, cond_a: r.cond_a, cond_b: r.cond_b, @@ -384,10 +358,9 @@ pub(super) fn post_build( // Plugin-shadow preflight (Claude Code): a staged skill name also discoverable // from an enabled plugin or the global skills dir contaminates the run. Scan the - // first staged env, not `ctx.stage_root` — under Cli the legacy single `env/` is - // never created, so the project-local `.claude/settings.json` enabledPlugins the - // scan reads must come from a real staged env. In-session's first target *is* - // `env/` (== `ctx.stage_root`), so this is unchanged there. + // first staged env, not `ctx.stage_root` — only the per-`(group, condition)` + // envs are created, so the project-local `.claude/settings.json` enabledPlugins + // the scan reads must come from a real staged env. if ctx.harness == Harness::ClaudeCode { let mut names: Vec<&str> = vec![ctx.skill_name.as_str()]; names.extend(ctx.sibling_skill_names.iter().map(String::as_str)); diff --git a/src/cli/run/orchestrate/envs.rs b/src/cli/run/orchestrate/envs.rs index c6f469f..424c65b 100644 --- a/src/cli/run/orchestrate/envs.rs +++ b/src/cli/run/orchestrate/envs.rs @@ -1,26 +1,20 @@ //! Env-layout planning: turn the computed isolation [`Group`]s into the concrete -//! environment directories a run stages into, which differs by dispatch mechanism. +//! environment directories a run stages into. //! -//! - **InSession** keeps a single `iteration-N/env/` that hosts *both* conditions -//! (the off-condition skill is removed by `switch-condition`) and the first -//! group's fixtures (later groups are swapped in by `reset-batch`). One env, one -//! session — byte-identical to the pre-grouping layout in the single-group case. -//! - **Cli** materializes one `iteration-N/env--/` per -//! `(group, condition)`: each subprocess `cd`s into its own env, which holds only -//! that condition's skill (or none) and that group's fixtures — real physical -//! isolation along both axes. +//! A run materializes one `iteration-N/env--/` per +//! `(group, condition)`: each subprocess `cd`s into its own env, which holds only +//! that condition's skill (or none) and that group's fixtures — real physical +//! isolation along both axes. use std::path::{Path, PathBuf}; -use crate::core::DispatchMechanism; - use super::super::grouping::Group; /// One environment directory to stage for a run. pub(super) struct EnvTarget { pub root: PathBuf, - /// `(condition name, that condition's skill path)` staged into this env. - /// InSession stages both conditions here; Cli stages exactly one. + /// `(condition name, that condition's skill path)` staged into this env — + /// exactly one per env. pub conditions: Vec<(&'static str, Option)>, /// Eval ids whose fixtures populate this env (its group's evals). pub eval_ids: Vec, @@ -29,7 +23,6 @@ pub(super) struct EnvTarget { /// Inputs to [`env_targets`]. pub(super) struct EnvLayoutInput<'a> { pub iteration_dir: &'a Path, - pub mechanism: DispatchMechanism, pub groups: &'a [Group], pub cond_a: &'static str, pub cond_b: &'static str, @@ -37,61 +30,31 @@ pub(super) struct EnvLayoutInput<'a> { pub skill_path_b: Option<&'a str>, } -/// The env dir a `(group, condition)` task runs in: the shared `env/` for -/// InSession, or the per-`(group, condition)` env for Cli. -pub(super) fn task_env_root( - iteration_dir: &Path, - mechanism: DispatchMechanism, - group_id: &str, - condition: &str, -) -> PathBuf { - match mechanism { - DispatchMechanism::InSession => iteration_dir.join("env"), - DispatchMechanism::Cli => iteration_dir.join(format!("env-{group_id}-{condition}")), - } +/// The env dir a `(group, condition)` task runs in. +pub(super) fn task_env_root(iteration_dir: &Path, group_id: &str, condition: &str) -> PathBuf { + iteration_dir.join(format!("env-{group_id}-{condition}")) } -/// Plan the environments to stage. InSession returns a single env hosting both -/// conditions and the *first* group's fixtures; Cli returns one env per -/// `(group, condition)`. +/// Plan the environments to stage: one env per `(group, condition)`. pub(super) fn env_targets(input: &EnvLayoutInput) -> Vec { let conds: [(&'static str, Option); 2] = [ (input.cond_a, input.skill_path_a.map(str::to_owned)), (input.cond_b, input.skill_path_b.map(str::to_owned)), ]; - match input.mechanism { - DispatchMechanism::InSession => { - // One env, staged for the first group; reset-batch swaps later groups in. - let first = input - .groups - .first() - .expect("at least one group is always computed"); - vec![EnvTarget { - root: task_env_root( - input.iteration_dir, - input.mechanism, - &first.id, - input.cond_a, - ), - conditions: conds.to_vec(), - eval_ids: first.eval_ids.clone(), - }] - } - DispatchMechanism::Cli => input - .groups - .iter() - .flat_map(|g| { - conds - .clone() - .into_iter() - .map(move |(cond, skill)| EnvTarget { - root: task_env_root(input.iteration_dir, input.mechanism, &g.id, cond), - conditions: vec![(cond, skill)], - eval_ids: g.eval_ids.clone(), - }) - }) - .collect(), - } + input + .groups + .iter() + .flat_map(|g| { + conds + .clone() + .into_iter() + .map(move |(cond, skill)| EnvTarget { + root: task_env_root(input.iteration_dir, &g.id, cond), + conditions: vec![(cond, skill)], + eval_ids: g.eval_ids.clone(), + }) + }) + .collect() } #[cfg(test)] @@ -114,39 +77,11 @@ mod tests { } #[test] - fn insession_single_env_hosts_both_conditions_and_first_group() { - let iter = Path::new("/w/iteration-1"); - let gs = groups(); - let targets = env_targets(&EnvLayoutInput { - iteration_dir: iter, - mechanism: DispatchMechanism::InSession, - groups: &gs, - cond_a: "with_skill", - cond_b: "without_skill", - skill_path_a: Some("/s/SKILL.md"), - skill_path_b: None, - }); - assert_eq!(targets.len(), 1); - assert_eq!(targets[0].root, Path::new("/w/iteration-1/env")); - assert_eq!( - targets[0] - .conditions - .iter() - .map(|(c, _)| *c) - .collect::>(), - vec!["with_skill", "without_skill"] - ); - // Only the first group's fixtures populate the env up front. - assert_eq!(targets[0].eval_ids, vec!["e1"]); - } - - #[test] - fn cli_one_env_per_group_condition_with_only_that_conditions_skill() { + fn one_env_per_group_condition_with_only_that_conditions_skill() { let iter = Path::new("/w/iteration-1"); let gs = groups(); let targets = env_targets(&EnvLayoutInput { iteration_dir: iter, - mechanism: DispatchMechanism::Cli, groups: &gs, cond_a: "with_skill", cond_b: "without_skill", @@ -181,14 +116,10 @@ mod tests { } #[test] - fn task_env_root_is_bare_env_for_insession_and_suffixed_for_cli() { + fn task_env_root_is_suffixed_by_group_and_condition() { let iter = Path::new("/w/iteration-1"); assert_eq!( - task_env_root(iter, DispatchMechanism::InSession, "g2", "without_skill"), - Path::new("/w/iteration-1/env") - ); - assert_eq!( - task_env_root(iter, DispatchMechanism::Cli, "g2", "without_skill"), + task_env_root(iter, "g2", "without_skill"), Path::new("/w/iteration-1/env-g2-without_skill") ); } diff --git a/src/cli/run/orchestrate/mod.rs b/src/cli/run/orchestrate/mod.rs index 6130f14..53c15f1 100644 --- a/src/cli/run/orchestrate/mod.rs +++ b/src/cli/run/orchestrate/mod.rs @@ -15,10 +15,10 @@ use std::path::PathBuf; use crate::adapters::{CliDispatchContext, adapter_for}; use crate::cli::command_target_args; -use crate::core::{DispatchMechanism, Eval, Mode, RunContext}; +use crate::core::{Eval, Mode, RunContext}; use super::RunError; -use super::util::{insession_isolated_handoff, mode_str}; +use super::util::mode_str; mod build; mod envs; @@ -156,16 +156,10 @@ fn print_next_steps(ctx: &RunContext, opts: &RunOptions, r: &Resolved, num_tasks r.iteration_dir.join("dispatch.json").display() ); - match ctx.run_mode.mechanism() { - DispatchMechanism::InSession => println!( - "Runbook: {} — start a fresh session in env/ and \"Read and follow RUNBOOK.md\".", - ctx.stage_root.join("RUNBOOK.md").display() - ), - DispatchMechanism::Cli => println!( - "Runbook: {} — a human-followed copy of the steps below.", - r.iteration_dir.join("RUNBOOK.md").display() - ), - } + println!( + "Runbook: {} — a human-followed copy of the steps below.", + r.iteration_dir.join("RUNBOOK.md").display() + ); let run_counts: Vec = r .selected_evals .iter() @@ -197,23 +191,14 @@ fn print_next_steps(ctx: &RunContext, opts: &RunOptions, r: &Resolved, num_tasks return; } let target_args = command_target_args(ctx); - match ctx.run_mode.mechanism() { - // In-session subagent dispatch (Claude Code's Task tool today). The env is - // built before the isolated session starts, so the summary just hands off: - // cd into env/, start a fresh session, "Read and follow RUNBOOK.md" — which - // carries the full dispatch → switch-condition → ingest → finalize loop. - DispatchMechanism::InSession => { - println!("\nNext: {}", insession_isolated_handoff(&ctx.stage_root)) - } - // One-shot CLI dispatch; the exact command is harness-specific. - DispatchMechanism::Cli => println!( - "{}", - adapter_for(ctx.harness).cli_next_steps(CliDispatchContext { - guard: opts.guard, - target_args: &target_args, - iteration, - agent_model: opts.agent_model, - }) - ), - } + // One-shot CLI dispatch; the exact command is harness-specific. + println!( + "{}", + adapter_for(ctx.harness).cli_next_steps(CliDispatchContext { + guard: opts.guard, + target_args: &target_args, + iteration, + agent_model: opts.agent_model, + }) + ); } diff --git a/src/cli/run/orchestrate/stage.rs b/src/cli/run/orchestrate/stage.rs index ce5014d..d6848e7 100644 --- a/src/cli/run/orchestrate/stage.rs +++ b/src/cli/run/orchestrate/stage.rs @@ -70,12 +70,10 @@ pub(super) fn stage_conditions( )); } - // The environments to stage: one shared `env/` for in-session (hosting both - // conditions + the first group's fixtures), or one per (group, condition) for - // Cli (each with only its condition's skill + its group's fixtures). + // The environments to stage: one per (group, condition), each with only its + // condition's skill + its group's fixtures. let targets = env_targets(&EnvLayoutInput { iteration_dir: &r.iteration_dir, - mechanism: ctx.run_mode.mechanism(), groups: &r.groups, cond_a: r.cond_a, cond_b: r.cond_b, @@ -89,7 +87,7 @@ pub(super) fn stage_conditions( for target in &targets { // Disarm a prior run's guard before re-staging, so a crashed run can't leave // the write-blocking hook armed across runs. Created unconditionally — even - // under --no-stage, fixtures (and the in-session RUNBOOK) still land here. + // under --no-stage, each env's fixtures still land here. teardown_guard(&target.root); fs::create_dir_all(&target.root)?; diff --git a/src/cli/run/runbook.rs b/src/cli/run/runbook.rs index ac0b921..d0265c6 100644 --- a/src/cli/run/runbook.rs +++ b/src/cli/run/runbook.rs @@ -2,31 +2,22 @@ //! iteration directory during `run`. //! //! The runbook turns the prep session's "what to do next" guidance into a file -//! a *fresh, isolated* session (or a human at a terminal) can read end-to-end: -//! "Read and follow RUNBOOK.md". Which template is used is keyed on the run mode's -//! [`DispatchMechanism`](crate::core::DispatchMechanism), not the harness: +//! a human at a terminal can read end-to-end: "Read and follow RUNBOOK.md". Every +//! run uses the shared [`HEADLESS_RUNBOOK_TEMPLATE`], whose harness-specific +//! dispatch + judge recipes come from the adapter's CLI generators. //! -//! - `InSession` (interactive) → the harness's interactive, agent-followed template. -//! - `Cli` (hybrid / headless) → the shared headless, human-followed template — -//! including Claude Code under `--run-mode hybrid`. -//! -//! The per-mode prose skeletons live in `profiles/` (checked in, loaded via -//! [`HarnessAdapter::runbook_template`](crate::adapters::HarnessAdapter::runbook_template)) -//! and carry `{{TOKEN}}` placeholders the renderer fills with run-specific values. -//! The generated `RUNBOOK.md` itself is a workspace artifact and is not version -//! controlled. +//! The prose skeleton lives in `profiles/` (checked in) and carries `{{TOKEN}}` +//! placeholders the renderer fills with run-specific values. The generated +//! `RUNBOOK.md` itself is a workspace artifact and is not version controlled. use std::path::Path; use crate::adapters::{ CliDispatchContext, CliJudgeContext, HEADLESS_RUNBOOK_TEMPLATE, adapter_for, }; -use crate::core::{DispatchMechanism, Harness, Mode, RunMode}; +use crate::core::{Harness, Mode}; -use super::util::{ - harness_label, insession_dispatch_batch, insession_dispatch_segment, insession_ingest_command, - insession_reset_batch_command, insession_switch_command, mode_str, -}; +use super::util::{harness_label, mode_str}; /// Run-specific values the renderer substitutes into a runbook template. Built by /// the orchestrator from the resolved run; kept as primitives so the renderer is @@ -34,7 +25,6 @@ use super::util::{ /// unit-testable on its own. pub(crate) struct RunbookContext<'a> { pub harness: Harness, - pub run_mode: RunMode, pub skill_name: &'a str, pub iteration: u32, pub iteration_dir: &'a Path, @@ -42,9 +32,6 @@ pub(crate) struct RunbookContext<'a> { pub cond_a: &'a str, pub cond_b: &'a str, pub num_tasks: usize, - /// Isolation-group ids in order. One entry → the byte-identical single-batch - /// dispatch; more → per-group batches with `reset-batch` barriers (in-session). - pub groups: &'a [String], /// The self-sufficient `--skill-dir … --skill …` selector (leading space), /// from [`command_target_args`](crate::cli::command_target_args). pub target_args: &'a str, @@ -52,50 +39,14 @@ pub(crate) struct RunbookContext<'a> { pub agent_model: Option<&'a str>, } -/// The per-condition dispatch block for the interactive runbook. A single group -/// renders the legacy single-batch instruction (byte-identical to the pre-grouping -/// runbook). Multiple groups render each group's batch with a `reset-batch` barrier -/// between them; `first_condition` suppresses the reset before the very first group -/// (condition A starts from the env already staged with group 1, while condition B -/// must restore group 1 after A's last group mutated the env). -fn insession_dispatch_block( - condition: &str, - groups: &[String], - target_args: &str, - iteration: u32, - first_condition: bool, -) -> String { - if groups.len() <= 1 { - return insession_dispatch_batch(condition); - } - let mut parts: Vec = Vec::new(); - for (i, group) in groups.iter().enumerate() { - if !(first_condition && i == 0) { - parts.push(format!( - "Reset the env to group `{group}` (wait for the previous batch to finish first):\n\n```\n{}\n```", - insession_reset_batch_command(target_args, iteration, group) - )); - } - parts.push(format!( - "Dispatch group `{group}`: {}", - insession_dispatch_segment(condition, group) - )); - } - parts.join("\n\n") -} - -/// Render `RUNBOOK.md` for a run: pick the harness's template (interactive vs. -/// headless) and fill its `{{TOKEN}}` placeholders with run-specific values. +/// Render `RUNBOOK.md` for a run: fill the shared headless template's +/// `{{TOKEN}}` placeholders with run-specific values. The harness-specific +/// dispatch + judge recipes come from the adapter's CLI generators, so the +/// runbook stays in lockstep with `dispatch-manifest.md` and the printed next +/// steps; pipeline commands carry `--harness`. pub(crate) fn build_runbook(ctx: &RunbookContext) -> String { let adapter = adapter_for(ctx.harness); - // The runbook template is mechanism-keyed, not harness-keyed: an in-session - // run uses the harness's interactive (agent-followed) template; every Cli run - // uses the shared headless (human-followed) one — including Claude Code in - // hybrid, whose `runbook_template()` is the interactive variant. - let template = match ctx.run_mode.mechanism() { - DispatchMechanism::InSession => adapter.runbook_template(), - DispatchMechanism::Cli => HEADLESS_RUNBOOK_TEMPLATE, - }; + let template = HEADLESS_RUNBOOK_TEMPLATE; let iteration = ctx.iteration.to_string(); let num_tasks = ctx.num_tasks.to_string(); @@ -122,77 +73,37 @@ pub(crate) fn build_runbook(ctx: &RunbookContext) -> String { ("BENCHMARK_PATH", &benchmark_path), ]; - // Mechanism-specific tokens. Owners outlive the `render` call below. - let (dispatch_cond_a, dispatch_cond_b, switch_cmd, ingest_cmd); - let (dispatch_recipe, judge_recipe, finalize_cmd, teardown_cmd); - match ctx.run_mode.mechanism() { - // Interactive: an agent dispatches in-session subagents one condition batch - // at a time, runs `switch-condition` between them, then runs the rest of the - // loop itself. Built from the same fragments as the post-`run` "Next:" - // message so the two can never drift on the dispatch / switch / ingest text. - DispatchMechanism::InSession => { - dispatch_cond_a = insession_dispatch_block( - ctx.cond_a, - ctx.groups, - ctx.target_args, - ctx.iteration, - true, - ); - dispatch_cond_b = insession_dispatch_block( - ctx.cond_b, - ctx.groups, - ctx.target_args, - ctx.iteration, - false, - ); - switch_cmd = insession_switch_command(ctx.target_args, ctx.iteration, ctx.cond_b); - ingest_cmd = insession_ingest_command(ctx.target_args, ctx.iteration); - finalize_cmd = format!( - "eval-magic finalize{} --iteration {}", - ctx.target_args, ctx.iteration - ); - teardown_cmd = format!("eval-magic teardown{}", ctx.target_args); - vars.push(("DISPATCH_COND_A", &dispatch_cond_a)); - vars.push(("DISPATCH_COND_B", &dispatch_cond_b)); - vars.push(("SWITCH_CMD", &switch_cmd)); - vars.push(("INGEST_CMD", &ingest_cmd)); - vars.push(("FINALIZE_CMD", &finalize_cmd)); - vars.push(("TEARDOWN_CMD", &teardown_cmd)); - } - // Headless: a human pastes commands. The harness-specific dispatch + - // judge recipes come from the adapter's existing CLI generators, so the - // runbook stays in lockstep with `dispatch-manifest.md` and the printed - // next steps; pipeline commands carry `--harness`. - DispatchMechanism::Cli => { - let label = harness_label(ctx.harness); - dispatch_recipe = adapter.cli_next_steps(CliDispatchContext { - guard: ctx.guard, - target_args: ctx.target_args, - iteration: ctx.iteration, - agent_model: ctx.agent_model, - }); - judge_recipe = adapter - .cli_judge_next_steps(CliJudgeContext { - guard: ctx.guard, - iteration_dir: ctx.iteration_dir, - }) - .unwrap_or_else(|| { - "Dispatch each judge task `ingest` listed through the same harness CLI, \ - capturing its transcript output, then finalize." - .to_string() - }); - finalize_cmd = format!( - "eval-magic finalize{} --iteration {} --harness {label}", - ctx.target_args, ctx.iteration - ); - teardown_cmd = format!("eval-magic teardown{} --harness {label}", ctx.target_args); - vars.push(("HARNESS", label)); - vars.push(("DISPATCH_RECIPE", &dispatch_recipe)); - vars.push(("JUDGE_RECIPE", &judge_recipe)); - vars.push(("FINALIZE_CMD", &finalize_cmd)); - vars.push(("TEARDOWN_CMD", &teardown_cmd)); - } - } + // A human pastes commands. The harness-specific dispatch + judge recipes come + // from the adapter's CLI generators, so the runbook stays in lockstep with + // `dispatch-manifest.md` and the printed next steps; pipeline commands carry + // `--harness`. Owners outlive the `render` call below. + let label = harness_label(ctx.harness); + let dispatch_recipe = adapter.cli_next_steps(CliDispatchContext { + guard: ctx.guard, + target_args: ctx.target_args, + iteration: ctx.iteration, + agent_model: ctx.agent_model, + }); + let judge_recipe = adapter + .cli_judge_next_steps(CliJudgeContext { + guard: ctx.guard, + iteration_dir: ctx.iteration_dir, + }) + .unwrap_or_else(|| { + "Dispatch each judge task `ingest` listed through the same harness CLI, \ + capturing its transcript output, then finalize." + .to_string() + }); + let finalize_cmd = format!( + "eval-magic finalize{} --iteration {} --harness {label}", + ctx.target_args, ctx.iteration + ); + let teardown_cmd = format!("eval-magic teardown{} --harness {label}", ctx.target_args); + vars.push(("HARNESS", label)); + vars.push(("DISPATCH_RECIPE", &dispatch_recipe)); + vars.push(("JUDGE_RECIPE", &judge_recipe)); + vars.push(("FINALIZE_CMD", &finalize_cmd)); + vars.push(("TEARDOWN_CMD", &teardown_cmd)); render(template, &vars) } @@ -238,137 +149,11 @@ mod tests { use super::*; use std::path::PathBuf; - fn claude_ctx(dir: &Path) -> RunbookContext<'_> { - RunbookContext { - harness: Harness::ClaudeCode, - run_mode: RunMode::Interactive, - skill_name: "widget-skill", - iteration: 5, - iteration_dir: dir, - mode: Mode::NewSkill, - cond_a: "with_skill", - cond_b: "without_skill", - num_tasks: 4, - groups: &[], - target_args: " --skill-dir /tmp/skills --skill widget-skill", - guard: true, - agent_model: None, - } - } - - #[test] - fn interactive_runbook_carries_run_specifics_and_full_loop() { - let dir = PathBuf::from("/work/.eval-magic/widget-skill/iteration-5"); - let book = build_runbook(&claude_ctx(&dir)); - - // Run-specific identity. - assert!(book.contains("widget-skill"), "names the skill: {book}"); - assert!(book.contains("iteration 5"), "names the iteration: {book}"); - assert!( - book.contains("with_skill") && book.contains("without_skill"), - "names both conditions: {book}" - ); - assert!(book.contains("new-skill"), "names the mode: {book}"); - - // The dispatch step reuses the in-session guidance (agent_description is - // the transcript-linking key). - assert!( - book.contains("agent_description"), - "carries the dispatch-loop guidance: {book}" - ); - - // The per-condition batch loop: each condition dispatched as its own batch, - // with a `switch-condition` barrier (naming the kept condition) between them. - assert!( - book.contains("`condition` is `with_skill`") - && book.contains("`condition` is `without_skill`"), - "dispatches each condition as its own batch: {book}" - ); - assert!( - book.contains( - "eval-magic switch-condition --skill-dir /tmp/skills --skill widget-skill --iteration 5 --condition without_skill" - ), - "carries the switch-condition barrier command: {book}" - ); - - // The full single-session loop: ingest → finalize → teardown, each a - // copy-pasteable command threaded with the target selector + iteration. - assert!( - book.contains( - "eval-magic ingest --skill-dir /tmp/skills --skill widget-skill --iteration 5" - ), - "carries the ingest command: {book}" - ); - assert!( - book.contains( - "eval-magic finalize --skill-dir /tmp/skills --skill widget-skill --iteration 5" - ), - "carries the finalize command: {book}" - ); - assert!( - book.contains("eval-magic teardown --skill-dir /tmp/skills --skill widget-skill"), - "carries the teardown command: {book}" - ); - assert!( - book.contains("benchmark.json"), - "points at the result: {book}" - ); - - // No interactive run is dispatched through a harness CLI — that is the - // headless path. - assert!( - !book.contains("codex exec"), - "interactive runbook is not a CLI-dispatch recipe: {book}" - ); - // Every template token must be filled. - assert!( - !book.contains("{{"), - "no unsubstituted tokens remain: {book}" - ); - } - - #[test] - fn interactive_runbook_with_multiple_groups_carries_reset_batch_barriers() { - let dir = PathBuf::from("/work/.eval-magic/widget-skill/iteration-5"); - let groups = ["g1".to_string(), "g2".to_string()]; - let book = build_runbook(&RunbookContext { - groups: &groups, - ..claude_ctx(&dir) - }); - - // Each group dispatches as its own segment, filtered by group. - assert!( - book.contains("`condition` is `with_skill` and `group` is `g1`") - && book.contains("`condition` is `with_skill` and `group` is `g2`"), - "with_skill dispatches each group separately: {book}" - ); - assert!( - book.contains("`condition` is `without_skill` and `group` is `g1`") - && book.contains("`condition` is `without_skill` and `group` is `g2`"), - "without_skill dispatches each group separately: {book}" - ); - // reset-batch barriers between groups, naming the group to seed. - assert!( - book.contains( - "eval-magic reset-batch --skill-dir /tmp/skills --skill widget-skill --iteration 5 --group g2" - ), - "carries the reset-batch barrier for g2: {book}" - ); - // The switch-condition barrier is still present, once, between conditions. - assert!( - book.contains("eval-magic switch-condition") - && book.contains("--condition without_skill"), - "still carries the switch-condition barrier: {book}" - ); - assert!(!book.contains("{{"), "no unsubstituted tokens: {book}"); - } - #[test] fn headless_runbook_is_human_followed_cli_recipe() { let dir = PathBuf::from("/work/.eval-magic/widget-skill/iteration-2"); let ctx = RunbookContext { harness: Harness::Codex, - run_mode: RunMode::Hybrid, skill_name: "widget-skill", iteration: 2, iteration_dir: &dir, @@ -376,7 +161,6 @@ mod tests { cond_a: "old_skill", cond_b: "new_skill", num_tasks: 6, - groups: &[], target_args: " --skill-dir /tmp/skills --skill widget-skill", guard: false, agent_model: Some("gpt-5-mini"), @@ -391,8 +175,7 @@ mod tests { "names both conditions: {book}" ); - // Human-followed framing (the shared headless template), not the agent - // in-session framing. + // Human-followed framing (the shared headless template). assert!( book.contains("human driving"), "frames the run for a human at a terminal: {book}" diff --git a/src/cli/run/steps.rs b/src/cli/run/steps.rs index 532184c..c213262 100644 --- a/src/cli/run/steps.rs +++ b/src/cli/run/steps.rs @@ -13,7 +13,7 @@ //! parameter; the production runner — which maps each [`StepKind`] to its stage //! handler — lives in [`crate::cli`] alongside those handlers. -use crate::core::{DispatchMechanism, Harness, RunMode}; +use crate::core::{Harness, RunMode}; /// Which post-dispatch stage a [`StepCommand`] runs. The production runner /// matches on this to call the corresponding handler; tests assert on it. @@ -37,13 +37,9 @@ pub struct StepCommand { pub skill: Option, pub iteration: u32, pub harness: Harness, - /// The run mode, re-derived at each stage so the transcript source matches - /// the dispatch mechanism. Round-trips through `CommonArgs` exactly like - /// `harness`, so ingest sub-stages don't silently re-default it. + /// The run mode, re-derived at each stage. Round-trips through `CommonArgs` + /// exactly like `harness`, so ingest sub-stages don't silently re-default it. pub run_mode: RunMode, - /// Only the transcript-reading stages (record-runs, fill-transcripts) carry a - /// subagents dir; the others leave it `None`. - pub subagents_dir: Option, pub workspace_dir: Option, } @@ -55,7 +51,6 @@ pub struct StepParams<'a> { pub iteration: u32, pub harness: Harness, pub run_mode: RunMode, - pub subagents_dir: Option<&'a str>, pub workspace_dir: Option<&'a str>, } @@ -66,20 +61,14 @@ impl Default for StepParams<'_> { skill: None, iteration: 0, harness: Harness::ClaudeCode, - run_mode: RunMode::Interactive, - subagents_dir: None, + run_mode: RunMode::Hybrid, workspace_dir: None, } } } impl StepParams<'_> { - fn step( - &self, - label: &'static str, - kind: StepKind, - subagents_dir: Option, - ) -> StepCommand { + fn step(&self, label: &'static str, kind: StepKind) -> StepCommand { StepCommand { label, kind, @@ -88,34 +77,28 @@ impl StepParams<'_> { iteration: self.iteration, harness: self.harness, run_mode: self.run_mode, - subagents_dir, workspace_dir: self.workspace_dir.map(str::to_string), } } } /// The ingest chain: record-runs → fill-transcripts → detect-stray-writes → -/// grade. Only the first two carry the subagents dir, and only for the -/// in-session dispatch mechanism (a Cli-dispatch harness reads its transcript -/// from each task's `outputs/` dir instead). +/// grade. Each stage reads its transcript from each task's `outputs/` events +/// file. pub fn build_ingest_commands(p: &StepParams) -> Vec { - let transcripts = match p.run_mode.mechanism() { - DispatchMechanism::InSession => p.subagents_dir.map(str::to_string), - DispatchMechanism::Cli => None, - }; vec![ - p.step("record-runs", StepKind::RecordRuns, transcripts.clone()), - p.step("fill-transcripts", StepKind::FillTranscripts, transcripts), - p.step("detect-stray-writes", StepKind::DetectStrayWrites, None), - p.step("grade", StepKind::Grade { finalize: false }, None), + p.step("record-runs", StepKind::RecordRuns), + p.step("fill-transcripts", StepKind::FillTranscripts), + p.step("detect-stray-writes", StepKind::DetectStrayWrites), + p.step("grade", StepKind::Grade { finalize: false }), ] } /// The finalize chain: grade --finalize → aggregate. pub fn build_finalize_commands(p: &StepParams) -> Vec { vec![ - p.step("grade --finalize", StepKind::Grade { finalize: true }, None), - p.step("aggregate", StepKind::Aggregate, None), + p.step("grade --finalize", StepKind::Grade { finalize: true }), + p.step("aggregate", StepKind::Aggregate), ] } @@ -146,7 +129,6 @@ mod tests { skill_dir: Some("/skills"), skill: Some("mr-review"), iteration: 2, - subagents_dir: Some("/subagents"), ..Default::default() } } @@ -178,15 +160,10 @@ mod tests { assert_eq!(s.skill.as_deref(), Some("mr-review")); assert_eq!(s.iteration, 2); } - // The transcript-reading steps get the subagents dir; the others must not. - assert_eq!(steps[0].subagents_dir.as_deref(), Some("/subagents")); - assert_eq!(steps[1].subagents_dir.as_deref(), Some("/subagents")); - assert_eq!(steps[2].subagents_dir, None); - assert_eq!(steps[3].subagents_dir, None); } #[test] - fn ingest_omits_subagents_for_codex() { + fn ingest_threads_harness_through_every_step() { let steps = build_ingest_commands(&StepParams { skill_dir: Some("/skills"), skill: Some("mr-review"), @@ -205,40 +182,6 @@ mod tests { ] ); assert!(steps.iter().all(|s| s.harness == Harness::Codex)); - assert_eq!(steps[0].subagents_dir, None); - assert_eq!(steps[1].subagents_dir, None); - } - - #[test] - fn ingest_omits_subagents_for_claude_hybrid() { - // Claude Code in hybrid mode dispatches via the CLI, so it reads each - // task's events file — not a subagents dir — even though the harness is - // ClaudeCode and a subagents dir was passed. - let steps = build_ingest_commands(&StepParams { - skill_dir: Some("/skills"), - skill: Some("mr-review"), - iteration: 2, - harness: Harness::ClaudeCode, - run_mode: RunMode::Hybrid, - subagents_dir: Some("/subagents"), - ..Default::default() - }); - assert!(steps.iter().all(|s| s.harness == Harness::ClaudeCode)); - assert!(steps.iter().all(|s| s.run_mode == RunMode::Hybrid)); - assert_eq!(steps[0].subagents_dir, None); - assert_eq!(steps[1].subagents_dir, None); - } - - #[test] - fn ingest_keeps_subagents_for_claude_interactive() { - // The default (interactive) Claude path still reads the subagents dir. - let steps = build_ingest_commands(&StepParams { - iteration: 2, - subagents_dir: Some("/subagents"), - ..Default::default() - }); - assert_eq!(steps[0].subagents_dir.as_deref(), Some("/subagents")); - assert_eq!(steps[1].subagents_dir.as_deref(), Some("/subagents")); } #[test] @@ -265,8 +208,7 @@ mod tests { skill: None, iteration: 0, harness: Harness::ClaudeCode, - run_mode: RunMode::Interactive, - subagents_dir: None, + run_mode: RunMode::Hybrid, workspace_dir: None, } } diff --git a/src/cli/run/util.rs b/src/cli/run/util.rs index 92e27ea..39d74c5 100644 --- a/src/cli/run/util.rs +++ b/src/cli/run/util.rs @@ -56,79 +56,6 @@ pub(crate) fn unguarded_notice(no_stage: bool) -> Option { ) } -/// The shared dispatch-instruction body, parameterized on the `tasks[]` filter so -/// the condition-only and condition+group variants stay in lockstep. -fn insession_dispatch_instruction(filter: &str) -> String { - format!( - "iterate the `tasks[]` entries in dispatch.json whose {filter} and \ - dispatch each as a subagent, passing its `agent_description` verbatim as the subagent \ - description (that string is the key that links each transcript back — without it tool \ - calls, tokens, and duration come back empty)." - ) -} - -/// Dispatch instruction for one condition batch: iterate the matching `tasks[]` -/// and dispatch each as a subagent with its `agent_description` verbatim. A building -/// block of the interactive runbook's per-condition steps ([`super::runbook`]). -pub(crate) fn insession_dispatch_batch(condition: &str) -> String { - insession_dispatch_instruction(&format!("`condition` is `{condition}`")) -} - -/// Dispatch instruction for one `(condition, group)` segment — used when a run has -/// more than one isolation group, so each group's batch dispatches separately with -/// a [`insession_reset_batch_command`] barrier between groups ([`super::runbook`]). -pub(crate) fn insession_dispatch_segment(condition: &str, group: &str) -> String { - insession_dispatch_instruction(&format!( - "`condition` is `{condition}` and `group` is `{group}`" - )) -} - -/// The `reset-batch` barrier command between isolation-group batches: wipe the -/// env working tree and re-seed it with `group`'s fixtures before dispatching it. -/// A building block of the interactive runbook ([`super::runbook`]). -pub(crate) fn insession_reset_batch_command( - target_args: &str, - iteration: u32, - group: &str, -) -> String { - format!("eval-magic reset-batch{target_args} --iteration {iteration} --group {group}") -} - -/// The `switch-condition` barrier command between batches: name the condition about -/// to be dispatched (the one to keep). A building block of the interactive runbook -/// ([`super::runbook`]). -pub(crate) fn insession_switch_command(target_args: &str, iteration: u32, keep: &str) -> String { - format!("eval-magic switch-condition{target_args} --iteration {iteration} --condition {keep}") -} - -/// The `ingest` hand-off command + its session-resolution hint. A building block of -/// the interactive runbook ([`super::runbook`]). -pub(crate) fn insession_ingest_command(target_args: &str, iteration: u32) -> String { - format!( - "eval-magic ingest{target_args} --iteration {iteration}\n\ - (ingest auto-resolves the subagents dir from CLAUDE_CODE_SESSION_ID; outside that \ - session, add --session-id or --subagents-dir .)" - ) -} - -/// The post-`run` handoff for the isolated in-session flow: cd into the env, start a -/// *fresh* Claude Code session there, and have it read `RUNBOOK.md` — which carries the -/// full dispatch → switch-condition → ingest → finalize loop. The env (incl. -/// `env/.claude/skills/`) is built before that session starts, so the fresh session is -/// structural, not a watcher workaround; the orchestrator no longer juggles the dispatch -/// loop itself. -pub(crate) fn insession_isolated_handoff(env_dir: &Path) -> String { - format!( - "start the isolated run in a fresh session:\n \ - 1. cd {env}\n \ - 2. start a fresh Claude Code session there (`claude`)\n \ - 3. say: Read and follow RUNBOOK.md\n\ - RUNBOOK.md walks the whole loop (dispatch → switch-condition → ingest → finalize) and \ - writes benchmark.json; resume here to read it.", - env = env_dir.display() - ) -} - /// Resolve the shared, harness-agnostic plan-mode procedure profile injected by /// `--plan-mode`. A compile-time bundled asset, mirroring the schema embedding in /// `validation`. @@ -168,10 +95,9 @@ pub(crate) fn validate_harness_run_options( } /// A per-run nonce (`-<6 hex>`) that namespaces dispatch -/// descriptions so transcripts can't collide across iterations sharing one parent -/// session's subagents dir. With no RNG crate, the low bits of the -/// sub-millisecond clock supply the entropy — enough, since the base36 millis -/// prefix already differs between runs. +/// descriptions so they stay unique across iterations of the same skill. With no +/// RNG crate, the low bits of the sub-millisecond clock supply the entropy — +/// enough, since the base36 millis prefix already differs between runs. pub(crate) fn make_run_nonce() -> String { let now = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -257,16 +183,6 @@ mod tests { assert!(validate_harness_run_options(&opts, &ctx).is_ok()); } - #[test] - fn claude_interactive_allows_guard() { - let (_t, ctx) = ctx_for(Harness::ClaudeCode, RunMode::Interactive); - let opts = RunOptions { - guard: true, - ..Default::default() - }; - assert!(validate_harness_run_options(&opts, &ctx).is_ok()); - } - #[test] fn unguarded_notice_when_no_stage() { let notice = unguarded_notice(true).unwrap(); @@ -285,30 +201,6 @@ mod tests { assert!(unguarded_notice(false).is_none()); } - #[test] - fn isolated_handoff_points_into_env_and_at_the_runbook() { - let env = Path::new("/work/.eval-magic/widget/iteration-3/env"); - let handoff = insession_isolated_handoff(env); - assert!( - handoff.contains("/work/.eval-magic/widget/iteration-3/env"), - "names the env to cd into: {handoff}" - ); - assert!(handoff.contains("cd "), "spells out the cd step: {handoff}"); - assert!( - handoff.contains("Read and follow RUNBOOK.md"), - "hands off to the runbook in a fresh session: {handoff}" - ); - assert!( - handoff.contains("fresh"), - "names the fresh isolated session: {handoff}" - ); - // The handoff replaces the old printed dispatch loop — it must not re-print it. - assert!( - !handoff.contains("one batch at a time"), - "the dispatch loop lives in RUNBOOK.md now, not the summary: {handoff}" - ); - } - #[test] fn plan_mode_profile_is_shared_and_harness_agnostic() { let profile = resolve_plan_mode_profile(); diff --git a/src/core/mod.rs b/src/core/mod.rs index a0d12b8..a78a0d1 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -2,7 +2,7 @@ //! //! - [`types`] — domain types (`Eval`, `RunRecord`, `Assertion`, `GradingResult`, …) //! - [`context`] — `RunContext` detection from parsed flags / environment -//! - [`run_mode`] — dispatch mechanism (in-session vs. one-shot CLI) +//! - [`run_mode`] — run mode (hybrid / headless) and per-harness capabilities //! - [`runtime`] — runtime helpers (git spawning) //! //! The submodules are re-exported flat here so downstream code writes @@ -14,8 +14,6 @@ pub mod runtime; pub mod types; pub use context::{ContextError, DetectInput, Harness, RunContext, detect_run_context}; -pub use run_mode::{ - DispatchMechanism, HarnessRunCapabilities, RunMode, capabilities_for, resolve_run_mode, -}; +pub use run_mode::{HarnessRunCapabilities, RunMode, capabilities_for, resolve_run_mode}; pub use runtime::{GitOutput, run_git}; pub use types::*; diff --git a/src/core/run_mode.rs b/src/core/run_mode.rs index a2457df..f1fe909 100644 --- a/src/core/run_mode.rs +++ b/src/core/run_mode.rs @@ -1,18 +1,12 @@ //! Run mode — *how* an eval is dispatched, independent of *which* harness runs //! it. //! -//! There are two dispatch **mechanisms** in the code today: -//! -//! - [`DispatchMechanism::InSession`] — the runner hands tasks to in-session -//! subagents (Claude Code's Task tool). The reference is Claude Code. -//! - [`DispatchMechanism::Cli`] — each task is dispatched through a one-shot -//! harness CLI subprocess (`codex exec`). The reference is Codex. -//! -//! These two mechanisms underpin the three *user-facing* run modes documented in -//! the README: **fully-interactive** rides on [`InSession`](DispatchMechanism::InSession); -//! **headless** and **hybrid** both ride on [`Cli`](DispatchMechanism::Cli), -//! differing only in whether a human/agent session drives the loop — not in how -//! a single task reaches the harness. +//! Every dispatch now rides a single mechanism: each task is delivered through a +//! one-shot harness CLI subprocess (`claude -p`, `codex exec`). The two +//! *user-facing* run modes documented in the README — **hybrid** and +//! **headless** — share that mechanism and differ only in whether an agent or a +//! human session drives the loop, not in how a single task reaches the harness. +//! (The vocabulary collapse that folds these two into one is tracked separately.) //! //! This is distinct from the comparison [`Mode`](crate::core::Mode) //! (`new-skill` / `revision`), which selects the two conditions being compared, @@ -22,29 +16,15 @@ use serde::{Deserialize, Serialize}; use crate::core::Harness; -/// How a single dispatch is delivered to a harness. The primary code axis for -/// run-mode concerns (next-steps guidance, transcript source). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DispatchMechanism { - /// In-session subagent dispatch (Claude Code's Task tool). - InSession, - /// One-shot harness CLI subprocess dispatch (`codex exec`). - Cli, -} - -/// The user-facing run mode — *who/what drives the loop* plus which dispatch -/// mechanism each task rides on. This is the parity vocabulary documented in the -/// README (§Run modes); it maps down to a [`DispatchMechanism`] via -/// [`RunMode::mechanism`]. `hybrid` and `headless` both ride on -/// [`Cli`](DispatchMechanism::Cli) and differ only in whether a session drives -/// the loop — a distinction we persist (in `conditions.json`) even though it -/// doesn't change how a single task reaches the harness. +/// The user-facing run mode — *who/what drives the loop*. Both modes dispatch +/// each task through the harness CLI; they differ only in whether an agent +/// session (`hybrid`) or a human (`headless`) drives the loop — a distinction we +/// persist (in `conditions.json`) even though it doesn't change how a single task +/// reaches the harness. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, clap::ValueEnum)] #[serde(rename_all = "kebab-case")] #[value(rename_all = "kebab-case")] pub enum RunMode { - /// In-session subagent dispatch (Claude Code's Task tool). - Interactive, /// An agent session orchestrates while each dispatch shells out to the /// harness CLI (`claude -p`, `codex exec`). Hybrid, @@ -54,29 +34,17 @@ pub enum RunMode { } impl RunMode { - /// The dispatch mechanism this run mode rides on. - pub fn mechanism(self) -> DispatchMechanism { - match self { - RunMode::Interactive => DispatchMechanism::InSession, - RunMode::Hybrid | RunMode::Headless => DispatchMechanism::Cli, - } - } - - /// The default run mode for a harness when `--run-mode` is omitted, chosen to - /// preserve today's behavior: Claude Code → interactive, the CLI-dispatch - /// harnesses → hybrid. - pub fn default_for(harness: Harness) -> RunMode { - match harness { - Harness::ClaudeCode => RunMode::Interactive, - Harness::Codex | Harness::OpenCode => RunMode::Hybrid, - } + /// The default run mode for a harness when `--run-mode` is omitted. Every + /// harness defaults to `hybrid`: an agent session drives the loop and each + /// dispatch shells out to the harness CLI. + pub fn default_for(_harness: Harness) -> RunMode { + RunMode::Hybrid } /// The kebab-case identifier (matches the `--run-mode` flag values and the /// serialized form in `conditions.json`). pub fn as_str(self) -> &'static str { match self { - RunMode::Interactive => "interactive", RunMode::Hybrid => "hybrid", RunMode::Headless => "headless", } @@ -89,12 +57,9 @@ impl RunMode { pub fn resolve_run_mode(harness: Harness, requested: Option) -> Result { let mode = requested.unwrap_or_else(|| RunMode::default_for(harness)); let supported: &[RunMode] = match harness { - // Claude Code wires every mode: in-session (interactive) plus both CLI - // modes (hybrid and headless ride the same `claude -p` mechanism). - Harness::ClaudeCode => &[RunMode::Interactive, RunMode::Hybrid, RunMode::Headless], - // Codex dispatches via subprocess, so in-session doesn't translate, but - // both CLI modes do (hybrid is agent-driven, headless human-driven). - Harness::Codex => &[RunMode::Hybrid, RunMode::Headless], + // Claude Code and Codex both wire the CLI mechanism, so both modes apply + // (hybrid is agent-driven, headless human-driven). + Harness::ClaudeCode | Harness::Codex => &[RunMode::Hybrid, RunMode::Headless], // OpenCode's CLI path is only partially wired (no transcript ingest), so // only hybrid is advertised for now. Harness::OpenCode => &[RunMode::Hybrid], @@ -131,7 +96,6 @@ fn harness_label(harness: Harness) -> &'static str { /// sequence starts. Harness-specific behavior still lives behind the adapter. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct HarnessRunCapabilities { - pub mechanism: DispatchMechanism, pub supports_guard: bool, pub supports_bootstrap_with_no_stage: bool, pub supports_stage_name_with_no_stage: bool, @@ -141,19 +105,16 @@ pub struct HarnessRunCapabilities { pub fn capabilities_for(harness: Harness) -> HarnessRunCapabilities { match harness { Harness::ClaudeCode => HarnessRunCapabilities { - mechanism: DispatchMechanism::InSession, supports_guard: true, supports_bootstrap_with_no_stage: true, supports_stage_name_with_no_stage: true, }, Harness::Codex => HarnessRunCapabilities { - mechanism: DispatchMechanism::Cli, supports_guard: true, supports_bootstrap_with_no_stage: false, supports_stage_name_with_no_stage: false, }, Harness::OpenCode => HarnessRunCapabilities { - mechanism: DispatchMechanism::Cli, supports_guard: false, supports_bootstrap_with_no_stage: true, supports_stage_name_with_no_stage: true, @@ -168,40 +129,24 @@ mod tests { #[test] fn capabilities_capture_run_option_support_by_harness() { let claude = capabilities_for(Harness::ClaudeCode); - assert_eq!(claude.mechanism, DispatchMechanism::InSession); assert!(claude.supports_guard); assert!(claude.supports_bootstrap_with_no_stage); assert!(claude.supports_stage_name_with_no_stage); let codex = capabilities_for(Harness::Codex); - assert_eq!(codex.mechanism, DispatchMechanism::Cli); assert!(codex.supports_guard); assert!(!codex.supports_bootstrap_with_no_stage); assert!(!codex.supports_stage_name_with_no_stage); let opencode = capabilities_for(Harness::OpenCode); - assert_eq!(opencode.mechanism, DispatchMechanism::Cli); assert!(!opencode.supports_guard); assert!(opencode.supports_bootstrap_with_no_stage); assert!(opencode.supports_stage_name_with_no_stage); } #[test] - fn run_mode_mechanism_maps_each_mode() { - assert_eq!( - RunMode::Interactive.mechanism(), - DispatchMechanism::InSession - ); - assert_eq!(RunMode::Hybrid.mechanism(), DispatchMechanism::Cli); - assert_eq!(RunMode::Headless.mechanism(), DispatchMechanism::Cli); - } - - #[test] - fn run_mode_default_per_harness_preserves_today() { - assert_eq!( - RunMode::default_for(Harness::ClaudeCode), - RunMode::Interactive - ); + fn run_mode_defaults_to_hybrid_for_every_harness() { + assert_eq!(RunMode::default_for(Harness::ClaudeCode), RunMode::Hybrid); assert_eq!(RunMode::default_for(Harness::Codex), RunMode::Hybrid); assert_eq!(RunMode::default_for(Harness::OpenCode), RunMode::Hybrid); } @@ -210,7 +155,7 @@ mod tests { fn resolve_run_mode_defaults_when_unspecified() { assert_eq!( resolve_run_mode(Harness::ClaudeCode, None).unwrap(), - RunMode::Interactive + RunMode::Hybrid ); assert_eq!( resolve_run_mode(Harness::Codex, None).unwrap(), @@ -227,11 +172,10 @@ mod tests { } #[test] - fn resolve_run_mode_rejects_interactive_for_cli_harnesses() { - let err = resolve_run_mode(Harness::Codex, Some(RunMode::Interactive)).unwrap_err(); - assert!(err.contains("interactive"), "message was: {err}"); - assert!(err.contains("codex"), "message was: {err}"); - assert!(resolve_run_mode(Harness::OpenCode, Some(RunMode::Interactive)).is_err()); + fn resolve_run_mode_rejects_headless_for_opencode() { + let err = resolve_run_mode(Harness::OpenCode, Some(RunMode::Headless)).unwrap_err(); + assert!(err.contains("headless"), "message was: {err}"); + assert!(err.contains("opencode"), "message was: {err}"); } #[test] diff --git a/src/core/types.rs b/src/core/types.rs index dddd9d7..bbd300b 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -128,8 +128,8 @@ pub struct ConditionsRecord { /// `None` on older artifacts written before run-mode selection existed. #[serde(skip_serializing_if = "Option::is_none")] pub run_mode: Option, - /// Per-run nonce; namespaces dispatch descriptions so transcripts can't - /// collide across iterations sharing one parent session's subagents dir. + /// Per-run nonce; namespaces dispatch descriptions so they stay unique across + /// iterations of the same skill. #[serde(skip_serializing_if = "Option::is_none")] pub run_nonce: Option, /// The `--runs` value the iteration was built with (provenance; per-eval diff --git a/src/pipeline/fill_transcripts.rs b/src/pipeline/fill_transcripts.rs index 06bd0b6..aed05cb 100644 --- a/src/pipeline/fill_transcripts.rs +++ b/src/pipeline/fill_transcripts.rs @@ -2,9 +2,10 @@ //! //! Walks the iteration's `eval-*` //! directories and, for each `(eval, condition)` `run.json`, populates -//! `tool_invocations` from the persisted transcript (Claude Code subagent JSONL -//! resolved by the task's `agent_description`, or Codex `codex-events.jsonl`). -//! Records that already carry invocations are skipped unless `overwrite`. +//! `tool_invocations` from the events file the harness CLI wrote under the task's +//! `outputs_dir` (e.g. Codex's `codex-events.jsonl`, Claude Code's +//! `claude-events.jsonl`). Records that already carry invocations are skipped +//! unless `overwrite`. use std::collections::HashMap; use std::fs; @@ -12,8 +13,8 @@ use std::path::Path; use serde::Deserialize; -use crate::adapters::{adapter_for, find_by_description}; -use crate::core::{ConditionsRecord, DispatchMechanism, Harness, RunRecord, ToolInvocation}; +use crate::adapters::adapter_for; +use crate::core::{ConditionsRecord, Harness, RunRecord, ToolInvocation}; use crate::pipeline::error::PipelineError; use crate::pipeline::io::write_json; use crate::pipeline::slots::{run_key, run_slots}; @@ -40,45 +41,14 @@ struct DispatchRef { #[serde(default)] run_index: Option, #[serde(default)] - agent_description: Option, - #[serde(default)] outputs_dir: Option, } -/// The canonical dispatch description for an `(eval, condition, run)` run. -/// -/// The runner writes a unique `agent_description` per task into `dispatch.json` -/// (namespaced with the iteration + run nonce); reading it back binds each run to -/// the exact agent that produced it. Falls back to the -/// `:[:r]` reconstruction when `dispatch.json` is absent, -/// malformed, or missing the task (hand-authored/operator runs). -pub fn resolve_agent_description( - iteration_dir: &Path, - eval_id: &str, - condition: &str, - run_index: Option, -) -> String { - let dispatch_path = iteration_dir.join("dispatch.json"); - if let Ok(raw) = fs::read_to_string(&dispatch_path) - && let Ok(env) = serde_json::from_str::(&raw) - && let Some(tasks) = env.tasks - && let Some(task) = tasks - .iter() - .find(|t| t.eval_id == eval_id && t.condition == condition && t.run_index == run_index) - && let Some(desc) = &task.agent_description - { - return desc.clone(); - } - run_key(eval_id, condition, run_index) -} - /// Populate `tool_invocations` for every `run.json` under `iteration_dir`. See /// the module docs for the transcript sources and overwrite semantics. pub fn fill_transcripts( iteration_dir: &Path, harness: Harness, - mechanism: DispatchMechanism, - subagents_dir: Option<&Path>, overwrite: bool, ) -> Result { let conditions_path = iteration_dir.join("conditions.json"); @@ -131,18 +101,8 @@ pub fn fill_transcripts( .cloned() .unwrap_or_else(|| slot.dir.join("outputs").to_string_lossy().into_owned()); - // Resolve the in-session description lazily — only the InSession - // branch needs it, so a Cli run skips the dispatch.json re-read. - let description = (mechanism == DispatchMechanism::InSession).then(|| { - resolve_agent_description(iteration_dir, eval_id, cond, slot.run_index) - }); - let Some(invocations) = invocations_for_run( - harness, - mechanism, - subagents_dir, - description.as_deref(), - Path::new(&outputs_dir), - ) else { + let Some(invocations) = invocations_for_run(harness, Path::new(&outputs_dir)) + else { result.missing += 1; continue; }; @@ -174,34 +134,15 @@ fn outputs_dirs_by_key(iteration_dir: &Path) -> HashMap { out } -/// Parse the invocations for one run, keyed on the dispatch mechanism: a -/// `Cli`-mechanism harness reads the events file its CLI wrote under -/// `outputs_dir` (e.g. Codex's `codex-events.jsonl`, Claude Code hybrid's -/// `claude-events.jsonl`); an `InSession` harness reads the subagent transcript -/// matched by `description` (resolved by the caller). -fn invocations_for_run( - harness: Harness, - mechanism: DispatchMechanism, - subagents_dir: Option<&Path>, - description: Option<&str>, - outputs_dir: &Path, -) -> Option> { - match mechanism { - DispatchMechanism::Cli => { - let events_path = outputs_dir.join(adapter_for(harness).cli_events_filename()?); - if !events_path.exists() { - return None; - } - adapter_for(harness).parse_cli_events(&events_path).ok() - } - DispatchMechanism::InSession => { - let subagent = - find_by_description(subagents_dir.unwrap_or_else(|| Path::new("")), description?)?; - adapter_for(harness) - .parse_transcript(&subagent.jsonl_path) - .ok() - } +/// Parse the invocations for one run: read the events file the harness CLI wrote +/// under `outputs_dir` (e.g. Codex's `codex-events.jsonl`, Claude Code's +/// `claude-events.jsonl`). Returns `None` when no events file is found. +fn invocations_for_run(harness: Harness, outputs_dir: &Path) -> Option> { + let events_path = outputs_dir.join(adapter_for(harness).cli_events_filename()?); + if !events_path.exists() { + return None; } + adapter_for(harness).parse_cli_events(&events_path).ok() } #[cfg(test)] @@ -244,66 +185,6 @@ mod tests { fs::write(path, serde_json::to_string_pretty(&record).unwrap()).unwrap(); } - // --- resolveAgentDescription --- - - #[test] - fn returns_the_namespaced_agent_description_from_dispatch() { - let root = TempDir::new().unwrap(); - let dir = root.path().join("iter-canonical"); - write_dispatch( - &dir, - json!([ - {"eval_id": "crash", "condition": "with_skill", "agent_description": "crash:with_skill:i3-abc123"}, - {"eval_id": "crash", "condition": "without_skill", "agent_description": "crash:without_skill:i3-abc123"} - ]), - ); - assert_eq!( - resolve_agent_description(&dir, "crash", "with_skill", None), - "crash:with_skill:i3-abc123" - ); - assert_eq!( - resolve_agent_description(&dir, "crash", "without_skill", None), - "crash:without_skill:i3-abc123" - ); - } - - #[test] - fn falls_back_to_legacy_reconstruction_when_dispatch_absent() { - let root = TempDir::new().unwrap(); - let dir = root.path().join("iter-no-dispatch"); - fs::create_dir_all(&dir).unwrap(); - assert_eq!( - resolve_agent_description(&dir, "crash", "with_skill", None), - "crash:with_skill" - ); - } - - #[test] - fn falls_back_when_task_missing_from_dispatch() { - let root = TempDir::new().unwrap(); - let dir = root.path().join("iter-partial"); - write_dispatch( - &dir, - json!([{"eval_id": "other", "condition": "with_skill", "agent_description": "other:with_skill:i1-x"}]), - ); - assert_eq!( - resolve_agent_description(&dir, "crash", "with_skill", None), - "crash:with_skill" - ); - } - - #[test] - fn falls_back_when_dispatch_malformed() { - let root = TempDir::new().unwrap(); - let dir = root.path().join("iter-malformed"); - fs::create_dir_all(&dir).unwrap(); - fs::write(dir.join("dispatch.json"), "{ not valid json").unwrap(); - assert_eq!( - resolve_agent_description(&dir, "crash", "with_skill", None), - "crash:with_skill" - ); - } - // --- fillTranscripts --- #[test] @@ -342,14 +223,7 @@ mod tests { ) .unwrap(); - let result = fill_transcripts( - &iteration_dir, - Harness::ClaudeCode, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = fill_transcripts(&iteration_dir, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.filled, 1); assert_eq!(result.missing, 0); @@ -393,14 +267,7 @@ mod tests { ) .unwrap(); - let result = fill_transcripts( - &iteration_dir, - Harness::Codex, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = fill_transcripts(&iteration_dir, Harness::Codex, false).unwrap(); assert_eq!(result.filled, 1); assert_eq!(result.missing, 0); @@ -443,14 +310,7 @@ mod tests { .unwrap(); } - let result = fill_transcripts( - &iteration_dir, - Harness::Codex, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = fill_transcripts(&iteration_dir, Harness::Codex, false).unwrap(); assert_eq!(result.filled, 2); assert_eq!(result.missing, 0); diff --git a/src/pipeline/mod.rs b/src/pipeline/mod.rs index 82c889d..1417b3c 100644 --- a/src/pipeline/mod.rs +++ b/src/pipeline/mod.rs @@ -21,7 +21,7 @@ pub use detect_stray_writes::{ detect_stray_writes_report, }; pub use error::PipelineError; -pub use fill_transcripts::{FillTranscriptsResult, fill_transcripts, resolve_agent_description}; +pub use fill_transcripts::{FillTranscriptsResult, fill_transcripts}; pub use grade::{GradeContext, emit_judge_tasks, finalize}; pub use record_runs::{RecordRunsResult, record_runs}; pub use slots::{RunSlot, run_slots}; diff --git a/src/pipeline/record_runs.rs b/src/pipeline/record_runs.rs index c97034b..b0cc3e2 100644 --- a/src/pipeline/record_runs.rs +++ b/src/pipeline/record_runs.rs @@ -5,8 +5,8 @@ //! from sources already on disk: carry-over fields from the dispatch task, the //! `final_message` (from `/final-message.md`, falling back to the //! transcript's last assistant text), and `tool_invocations`/tokens/duration from -//! the persisted transcript (Claude Code subagent JSONL or Codex -//! `codex-events.jsonl`). +//! each task's events file (`outputs/-events.jsonl` — Claude Code's +//! `claude -p` stream-json or Codex's `codex-events.jsonl`). //! //! Existing records always win: an agent/operator-written `run.json` is skipped //! without `overwrite`, and `timing.json` is backfill-only — completion-event @@ -18,8 +18,8 @@ use std::path::Path; use serde::Deserialize; -use crate::adapters::{TranscriptSummary, adapter_for, find_by_description}; -use crate::core::{DispatchMechanism, Harness, RunRecord, TimingRecord, TimingSource}; +use crate::adapters::{TranscriptSummary, adapter_for}; +use crate::core::{Harness, RunRecord, TimingRecord, TimingSource}; use crate::pipeline::error::PipelineError; use crate::pipeline::io::write_json; use crate::validation::{SchemaName, validate_against_schema}; @@ -44,7 +44,6 @@ struct DispatchTask { outputs_dir: String, run_record_path: String, timing_path: String, - agent_description: String, #[serde(default)] dispatch_prompt_path: String, } @@ -63,14 +62,9 @@ impl RecordRunsResult { /// A loud, actionable warning when runs were recorded from `final-message.md` /// but their transcripts didn't link — leaving `tool_invocations`/tokens/ /// duration empty so `transcript_check` assertions silently grade - /// unverifiable. `None` when every run matched its transcript. The hint is - /// tailored to how the harness correlates transcripts (description match vs. - /// the Codex events file). - pub fn transcript_warning( - &self, - harness: Harness, - mechanism: DispatchMechanism, - ) -> Option { + /// unverifiable. `None` when every run matched its transcript. The hint names + /// the per-task events file the harness CLI was expected to write. + pub fn transcript_warning(&self, harness: Harness) -> Option { if self.missing_transcript == 0 { return None; } @@ -82,23 +76,10 @@ impl RecordRunsResult { } else { format!("⚠ {n} run{plural} missing a transcript") }; - // The cause is keyed on the dispatch mechanism, not the harness: a - // Cli-dispatch run (Codex, or Claude Code in hybrid/headless) misses the - // per-task events file; an in-session run misses the subagent transcript. - let cause = match mechanism { - DispatchMechanism::Cli => { - let file = adapter_for(harness) - .cli_events_filename() - .unwrap_or("the events file"); - format!("expected `outputs/{file}` was not found") - } - DispatchMechanism::InSession => { - "did you pass each task's `agent_description` verbatim as the subagent \ - description? If so, confirm `--subagents-dir` points at the parent session's \ - subagents dir" - .to_string() - } - }; + let file = adapter_for(harness) + .cli_events_filename() + .unwrap_or("the events file"); + let cause = format!("expected `outputs/{file}` was not found"); Some(format!( "{lead} — {cause}; tool_invocations/tokens/duration are empty, so transcript_check \ assertions will grade unverifiable." @@ -130,8 +111,6 @@ impl RecordRunsResult { pub fn record_runs( iteration_dir: &Path, harness: Harness, - mechanism: DispatchMechanism, - subagents_dir: Option<&Path>, overwrite: bool, ) -> Result { let dispatch_path = iteration_dir.join("dispatch.json"); @@ -148,7 +127,7 @@ pub fn record_runs( let mut result = RecordRunsResult::default(); for task in &tasks { - let summary = transcript_summary_for_task(harness, mechanism, subagents_dir, task); + let summary = transcript_summary_for_task(harness, task); if summary.is_none() { result.missing_transcript += 1; } @@ -230,7 +209,7 @@ pub fn record_runs( /// call returned the prompt's content (its distinctive first-line `sentinel`). /// /// A run that never references the prompt path is NOT flagged — absence is not -/// proof of failure (an in-session subagent can receive the prompt another way), +/// proof of failure (the agent can receive the prompt another way), /// and requiring positive evidence keeps the check free of false positives. /// Returns `false` when `sentinel` is empty (the prompt file was missing or /// unreadable, so the read cannot be judged). @@ -277,38 +256,18 @@ fn prompt_sentinel(prompt_path: &str) -> String { .unwrap_or_default() } -/// Resolve a task's transcript summary, keyed on the dispatch mechanism: a -/// `Cli`-mechanism harness reads the events file its CLI wrote under the task's -/// outputs dir (e.g. Codex's `codex-events.jsonl`); an `InSession` harness reads -/// the subagent transcript matched by the task's `agent_description`. Returns -/// `None` when no transcript is found. -fn transcript_summary_for_task( - harness: Harness, - mechanism: DispatchMechanism, - subagents_dir: Option<&Path>, - task: &DispatchTask, -) -> Option { - match mechanism { - DispatchMechanism::Cli => { - let events_path = - Path::new(&task.outputs_dir).join(adapter_for(harness).cli_events_filename()?); - if !events_path.exists() { - return None; - } - adapter_for(harness) - .parse_cli_events_full(&events_path) - .ok() - } - DispatchMechanism::InSession => { - let subagent = find_by_description( - subagents_dir.unwrap_or_else(|| Path::new("")), - &task.agent_description, - )?; - adapter_for(harness) - .parse_transcript_full(&subagent.jsonl_path) - .ok() - } - } +/// Resolve a task's transcript summary: read the events file the harness CLI +/// wrote under the task's outputs dir (e.g. Codex's `codex-events.jsonl`, Claude +/// Code's `claude-events.jsonl`). Returns `None` when no transcript is found. +fn transcript_summary_for_task(harness: Harness, task: &DispatchTask) -> Option { + let events_path = + Path::new(&task.outputs_dir).join(adapter_for(harness).cli_events_filename()?); + if !events_path.exists() { + return None; + } + adapter_for(harness) + .parse_cli_events_full(&events_path) + .ok() } #[cfg(test)] @@ -318,12 +277,6 @@ mod tests { use std::path::PathBuf; use tempfile::TempDir; - /// Token math for `transcript_lines`: msg_1 (100+20+30+50) + msg_2 - /// (200+40+0+60) = 500. - const TRANSCRIPT_TOKENS: i64 = 500; - /// 10:00:00.000 → 10:01:00.000. - const TRANSCRIPT_DURATION_MS: i64 = 60_000; - fn jsonl(lines: &[Value]) -> String { let body = lines .iter() @@ -333,33 +286,6 @@ mod tests { format!("{body}\n") } - /// A minimal transcript with usage, timestamps, one tool call, and final text. - fn transcript_lines(final_text: &str) -> Vec { - vec![ - json!({"type": "user", "timestamp": "2026-06-04T10:00:00.000Z", "message": {"role": "user", "content": "go"}}), - json!({"type": "assistant", "timestamp": "2026-06-04T10:00:10.000Z", "message": { - "id": "msg_1", "role": "assistant", - "usage": {"input_tokens": 100, "output_tokens": 20, "cache_creation_input_tokens": 30, "cache_read_input_tokens": 50}, - "content": [{"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}}] - }}), - json!({"type": "user", "timestamp": "2026-06-04T10:00:12.000Z", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "ok"}]}}), - json!({"type": "assistant", "timestamp": "2026-06-04T10:01:00.000Z", "message": { - "id": "msg_2", "role": "assistant", - "usage": {"input_tokens": 200, "output_tokens": 40, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 60}, - "content": [{"type": "text", "text": final_text}] - }}), - ] - } - - fn write_subagent(subagents_dir: &Path, name: &str, description: &str, lines: &[Value]) { - fs::write( - subagents_dir.join(format!("{name}.meta.json")), - json!({"agentType": "general-purpose", "description": description}).to_string(), - ) - .unwrap(); - fs::write(subagents_dir.join(format!("{name}.jsonl")), jsonl(lines)).unwrap(); - } - fn write_codex_events(outputs_dir: &Path, final_text: &str) { let lines = vec![ json!({"type": "thread.started", "timestamp": "2026-06-04T10:00:00.000Z"}), @@ -438,14 +364,7 @@ mod tests { "I could not read the prompt file.", ); - let result = record_runs( - iter, - Harness::ClaudeCode, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = record_runs(iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.skipped_prompt_unread, 1); assert_eq!(result.recorded, 0); @@ -482,14 +401,7 @@ mod tests { "Done.", ); - let result = record_runs( - iter, - Harness::ClaudeCode, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = record_runs(iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.recorded, 1); assert_eq!(result.skipped_prompt_unread, 0); @@ -591,20 +503,18 @@ mod tests { .exists() } - /// `(iteration_dir, subagents_dir)` under a fresh temp root. - fn dirs(root: &TempDir) -> (PathBuf, PathBuf) { + /// The iteration dir under a fresh temp root. + fn dirs(root: &TempDir) -> PathBuf { let iteration_dir = root.path().join("iter"); - let subagents_dir = root.path().join("sub"); fs::create_dir_all(&iteration_dir).unwrap(); - fs::create_dir_all(&subagents_dir).unwrap(); - (iteration_dir, subagents_dir) + iteration_dir } #[test] fn assembles_run_and_timing_for_every_task_from_disk() { let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); - write_iteration( + let iter = dirs(&root); + let paths = write_iteration( &iter, &[ FixtureTask { @@ -619,27 +529,10 @@ mod tests { }, ], ); - write_subagent( - &sub, - "agent-a", - "crash:with_skill:i1-nonce1", - &transcript_lines("unused"), - ); - write_subagent( - &sub, - "agent-b", - "crash:without_skill:i1-nonce1", - &transcript_lines("unused"), - ); + write_claude_events(&paths[0].outputs_dir, "unused"); + write_claude_events(&paths[1].outputs_dir, "unused"); - let result = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap(); + let result = record_runs(&iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.recorded, 2); assert_eq!(result.missing_transcript, 0); @@ -661,15 +554,15 @@ mod tests { ); let timing = read_timing_value(&iter, "crash", "with_skill"); - assert_eq!(timing["total_tokens"], json!(TRANSCRIPT_TOKENS)); - assert_eq!(timing["duration_ms"], json!(TRANSCRIPT_DURATION_MS)); + assert_eq!(timing["total_tokens"], json!(125)); + assert_eq!(timing["duration_ms"], json!(30_000)); assert_eq!(timing["source"], json!("transcript")); } #[test] fn carries_run_index_from_dispatch_task_into_each_run_record() { let root = TempDir::new().unwrap(); - let (iter, _sub) = dirs(&root); + let iter = dirs(&root); let cond_dir = iter.join("eval-crash").join("with_skill"); let mut serialized = Vec::new(); for k in [1u32, 2] { @@ -702,8 +595,7 @@ mod tests { ) .unwrap(); - let result = - record_runs(&iter, Harness::Codex, DispatchMechanism::Cli, None, false).unwrap(); + let result = record_runs(&iter, Harness::Codex, false).unwrap(); assert_eq!(result.recorded, 2); for k in [1u32, 2] { @@ -718,7 +610,7 @@ mod tests { #[test] fn assembles_codex_records_from_each_tasks_events() { let root = TempDir::new().unwrap(); - let (iter, _sub) = dirs(&root); + let iter = dirs(&root); let paths = write_iteration( &iter, &[FixtureTask { @@ -729,8 +621,7 @@ mod tests { ); write_codex_events(&paths[0].outputs_dir, "Codex final."); - let result = - record_runs(&iter, Harness::Codex, DispatchMechanism::Cli, None, false).unwrap(); + let result = record_runs(&iter, Harness::Codex, false).unwrap(); assert_eq!(result.recorded, 1); assert_eq!(result.missing_transcript, 0); @@ -751,7 +642,7 @@ mod tests { #[test] fn falls_back_to_codex_final_agent_message_when_final_message_md_missing() { let root = TempDir::new().unwrap(); - let (iter, _sub) = dirs(&root); + let iter = dirs(&root); let paths = write_iteration( &iter, &[FixtureTask { @@ -762,8 +653,7 @@ mod tests { ); write_codex_events(&paths[0].outputs_dir, "Closing summary from Codex."); - let result = - record_runs(&iter, Harness::Codex, DispatchMechanism::Cli, None, false).unwrap(); + let result = record_runs(&iter, Harness::Codex, false).unwrap(); assert_eq!(result.recorded, 1); assert_eq!( read_run(&iter, "crash", "with_skill").final_message, @@ -774,7 +664,7 @@ mod tests { #[test] fn skips_existing_run_without_overwrite_then_replaces_with_it() { let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); + let iter = dirs(&root); let paths = write_iteration( &iter, &[FixtureTask { @@ -783,12 +673,7 @@ mod tests { final_message: Some("New."), }], ); - write_subagent( - &sub, - "agent-a", - "crash:with_skill:i1-nonce1", - &transcript_lines("unused"), - ); + write_claude_events(&paths[0].outputs_dir, "unused"); let hand_written = json!({ "eval_id": "crash", "condition": "with_skill", "skill_path": "/staged/skill/SKILL.md", "prompt": "Do the crash task", @@ -796,14 +681,7 @@ mod tests { }); fs::write(&paths[0].run_record_path, hand_written.to_string()).unwrap(); - let skipped = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap(); + let skipped = record_runs(&iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(skipped.recorded, 0); assert_eq!(skipped.skipped_existing, 1); assert_eq!( @@ -811,14 +689,7 @@ mod tests { "Agent-authored." ); - let replaced = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - true, - ) - .unwrap(); + let replaced = record_runs(&iter, Harness::ClaudeCode, true).unwrap(); assert_eq!(replaced.recorded, 1); assert_eq!(read_run(&iter, "crash", "with_skill").final_message, "New."); } @@ -826,7 +697,7 @@ mod tests { #[test] fn backfills_timing_only_when_absent() { let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); + let iter = dirs(&root); let paths = write_iteration( &iter, &[FixtureTask { @@ -835,26 +706,14 @@ mod tests { final_message: Some("Done."), }], ); - write_subagent( - &sub, - "agent-a", - "crash:with_skill:i1-nonce1", - &transcript_lines("unused"), - ); + write_claude_events(&paths[0].outputs_dir, "unused"); fs::write( &paths[0].timing_path, json!({"total_tokens": 12345, "duration_ms": 9000}).to_string(), ) .unwrap(); - record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap(); + record_runs(&iter, Harness::ClaudeCode, false).unwrap(); // Agent-captured completion-event timing wins; not overwritten. let timing = read_timing_value(&iter, "crash", "with_skill"); @@ -863,44 +722,10 @@ mod tests { assert!(timing.get("source").is_none()); } - #[test] - fn falls_back_to_transcript_final_assistant_text_when_final_message_md_missing() { - let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); - write_iteration( - &iter, - &[FixtureTask { - eval_id: "crash", - condition: "with_skill", - final_message: None, - }], - ); - write_subagent( - &sub, - "agent-a", - "crash:with_skill:i1-nonce1", - &transcript_lines("Closing summary from transcript."), - ); - - let result = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap(); - assert_eq!(result.recorded, 1); - assert_eq!( - read_run(&iter, "crash", "with_skill").final_message, - "Closing summary from transcript." - ); - } - #[test] fn skips_the_slot_entirely_when_no_final_message_source_exists() { let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); + let iter = dirs(&root); write_iteration( &iter, &[FixtureTask { @@ -911,14 +736,7 @@ mod tests { ); // No final-message.md, no transcript. - let result = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap(); + let result = record_runs(&iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.recorded, 0); assert_eq!(result.skipped_no_final_message, 1); assert!(!run_exists(&iter, "crash", "with_skill")); @@ -928,7 +746,7 @@ mod tests { #[test] fn writes_empty_invocations_and_no_timing_when_transcript_missing() { let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); + let iter = dirs(&root); write_iteration( &iter, &[FixtureTask { @@ -937,16 +755,9 @@ mod tests { final_message: Some("Done."), }], ); - // final-message.md exists but no subagent transcript matches. + // final-message.md exists but no events file is present. - let result = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap(); + let result = record_runs(&iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.recorded, 1); assert_eq!(result.missing_transcript, 1); @@ -959,16 +770,9 @@ mod tests { #[test] fn errors_when_dispatch_json_is_absent() { let root = TempDir::new().unwrap(); - let (iter, sub) = dirs(&root); + let iter = dirs(&root); // Hand-authored/operator runs have no dispatch.json — the manual path owns them. - let err = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::InSession, - Some(&sub), - false, - ) - .unwrap_err(); + let err = record_runs(&iter, Harness::ClaudeCode, false).unwrap_err(); assert!( err.to_string().contains("dispatch.json"), "error was: {err}" @@ -982,36 +786,7 @@ mod tests { missing_transcript: 0, ..Default::default() }; - assert!( - result - .transcript_warning(Harness::ClaudeCode, DispatchMechanism::InSession) - .is_none() - ); - } - - #[test] - fn claude_code_warning_names_agent_description_when_all_runs_miss() { - let result = RecordRunsResult { - recorded: 8, - missing_transcript: 8, - ..Default::default() - }; - let warning = result - .transcript_warning(Harness::ClaudeCode, DispatchMechanism::InSession) - .unwrap(); - assert!(warning.contains("8"), "names the count: {warning}"); - assert!( - warning.contains("agent_description"), - "points at the load-bearing key: {warning}" - ); - assert!( - warning.to_lowercase().contains("verbatim"), - "says pass it verbatim: {warning}" - ); - assert!( - warning.contains("--subagents-dir"), - "offers the other likely cause: {warning}" - ); + assert!(result.transcript_warning(Harness::ClaudeCode).is_none()); } #[test] @@ -1021,9 +796,7 @@ mod tests { missing_transcript: 1, ..Default::default() }; - let warning = result - .transcript_warning(Harness::ClaudeCode, DispatchMechanism::InSession) - .unwrap(); + let warning = result.transcript_warning(Harness::ClaudeCode).unwrap(); assert!(warning.contains('1'), "names the count: {warning}"); } @@ -1034,9 +807,7 @@ mod tests { missing_transcript: 2, ..Default::default() }; - let warning = result - .transcript_warning(Harness::Codex, DispatchMechanism::Cli) - .unwrap(); + let warning = result.transcript_warning(Harness::Codex).unwrap(); assert!( warning.contains("codex-events.jsonl"), "names the Codex source: {warning}" @@ -1050,7 +821,7 @@ mod tests { #[test] fn assembles_claude_hybrid_records_from_each_tasks_events() { let root = TempDir::new().unwrap(); - let (iter, _sub) = dirs(&root); + let iter = dirs(&root); let paths = write_iteration( &iter, &[FixtureTask { @@ -1061,14 +832,7 @@ mod tests { ); write_claude_events(&paths[0].outputs_dir, "Closing summary."); - let result = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = record_runs(&iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.recorded, 1); assert_eq!(result.missing_transcript, 0); @@ -1091,7 +855,7 @@ mod tests { // Claude `-p` has no --output-last-message, so the result event's text is // the primary final-message source. let root = TempDir::new().unwrap(); - let (iter, _sub) = dirs(&root); + let iter = dirs(&root); let paths = write_iteration( &iter, &[FixtureTask { @@ -1102,14 +866,7 @@ mod tests { ); write_claude_events(&paths[0].outputs_dir, "Closing summary from claude -p."); - let result = record_runs( - &iter, - Harness::ClaudeCode, - DispatchMechanism::Cli, - None, - false, - ) - .unwrap(); + let result = record_runs(&iter, Harness::ClaudeCode, false).unwrap(); assert_eq!(result.recorded, 1); assert_eq!( read_run(&iter, "crash", "with_skill").final_message, @@ -1124,9 +881,7 @@ mod tests { missing_transcript: 2, ..Default::default() }; - let warning = result - .transcript_warning(Harness::ClaudeCode, DispatchMechanism::Cli) - .unwrap(); + let warning = result.transcript_warning(Harness::ClaudeCode).unwrap(); assert!( warning.contains("claude-events.jsonl"), "names the Claude hybrid source: {warning}" diff --git a/tests/run/claude_cli.rs b/tests/run/claude_cli.rs index 55a167b..69f551f 100644 --- a/tests/run/claude_cli.rs +++ b/tests/run/claude_cli.rs @@ -1,7 +1,6 @@ //! Claude Code CLI run modes (`--run-mode hybrid` / `headless`): `claude -p` //! stream-json dispatch guidance, run-mode persistence + defaulting, the -//! human-followed runbook, the write guard under Cli dispatch, and the remaining -//! run-mode combo rejections (Codex interactive). +//! human-followed runbook, and the write guard under Cli dispatch. use crate::helpers::*; use predicates::str::contains; @@ -72,31 +71,6 @@ fn claude_hybrid_dispatch_guidance_includes_agent_model_when_provided() { assert!(stdout.contains("--model opus")); } -#[test] -fn claude_defaults_to_interactive_handoff() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "claude-code", - "--dry-run", - ]) - .assert() - .success(); - - // No --run-mode → interactive default; no CLI recipe in the manifest. - let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); - assert_eq!(conditions["run_mode"], "interactive"); - let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); - assert!(!manifest.contains("claude -p")); -} - #[test] fn claude_hybrid_runbook_is_human_followed_cli_recipe() { let tmp = tempfile::TempDir::new().unwrap(); @@ -226,29 +200,6 @@ fn claude_hybrid_record_runs_does_not_require_a_session_id() { .stdout(contains("Recorded:")); } -#[test] -fn codex_rejects_run_mode_interactive() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "codex", - "--run-mode", - "interactive", - "--dry-run", - ]) - .assert() - .failure() - .stderr(contains("interactive")) - .stderr(contains("codex")); -} - #[test] fn claude_cli_guard_installs_project_hook() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/tests/run/env_layout.rs b/tests/run/env_layout.rs index f0f6073..86dd8e4 100644 --- a/tests/run/env_layout.rs +++ b/tests/run/env_layout.rs @@ -1,6 +1,7 @@ -//! Isolated-run env builder: staging redirects into the per-iteration -//! `env/` dir, fixtures are copied in like a real repo, and `RUNBOOK.md` lives in -//! the env. eval-magic meta stays above the env in `iteration-N/`. +//! Isolated-run env builder: staging redirects into the per-`(group, condition)` +//! `env--/` dirs, fixtures are copied into each like a real repo, +//! and `RUNBOOK.md` lives above them in `iteration-N/`. eval-magic meta stays above +//! the envs in `iteration-N/`. use crate::helpers::*; use serde_json::json; @@ -19,7 +20,8 @@ fn stages_into_env_not_cwd() { .assert() .success(); - // The staged skill lands under env/.claude/skills, not the invocation cwd. + // The staged skill lands under env-g1-with_skill/.claude/skills, not the + // invocation cwd. assert_eq!( env_staged_entries(&cwd), vec!["slow-powers-eval-1-with_skill__mr-review"] @@ -30,7 +32,11 @@ fn stages_into_env_not_cwd() { ); // eval-magic meta stays above the env, in iteration-N/. assert!(iteration_dir(&cwd).join("dispatch.json").exists()); - assert!(!env_dir(&cwd).join("dispatch.json").exists()); + assert!( + !cli_env_dir(&cwd, "g1", "with_skill") + .join("dispatch.json") + .exists() + ); } #[test] @@ -52,8 +58,10 @@ fn env_dir_created_even_with_no_stage() { .assert() .success(); - // Even with staging disabled, the env must exist for fixtures + RUNBOOK. - assert!(env_dir(&cwd).is_dir()); + // Even with staging disabled, each per-(group, condition) env must exist for + // fixtures + the per-env guard. + assert!(cli_env_dir(&cwd, "g1", "with_skill").is_dir()); + assert!(cli_env_dir(&cwd, "g1", "without_skill").is_dir()); } #[test] @@ -77,10 +85,14 @@ fn fixtures_copied_into_env_like_a_real_repo() { .assert() .success(); - // Structure preserved under env/, not flattened into an inputs/ bucket. - assert_eq!(read_str(&env_dir(&cwd).join("src/main.rs")), "fn main() {}"); - assert_eq!(read_str(&env_dir(&cwd).join("data/x.json")), "{}"); - assert!(!env_dir(&cwd).join("inputs").exists()); + // Structure preserved under each per-condition env, not flattened into an + // inputs/ bucket. Fixtures are copied into every relevant env (per its group). + for cond in ["with_skill", "without_skill"] { + let env = cli_env_dir(&cwd, "g1", cond); + assert_eq!(read_str(&env.join("src/main.rs")), "fn main() {}"); + assert_eq!(read_str(&env.join("data/x.json")), "{}"); + assert!(!env.join("inputs").exists()); + } // The dispatch prompt lists fixtures env-relative — the agent's cwd is env. let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); @@ -154,15 +166,17 @@ fn dispatch_outputs_live_under_env() { // Canonicalize to compare across the macOS /var → /private/var symlink: // dispatch.json stores resolved paths, but the test roots come from the raw // tempdir, so a lexical starts_with would mismatch. - let env = fs::canonicalize(env_dir(&cwd)).unwrap(); let iter = fs::canonicalize(iteration_dir(&cwd)).unwrap(); - let outputs_root = env.join(".eval-magic-outputs"); for task in tasks { - // The agent-under-test (cwd = env/) writes only inside its env. + // The agent-under-test (cwd = its per-(group, condition) env) writes only + // inside that env's .eval-magic-outputs/. + let cond = task["condition"].as_str().unwrap(); + let env = fs::canonicalize(cli_env_dir(&cwd, "g1", cond)).unwrap(); + let outputs_root = env.join(".eval-magic-outputs"); let outputs_dir = fs::canonicalize(task["outputs_dir"].as_str().unwrap()).unwrap(); assert!( outputs_dir.starts_with(&outputs_root), - "outputs_dir under env/.eval-magic-outputs/: {}", + "outputs_dir under env-g1-{cond}/.eval-magic-outputs/: {}", outputs_dir.display() ); // run.json / timing.json are eval-magic meta: above the env, in iteration-N/. @@ -208,8 +222,14 @@ fn shared_fixture_copied_once_across_conditions_and_runs() { .assert() .success(); - // One shared copy in env, referenced env-relative by every condition × run. - assert_eq!(read_str(&env_dir(&cwd).join("fixture.txt")), "DATA"); + // One copy per env, shared env-relative by that env's runs. Each condition env + // carries its own copy, referenced env-relative ("fixture.txt") by every task. + for cond in ["with_skill", "without_skill"] { + assert_eq!( + read_str(&cli_env_dir(&cwd, "g1", cond).join("fixture.txt")), + "DATA" + ); + } let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); let tasks = dispatch["tasks"].as_array().unwrap(); assert_eq!(tasks.len(), 4, "1 eval × 2 conditions × 2 runs"); @@ -238,8 +258,12 @@ fn two_evals_sharing_a_fixture_declaration_succeeds() { .assert() .success(); - // Two evals declaring the same fixture from the same source is an idempotent share. - assert_eq!(read_str(&env_dir(&cwd).join("shared.txt")), "SHARED"); + // Two evals declaring the same fixture from the same source is an idempotent + // share: the with_skill env carries a single copy. + assert_eq!( + read_str(&cli_env_dir(&cwd, "g1", "with_skill").join("shared.txt")), + "SHARED" + ); let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); for id in ["e1", "e2"] { let task = dispatch["tasks"] @@ -276,14 +300,14 @@ fn env_contains_only_the_staged_skill_no_repo_leakage() { .assert() .success(); - // env/.claude/skills holds only the staged skill-under-test. + // env-g1-with_skill/.claude/skills holds only the staged skill-under-test. assert_eq!( env_staged_entries(&cwd), vec!["slow-powers-eval-1-with_skill__mr-review"] ); - // The unrelated cwd skill is absent from env. + // The unrelated cwd skill is absent from the env. assert!( - !env_dir(&cwd) + !cli_env_dir(&cwd, "g1", "with_skill") .join(".claude/skills/unrelated-skill") .exists() ); @@ -301,9 +325,12 @@ fn guard_marker_allowed_roots_cover_meta_above_env() { .assert() .success(); - // The guard arms inside env, but its allowedRoots include the workspace root above env, - // so eval-magic can still write meta (benchmark.json, dispatch.json) into iteration-N/. - let marker = read_json(&env_dir(&cwd).join(".claude/skills/.slow-powers-eval-guard.json")); + // The guard arms inside each env, but its allowedRoots include the workspace root + // above env, so eval-magic can still write meta (benchmark.json, dispatch.json) + // into iteration-N/. + let marker = read_json( + &cli_env_dir(&cwd, "g1", "with_skill").join(".claude/skills/.slow-powers-eval-guard.json"), + ); let roots = marker["allowedRoots"].as_array().unwrap(); let iter = iteration_dir(&cwd); assert!( diff --git a/tests/run/grouping.rs b/tests/run/grouping.rs index e941409..d018a34 100644 --- a/tests/run/grouping.rs +++ b/tests/run/grouping.rs @@ -1,7 +1,8 @@ //! Isolation-group batching during `run`: how the setup phase groups evals into -//! environments and records the plan in `dispatch.json`. Covers the in-session -//! single-env (byte-compat) path, the Cli per-(group, condition) split that closes -//! the condition-isolation gap, and the explicit `isolation: isolated` hint. +//! environments and records the plan in `dispatch.json`. Covers the per-(group, +//! condition) env split that closes the condition-isolation gap — emitted for every +//! run now, including the bare default invocation — and the explicit +//! `isolation: isolated` hint that fans a second group out into its own envs. use crate::helpers::*; use serde_json::json; @@ -17,7 +18,7 @@ fn write_fixtures(skill_dir: &std::path::Path) { } #[test] -fn insession_single_group_omits_groups_key_and_stays_bare_env() { +fn single_group_emits_groups_key_and_per_condition_envs() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); skill_eval() @@ -28,17 +29,27 @@ fn insession_single_group_omits_groups_key_and_stays_bare_env() { .assert() .success(); - // The common no-conflict in-session case is byte-identical to the pre-grouping - // shape: a bare env/, no `groups` summary, and no per-task group/eval_root keys. - assert!(env_dir(&cwd).exists()); + // Even the bare default invocation now splits the env per (group, condition) and + // always records a `groups` summary — the single-env, no-groups shape is gone. + assert!(cli_env_dir(&cwd, "g1", "with_skill").exists()); + assert!(cli_env_dir(&cwd, "g1", "without_skill").exists()); let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); - assert!( - dispatch.get("groups").is_none(), - "single-group in-session omits the groups summary: {dispatch}" - ); + let groups = dispatch["groups"] + .as_array() + .expect("groups summary present even for a single group"); + assert_eq!(groups.len(), 1); + assert_eq!(groups[0]["id"], "g1"); + + // A single group means no per-task group tag, but each task still carries the + // per-condition env it runs in via eval_root. for task in dispatch["tasks"].as_array().unwrap() { - assert!(task.get("group").is_none(), "no group tag: {task}"); - assert!(task.get("eval_root").is_none(), "no eval_root: {task}"); + assert!(task.get("group").is_none(), "single group: no tag: {task}"); + let cond = task["condition"].as_str().unwrap(); + let eval_root = task["eval_root"].as_str().expect("task carries eval_root"); + assert!( + eval_root.ends_with(&format!("env-g1-{cond}")), + "eval_root points at the per-condition env: {eval_root}" + ); } } @@ -105,7 +116,7 @@ fn cli_single_group_emits_groups_and_splits_env_per_condition() { } #[test] -fn isolated_hint_splits_into_two_groups_in_session() { +fn isolated_hint_splits_into_two_groups() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), TWO_EVALS_ONE_ISOLATED); write_fixtures(&skill_dir); @@ -133,7 +144,7 @@ fn isolated_hint_splits_into_two_groups_in_session() { groups[1]["rationale"] ); - // Tasks are tagged with their group. + // With two groups, tasks are tagged with their group. let e2_task = dispatch["tasks"] .as_array() .unwrap() @@ -142,12 +153,23 @@ fn isolated_hint_splits_into_two_groups_in_session() { .unwrap(); assert_eq!(e2_task["group"], "g2"); - // In-session stages only the FIRST group's fixtures into the one env up front; - // the isolated group's fixtures are swapped in later by reset-batch. - assert_eq!(read_str(&env_dir(&cwd).join("a.txt")), "AAA"); + // Each group gets its own per-condition envs, holding only that group's fixtures — + // g1's a.txt never leaks into g2's env and vice versa. + assert_eq!( + read_str(&cli_env_dir(&cwd, "g1", "with_skill").join("a.txt")), + "AAA" + ); + assert!( + !cli_env_dir(&cwd, "g1", "with_skill").join("b.txt").exists(), + "the isolated group's fixture is not staged into g1's env" + ); + assert_eq!( + read_str(&cli_env_dir(&cwd, "g2", "with_skill").join("b.txt")), + "BBB" + ); assert!( - !env_dir(&cwd).join("b.txt").exists(), - "the isolated group's fixture is not staged into the shared env up front" + !cli_env_dir(&cwd, "g2", "with_skill").join("a.txt").exists(), + "g1's fixture is not staged into the isolated group's env" ); } diff --git a/tests/run/helpers.rs b/tests/run/helpers.rs index 1c5caab..5e3a96d 100644 --- a/tests/run/helpers.rs +++ b/tests/run/helpers.rs @@ -36,24 +36,20 @@ pub fn iteration_dir(cwd: &Path) -> PathBuf { .join("iteration-1") } -/// The isolated env dir that becomes the agent-under-test's cwd (in-session -/// dispatch): staging, fixtures, and `RUNBOOK.md` all land under here, below -/// `iteration_dir`. -pub fn env_dir(cwd: &Path) -> PathBuf { - iteration_dir(cwd).join("env") -} - -/// A per-`(group, condition)` Cli env dir — the cwd each `claude -p`/`codex exec` +/// A per-`(group, condition)` env dir — the cwd each `claude -p`/`codex exec` /// subprocess runs from: `iteration-N/env--/`. Each holds only /// that condition's skill (or none, for the control arm) and its group's fixtures. +/// Staging, fixtures, and the guard marker all land under here, below +/// `iteration_dir`; `RUNBOOK.md` lives above it in `iteration_dir`. pub fn cli_env_dir(cwd: &Path, group: &str, condition: &str) -> PathBuf { iteration_dir(cwd).join(format!("env-{group}-{condition}")) } -/// Staged skill names under the env's harness skills dir (`env/.claude/skills`), -/// excluding the staging manifest, sorted. +/// Staged skill names under the default single-group `with_skill` env's harness +/// skills dir (`env-g1-with_skill/.claude/skills`), excluding the staging +/// manifest, sorted. pub fn env_staged_entries(cwd: &Path) -> Vec { - staged_entries(&env_dir(cwd).join(".claude/skills")) + staged_entries(&cli_env_dir(cwd, "g1", "with_skill").join(".claude/skills")) } pub fn read_json(path: &Path) -> Value { diff --git a/tests/run/lifecycle.rs b/tests/run/lifecycle.rs index f7a1b15..42f69d5 100644 --- a/tests/run/lifecycle.rs +++ b/tests/run/lifecycle.rs @@ -11,8 +11,8 @@ use std::path::Path; fn guard_installs_pretooluse_hook_under_env() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // The guard arms inside the isolated env — the agent-under-test's cwd. - let settings = env_dir(&cwd).join(".claude/settings.local.json"); + // The guard arms inside each per-(group, condition) env — the agent-under-test's cwd. + let settings = cli_env_dir(&cwd, "g1", "with_skill").join(".claude/settings.local.json"); skill_eval() .current_dir(&cwd) @@ -48,13 +48,16 @@ fn guard_installs_pretooluse_hook_under_env() { } #[test] -fn finalize_does_not_warn_about_env_scoped_guard_from_cwd() { +fn finalize_warns_about_armed_per_env_guard_for_default_run() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // The guard arms inside the env; `finalize` checks the invocation cwd, where no - // guard lives, so it does not warn. The env-scoped guard is harmless to the operator's - // cwd (it only loads when cwd = env); the in-env loop handles it within the session. - let marker = env_dir(&cwd).join(".claude/skills/.slow-powers-eval-guard.json"); + // The bare default run is hybrid: `--guard` arms a marker in each per-(group, + // condition) env. `finalize` runs from the invocation cwd, not inside any env, but + // the reworked finalize walks the per-env markers, so it reminds the operator the + // guard is still armed. (finalize only warns; `teardown` disarms — the marker + // survives finalize.) + let marker = + cli_env_dir(&cwd, "g1", "with_skill").join(".claude/skills/.slow-powers-eval-guard.json"); skill_eval() .current_dir(&cwd) @@ -72,7 +75,7 @@ fn finalize_does_not_warn_about_env_scoped_guard_from_cwd() { .args(["--skill", "mr-review", "--iteration", "1"]) .assert() .success() - .stdout(contains("Guard still armed").not()); + .stdout(contains("Guard still armed")); assert!(marker.exists()); } @@ -105,8 +108,8 @@ fn finalize_does_not_warn_when_guard_is_not_armed() { fn teardown_reclaims_workspace_and_env_guard() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - let settings = env_dir(&cwd).join(".claude/settings.local.json"); - let staged = env_dir(&cwd).join(".claude/skills"); + let settings = cli_env_dir(&cwd, "g1", "with_skill").join(".claude/settings.local.json"); + let staged = cli_env_dir(&cwd, "g1", "with_skill").join(".claude/skills"); skill_eval() .current_dir(&cwd) @@ -355,8 +358,8 @@ fn runs_flag_expands_dispatches_into_run_dirs() { .join(cond) .join(format!("run-{k}")); assert!(run_dir.is_dir(), "missing meta run dir {run_dir:?}"); - // Per-run outputs dir inside the env. - let out_dir = env_dir(&cwd) + // Per-run outputs dir inside the condition's env. + let out_dir = cli_env_dir(&cwd, "g1", cond) .join(".eval-magic-outputs") .join(format!("eval-{eval}")) .join(cond) @@ -396,8 +399,10 @@ fn runs_one_keeps_flat_single_run_layout() { let cond_dir = iteration_dir(&cwd).join("eval-e1").join("with_skill"); assert!(cond_dir.is_dir()); assert!(!cond_dir.join("run-1").exists()); - // Outputs live inside the env, flat (no run-1/ segment) for a single-run cell. - let out_dir = env_dir(&cwd).join(".eval-magic-outputs/eval-e1/with_skill"); + // Outputs live inside the condition's env, flat (no run-1/ segment) for a + // single-run cell. + let out_dir = + cli_env_dir(&cwd, "g1", "with_skill").join(".eval-magic-outputs/eval-e1/with_skill"); assert!(out_dir.is_dir()); assert!(!out_dir.join("run-1").exists()); } diff --git a/tests/run/runbook.rs b/tests/run/runbook.rs index e4c4a62..b1eb688 100644 --- a/tests/run/runbook.rs +++ b/tests/run/runbook.rs @@ -2,65 +2,6 @@ //! artifact, and the post-run pointer at it. use crate::helpers::*; -use predicates::prelude::PredicateBooleanExt; -use predicates::str::contains; - -#[test] -fn run_writes_interactive_runbook_and_points_at_it() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // A real run (not --dry-run) so the post-run "Next:" handoff prints; --dry-run - // stops before next steps by contract. - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args(["--skill", "mr-review"]) - .assert() - .success() - // The summary hands off to a fresh isolated session: cd into env/, then - // "Read and follow RUNBOOK.md". It must not re-print the dispatch loop — - // that lives only in RUNBOOK.md now (the session-juggling apparatus is gone). - // (The exact env path in the handoff is locked by the util.rs unit test; - // here we just confirm the handoff is wired into stdout.) - .stdout(contains("Read and follow RUNBOOK.md")) - .stdout(contains("1. cd ")) - .stdout(contains("one batch at a time").not()); - - // The runbook lives inside the isolated env — the session's cwd reads it. - assert!(!iteration_dir(&cwd).join("RUNBOOK.md").exists()); - let book = read_str(&env_dir(&cwd).join("RUNBOOK.md")); - assert!(book.contains("mr-review"), "names the skill: {book}"); - assert!( - book.contains("with_skill") && book.contains("without_skill"), - "names both conditions: {book}" - ); - assert!( - book.contains("agent_description"), - "carries the in-session dispatch guidance: {book}" - ); - // The per-condition batch loop: a switch-condition barrier between the two - // batches, carrying the absolute --workspace-dir so it resolves from env/. - assert!( - book.contains("eval-magic switch-condition --skill-dir") - && book.contains("--workspace-dir") - && book.contains("--condition without_skill"), - "carries the switch-condition barrier between batches: {book}" - ); - assert!( - book.contains("eval-magic ingest --skill-dir"), - "carries the ingest command: {book}" - ); - assert!( - book.contains("eval-magic finalize --skill-dir"), - "carries the finalize command: {book}" - ); - assert!( - book.contains("benchmark.json"), - "points at the result: {book}" - ); - assert!(!book.contains("{{"), "no unsubstituted tokens: {book}"); -} #[test] fn run_writes_headless_runbook_for_codex() { @@ -75,8 +16,12 @@ fn run_writes_headless_runbook_for_codex() { .success(); // Cli dispatches from per-(group, condition) envs, so the human-followed runbook - // lives in the iteration dir (there is no single env/). - assert!(!env_dir(&cwd).join("RUNBOOK.md").exists()); + // lives in the iteration dir, not inside any env. + assert!( + !cli_env_dir(&cwd, "g1", "with_skill") + .join("RUNBOOK.md") + .exists() + ); let book = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); assert!( book.contains("human driving"), diff --git a/tests/run/staging.rs b/tests/run/staging.rs index aafc2b5..e0f3bc0 100644 --- a/tests/run/staging.rs +++ b/tests/run/staging.rs @@ -64,9 +64,9 @@ fn run_from_skill_dir_defaults_to_new_skill_without_staging_siblings() { .assert() .success() .stdout(contains("Preparing mr-review iteration-1 (new-skill)")) - // The run summary now hands off to the isolated session; the pipeline - // commands live in the RUNBOOK (asserted below), not the printed summary. - .stdout(contains("Read and follow RUNBOOK.md")); + // The run summary points at the human-followed RUNBOOK (a copy of the dispatch + // steps); the auto-derived pipeline commands are threaded into it (asserted below). + .stdout(contains("RUNBOOK.md")); assert!( direct_iteration_dir(&skill_sub) @@ -79,13 +79,10 @@ fn run_from_skill_dir_defaults_to_new_skill_without_staging_siblings() { ); // Run from inside the skill dir with no args: the auto-derived target selector - // (`command_target_args`) is threaded into the RUNBOOK's pipeline commands. - let runbook = read_str( - &direct_iteration_dir(&skill_sub) - .join("env") - .join("RUNBOOK.md"), - ); - assert!(runbook.contains("eval-magic ingest --skill-dir")); + // (`command_target_args`) is threaded into the RUNBOOK's pipeline commands. The + // RUNBOOK lives in the iteration dir (Cli dispatch has no single env/). + let runbook = read_str(&direct_iteration_dir(&skill_sub).join("RUNBOOK.md")); + assert!(runbook.contains("ingest --skill-dir")); assert!(runbook.contains("--skill mr-review --workspace-dir")); assert!(runbook.contains("--iteration 1")); @@ -194,7 +191,7 @@ fn stage_name_threads_verbatim_name_and_registers_cleanup() { .assert() .success(); - let skills_dir = env_dir(&cwd).join(".claude/skills"); + let skills_dir = cli_env_dir(&cwd, "g1", "with_skill").join(".claude/skills"); assert_eq!(staged_entries(&skills_dir), vec!["mr-review"]); let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); @@ -231,10 +228,10 @@ fn stage_name_threads_verbatim_name_and_registers_cleanup() { fn stage_name_refuses_to_clobber_preexisting_dir() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // Staging now lands in env/.claude/skills, which is fresh per iteration. - // The clobber guard still matters on a re-run (--iteration 1) where the env - // already holds an untracked skill dir; pre-seed that and confirm it is preserved. - let preexisting = env_dir(&cwd).join(".claude/skills/my-real-skill"); + // Staging now lands in env-g1-with_skill/.claude/skills, which is fresh per + // iteration. The clobber guard still matters on a re-run (--iteration 1) where the + // env already holds an untracked skill dir; pre-seed that and confirm it is preserved. + let preexisting = cli_env_dir(&cwd, "g1", "with_skill").join(".claude/skills/my-real-skill"); fs::create_dir_all(&preexisting).unwrap(); fs::write(preexisting.join("SKILL.md"), "USER OWNED").unwrap(); From 34f325dcc958303ac76243e3375b2588e6817f30 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Wed, 1 Jul 2026 20:39:27 -0400 Subject: [PATCH 04/45] feat(instant-magic): retire run mode, collapse docs --- README.md | 128 +++++------ docs/harness-parity.md | 58 ++--- .../{runbook-headless.md => runbook.md} | 0 src/adapters/claude_cli.rs | 3 +- src/adapters/harness.rs | 11 +- src/adapters/mod.rs | 4 +- src/adapters/plugin_shadow.rs | 35 ++- src/cli/args.rs | 13 +- src/cli/commands/pipeline.rs | 3 - src/cli/mod.rs | 5 +- src/cli/run/mod.rs | 2 +- src/cli/run/orchestrate/build.rs | 2 - src/cli/run/runbook.rs | 16 +- src/cli/run/steps.rs | 10 +- src/cli/run/util.rs | 23 +- src/core/capabilities.rs | 66 ++++++ src/core/context.rs | 11 - src/core/mod.rs | 12 +- src/core/run_mode.rs | 206 ------------------ src/core/types.rs | 7 - src/pipeline/fill_transcripts.rs | 5 +- src/pipeline/record_runs.rs | 8 +- tests/run/claude_cli.rs | 170 +++++++-------- tests/run/codex.rs | 24 +- tests/run/env_layout.rs | 6 +- tests/run/grouping.rs | 12 +- tests/run/lifecycle.rs | 19 +- tests/run/runbook.rs | 9 +- 28 files changed, 294 insertions(+), 574 deletions(-) rename profiles/shared/{runbook-headless.md => runbook.md} (100%) create mode 100644 src/core/capabilities.rs delete mode 100644 src/core/run_mode.rs diff --git a/README.md b/README.md index 36ad482..cff6dc5 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ An eval dispatches a fresh subagent twice per test case — once with the skill `eval-magic` ships as a dependency-less prebuilt binary under the command name `eval-magic`. Every artifact follows a documented JSON Schema, so records grade the same way regardless of where they were authored. **Claude Code and Codex CLI are wired harnesses today**; OpenCode has foundational harness selection and staging support; see [Harnesses](#harnesses) for per-harness fidelity and caveats. From inside an agent session, running an eval is as simple as: *"Install eval-magic and help me run an eval on my-skill."* -This README is the complete operating guide: install, author cases, run both modes, drive the loop, read results, and keep a baseline. For the full flag-by-flag reference, run `eval-magic --help` (and `eval-magic --help`). For *when and why* to write an eval at all — the methodology, the decision to test, designing cases under pressure — see the [`slow-powers`](https://github.com/slowdini/slow-powers) plugin's `evaluating-skills` skill, which owns that craft. +This README is the complete operating guide: install, author cases, run the loop, read results, and keep a baseline. For the full flag-by-flag reference, run `eval-magic --help` (and `eval-magic --help`). For *when and why* to write an eval at all — the methodology, the decision to test, designing cases under pressure — see the [`slow-powers`](https://github.com/slowdini/slow-powers) plugin's `evaluating-skills` skill, which owns that craft. ## Install @@ -41,7 +41,7 @@ cargo build --release # binary at target/release/eval-magic ## How an eval works -For each test case, the runner sets up two conditions and a fresh subagent runs each with clean context — *how* that subagent is dispatched (in-session vs. one-shot CLI) is the run-mode axis covered under [Harnesses](#harnesses): +For each test case, the runner sets up two conditions and a fresh subagent runs each with clean context. Each subagent is dispatched through the harness's one-shot CLI (`claude -p`, `codex exec`) — the specifics per harness are covered under [Harnesses](#harnesses): - **Mode A — new skill:** `with_skill` vs `without_skill`. Validates a brand-new skill beats baseline behavior with no skill loaded. - **Mode B — revision (the common case):** `old_skill` vs `new_skill`. Tests a language change to an existing skill — you snapshot the old `SKILL.md`, then run both variants against the same prompts. A negative or zero `delta.pass_rate` is a signal to revert. @@ -85,35 +85,32 @@ environment. ### Mode A — new skill (with vs. without) ```bash -# 1. Build the iteration's isolated env (arm --guard — see Cost & confirmation). -# run stages skills into .eval-magic/my-skill/iteration-1/env/, copies -# fixtures in, and writes RUNBOOK.md. It does NOT dispatch — it prints a handoff. -# Add --runs to dispatch every eval N times per condition for variance -# reduction (a per-eval "runs" field in evals.json overrides the flag). +# 1. Build the iteration's isolated envs (arm --guard — see Cost & confirmation). +# run stages skills into one env per (group, condition) under +# .eval-magic/my-skill/iteration-1/, copies fixtures in, and writes RUNBOOK.md. +# It does NOT dispatch — it prints a handoff. Add --runs to dispatch every +# eval N times per condition for variance reduction (a per-eval "runs" field in +# evals.json overrides the flag). eval-magic run --guard -# 2. Enter the isolated env and follow the runbook. cd into iteration-1/env/ and -# start a fresh agent session there (interactive Claude Code: the staged skills -# must be present at session start), then say: "Read and follow RUNBOOK.md". -# That session drives the whole loop below — dispatch → switch-condition → -# ingest → finalize — and writes benchmark.json into iteration-1/. (Headless: -# you, a human, follow the same RUNBOOK.md top to bottom; hybrid: the session -# shells out a `claude -p` / `codex exec` recipe per task.) See Claude Code -# below for the plugin-isolation and transcript specifics. +# 2. Follow the runbook. From the iteration dir, read RUNBOOK.md end to end. It +# drives the whole loop below — dispatch → ingest → dispatch judges → finalize — +# dispatching each task through the harness CLI (`claude -p` / `codex exec`) and +# writing benchmark.json into iteration-1/. An agent session can drive the runbook +# ("Read and follow RUNBOOK.md"), or you can follow it by hand — the steps are +# identical. See Claude Code below for the plugin-isolation and transcript specifics. -# Steps 3–5 are driven from inside the runbook — shown here for reference: +# Steps 3–5 are driven from the runbook — shown here for reference: # 3. ingest assembles records, detects stray writes, and grades, stopping at the -# judge hand-off. In-session it auto-resolves transcripts from -# CLAUDE_CODE_SESSION_ID; hybrid/headless read each task's events file instead. +# judge hand-off. It reads each task's events file (outputs/-events.jsonl). eval-magic ingest # 4. Dispatch the judge tasks ingest lists, then finalize. If --guard is still # armed, finalize reminds you to run teardown-guard before editing source. eval-magic finalize -# 5. Read .eval-magic/my-skill/iteration-1/benchmark.json (the prep session -# resumes here), then clean up: +# 5. Read .eval-magic/my-skill/iteration-1/benchmark.json, then clean up: eval-magic teardown ``` @@ -131,41 +128,36 @@ If you snapshot *before* editing, omit `--ref` (it then reads the working tree) ## The run loop -A run is one canonical workflow. `run` *prepares* an isolated env and hands off; a session entered in that env drives the rest of the loop to `benchmark.json`: +A run is one canonical workflow. `run` *prepares* the isolated envs and hands off; the runbook then drives the rest of the loop to `benchmark.json`. An agent session or a human at a terminal can drive it — the steps are identical: ``` -run (prepare env/ + RUNBOOK.md) - └─► [in env/, runbook-driven] dispatch batch A → switch-condition → dispatch batch B +run (prepare per-(group,condition) envs + RUNBOOK.md) + └─► [runbook-driven] dispatch condition A → dispatch condition B → ingest → dispatch judges → finalize ──► benchmark.json teardown ``` -1. **`run` prepares — it does not dispatch.** It builds the iteration workspace (`iteration-N/`), snapshots the `SKILL.md`, stages skills into the isolated env `iteration-N/env/` (the agent's cwd), copies fixtures in so it reads like a real repo, emits `dispatch.json` (machine-readable) alongside `dispatch-manifest.md` (human-readable), and writes `RUNBOOK.md` into `env/`. Then it prints a handoff, not a dispatch. -2. **Enter the isolated env.** `cd` into `iteration-N/env/`, begin a run session there, and say *Read and follow `RUNBOOK.md`*. How you enter differs by run mode (see [Run modes](#run-modes)): - - **Interactive (Claude Code):** start a *fresh* Claude Code session in `env/` so the staged skills are present at session start; it dispatches in-session subagents and runs the rest of the loop itself. - - **Hybrid (Claude Code / Codex):** an orchestrating session follows `RUNBOOK.md`, shelling out a `claude -p` / `codex exec` recipe per task. - - **Headless (Claude Code / Codex):** no session — a human follows the same `RUNBOOK.md`, pasting each recipe and command top to bottom. -3. **Dispatch agents (runbook-driven).** Read `dispatch.json`. Each task object points at a `dispatch_prompt_path` (the full prompt lives in a file so you never reproduce kilobytes inline), an `agent_description` to pass through *verbatim* as the dispatch description, and the exact `run_record_path` / `timing_path`. For each task, dispatch a fresh subagent told to read the file at `dispatch_prompt_path` and follow it exactly. The `agent_description` is namespaced with the iteration and a per-run nonce (`:[:r]:i-`; the `r` segment appears only in multi-run cells, see `run --help` on `--runs`) — passing it through unchanged is what lets transcripts correlate to runs. -4. **`switch-condition` between condition batches.** Conditions run as sequential batches, never interleaved. After joining *all* of the first batch's subagents, run `eval-magic switch-condition --condition ` to remove the off-condition's staged skill from `env/.claude/skills/`, so the next batch can't read it — the read-isolation barrier. When a run has more than one **isolation group** (see below), the runbook also dispatches each group as its own batch and runs `eval-magic reset-batch --group ` between groups — it wipes the shared `env/` working tree and re-seeds it with that group's fixtures, the per-group isolation barrier. The runbook spells out the exact `switch-condition` / `reset-batch` sequence; you never work it out yourself. -5. **`ingest`** (a fixed-order chain: record-runs → fill-transcripts → detect-stray-writes → grade), run from inside `env/`, assembles each task's `run.json` and `timing.json` from `dispatch.json` + the subagent's `outputs/final-message.md` + the persisted transcript, scans for stray writes, and grades the `transcript_check` assertions. It stops at the judge hand-off, listing a judge task per `llm_judge` assertion. -6. **Dispatch judges.** Same pattern as step 3: dispatch a fresh subagent for each judge task to read its prompt file and write its verdict back. -7. **`finalize`** (grade `--finalize` → aggregate) merges the judge verdicts and writes `benchmark.json` into `iteration-N/`, *above* `env/`. Read it. If a `--guard` marker is still live, it also reminds you to run `teardown-guard` before editing source. -8. **`teardown`** disarms the guard, removes the staged skill set, and reclaims the workspace artifacts that are safe to delete. +1. **`run` prepares — it does not dispatch.** It builds the iteration workspace (`iteration-N/`), snapshots the `SKILL.md`, stages skills into one isolated env per `(group, condition)` (`iteration-N/env--/`, the cwd each dispatch runs from), copies fixtures in so each reads like a real repo, emits `dispatch.json` (machine-readable) alongside `dispatch-manifest.md` (human-readable), and writes `RUNBOOK.md` into `iteration-N/`. Then it prints a handoff, not a dispatch. +2. **Follow the runbook.** From `iteration-N/`, read `RUNBOOK.md` end to end. An agent session can drive it (*Read and follow `RUNBOOK.md`*) or you can follow it by hand — the commands are identical. It carries the exact per-task dispatch recipe plus the `ingest` / `finalize` commands, each already threaded with `--harness`. +3. **Dispatch agents (runbook-driven).** Read `dispatch.json`. Each task object points at a `dispatch_prompt_path` (the full prompt lives in a file so you never reproduce kilobytes inline), the `eval_root` env to dispatch from, and the exact `run_record_path` / `timing_path`. For each task, run the harness CLI recipe from its `eval_root`, pointing the dispatched subagent at `dispatch_prompt_path` to read and follow exactly, and capture the events transcript into `outputs/`. Conditions are physically isolated — the `with_skill` env holds the staged skill, the control arm's env holds none — so there is no runtime "switch" step to get wrong. +4. **`ingest`** (a fixed-order chain: record-runs → fill-transcripts → detect-stray-writes → grade) assembles each task's `run.json` and `timing.json` from `dispatch.json` + the subagent's `outputs/final-message.md` + each task's events transcript, scans for stray writes, and grades the `transcript_check` assertions. It stops at the judge hand-off, listing a judge task per `llm_judge` assertion. +5. **Dispatch judges.** Same pattern as step 3: run the CLI recipe for each judge task to read its prompt file and write its verdict back. +6. **`finalize`** (grade `--finalize` → aggregate) merges the judge verdicts and writes `benchmark.json` into `iteration-N/`, *above* the envs. Read it. If a `--guard` marker is still live, it also reminds you to run `teardown-guard` before editing source. +7. **`teardown`** disarms the guard, removes the staged skill set, and reclaims the workspace artifacts that are safe to delete. -The chains run in-process and stop at the first failure; re-running after a fix is safe — every sub-step skips work that's already done. The individual steps (`record-runs`, `fill-transcripts`, `detect-stray-writes`, `grade`, `aggregate`) remain callable for inspection or recovery. Under the `Cli` mechanism (Claude Code hybrid/headless, Codex), the per-task dispatch recipe lives in `RUNBOOK.md` and `ingest` reads each task's events file (`claude-events.jsonl` / `codex-events.jsonl`) instead of an in-session transcript; un-wired harnesses still write records by hand until their adapters land. +The chains run in-process and stop at the first failure; re-running after a fix is safe — every sub-step skips work that's already done. The individual steps (`record-runs`, `fill-transcripts`, `detect-stray-writes`, `grade`, `aggregate`) remain callable for inspection or recovery. The per-task dispatch recipe lives in `RUNBOOK.md` and `dispatch-manifest.md`, and `ingest` reads each task's events file (`claude-events.jsonl` / `codex-events.jsonl`); un-wired harnesses still write records by hand until their adapters land. ### Isolation grouping (which agents batch together) -`run` decides at **setup** time which evals can share an environment and which need their own, writes the plan into `dispatch.json` (a `groups[]` summary plus a per-task `group`/`eval_root`), and the runbook follows it — the executing session does no isolation reasoning itself. By default every eval shares one group (today's behavior, unchanged). Two things create a separate group: evals whose fixtures would clobber each other, and an eval that opts out explicitly with `"isolation": "isolated"` in `evals.json` (use it when an eval's agent *mutates* a fixture another eval reads). How groups are realized depends on the run mode: +`run` decides at **setup** time which evals can share an environment and which need their own, writes the plan into `dispatch.json` (a `groups[]` summary plus a per-task `group`/`eval_root`), and the runbook follows it — whoever drives the loop does no isolation reasoning themselves. By default every eval shares one group. Two things create a separate group: evals whose fixtures would clobber each other, and an eval that opts out explicitly with `"isolation": "isolated"` in `evals.json` (use it when an eval's agent *mutates* a fixture another eval reads). -- **Interactive (in-session):** one `env/`; groups are dispatched as sequential batches with a `reset-batch` barrier between them. -- **Hybrid / headless (Cli):** one env per `(group, condition)` — `iteration-N/env--/` — so each subprocess `cd`s into a fully-isolated cwd. This also gives the Cli path real per-condition isolation: the control arm's env contains no skill at all. +Each `(group, condition)` gets its own env — `iteration-N/env--/` — so every dispatch `cd`s into a fully-isolated cwd holding only that group's fixtures, plus the staged skill for the `with_skill` arm (the control arm's env holds no skill at all). This structural split *is* the per-condition read-isolation barrier — there is no runtime switch step. ## Cost & confirmation An eval run is not free: an N-case suite is **2N full agent sessions**, plus a judge dispatch per `llm_judge` assertion — real wall-clock time and real tokens. A subagent under test runs the real skill, and some skills write to disk, so it can write outside its sandbox. -If you are an agent driving this tool, **never kick off a run silently.** Present the user a run summary — skill, mode, eval cases, the models that will run the agents and the judge, the cost, and the guard status — and wait for explicit confirmation. For CLI-dispatch harnesses, pass `--agent-model ` and `--judge-model ` to have the generated command recipes select those models when the harness adapter supports model selection; for in-session dispatch, those flags are still provenance because the runner does not choose the parent session's model. Arm `--guard` unless the user actively opts out; unguarded, stray writes are only *detected* after the fact by `detect-stray-writes`, never blocked. +If you are an agent driving this tool, **never kick off a run silently.** Present the user a run summary — skill, mode, eval cases, the models that will run the agents and the judge, the cost, and the guard status — and wait for explicit confirmation. Pass `--agent-model ` and `--judge-model ` to have the generated command recipes select those models when the harness adapter supports model selection (Codex today); otherwise they are recorded as provenance. Arm `--guard` unless the user actively opts out; unguarded, stray writes are only *detected* after the fact by `detect-stray-writes`, never blocked. The judgment of *whether* a change needs an eval, and how to design cases that actually measure it, lives in the [`slow-powers`](https://github.com/slowdini/slow-powers) plugin's `evaluating-skills` skill. @@ -264,53 +256,35 @@ For the `without_skill` / baseline condition, the dispatch reflects "this skill Every artifact follows a JSON Schema in [`schema/`](schema/), so a run record means the same thing regardless of which harness produced it. **Claude Code** and **Codex** are wired harnesses, with harness-specific fidelity notes below. The parity-audit framework for bringing another harness up to the supported feature set is in [docs/harness-parity.md](docs/harness-parity.md). -### Run modes +### How dispatch works -*How* an eval gets dispatched is the primary parity axis (see [docs/harness-parity.md](docs/harness-parity.md)), distinct from *which* harness runs it. There are three run modes — pick whichever fits your account and plan: - -- **Headless** — you never start an agent session. You run a series of `eval-magic` commands, and every eval test and judge is dispatched through the harness's one-shot CLI (`claude -p`, `codex exec`), writing transcripts to disk. The run ends in a written report. -- **Fully interactive** — you start an agent session and ask it to run an eval. The agent runs the `eval-magic` commands and dispatches tests and judges as **in-session subagents**, then hands you the report. -- **Hybrid** — like interactive, but the agent guides the process and issues headless CLI dispatches (`claude -p` / `codex exec`) for some or all tests and judges — useful for working through iterations. - -Under the hood these three modes ride on **two dispatch mechanisms** (`DispatchMechanism` in `src/core/run_mode.rs`): *fully-interactive* dispatches **in-session** subagents, while *headless* and *hybrid* both dispatch through the **one-shot CLI** — they differ only in whether a session drives the loop, not in how a single task reaches the harness. +Every eval test and judge is dispatched the same way: through the harness's one-shot CLI (`claude -p`, `codex exec`), one subprocess per task, each `cd`'d into its `(group, condition)` env and writing its events transcript to disk. `run` prepares the envs and `RUNBOOK.md`; from there an **agent session** can drive the loop (reading the runbook and shelling out each recipe) or a **human** can follow the same runbook by hand. Support today: -| Harness | Headless | Fully interactive | Hybrid | -|---------|:--------:|:-----------------:|:------:| +| Harness | CLI dispatch | Transcript ingest | `--guard` | +|---------|:------------:|:-----------------:|:---------:| | **Claude Code** | ✅ | ✅ | ✅ | -| **Codex** | ✅ | ❔ likely N/A¹ | ✅ | -| **OpenCode** | ❌ | ❌ | ❌² | - -¹ Codex dispatches via subprocess (`codex exec`), not in-session subagents, so a "fully interactive" Codex mode may not translate. ² OpenCode foundational harness support is wired: `--harness opencode` stages skills under `.opencode/skills/` and emits native dispatch prompts. Transcript ingest, auto-record, and `--guard` are pending. - -**Cost and billing.** Mode choice has billing consequences: +| **Codex** | ✅ | ✅ | ✅ | +| **OpenCode** | ⚠️ partial¹ | ❌ | ❌ | -- **Claude Code, fully interactive** (Task-tool subagents) — billed under normal interactive session usage/limits (your subscription's interactive pool, or your API key). -- **Claude Code, headless / hybrid** (`claude -p`) — same token-based pricing, but on **subscription plans, starting June 15 2026**, `claude -p` (Agent SDK) usage draws from a **separate monthly Agent SDK credit pool**, distinct from interactive limits. Headless JSON output exposes `total_cost_usd` per invocation, so the runner can record per-task cost — something the in-session Task-tool path can't easily capture. -- **Codex, hybrid** (`codex exec`) — billed under your Codex usage. +¹ OpenCode has foundational harness support: `--harness opencode` stages skills under `.opencode/skills/` and emits native dispatch prompts, but eval-magic does not yet drive OpenCode dispatches or ingest their transcripts. -**Intended end state:** all three modes first-class on Claude Code and Codex (where the mode translates) — reached as of this release; OpenCode wired as a third harness remains. Progress is tracked in [GitHub issues](https://github.com/slowdini/eval-magic/issues). +### Claude Code -### Claude Code (fully wired) +The run loop above *is* the Claude Code loop: each eval test and judge is dispatched through the `claude -p` one-shot CLI. `eval-magic run` only *prepares* the isolated envs (`.eval-magic//iteration-N/env--/`) and writes `RUNBOOK.md` into `iteration-N/`, then prints a handoff. An agent session can drive the runbook (*Read and follow RUNBOOK.md*), or you can follow it by hand — either way each task shells out the same `claude -p` recipe, and `ingest` → `finalize` assemble `benchmark.json`. These are the Claude-Code-specific details: -The run loop above *is* the Claude Code loop. By default this is the **fully-interactive** run mode (see [Run modes](#run-modes)) — subagents are dispatched in-session via the Task tool; the **hybrid** and **headless** (`claude -p`) modes are now wired too (pass `--run-mode hybrid` or `--run-mode headless`, see below). `eval-magic run` itself only *prepares* the isolated env (`.eval-magic//iteration-N/env/`) and writes `RUNBOOK.md` into it, then prints a handoff: `cd` into `env/`, start a **fresh** Claude Code session there, and say *Read and follow RUNBOOK.md*. That fresh session — clean cwd, staged skills present at session start — drives the whole dispatch → switch-condition → ingest → finalize loop and writes `benchmark.json`, which the prep session resumes on. These are the Claude-Code-specific details: +**Isolating from installed plugins.** Read this first if the skill you're evaluating shares a name with one an installed, enabled plugin provides. Each `claude -p` dispatch loads *your* user/global plugins and the global skills dir from its Claude config — the staging slug avoids an on-disk collision but does not stop the installed copy from being discoverable, contaminating both arms (the `without_skill` arm is then not truly skill-absent). The runner can only *detect and warn* (the plugin-shadow banner). To actually isolate, constrain the config each `claude -p` dispatch loads, one of these ways: -**Isolating from installed plugins.** Read this first if the skill you're evaluating shares a name with one an installed, enabled plugin provides. Subagents are dispatched via the **Task tool**, so they inherit *this session's* enabled plugins — the staging slug avoids an on-disk collision but does not stop the installed copy from being discoverable, contaminating both arms (the `without_skill` arm is then not truly skill-absent). Plugins load at session start and can't be unloaded mid-session, so the runner only *detects and warns* (the plugin-shadow banner). The isolated env gives a clean *cwd* but does not unload user/global plugins, so this still applies. To actually isolate, launch the **fresh session you start in `env/`** one of these ways — subagents inherit it: +- **Drop user-scope plugins, keep auth:** add `--setting-sources project,local` to the dispatch. User-scope `enabledPlugins` isn't loaded; auth is unaffected. +- **Disable the specific plugin:** set `"enabledPlugins": { "@": false }` in a settings source the dispatch loads. +- **Clean config dir (strips everything):** run each dispatch under `CLAUDE_CONFIG_DIR="$(mktemp -d)"`. No installed plugins or global skills load at all. Auth caveat: OAuth lives in `~/.claude.json`, which a relocated config dir may not carry — set `ANTHROPIC_API_KEY` or re-authenticate once in the fresh dir. -- **Drop user-scope plugins, keep auth:** `claude --setting-sources project,local`. User-scope `enabledPlugins` isn't loaded; auth is unaffected. -- **Disable the specific plugin, then restart:** set `"enabledPlugins": { "@": false }` in a settings source that loads at startup, and start a fresh session. -- **Clean config dir (strips everything):** `CLAUDE_CONFIG_DIR="$(mktemp -d)" claude`. No installed plugins or global skills load at all. Auth caveat: OAuth lives in `~/.claude.json`, which a relocated config dir may not carry — set `ANTHROPIC_API_KEY` or re-authenticate once in the fresh dir. +Project-local staged skills live in each env at `.claude/skills/`, independent of installed plugins, so they still load and the meta-check still resolves the slug under all three. -Project-local staged skills live in the isolated env at `env/.claude/skills/`, independent of installed plugins, so they still load and the meta-check still resolves the slug under all three. +**Discovery is structural.** Claude Code discovers the project-local skills present in the dispatch's cwd. Because `eval-magic run` builds each env's `.claude/skills/` *before* the dispatch runs, the staged skills are present from the start — there is no mid-session staging hazard. `--no-stage` remains for harnesses without project-local skill discovery: each `SKILL.md` is inlined into its dispatch prompt instead of staged. Regardless, run `detect-stray-writes` (folded into `ingest`) before trusting a result. -**Discovery is structural now.** Claude Code only watches skill directories that existed when the session started. Because `eval-magic run` builds `env/.claude/skills/` *before* you start the fresh session in `env/`, the staged skills are present at session start and discovered normally — there is no mid-session staging, so the old "did the dir exist when your session started?" hazard (and the build-time warning it once printed) no longer applies. `--no-stage` remains for harnesses without project-local skill discovery: each `SKILL.md` is inlined into its dispatch prompt instead of staged. Regardless, run `detect-stray-writes` (folded into `ingest`) before trusting a result. - -**Where transcripts live.** Claude Code persists subagent transcripts under `~/.claude/projects///subagents/`. `ingest` auto-resolves this from the `CLAUDE_CODE_SESSION_ID` the orchestrating session exports (deriving `` from the cwd and scanning `projects/*` if needed), so you normally don't pass `--subagents-dir` at all. When running outside that session — or to target a past session — pass `--session-id `, or override the lookup entirely with `--subagents-dir `. Besides out-of-bounds writes, `detect-stray-writes` also flags **live-source reads**: an arm whose subagent read the live skill source instead of its staged copy. That usually means the Skill tool couldn't resolve the staged slug yet and the agent improvised — fatal in revision mode, where the `old_skill` arm then sees new-skill content. Treat a flagged cell's arm as contaminated. - -**Dispatching via the Task tool.** `dispatch.json` is a top-level object (`{ skill_name, iteration, run_nonce, …, tasks: [...] }`); iterate `tasks[]`. For each task, dispatch a fresh subagent via the Task tool with the prompt `Read the file at and follow its instructions exactly.` (substituting the task's `dispatch_prompt_path`), and pass `agent_description` *verbatim* as the description — it's namespaced `::i-`, and passing it unchanged is what lets transcript correlation work. (The Task tool documents `description` as "short", but pass the full string regardless — correlation depends on the exact value.) You do **not** write `run.json`/`timing.json` yourself; the subagent writes `outputs/final-message.md`, and `ingest` (`record-runs`) assembles both records from disk. For a plan-mode-relevant skill, add `--plan-mode` to inject the shared plan-mode procedure as a `` operating-context layer. - -**Hybrid mode (`--run-mode hybrid`).** Pass `--run-mode hybrid` to dispatch each task through the `claude -p` one-shot CLI instead of in-session subagents — the same shape as Codex's hybrid flow, where an agent session orchestrates while each test/judge shells out to the CLI. `run` then prints (and `dispatch-manifest.md` / `RUNBOOK.md` carry) a `claude -p` recipe per task: +**Dispatching via `claude -p`.** `dispatch.json` is a top-level object (`{ skill_name, iteration, run_nonce, …, tasks: [...] }`); iterate `tasks[]`. `run` prints (and `dispatch-manifest.md` / `RUNBOOK.md` carry) a `claude -p` recipe per task: ```bash cd && claude -p --output-format stream-json --verbose --permission-mode acceptEdits \ @@ -320,13 +294,15 @@ cd && claude -p --output-format stream-json --verbose --permission-m 2> /claude-stderr.log ``` -Three details differ from Codex's `codex exec`: `--output-format stream-json` **requires `--verbose`** in `-p` mode; `claude` has **no `--cd` flag**, so each dispatch must run from the env dir (`cd &&`) — staged-skill discovery is cwd-relative, so getting this wrong makes the `with_skill` arm behave like `without_skill`; and there is **no `--output-last-message`**, so the final message is recovered from the stream-json `result` event rather than a file. Detach stdin with ``), it loads and enforces the hook exactly as an interactive session would (the recipe never passes `--bare`, which would skip hook discovery). A deny aborts the offending dispatch; `detect-stray-writes` (folded into `ingest`) remains the after-the-fact backstop. +Three details differ from Codex's `codex exec`: `--output-format stream-json` **requires `--verbose`** in `-p` mode; `claude` has **no `--cd` flag**, so each dispatch must run from its env dir (`cd &&`) — staged-skill discovery is cwd-relative, so getting this wrong makes the `with_skill` arm behave like `without_skill`; and there is **no `--output-last-message`**, so the final message is recovered from the stream-json `result` event rather than a file. Detach stdin with `` operating-context layer. + +Besides out-of-bounds writes, `detect-stray-writes` also flags **live-source reads**: an arm whose subagent read the live skill source instead of its staged copy. That usually means the Skill tool couldn't resolve the staged slug and the agent improvised — fatal in revision mode, where the `old_skill` arm then sees new-skill content. Treat a flagged cell's arm as contaminated. -**Headless mode (`--run-mode headless`).** The same `claude -p` dispatch as hybrid, but no agent session drives the loop — you (a human) paste each `eval-magic` command and the `claude -p` recipe yourself, ending in a written `benchmark.json`. `run` writes the same human-followed `RUNBOOK.md` (the shared headless template) into `env/`; work from that directory and copy-paste top to bottom: dispatch the tests → `ingest` → dispatch the judges → `finalize` → read the result → `teardown`. Pass `--run-mode headless` to every command of the run (the printed next steps and the runbook already carry it). `--guard` behaves exactly as it does under hybrid. +**Write guard.** `--guard` stages a `PreToolUse` hook in each env's `.claude/settings.local.json`; because every `claude -p` dispatch runs from its env (`cd `), it loads and enforces the hook (the recipe never passes `--bare`, which would skip hook discovery). A deny aborts the offending dispatch; `detect-stray-writes` (folded into `ingest`) remains the after-the-fact backstop. ### Codex -Codex's `codex --ask-for-approval never exec --json` flow powers both CLI run modes (see [Run modes](#run-modes)): **hybrid** — an agent session orchestrates while each dispatch shells out to the CLI — and **headless** (`--run-mode headless`), where eval-magic drives the whole loop with no session, writing the same human-followed `RUNBOOK.md`. A **fully-interactive** mode likely doesn't translate, since Codex dispatches via subprocess rather than in-session subagents. +Codex dispatches each task through the `codex --ask-for-approval never exec --json` one-shot CLI — the same single dispatch path as Claude Code (see [How dispatch works](#how-dispatch-works)). An agent session can drive the runbook, or you can follow it by hand; either way each test and judge shells out the same `codex exec` recipe. Pass `--harness codex`: skills stage under repo-local `.agents/skills/` (the staged skill-under-test's frontmatter `name:` is rewritten to the eval slug so Codex's repo-local discovery sees it), and `conditions.json` / `dispatch.json` record `"harness": "codex"`. Dispatch each task with a fresh `codex --ask-for-approval never exec --json` execution, capturing the event stream: diff --git a/docs/harness-parity.md b/docs/harness-parity.md index fdee31f..45a4c42 100644 --- a/docs/harness-parity.md +++ b/docs/harness-parity.md @@ -4,21 +4,15 @@ You are an agent running inside one of the eval runner's supported harnesses. Th This file covers the **`eval-magic` runner only** — the infrastructure in `eval-magic` that dispatches, records, and grades skill evals. -Read the file end-to-end before acting. Steps 4a and 4b are the source of truth for what "eval-magic parity" means today — when the runner gains a new dispatch mechanism or the `HarnessAdapter` trait gains a method, those steps are updated and this file stays evergreen. +Read the file end-to-end before acting. Steps 4a and 4b are the source of truth for what "eval-magic parity" means today — when the `HarnessAdapter` trait gains a method, those steps are updated and this file stays evergreen. -## The two parity axes +## The parity surface -Parity is organized around **run mode** — *how* an eval is dispatched — as the primary axis, with the harness adapter as the plug-in surface beneath it: +Every dispatch now rides a single mechanism: each task is delivered through a one-shot harness CLI subprocess (`claude -p`, `codex exec`), one subprocess per task, writing its events transcript to disk. **Codex** and **Claude Code** are the references. Whether an agent session or a human drives the runbook doesn't change how a single task reaches the harness — see the README's [How dispatch works](../README.md#how-dispatch-works) section. -1. **Run-mode / dispatch-mechanism parity (primary).** Every dispatch uses one of two mechanisms, defined as `DispatchMechanism` in `src/core/run_mode.rs`: - - **`InSession`** — the runner hands tasks to in-session subagents (Claude Code's Task tool). **Claude Code** is the reference. - - **`Cli`** — each task is dispatched through a one-shot harness CLI subprocess (`codex exec`). **Codex** is the reference. +Parity is therefore a single axis: **harness-adapter feature parity.** Each harness plugs into the runner through one impl of the **`HarnessAdapter`** trait in `src/adapters/harness.rs`, resolved by `adapter_for(harness)`. The trait's methods *are* the feature surface: skill-list rendering, transcript parsing, staged-skills dir, plan-mode profile, the write-guard hook, and the CLI dispatch guidance. A harness reaches parity by implementing the trait methods the CLI dispatch path consumes. `capabilities_for(harness)` (`src/core/capabilities.rs`) carries the narrow run-option capabilities the generic `run` preflight validates. - These two mechanisms underpin the three *user-facing* run modes (the `RunMode` enum, same file) in the README's [Run modes](../README.md#run-modes) section: **fully-interactive** rides on `InSession`; **headless** and **hybrid** both ride on `Cli` (`RunMode::mechanism()`). The mechanism is selected per run by the `--run-mode` flag, not fixed by the harness: `resolve_run_mode(harness, requested)` defaults the mode per harness (Claude Code → interactive, Codex/OpenCode → hybrid) and rejects combinations a harness doesn't support. A single harness can therefore wire more than one mechanism — Claude Code wires both `InSession` (interactive) and `Cli` (hybrid + headless). `capabilities_for(harness)` (same file) carries the narrow run-option capabilities the generic `run` preflight validates. - -2. **Harness-adapter feature parity (the plug-in surface).** Each harness plugs into the runner through one impl of the **`HarnessAdapter`** trait in `src/adapters/harness.rs`, resolved by `adapter_for(harness)`. The trait's methods *are* the feature surface: skill-list rendering, transcript parsing, staged-skills dir, plan-mode profile, the write-guard hook, and the `Cli`-mechanism dispatch guidance. A harness reaches parity for a mechanism by implementing the trait methods that mechanism consumes. - -A harness can wire a mechanism yet leave some adapter methods at their stub/default (lower fidelity), or implement rich adapter methods while only supporting one mechanism. Step 4 audits both axes. +A harness can wire the dispatch path yet leave some adapter methods at their stub/default (lower fidelity). Step 4 audits how complete each is. --- @@ -42,9 +36,9 @@ Read these in order. Paths are relative to the repository root. | Source | What to look for | |--------|------------------| -| `src/core/run_mode.rs` | `DispatchMechanism`, `RunMode`, `resolve_run_mode`, and `capabilities_for`. The two dispatch mechanisms, the user-facing run modes that select them (and which a harness supports), and the focused run-option capabilities generic preflight validates | -| `src/adapters/harness.rs` | The `HarnessAdapter` trait (the feature surface), the three impls, and `adapter_for`. The reference impls are `ClaudeCodeAdapter` (`InSession` *and* `Cli` via hybrid/headless) and `CodexAdapter` (`Cli`) — read the one that matches your mechanism | -| `src/adapters/claude_code_transcript.rs`, `src/adapters/claude_stream_json.rs`, and `src/adapters/codex_transcript.rs` | The reference transcript parsers. The `InSession` branch uses `parse_transcript*`; the `Cli` branch uses `parse_cli_events*` (default-delegating to `parse_transcript*`, so Codex's events parser serves both). Claude Code splits them: `parse_transcript*` parses the in-session subagent JSONL, `parse_cli_events*` parses `claude -p` stream-json (`claude_stream_json.rs`). A second harness translates its transcript shape into the same `ToolInvocation` list / `TranscriptSummary` | +| `src/core/capabilities.rs` | `HarnessRunCapabilities` and `capabilities_for` — the focused run-option capabilities the generic `run` preflight validates per harness | +| `src/adapters/harness.rs` | The `HarnessAdapter` trait (the feature surface), the three impls, and `adapter_for`. The reference impls are `ClaudeCodeAdapter` and `CodexAdapter` — read the one closest to your harness | +| `src/adapters/claude_stream_json.rs` and `src/adapters/codex_transcript.rs` | The reference CLI events parsers. `parse_cli_events*` translates each harness's `outputs/-events.jsonl` into the same `ToolInvocation` list / `TranscriptSummary` every downstream stage consumes (`claude_stream_json.rs` parses `claude -p` stream-json; Codex parses its `codex-events.jsonl`). A second harness translates its transcript shape into the same types | | `eval-magic --help` and the README's [Environment parity](../README.md#environment-parity) / [Harnesses](../README.md#harnesses) sections | The cross-harness breadcrumbs and the flag-by-flag reference. Treat the breadcrumbs as starting points, not specifications | Do not skim. The parity report you produce in Step 4 is only as good as the reference you internalized here. @@ -55,7 +49,7 @@ Do not skim. The parity report you produce in Step 4 is only as good as the refe Enumerate, using ordinary file search, what already exists for your harness. Do not rely on memory — search the working tree. Useful heuristics: -- Your harness's arm in `adapter_for`, its mode support in `resolve_run_mode`, and its `HarnessAdapter` impl in `src/adapters/harness.rs` +- Your harness's arm in `adapter_for` and its `HarnessAdapter` impl in `src/adapters/harness.rs` - The harness name anywhere inside `src/` (especially `src/adapters/`, `src/core/context.rs`) and `profiles/` - A per-harness section in the README, or tests exercising the runner for the harness (`tests/`, e.g. `tests/run/codex.rs`, `tests/run/opencode.rs`) @@ -63,14 +57,11 @@ Record every path you find. You will reference them in Step 4. --- -## Step 4a — Audit run-mode / mechanism parity - -State which `DispatchMechanism`(s) your harness supports today, per `resolve_run_mode` (the run modes it accepts), and whether each mechanism's path is end-to-end: +## Step 4a — Audit CLI-dispatch parity -- **`InSession`** consumes: `parse_transcript` / `parse_transcript_full` against a subagents-dir transcript source (no `cli_events_filename`), and the in-session next-steps guidance (mechanism-level, not adapter-supplied). -- **`Cli`** consumes these `HarnessAdapter` methods: `parse_cli_events` / `parse_cli_events_full` (the events parser, defaulting to `parse_transcript*`), `cli_events_filename` (the per-task transcript file the CLI writes), `cli_model_flag` (the harness-native model-selection flag, when supported), `cli_next_steps` (the post-`run` dispatch guidance), `cli_manifest_section` (the dispatch-manifest recipe, gated on the resolved mechanism), and `cli_judge_next_steps` (the post-`grade` / post-`ingest` judge dispatch recipe). +Confirm your harness's CLI dispatch path runs end-to-end. It consumes these `HarnessAdapter` methods: `parse_cli_events` / `parse_cli_events_full` (the events parser), `cli_events_filename` (the per-task transcript file the CLI writes), `cli_model_flag` (the harness-native model-selection flag, when supported), `cli_next_steps` (the post-`run` dispatch guidance), `cli_manifest_section` (the dispatch-manifest recipe), and `cli_judge_next_steps` (the post-`grade` / post-`ingest` judge dispatch recipe). -A harness reaches mechanism parity when its mechanism's path runs end-to-end: dispatch guidance is emitted, the transcript is found and parsed, and `record-runs` / `fill-transcripts` assemble records. The intended end state is each harness supporting every mechanism that translates to it (the README's [Run modes](../README.md#run-modes) matrix tracks current support). +A harness reaches dispatch parity when its path runs end-to-end: dispatch guidance is emitted, each task's events transcript is found and parsed, and `record-runs` / `fill-transcripts` assemble records. The README's [How dispatch works](../README.md#how-dispatch-works) matrix tracks current support. ## Step 4b — Audit harness-adapter feature parity @@ -79,16 +70,16 @@ For each `HarnessAdapter` method below, compare your harness's impl against the | Adapter capability | Trait method(s) | Reference behavior | |--------------------|-----------------|--------------------| | Realistic eval environment (skill staging) | `skills_dir`, `render_available_skills_block`, `rewrites_frontmatter_name`, `advertises_staged_slug_name`, `skill_surface_phrase`, `skill_unresolved_phrase` | Stage skills under the harness's project-local dir and render the discoverable-skills block in the harness's **native** presentation, so a dispatch reads like a real session in that harness, not an eval. Claude Code: `.claude/skills/` + `The following skills are available for use with the Skill tool:`. The `--bootstrap` `` wrapper and the slug-disambiguation framing are shared in `src/cli/run/dispatch.rs` | -| Skill-eval transcript adapter | `parse_transcript`, `parse_transcript_full` (InSession); `parse_cli_events`, `parse_cli_events_full` (Cli) | Translate the harness's persisted transcript into the same `ToolInvocation` list and `TranscriptSummary` (final message, tool invocations, deduped usage/timing) every downstream stage consumes. The Cli pair defaults to the InSession pair, so a Cli-only harness implements one parser; a harness wiring **both** mechanisms (Claude Code) implements each — `parse_transcript*` for its in-session transcript, `parse_cli_events*` for its CLI events file | -| Skill-eval auto-record (run/timing assembly) | (consumes `parse_transcript_full` / `parse_cli_events_full` + `cli_events_filename`) | `src/pipeline/record_runs.rs` assembles each task's `run.json` + `timing.json` from disk: carry-over fields from `dispatch.json`, `final_message` from `outputs/final-message.md` (falling back to the transcript's final text — the primary path for `claude -p`, which has no `--output-last-message`), and tool invocations/tokens/duration from the parsed transcript. A harness closes this gap by supplying the transcript its parser consumes (the portable fallback — hand-authored records against `run-record.schema.json` — is unchanged) | -| Cli model selection | `cli_model_flag`, `cli_next_steps`, `cli_manifest_section`, `cli_judge_next_steps` | For one-shot CLI dispatch, `run --agent-model` is rendered into the agent command recipe and `run --judge-model` becomes the default model in `judge-tasks.json`; assertion-level `llm_judge.model` remains the most specific override. Codex is the reference: `cli_model_flag` returns `-m`, agent recipes render `codex --ask-for-approval never exec ... -m `, and judge recipes read each task's resolved `model` and pass `-m "$model"` only when present | +| Skill-eval transcript adapter | `parse_cli_events`, `parse_cli_events_full` | Translate the harness's `outputs/-events.jsonl` into the same `ToolInvocation` list and `TranscriptSummary` (final message, tool invocations, deduped usage/timing) every downstream stage consumes. Claude Code parses `claude -p` stream-json (`claude_stream_json.rs`); Codex parses its `codex-events.jsonl`. A new harness implements this pair for its own events shape | +| Skill-eval auto-record (run/timing assembly) | (consumes `parse_cli_events_full` + `cli_events_filename`) | `src/pipeline/record_runs.rs` assembles each task's `run.json` + `timing.json` from disk: carry-over fields from `dispatch.json`, `final_message` from `outputs/final-message.md` (falling back to the transcript's final text — the primary path for `claude -p`, which has no `--output-last-message`), and tool invocations/tokens/duration from the parsed transcript. A harness closes this gap by supplying the transcript its parser consumes (the portable fallback — hand-authored records against `run-record.schema.json` — is unchanged) | +| CLI model selection | `cli_model_flag`, `cli_next_steps`, `cli_manifest_section`, `cli_judge_next_steps` | For one-shot CLI dispatch, `run --agent-model` is rendered into the agent command recipe and `run --judge-model` becomes the default model in `judge-tasks.json`; assertion-level `llm_judge.model` remains the most specific override. Codex is the reference: `cli_model_flag` returns `-m`, agent recipes render `codex --ask-for-approval never exec ... -m `, and judge recipes read each task's resolved `model` and pass `-m "$model"` only when present | | Eval subagent write enforcement | `install_guard` | Opt-in `--guard` stages a pre-tool hook (`src/sandbox/`) that *denies* subagent writes/installs outside the eval sandbox while dispatches run. Portable fallback for every harness: the `eval-magic detect-stray-writes` post-pass (`src/pipeline/detect_stray_writes.rs`) flags out-of-bounds writes from the parsed transcript after the fact | | Eval plan-mode operating context | `render_plan_mode_context` | Opt-in `--plan-mode` injects the shared `profiles/shared/plan-mode.md` (embedded at compile time) as a `` operating-context layer in every dispatch — identical text for every harness | -| Harness-details operator guide | (docs, not a trait method) | The README's per-harness section, e.g. [Claude Code](../README.md#claude-code-fully-wired) | +| Harness-details operator guide | (docs, not a trait method) | The README's per-harness section, e.g. [Claude Code](../README.md#claude-code) | -**Note on the transcript adapter (raised bar).** Baseline eval suites use `transcript_check` assertions — deterministic regex checks against a run's tool invocations (e.g. "a test command ran", "the sibling skill was loaded"). These only grade when `parse_transcript` is implemented for your harness. A harness without it still functions: those assertions grade as *unverifiable* and the `llm_judge` assertions carry the substantive measurement. But adapter richness is an explicit parity target, not optional polish — implementing or enriching `parse_transcript*` lets more of a baseline suite grade mechanically. Treat it as a goal to aim at, not a box already checked. +**Note on the transcript adapter (raised bar).** Baseline eval suites use `transcript_check` assertions — deterministic regex checks against a run's tool invocations (e.g. "a test command ran", "the sibling skill was loaded"). These only grade when `parse_cli_events` is implemented for your harness. A harness without it still functions: those assertions grade as *unverifiable* and the `llm_judge` assertions carry the substantive measurement. But adapter richness is an explicit parity target, not optional polish — implementing or enriching `parse_cli_events*` lets more of a baseline suite grade mechanically. Treat it as a goal to aim at, not a box already checked. -**Note on write enforcement (parity goal).** Eval subagents are instructed to write only inside their `outputs/` dir, but nothing in the portable contract *enforces* it — a misbehaving subagent can edit the real repo or install packages, silently tainting the run. Two layers address this: the portable `detect-stray-writes` post-pass (available to every harness, since it works off the same parsed transcript) and an opt-in harness-native `install_guard` that stages a pre-tool hook to *block* the write before it happens. Claude Code and Codex both wire this through their `PreToolUse` hook surfaces — for Claude Code across *both* mechanisms, since `claude -p` loads the same project-local `settings.local.json` hook from the env cwd each dispatch runs in. **Harness-level tool enforcement is an explicit parity goal, not optional polish.** A harness that can express a pre-tool guard (a hook, a permission rule, a sandboxed cwd) should wire `install_guard`; until then, `detect-stray-writes` is the honest fallback. +**Note on write enforcement (parity goal).** Eval subagents are instructed to write only inside their `outputs/` dir, but nothing in the portable contract *enforces* it — a misbehaving subagent can edit the real repo or install packages, silently tainting the run. Two layers address this: the portable `detect-stray-writes` post-pass (available to every harness, since it works off the same parsed transcript) and an opt-in harness-native `install_guard` that stages a pre-tool hook to *block* the write before it happens. Claude Code and Codex both wire this through their `PreToolUse` hook surfaces — `claude -p` loads the same project-local `settings.local.json` hook from the env cwd each dispatch runs in. **Harness-level tool enforcement is an explicit parity goal, not optional polish.** A harness that can express a pre-tool guard (a hook, a permission rule, a sandboxed cwd) should wire `install_guard`; until then, `detect-stray-writes` is the honest fallback. **Note on plan-mode fidelity (residual parity goal).** `--plan-mode` injects a shared, harness-agnostic plan-mode procedure as operating context, the closest a harness's eval runner can get to reproducing the wild failure where a real plan mode makes loading a skill feel redundant. It is **not** the real mode: it is still text the dispatched subagent reads, not a state the harness places it under, so a pass remains necessary-not-sufficient (the seeding ceiling is explained in the [`slow-powers`](https://github.com/slowdini/slow-powers) `evaluating-skills` skill). A harness that can dispatch an eval subagent *into* its own plan/research mode would close this gap; `--plan-mode` (a profile + renderer) is the approximation every harness can reach in the meantime. @@ -97,9 +88,8 @@ Surface your findings inline using this template: ``` ## Eval-Magic Parity Report: -### Run-mode / mechanism parity -- Mechanism(s) wired: (per resolve_run_mode) -- End-to-end? +### CLI-dispatch parity +- Dispatch path wired end-to-end? ### Harness-adapter feature parity - **Skill staging + native skill block** — ✅ Implemented / ⚠️ Partial / ❌ Missing / N/A @@ -127,7 +117,7 @@ The agent reports inline by default. If the user asks for a persistent artifact, ## Step 5 — Pick a gap and prep to close it -Surface the report to the user and propose **one or two** gaps worth closing this session. Bias toward the smallest gap with the highest user impact — typically a `parse_transcript` impl or an operator-guide section, not a wholesale runner rework. +Surface the report to the user and propose **one or two** gaps worth closing this session. Bias toward the smallest gap with the highest user impact — typically a `parse_cli_events` impl or an operator-guide section, not a wholesale runner rework. Once the user picks a gap: @@ -140,8 +130,8 @@ Once the user picks a gap: ## Guardrails -- **Cross-harness compatibility is enforced.** A change for your harness MUST NOT break or degrade any other harness. Keep harness-specific behavior behind your `HarnessAdapter` impl; generic run-mode code goes through the trait. +- **Cross-harness compatibility is enforced.** A change for your harness MUST NOT break or degrade any other harness. Keep harness-specific behavior behind your `HarnessAdapter` impl; generic dispatch code goes through the trait. - **One problem per PR.** A parity-closing PR should wire one capability for one harness. -- **Do not fabricate features that don't exist in any harness yet.** Parity means "catch up to a reference mechanism/adapter," not "invent something new." +- **Do not fabricate features that don't exist in any harness yet.** Parity means "catch up to a reference adapter," not "invent something new." - **Do not guess at harness-specific details.** If your harness's docs don't confirm something, ask the user before proceeding. -- **Keep this file evergreen.** If you add a dispatch mechanism (`DispatchMechanism`) or a `HarnessAdapter` method, update Step 4a / 4b here in the same PR. +- **Keep this file evergreen.** If the `HarnessAdapter` trait gains a method, update Step 4a / 4b here in the same PR. diff --git a/profiles/shared/runbook-headless.md b/profiles/shared/runbook.md similarity index 100% rename from profiles/shared/runbook-headless.md rename to profiles/shared/runbook.md diff --git a/src/adapters/claude_cli.rs b/src/adapters/claude_cli.rs index 24b22d0..15837f6 100644 --- a/src/adapters/claude_cli.rs +++ b/src/adapters/claude_cli.rs @@ -1,5 +1,4 @@ -//! Claude Code `claude -p` command rendering for dispatch guidance -//! (hybrid / headless run modes). +//! Claude Code `claude -p` command rendering for CLI dispatch guidance. //! //! Differences from the Codex recipe, all forced by the `claude` CLI: //! `--output-format stream-json` requires `--verbose` in `-p` mode; there is no diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 9269d07..4463607 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -1,4 +1,4 @@ -//! The harness adapter API — the single seam between generic run-mode code and +//! The harness adapter API — the single seam between generic dispatch code and //! harness-specific behavior. //! //! Every harness-specific concern hangs off the [`HarnessAdapter`] trait: how @@ -27,7 +27,7 @@ use super::{ render_opencode_available_skills_block, }; -/// The behavior that varies by harness. Generic run-mode code depends on this +/// The behavior that varies by harness. Generic dispatch code depends on this /// trait, never on a concrete harness variant. pub trait HarnessAdapter { /// The kebab-case identifier used in CLI flags, `dispatch.json`, and the @@ -135,8 +135,7 @@ pub trait HarnessAdapter { /// The shared (human-followed) `RUNBOOK.md` template used by every run, /// regardless of harness (Claude Code, Codex, OpenCode). -pub const HEADLESS_RUNBOOK_TEMPLATE: &str = - include_str!("../../profiles/shared/runbook-headless.md"); +pub const RUNBOOK_TEMPLATE: &str = include_str!("../../profiles/shared/runbook.md"); pub struct ClaudeCodeAdapter; pub struct CodexAdapter; @@ -203,7 +202,7 @@ impl HarnessAdapter for ClaudeCodeAdapter { } fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { Some(vec![ - "After all dispatches (Claude Code hybrid):".to_string(), + "After all dispatches (Claude Code):".to_string(), String::new(), "Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with ``. @@ -20,7 +20,7 @@ pub mod plugin_shadow; pub use harness::{ ClaudeCodeAdapter, CliDispatchContext, CliJudgeContext, CliManifestContext, CodexAdapter, - HEADLESS_RUNBOOK_TEMPLATE, HarnessAdapter, OpenCodeAdapter, adapter_for, + HarnessAdapter, OpenCodeAdapter, RUNBOOK_TEMPLATE, adapter_for, }; pub use claude_code_session::{render_available_skills_block, render_plan_mode_context}; diff --git a/src/adapters/plugin_shadow.rs b/src/adapters/plugin_shadow.rs index ffce295..9286e5d 100644 --- a/src/adapters/plugin_shadow.rs +++ b/src/adapters/plugin_shadow.rs @@ -1,13 +1,12 @@ //! Plugin-shadow detector (Claude Code). //! -//! The runner stages eval skills into the -//! project-local `.claude/skills/` dir, but eval subagents are dispatched via the -//! Task tool and run in-process — so they ALSO inherit whatever skills the -//! orchestrator session loaded from installed plugins and the global skills dir. -//! When a staged skill name collides with one of those, both copies are -//! discoverable and the with/without comparison is contaminated. The runner -//! cannot unload a plugin from a running session, so this module only *detects -//! and reports* the overlap, reading declared settings as a best-effort proxy. +//! The runner stages eval skills into each dispatch's project-local +//! `.claude/skills/` dir, but every `claude -p` dispatch ALSO loads the user/global +//! plugins and the global skills dir from its Claude config. When a staged skill +//! name collides with one of those, both copies are discoverable and the +//! with/without comparison is contaminated. The runner cannot strip an installed +//! plugin from a dispatch, so this module only *detects and reports* the overlap, +//! reading declared settings as a best-effort proxy. use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -15,7 +14,7 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -const ISOLATION_DOC: &str = "docs/harness-claude-code.md → \"Isolating from installed plugins\""; +const ISOLATION_DOC: &str = "README.md → Claude Code → \"Isolating from installed plugins\""; /// A staged skill that is also discoverable from the live environment. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -206,9 +205,9 @@ pub fn shadow_validity_warnings(report: &PluginShadowReport) -> Vec { .iter() .map(|s| { format!( - "staged skill '{}' is also provided by {} — eval subagents could discover both \ - copies, so with/without results may be contaminated. Re-run from an isolated \ - session (see {}).", + "staged skill '{}' is also provided by {} — each claude -p dispatch could discover \ + both copies, so with/without results may be contaminated. Isolate each dispatch's \ + Claude config (see {}).", s.skill_name(), s.source_label(), ISOLATION_DOC @@ -231,21 +230,19 @@ pub fn format_shadow_banner(report: &PluginShadowReport) -> String { lines.push(format!(" • {} — {}", s.skill_name(), s.source_label())); } lines.push( - " Eval subagents (dispatched via the Task tool) inherit this session's plugins," - .to_string(), - ); - lines.push( - " so both the staged copy and the installed copy are discoverable — the".to_string(), + " Each `claude -p` dispatch loads your user/global plugins and skills, so".to_string(), ); + lines.push(" both the staged copy and the installed copy are discoverable — the".to_string()); lines.push( " with/without comparison may be contaminated and the control arm is not truly" .to_string(), ); lines.push( - " skill-absent. The runner cannot unload a plugin from a running session.".to_string(), + " skill-absent. The runner cannot strip an installed plugin from the dispatch." + .to_string(), ); lines.push(format!( - " Re-run from an isolated session — see {ISOLATION_DOC}." + " Isolate each dispatch's Claude config — see {ISOLATION_DOC}." )); lines.join("\n") } diff --git a/src/cli/args.rs b/src/cli/args.rs index e68c2e4..a5d8c50 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -6,7 +6,7 @@ use clap::{Args, Parser, Subcommand}; -use crate::core::{Harness, RunMode}; +use crate::core::Harness; /// Run skill evals — measure whether an agent skill actually shifts behavior. /// @@ -81,17 +81,6 @@ pub struct CommonArgs { /// `--guard` are not yet wired for OpenCode. #[arg(long)] pub harness: Option, - /// Run mode: `hybrid` (an agent orchestrates while each dispatch shells out to - /// the harness CLI) or `headless` (CLI-only, no session). - /// - /// Every harness defaults to `hybrid`. Both modes dispatch through the harness - /// CLI (`claude -p`, `codex exec`) and read each task's - /// `outputs/-events.jsonl`; they differ only in whether an agent or a - /// human drives the loop. Claude Code and Codex wire both modes; OpenCode wires - /// `hybrid` only. Pass the same value to every command of a run; the printed - /// next-step commands already carry it. - #[arg(long)] - pub run_mode: Option, /// Workspace directory (defaults to `/.eval-magic`). /// /// The artifact root. Pass the same value to every command of a run, including diff --git a/src/cli/commands/pipeline.rs b/src/cli/commands/pipeline.rs index 87fca42..2c424b6 100644 --- a/src/cli/commands/pipeline.rs +++ b/src/cli/commands/pipeline.rs @@ -45,7 +45,6 @@ fn run_step(step: &run::steps::StepCommand) -> anyhow::Result<()> { iteration: Some(step.iteration), mode: None, harness: Some(step.harness), - run_mode: Some(step.run_mode), workspace_dir: step.workspace_dir.clone(), only: None, skip: None, @@ -75,7 +74,6 @@ pub(crate) fn run_ingest(args: CommonArgs) -> anyhow::Result<()> { skill: args.skill.as_deref(), iteration, harness: ctx.harness, - run_mode: ctx.run_mode, workspace_dir: args.workspace_dir.as_deref(), }); if let Some(failed) = run::steps::run_steps(&steps, run_step) { @@ -119,7 +117,6 @@ pub(crate) fn run_finalize(args: CommonArgs) -> anyhow::Result<()> { skill: args.skill.as_deref(), iteration, harness: ctx.harness, - run_mode: ctx.run_mode, workspace_dir: args.workspace_dir.as_deref(), }); if let Some(failed) = run::steps::run_steps(&steps, run_step) { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index fb7b44d..483ffaa 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -36,7 +36,6 @@ fn dispatch(command: Option) -> anyhow::Result<()> { iteration: None, mode: None, harness: None, - run_mode: None, workspace_dir: None, only: None, skip: None, @@ -93,7 +92,6 @@ pub(crate) fn run_context_with_bootstrap( bootstrap, workspace_dir: args.workspace_dir.clone(), harness: args.harness, - run_mode: args.run_mode, cwd: None, })?) } @@ -119,11 +117,10 @@ pub(crate) fn parse_id_list(v: Option<&str>) -> Option> { /// and the iteration tree above the env would not resolve. pub(crate) fn command_target_args(ctx: &RunContext) -> String { format!( - " --skill-dir {} --skill {} --workspace-dir {} --run-mode {}", + " --skill-dir {} --skill {} --workspace-dir {}", ctx.skill_dir.display(), ctx.skill_name, ctx.workspace_root.display(), - ctx.run_mode.as_str() ) } diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index 0121503..5aebf6a 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -1,4 +1,4 @@ -//! The `run` orchestrator and its run-mode variants. +//! The `run` orchestrator. //! //! Split into focused sub-orchestrators: //! diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index 48c5a90..a9618a1 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -53,7 +53,6 @@ pub(super) fn write_dispatch( ], timestamp: now_iso8601(), harness: Some(ctx.harness), - run_mode: Some(ctx.run_mode), run_nonce: Some(r.run_nonce.clone()), runs: Some(opts.runs), agent_model: opts.agent_model.map(str::to_owned), @@ -255,7 +254,6 @@ pub(super) fn write_dispatch( "label": conditions.label, "conditions": conditions.conditions, "harness": ctx.harness, - "run_mode": ctx.run_mode, "tasks": tasks, }); // The isolation-batch plan the executing session/human follows: which evals diff --git a/src/cli/run/runbook.rs b/src/cli/run/runbook.rs index d0265c6..aaa35e9 100644 --- a/src/cli/run/runbook.rs +++ b/src/cli/run/runbook.rs @@ -3,8 +3,8 @@ //! //! The runbook turns the prep session's "what to do next" guidance into a file //! a human at a terminal can read end-to-end: "Read and follow RUNBOOK.md". Every -//! run uses the shared [`HEADLESS_RUNBOOK_TEMPLATE`], whose harness-specific -//! dispatch + judge recipes come from the adapter's CLI generators. +//! run uses the shared [`RUNBOOK_TEMPLATE`], whose harness-specific dispatch + +//! judge recipes come from the adapter's CLI generators. //! //! The prose skeleton lives in `profiles/` (checked in) and carries `{{TOKEN}}` //! placeholders the renderer fills with run-specific values. The generated @@ -12,9 +12,7 @@ use std::path::Path; -use crate::adapters::{ - CliDispatchContext, CliJudgeContext, HEADLESS_RUNBOOK_TEMPLATE, adapter_for, -}; +use crate::adapters::{CliDispatchContext, CliJudgeContext, RUNBOOK_TEMPLATE, adapter_for}; use crate::core::{Harness, Mode}; use super::util::{harness_label, mode_str}; @@ -39,14 +37,14 @@ pub(crate) struct RunbookContext<'a> { pub agent_model: Option<&'a str>, } -/// Render `RUNBOOK.md` for a run: fill the shared headless template's +/// Render `RUNBOOK.md` for a run: fill the shared runbook template's /// `{{TOKEN}}` placeholders with run-specific values. The harness-specific /// dispatch + judge recipes come from the adapter's CLI generators, so the /// runbook stays in lockstep with `dispatch-manifest.md` and the printed next /// steps; pipeline commands carry `--harness`. pub(crate) fn build_runbook(ctx: &RunbookContext) -> String { let adapter = adapter_for(ctx.harness); - let template = HEADLESS_RUNBOOK_TEMPLATE; + let template = RUNBOOK_TEMPLATE; let iteration = ctx.iteration.to_string(); let num_tasks = ctx.num_tasks.to_string(); @@ -150,7 +148,7 @@ mod tests { use std::path::PathBuf; #[test] - fn headless_runbook_is_human_followed_cli_recipe() { + fn runbook_is_human_followed_cli_recipe() { let dir = PathBuf::from("/work/.eval-magic/widget-skill/iteration-2"); let ctx = RunbookContext { harness: Harness::Codex, @@ -175,7 +173,7 @@ mod tests { "names both conditions: {book}" ); - // Human-followed framing (the shared headless template). + // Human-followed framing (the shared runbook template). assert!( book.contains("human driving"), "frames the run for a human at a terminal: {book}" diff --git a/src/cli/run/steps.rs b/src/cli/run/steps.rs index c213262..cd910d5 100644 --- a/src/cli/run/steps.rs +++ b/src/cli/run/steps.rs @@ -13,7 +13,7 @@ //! parameter; the production runner — which maps each [`StepKind`] to its stage //! handler — lives in [`crate::cli`] alongside those handlers. -use crate::core::{Harness, RunMode}; +use crate::core::Harness; /// Which post-dispatch stage a [`StepCommand`] runs. The production runner /// matches on this to call the corresponding handler; tests assert on it. @@ -37,9 +37,6 @@ pub struct StepCommand { pub skill: Option, pub iteration: u32, pub harness: Harness, - /// The run mode, re-derived at each stage. Round-trips through `CommonArgs` - /// exactly like `harness`, so ingest sub-stages don't silently re-default it. - pub run_mode: RunMode, pub workspace_dir: Option, } @@ -50,7 +47,6 @@ pub struct StepParams<'a> { pub skill: Option<&'a str>, pub iteration: u32, pub harness: Harness, - pub run_mode: RunMode, pub workspace_dir: Option<&'a str>, } @@ -61,7 +57,6 @@ impl Default for StepParams<'_> { skill: None, iteration: 0, harness: Harness::ClaudeCode, - run_mode: RunMode::Hybrid, workspace_dir: None, } } @@ -76,7 +71,6 @@ impl StepParams<'_> { skill: self.skill.map(str::to_string), iteration: self.iteration, harness: self.harness, - run_mode: self.run_mode, workspace_dir: self.workspace_dir.map(str::to_string), } } @@ -169,7 +163,6 @@ mod tests { skill: Some("mr-review"), iteration: 2, harness: Harness::Codex, - run_mode: RunMode::Hybrid, ..Default::default() }); assert_eq!( @@ -208,7 +201,6 @@ mod tests { skill: None, iteration: 0, harness: Harness::ClaudeCode, - run_mode: RunMode::Hybrid, workspace_dir: None, } } diff --git a/src/cli/run/util.rs b/src/cli/run/util.rs index 39d74c5..eaa1aba 100644 --- a/src/cli/run/util.rs +++ b/src/cli/run/util.rs @@ -137,11 +137,11 @@ pub(crate) fn harness_label(harness: Harness) -> &'static str { #[cfg(test)] mod tests { use super::*; - use crate::core::{DetectInput, RunMode, detect_run_context}; + use crate::core::{DetectInput, detect_run_context}; use std::fs; - /// Build a `RunContext` for `harness`/`run_mode` against a throwaway skill dir. - fn ctx_for(harness: Harness, run_mode: RunMode) -> (tempfile::TempDir, RunContext) { + /// Build a `RunContext` for `harness` against a throwaway skill dir. + fn ctx_for(harness: Harness) -> (tempfile::TempDir, RunContext) { let tmp = tempfile::TempDir::new().unwrap(); let skill = tmp.path().join("widget"); fs::create_dir_all(&skill).unwrap(); @@ -153,7 +153,6 @@ mod tests { let ctx = detect_run_context(DetectInput { skill: Some(skill.display().to_string()), harness: Some(harness), - run_mode: Some(run_mode), cwd: Some(tmp.path().to_path_buf()), ..Default::default() }) @@ -162,20 +161,10 @@ mod tests { } #[test] - fn claude_hybrid_allows_guard() { + fn claude_allows_guard() { // `claude -p` loads the project `.claude/settings.local.json` PreToolUse - // hook from its cwd, so the write guard fires under Cli dispatch too. - let (_t, ctx) = ctx_for(Harness::ClaudeCode, RunMode::Hybrid); - let opts = RunOptions { - guard: true, - ..Default::default() - }; - assert!(validate_harness_run_options(&opts, &ctx).is_ok()); - } - - #[test] - fn claude_headless_allows_guard() { - let (_t, ctx) = ctx_for(Harness::ClaudeCode, RunMode::Headless); + // hook from its cwd, so the write guard fires under CLI dispatch. + let (_t, ctx) = ctx_for(Harness::ClaudeCode); let opts = RunOptions { guard: true, ..Default::default() diff --git a/src/core/capabilities.rs b/src/core/capabilities.rs new file mode 100644 index 0000000..160b2e8 --- /dev/null +++ b/src/core/capabilities.rs @@ -0,0 +1,66 @@ +//! Per-harness run-option capabilities. +//! +//! Every dispatch rides a single mechanism: each task is delivered through a +//! one-shot harness CLI subprocess (`claude -p`, `codex exec`). What still varies +//! by harness is which *run options* the generic `run` preflight may accept — +//! captured here as a narrow capability table, independent of the comparison +//! [`Mode`](crate::core::Mode) (`new-skill` / `revision`), which selects the two +//! conditions being compared. + +use crate::core::Harness; + +/// Run-option support for a harness's dispatch path. +/// +/// This is intentionally narrower than full harness parity: it only describes +/// options the generic `run` preflight must accept or reject before the build +/// sequence starts. Harness-specific behavior still lives behind the adapter. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct HarnessRunCapabilities { + pub supports_guard: bool, + pub supports_bootstrap_with_no_stage: bool, + pub supports_stage_name_with_no_stage: bool, +} + +/// The focused capability table for generic `run` option validation. +pub fn capabilities_for(harness: Harness) -> HarnessRunCapabilities { + match harness { + Harness::ClaudeCode => HarnessRunCapabilities { + supports_guard: true, + supports_bootstrap_with_no_stage: true, + supports_stage_name_with_no_stage: true, + }, + Harness::Codex => HarnessRunCapabilities { + supports_guard: true, + supports_bootstrap_with_no_stage: false, + supports_stage_name_with_no_stage: false, + }, + Harness::OpenCode => HarnessRunCapabilities { + supports_guard: false, + supports_bootstrap_with_no_stage: true, + supports_stage_name_with_no_stage: true, + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capabilities_capture_run_option_support_by_harness() { + let claude = capabilities_for(Harness::ClaudeCode); + assert!(claude.supports_guard); + assert!(claude.supports_bootstrap_with_no_stage); + assert!(claude.supports_stage_name_with_no_stage); + + let codex = capabilities_for(Harness::Codex); + assert!(codex.supports_guard); + assert!(!codex.supports_bootstrap_with_no_stage); + assert!(!codex.supports_stage_name_with_no_stage); + + let opencode = capabilities_for(Harness::OpenCode); + assert!(!opencode.supports_guard); + assert!(opencode.supports_bootstrap_with_no_stage); + assert!(opencode.supports_stage_name_with_no_stage); + } +} diff --git a/src/core/context.rs b/src/core/context.rs index 9939f8c..e67bb72 100644 --- a/src/core/context.rs +++ b/src/core/context.rs @@ -9,8 +9,6 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; -use crate::core::run_mode::{RunMode, resolve_run_mode}; - /// The agent harness an eval runs against. Single source of truth, shared with /// the CLI layer (it derives `clap::ValueEnum` so flags can parse it directly). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, clap::ValueEnum)] @@ -39,9 +37,6 @@ pub struct RunContext { pub stage_root: PathBuf, pub bootstrap_path: Option, pub harness: Harness, - /// The resolved run mode (the dispatch mechanism + who drives the loop). - /// Resolved per harness from the `--run-mode` flag in [`detect_run_context`]. - pub run_mode: RunMode, } /// Already-parsed flag values handed to [`detect_run_context`]. `clap` owns the @@ -54,7 +49,6 @@ pub struct DetectInput { pub bootstrap: Option, pub workspace_dir: Option, pub harness: Option, - pub run_mode: Option, pub cwd: Option, } @@ -77,8 +71,6 @@ pub enum ContextError { SkillNotFound(String), #[error("--bootstrap file not found: {0}")] BootstrapNotFound(String), - #[error("{0}")] - UnsupportedRunMode(String), #[error("io error: {0}")] Io(#[from] std::io::Error), } @@ -212,8 +204,6 @@ pub fn detect_run_context(input: DetectInput) -> Result Result RunMode { - RunMode::Hybrid - } - - /// The kebab-case identifier (matches the `--run-mode` flag values and the - /// serialized form in `conditions.json`). - pub fn as_str(self) -> &'static str { - match self { - RunMode::Hybrid => "hybrid", - RunMode::Headless => "headless", - } - } -} - -/// Resolve the effective run mode for a harness, defaulting per harness when -/// unspecified and rejecting unsupported `(harness, mode)` combinations. The -/// `Err` string is operator-facing. -pub fn resolve_run_mode(harness: Harness, requested: Option) -> Result { - let mode = requested.unwrap_or_else(|| RunMode::default_for(harness)); - let supported: &[RunMode] = match harness { - // Claude Code and Codex both wire the CLI mechanism, so both modes apply - // (hybrid is agent-driven, headless human-driven). - Harness::ClaudeCode | Harness::Codex => &[RunMode::Hybrid, RunMode::Headless], - // OpenCode's CLI path is only partially wired (no transcript ingest), so - // only hybrid is advertised for now. - Harness::OpenCode => &[RunMode::Hybrid], - }; - if supported.contains(&mode) { - return Ok(mode); - } - let supported_list = supported - .iter() - .map(|m| m.as_str()) - .collect::>() - .join(", "); - Err(format!( - "--run-mode {} is not supported for --harness {}; supported: {}", - mode.as_str(), - harness_label(harness), - supported_list, - )) -} - -/// The kebab-case CLI identifier for a harness (for operator-facing messages). -fn harness_label(harness: Harness) -> &'static str { - match harness { - Harness::ClaudeCode => "claude-code", - Harness::Codex => "codex", - Harness::OpenCode => "opencode", - } -} - -/// Run-option support for a harness's currently wired dispatch mechanism. -/// -/// This is intentionally narrower than full harness parity: it only describes -/// options the generic `run` preflight must accept or reject before the build -/// sequence starts. Harness-specific behavior still lives behind the adapter. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct HarnessRunCapabilities { - pub supports_guard: bool, - pub supports_bootstrap_with_no_stage: bool, - pub supports_stage_name_with_no_stage: bool, -} - -/// The focused capability table for generic `run` option validation. -pub fn capabilities_for(harness: Harness) -> HarnessRunCapabilities { - match harness { - Harness::ClaudeCode => HarnessRunCapabilities { - supports_guard: true, - supports_bootstrap_with_no_stage: true, - supports_stage_name_with_no_stage: true, - }, - Harness::Codex => HarnessRunCapabilities { - supports_guard: true, - supports_bootstrap_with_no_stage: false, - supports_stage_name_with_no_stage: false, - }, - Harness::OpenCode => HarnessRunCapabilities { - supports_guard: false, - supports_bootstrap_with_no_stage: true, - supports_stage_name_with_no_stage: true, - }, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn capabilities_capture_run_option_support_by_harness() { - let claude = capabilities_for(Harness::ClaudeCode); - assert!(claude.supports_guard); - assert!(claude.supports_bootstrap_with_no_stage); - assert!(claude.supports_stage_name_with_no_stage); - - let codex = capabilities_for(Harness::Codex); - assert!(codex.supports_guard); - assert!(!codex.supports_bootstrap_with_no_stage); - assert!(!codex.supports_stage_name_with_no_stage); - - let opencode = capabilities_for(Harness::OpenCode); - assert!(!opencode.supports_guard); - assert!(opencode.supports_bootstrap_with_no_stage); - assert!(opencode.supports_stage_name_with_no_stage); - } - - #[test] - fn run_mode_defaults_to_hybrid_for_every_harness() { - assert_eq!(RunMode::default_for(Harness::ClaudeCode), RunMode::Hybrid); - assert_eq!(RunMode::default_for(Harness::Codex), RunMode::Hybrid); - assert_eq!(RunMode::default_for(Harness::OpenCode), RunMode::Hybrid); - } - - #[test] - fn resolve_run_mode_defaults_when_unspecified() { - assert_eq!( - resolve_run_mode(Harness::ClaudeCode, None).unwrap(), - RunMode::Hybrid - ); - assert_eq!( - resolve_run_mode(Harness::Codex, None).unwrap(), - RunMode::Hybrid - ); - } - - #[test] - fn resolve_run_mode_accepts_claude_hybrid() { - assert_eq!( - resolve_run_mode(Harness::ClaudeCode, Some(RunMode::Hybrid)).unwrap(), - RunMode::Hybrid - ); - } - - #[test] - fn resolve_run_mode_rejects_headless_for_opencode() { - let err = resolve_run_mode(Harness::OpenCode, Some(RunMode::Headless)).unwrap_err(); - assert!(err.contains("headless"), "message was: {err}"); - assert!(err.contains("opencode"), "message was: {err}"); - } - - #[test] - fn resolve_run_mode_accepts_claude_headless() { - assert_eq!( - resolve_run_mode(Harness::ClaudeCode, Some(RunMode::Headless)).unwrap(), - RunMode::Headless - ); - } - - #[test] - fn resolve_run_mode_accepts_codex_headless() { - assert_eq!( - resolve_run_mode(Harness::Codex, Some(RunMode::Headless)).unwrap(), - RunMode::Headless - ); - } - - #[test] - fn run_mode_serde_roundtrips_kebab_case() { - assert_eq!( - serde_json::to_string(&RunMode::Hybrid).unwrap(), - "\"hybrid\"" - ); - let parsed: RunMode = serde_json::from_str("\"headless\"").unwrap(); - assert_eq!(parsed, RunMode::Headless); - } -} diff --git a/src/core/types.rs b/src/core/types.rs index bbd300b..f2a6b3b 100644 --- a/src/core/types.rs +++ b/src/core/types.rs @@ -10,7 +10,6 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::core::context::Harness; -use crate::core::run_mode::RunMode; /// Meta-assertion id reserved for the skill-invocation check. pub const SKILL_INVOKED_META_ID: &str = "__skill_invoked"; @@ -124,10 +123,6 @@ pub struct ConditionsRecord { pub timestamp: String, #[serde(skip_serializing_if = "Option::is_none")] pub harness: Option, - /// The run mode this iteration was built with (provenance + recoverability). - /// `None` on older artifacts written before run-mode selection existed. - #[serde(skip_serializing_if = "Option::is_none")] - pub run_mode: Option, /// Per-run nonce; namespaces dispatch descriptions so they stay unique across /// iterations of the same skill. #[serde(skip_serializing_if = "Option::is_none")] @@ -406,7 +401,6 @@ mod tests { conditions: vec![], timestamp: "2026-06-08T00:00:00Z".into(), harness: Some(Harness::ClaudeCode), - run_mode: Some(RunMode::Hybrid), run_nonce: None, runs: None, agent_model: None, @@ -419,7 +413,6 @@ mod tests { out.get("harness"), Some(&Value::String("claude-code".into())) ); - assert_eq!(out.get("run_mode"), Some(&Value::String("hybrid".into()))); // Absent optionals omitted. assert!(out.get("baseline").is_none()); assert!(out.get("run_nonce").is_none()); diff --git a/src/pipeline/fill_transcripts.rs b/src/pipeline/fill_transcripts.rs index aed05cb..afe5df0 100644 --- a/src/pipeline/fill_transcripts.rs +++ b/src/pipeline/fill_transcripts.rs @@ -188,7 +188,7 @@ mod tests { // --- fillTranscripts --- #[test] - fn fills_a_claude_hybrid_run_record_from_outputs_events() { + fn fills_a_claude_run_record_from_outputs_events() { let root = TempDir::new().unwrap(); let iteration_dir: PathBuf = root.path().join("iter-claude-fill"); let cond_dir = iteration_dir.join("eval-crash").join("with_skill"); @@ -202,8 +202,7 @@ mod tests { "mode": "new-skill", "conditions": [{"name": "with_skill", "skill_path": "/skill/SKILL.md"}], "timestamp": "2026-06-07T00:00:00.000Z", - "harness": "claude-code", - "run_mode": "hybrid" + "harness": "claude-code" }) .to_string(), ) diff --git a/src/pipeline/record_runs.rs b/src/pipeline/record_runs.rs index b0cc3e2..f1b0661 100644 --- a/src/pipeline/record_runs.rs +++ b/src/pipeline/record_runs.rs @@ -819,7 +819,7 @@ mod tests { } #[test] - fn assembles_claude_hybrid_records_from_each_tasks_events() { + fn assembles_claude_records_from_each_tasks_events() { let root = TempDir::new().unwrap(); let iter = dirs(&root); let paths = write_iteration( @@ -875,7 +875,7 @@ mod tests { } #[test] - fn claude_hybrid_warning_points_at_events_file() { + fn claude_warning_points_at_events_file() { let result = RecordRunsResult { recorded: 2, missing_transcript: 2, @@ -884,11 +884,11 @@ mod tests { let warning = result.transcript_warning(Harness::ClaudeCode).unwrap(); assert!( warning.contains("claude-events.jsonl"), - "names the Claude hybrid source: {warning}" + "names the Claude CLI events source: {warning}" ); assert!( !warning.contains("agent_description"), - "hybrid doesn't use agent_description: {warning}" + "CLI dispatch doesn't use agent_description: {warning}" ); } } diff --git a/tests/run/claude_cli.rs b/tests/run/claude_cli.rs index 69f551f..4ec48bd 100644 --- a/tests/run/claude_cli.rs +++ b/tests/run/claude_cli.rs @@ -1,13 +1,12 @@ -//! Claude Code CLI run modes (`--run-mode hybrid` / `headless`): `claude -p` -//! stream-json dispatch guidance, run-mode persistence + defaulting, the -//! human-followed runbook, and the write guard under Cli dispatch. +//! Claude Code CLI dispatch: `claude -p` stream-json dispatch guidance, the +//! human-followed runbook, and the write guard under CLI dispatch. use crate::helpers::*; use predicates::str::contains; use std::fs; #[test] -fn claude_hybrid_dispatch_guidance_uses_claude_p() { +fn claude_dispatch_guidance_uses_claude_p() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); let assert = skill_eval() @@ -21,8 +20,6 @@ fn claude_hybrid_dispatch_guidance_uses_claude_p() { "new-skill", "--harness", "claude-code", - "--run-mode", - "hybrid", ]) .assert() .success(); @@ -41,13 +38,10 @@ fn claude_hybrid_dispatch_guidance_uses_claude_p() { let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); assert_eq!(conditions["harness"], "claude-code"); - assert_eq!(conditions["run_mode"], "hybrid"); - let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); - assert_eq!(dispatch["run_mode"], "hybrid"); } #[test] -fn claude_hybrid_dispatch_guidance_includes_agent_model_when_provided() { +fn claude_dispatch_guidance_includes_agent_model_when_provided() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); let assert = skill_eval() @@ -59,8 +53,6 @@ fn claude_hybrid_dispatch_guidance_includes_agent_model_when_provided() { "mr-review", "--harness", "claude-code", - "--run-mode", - "hybrid", "--agent-model", "opus", ]) @@ -72,7 +64,7 @@ fn claude_hybrid_dispatch_guidance_includes_agent_model_when_provided() { } #[test] -fn claude_hybrid_runbook_is_human_followed_cli_recipe() { +fn claude_run_writes_human_followed_runbook() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); skill_eval() @@ -84,62 +76,21 @@ fn claude_hybrid_runbook_is_human_followed_cli_recipe() { "mr-review", "--harness", "claude-code", - "--run-mode", - "hybrid", "--dry-run", ]) .assert() .success(); - // Cli dispatches from per-(group, condition) envs, so the human-followed - // runbook lives in the iteration dir, not a single env/. - let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); - assert!( - runbook.contains("human driving"), - "hybrid uses the human-followed template: {runbook}" - ); - assert!( - runbook.contains("claude -p"), - "carries the claude -p dispatch recipe: {runbook}" - ); -} - -#[test] -fn claude_headless_records_mode_and_human_runbook() { - let tmp = tempfile::TempDir::new().unwrap(); - let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - skill_eval() - .current_dir(&cwd) - .args(["run", "--skill-dir"]) - .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "claude-code", - "--run-mode", - "headless", - "--dry-run", - ]) - .assert() - .success(); - - // Headless rides the same Cli mechanism as hybrid; the run mode is persisted - // distinctly so every post-dispatch command can carry it. - let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); - assert_eq!(conditions["run_mode"], "headless"); - let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); - assert_eq!(dispatch["run_mode"], "headless"); let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); assert!(manifest.contains("claude -p --output-format stream-json")); - // The runbook is the shared human-followed template carrying the claude -p - // recipe and headless-threaded pipeline commands. Cli has no single env/, so - // it lives in the iteration dir. + // Each task dispatches from its own per-(group, condition) env, so the shared + // human-followed runbook lives in the iteration dir, above those envs, and + // carries the claude -p recipe plus the --harness-threaded pipeline commands. let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); assert!( runbook.contains("human driving"), - "headless uses the human-followed template: {runbook}" + "uses the human-followed template: {runbook}" ); assert!( runbook.contains("claude -p"), @@ -149,10 +100,6 @@ fn claude_headless_records_mode_and_human_runbook() { runbook.contains("--harness claude-code"), "pipeline commands carry --harness claude-code: {runbook}" ); - assert!( - runbook.contains("--run-mode headless"), - "pipeline commands carry the headless run mode: {runbook}" - ); assert!( !runbook.contains("{{"), "no unsubstituted tokens: {runbook}" @@ -160,33 +107,22 @@ fn claude_headless_records_mode_and_human_runbook() { } #[test] -fn claude_hybrid_record_runs_does_not_require_a_session_id() { - // Regression: hybrid/headless ride the Cli mechanism and read each task's - // claude-events.jsonl, never the in-session subagents dir. Resolving that dir - // is gated on the dispatch mechanism, not the harness, so `record-runs` in - // hybrid mode must NOT bail on a missing CLAUDE_CODE_SESSION_ID — the way the - // old harness-keyed gate did for `--harness claude-code`. This is the - // documented headless path (no session at all). +fn claude_record_runs_does_not_require_a_session_id() { + // Regression: CLI dispatch reads each task's claude-events.jsonl, never an + // in-session subagents dir, so `record-runs --harness claude-code` must NOT + // bail on a missing CLAUDE_CODE_SESSION_ID. let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); skill_eval() .current_dir(&cwd) .args(["run", "--skill-dir"]) .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "claude-code", - "--run-mode", - "hybrid", - ]) + .args(["--skill", "mr-review", "--harness", "claude-code"]) .assert() .success(); - // No session id in the environment, and none passed — the pre-fix code aborted - // here with "could not auto-resolve the subagents dir". The fix returns early - // for the Cli mechanism, so record-runs proceeds to its summary. + // No session id in the environment, and none passed — record-runs proceeds to + // its summary rather than aborting on an unresolved subagents dir. skill_eval() .current_dir(&cwd) .env_remove("CLAUDE_CODE_SESSION_ID") @@ -194,7 +130,7 @@ fn claude_hybrid_record_runs_does_not_require_a_session_id() { .arg(&skill_dir) .args(["--skill", "mr-review", "--workspace-dir"]) .arg(cwd.join(".eval-magic")) - .args(["--harness", "claude-code", "--run-mode", "hybrid"]) + .args(["--harness", "claude-code"]) .assert() .success() .stdout(contains("Recorded:")); @@ -213,8 +149,6 @@ fn claude_cli_guard_installs_project_hook() { "mr-review", "--harness", "claude-code", - "--run-mode", - "hybrid", "--guard", ]) .assert() @@ -302,19 +236,81 @@ fn cli_plugin_shadow_preflight_reads_per_env_project_settings() { .env("CLAUDE_CONFIG_DIR", &config) .args(["run", "--skill-dir"]) .arg(&skill_dir) + .args(["--skill", "mr-review", "--harness", "claude-code"]) + .assert() + .success(); + + assert!( + iteration_dir(&cwd).join("plugin-shadow.json").exists(), + "preflight detected the project-enabled plugin shadow by scanning the staged env" + ); +} + +#[test] +fn run_omits_run_mode_from_every_artifact_and_command() { + // The run-mode vocabulary is retired: there is one CLI dispatch path, so no + // artifact records a run mode and no printed/threaded command carries the flag. + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + let assert = skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) .args([ "--skill", "mr-review", "--harness", "claude-code", - "--run-mode", - "hybrid", + "--dry-run", ]) .assert() .success(); + let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); + assert!( + !stdout.contains("--run-mode"), + "printed next-step commands carry no --run-mode: {stdout}" + ); + let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); assert!( - iteration_dir(&cwd).join("plugin-shadow.json").exists(), - "preflight detected the project-enabled plugin shadow by scanning the staged env" + conditions.get("run_mode").is_none(), + "conditions.json carries no run_mode: {conditions}" + ); + let dispatch = read_json(&iteration_dir(&cwd).join("dispatch.json")); + assert!( + dispatch.get("run_mode").is_none(), + "dispatch.json carries no run_mode: {dispatch}" ); + let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); + assert!( + !runbook.contains("--run-mode"), + "runbook pipeline commands carry no --run-mode: {runbook}" + ); + let manifest = read_str(&iteration_dir(&cwd).join("dispatch-manifest.md")); + assert!( + !manifest.contains("--run-mode"), + "dispatch manifest carries no --run-mode: {manifest}" + ); +} + +#[test] +fn run_mode_flag_is_rejected() { + // `--run-mode` is fully removed, not a hidden no-op: clap rejects it. + let tmp = tempfile::TempDir::new().unwrap(); + let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); + skill_eval() + .current_dir(&cwd) + .args(["run", "--skill-dir"]) + .arg(&skill_dir) + .args([ + "--skill", + "mr-review", + "--harness", + "claude-code", + "--run-mode", + "hybrid", + "--dry-run", + ]) + .assert() + .failure(); } diff --git a/tests/run/codex.rs b/tests/run/codex.rs index f5842a3..cf3cf3b 100644 --- a/tests/run/codex.rs +++ b/tests/run/codex.rs @@ -305,42 +305,28 @@ fn codex_dispatch_guidance_omits_hook_bypass_when_unguarded() { } #[test] -fn codex_headless_records_mode_and_human_runbook() { +fn codex_run_writes_human_followed_runbook() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); skill_eval() .current_dir(&cwd) .args(["run", "--skill-dir"]) .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "codex", - "--run-mode", - "headless", - "--dry-run", - ]) + .args(["--skill", "mr-review", "--harness", "codex", "--dry-run"]) .assert() .success(); - let conditions = read_json(&iteration_dir(&cwd).join("conditions.json")); - assert_eq!(conditions["run_mode"], "headless"); - - // Cli has no single env/, so the human-followed runbook lives in the iteration dir. + // Each task dispatches from its own per-(group, condition) env, so the + // human-followed runbook lives in the iteration dir, above those envs. let runbook = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); assert!( runbook.contains("human driving"), - "headless uses the human-followed template: {runbook}" + "uses the human-followed template: {runbook}" ); assert!( runbook.contains("codex --ask-for-approval never exec"), "carries the Codex CLI dispatch recipe: {runbook}" ); - assert!( - runbook.contains("--run-mode headless"), - "pipeline commands carry the headless run mode: {runbook}" - ); } #[test] diff --git a/tests/run/env_layout.rs b/tests/run/env_layout.rs index 86dd8e4..86f1264 100644 --- a/tests/run/env_layout.rs +++ b/tests/run/env_layout.rs @@ -133,9 +133,9 @@ fn dispatch_tasks_grouped_by_condition() { .collect(); assert_eq!(conds.len(), 4, "2 evals × 2 conditions: {conds:?}"); - // All with_skill tasks precede all without_skill tasks, so the runbook's - // "dispatch all of cond A → switch-condition → dispatch all of cond B" batches - // map to a straight top-to-bottom read of tasks[]. + // All with_skill tasks precede all without_skill tasks, so a straight + // top-to-bottom read of tasks[] dispatches condition A's batch, then + // condition B's — each from its own per-(group, condition) env. let first_b = conds.iter().position(|c| c == "without_skill").unwrap(); assert!( conds[..first_b].iter().all(|c| c == "with_skill"), diff --git a/tests/run/grouping.rs b/tests/run/grouping.rs index d018a34..672a2e3 100644 --- a/tests/run/grouping.rs +++ b/tests/run/grouping.rs @@ -68,8 +68,6 @@ fn cli_single_group_emits_groups_and_splits_env_per_condition() { "new-skill", "--harness", "claude-code", - "--run-mode", - "hybrid", "--dry-run", ]) .assert() @@ -182,15 +180,7 @@ fn isolated_hint_splits_into_separate_envs_cli() { .current_dir(&cwd) .args(["run", "--skill-dir"]) .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--harness", - "codex", - "--run-mode", - "hybrid", - "--dry-run", - ]) + .args(["--skill", "mr-review", "--harness", "codex", "--dry-run"]) .assert() .success(); diff --git a/tests/run/lifecycle.rs b/tests/run/lifecycle.rs index 42f69d5..0d8f12f 100644 --- a/tests/run/lifecycle.rs +++ b/tests/run/lifecycle.rs @@ -519,7 +519,7 @@ fn teardown_disarms_per_group_condition_cli_guards() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // Cli (hybrid) materializes one env per (group, condition); `--guard` arms a marker + // CLI dispatch materializes one env per (group, condition); `--guard` arms a marker // in each. The human runs teardown from the iteration dir, not from inside any env, // so the cwd-only disarm never reaches these per-env markers. skill_eval() @@ -531,8 +531,6 @@ fn teardown_disarms_per_group_condition_cli_guards() { "mr-review", "--harness", "claude-code", - "--run-mode", - "hybrid", "--guard", ]) .assert() @@ -557,7 +555,7 @@ fn teardown_disarms_per_group_condition_cli_guards() { .current_dir(&cwd) .args(["teardown", "--skill-dir"]) .arg(&skill_dir) - .args(["--skill", "mr-review", "--run-mode", "hybrid"]) + .args(["--skill", "mr-review"]) .assert() .success() .stdout(contains("write guard disarmed")); @@ -575,7 +573,7 @@ fn finalize_warns_about_armed_cli_per_env_guard() { let tmp = tempfile::TempDir::new().unwrap(); let (skill_dir, cwd) = setup(tmp.path(), DEFAULT_EVALS); - // Cli (hybrid) arms a guard in each per-(group, condition) env. finalize runs from + // CLI dispatch arms a guard in each per-(group, condition) env. finalize runs from // the iteration dir, not an env, so the cwd-only check misses them; it must walk the // per-env markers and remind the operator. skill_eval() @@ -587,8 +585,6 @@ fn finalize_warns_about_armed_cli_per_env_guard() { "mr-review", "--harness", "claude-code", - "--run-mode", - "hybrid", "--guard", ]) .assert() @@ -598,14 +594,7 @@ fn finalize_warns_about_armed_cli_per_env_guard() { .current_dir(&cwd) .args(["finalize", "--skill-dir"]) .arg(&skill_dir) - .args([ - "--skill", - "mr-review", - "--run-mode", - "hybrid", - "--iteration", - "1", - ]) + .args(["--skill", "mr-review", "--iteration", "1"]) .assert() .success() .stdout(contains("Guard still armed")); diff --git a/tests/run/runbook.rs b/tests/run/runbook.rs index b1eb688..abb93ef 100644 --- a/tests/run/runbook.rs +++ b/tests/run/runbook.rs @@ -51,18 +51,15 @@ fn run_writes_headless_runbook_for_claude() { "mr-review", "--harness", "claude-code", - "--run-mode", - "headless", "--dry-run", ]) .assert() .success(); let book = read_str(&iteration_dir(&cwd).join("RUNBOOK.md")); - // A Claude Code Cli-mode run uses the shared human-followed template, NOT - // Claude's interactive (agent-followed) one — so the in-session switch-condition - // batch loop is absent and the claude -p recipe is present. Cli has no single - // env/, so the runbook lives in the iteration dir. + // A Claude Code run uses the shared human-followed template carrying the + // `claude -p` recipe. Each task dispatches from its own per-(group, condition) + // env, so the runbook lives in the iteration dir, above those envs. assert!( book.contains("human driving"), "frames the run for a human at a terminal: {book}" From 3b1f98e1cda079980e5f0186bd54e7e44a9c1c8b Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sun, 5 Jul 2026 22:46:32 -0400 Subject: [PATCH 05/45] refactor(adapters): move harness-specific code into per-harness modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restructure src/adapters/ into per-harness module trees (claude_code/, codex/, opencode/) so every harness-specific reference lives in a clearly-named file: - Adapter impls move out of harness.rs into /mod.rs; harness.rs keeps the trait, dispatch contexts, adapter_for(), and the cross-harness tests. - The shared TranscriptSummary moves out of the Claude transcript module into adapters/transcript.rs — it is the harness-neutral shape every parser produces. - Guard installers and verdict shaping move out of src/sandbox into /guard.rs (install_claude_guard/install_codex_guard, hook matchers, guard_decision/codex_guard_decision). sandbox/install.rs keeps only the shared marker/manifest/teardown machinery; the generic install_guard() wrapper (hardcoded to Claude) is deleted — production code already goes through adapter.install_guard(). - teardown_guard/guard_is_armed and the hook entry points' default marker paths derive from Harness::ALL + adapter skills_dir() instead of hardcoded .claude/.agents paths. - OpenCode slug/naming helpers move out of staging into adapters/opencode (taking the staged prefix as a parameter). No behavior change; unit tests move with their code. The guard / guard-codex subcommand names, marker/manifest filenames, and manifest fields are stable on-disk contracts and are unchanged. Part of #121. Co-Authored-By: Claude Fable 5 --- .../{claude_cli.rs => claude_code/cli.rs} | 2 +- src/adapters/claude_code/guard.rs | 348 +++++++++++++ src/adapters/claude_code/mod.rs | 153 ++++++ .../{ => claude_code}/plugin_shadow.rs | 0 .../session.rs} | 0 .../stream_json.rs} | 7 +- .../transcript.rs} | 20 +- src/adapters/{codex_cli.rs => codex/cli.rs} | 2 +- src/adapters/codex/guard.rs | 260 ++++++++++ src/adapters/codex/mod.rs | 130 +++++ .../{codex_session.rs => codex/session.rs} | 0 .../transcript.rs} | 2 +- src/adapters/harness.rs | 303 +---------- src/adapters/mod.rs | 37 +- src/adapters/opencode/mod.rs | 192 +++++++ .../session.rs} | 0 src/adapters/transcript.rs | 22 + src/cli/commands/guard.rs | 50 +- src/cli/run/staging/mod.rs | 87 +--- src/cli/run/staging/tests/stage.rs | 23 +- src/core/context.rs | 6 + src/sandbox/guard.rs | 135 +---- src/sandbox/install.rs | 476 ++---------------- src/sandbox/mod.rs | 17 +- 24 files changed, 1249 insertions(+), 1023 deletions(-) rename src/adapters/{claude_cli.rs => claude_code/cli.rs} (99%) create mode 100644 src/adapters/claude_code/guard.rs create mode 100644 src/adapters/claude_code/mod.rs rename src/adapters/{ => claude_code}/plugin_shadow.rs (100%) rename src/adapters/{claude_code_session.rs => claude_code/session.rs} (100%) rename src/adapters/{claude_stream_json.rs => claude_code/stream_json.rs} (98%) rename src/adapters/{claude_code_transcript.rs => claude_code/transcript.rs} (94%) rename src/adapters/{codex_cli.rs => codex/cli.rs} (99%) create mode 100644 src/adapters/codex/guard.rs create mode 100644 src/adapters/codex/mod.rs rename src/adapters/{codex_session.rs => codex/session.rs} (100%) rename src/adapters/{codex_transcript.rs => codex/transcript.rs} (99%) create mode 100644 src/adapters/opencode/mod.rs rename src/adapters/{opencode_session.rs => opencode/session.rs} (100%) create mode 100644 src/adapters/transcript.rs diff --git a/src/adapters/claude_cli.rs b/src/adapters/claude_code/cli.rs similarity index 99% rename from src/adapters/claude_cli.rs rename to src/adapters/claude_code/cli.rs index 15837f6..88d7056 100644 --- a/src/adapters/claude_cli.rs +++ b/src/adapters/claude_code/cli.rs @@ -8,7 +8,7 @@ //! written to a file. `, +) -> io::Result { + let skills_dir = stage_root.join(".claude").join("skills"); + fs::create_dir_all(&skills_dir)?; + + let marker_path = skills_dir.join(GUARD_MARKER); + write_marker(&marker_path, stage_root, ttl)?; + + let settings_path = stage_root.join(".claude").join("settings.local.json"); + let settings_existed = settings_path.exists(); + let backup = if settings_existed { + Some(fs::read_to_string(&settings_path)?) + } else { + None + }; + + // Start from the existing settings (or an empty object), preserving key + // order, then append the PreToolUse hook entry. + let mut settings: Value = backup + .as_deref() + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or_else(|| json!({})); + let hooks = settings + .as_object_mut() + .expect("settings is a JSON object") + .entry("hooks") + .or_insert_with(|| json!({})); + let pre = hooks + .as_object_mut() + .expect("hooks is a JSON object") + .entry("PreToolUse") + .or_insert_with(|| json!([])); + let command = format!( + "\"{}\" guard \"{}\"", + guard_exe.display(), + marker_path.display() + ); + pre.as_array_mut() + .expect("PreToolUse is an array") + .push(json!({ + "matcher": HOOK_MATCHER, + "hooks": [ { "type": "command", "command": command } ], + })); + write_json(&settings_path, &settings)?; + + write_manifest( + &skills_dir.join(GUARD_MANIFEST), + &settings_path, + settings_existed, + backup, + &marker_path, + )?; + + Ok(marker_path) +} + +/// Evaluate a PreToolUse hook `payload` (the JSON Claude Code sends on stdin) +/// against `marker`. Returns the serialized deny verdict to print on stdout when +/// the call is blocked — Claude Code's native `hookSpecificOutput` shape — or +/// `None` to allow (print nothing). An empty or malformed payload is treated as +/// allow. +pub(crate) fn guard_decision(payload: &str, marker: Option) -> Option { + let (tool_name, tool_input) = parse_tool_call(payload)?; + + let decision = decide(&tool_name, &tool_input, marker.as_ref(), now_ms()); + if decision.allow { + return None; + } + Some( + serde_json::to_string(&json!({ + "hookSpecificOutput": { + "hookEventName": "PreToolUse", + "permissionDecision": "deny", + "permissionDecisionReason": decision.reason, + } + })) + .expect("deny verdict serializes"), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sandbox::install::{iso_millis, teardown_guard}; + use crate::sandbox::{guard_is_armed, now_ms}; + use chrono::DateTime; + use tempfile::TempDir; + + struct Case { + _tmp: TempDir, + stage_root: PathBuf, + } + + fn setup() -> Case { + let tmp = TempDir::new().unwrap(); + let stage_root = tmp.path().join("stage"); + fs::create_dir_all(&stage_root).unwrap(); + Case { + _tmp: tmp, + stage_root, + } + } + + fn skills_dir(stage_root: &Path) -> PathBuf { + stage_root.join(".claude").join("skills") + } + + fn settings_path(stage_root: &Path) -> PathBuf { + stage_root.join(".claude").join("settings.local.json") + } + + fn read_json(path: &Path) -> Value { + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap() + } + + fn absolutize(p: &Path) -> PathBuf { + std::path::absolute(p).unwrap_or_else(|_| p.to_path_buf()) + } + + /// A live marker (active, no expiry → unexpired) scoped to one root. + fn marker() -> GuardMarker { + GuardMarker { + active: Some(true), + allowed_roots: Some(vec!["/work/.eval-magic".to_string()]), + expires_at: None, + } + } + + #[test] + fn install_writes_an_active_marker_hook_and_manifest() { + let c = setup(); + let exe = Path::new("/g/eval-magic"); + install_guard(&c.stage_root, exe, None).unwrap(); + + let marker = read_json(&skills_dir(&c.stage_root).join(GUARD_MARKER)); + assert_eq!(marker["active"], json!(true)); + let expires = marker["expiresAt"].as_str().unwrap(); + let exp_ms = DateTime::parse_from_rfc3339(expires) + .unwrap() + .timestamp_millis(); + assert!(exp_ms > now_ms()); + let env = absolutize(&c.stage_root).display().to_string(); + assert!( + marker["allowedRoots"] + .as_array() + .unwrap() + .iter() + .any(|r| r.as_str().unwrap() == env) + ); + + let settings = read_json(&settings_path(&c.stage_root)); + let hook = &settings["hooks"]["PreToolUse"][0]; + assert!(hook["matcher"].as_str().unwrap().contains("Write")); + assert!( + hook["hooks"][0]["command"] + .as_str() + .unwrap() + .contains("guard") + ); + + assert!(skills_dir(&c.stage_root).join(GUARD_MANIFEST).exists()); + } + + #[test] + fn marker_scopes_allowed_roots_to_the_env_and_temp_only() { + let c = setup(); + let exe = Path::new("/g/eval-magic"); + install_guard(&c.stage_root, exe, None).unwrap(); + + let marker = read_json(&skills_dir(&c.stage_root).join(GUARD_MARKER)); + let roots: Vec = marker["allowedRoots"] + .as_array() + .unwrap() + .iter() + .map(|r| r.as_str().unwrap().to_string()) + .collect(); + + // The guard boundary is the isolated env (stage_root) plus temp — nothing + // above it. The parent workspace tree must NOT be an allowed root, or the + // agent could write into sibling iterations / the meta dir above `env/`. + let env = absolutize(&c.stage_root).display().to_string(); + let temp = absolutize(&std::env::temp_dir()).display().to_string(); + assert_eq!(roots, vec![env, temp]); + assert!( + !roots.iter().any(|r| r.ends_with(".eval-magic")), + "workspace_root must not be an allowed root: {roots:?}" + ); + } + + #[test] + fn hook_command_invokes_the_binary_guard_subcommand() { + let c = setup(); + let exe = Path::new("/g/eval-magic"); + let marker = install_guard(&c.stage_root, exe, None).unwrap(); + let settings = read_json(&settings_path(&c.stage_root)); + let command = settings["hooks"]["PreToolUse"][0]["hooks"][0]["command"] + .as_str() + .unwrap() + .to_string(); + assert_eq!( + command, + format!("\"/g/eval-magic\" guard \"{}\"", marker.display()) + ); + } + + #[test] + fn teardown_deletes_settings_it_created() { + let c = setup(); + let exe = Path::new("/g/eval-magic"); + install_guard(&c.stage_root, exe, None).unwrap(); + assert!(settings_path(&c.stage_root).exists()); + + assert!(teardown_guard(&c.stage_root)); + assert!(!settings_path(&c.stage_root).exists()); + assert!(!skills_dir(&c.stage_root).join(GUARD_MARKER).exists()); + assert!(!skills_dir(&c.stage_root).join(GUARD_MANIFEST).exists()); + } + + #[test] + fn teardown_restores_a_pre_existing_settings_verbatim() { + let c = setup(); + fs::create_dir_all(c.stage_root.join(".claude")).unwrap(); + let original = format!( + "{}\n", + serde_json::to_string_pretty(&json!({ + "permissions": { "allow": ["Bash(ls)"] } + })) + .unwrap() + ); + fs::write(settings_path(&c.stage_root), &original).unwrap(); + + let exe = Path::new("/g/eval-magic"); + install_guard(&c.stage_root, exe, None).unwrap(); + // hook present while armed + assert!( + fs::read_to_string(settings_path(&c.stage_root)) + .unwrap() + .contains("PreToolUse") + ); + + teardown_guard(&c.stage_root); + assert_eq!( + fs::read_to_string(settings_path(&c.stage_root)).unwrap(), + original + ); + } + + #[test] + fn guard_is_armed_ignores_missing_inactive_expired_and_malformed_markers() { + let c = setup(); + let marker_path = skills_dir(&c.stage_root).join(GUARD_MARKER); + fs::create_dir_all(skills_dir(&c.stage_root)).unwrap(); + + assert!(!guard_is_armed(&c.stage_root)); + + fs::write( + &marker_path, + serde_json::to_string(&json!({ "active": false })).unwrap(), + ) + .unwrap(); + assert!(!guard_is_armed(&c.stage_root)); + + fs::write( + &marker_path, + serde_json::to_string(&json!({ + "active": true, + "expiresAt": iso_millis(now_ms() - 60_000), + })) + .unwrap(), + ) + .unwrap(); + assert!(!guard_is_armed(&c.stage_root)); + + fs::write(&marker_path, "not json").unwrap(); + assert!(!guard_is_armed(&c.stage_root)); + } + + #[test] + fn teardown_sweeps_a_stray_marker_even_without_a_manifest() { + let c = setup(); + fs::create_dir_all(skills_dir(&c.stage_root)).unwrap(); + fs::write(skills_dir(&c.stage_root).join(GUARD_MARKER), "{}").unwrap(); + assert!(teardown_guard(&c.stage_root)); + assert!(!skills_dir(&c.stage_root).join(GUARD_MARKER).exists()); + } + + #[test] + fn allows_returns_none() { + let payload = r#"{ "tool_name": "Read", "tool_input": { "file_path": "/etc/passwd" } }"#; + assert_eq!(guard_decision(payload, Some(marker())), None); + } + + #[test] + fn deny_returns_pretooluse_deny_json() { + let payload = r#"{ "tool_name": "Write", "tool_input": { "file_path": "/etc/passwd" } }"#; + let out = guard_decision(payload, Some(marker())).expect("should deny"); + let v: Value = serde_json::from_str(&out).unwrap(); + assert_eq!(v["hookSpecificOutput"]["hookEventName"], "PreToolUse"); + assert_eq!(v["hookSpecificOutput"]["permissionDecision"], "deny"); + assert!( + v["hookSpecificOutput"]["permissionDecisionReason"] + .as_str() + .unwrap() + .contains("outside") + ); + } + + #[test] + fn no_marker_allows_everything() { + let payload = r#"{ "tool_name": "Write", "tool_input": { "file_path": "/etc/passwd" } }"#; + assert_eq!(guard_decision(payload, None), None); + } + + #[test] + fn empty_or_malformed_payload_fails_open() { + assert_eq!(guard_decision("", Some(marker())), None); + assert_eq!(guard_decision("not json", Some(marker())), None); + } +} diff --git a/src/adapters/claude_code/mod.rs b/src/adapters/claude_code/mod.rs new file mode 100644 index 0000000..6c6fd44 --- /dev/null +++ b/src/adapters/claude_code/mod.rs @@ -0,0 +1,153 @@ +//! Claude Code harness support — the default harness. +//! +//! Everything Claude-Code-specific lives in this module tree: the adapter impl +//! (this file), dispatch-recipe rendering ([`cli`]), the native skills block +//! ([`session`]), `claude -p` stream-json transcript parsing ([`stream_json`] + +//! [`transcript`]), plugin-shadow detection ([`plugin_shadow`]), and the +//! write-guard hook ([`guard`]). + +mod cli; +pub(crate) mod guard; +pub mod plugin_shadow; +pub mod session; +pub mod stream_json; +pub mod transcript; + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::core::{AvailableSkill, ToolInvocation}; + +use super::TranscriptSummary; +use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; +use cli::{ + claude_exec_command_template, claude_judge_dispatch_recipe, claude_parallel_dispatch_recipe, +}; +use session::render_available_skills_block; +use stream_json::{parse_claude_stream_json, parse_claude_stream_json_full}; + +pub struct ClaudeCodeAdapter; + +impl HarnessAdapter for ClaudeCodeAdapter { + fn label(&self) -> &'static str { + "claude-code" + } + fn skills_dir(&self, repo_root: &Path) -> PathBuf { + repo_root.join(".claude").join("skills") + } + fn rewrites_frontmatter_name(&self) -> bool { + false + } + fn advertises_staged_slug_name(&self) -> bool { + false + } + fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { + render_available_skills_block(skills) + } + fn skill_surface_phrase(&self) -> &'static str { + "via the Skill tool" + } + fn skill_unresolved_phrase(&self) -> &'static str { + "If the Skill tool cannot resolve that identifier" + } + fn cli_events_filename(&self) -> Option<&'static str> { + Some("claude-events.jsonl") + } + fn cli_model_flag(&self) -> Option<&'static str> { + Some("--model") + } + fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { + format!( + "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness claude-code`.", + claude_exec_command_template(self.cli_model_flag(), ctx.agent_model), + target_args = ctx.target_args, + iteration = ctx.iteration + ) + } + fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { + Some(vec![ + "After all dispatches (Claude Code):".to_string(), + String::new(), + "Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `) -> Option { + Some(claude_judge_dispatch_recipe( + self.cli_model_flag(), + ctx.iteration_dir, + )) + } + fn parse_cli_events(&self, path: &Path) -> io::Result> { + parse_claude_stream_json(path) + } + fn parse_cli_events_full(&self, path: &Path) -> io::Result { + parse_claude_stream_json_full(path) + } + fn install_guard( + &self, + stage_root: &Path, + guard_exe: &Path, + ttl: Option, + ) -> io::Result { + guard::install_guard(stage_root, guard_exe, ttl) + } + fn guard_armed_message(&self) -> Option<&'static str> { + Some( + "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches.\n Each `claude -p` dispatch loads the hook from the env cwd it runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", + ) + } +} + +#[cfg(test)] +mod tests { + use crate::adapters::adapter_for; + use crate::core::Harness; + + #[test] + fn claude_adapter_advertises_cli_events_file_and_model_flag() { + let a = adapter_for(Harness::ClaudeCode); + assert_eq!(a.cli_events_filename(), Some("claude-events.jsonl")); + assert_eq!(a.cli_model_flag(), Some("--model")); + } + + #[test] + fn claude_parse_cli_events_full_reads_stream_json_result_event() { + use serde_json::json; + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("claude-events.jsonl"); + // No per-line timestamps; the result event is the only source of duration. + let lines = [ + json!({"type": "assistant", "message": {"id": "msg_1", "role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}} + ]}}), + json!({"type": "result", "subtype": "success", "is_error": false, "result": "Done", "duration_ms": 5637, "usage": {"input_tokens": 1, "output_tokens": 2, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}), + ]; + let body = lines + .iter() + .map(|l| l.to_string()) + .collect::>() + .join("\n"); + std::fs::write(&path, format!("{body}\n")).unwrap(); + + let a = adapter_for(Harness::ClaudeCode); + let summary = a.parse_cli_events_full(&path).unwrap(); + assert_eq!(summary.final_text, Some("Done".into())); + assert_eq!(summary.duration_ms, Some(5637)); + assert_eq!(summary.tool_invocations.len(), 1); + assert_eq!(summary.tool_invocations[0].name, "Bash"); + } +} diff --git a/src/adapters/plugin_shadow.rs b/src/adapters/claude_code/plugin_shadow.rs similarity index 100% rename from src/adapters/plugin_shadow.rs rename to src/adapters/claude_code/plugin_shadow.rs diff --git a/src/adapters/claude_code_session.rs b/src/adapters/claude_code/session.rs similarity index 100% rename from src/adapters/claude_code_session.rs rename to src/adapters/claude_code/session.rs diff --git a/src/adapters/claude_stream_json.rs b/src/adapters/claude_code/stream_json.rs similarity index 98% rename from src/adapters/claude_stream_json.rs rename to src/adapters/claude_code/stream_json.rs index 429937d..4615f40 100644 --- a/src/adapters/claude_stream_json.rs +++ b/src/adapters/claude_code/stream_json.rs @@ -4,7 +4,7 @@ //! --output-format stream-json --verbose` writes (captured per task as //! `outputs/claude-events.jsonl`). The `assistant`/`user` events wrap a full //! Anthropic Messages object under `message`, so tool-call extraction is shared -//! with the [`claude_code_transcript`](super::claude_code_transcript) record +//! with the [`transcript`](super::transcript) record //! types. The differences are all in the envelope: there are no per-line //! timestamps, and a terminal `result` event carries the authoritative final //! text, wall-clock duration, and token usage. `system`, `rate_limit_event`, and @@ -17,8 +17,9 @@ use std::path::Path; use crate::core::ToolInvocation; -use super::TranscriptSummary; -use super::claude_code_transcript::{ +use crate::adapters::TranscriptSummary; + +use super::transcript::{ TranscriptRecord, UsageRecord, extract_invocations, last_assistant_text, read_records, }; diff --git a/src/adapters/claude_code_transcript.rs b/src/adapters/claude_code/transcript.rs similarity index 94% rename from src/adapters/claude_code_transcript.rs rename to src/adapters/claude_code/transcript.rs index 811ec5e..0413349 100644 --- a/src/adapters/claude_code_transcript.rs +++ b/src/adapters/claude_code/transcript.rs @@ -3,11 +3,12 @@ //! Defines the JSONL record shapes and the shared extractors — ordered //! [`ToolInvocation`]s (matching `tool_result` blocks back to their `tool_use` by //! id) and the last assistant text — reused by the `claude -p` stream-json parser -//! ([`claude_stream_json`](super::claude_stream_json)), plus the -//! [`TranscriptSummary`] the pipeline consumes. +//! ([`stream_json`](super::stream_json)). The harness-neutral +//! [`TranscriptSummary`](crate::adapters::TranscriptSummary) the pipeline +//! consumes lives in `crate::adapters::transcript`. use crate::core::ToolInvocation; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use serde_json::Value; use std::collections::HashMap; use std::fs; @@ -163,19 +164,6 @@ pub(crate) fn last_assistant_text(records: &[TranscriptRecord]) -> Option, - /// Total token usage (input + output + cache creation/read), as reported by - /// the run's terminal `result` event. - pub total_tokens: Option, - /// Wall-clock duration, as reported by the run's terminal `result` event. - pub duration_ms: Option, - /// Concatenated text blocks of the last assistant message. - pub final_text: Option, -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/adapters/codex_cli.rs b/src/adapters/codex/cli.rs similarity index 99% rename from src/adapters/codex_cli.rs rename to src/adapters/codex/cli.rs index a06db69..860ec1b 100644 --- a/src/adapters/codex_cli.rs +++ b/src/adapters/codex/cli.rs @@ -1,6 +1,6 @@ //! Codex CLI command rendering (`codex exec`) for dispatch guidance. -use super::cli_command::render_cli_model_arg; +use crate::adapters::cli_command::render_cli_model_arg; use std::path::Path; /// Copy/pasteable Codex dispatch command template. Stdin is detached so a diff --git a/src/adapters/codex/guard.rs b/src/adapters/codex/guard.rs new file mode 100644 index 0000000..05a3c4f --- /dev/null +++ b/src/adapters/codex/guard.rs @@ -0,0 +1,260 @@ +//! Codex write-guard hook: install + verdict shape. +//! +//! Arms the guard by merging a `PreToolUse` hook into the env's +//! `.codex/hooks.json`; dispatches must pass `--dangerously-bypass-hook-trust` +//! so the vetted project-local hook actually runs. The hook invokes the hidden +//! `guard-codex` subcommand (a stable on-disk contract), whose block verdict +//! uses Codex's native `{ "decision": "block", "reason": "..." }` shape. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use crate::sandbox::decide::{GuardMarker, decide}; +use crate::sandbox::install::{ + GUARD_MANIFEST, GUARD_MARKER, write_json, write_manifest, write_marker, +}; +use crate::sandbox::{now_ms, parse_tool_call}; + +/// Tool names the Codex PreToolUse hook fires on. +const HOOK_MATCHER: &str = "^Bash$|^apply_patch$|^Edit$|^Write$"; + +/// Arm the write guard using Codex's project-local hook surface. Returns the +/// staged marker path. +pub(crate) fn install_guard( + stage_root: &Path, + guard_exe: &Path, + ttl: Option, +) -> io::Result { + let skills_dir = stage_root.join(".agents").join("skills"); + fs::create_dir_all(&skills_dir)?; + + let marker_path = skills_dir.join(GUARD_MARKER); + write_marker(&marker_path, stage_root, ttl)?; + + let hooks_path = stage_root.join(".codex").join("hooks.json"); + if let Some(parent) = hooks_path.parent() { + fs::create_dir_all(parent)?; + } + let hooks_existed = hooks_path.exists(); + let backup = if hooks_existed { + Some(fs::read_to_string(&hooks_path)?) + } else { + None + }; + + let mut hooks: Value = backup + .as_deref() + .and_then(|s| serde_json::from_str(s).ok()) + .unwrap_or_else(|| json!({})); + let hooks_obj = hooks + .as_object_mut() + .expect("hooks.json root is a JSON object") + .entry("hooks") + .or_insert_with(|| json!({})); + let pre = hooks_obj + .as_object_mut() + .expect("hooks is a JSON object") + .entry("PreToolUse") + .or_insert_with(|| json!([])); + let command = format!( + "\"{}\" guard-codex \"{}\"", + guard_exe.display(), + marker_path.display() + ); + pre.as_array_mut() + .expect("PreToolUse is an array") + .push(json!({ + "matcher": HOOK_MATCHER, + "hooks": [ + { + "type": "command", + "command": command, + "timeout": 30, + "statusMessage": "Checking eval write boundary", + } + ], + })); + write_json(&hooks_path, &hooks)?; + + write_manifest( + &skills_dir.join(GUARD_MANIFEST), + &hooks_path, + hooks_existed, + backup, + &marker_path, + )?; + + Ok(marker_path) +} + +/// The hook-config dir the Codex guard writes under `stage_root`; teardown +/// prunes it when the restored config leaves it empty. +pub(crate) fn hook_cleanup_dir(stage_root: &Path) -> PathBuf { + stage_root.join(".codex") +} + +/// Evaluate a PreToolUse hook `payload` (the JSON Codex sends on stdin) against +/// `marker`. Codex's hook contract blocks by returning `{ "decision": "block", +/// "reason": "..." }` on stdout — kept separate from Claude Code's +/// `hookSpecificOutput` shape so both harnesses use their native conventions. +pub(crate) fn guard_decision(payload: &str, marker: Option) -> Option { + let (tool_name, tool_input) = parse_tool_call(payload)?; + let decision = decide(&tool_name, &tool_input, marker.as_ref(), now_ms()); + if decision.allow { + return None; + } + Some( + serde_json::to_string(&json!({ + "decision": "block", + "reason": decision.reason, + })) + .expect("Codex block verdict serializes"), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::sandbox::install::teardown_guard; + use tempfile::TempDir; + + struct Case { + _tmp: TempDir, + stage_root: PathBuf, + } + + fn setup() -> Case { + let tmp = TempDir::new().unwrap(); + let stage_root = tmp.path().join("stage"); + fs::create_dir_all(&stage_root).unwrap(); + Case { + _tmp: tmp, + stage_root, + } + } + + fn codex_hooks_path(stage_root: &Path) -> PathBuf { + stage_root.join(".codex").join("hooks.json") + } + + fn read_json(path: &Path) -> Value { + serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap() + } + + fn absolutize(p: &Path) -> PathBuf { + std::path::absolute(p).unwrap_or_else(|_| p.to_path_buf()) + } + + /// A live marker (active, no expiry → unexpired) scoped to one root. + fn marker() -> GuardMarker { + GuardMarker { + active: Some(true), + allowed_roots: Some(vec!["/work/.eval-magic".to_string()]), + expires_at: None, + } + } + + #[test] + fn codex_install_writes_project_hook_marker_and_manifest() { + let c = setup(); + let exe = Path::new("/g/eval-magic"); + install_guard(&c.stage_root, exe, None).unwrap(); + + let marker = read_json( + &c.stage_root + .join(".agents") + .join("skills") + .join(GUARD_MARKER), + ); + assert_eq!(marker["active"], json!(true)); + // The Codex guard shares the env-scoped roots: the staged `.agents/skills` + // dir lives inside `stage_root`, so the single env root already covers it. + let env = absolutize(&c.stage_root).display().to_string(); + assert!( + marker["allowedRoots"] + .as_array() + .unwrap() + .iter() + .any(|r| r.as_str().unwrap() == env) + ); + + let hooks = read_json(&codex_hooks_path(&c.stage_root)); + let hook = &hooks["hooks"]["PreToolUse"][0]; + assert!(hook["matcher"].as_str().unwrap().contains("apply_patch")); + assert!( + hook["hooks"][0]["command"] + .as_str() + .unwrap() + .contains("guard-codex") + ); + assert!( + c.stage_root + .join(".agents") + .join("skills") + .join(GUARD_MANIFEST) + .exists() + ); + } + + #[test] + fn codex_teardown_restores_pre_existing_hooks_json_verbatim() { + let c = setup(); + fs::create_dir_all(c.stage_root.join(".codex")).unwrap(); + let original = format!( + "{}\n", + serde_json::to_string_pretty(&json!({ + "hooks": { + "PostToolUse": [ + { + "matcher": "Bash", + "hooks": [{ "type": "command", "command": "echo ok" }] + } + ] + } + })) + .unwrap() + ); + fs::write(codex_hooks_path(&c.stage_root), &original).unwrap(); + + install_guard(&c.stage_root, Path::new("/g/eval-magic"), None).unwrap(); + assert!( + fs::read_to_string(codex_hooks_path(&c.stage_root)) + .unwrap() + .contains("guard-codex") + ); + + teardown_guard(&c.stage_root); + assert_eq!( + fs::read_to_string(codex_hooks_path(&c.stage_root)).unwrap(), + original + ); + } + + #[test] + fn codex_deny_returns_decision_block_json() { + let payload = r#"{ "hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": { "command": "npm install left-pad" } }"#; + let out = guard_decision(payload, Some(marker())).expect("should block"); + let v: Value = serde_json::from_str(&out).unwrap(); + assert_eq!(v["decision"], "block"); + assert!(v["reason"].as_str().unwrap().contains("blocked Bash")); + } + + #[test] + fn codex_apply_patch_outside_allowed_roots_blocks() { + let payload = r#"{ "hook_event_name": "PreToolUse", "tool_name": "apply_patch", "tool_input": { "files": ["/etc/passwd"] } }"#; + let out = guard_decision(payload, Some(marker())).expect("should block"); + let v: Value = serde_json::from_str(&out).unwrap(); + assert_eq!(v["decision"], "block"); + assert!(v["reason"].as_str().unwrap().contains("apply_patch")); + } + + #[test] + fn codex_apply_patch_inside_allowed_roots_allows() { + let payload = r#"{ "hook_event_name": "PreToolUse", "tool_name": "apply_patch", "tool_input": { "files": ["/work/.eval-magic/out.md"] } }"#; + assert_eq!(guard_decision(payload, Some(marker())), None); + } +} diff --git a/src/adapters/codex/mod.rs b/src/adapters/codex/mod.rs new file mode 100644 index 0000000..fa52d96 --- /dev/null +++ b/src/adapters/codex/mod.rs @@ -0,0 +1,130 @@ +//! Codex harness support. +//! +//! Everything Codex-specific lives in this module tree: the adapter impl (this +//! file), `codex exec` dispatch-recipe rendering ([`cli`]), the `## Skills` +//! block ([`session`]), `item.completed` event-stream parsing ([`transcript`]), +//! and the write-guard hook ([`guard`]). + +mod cli; +pub(crate) mod guard; +pub mod session; +pub mod transcript; + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::core::{AvailableSkill, ToolInvocation}; + +use super::TranscriptSummary; +use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; +use cli::{ + codex_exec_command_template, codex_judge_dispatch_recipe, codex_parallel_dispatch_recipe, +}; +use session::render_codex_available_skills_block; +use transcript::{parse_codex_events, parse_codex_events_full}; + +pub struct CodexAdapter; + +impl HarnessAdapter for CodexAdapter { + fn label(&self) -> &'static str { + "codex" + } + fn skills_dir(&self, repo_root: &Path) -> PathBuf { + repo_root.join(".agents").join("skills") + } + fn rewrites_frontmatter_name(&self) -> bool { + true + } + fn advertises_staged_slug_name(&self) -> bool { + true + } + fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { + render_codex_available_skills_block(skills) + } + fn skill_surface_phrase(&self) -> &'static str { + "as a Codex skill" + } + fn skill_unresolved_phrase(&self) -> &'static str { + "If it does not load as a Codex skill" + } + fn cli_events_filename(&self) -> Option<&'static str> { + Some("codex-events.jsonl") + } + fn cli_model_flag(&self) -> Option<&'static str> { + Some("-m") + } + fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { + format!( + "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness codex`.", + codex_exec_command_template(self.cli_model_flag(), ctx.guard, ctx.agent_model), + target_args = ctx.target_args, + iteration = ctx.iteration + ) + } + fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { + Some(vec![ + "After all dispatches (Codex):".to_string(), + String::new(), + "Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `) -> Option { + Some(codex_judge_dispatch_recipe( + self.cli_model_flag(), + ctx.guard, + ctx.iteration_dir, + )) + } + fn parse_cli_events(&self, path: &Path) -> io::Result> { + parse_codex_events(path) + } + fn parse_cli_events_full(&self, path: &Path) -> io::Result { + parse_codex_events_full(path) + } + fn install_guard( + &self, + stage_root: &Path, + guard_exe: &Path, + ttl: Option, + ) -> io::Result { + guard::install_guard(stage_root, guard_exe, ttl) + } + fn guard_armed_message(&self) -> Option<&'static str> { + Some( + "\n🛡 Write guard armed: a PreToolUse hook is staged in .codex/hooks.json\n and will block writes/installs outside the eval sandbox during Codex dispatches.\n Dispatch with codex --ask-for-approval never exec --dangerously-bypass-hook-trust so the vetted eval hook runs.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", + ) + } +} + +#[cfg(test)] +mod tests { + use crate::adapters::adapter_for; + use crate::core::Harness; + + #[test] + fn codex_parse_cli_events_delegates_to_events_parser() { + use serde_json::json; + let dir = tempfile::TempDir::new().unwrap(); + let path = dir.path().join("codex-events.jsonl"); + let line = json!({"type": "item.completed", "item": {"id": "i1", "type": "command_execution", "command": "bun test", "output": "ok"}}); + std::fs::write(&path, format!("{line}\n")).unwrap(); + + let inv = adapter_for(Harness::Codex).parse_cli_events(&path).unwrap(); + assert_eq!(inv.len(), 1); + assert_eq!(inv[0].name, "command_execution"); + } +} diff --git a/src/adapters/codex_session.rs b/src/adapters/codex/session.rs similarity index 100% rename from src/adapters/codex_session.rs rename to src/adapters/codex/session.rs diff --git a/src/adapters/codex_transcript.rs b/src/adapters/codex/transcript.rs similarity index 99% rename from src/adapters/codex_transcript.rs rename to src/adapters/codex/transcript.rs index 83f6e3e..1150dbc 100644 --- a/src/adapters/codex_transcript.rs +++ b/src/adapters/codex/transcript.rs @@ -6,7 +6,7 @@ //! [`TranscriptSummary`] shape as the Claude adapter, but with Codex's token //! accounting (excludes cached input tokens). -use crate::adapters::claude_code_transcript::TranscriptSummary; +use crate::adapters::TranscriptSummary; use crate::core::ToolInvocation; use serde_json::{Map, Value}; use std::fs; diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 4463607..7c399b2 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -6,7 +6,9 @@ //! transcript is parsed, where staged skills live, and which native hook the //! write guard installs. Generic code resolves an adapter with [`adapter_for`] //! and then calls the trait — so [`adapter_for`] is the one place that names a -//! concrete harness for this surface. +//! concrete harness for this surface. The impls live in the per-harness +//! modules ([`claude_code`](super::claude_code), [`codex`](super::codex), +//! [`opencode`](super::opencode)). use std::io; use std::path::{Path, PathBuf}; @@ -15,17 +17,6 @@ use std::time::Duration; use crate::core::{AvailableSkill, Harness, ToolInvocation}; use super::TranscriptSummary; -use super::claude_cli::{ - claude_exec_command_template, claude_judge_dispatch_recipe, claude_parallel_dispatch_recipe, -}; -use super::codex_cli::{ - codex_exec_command_template, codex_judge_dispatch_recipe, codex_parallel_dispatch_recipe, -}; -use super::{ - parse_claude_stream_json, parse_claude_stream_json_full, parse_codex_events, - parse_codex_events_full, render_available_skills_block, render_codex_available_skills_block, - render_opencode_available_skills_block, -}; /// The behavior that varies by harness. Generic dispatch code depends on this /// trait, never on a concrete harness variant. @@ -137,10 +128,6 @@ pub trait HarnessAdapter { /// regardless of harness (Claude Code, Codex, OpenCode). pub const RUNBOOK_TEMPLATE: &str = include_str!("../../profiles/shared/runbook.md"); -pub struct ClaudeCodeAdapter; -pub struct CodexAdapter; -pub struct OpenCodeAdapter; - /// Context for rendering a harness's one-shot CLI agent-dispatch guidance. #[derive(Debug, Clone, Copy)] pub struct CliDispatchContext<'a> { @@ -164,243 +151,14 @@ pub struct CliJudgeContext<'a> { pub iteration_dir: &'a Path, } -impl HarnessAdapter for ClaudeCodeAdapter { - fn label(&self) -> &'static str { - "claude-code" - } - fn skills_dir(&self, repo_root: &Path) -> PathBuf { - repo_root.join(".claude").join("skills") - } - fn rewrites_frontmatter_name(&self) -> bool { - false - } - fn advertises_staged_slug_name(&self) -> bool { - false - } - fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { - render_available_skills_block(skills) - } - fn skill_surface_phrase(&self) -> &'static str { - "via the Skill tool" - } - fn skill_unresolved_phrase(&self) -> &'static str { - "If the Skill tool cannot resolve that identifier" - } - fn cli_events_filename(&self) -> Option<&'static str> { - Some("claude-events.jsonl") - } - fn cli_model_flag(&self) -> Option<&'static str> { - Some("--model") - } - fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { - format!( - "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness claude-code`.", - claude_exec_command_template(self.cli_model_flag(), ctx.agent_model), - target_args = ctx.target_args, - iteration = ctx.iteration - ) - } - fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { - Some(vec![ - "After all dispatches (Claude Code):".to_string(), - String::new(), - "Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `) -> Option { - Some(claude_judge_dispatch_recipe( - self.cli_model_flag(), - ctx.iteration_dir, - )) - } - fn parse_cli_events(&self, path: &Path) -> io::Result> { - parse_claude_stream_json(path) - } - fn parse_cli_events_full(&self, path: &Path) -> io::Result { - parse_claude_stream_json_full(path) - } - fn install_guard( - &self, - stage_root: &Path, - guard_exe: &Path, - ttl: Option, - ) -> io::Result { - crate::sandbox::install::install_claude_guard(stage_root, guard_exe, ttl) - } - fn guard_armed_message(&self) -> Option<&'static str> { - Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches.\n Each `claude -p` dispatch loads the hook from the env cwd it runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", - ) - } -} - -impl HarnessAdapter for CodexAdapter { - fn label(&self) -> &'static str { - "codex" - } - fn skills_dir(&self, repo_root: &Path) -> PathBuf { - repo_root.join(".agents").join("skills") - } - fn rewrites_frontmatter_name(&self) -> bool { - true - } - fn advertises_staged_slug_name(&self) -> bool { - true - } - fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { - render_codex_available_skills_block(skills) - } - fn skill_surface_phrase(&self) -> &'static str { - "as a Codex skill" - } - fn skill_unresolved_phrase(&self) -> &'static str { - "If it does not load as a Codex skill" - } - fn cli_events_filename(&self) -> Option<&'static str> { - Some("codex-events.jsonl") - } - fn cli_model_flag(&self) -> Option<&'static str> { - Some("-m") - } - fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { - format!( - "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness codex`.", - codex_exec_command_template(self.cli_model_flag(), ctx.guard, ctx.agent_model), - target_args = ctx.target_args, - iteration = ctx.iteration - ) - } - fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { - Some(vec![ - "After all dispatches (Codex):".to_string(), - String::new(), - "Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `) -> Option { - Some(codex_judge_dispatch_recipe( - self.cli_model_flag(), - ctx.guard, - ctx.iteration_dir, - )) - } - fn parse_cli_events(&self, path: &Path) -> io::Result> { - parse_codex_events(path) - } - fn parse_cli_events_full(&self, path: &Path) -> io::Result { - parse_codex_events_full(path) - } - fn install_guard( - &self, - stage_root: &Path, - guard_exe: &Path, - ttl: Option, - ) -> io::Result { - crate::sandbox::install::install_codex_guard(stage_root, guard_exe, ttl) - } - fn guard_armed_message(&self) -> Option<&'static str> { - Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .codex/hooks.json\n and will block writes/installs outside the eval sandbox during Codex dispatches.\n Dispatch with codex --ask-for-approval never exec --dangerously-bypass-hook-trust so the vetted eval hook runs.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", - ) - } -} - -impl HarnessAdapter for OpenCodeAdapter { - fn label(&self) -> &'static str { - "opencode" - } - fn skills_dir(&self, repo_root: &Path) -> PathBuf { - repo_root.join(".opencode").join("skills") - } - fn rewrites_frontmatter_name(&self) -> bool { - true - } - fn advertises_staged_slug_name(&self) -> bool { - false - } - fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { - render_opencode_available_skills_block(skills) - } - fn skill_surface_phrase(&self) -> &'static str { - "as an OpenCode skill" - } - fn skill_unresolved_phrase(&self) -> &'static str { - "If it does not load as an OpenCode skill" - } - fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { - let model_note = if ctx.agent_model.is_some() { - " Model selection was recorded as provenance, but the OpenCode adapter has no CLI model flag wired yet." - } else { - "" - }; - format!( - "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task with `opencode run`.{model_note} OpenCode transcript ingest is not yet wired, so assemble each task's `run.json`/`timing.json` manually (or capture `opencode run --format json` / `opencode export` output), then run `ingest{target_args} --iteration {iteration} --harness opencode`.", - target_args = ctx.target_args, - iteration = ctx.iteration - ) - } - // OpenCode transcript ingest is not yet wired: its `cli_events_filename` is - // `None`, so the ingest pipeline never reaches these parsers. They error - // rather than parse until OpenCode ingest lands. - fn parse_cli_events(&self, _path: &Path) -> io::Result> { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "opencode transcript ingest is not yet wired", - )) - } - fn parse_cli_events_full(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "opencode transcript ingest is not yet wired", - )) - } - fn install_guard( - &self, - _stage_root: &Path, - _guard_exe: &Path, - _ttl: Option, - ) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "--guard is not yet supported for the opencode harness", - )) - } -} - /// Resolve the adapter for a [`Harness`]. This is the single dispatch point on /// the harness variant for all harness-specific behavior; every other module /// goes through the returned trait object. pub fn adapter_for(harness: Harness) -> &'static dyn HarnessAdapter { match harness { - Harness::ClaudeCode => &ClaudeCodeAdapter, - Harness::Codex => &CodexAdapter, - Harness::OpenCode => &OpenCodeAdapter, + Harness::ClaudeCode => &super::claude_code::ClaudeCodeAdapter, + Harness::Codex => &super::codex::CodexAdapter, + Harness::OpenCode => &super::opencode::OpenCodeAdapter, } } @@ -441,20 +199,13 @@ mod tests { #[test] fn plan_mode_context_wraps_in_system_reminder_for_every_harness() { - for h in [Harness::ClaudeCode, Harness::Codex, Harness::OpenCode] { + for h in Harness::ALL { let out = adapter_for(h).render_plan_mode_context("BODY"); assert_eq!(out, "\nBODY\n"); assert_eq!(adapter_for(h).render_plan_mode_context(" "), ""); } } - #[test] - fn claude_adapter_advertises_cli_events_file_and_model_flag() { - let a = adapter_for(Harness::ClaudeCode); - assert_eq!(a.cli_events_filename(), Some("claude-events.jsonl")); - assert_eq!(a.cli_model_flag(), Some("--model")); - } - #[test] fn guard_armed_message_is_harness_specific_and_absent_for_opencode() { // The post-arm `--guard` banner names the harness's native hook surface, @@ -479,44 +230,4 @@ mod tests { // banner to print. assert_eq!(adapter_for(Harness::OpenCode).guard_armed_message(), None); } - - #[test] - fn claude_parse_cli_events_full_reads_stream_json_result_event() { - use serde_json::json; - let dir = tempfile::TempDir::new().unwrap(); - let path = dir.path().join("claude-events.jsonl"); - // No per-line timestamps; the result event is the only source of duration. - let lines = [ - json!({"type": "assistant", "message": {"id": "msg_1", "role": "assistant", "content": [ - {"type": "tool_use", "id": "toolu_1", "name": "Bash", "input": {"command": "ls"}} - ]}}), - json!({"type": "result", "subtype": "success", "is_error": false, "result": "Done", "duration_ms": 5637, "usage": {"input_tokens": 1, "output_tokens": 2, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}), - ]; - let body = lines - .iter() - .map(|l| l.to_string()) - .collect::>() - .join("\n"); - std::fs::write(&path, format!("{body}\n")).unwrap(); - - let a = adapter_for(Harness::ClaudeCode); - let summary = a.parse_cli_events_full(&path).unwrap(); - assert_eq!(summary.final_text, Some("Done".into())); - assert_eq!(summary.duration_ms, Some(5637)); - assert_eq!(summary.tool_invocations.len(), 1); - assert_eq!(summary.tool_invocations[0].name, "Bash"); - } - - #[test] - fn codex_parse_cli_events_delegates_to_events_parser() { - use serde_json::json; - let dir = tempfile::TempDir::new().unwrap(); - let path = dir.path().join("codex-events.jsonl"); - let line = json!({"type": "item.completed", "item": {"id": "i1", "type": "command_execution", "command": "bun test", "output": "ok"}}); - std::fs::write(&path, format!("{line}\n")).unwrap(); - - let inv = adapter_for(Harness::Codex).parse_cli_events(&path).unwrap(); - assert_eq!(inv.len(), 1); - assert_eq!(inv[0].name, "command_execution"); - } } diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 0244ce0..aa3c38c 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -1,37 +1,26 @@ //! The harness adapter layer. //! //! [`harness`] defines the [`HarnessAdapter`] trait — the single API generic -//! dispatch code uses to reach harness-specific behavior. The per-harness -//! session renderers + transcript parsers it delegates to live in the sibling -//! submodules, plus plugin-shadow detection. The submodules are re-exported -//! flat so downstream code writes `crate::adapters::`. +//! dispatch code uses to reach harness-specific behavior. Everything specific +//! to one harness lives in that harness's module tree ([`claude_code`], +//! [`codex`], [`opencode`]): the adapter impl, session renderers, transcript +//! parsers, dispatch-recipe rendering, and write-guard hooks. Generic code +//! resolves an adapter with [`adapter_for`] and calls the trait. -mod claude_cli; -pub mod claude_code_session; -pub mod claude_code_transcript; -pub mod claude_stream_json; +pub mod claude_code; mod cli_command; -mod codex_cli; -pub mod codex_session; -pub mod codex_transcript; +pub mod codex; pub mod harness; -pub mod opencode_session; -pub mod plugin_shadow; +pub mod opencode; +pub mod transcript; pub use harness::{ - ClaudeCodeAdapter, CliDispatchContext, CliJudgeContext, CliManifestContext, CodexAdapter, - HarnessAdapter, OpenCodeAdapter, RUNBOOK_TEMPLATE, adapter_for, + CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, RUNBOOK_TEMPLATE, + adapter_for, }; +pub use transcript::TranscriptSummary; -pub use claude_code_session::{render_available_skills_block, render_plan_mode_context}; -pub use claude_code_transcript::TranscriptSummary; -pub use claude_stream_json::{parse_claude_stream_json, parse_claude_stream_json_full}; -pub use codex_session::{render_codex_available_skills_block, render_codex_plan_mode_context}; -pub use codex_transcript::{parse_codex_events, parse_codex_events_full}; -pub use opencode_session::{ - render_opencode_available_skills_block, render_opencode_plan_mode_context, -}; -pub use plugin_shadow::{ +pub use claude_code::plugin_shadow::{ PluginShadowReport, ShadowSource, config_dir_from_env, detect_plugin_shadows, format_shadow_banner, resolve_config_dir, shadow_validity_warnings, }; diff --git a/src/adapters/opencode/mod.rs b/src/adapters/opencode/mod.rs new file mode 100644 index 0000000..d51e23a --- /dev/null +++ b/src/adapters/opencode/mod.rs @@ -0,0 +1,192 @@ +//! OpenCode harness support. +//! +//! Everything OpenCode-specific lives in this module tree: the adapter impl and +//! slug/naming rules (this file) and the `` XML block +//! ([`session`]). Transcript ingest, the model flag, and the write guard are +//! not wired yet; the adapter's error stubs say so. + +pub mod session; + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::core::{AvailableSkill, ToolInvocation}; + +use super::TranscriptSummary; +use super::harness::{CliDispatchContext, HarnessAdapter}; +use session::render_opencode_available_skills_block; + +pub struct OpenCodeAdapter; + +impl HarnessAdapter for OpenCodeAdapter { + fn label(&self) -> &'static str { + "opencode" + } + fn skills_dir(&self, repo_root: &Path) -> PathBuf { + repo_root.join(".opencode").join("skills") + } + fn rewrites_frontmatter_name(&self) -> bool { + true + } + fn advertises_staged_slug_name(&self) -> bool { + false + } + fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { + render_opencode_available_skills_block(skills) + } + fn skill_surface_phrase(&self) -> &'static str { + "as an OpenCode skill" + } + fn skill_unresolved_phrase(&self) -> &'static str { + "If it does not load as an OpenCode skill" + } + fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { + let model_note = if ctx.agent_model.is_some() { + " Model selection was recorded as provenance, but the OpenCode adapter has no CLI model flag wired yet." + } else { + "" + }; + format!( + "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task with `opencode run`.{model_note} OpenCode transcript ingest is not yet wired, so assemble each task's `run.json`/`timing.json` manually (or capture `opencode run --format json` / `opencode export` output), then run `ingest{target_args} --iteration {iteration} --harness opencode`.", + target_args = ctx.target_args, + iteration = ctx.iteration + ) + } + // OpenCode transcript ingest is not yet wired: its `cli_events_filename` is + // `None`, so the ingest pipeline never reaches these parsers. They error + // rather than parse until OpenCode ingest lands. + fn parse_cli_events(&self, _path: &Path) -> io::Result> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "opencode transcript ingest is not yet wired", + )) + } + fn parse_cli_events_full(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "opencode transcript ingest is not yet wired", + )) + } + fn install_guard( + &self, + _stage_root: &Path, + _guard_exe: &Path, + _ttl: Option, + ) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + "--guard is not yet supported for the opencode harness", + )) + } +} + +/// True when `name` satisfies OpenCode's skill-name rules: +/// - 1–64 characters +/// - lowercase alphanumeric with single-hyphen separators +/// - no leading/trailing/consecutive hyphens +pub(crate) fn is_valid_opencode_name(name: &str) -> bool { + if name.is_empty() || name.len() > 64 { + return false; + } + let mut prev = '-'; + for ch in name.chars() { + if ch == '-' { + if prev == '-' { + return false; + } + } else if !ch.is_ascii_lowercase() && !ch.is_ascii_digit() { + return false; + } + prev = ch; + } + !name.starts_with('-') && !name.ends_with('-') +} + +/// Sanitize an arbitrary identifier so it is a valid OpenCode skill name. +fn sanitize_opencode_name(name: &str) -> String { + let mut out = String::new(); + let mut prev_hyphen = false; + for ch in name.to_ascii_lowercase().chars() { + if ch.is_ascii_lowercase() || ch.is_ascii_digit() { + out.push(ch); + prev_hyphen = false; + } else if !prev_hyphen { + out.push('-'); + prev_hyphen = true; + } + } + while out.ends_with('-') { + out.pop(); + } + while out.starts_with('-') { + out.remove(0); + } + if out.is_empty() { + out.push_str("skill"); + } + if out.len() > 64 { + out.truncate(64); + while out.ends_with('-') { + out.pop(); + } + } + out +} + +/// Build a slug that is valid for OpenCode's skill directory + frontmatter name +/// constraints. `prefix` is the conspicuous staged-skill prefix, preserved so +/// cleanup prefix-scans still find it. +pub(crate) fn opencode_slug( + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, +) -> String { + let condition = sanitize_opencode_name(condition); + let skill = sanitize_opencode_name(skill_name); + let base = format!("{prefix}{iteration}-{condition}-{skill}"); + if base.len() <= 64 && is_valid_opencode_name(&base) { + return base; + } + // If the combined slug is too long, truncate the skill portion. + let prefix = format!("{prefix}{iteration}-{condition}-"); + let budget = 64usize.saturating_sub(prefix.len()); + let mut truncated = skill.clone(); + truncated.truncate(budget); + while truncated.ends_with('-') { + truncated.pop(); + } + if truncated.is_empty() { + truncated.push_str("skill"); + } + format!("{prefix}{truncated}") +} + +#[cfg(test)] +mod tests { + use super::*; + + const PREFIX: &str = "slow-powers-eval-"; + + #[test] + fn opencode_slug_sanitizes_underscores_and_special_characters() { + assert_eq!( + opencode_slug(PREFIX, 1, "with_skill", "My_Skill!"), + "slow-powers-eval-1-with-skill-my-skill" + ); + assert_eq!( + opencode_slug(PREFIX, 2, "without_skill", "snake_case"), + "slow-powers-eval-2-without-skill-snake-case" + ); + } + + #[test] + fn opencode_slug_truncates_to_valid_max_length() { + let very_long = "a".repeat(200); + let slug = opencode_slug(PREFIX, 1, "with_skill", &very_long); + assert!(slug.len() <= 64); + assert!(is_valid_opencode_name(&slug)); + assert!(slug.starts_with("slow-powers-eval-1-with-skill-")); + } +} diff --git a/src/adapters/opencode_session.rs b/src/adapters/opencode/session.rs similarity index 100% rename from src/adapters/opencode_session.rs rename to src/adapters/opencode/session.rs diff --git a/src/adapters/transcript.rs b/src/adapters/transcript.rs new file mode 100644 index 0000000..c31b78b --- /dev/null +++ b/src/adapters/transcript.rs @@ -0,0 +1,22 @@ +//! Harness-neutral transcript types. +//! +//! Every harness's transcript parser reduces its native events file to a +//! [`TranscriptSummary`]; the pipeline consumes only this shape, never a +//! harness's raw record types. + +use serde::{Deserialize, Serialize}; + +use crate::core::ToolInvocation; + +/// A transcript boiled down to the artifacts the pipeline needs. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct TranscriptSummary { + pub tool_invocations: Vec, + /// Total token usage (input + output + cache creation/read), as reported by + /// the run's terminal `result` event. + pub total_tokens: Option, + /// Wall-clock duration, as reported by the run's terminal `result` event. + pub duration_ms: Option, + /// Concatenated text blocks of the last assistant message. + pub final_text: Option, +} diff --git a/src/cli/commands/guard.rs b/src/cli/commands/guard.rs index c4211d0..47b47b8 100644 --- a/src/cli/commands/guard.rs +++ b/src/cli/commands/guard.rs @@ -1,21 +1,29 @@ -//! Write-guard command handlers: the hidden `guard` PreToolUse hook entry point -//! and the user-facing `teardown-guard`. +//! Write-guard command handlers: the hidden per-harness PreToolUse hook entry +//! points and the user-facing `teardown-guard`. +//! +//! The `guard` / `guard-codex` subcommand names are a **stable on-disk +//! contract**: armed hooks staged into harness config reference them by name, +//! so renaming either would break every already-armed guard. use std::io; use std::path::PathBuf; +use crate::adapters::{adapter_for, claude_code, codex}; +use crate::core::Harness; use crate::sandbox; -/// The hidden PreToolUse hook entry point. Reads the hook payload from stdin and -/// the marker path from argv, then prints a deny verdict for out-of-bounds calls. -/// It **fails open** — every error path allows the call and exits 0, so the -/// guard can never brick a session. +/// The hidden Claude Code PreToolUse hook entry point. Reads the hook payload +/// from stdin and the marker path from argv, then prints a deny verdict for +/// out-of-bounds calls. It **fails open** — every error path allows the call +/// and exits 0, so the guard can never brick a session. pub(crate) fn run_guard(marker: Option) -> anyhow::Result<()> { let marker_path = marker .map(PathBuf::from) - .unwrap_or_else(default_marker_path); + .unwrap_or_else(|| default_marker_path(Harness::ClaudeCode)); let payload = io::read_to_string(io::stdin()).unwrap_or_default(); - if let Some(verdict) = sandbox::guard_decision(&payload, sandbox::read_marker(&marker_path)) { + if let Some(verdict) = + claude_code::guard::guard_decision(&payload, sandbox::read_marker(&marker_path)) + { print!("{verdict}"); } Ok(()) @@ -27,10 +35,10 @@ pub(crate) fn run_guard(marker: Option) -> anyhow::Result<()> { pub(crate) fn run_guard_codex(marker: Option) -> anyhow::Result<()> { let marker_path = marker .map(PathBuf::from) - .unwrap_or_else(default_codex_marker_path); + .unwrap_or_else(|| default_marker_path(Harness::Codex)); let payload = io::read_to_string(io::stdin()).unwrap_or_default(); if let Some(verdict) = - sandbox::codex_guard_decision(&payload, sandbox::read_marker(&marker_path)) + codex::guard::guard_decision(&payload, sandbox::read_marker(&marker_path)) { print!("{verdict}"); } @@ -53,22 +61,10 @@ pub(crate) fn run_teardown_guard() -> anyhow::Result<()> { Ok(()) } -/// The marker path the guard reads when argv carries none: -/// `/.claude/skills/.slow-powers-eval-guard.json`. -fn default_marker_path() -> PathBuf { - std::env::current_dir() - .unwrap_or_default() - .join(".claude") - .join("skills") - .join(sandbox::GUARD_MARKER) -} - -/// The marker path the Codex guard reads when argv carries none: -/// `/.agents/skills/.slow-powers-eval-guard.json`. -fn default_codex_marker_path() -> PathBuf { - std::env::current_dir() - .unwrap_or_default() - .join(".agents") - .join("skills") +/// The marker path a guard hook reads when argv carries none: the harness's +/// skills dir under the cwd, e.g. `/.claude/skills/.slow-powers-eval-guard.json`. +fn default_marker_path(harness: Harness) -> PathBuf { + adapter_for(harness) + .skills_dir(&std::env::current_dir().unwrap_or_default()) .join(sandbox::GUARD_MARKER) } diff --git a/src/cli/run/staging/mod.rs b/src/cli/run/staging/mod.rs index d15ba80..a502f84 100644 --- a/src/cli/run/staging/mod.rs +++ b/src/cli/run/staging/mod.rs @@ -142,96 +142,27 @@ fn prune_if_empty(dir: &Path) -> Result<(), RunError> { Ok(()) } -/// True when `name` satisfies OpenCode's skill-name rules: -/// - 1–64 characters -/// - lowercase alphanumeric with single-hyphen separators -/// - no leading/trailing/consecutive hyphens -fn is_valid_opencode_name(name: &str) -> bool { - if name.is_empty() || name.len() > 64 { - return false; - } - let mut prev = '-'; - for ch in name.chars() { - if ch == '-' { - if prev == '-' { - return false; - } - } else if !ch.is_ascii_lowercase() && !ch.is_ascii_digit() { - return false; - } - prev = ch; - } - !name.starts_with('-') && !name.ends_with('-') -} - -/// Sanitize an arbitrary identifier so it is a valid OpenCode skill name. -fn sanitize_opencode_name(name: &str) -> String { - let mut out = String::new(); - let mut prev_hyphen = false; - for ch in name.to_ascii_lowercase().chars() { - if ch.is_ascii_lowercase() || ch.is_ascii_digit() { - out.push(ch); - prev_hyphen = false; - } else if !prev_hyphen { - out.push('-'); - prev_hyphen = true; - } - } - while out.ends_with('-') { - out.pop(); - } - while out.starts_with('-') { - out.remove(0); - } - if out.is_empty() { - out.push_str("skill"); - } - if out.len() > 64 { - out.truncate(64); - while out.ends_with('-') { - out.pop(); - } - } - out -} - -/// Build a slug that is valid for OpenCode's skill directory + frontmatter name -/// constraints. The prefix is preserved so cleanup prefix-scans still find it. -fn opencode_slug(iteration: u32, condition: &str, skill_name: &str) -> String { - let condition = sanitize_opencode_name(condition); - let skill = sanitize_opencode_name(skill_name); - let base = format!("{STAGED_SKILL_PREFIX}{iteration}-{condition}-{skill}"); - if base.len() <= 64 && is_valid_opencode_name(&base) { - return base; - } - // If the combined slug is too long, truncate the skill portion. - let prefix = format!("{STAGED_SKILL_PREFIX}{iteration}-{condition}-"); - let budget = 64usize.saturating_sub(prefix.len()); - let mut truncated = skill.clone(); - truncated.truncate(budget); - while truncated.ends_with('-') { - truncated.pop(); - } - if truncated.is_empty() { - truncated.push_str("skill"); - } - format!("{prefix}{truncated}") -} - /// Stage one skill under the harness's skills dir and return its slug. For Codex /// and OpenCode the frontmatter `name:` is rewritten to the slug. pub fn stage_skill_for_harness(opts: &StageSkillOpts) -> Result { let slug = match opts.stage_name_override { Some(name) => name.to_string(), None => match opts.harness { - Harness::OpenCode => opencode_slug(opts.iteration, opts.condition, opts.skill_name), + Harness::OpenCode => crate::adapters::opencode::opencode_slug( + STAGED_SKILL_PREFIX, + opts.iteration, + opts.condition, + opts.skill_name, + ), _ => format!( "{STAGED_SKILL_PREFIX}{}-{}__{}", opts.iteration, opts.condition, opts.skill_name ), }, }; - if opts.harness == Harness::OpenCode && !is_valid_opencode_name(&slug) { + if opts.harness == Harness::OpenCode + && !crate::adapters::opencode::is_valid_opencode_name(&slug) + { return Err(RunError::msg(format!( "OpenCode skill name \"{slug}\" is invalid; names must be 1-64 lowercase alphanumeric characters separated by single hyphens" ))); diff --git a/src/cli/run/staging/tests/stage.rs b/src/cli/run/staging/tests/stage.rs index e24b653..835a445 100644 --- a/src/cli/run/staging/tests/stage.rs +++ b/src/cli/run/staging/tests/stage.rs @@ -209,27 +209,8 @@ fn codex_stage_name_override_is_dir_and_frontmatter_name() { } // ── stage_skill_for_harness (opencode) ──────────────────────────────── - -#[test] -fn opencode_slug_sanitizes_underscores_and_special_characters() { - assert_eq!( - opencode_slug(1, "with_skill", "My_Skill!"), - "slow-powers-eval-1-with-skill-my-skill" - ); - assert_eq!( - opencode_slug(2, "without_skill", "snake_case"), - "slow-powers-eval-2-without-skill-snake-case" - ); -} - -#[test] -fn opencode_slug_truncates_to_valid_max_length() { - let very_long = "a".repeat(200); - let slug = opencode_slug(1, "with_skill", &very_long); - assert!(slug.len() <= 64); - assert!(is_valid_opencode_name(&slug)); - assert!(slug.starts_with("slow-powers-eval-1-with-skill-")); -} +// The opencode slug/naming unit tests live with the slug functions in +// `crate::adapters::opencode`. #[test] fn opencode_stages_under_opencode_skills_and_rewrites_frontmatter_name() { diff --git a/src/core/context.rs b/src/core/context.rs index e67bb72..ab7ae8e 100644 --- a/src/core/context.rs +++ b/src/core/context.rs @@ -23,6 +23,12 @@ pub enum Harness { OpenCode, } +impl Harness { + /// Every supported harness, for code that must sweep all of them (e.g. + /// guard teardown scans each harness's skills dir for a marker). + pub const ALL: [Harness; 3] = [Harness::ClaudeCode, Harness::Codex, Harness::OpenCode]; +} + /// The resolved environment for a run: validated skill location, sibling skills, /// workspace/stage roots, optional bootstrap file, and the target harness. Built /// by [`detect_run_context`]; held in memory and never (de)serialized. diff --git a/src/sandbox/guard.rs b/src/sandbox/guard.rs index 51af2db..43687f1 100644 --- a/src/sandbox/guard.rs +++ b/src/sandbox/guard.rs @@ -1,17 +1,17 @@ -//! Guard hook evaluation. +//! Shared guard-hook plumbing. //! -//! Library logic behind the binary's hidden `guard` subcommand: the CLI handler -//! reads the hook payload from stdin and the marker path from argv, calls -//! [`guard_decision`], and writes any deny verdict to stdout. Both layers fail -//! open — a malformed payload or unreadable marker yields "allow", so the guard -//! can never brick a session. +//! The harness-specific verdict shaping lives with each harness's guard module +//! (`crate::adapters::::guard`): the CLI handler reads the hook +//! payload from stdin and the marker path from argv, and the harness guard +//! calls [`parse_tool_call`] + `decide` to evaluate it. Both layers fail open — +//! a malformed payload or unreadable marker yields "allow", so the guard can +//! never brick a session. use std::path::Path; -use serde_json::{Map, Value, json}; +use serde_json::{Map, Value}; -use super::decide::{GuardMarker, decide}; -use super::now_ms; +use super::decide::GuardMarker; /// Read and parse the guard marker. Missing or unparseable → `None` (the guard /// then allows everything — fail open). @@ -20,48 +20,10 @@ pub fn read_marker(path: &Path) -> Option { serde_json::from_str(&text).ok() } -/// Evaluate a PreToolUse hook `payload` (the JSON the harness sends on stdin) -/// against `marker`. Returns the serialized deny verdict to print on stdout when -/// the call is blocked, or `None` to allow (print nothing). An empty or malformed -/// payload is treated as allow. -pub fn guard_decision(payload: &str, marker: Option) -> Option { - let (tool_name, tool_input) = parse_tool_call(payload)?; - - let decision = decide(&tool_name, &tool_input, marker.as_ref(), now_ms()); - if decision.allow { - return None; - } - Some( - serde_json::to_string(&json!({ - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "permissionDecision": "deny", - "permissionDecisionReason": decision.reason, - } - })) - .expect("deny verdict serializes"), - ) -} - -/// Codex's hook contract blocks by returning `{ "decision": "block", "reason": -/// "..." }` on stdout. Keep this separate from Claude Code's -/// `hookSpecificOutput` shape so both harnesses use their native conventions. -pub fn codex_guard_decision(payload: &str, marker: Option) -> Option { - let (tool_name, tool_input) = parse_tool_call(payload)?; - let decision = decide(&tool_name, &tool_input, marker.as_ref(), now_ms()); - if decision.allow { - return None; - } - Some( - serde_json::to_string(&json!({ - "decision": "block", - "reason": decision.reason, - })) - .expect("Codex block verdict serializes"), - ) -} - -fn parse_tool_call(payload: &str) -> Option<(String, Value)> { +/// Extract the tool name + input from a PreToolUse hook payload (the JSON the +/// harness sends on stdin). `None` for an empty or malformed payload — treated +/// as allow by every caller. +pub(crate) fn parse_tool_call(payload: &str) -> Option<(String, Value)> { let trimmed = payload.trim(); let parsed: Value = serde_json::from_str(if trimmed.is_empty() { "{}" } else { trimmed }).ok()?; @@ -109,74 +71,3 @@ fn merge_top_level_files(input: Value, parsed: &Value) -> Value { other => other, } } - -#[cfg(test)] -mod tests { - use super::*; - - /// A live marker (active, no expiry → unexpired) scoped to one root. - fn marker() -> GuardMarker { - GuardMarker { - active: Some(true), - allowed_roots: Some(vec!["/work/.eval-magic".to_string()]), - expires_at: None, - } - } - - #[test] - fn allows_returns_none() { - let payload = r#"{ "tool_name": "Read", "tool_input": { "file_path": "/etc/passwd" } }"#; - assert_eq!(guard_decision(payload, Some(marker())), None); - } - - #[test] - fn deny_returns_pretooluse_deny_json() { - let payload = r#"{ "tool_name": "Write", "tool_input": { "file_path": "/etc/passwd" } }"#; - let out = guard_decision(payload, Some(marker())).expect("should deny"); - let v: Value = serde_json::from_str(&out).unwrap(); - assert_eq!(v["hookSpecificOutput"]["hookEventName"], "PreToolUse"); - assert_eq!(v["hookSpecificOutput"]["permissionDecision"], "deny"); - assert!( - v["hookSpecificOutput"]["permissionDecisionReason"] - .as_str() - .unwrap() - .contains("outside") - ); - } - - #[test] - fn no_marker_allows_everything() { - let payload = r#"{ "tool_name": "Write", "tool_input": { "file_path": "/etc/passwd" } }"#; - assert_eq!(guard_decision(payload, None), None); - } - - #[test] - fn codex_deny_returns_decision_block_json() { - let payload = r#"{ "hook_event_name": "PreToolUse", "tool_name": "Bash", "tool_input": { "command": "npm install left-pad" } }"#; - let out = codex_guard_decision(payload, Some(marker())).expect("should block"); - let v: Value = serde_json::from_str(&out).unwrap(); - assert_eq!(v["decision"], "block"); - assert!(v["reason"].as_str().unwrap().contains("blocked Bash")); - } - - #[test] - fn codex_apply_patch_outside_allowed_roots_blocks() { - let payload = r#"{ "hook_event_name": "PreToolUse", "tool_name": "apply_patch", "tool_input": { "files": ["/etc/passwd"] } }"#; - let out = codex_guard_decision(payload, Some(marker())).expect("should block"); - let v: Value = serde_json::from_str(&out).unwrap(); - assert_eq!(v["decision"], "block"); - assert!(v["reason"].as_str().unwrap().contains("apply_patch")); - } - - #[test] - fn codex_apply_patch_inside_allowed_roots_allows() { - let payload = r#"{ "hook_event_name": "PreToolUse", "tool_name": "apply_patch", "tool_input": { "files": ["/work/.eval-magic/out.md"] } }"#; - assert_eq!(codex_guard_decision(payload, Some(marker())), None); - } - - #[test] - fn empty_or_malformed_payload_fails_open() { - assert_eq!(guard_decision("", Some(marker())), None); - assert_eq!(guard_decision("not json", Some(marker())), None); - } -} diff --git a/src/sandbox/install.rs b/src/sandbox/install.rs index 6f9d774..7b57c0d 100644 --- a/src/sandbox/install.rs +++ b/src/sandbox/install.rs @@ -1,9 +1,10 @@ -//! Arm / disarm the write guard. +//! Shared write-guard arm/disarm machinery. //! -//! [`install_guard`] writes a marker listing the allowed roots and merges a -//! `PreToolUse` hook into the target harness's project config. The original hook -//! file is backed up verbatim in a manifest so [`teardown_guard`] restores it -//! exactly. +//! Each harness's installer (in its adapter module, e.g. +//! `crate::adapters::claude_code::guard`) writes a marker listing the allowed +//! roots and merges a `PreToolUse` hook into that harness's project config, +//! using the marker/manifest helpers here. The original hook file is backed up +//! verbatim in a manifest so [`teardown_guard`] restores it exactly. //! //! The hook command points at the running binary (`std::env::current_exe`), so //! there is no separate hook script to ship and no interpreter to select. @@ -15,7 +16,10 @@ use std::time::Duration; use chrono::{DateTime, SecondsFormat}; use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; +use serde_json::Value; +use serde_json::json; + +use crate::core::Harness; use super::now_ms; use super::{guard::read_marker, marker_is_armed}; @@ -29,11 +33,6 @@ pub const GUARD_MANIFEST: &str = ".slow-powers-eval-guard-manifest.json"; /// linger before it is treated as expired (see `super::decide`). const GUARD_TTL: Duration = Duration::from_secs(6 * 60 * 60); // 6h -/// Tool names the Claude Code PreToolUse hook fires on. -const CLAUDE_HOOK_MATCHER: &str = "Write|Edit|MultiEdit|NotebookEdit|Bash"; -/// Tool names the Codex PreToolUse hook fires on. -const CODEX_HOOK_MATCHER: &str = "^Bash$|^apply_patch$|^Edit$|^Write$"; - /// Restoration record written beside the marker. The field names are the /// on-disk manifest format — keep them stable so older manifests stay readable. #[derive(Debug, Serialize, Deserialize)] @@ -47,7 +46,7 @@ struct GuardManifest { /// Format epoch milliseconds as `2026-06-08T12:00:00.000Z` — RFC 3339 with /// millisecond precision, the timestamp format every artifact uses. -fn iso_millis(ms: i64) -> String { +pub(crate) fn iso_millis(ms: i64) -> String { DateTime::from_timestamp_millis(ms) .unwrap_or_default() .to_rfc3339_opts(SecondsFormat::Millis, true) @@ -61,7 +60,7 @@ fn absolutize(p: &Path) -> PathBuf { /// Write `value` as 2-space-pretty JSON with a trailing newline — the stable /// on-disk format for every artifact this binary writes. -fn write_json(path: &Path, value: &Value) -> io::Result<()> { +pub(crate) fn write_json(path: &Path, value: &Value) -> io::Result<()> { let mut text = serde_json::to_string_pretty(value) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; text.push('\n'); @@ -69,14 +68,14 @@ fn write_json(path: &Path, value: &Value) -> io::Result<()> { } /// The guard's allowed write roots: the isolated env (`stage_root`, the -/// agent-under-test's cwd) and the OS temp dir. The staged skills dir -/// (`stage_root/.claude/skills` or `.agents/skills`) and the per-task outputs dir -/// both live *inside* `stage_root`, so a single env root covers every legitimate -/// agent write. Scoping to the env — not the parent `.eval-magic/` — keeps the -/// guard boundary identical to the isolation boundary: the agent can't reach a -/// sibling iteration or the `iteration-N/` meta tree above its cwd. eval-magic's own -/// above-env writes (e.g. `benchmark.json`) are not gated here: they run as -/// non-mutating `eval-magic` subprocesses the guard's Bash classifier passes. +/// agent-under-test's cwd) and the OS temp dir. The staged skills dir and the +/// per-task outputs dir both live *inside* `stage_root`, so a single env root +/// covers every legitimate agent write. Scoping to the env — not the parent +/// `.eval-magic/` — keeps the guard boundary identical to the isolation +/// boundary: the agent can't reach a sibling iteration or the `iteration-N/` +/// meta tree above its cwd. eval-magic's own above-env writes (e.g. +/// `benchmark.json`) are not gated here: they run as non-mutating `eval-magic` +/// subprocesses the guard's Bash classifier passes. fn marker_allowed_roots(stage_root: &Path) -> Vec { vec![ absolutize(stage_root).display().to_string(), @@ -84,7 +83,14 @@ fn marker_allowed_roots(stage_root: &Path) -> Vec { ] } -fn write_marker(marker_path: &Path, stage_root: &Path, ttl: Option) -> io::Result<()> { +/// Write the guard marker that arms the hook for `stage_root`. The guard is a +/// no-op until this marker exists and is unexpired, so the hook is inert +/// outside an active run. `ttl` overrides the default 6h lifetime. +pub(crate) fn write_marker( + marker_path: &Path, + stage_root: &Path, + ttl: Option, +) -> io::Result<()> { let expires_ms = now_ms() + ttl.unwrap_or(GUARD_TTL).as_millis() as i64; write_json( marker_path, @@ -96,7 +102,8 @@ fn write_marker(marker_path: &Path, stage_root: &Path, ttl: Option) -> ) } -fn write_manifest( +/// Write the restoration manifest recording what install changed. +pub(crate) fn write_manifest( manifest_path: &Path, settings_path: &Path, settings_existed: bool, @@ -117,163 +124,29 @@ fn write_manifest( ) } -/// Arm the write guard for an eval run. Returns the marker path. The guard is a -/// no-op until this marker exists and is unexpired, so the hook is inert outside -/// an active run. `guard_exe` is the path the hook invokes (normally -/// `std::env::current_exe()`); `ttl` overrides the default 6h lifetime. -pub fn install_guard( - stage_root: &Path, - guard_exe: &Path, - ttl: Option, -) -> io::Result { - install_claude_guard(stage_root, guard_exe, ttl) -} - -pub(crate) fn install_claude_guard( - stage_root: &Path, - guard_exe: &Path, - ttl: Option, -) -> io::Result { - let skills_dir = stage_root.join(".claude").join("skills"); - fs::create_dir_all(&skills_dir)?; - - let marker_path = skills_dir.join(GUARD_MARKER); - write_marker(&marker_path, stage_root, ttl)?; - - let settings_path = stage_root.join(".claude").join("settings.local.json"); - let settings_existed = settings_path.exists(); - let backup = if settings_existed { - Some(fs::read_to_string(&settings_path)?) - } else { - None - }; - - // Start from the existing settings (or an empty object), preserving key - // order, then append the PreToolUse hook entry. - let mut settings: Value = backup - .as_deref() - .and_then(|s| serde_json::from_str(s).ok()) - .unwrap_or_else(|| json!({})); - let hooks = settings - .as_object_mut() - .expect("settings is a JSON object") - .entry("hooks") - .or_insert_with(|| json!({})); - let pre = hooks - .as_object_mut() - .expect("hooks is a JSON object") - .entry("PreToolUse") - .or_insert_with(|| json!([])); - let command = format!( - "\"{}\" guard \"{}\"", - guard_exe.display(), - marker_path.display() - ); - pre.as_array_mut() - .expect("PreToolUse is an array") - .push(json!({ - "matcher": CLAUDE_HOOK_MATCHER, - "hooks": [ { "type": "command", "command": command } ], - })); - write_json(&settings_path, &settings)?; - - write_manifest( - &skills_dir.join(GUARD_MANIFEST), - &settings_path, - settings_existed, - backup, - &marker_path, - )?; - - Ok(marker_path) -} - -pub(crate) fn install_codex_guard( - stage_root: &Path, - guard_exe: &Path, - ttl: Option, -) -> io::Result { - let skills_dir = stage_root.join(".agents").join("skills"); - fs::create_dir_all(&skills_dir)?; - - let marker_path = skills_dir.join(GUARD_MARKER); - write_marker(&marker_path, stage_root, ttl)?; - - let hooks_path = stage_root.join(".codex").join("hooks.json"); - if let Some(parent) = hooks_path.parent() { - fs::create_dir_all(parent)?; - } - let hooks_existed = hooks_path.exists(); - let backup = if hooks_existed { - Some(fs::read_to_string(&hooks_path)?) - } else { - None - }; - - let mut hooks: Value = backup - .as_deref() - .and_then(|s| serde_json::from_str(s).ok()) - .unwrap_or_else(|| json!({})); - let hooks_obj = hooks - .as_object_mut() - .expect("hooks.json root is a JSON object") - .entry("hooks") - .or_insert_with(|| json!({})); - let pre = hooks_obj - .as_object_mut() - .expect("hooks is a JSON object") - .entry("PreToolUse") - .or_insert_with(|| json!([])); - let command = format!( - "\"{}\" guard-codex \"{}\"", - guard_exe.display(), - marker_path.display() - ); - pre.as_array_mut() - .expect("PreToolUse is an array") - .push(json!({ - "matcher": CODEX_HOOK_MATCHER, - "hooks": [ - { - "type": "command", - "command": command, - "timeout": 30, - "statusMessage": "Checking eval write boundary", - } - ], - })); - write_json(&hooks_path, &hooks)?; - - write_manifest( - &skills_dir.join(GUARD_MANIFEST), - &hooks_path, - hooks_existed, - backup, - &marker_path, - )?; - - Ok(marker_path) -} - /// Disarm the guard: restore the original harness hook file (or delete it if we -/// created it) and remove the marker + manifest. Safe to call when no guard is -/// installed. Returns true if a guard was found and torn down. +/// created it) and remove the marker + manifest, for every harness's skills +/// dir. Safe to call when no guard is installed. Returns true if a guard was +/// found and torn down. pub fn teardown_guard(stage_root: &Path) -> bool { - let torn_claude = teardown_guard_from_skills_dir(&stage_root.join(".claude").join("skills")); - let torn_codex = teardown_guard_from_skills_dir(&stage_root.join(".agents").join("skills")); - let _ = prune_if_empty(&stage_root.join(".codex")); - torn_claude || torn_codex + let mut torn = false; + for harness in Harness::ALL { + let skills_dir = crate::adapters::adapter_for(harness).skills_dir(stage_root); + torn |= teardown_guard_from_skills_dir(&skills_dir); + } + let _ = prune_if_empty(&crate::adapters::codex::guard::hook_cleanup_dir(stage_root)); + torn } -/// True when either harness has a live guard marker under `stage_root`. +/// True when any harness has a live guard marker under `stage_root`. pub(crate) fn guard_is_armed(stage_root: &Path) -> bool { let now = now_ms(); - [ - stage_root.join(".claude").join("skills").join(GUARD_MARKER), - stage_root.join(".agents").join("skills").join(GUARD_MARKER), - ] - .iter() - .any(|path| marker_is_armed(read_marker(path).as_ref(), now)) + Harness::ALL.iter().any(|&harness| { + let marker_path = crate::adapters::adapter_for(harness) + .skills_dir(stage_root) + .join(GUARD_MARKER); + marker_is_armed(read_marker(&marker_path).as_ref(), now) + }) } fn teardown_guard_from_skills_dir(skills_dir: &Path) -> bool { @@ -336,141 +209,6 @@ mod tests { } } - fn skills_dir(stage_root: &Path) -> PathBuf { - stage_root.join(".claude").join("skills") - } - - fn settings_path(stage_root: &Path) -> PathBuf { - stage_root.join(".claude").join("settings.local.json") - } - - fn codex_hooks_path(stage_root: &Path) -> PathBuf { - stage_root.join(".codex").join("hooks.json") - } - - fn read_json(path: &Path) -> Value { - serde_json::from_str(&fs::read_to_string(path).unwrap()).unwrap() - } - - #[test] - fn install_writes_an_active_marker_hook_and_manifest() { - let c = setup(); - let exe = Path::new("/g/eval-magic"); - install_guard(&c.stage_root, exe, None).unwrap(); - - let marker = read_json(&skills_dir(&c.stage_root).join(GUARD_MARKER)); - assert_eq!(marker["active"], json!(true)); - let expires = marker["expiresAt"].as_str().unwrap(); - let exp_ms = DateTime::parse_from_rfc3339(expires) - .unwrap() - .timestamp_millis(); - assert!(exp_ms > now_ms()); - let env = absolutize(&c.stage_root).display().to_string(); - assert!( - marker["allowedRoots"] - .as_array() - .unwrap() - .iter() - .any(|r| r.as_str().unwrap() == env) - ); - - let settings = read_json(&settings_path(&c.stage_root)); - let hook = &settings["hooks"]["PreToolUse"][0]; - assert!(hook["matcher"].as_str().unwrap().contains("Write")); - assert!( - hook["hooks"][0]["command"] - .as_str() - .unwrap() - .contains("guard") - ); - - assert!(skills_dir(&c.stage_root).join(GUARD_MANIFEST).exists()); - } - - #[test] - fn marker_scopes_allowed_roots_to_the_env_and_temp_only() { - let c = setup(); - let exe = Path::new("/g/eval-magic"); - install_guard(&c.stage_root, exe, None).unwrap(); - - let marker = read_json(&skills_dir(&c.stage_root).join(GUARD_MARKER)); - let roots: Vec = marker["allowedRoots"] - .as_array() - .unwrap() - .iter() - .map(|r| r.as_str().unwrap().to_string()) - .collect(); - - // The guard boundary is the isolated env (stage_root) plus temp — nothing - // above it. The parent workspace tree must NOT be an allowed root, or the - // agent could write into sibling iterations / the meta dir above `env/`. - let env = absolutize(&c.stage_root).display().to_string(); - let temp = absolutize(&std::env::temp_dir()).display().to_string(); - assert_eq!(roots, vec![env, temp]); - assert!( - !roots.iter().any(|r| r.ends_with(".eval-magic")), - "workspace_root must not be an allowed root: {roots:?}" - ); - } - - #[test] - fn hook_command_invokes_the_binary_guard_subcommand() { - let c = setup(); - let exe = Path::new("/g/eval-magic"); - let marker = install_guard(&c.stage_root, exe, None).unwrap(); - let settings = read_json(&settings_path(&c.stage_root)); - let command = settings["hooks"]["PreToolUse"][0]["hooks"][0]["command"] - .as_str() - .unwrap() - .to_string(); - assert_eq!( - command, - format!("\"/g/eval-magic\" guard \"{}\"", marker.display()) - ); - } - - #[test] - fn teardown_deletes_settings_it_created() { - let c = setup(); - let exe = Path::new("/g/eval-magic"); - install_guard(&c.stage_root, exe, None).unwrap(); - assert!(settings_path(&c.stage_root).exists()); - - assert!(teardown_guard(&c.stage_root)); - assert!(!settings_path(&c.stage_root).exists()); - assert!(!skills_dir(&c.stage_root).join(GUARD_MARKER).exists()); - assert!(!skills_dir(&c.stage_root).join(GUARD_MANIFEST).exists()); - } - - #[test] - fn teardown_restores_a_pre_existing_settings_verbatim() { - let c = setup(); - fs::create_dir_all(c.stage_root.join(".claude")).unwrap(); - let original = format!( - "{}\n", - serde_json::to_string_pretty(&json!({ - "permissions": { "allow": ["Bash(ls)"] } - })) - .unwrap() - ); - fs::write(settings_path(&c.stage_root), &original).unwrap(); - - let exe = Path::new("/g/eval-magic"); - install_guard(&c.stage_root, exe, None).unwrap(); - // hook present while armed - assert!( - fs::read_to_string(settings_path(&c.stage_root)) - .unwrap() - .contains("PreToolUse") - ); - - teardown_guard(&c.stage_root); - assert_eq!( - fs::read_to_string(settings_path(&c.stage_root)).unwrap(), - original - ); - } - #[test] fn teardown_is_a_safe_no_op_when_nothing_is_installed() { let c = setup(); @@ -480,127 +218,13 @@ mod tests { #[test] fn guard_is_armed_detects_claude_or_codex_marker() { let c = setup(); - install_guard(&c.stage_root, Path::new("/g/eval-magic"), None).unwrap(); + let exe = Path::new("/g/eval-magic"); + crate::adapters::claude_code::guard::install_guard(&c.stage_root, exe, None).unwrap(); assert!(guard_is_armed(&c.stage_root)); teardown_guard(&c.stage_root); assert!(!guard_is_armed(&c.stage_root)); - install_codex_guard(&c.stage_root, Path::new("/g/eval-magic"), None).unwrap(); + crate::adapters::codex::guard::install_guard(&c.stage_root, exe, None).unwrap(); assert!(guard_is_armed(&c.stage_root)); } - - #[test] - fn guard_is_armed_ignores_missing_inactive_expired_and_malformed_markers() { - let c = setup(); - let marker_path = skills_dir(&c.stage_root).join(GUARD_MARKER); - fs::create_dir_all(skills_dir(&c.stage_root)).unwrap(); - - assert!(!guard_is_armed(&c.stage_root)); - - fs::write( - &marker_path, - serde_json::to_string(&json!({ "active": false })).unwrap(), - ) - .unwrap(); - assert!(!guard_is_armed(&c.stage_root)); - - fs::write( - &marker_path, - serde_json::to_string(&json!({ - "active": true, - "expiresAt": iso_millis(now_ms() - 60_000), - })) - .unwrap(), - ) - .unwrap(); - assert!(!guard_is_armed(&c.stage_root)); - - fs::write(&marker_path, "not json").unwrap(); - assert!(!guard_is_armed(&c.stage_root)); - } - - #[test] - fn teardown_sweeps_a_stray_marker_even_without_a_manifest() { - let c = setup(); - fs::create_dir_all(skills_dir(&c.stage_root)).unwrap(); - fs::write(skills_dir(&c.stage_root).join(GUARD_MARKER), "{}").unwrap(); - assert!(teardown_guard(&c.stage_root)); - assert!(!skills_dir(&c.stage_root).join(GUARD_MARKER).exists()); - } - - #[test] - fn codex_install_writes_project_hook_marker_and_manifest() { - let c = setup(); - let exe = Path::new("/g/eval-magic"); - install_codex_guard(&c.stage_root, exe, None).unwrap(); - - let marker = read_json( - &c.stage_root - .join(".agents") - .join("skills") - .join(GUARD_MARKER), - ); - assert_eq!(marker["active"], json!(true)); - // The Codex guard shares the env-scoped roots: the staged `.agents/skills` - // dir lives inside `stage_root`, so the single env root already covers it. - let env = absolutize(&c.stage_root).display().to_string(); - assert!( - marker["allowedRoots"] - .as_array() - .unwrap() - .iter() - .any(|r| r.as_str().unwrap() == env) - ); - - let hooks = read_json(&codex_hooks_path(&c.stage_root)); - let hook = &hooks["hooks"]["PreToolUse"][0]; - assert!(hook["matcher"].as_str().unwrap().contains("apply_patch")); - assert!( - hook["hooks"][0]["command"] - .as_str() - .unwrap() - .contains("guard-codex") - ); - assert!( - c.stage_root - .join(".agents") - .join("skills") - .join(GUARD_MANIFEST) - .exists() - ); - } - - #[test] - fn codex_teardown_restores_pre_existing_hooks_json_verbatim() { - let c = setup(); - fs::create_dir_all(c.stage_root.join(".codex")).unwrap(); - let original = format!( - "{}\n", - serde_json::to_string_pretty(&json!({ - "hooks": { - "PostToolUse": [ - { - "matcher": "Bash", - "hooks": [{ "type": "command", "command": "echo ok" }] - } - ] - } - })) - .unwrap() - ); - fs::write(codex_hooks_path(&c.stage_root), &original).unwrap(); - - install_codex_guard(&c.stage_root, Path::new("/g/eval-magic"), None).unwrap(); - assert!( - fs::read_to_string(codex_hooks_path(&c.stage_root)) - .unwrap() - .contains("guard-codex") - ); - - teardown_guard(&c.stage_root); - assert_eq!( - fs::read_to_string(codex_hooks_path(&c.stage_root)).unwrap(), - original - ); - } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 7537227..66152e1 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -1,9 +1,11 @@ -//! Execution sandbox: write-guard install/teardown and write-boundary policy. +//! Execution sandbox: shared write-guard machinery and write-boundary policy. //! -//! The hook entry points are hidden subcommands on this binary (see [`guard`] and -//! `cli`), so the installed PreToolUse hook invokes `eval-magic guard ` -//! or `eval-magic guard-codex ` — no separate hook script to ship or -//! locate. +//! The hook entry points are hidden subcommands on this binary (see `cli`), so +//! the installed PreToolUse hook invokes `eval-magic guard ` or +//! `eval-magic guard-codex ` — no separate hook script to ship or +//! locate. Each harness's installer and verdict shape live in its adapter +//! module (`crate::adapters::::guard`); this module holds the shared +//! marker/manifest/teardown machinery and the boundary policy. pub mod decide; pub mod guard; @@ -12,9 +14,10 @@ pub mod policy; pub(crate) use decide::marker_is_armed; pub use decide::{GuardDecision, GuardMarker, decide}; -pub use guard::{codex_guard_decision, guard_decision, read_marker}; +pub(crate) use guard::parse_tool_call; +pub use guard::read_marker; pub(crate) use install::guard_is_armed; -pub use install::{GUARD_MANIFEST, GUARD_MARKER, install_guard, teardown_guard}; +pub use install::{GUARD_MANIFEST, GUARD_MARKER, teardown_guard}; pub use policy::{WRITE_TOOLS, classify_bash, is_under, is_under_any, is_write_tool, path_arg}; use std::time::{SystemTime, UNIX_EPOCH}; From cc009de6ed593ea388efd97f2e0fc627f760baaf Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sun, 5 Jul 2026 22:54:18 -0400 Subject: [PATCH 06/45] refactor(adapters): tier the HarnessAdapter trait into baseline + enhancements Reorganize the trait around the instant-magic contract (#113): a minimal baseline every harness must satisfy plus optional enhancements that raise fidelity. Behavior-neutral. - Baseline (required) is now just label() + skills_dir(); the module and trait docs state the contract, and every method's doc comment names its tier, the enhancement it belongs to, and the fallback. - OpenCode's hand-written "not wired" stubs become trait defaults: parse_cli_events/parse_cli_events_full and install_guard now default to Unsupported errors that name the enhancement and the harness. rewrites_frontmatter_name / advertises_staged_slug_name default to false; render_available_skills_block and the skill phrases get neutral generic defaults (all current overrides kept). - New defaulted seam methods absorb the remaining harness matches in generic code: - staged_slug + validate_stage_name (staging's OpenCode slug match and naming validation move behind the adapter) - run_capabilities (core::capabilities_for's harness match deleted; the struct stays in core, the values live with each adapter, and an invariant test keeps supports_guard in lockstep with guard_armed_message) - config_dir_names (staging's hardcoded .claude/.agents/.codex/ .opencode sibling-asset filter now unions the adapters' lists) - guard_hook_cleanup_dir (teardown's hardcoded .codex prune) - transcript_surfaces_skill_invocation (judge_tasks' Codex branch for the __skill_invoked code check) Part of #121. Co-Authored-By: Claude Fable 5 --- src/adapters/claude_code/mod.rs | 14 +- src/adapters/codex/mod.rs | 20 +- src/adapters/harness.rs | 393 ++++++++++++++++++++++++------ src/adapters/opencode/mod.rs | 62 +++-- src/cli/run/staging/mod.rs | 60 +++-- src/cli/run/util.rs | 4 +- src/core/capabilities.rs | 64 +---- src/core/mod.rs | 2 +- src/pipeline/grade/judge_tasks.rs | 10 +- src/sandbox/install.rs | 8 +- 10 files changed, 426 insertions(+), 211 deletions(-) diff --git a/src/adapters/claude_code/mod.rs b/src/adapters/claude_code/mod.rs index 6c6fd44..11efa9f 100644 --- a/src/adapters/claude_code/mod.rs +++ b/src/adapters/claude_code/mod.rs @@ -17,7 +17,7 @@ use std::io; use std::path::{Path, PathBuf}; use std::time::Duration; -use crate::core::{AvailableSkill, ToolInvocation}; +use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; @@ -36,11 +36,15 @@ impl HarnessAdapter for ClaudeCodeAdapter { fn skills_dir(&self, repo_root: &Path) -> PathBuf { repo_root.join(".claude").join("skills") } - fn rewrites_frontmatter_name(&self) -> bool { - false + fn run_capabilities(&self) -> HarnessRunCapabilities { + HarnessRunCapabilities { + supports_guard: true, + supports_bootstrap_with_no_stage: true, + supports_stage_name_with_no_stage: true, + } } - fn advertises_staged_slug_name(&self) -> bool { - false + fn config_dir_names(&self) -> &'static [&'static str] { + &[".claude"] } fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { render_available_skills_block(skills) diff --git a/src/adapters/codex/mod.rs b/src/adapters/codex/mod.rs index fa52d96..574ed78 100644 --- a/src/adapters/codex/mod.rs +++ b/src/adapters/codex/mod.rs @@ -14,7 +14,7 @@ use std::io; use std::path::{Path, PathBuf}; use std::time::Duration; -use crate::core::{AvailableSkill, ToolInvocation}; +use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; @@ -33,6 +33,16 @@ impl HarnessAdapter for CodexAdapter { fn skills_dir(&self, repo_root: &Path) -> PathBuf { repo_root.join(".agents").join("skills") } + fn run_capabilities(&self) -> HarnessRunCapabilities { + HarnessRunCapabilities { + supports_guard: true, + supports_bootstrap_with_no_stage: false, + supports_stage_name_with_no_stage: false, + } + } + fn config_dir_names(&self) -> &'static [&'static str] { + &[".agents", ".codex"] + } fn rewrites_frontmatter_name(&self) -> bool { true } @@ -95,6 +105,11 @@ impl HarnessAdapter for CodexAdapter { fn parse_cli_events_full(&self, path: &Path) -> io::Result { parse_codex_events_full(path) } + // Codex's JSONL exposes no deterministic skill-tool event, so the + // `__skill_invoked` meta-check uses the LLM-judge fallback. + fn transcript_surfaces_skill_invocation(&self) -> bool { + false + } fn install_guard( &self, stage_root: &Path, @@ -103,6 +118,9 @@ impl HarnessAdapter for CodexAdapter { ) -> io::Result { guard::install_guard(stage_root, guard_exe, ttl) } + fn guard_hook_cleanup_dir(&self, stage_root: &Path) -> Option { + Some(guard::hook_cleanup_dir(stage_root)) + } fn guard_armed_message(&self) -> Option<&'static str> { Some( "\n🛡 Write guard armed: a PreToolUse hook is staged in .codex/hooks.json\n and will block writes/installs outside the eval sandbox during Codex dispatches.\n Dispatch with codex --ask-for-approval never exec --dangerously-bypass-hook-trust so the vetted eval hook runs.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 7c399b2..ed16520 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -1,125 +1,284 @@ //! The harness adapter API — the single seam between generic dispatch code and //! harness-specific behavior. //! -//! Every harness-specific concern hangs off the [`HarnessAdapter`] trait: how -//! discoverable skills are presented in a dispatch prompt, how a persisted -//! transcript is parsed, where staged skills live, and which native hook the -//! write guard installs. Generic code resolves an adapter with [`adapter_for`] -//! and then calls the trait — so [`adapter_for`] is the one place that names a -//! concrete harness for this surface. The impls live in the per-harness -//! modules ([`claude_code`](super::claude_code), [`codex`](super::codex), +//! The trait is tiered into a **baseline** every harness must implement and +//! **enhancements** that raise fidelity when a harness has the native support: +//! +//! - **Baseline (required):** [`label`](HarnessAdapter::label) and +//! [`skills_dir`](HarnessAdapter::skills_dir). A new harness compiles with +//! just these two methods; dispatched through its one-shot CLI (with +//! `--no-stage` inlining the skill when native staging isn't wired), it +//! already supports `llm_judge` grading and the `detect-stray-writes` +//! post-pass. +//! - **Enhancements (defaulted):** every other method has a default — either a +//! working generic fallback (e.g. the plain available-skills block) or an +//! `Unsupported` error naming the enhancement it belongs to (e.g. transcript +//! ingest, the write guard). Override the methods of an enhancement to wire +//! it for a harness. +//! +//! Generic code resolves an adapter with [`adapter_for`] and then calls the +//! trait — so [`adapter_for`] is the one place that names a concrete harness +//! for this surface. The impls live in the per-harness modules +//! ([`claude_code`](super::claude_code), [`codex`](super::codex), //! [`opencode`](super::opencode)). use std::io; use std::path::{Path, PathBuf}; use std::time::Duration; -use crate::core::{AvailableSkill, Harness, ToolInvocation}; +use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; /// The behavior that varies by harness. Generic dispatch code depends on this -/// trait, never on a concrete harness variant. +/// trait, never on a concrete harness variant. See the module docs for the +/// baseline-vs-enhancement contract. pub trait HarnessAdapter { - /// The kebab-case identifier used in CLI flags, `dispatch.json`, and the - /// staged `conditions.json`. + // ── Baseline (required) — every harness implements these ──────────────── + + /// **Baseline.** The kebab-case identifier used in CLI flags, + /// `dispatch.json`, and the staged `conditions.json`. fn label(&self) -> &'static str; - /// The project-local directory staged skills live under for this harness. + /// **Baseline.** The project-local directory staged skills live under for + /// this harness. Under `--no-stage` nothing is staged into it, so a + /// baseline harness may point this at any repo-local path its discovery + /// would read. fn skills_dir(&self, repo_root: &Path) -> PathBuf; - /// Whether a staged skill's frontmatter `name:` is rewritten to its slug so - /// the harness's repo-local discovery resolves the staged copy. - fn rewrites_frontmatter_name(&self) -> bool; + // ── Run-option capabilities (defaulted) ────────────────────────────────── + + /// The run options the generic `run` preflight may accept for this + /// harness. The default is the baseline: no write guard, `--bootstrap` and + /// `--stage-name` allowed alongside `--no-stage`. Override alongside the + /// enhancement that changes support (e.g. wiring the write guard flips + /// `supports_guard`). + fn run_capabilities(&self) -> HarnessRunCapabilities { + HarnessRunCapabilities { + supports_guard: false, + supports_bootstrap_with_no_stage: true, + supports_stage_name_with_no_stage: true, + } + } - /// Whether the skill-under-test is advertised in the available-skills block - /// under its staged slug (vs. its natural name). True for Codex, whose - /// repo-local discovery keys on the rewritten frontmatter name. (OpenCode - /// also rewrites the frontmatter to the slug yet still advertises the natural - /// name — a known inconsistency tracked for a separate fix.) - fn advertises_staged_slug_name(&self) -> bool; + /// The project-local config dir names this harness reads or the adapter + /// writes (e.g. `.claude`). Staging excludes every harness's config dirs + /// when copying a skill's sibling assets, so a stray checked-in config dir + /// never rides into a staged env. List the parent of + /// [`skills_dir`](Self::skills_dir) plus any hook/config dirs the adapter + /// writes. + fn config_dir_names(&self) -> &'static [&'static str] { + &[] + } - /// Render the discoverable skills the way this harness natively surfaces - /// them (e.g. Claude Code's Skill-tool list, Codex's `## Skills`, OpenCode's - /// `` XML). - fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String; + // ── Enhancement: native skill staging (defaulted) ──────────────────────── + // Fallback without it: `--no-stage` inlines each SKILL.md into its + // dispatch prompt instead of staging files for native discovery. - /// How a staged skill is described as discoverable in the neutral - /// slug-disambiguation line (e.g. "via the Skill tool"). - fn skill_surface_phrase(&self) -> &'static str; + /// **Enhancement: native staging.** Build the conspicuous staged-skill + /// slug. The default underscore form is fine for any harness without + /// naming rules; a harness with constrained skill names (e.g. OpenCode) + /// overrides it. `prefix` must be preserved so cleanup prefix-scans still + /// find the staged dir. + fn staged_slug( + &self, + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, + ) -> String { + format!("{prefix}{iteration}-{condition}__{skill_name}") + } - /// The lead-in for the fallback "read the skill from ``" instruction - /// when the staged identifier can't be resolved. - fn skill_unresolved_phrase(&self) -> &'static str; + /// **Enhancement: native staging.** Validate a staged-skill identifier + /// (generated slug or `--stage-name` override) against this harness's + /// naming rules. The default accepts anything. + fn validate_stage_name(&self, _name: &str) -> Result<(), String> { + Ok(()) + } - /// Wrap a plan-mode profile as a `` operating-context - /// layer. The default usually suffices. - fn render_plan_mode_context(&self, profile_text: &str) -> String { - let trimmed = profile_text.trim(); - if trimmed.is_empty() { + /// **Enhancement: native staging.** Whether a staged skill's frontmatter + /// `name:` is rewritten to its slug so the harness's repo-local discovery + /// resolves the staged copy. + fn rewrites_frontmatter_name(&self) -> bool { + false + } + + /// **Enhancement: native staging.** Whether the skill-under-test is + /// advertised in the available-skills block under its staged slug (vs. its + /// natural name). True for Codex, whose repo-local discovery keys on the + /// rewritten frontmatter name. (OpenCode also rewrites the frontmatter to + /// the slug yet still advertises the natural name — a known inconsistency + /// tracked for a separate fix.) + fn advertises_staged_slug_name(&self) -> bool { + false + } + + /// **Enhancement: native staging.** Render the discoverable skills the way + /// this harness natively surfaces them (e.g. Claude Code's Skill-tool + /// list, Codex's `## Skills`, OpenCode's `` XML). The + /// default is a neutral bulleted list. + fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { + if skills.is_empty() { return String::new(); } - format!("\n{trimmed}\n") + let mut sorted: Vec<&AvailableSkill> = skills.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + let mut out = String::from("The following skills are available in this session:\n"); + for s in sorted { + out.push_str(&format!("\n- {}: {}", s.name, s.description)); + } + out + } + + /// **Enhancement: native staging.** How a staged skill is described as + /// discoverable in the neutral slug-disambiguation line (e.g. "via the + /// Skill tool"). + fn skill_surface_phrase(&self) -> &'static str { + "as a discoverable skill" } - /// The filename (under a task's `outputs/` dir) this harness's one-shot CLI - /// writes the captured transcript to. `None` when the harness has no - /// transcript ingest wired yet (e.g. OpenCode). + /// **Enhancement: native staging.** The lead-in for the fallback "read the + /// skill from ``" instruction when the staged identifier can't be + /// resolved. + fn skill_unresolved_phrase(&self) -> &'static str { + "If the staged skill cannot be resolved" + } + + // ── Enhancement: transcript parser (defaulted) ─────────────────────────── + // Fallback without it: `transcript_check` assertions grade as + // unverifiable, `llm_judge` carries the grading, token/cost/duration go + // unrecorded, and run records are assembled by hand (or from + // `outputs/final-message.md`) instead of auto-ingested. + + /// **Enhancement: transcript parser.** The filename (under a task's + /// `outputs/` dir) this harness's one-shot CLI writes the captured + /// transcript to. `None` when no transcript ingest is wired — the ingest + /// pipeline then never calls the parsers below. fn cli_events_filename(&self) -> Option<&'static str> { None } - /// The native model-selection flag accepted by this harness's CLI. `None` - /// means the adapter has no model-selection support wired yet. + /// **Enhancement: transcript parser.** Parse the events file this + /// harness's one-shot CLI wrote (the captured transcript) into ordered + /// tool invocations. + fn parse_cli_events(&self, _path: &Path) -> io::Result> { + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "transcript ingest is not wired for the {} harness", + self.label() + ), + )) + } + + /// **Enhancement: transcript parser.** The full-summary counterpart of + /// [`parse_cli_events`](Self::parse_cli_events): tool invocations, deduped + /// token usage, duration, and final message text. + fn parse_cli_events_full(&self, _path: &Path) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "transcript ingest is not wired for the {} harness", + self.label() + ), + )) + } + + /// **Enhancement: transcript parser.** Whether the parsed transcript + /// exposes a deterministic skill-invocation event the `__skill_invoked` + /// meta-check can match. False for Codex (its JSONL has no skill-tool + /// event), which routes the meta-check to the LLM-judge fallback. + fn transcript_surfaces_skill_invocation(&self) -> bool { + true + } + + // ── Enhancement: model flag (defaulted) ────────────────────────────────── + // Fallback without it: `--agent-model` / `--judge-model` are recorded as + // provenance only; dispatches run on the harness's default model. + + /// **Enhancement: model flag.** The native model-selection flag accepted + /// by this harness's CLI. `None` means no model-selection support is + /// wired. fn cli_model_flag(&self) -> Option<&'static str> { None } - /// The `Next:` guidance printed after `run`: how to dispatch each task through - /// this harness's one-shot CLI and then ingest. Empty when the adapter has no - /// dispatch recipe wired. - fn cli_next_steps(&self, _ctx: CliDispatchContext<'_>) -> String { - String::new() + // ── Enhancement: write guard (defaulted) ───────────────────────────────── + // Fallback without it: the `detect-stray-writes` post-pass (folded into + // `ingest`) audits out-of-bounds writes after the fact. + + /// **Enhancement: write guard.** Arm the write guard using this harness's + /// native pre-tool hook surface, returning the staged marker path. The + /// guard's allowed roots are derived from `stage_root` (the isolated env / + /// agent cwd), so it bounds the agent to the same env boundary that + /// isolates its reads. + fn install_guard( + &self, + _stage_root: &Path, + _guard_exe: &Path, + _ttl: Option, + ) -> io::Result { + Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("--guard is not supported for the {} harness", self.label()), + )) } - /// Extra `dispatch-manifest.md` lines describing this harness's dispatch - /// recipe (command template, parallel recipe, ingest note). `None` when the - /// harness contributes no manifest section. - fn cli_manifest_section(&self, _ctx: CliManifestContext<'_>) -> Option> { + /// **Enhancement: write guard.** The banner printed after `--guard` + /// successfully arms, describing the harness's native hook surface and how + /// to remove it. `None` for a harness with no write guard (its + /// [`install_guard`](Self::install_guard) errors), in which case no banner + /// is printed. + fn guard_armed_message(&self) -> Option<&'static str> { None } - /// The post-`grade` / post-`ingest` judge dispatch guidance for this harness. - /// `None` leaves the generic judge handoff in place. - fn cli_judge_next_steps(&self, _ctx: CliJudgeContext<'_>) -> Option { + /// **Enhancement: write guard.** A hook-config dir the guard install + /// created outside [`skills_dir`](Self::skills_dir) (e.g. Codex's + /// `.codex/`), which teardown prunes when restoring the original config + /// leaves it empty. `None` when the guard writes only under existing dirs. + fn guard_hook_cleanup_dir(&self, _stage_root: &Path) -> Option { None } - /// Parse the events file this harness's one-shot CLI wrote (the captured - /// transcript) into ordered tool invocations. - fn parse_cli_events(&self, path: &Path) -> io::Result>; + // ── Enhancement: plan-mode context (defaulted) ─────────────────────────── + + /// **Enhancement: plan-mode context.** Wrap a plan-mode profile as an + /// operating-context layer. The shared `` default + /// usually suffices; a harness with a real native plan mode could inject + /// it differently. + fn render_plan_mode_context(&self, profile_text: &str) -> String { + let trimmed = profile_text.trim(); + if trimmed.is_empty() { + return String::new(); + } + format!("\n{trimmed}\n") + } + + // ── Enhancement: dispatch recipes (defaulted) ──────────────────────────── + // Fallback without them: `run` prints the generic handoff and the runbook + // carries no copy-pasteable per-task command. + + /// **Enhancement: dispatch recipes.** The `Next:` guidance printed after + /// `run`: how to dispatch each task through this harness's one-shot CLI + /// and then ingest. Empty when no dispatch recipe is wired. + fn cli_next_steps(&self, _ctx: CliDispatchContext<'_>) -> String { + String::new() + } - /// The full-summary counterpart of [`parse_cli_events`](Self::parse_cli_events): - /// tool invocations, deduped token usage, duration, and final message text. - fn parse_cli_events_full(&self, path: &Path) -> io::Result; + /// **Enhancement: dispatch recipes.** Extra `dispatch-manifest.md` lines + /// describing this harness's dispatch recipe (command template, parallel + /// recipe, ingest note). `None` when the harness contributes no manifest + /// section. + fn cli_manifest_section(&self, _ctx: CliManifestContext<'_>) -> Option> { + None + } - /// Arm the write guard using this harness's native pre-tool hook surface, - /// returning the staged marker path. The guard's allowed roots are derived - /// from `stage_root` (the isolated env / agent cwd), so it bounds the agent to - /// the same env boundary that isolates its reads. - fn install_guard( - &self, - stage_root: &Path, - guard_exe: &Path, - ttl: Option, - ) -> io::Result; - - /// The banner printed after `--guard` successfully arms, describing the - /// harness's native hook surface and how to remove it. Harness-specific text, - /// so it lives here rather than in generic run code. `None` for a harness with - /// no write guard (its [`install_guard`](Self::install_guard) errors), in which - /// case no banner is printed. - fn guard_armed_message(&self) -> Option<&'static str> { + /// **Enhancement: dispatch recipes.** The post-`grade` / post-`ingest` + /// judge dispatch guidance for this harness. `None` leaves the generic + /// judge handoff in place. + fn cli_judge_next_steps(&self, _ctx: CliJudgeContext<'_>) -> Option { None } } @@ -206,6 +365,40 @@ mod tests { } } + #[test] + fn run_capabilities_capture_run_option_support_by_harness() { + let claude = adapter_for(Harness::ClaudeCode).run_capabilities(); + assert!(claude.supports_guard); + assert!(claude.supports_bootstrap_with_no_stage); + assert!(claude.supports_stage_name_with_no_stage); + + let codex = adapter_for(Harness::Codex).run_capabilities(); + assert!(codex.supports_guard); + assert!(!codex.supports_bootstrap_with_no_stage); + assert!(!codex.supports_stage_name_with_no_stage); + + let opencode = adapter_for(Harness::OpenCode).run_capabilities(); + assert!(!opencode.supports_guard); + assert!(opencode.supports_bootstrap_with_no_stage); + assert!(opencode.supports_stage_name_with_no_stage); + } + + #[test] + fn guard_support_and_guard_banner_stay_in_lockstep() { + // `run_capabilities().supports_guard` gates the `--guard` preflight and + // `guard_armed_message()` is the post-arm banner — an adapter wiring or + // dropping the write-guard enhancement must update both. + for h in Harness::ALL { + let a = adapter_for(h); + assert_eq!( + a.run_capabilities().supports_guard, + a.guard_armed_message().is_some(), + "guard capability and banner disagree for {}", + a.label() + ); + } + } + #[test] fn guard_armed_message_is_harness_specific_and_absent_for_opencode() { // The post-arm `--guard` banner names the harness's native hook surface, @@ -230,4 +423,50 @@ mod tests { // banner to print. assert_eq!(adapter_for(Harness::OpenCode).guard_armed_message(), None); } + + #[test] + fn staged_slug_default_and_opencode_override_preserve_the_prefix() { + let prefix = "slow-powers-eval-"; + for h in Harness::ALL { + let slug = adapter_for(h).staged_slug(prefix, 2, "with_skill", "my-skill"); + assert!(slug.starts_with(prefix), "{slug}"); + assert!( + adapter_for(h).validate_stage_name(&slug).is_ok(), + "each harness's generated slug passes its own naming rules: {slug}" + ); + } + assert_eq!( + adapter_for(Harness::ClaudeCode).staged_slug(prefix, 2, "with_skill", "my-skill"), + "slow-powers-eval-2-with_skill__my-skill" + ); + assert_eq!( + adapter_for(Harness::OpenCode).staged_slug(prefix, 2, "with_skill", "my-skill"), + "slow-powers-eval-2-with-skill-my-skill" + ); + } + + #[test] + fn every_config_dir_list_covers_the_skills_dir_parent() { + // Staging's sibling-asset filter unions config_dir_names() across all + // harnesses; each adapter must at least name the dir its skills live + // under so a checked-in copy never rides into a staged env. + let root = Path::new(""); + for h in Harness::ALL { + let a = adapter_for(h); + let skills_dir = a.skills_dir(root); + let top = skills_dir + .components() + .next() + .expect("skills dir has a leading component") + .as_os_str() + .to_string_lossy() + .into_owned(); + assert!( + a.config_dir_names().contains(&top.as_str()), + "{} config_dir_names {:?} misses {top}", + a.label(), + a.config_dir_names() + ); + } + } } diff --git a/src/adapters/opencode/mod.rs b/src/adapters/opencode/mod.rs index d51e23a..77e5ba4 100644 --- a/src/adapters/opencode/mod.rs +++ b/src/adapters/opencode/mod.rs @@ -3,17 +3,14 @@ //! Everything OpenCode-specific lives in this module tree: the adapter impl and //! slug/naming rules (this file) and the `` XML block //! ([`session`]). Transcript ingest, the model flag, and the write guard are -//! not wired yet; the adapter's error stubs say so. +//! not wired yet; the trait's enhancement defaults cover them. pub mod session; -use std::io; use std::path::{Path, PathBuf}; -use std::time::Duration; -use crate::core::{AvailableSkill, ToolInvocation}; +use crate::core::AvailableSkill; -use super::TranscriptSummary; use super::harness::{CliDispatchContext, HarnessAdapter}; use session::render_opencode_available_skills_block; @@ -26,12 +23,32 @@ impl HarnessAdapter for OpenCodeAdapter { fn skills_dir(&self, repo_root: &Path) -> PathBuf { repo_root.join(".opencode").join("skills") } + fn config_dir_names(&self) -> &'static [&'static str] { + &[".opencode"] + } + // OpenCode's constrained naming rules need a sanitized slug; the generated + // slug and any --stage-name override are validated against the same rules. + fn staged_slug( + &self, + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, + ) -> String { + opencode_slug(prefix, iteration, condition, skill_name) + } + fn validate_stage_name(&self, name: &str) -> Result<(), String> { + if is_valid_opencode_name(name) { + Ok(()) + } else { + Err(format!( + "OpenCode skill name \"{name}\" is invalid; names must be 1-64 lowercase alphanumeric characters separated by single hyphens" + )) + } + } fn rewrites_frontmatter_name(&self) -> bool { true } - fn advertises_staged_slug_name(&self) -> bool { - false - } fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { render_opencode_available_skills_block(skills) } @@ -53,32 +70,9 @@ impl HarnessAdapter for OpenCodeAdapter { iteration = ctx.iteration ) } - // OpenCode transcript ingest is not yet wired: its `cli_events_filename` is - // `None`, so the ingest pipeline never reaches these parsers. They error - // rather than parse until OpenCode ingest lands. - fn parse_cli_events(&self, _path: &Path) -> io::Result> { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "opencode transcript ingest is not yet wired", - )) - } - fn parse_cli_events_full(&self, _path: &Path) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "opencode transcript ingest is not yet wired", - )) - } - fn install_guard( - &self, - _stage_root: &Path, - _guard_exe: &Path, - _ttl: Option, - ) -> io::Result { - Err(io::Error::new( - io::ErrorKind::Unsupported, - "--guard is not yet supported for the opencode harness", - )) - } + // Transcript ingest, the model flag, and the write guard are not wired for + // OpenCode: the trait's enhancement defaults (unsupported-errors and + // `None`s) apply as-is. } /// True when `name` satisfies OpenCode's skill-name rules: diff --git a/src/cli/run/staging/mod.rs b/src/cli/run/staging/mod.rs index a502f84..d84a6cc 100644 --- a/src/cli/run/staging/mod.rs +++ b/src/cli/run/staging/mod.rs @@ -111,6 +111,17 @@ pub(crate) fn skills_dir_for_harness(repo_root: &Path, harness: Harness) -> Path adapter_for(harness).skills_dir(repo_root) } +/// True when `name` is any harness's project-local config dir (`.claude`, +/// `.agents`, …). Staging excludes every harness's config dirs when copying a +/// skill's sibling assets — regardless of the active harness — so a checked-in +/// config dir never rides into a staged env. +fn is_harness_config_dir(name: &str) -> bool { + Harness::ALL + .iter() + .flat_map(|&h| adapter_for(h).config_dir_names()) + .any(|&dir| dir == name) +} + /// Rewrite (or insert) the `name:` frontmatter field so a Codex-staged skill's /// declared name matches its staged slug. fn rewrite_frontmatter_name(content: &str, name: &str) -> String { @@ -142,35 +153,25 @@ fn prune_if_empty(dir: &Path) -> Result<(), RunError> { Ok(()) } -/// Stage one skill under the harness's skills dir and return its slug. For Codex -/// and OpenCode the frontmatter `name:` is rewritten to the slug. +/// Stage one skill under the harness's skills dir and return its slug. For +/// harnesses whose adapter opts in, the frontmatter `name:` is rewritten to the +/// slug. pub fn stage_skill_for_harness(opts: &StageSkillOpts) -> Result { + let adapter = adapter_for(opts.harness); let slug = match opts.stage_name_override { Some(name) => name.to_string(), - None => match opts.harness { - Harness::OpenCode => crate::adapters::opencode::opencode_slug( - STAGED_SKILL_PREFIX, - opts.iteration, - opts.condition, - opts.skill_name, - ), - _ => format!( - "{STAGED_SKILL_PREFIX}{}-{}__{}", - opts.iteration, opts.condition, opts.skill_name - ), - }, + None => adapter.staged_slug( + STAGED_SKILL_PREFIX, + opts.iteration, + opts.condition, + opts.skill_name, + ), }; - if opts.harness == Harness::OpenCode - && !crate::adapters::opencode::is_valid_opencode_name(&slug) - { - return Err(RunError::msg(format!( - "OpenCode skill name \"{slug}\" is invalid; names must be 1-64 lowercase alphanumeric characters separated by single hyphens" - ))); - } + adapter.validate_stage_name(&slug).map_err(RunError::msg)?; let skill_dir = skills_dir_for_harness(opts.repo_root, opts.harness).join(&slug); fs::create_dir_all(&skill_dir)?; - let content = if adapter_for(opts.harness).rewrites_frontmatter_name() { + let content = if adapter.rewrites_frontmatter_name() { rewrite_frontmatter_name(opts.content, &slug) } else { opts.content.to_string() @@ -183,17 +184,12 @@ pub fn stage_skill_for_harness(opts: &StageSkillOpts) -> Result Result<(), RunError> { - let capabilities = capabilities_for(ctx.harness); + let capabilities = adapter_for(ctx.harness).run_capabilities(); let mut unsupported: Vec<&str> = Vec::new(); if opts.guard && !capabilities.supports_guard { unsupported.push("--guard"); diff --git a/src/core/capabilities.rs b/src/core/capabilities.rs index 160b2e8..bcf5af9 100644 --- a/src/core/capabilities.rs +++ b/src/core/capabilities.rs @@ -1,66 +1,22 @@ //! Per-harness run-option capabilities. //! //! Every dispatch rides a single mechanism: each task is delivered through a -//! one-shot harness CLI subprocess (`claude -p`, `codex exec`). What still varies -//! by harness is which *run options* the generic `run` preflight may accept — -//! captured here as a narrow capability table, independent of the comparison -//! [`Mode`](crate::core::Mode) (`new-skill` / `revision`), which selects the two -//! conditions being compared. - -use crate::core::Harness; +//! one-shot harness CLI subprocess. What still varies by harness is which *run +//! options* the generic `run` preflight may accept — captured as this narrow +//! capability struct, reported per harness by +//! `HarnessAdapter::run_capabilities` and independent of the comparison +//! [`Mode`](crate::core::Mode) (`new-skill` / `revision`), which selects the +//! two conditions being compared. /// Run-option support for a harness's dispatch path. /// -/// This is intentionally narrower than full harness parity: it only describes -/// options the generic `run` preflight must accept or reject before the build -/// sequence starts. Harness-specific behavior still lives behind the adapter. +/// This is intentionally narrower than the full enhancement surface: it only +/// describes options the generic `run` preflight must accept or reject before +/// the build sequence starts. Harness-specific behavior still lives behind the +/// adapter. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct HarnessRunCapabilities { pub supports_guard: bool, pub supports_bootstrap_with_no_stage: bool, pub supports_stage_name_with_no_stage: bool, } - -/// The focused capability table for generic `run` option validation. -pub fn capabilities_for(harness: Harness) -> HarnessRunCapabilities { - match harness { - Harness::ClaudeCode => HarnessRunCapabilities { - supports_guard: true, - supports_bootstrap_with_no_stage: true, - supports_stage_name_with_no_stage: true, - }, - Harness::Codex => HarnessRunCapabilities { - supports_guard: true, - supports_bootstrap_with_no_stage: false, - supports_stage_name_with_no_stage: false, - }, - Harness::OpenCode => HarnessRunCapabilities { - supports_guard: false, - supports_bootstrap_with_no_stage: true, - supports_stage_name_with_no_stage: true, - }, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn capabilities_capture_run_option_support_by_harness() { - let claude = capabilities_for(Harness::ClaudeCode); - assert!(claude.supports_guard); - assert!(claude.supports_bootstrap_with_no_stage); - assert!(claude.supports_stage_name_with_no_stage); - - let codex = capabilities_for(Harness::Codex); - assert!(codex.supports_guard); - assert!(!codex.supports_bootstrap_with_no_stage); - assert!(!codex.supports_stage_name_with_no_stage); - - let opencode = capabilities_for(Harness::OpenCode); - assert!(!opencode.supports_guard); - assert!(opencode.supports_bootstrap_with_no_stage); - assert!(opencode.supports_stage_name_with_no_stage); - } -} diff --git a/src/core/mod.rs b/src/core/mod.rs index eeac591..9ca390b 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -13,7 +13,7 @@ pub mod context; pub mod runtime; pub mod types; -pub use capabilities::{HarnessRunCapabilities, capabilities_for}; +pub use capabilities::HarnessRunCapabilities; pub use context::{ContextError, DetectInput, Harness, RunContext, detect_run_context}; pub use runtime::{GitOutput, run_git}; pub use types::*; diff --git a/src/pipeline/grade/judge_tasks.rs b/src/pipeline/grade/judge_tasks.rs index 0b760fa..659dd20 100644 --- a/src/pipeline/grade/judge_tasks.rs +++ b/src/pipeline/grade/judge_tasks.rs @@ -13,7 +13,7 @@ use std::path::Path; use serde::Serialize; use serde_json::json; -use crate::core::{Assertion, Harness, RunRecord, SKILL_INVOKED_META_ID, ToolInvocation}; +use crate::core::{Assertion, RunRecord, SKILL_INVOKED_META_ID, ToolInvocation}; use crate::pipeline::error::PipelineError; use crate::pipeline::io::{now_iso8601, write_json}; use crate::pipeline::slots::run_slots; @@ -223,7 +223,13 @@ pub fn emit_judge_tasks(ctx: &GradeContext) -> Result = Vec::new(); diff --git a/src/sandbox/install.rs b/src/sandbox/install.rs index 7b57c0d..0c9a600 100644 --- a/src/sandbox/install.rs +++ b/src/sandbox/install.rs @@ -131,10 +131,12 @@ pub(crate) fn write_manifest( pub fn teardown_guard(stage_root: &Path) -> bool { let mut torn = false; for harness in Harness::ALL { - let skills_dir = crate::adapters::adapter_for(harness).skills_dir(stage_root); - torn |= teardown_guard_from_skills_dir(&skills_dir); + let adapter = crate::adapters::adapter_for(harness); + torn |= teardown_guard_from_skills_dir(&adapter.skills_dir(stage_root)); + if let Some(hook_dir) = adapter.guard_hook_cleanup_dir(stage_root) { + let _ = prune_if_empty(&hook_dir); + } } - let _ = prune_if_empty(&crate::adapters::codex::guard::hook_cleanup_dir(stage_root)); torn } From f38e1cfadcf5e998958a4fed4961f865f05d2231 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sun, 5 Jul 2026 23:13:33 -0400 Subject: [PATCH 07/45] refactor(adapters): drop the orphaned per-harness plan-mode renderers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up: the per-harness render_*_plan_mode_context session fns were four identical copies of the trait's default with no production callers — production always went through the trait method. Delete them (their trim-behavior coverage moves into the cross-harness trait test), tighten the opencode slug helpers to private now that the staged_slug/validate_stage_name seams are their only callers, and drop a breadcrumb comment from the staging tests. Co-Authored-By: Claude Fable 5 --- src/adapters/claude_code/session.rs | 38 +++-------------------------- src/adapters/codex/session.rs | 20 --------------- src/adapters/harness.rs | 4 +++ src/adapters/opencode/mod.rs | 9 ++----- src/adapters/opencode/session.rs | 32 ------------------------ src/cli/run/staging/tests/stage.rs | 2 -- 6 files changed, 9 insertions(+), 96 deletions(-) diff --git a/src/adapters/claude_code/session.rs b/src/adapters/claude_code/session.rs index 376c037..37d0534 100644 --- a/src/adapters/claude_code/session.rs +++ b/src/adapters/claude_code/session.rs @@ -3,9 +3,9 @@ //! The available-skills reminder is //! a *harness-specific* surface: Claude Code presents discoverable skills to an //! agent as "The following skills are available for use with the Skill tool:" -//! followed by `- name: description` bullets. Plan-mode context is injected as a -//! `` block. Both live in an adapter rather than the harness- -//! agnostic orchestrator so a new harness adds its own renderer alongside. +//! followed by `- name: description` bullets. It lives in an adapter rather than +//! the harness-agnostic orchestrator so a new harness adds its own renderer +//! alongside. use crate::core::AvailableSkill; @@ -26,18 +26,6 @@ pub fn render_available_skills_block(skills: &[AvailableSkill]) -> String { out } -/// Render a plan-mode profile the way Claude Code injects an operating mode into -/// a live session: as a `` block the agent is told it is -/// operating under, not prose it merely reads. Returns an empty string for empty -/// input so the caller can omit the section entirely. -pub fn render_plan_mode_context(profile_text: &str) -> String { - let trimmed = profile_text.trim(); - if trimmed.is_empty() { - return String::new(); - } - format!("\n{trimmed}\n") -} - #[cfg(test)] mod tests { use super::*; @@ -71,24 +59,4 @@ mod tests { fn empty_list_renders_empty_string() { assert_eq!(render_available_skills_block(&[]), ""); } - - #[test] - fn plan_mode_wraps_in_system_reminder() { - let block = render_plan_mode_context("Plan mode is active. Do not edit."); - assert!(block.contains("")); - assert!(block.contains("")); - assert!(block.contains("Plan mode is active. Do not edit.")); - } - - #[test] - fn plan_mode_trims_surrounding_whitespace() { - let block = render_plan_mode_context("\n\n PROFILE-BODY \n\n"); - assert_eq!(block, "\nPROFILE-BODY\n"); - } - - #[test] - fn plan_mode_empty_or_whitespace_renders_empty_string() { - assert_eq!(render_plan_mode_context(""), ""); - assert_eq!(render_plan_mode_context(" \n "), ""); - } } diff --git a/src/adapters/codex/session.rs b/src/adapters/codex/session.rs index 0c757ab..e0761a6 100644 --- a/src/adapters/codex/session.rs +++ b/src/adapters/codex/session.rs @@ -26,17 +26,6 @@ pub fn render_codex_available_skills_block(skills: &[AvailableSkill]) -> String out } -/// Render a Codex plan-mode profile as an operating-context reminder. Codex's -/// non-interactive `exec` surface has no documented real plan-mode switch, so -/// this mirrors the portable approximation: text injected ahead of the task. -pub fn render_codex_plan_mode_context(profile_text: &str) -> String { - let trimmed = profile_text.trim(); - if trimmed.is_empty() { - return String::new(); - } - format!("\n{trimmed}\n") -} - #[cfg(test)] mod tests { use super::*; @@ -71,13 +60,4 @@ mod tests { fn empty_list_renders_empty_string() { assert_eq!(render_codex_available_skills_block(&[]), ""); } - - #[test] - fn plan_mode_wraps_in_system_reminder_with_codex_profile_text() { - let block = render_codex_plan_mode_context("Codex plan mode is active."); - assert_eq!( - block, - "\nCodex plan mode is active.\n" - ); - } } diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index ed16520..7f47fe6 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -362,6 +362,10 @@ mod tests { let out = adapter_for(h).render_plan_mode_context("BODY"); assert_eq!(out, "\nBODY\n"); assert_eq!(adapter_for(h).render_plan_mode_context(" "), ""); + assert_eq!( + adapter_for(h).render_plan_mode_context("\n\n BODY \n\n"), + "\nBODY\n" + ); } } diff --git a/src/adapters/opencode/mod.rs b/src/adapters/opencode/mod.rs index 77e5ba4..e8b2423 100644 --- a/src/adapters/opencode/mod.rs +++ b/src/adapters/opencode/mod.rs @@ -79,7 +79,7 @@ impl HarnessAdapter for OpenCodeAdapter { /// - 1–64 characters /// - lowercase alphanumeric with single-hyphen separators /// - no leading/trailing/consecutive hyphens -pub(crate) fn is_valid_opencode_name(name: &str) -> bool { +fn is_valid_opencode_name(name: &str) -> bool { if name.is_empty() || name.len() > 64 { return false; } @@ -131,12 +131,7 @@ fn sanitize_opencode_name(name: &str) -> String { /// Build a slug that is valid for OpenCode's skill directory + frontmatter name /// constraints. `prefix` is the conspicuous staged-skill prefix, preserved so /// cleanup prefix-scans still find it. -pub(crate) fn opencode_slug( - prefix: &str, - iteration: u32, - condition: &str, - skill_name: &str, -) -> String { +fn opencode_slug(prefix: &str, iteration: u32, condition: &str, skill_name: &str) -> String { let condition = sanitize_opencode_name(condition); let skill = sanitize_opencode_name(skill_name); let base = format!("{prefix}{iteration}-{condition}-{skill}"); diff --git a/src/adapters/opencode/session.rs b/src/adapters/opencode/session.rs index 64463ae..a968fee 100644 --- a/src/adapters/opencode/session.rs +++ b/src/adapters/opencode/session.rs @@ -28,17 +28,6 @@ pub fn render_opencode_available_skills_block(skills: &[AvailableSkill]) -> Stri out } -/// Render an OpenCode plan-mode profile as an operating-context reminder. The -/// real OpenCode plan agent is a primary agent mode, not text, so this is the -/// portable approximation: a `` the dispatch reads. -pub fn render_opencode_plan_mode_context(profile_text: &str) -> String { - let trimmed = profile_text.trim(); - if trimmed.is_empty() { - return String::new(); - } - format!("\n{trimmed}\n") -} - #[cfg(test)] mod tests { use super::*; @@ -77,25 +66,4 @@ mod tests { fn empty_list_renders_empty_string() { assert_eq!(render_opencode_available_skills_block(&[]), ""); } - - #[test] - fn plan_mode_wraps_in_system_reminder() { - let block = render_opencode_plan_mode_context("OpenCode plan mode is active."); - assert_eq!( - block, - "\nOpenCode plan mode is active.\n" - ); - } - - #[test] - fn plan_mode_trims_surrounding_whitespace() { - let block = render_opencode_plan_mode_context("\n\n PROFILE-BODY \n\n"); - assert_eq!(block, "\nPROFILE-BODY\n"); - } - - #[test] - fn plan_mode_empty_or_whitespace_renders_empty_string() { - assert_eq!(render_opencode_plan_mode_context(""), ""); - assert_eq!(render_opencode_plan_mode_context(" \n "), ""); - } } diff --git a/src/cli/run/staging/tests/stage.rs b/src/cli/run/staging/tests/stage.rs index 835a445..eb29cf7 100644 --- a/src/cli/run/staging/tests/stage.rs +++ b/src/cli/run/staging/tests/stage.rs @@ -209,8 +209,6 @@ fn codex_stage_name_override_is_dir_and_frontmatter_name() { } // ── stage_skill_for_harness (opencode) ──────────────────────────────── -// The opencode slug/naming unit tests live with the slug functions in -// `crate::adapters::opencode`. #[test] fn opencode_stages_under_opencode_skills_and_rewrites_frontmatter_name() { From 0168eba22b7700c3ba7b46ef7fca3ec1f2c49256 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Sun, 5 Jul 2026 23:02:22 -0400 Subject: [PATCH 08/45] docs(harness): replace the parity doc with progressive-enhancement docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #121. Reframes harness documentation around the instant-magic contract: a minimal baseline plus progressive enhancements, instead of a parity audit. - Delete docs/harness-parity.md — the codebase (tiered trait + invariant tests) now carries parity information; no agent-facing audit procedure or report template survives. - Add docs/progressive-enhancements.md (dev doc): the single dispatch mechanism, the baseline contract, one section per enhancement (why it needs harness code, what it unlocks, the fallback, its trait methods), an adding-a-new-harness checklist, and the guardrails salvaged from the parity doc. - Add docs/claude-notes.md, docs/codex-notes.md, docs/opencode-notes.md (dev docs): per-harness implementation notes harvested from the README's per-harness sections. - README: the Harnesses section slims to an enhancement-framed intro + "How dispatch works" + a support table declared the source of truth; the per-harness sections are removed — their dispatch recipes duplicated the generated RUNBOOK.md / dispatch-manifest.md, which the README now names as the runtime reference. Fixes the dangling #claude-code-fully-wired anchor and the stale "(Codex today)" model-flag note (Claude Code has --model wired). - Fold the three plugin-isolation recipes (--setting-sources, enabledPlugins, CLAUDE_CONFIG_DIR) into the plugin-shadow banner so operators see them exactly when contamination is detected; the banner and validity warnings now point at docs/claude-notes.md (test-driven: banner tests assert the recipes and the doc pointer). - AGENTS.md/CLAUDE.md: docs/ is now described as development docs. Co-Authored-By: Claude Fable 5 --- AGENTS.md | 7 +- README.md | 126 +++-------------- docs/claude-notes.md | 76 ++++++++++ docs/codex-notes.md | 69 ++++++++++ docs/harness-parity.md | 137 ------------------ docs/opencode-notes.md | 56 ++++++++ docs/progressive-enhancements.md | 160 ++++++++++++++++++++++ src/adapters/claude_code/plugin_shadow.rs | 56 +++++++- 8 files changed, 438 insertions(+), 249 deletions(-) create mode 100644 docs/claude-notes.md create mode 100644 docs/codex-notes.md delete mode 100644 docs/harness-parity.md create mode 100644 docs/opencode-notes.md create mode 100644 docs/progressive-enhancements.md diff --git a/AGENTS.md b/AGENTS.md index e764888..4da5844 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,9 +4,10 @@ `eval-magic` is a Rust CLI crate. The binary entry point is `src/main.rs`; reusable logic lives in `src/lib.rs` and submodules such as `cli/`, `pipeline/`, `sandbox/`, `validation/`, and -`workspace/`. JSON schemas are tracked in `schema/`, harness profiles in `profiles/`, and design or -parity notes in `docs/`. Integration tests are split by surface area under `tests/cli/` and -`tests/run/`; unit tests usually live beside the module they exercise. +`workspace/`. JSON schemas are tracked in `schema/`, harness profiles in `profiles/`, and +development docs (the harness enhancement contract, per-harness notes) in `docs/`. Integration +tests are split by surface area under `tests/cli/` and `tests/run/`; unit tests usually live +beside the module they exercise. ## Build, Test, and Development Commands diff --git a/README.md b/README.md index cff6dc5..00ce833 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ An eval dispatches a fresh subagent twice per test case — once with the skill loaded, once without (or old version vs. new) — and grades both outputs against assertions. The pass-rate delta tells you whether the skill is worth shipping or the change is worth landing. The runner builds the workspace, stages skills for discovery, generates dispatch prompts, assembles run records from transcripts, grades, and aggregates; your agent harness supplies the one thing the runner never does itself: dispatching the subagents. -`eval-magic` ships as a dependency-less prebuilt binary under the command name `eval-magic`. Every artifact follows a documented JSON Schema, so records grade the same way regardless of where they were authored. **Claude Code and Codex CLI are wired harnesses today**; OpenCode has foundational harness selection and staging support; see [Harnesses](#harnesses) for per-harness fidelity and caveats. From inside an agent session, running an eval is as simple as: *"Install eval-magic and help me run an eval on my-skill."* +`eval-magic` ships as a dependency-less prebuilt binary under the command name `eval-magic`. Every artifact follows a documented JSON Schema, so records grade the same way regardless of where they were authored. **Claude Code and Codex CLI are fully wired harnesses today**; OpenCode has native staging support; see [Harnesses](#harnesses) for per-harness enhancement support. From inside an agent session, running an eval is as simple as: *"Install eval-magic and help me run an eval on my-skill."* This README is the complete operating guide: install, author cases, run the loop, read results, and keep a baseline. For the full flag-by-flag reference, run `eval-magic --help` (and `eval-magic --help`). For *when and why* to write an eval at all — the methodology, the decision to test, designing cases under pressure — see the [`slow-powers`](https://github.com/slowdini/slow-powers) plugin's `evaluating-skills` skill, which owns that craft. @@ -98,7 +98,7 @@ eval-magic run --guard # dispatching each task through the harness CLI (`claude -p` / `codex exec`) and # writing benchmark.json into iteration-1/. An agent session can drive the runbook # ("Read and follow RUNBOOK.md"), or you can follow it by hand — the steps are -# identical. See Claude Code below for the plugin-isolation and transcript specifics. +# identical. The runbook carries the exact dispatch recipes for your harness. # Steps 3–5 are driven from the runbook — shown here for reference: @@ -145,7 +145,7 @@ teardown 6. **`finalize`** (grade `--finalize` → aggregate) merges the judge verdicts and writes `benchmark.json` into `iteration-N/`, *above* the envs. Read it. If a `--guard` marker is still live, it also reminds you to run `teardown-guard` before editing source. 7. **`teardown`** disarms the guard, removes the staged skill set, and reclaims the workspace artifacts that are safe to delete. -The chains run in-process and stop at the first failure; re-running after a fix is safe — every sub-step skips work that's already done. The individual steps (`record-runs`, `fill-transcripts`, `detect-stray-writes`, `grade`, `aggregate`) remain callable for inspection or recovery. The per-task dispatch recipe lives in `RUNBOOK.md` and `dispatch-manifest.md`, and `ingest` reads each task's events file (`claude-events.jsonl` / `codex-events.jsonl`); un-wired harnesses still write records by hand until their adapters land. +The chains run in-process and stop at the first failure; re-running after a fix is safe — every sub-step skips work that's already done. The individual steps (`record-runs`, `fill-transcripts`, `detect-stray-writes`, `grade`, `aggregate`) remain callable for inspection or recovery. The per-task dispatch recipe lives in `RUNBOOK.md` and `dispatch-manifest.md`, and `ingest` reads each task's events file (`outputs/-events.jsonl`); harnesses without transcript ingest still write records by hand until their adapters land. ### Isolation grouping (which agents batch together) @@ -157,7 +157,7 @@ Each `(group, condition)` gets its own env — `iteration-N/env--` and `--judge-model ` to have the generated command recipes select those models when the harness adapter supports model selection (Codex today); otherwise they are recorded as provenance. Arm `--guard` unless the user actively opts out; unguarded, stray writes are only *detected* after the fact by `detect-stray-writes`, never blocked. +If you are an agent driving this tool, **never kick off a run silently.** Present the user a run summary — skill, mode, eval cases, the models that will run the agents and the judge, the cost, and the guard status — and wait for explicit confirmation. Pass `--agent-model ` and `--judge-model ` to have the generated command recipes select those models when the harness adapter supports model selection (see the [Harnesses](#harnesses) table); otherwise they are recorded as provenance. Arm `--guard` unless the user actively opts out; unguarded, stray writes are only *detected* after the fact by `detect-stray-writes`, never blocked. The judgment of *whether* a change needs an eval, and how to design cases that actually measure it, lives in the [`slow-powers`](https://github.com/slowdini/slow-powers) plugin's `evaluating-skills` skill. @@ -165,7 +165,7 @@ The judgment of *whether* a change needs an eval, and how to design cases that a After you've seen what iteration 1 produces, add **assertions** to `evals.json` and re-grade without re-dispatching. Two types: -- **`transcript_check` — mechanical.** Regex matched against a run's tool invocations. Fast, deterministic, cheap. Use for "did the agent run X" or "did file Y get written." Requires a transcript adapter (wired for Claude Code and Codex event streams today). +- **`transcript_check` — mechanical.** Regex matched against a run's tool invocations. Fast, deterministic, cheap. Use for "did the agent run X" or "did file Y get written." Requires the harness's transcript-ingest enhancement (see the [Harnesses](#harnesses) table); without it these grade as unverifiable. - **`llm_judge` — judged.** Soft criteria a model evaluates. Use for "did the response quote actual evidence." Graded by a dispatched judge subagent. Harness-independent. Exact schemas are in [`schema/`](schema/); the assertion shapes and the grading output are detailed in `eval-magic grade --help`. Every with-skill run also gets an automatic **skill-invocation meta-check** — did the skill actually influence behavior? — surfaced as an `invocation_rate` per condition; a run where the skill wasn't invoked is a non-data-point, not evidence the skill is bad. Guidance on *what makes a good assertion* lives in the slow-powers `evaluating-skills` skill. @@ -186,7 +186,7 @@ Exact schemas are in [`schema/`](schema/); the assertion shapes and the grading A skill that adds 13 seconds and 1700 tokens but improves pass rate by 50 points is probably worth it; one that doubles tokens for a 2-point gain is probably not. For Mode B the keys are `old_skill` / `new_skill`, and a positive `delta.pass_rate` means the revision is an improvement. -Read `validity_warnings` **before** trusting any delta — a low skill-invocation rate (or a flagged stray write) means the result may not reflect the skill at all. +Read `validity_warnings` **before** trusting any delta — a low skill-invocation rate, a flagged stray write, or a flagged live-source read (an arm that read the live skill source instead of its staged copy) means the result may not reflect the skill at all. ## Workspace layout @@ -250,123 +250,39 @@ A subagent that runs an eval should start in an environment that mirrors a real For the `without_skill` / baseline condition, the dispatch reflects "this skill is unavailable, others remain" when siblings were opted in with `--skill-dir`; otherwise it measures the skill against a clean no-skill baseline. `--bootstrap` is separate from parity: it injects product-specific framing inside the `` block and does not enumerate skills. -**Parity is only as clean as your session.** Staging controls what the runner *adds*, not what your session already *loaded*. Subagents dispatched in-process share the parent session's plugins, so an installed plugin exposing a same-named skill is still discoverable and contaminates both arms — the staging slug stops an on-disk collision, not runtime discovery. The runner can't unload a live plugin; on Claude Code it emits a build-time *plugin-shadow* warning (also surfaced in `benchmark.json`'s `validity_warnings`). Closing it is a launch-time step — see [Claude Code](#claude-code-fully-wired) below. +**Parity is only as clean as your session.** Staging controls what the runner *adds*, not what your session already *loaded*. Subagents dispatched in-process share the parent session's plugins, so an installed plugin exposing a same-named skill is still discoverable and contaminates both arms — the staging slug stops an on-disk collision, not runtime discovery. The runner can't unload a live plugin; on Claude Code it emits a build-time *plugin-shadow* warning (also surfaced in `benchmark.json`'s `validity_warnings`) that lists the per-dispatch isolation options inline. Closing it is a launch-time step for whoever dispatches. ## Harnesses -Every artifact follows a JSON Schema in [`schema/`](schema/), so a run record means the same thing regardless of which harness produced it. **Claude Code** and **Codex** are wired harnesses, with harness-specific fidelity notes below. The parity-audit framework for bringing another harness up to the supported feature set is in [docs/harness-parity.md](docs/harness-parity.md). +Every artifact follows a JSON Schema in [`schema/`](schema/), so a run record means the same thing regardless of which harness produced it. Harness support is **a minimal baseline plus progressive enhancements**: any harness with a headless one-shot CLI can run evals at baseline (`--no-stage` inlines the skill into each dispatch prompt, `llm_judge` grades, `detect-stray-writes` audits), and each enhancement a harness's adapter wires raises fidelity from there. What each enhancement is, why it needs harness-specific support, and what its fallback is are documented in [docs/progressive-enhancements.md](docs/progressive-enhancements.md). ### How dispatch works -Every eval test and judge is dispatched the same way: through the harness's one-shot CLI (`claude -p`, `codex exec`), one subprocess per task, each `cd`'d into its `(group, condition)` env and writing its events transcript to disk. `run` prepares the envs and `RUNBOOK.md`; from there an **agent session** can drive the loop (reading the runbook and shelling out each recipe) or a **human** can follow the same runbook by hand. +Every eval test and judge is dispatched the same way: through the harness's one-shot CLI (`claude -p`, `codex exec`), one subprocess per task, each `cd`'d into its `(group, condition)` env and writing its events transcript to disk. `run` prepares the envs and `RUNBOOK.md`; from there an **agent session** can drive the loop (reading the runbook and shelling out each recipe) or a **human** can follow the same runbook by hand. The generated `RUNBOOK.md` / `dispatch-manifest.md` carry the exact per-task recipes for the selected harness — they, not this README, are the runtime reference for dispatch commands. -Support today: +### Support -| Harness | CLI dispatch | Transcript ingest | `--guard` | -|---------|:------------:|:-----------------:|:---------:| -| **Claude Code** | ✅ | ✅ | ✅ | -| **Codex** | ✅ | ✅ | ✅ | -| **OpenCode** | ⚠️ partial¹ | ❌ | ❌ | +This table is the source of truth for per-harness enhancement support: -¹ OpenCode has foundational harness support: `--harness opencode` stages skills under `.opencode/skills/` and emits native dispatch prompts, but eval-magic does not yet drive OpenCode dispatches or ingest their transcripts. +| Harness | Native staging | Dispatch recipes | Transcript ingest | Model flag | Write guard | +|---------|:--------------:|:----------------:|:-----------------:|:----------:|:-----------:| +| **Claude Code** | ✅ | ✅ | ✅ | ✅ | ✅ | +| **Codex** | ✅ | ✅ | ✅ | ✅ | ✅ | +| **OpenCode** | ✅ | ❌¹ | ❌ | ❌ | ❌ | -### Claude Code +¹ `run --harness opencode` stages skills and emits native dispatch prompts, but prints manual `opencode run` guidance instead of a copy-pasteable recipe. -The run loop above *is* the Claude Code loop: each eval test and judge is dispatched through the `claude -p` one-shot CLI. `eval-magic run` only *prepares* the isolated envs (`.eval-magic//iteration-N/env--/`) and writes `RUNBOOK.md` into `iteration-N/`, then prints a handoff. An agent session can drive the runbook (*Read and follow RUNBOOK.md*), or you can follow it by hand — either way each task shells out the same `claude -p` recipe, and `ingest` → `finalize` assemble `benchmark.json`. These are the Claude-Code-specific details: +A missing enhancement degrades fidelity, never correctness — every column has a fallback: without native staging, `--no-stage` inlines each `SKILL.md` into its dispatch prompt; without transcript ingest, `transcript_check` assertions grade as unverifiable and `llm_judge` carries the grading (tokens and duration go unrecorded); without a model flag, `--agent-model` / `--judge-model` are recorded as provenance only; without a write guard, `--guard` is rejected and `detect-stray-writes` audits after the fact. -**Isolating from installed plugins.** Read this first if the skill you're evaluating shares a name with one an installed, enabled plugin provides. Each `claude -p` dispatch loads *your* user/global plugins and the global skills dir from its Claude config — the staging slug avoids an on-disk collision but does not stop the installed copy from being discoverable, contaminating both arms (the `without_skill` arm is then not truly skill-absent). The runner can only *detect and warn* (the plugin-shadow banner). To actually isolate, constrain the config each `claude -p` dispatch loads, one of these ways: - -- **Drop user-scope plugins, keep auth:** add `--setting-sources project,local` to the dispatch. User-scope `enabledPlugins` isn't loaded; auth is unaffected. -- **Disable the specific plugin:** set `"enabledPlugins": { "@": false }` in a settings source the dispatch loads. -- **Clean config dir (strips everything):** run each dispatch under `CLAUDE_CONFIG_DIR="$(mktemp -d)"`. No installed plugins or global skills load at all. Auth caveat: OAuth lives in `~/.claude.json`, which a relocated config dir may not carry — set `ANTHROPIC_API_KEY` or re-authenticate once in the fresh dir. - -Project-local staged skills live in each env at `.claude/skills/`, independent of installed plugins, so they still load and the meta-check still resolves the slug under all three. - -**Discovery is structural.** Claude Code discovers the project-local skills present in the dispatch's cwd. Because `eval-magic run` builds each env's `.claude/skills/` *before* the dispatch runs, the staged skills are present from the start — there is no mid-session staging hazard. `--no-stage` remains for harnesses without project-local skill discovery: each `SKILL.md` is inlined into its dispatch prompt instead of staged. Regardless, run `detect-stray-writes` (folded into `ingest`) before trusting a result. - -**Dispatching via `claude -p`.** `dispatch.json` is a top-level object (`{ skill_name, iteration, run_nonce, …, tasks: [...] }`); iterate `tasks[]`. `run` prints (and `dispatch-manifest.md` / `RUNBOOK.md` carry) a `claude -p` recipe per task: - -```bash -cd && claude -p --output-format stream-json --verbose --permission-mode acceptEdits \ - "Read the file at and follow its instructions exactly. …" \ - /claude-events.jsonl \ - 2> /claude-stderr.log -``` - -Three details differ from Codex's `codex exec`: `--output-format stream-json` **requires `--verbose`** in `-p` mode; `claude` has **no `--cd` flag**, so each dispatch must run from its env dir (`cd &&`) — staged-skill discovery is cwd-relative, so getting this wrong makes the `with_skill` arm behave like `without_skill`; and there is **no `--output-last-message`**, so the final message is recovered from the stream-json `result` event rather than a file. Detach stdin with `` operating-context layer. - -Besides out-of-bounds writes, `detect-stray-writes` also flags **live-source reads**: an arm whose subagent read the live skill source instead of its staged copy. That usually means the Skill tool couldn't resolve the staged slug and the agent improvised — fatal in revision mode, where the `old_skill` arm then sees new-skill content. Treat a flagged cell's arm as contaminated. - -**Write guard.** `--guard` stages a `PreToolUse` hook in each env's `.claude/settings.local.json`; because every `claude -p` dispatch runs from its env (`cd `), it loads and enforces the hook (the recipe never passes `--bare`, which would skip hook discovery). A deny aborts the offending dispatch; `detect-stray-writes` (folded into `ingest`) remains the after-the-fact backstop. - -### Codex - -Codex dispatches each task through the `codex --ask-for-approval never exec --json` one-shot CLI — the same single dispatch path as Claude Code (see [How dispatch works](#how-dispatch-works)). An agent session can drive the runbook, or you can follow it by hand; either way each test and judge shells out the same `codex exec` recipe. - -Pass `--harness codex`: skills stage under repo-local `.agents/skills/` (the staged skill-under-test's frontmatter `name:` is rewritten to the eval slug so Codex's repo-local discovery sees it), and `conditions.json` / `dispatch.json` record `"harness": "codex"`. Dispatch each task with a fresh `codex --ask-for-approval never exec --json` execution, capturing the event stream: - -```bash -codex --ask-for-approval never exec --cd --sandbox workspace-write --json \ - --output-last-message /final-message.md \ - "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ - /codex-events.jsonl \ - 2> /codex-stderr.log -``` - -When `run --agent-model ` is set, the generated Codex recipes insert `-m ` before `--json`: - -```bash -codex --ask-for-approval never exec --cd --sandbox workspace-write -m --json \ - --output-last-message /final-message.md \ - "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ - /codex-events.jsonl \ - 2> /codex-stderr.log -``` - -When the run was armed with `--guard`, add `--dangerously-bypass-hook-trust` to that `codex --ask-for-approval never exec` command so the vetted project-local `PreToolUse` hook staged in `.codex/hooks.json` actually runs: - -```bash -codex --ask-for-approval never exec --cd --sandbox workspace-write --dangerously-bypass-hook-trust --json \ - --output-last-message /final-message.md \ - "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ - /codex-events.jsonl \ - 2> /codex-stderr.log -``` - -The `` becomes the default `model` in `judge-tasks.json`; an individual `llm_judge.model` overrides it. The Codex judge recipe reads each task's resolved `model` and passes `-m "$model"` only when one is present. - -`finalize` and `teardown` work the same with `--harness codex`. Codex results are lower fidelity than Claude Code in a few places: `transcript_check` matches parsed `item.completed` entries (`command_execution`, `file_change`, `web_search`, MCP items); the automatic `__skill_invoked` meta-check uses the LLM-judge fallback (Codex's JSONL exposes no deterministic skill-tool event); and `--plan-mode` injects the shared plan-mode procedure as text rather than launching `codex exec` into the interactive CLI's real `/plan` mode. `--guard` stages a Codex `PreToolUse` hook that blocks out-of-sandbox `Bash` mutations and `apply_patch` targets before they run; `detect-stray-writes` remains the post-run audit. Bias Codex suites toward `llm_judge` assertions for behavior and `transcript_check` for tool events. - -When running `eval-magic run --harness codex` from inside Codex itself, staging writes `.agents/skills`; adding `--guard` also writes `.codex/hooks.json`. Those project-local Codex config paths are protected by Codex's default workspace-write sandbox, so the runner may need approval/escalation or an external terminal invocation. That approval is Codex's own permission boundary, not something eval-magic bypasses. - -### OpenCode - -OpenCode is wired for **foundational harness selection and staging**. Pass `--harness opencode`: skills stage under repo-local `.opencode/skills/` (OpenCode's native project-local skills directory), the staged skill-under-test's frontmatter `name:` is rewritten to a sanitized OpenCode-valid slug, and `conditions.json` / `dispatch.json` record `"harness": "opencode"`. The dispatch prompt renders OpenCode's native `` XML block and a plan-mode approximation via ``. - -OpenCode skill names must be lowercase alphanumeric with single-hyphen separators, match the containing directory name, and be 1–64 characters. The runner sanitizes the eval-generated slug for the staged copy; sibling skills are staged at their natural names and must already satisfy OpenCode's naming rules. - -**Dispatching.** Eval-magic does not yet drive OpenCode dispatches automatically. Iterate `tasks[]` in `dispatch.json` and dispatch each task with `opencode run`, passing the prompt at `dispatch_prompt_path`. Capture the result and assemble `run.json`/`timing.json` manually, or record `opencode run --format json` / `opencode export` output for the upcoming transcript adapter. - -**Fidelity notes.** Transcript ingest, auto-record, and `--guard` are not yet wired for OpenCode. The automatic `__skill_invoked` meta-check uses the LLM-judge fallback until a transcript adapter lands, and `transcript_check` assertions grade as *unverifiable*. `detect-stray-writes` still audits out-of-bounds writes from any parsed transcript. `--guard` is rejected with a clear message for OpenCode; use `detect-stray-writes` as the audit fallback. +Per-harness implementation notes for developers wiring features live in [docs/claude-notes.md](docs/claude-notes.md), [docs/codex-notes.md](docs/codex-notes.md), and [docs/opencode-notes.md](docs/opencode-notes.md). ## Documentation | Where | What's in it | |-------|--------------| | `eval-magic --help` / `eval-magic --help` | The flag-by-flag reference: every subcommand and flag, worked examples, the `--skill-dir` model, the skill-invocation meta-check | -| [docs/harness-parity.md](docs/harness-parity.md) | The parity-audit framework for bringing a new harness up to the supported feature set | +| [docs/progressive-enhancements.md](docs/progressive-enhancements.md) | Development doc: the harness baseline-vs-enhancement contract — what each enhancement unlocks, why it needs harness-specific code, and its fallback | +| [docs/claude-notes.md](docs/claude-notes.md) / [docs/codex-notes.md](docs/codex-notes.md) / [docs/opencode-notes.md](docs/opencode-notes.md) | Development docs: per-harness implementation notes for working on eval-magic's harness support | | [GitHub issues](https://github.com/slowdini/eval-magic/issues) | Planned features and known limitations | ## Bundled assets diff --git a/docs/claude-notes.md b/docs/claude-notes.md new file mode 100644 index 0000000..8ea96ed --- /dev/null +++ b/docs/claude-notes.md @@ -0,0 +1,76 @@ +# Claude Code — harness implementation notes + +> **Audience:** developers working on eval-magic's Claude Code support. Runtime usage lives in the +> README, `--help`, and the generated `RUNBOOK.md`; the enhancement model is in +> [progressive-enhancements.md](progressive-enhancements.md). + +## Code map + +Everything Claude-Code-specific lives under `src/adapters/claude_code/`: + +| File | What's in it | +|------|--------------| +| `mod.rs` | `ClaudeCodeAdapter` — the trait impl | +| `cli.rs` | `claude -p` exec / parallel / judge recipe rendering | +| `session.rs` | native available-skills block (Skill-tool list) | +| `stream_json.rs` | `-p --output-format stream-json` transcript parsing | +| `transcript.rs` | JSONL record shapes + shared tool-call extractors | +| `plugin_shadow.rs` | plugin-shadow detection + isolation banner | +| `guard.rs` | write-guard hook install + `hookSpecificOutput` deny verdict | + +## Dispatch quirks (all forced by the `claude` CLI) + +- `--output-format stream-json` **requires `--verbose`** in `-p` mode. +- There is **no `--cd` flag**: every dispatch must run from its env dir (`cd &&`). + Staged-skill discovery is cwd-relative, so getting this wrong makes the `with_skill` arm behave + like `without_skill`. +- There is **no `--output-last-message`**: the final message is recovered from the stream-json + `result` event rather than a file. +- `@": false }` in a + settings source the dispatch loads. +- **Clean config dir (strips everything):** run each dispatch under + `CLAUDE_CONFIG_DIR="$(mktemp -d)"`. No installed plugins or global skills load. Auth caveat: + OAuth lives in `~/.claude.json`, which a relocated config dir may not carry — set + `ANTHROPIC_API_KEY` or re-authenticate once in the fresh dir. + +Project-local staged skills are independent of installed plugins, so they still load and the +meta-check still resolves the slug under all three options. + +## Write guard + +`--guard` merges a `PreToolUse` hook into each env's `.claude/settings.local.json` (matcher: +`Write|Edit|MultiEdit|NotebookEdit|Bash`). Every dispatch runs from its env, so it loads and +enforces the hook — the recipe never passes `--bare`, which would skip hook discovery. The hook +invokes the hidden `guard` subcommand (**stable on-disk contract — never rename**), which denies +via Claude Code's `hookSpecificOutput` JSON shape and stays silent to allow. Both layers fail open. +A deny aborts the offending dispatch; `detect-stray-writes` remains the after-the-fact backstop. diff --git a/docs/codex-notes.md b/docs/codex-notes.md new file mode 100644 index 0000000..7c8fd3b --- /dev/null +++ b/docs/codex-notes.md @@ -0,0 +1,69 @@ +# Codex — harness implementation notes + +> **Audience:** developers working on eval-magic's Codex support. Runtime usage lives in the +> README, `--help`, and the generated `RUNBOOK.md`; the enhancement model is in +> [progressive-enhancements.md](progressive-enhancements.md). + +## Code map + +Everything Codex-specific lives under `src/adapters/codex/`: + +| File | What's in it | +|------|--------------| +| `mod.rs` | `CodexAdapter` — the trait impl | +| `cli.rs` | `codex exec` exec / parallel / judge recipe rendering | +| `session.rs` | native `## Skills` markdown block | +| `transcript.rs` | `item.completed` event-stream parsing | +| `guard.rs` | write-guard hook install + `{"decision": "block"}` verdict | + +## Dispatch (`codex exec`) + +- The approval policy must come **before** `exec`: `codex --ask-for-approval never exec …` + (codex-cli rejects it after `exec`). +- `--cd ` sets the dispatch cwd (no shell `cd` needed, unlike Claude Code). +- `--sandbox workspace-write` bounds writes to the env. +- `--json` streams events to stdout — captured as `outputs/codex-events.jsonl`; stderr goes to + `codex-stderr.log` so progress/status text (e.g. stdin notices) stays out of the JSONL. +- `--output-last-message /final-message.md` writes the final-message file the + pipeline reads. +- `` is inserted before `--json` when `run --agent-model` is set. Judge recipes read each +task's resolved `model` from `judge-tasks.json` (`run --judge-model` sets the default; a +per-assertion `llm_judge.model` overrides it) and pass `-m "$model"` only when one is present. + +## Staging & discovery + +Skills stage under repo-local `.agents/skills/`. Codex keys discovery on the frontmatter `name:`, +so the staged skill-under-test's frontmatter is rewritten to the eval slug +(`rewrites_frontmatter_name` true) and the available-skills block advertises the slug +(`advertises_staged_slug_name` true). + +## Transcript (`item.completed`) + +`item.completed` events whose item type is not an agent message / reasoning / plan update become +tool invocations: `command_execution`, `file_change`, `web_search`, and MCP items. +`transcript_check` matches these parsed items. The JSONL exposes **no deterministic skill-tool +event**, so `transcript_surfaces_skill_invocation()` is false and the `__skill_invoked` meta-check +uses the LLM-judge fallback. Token accounting excludes cached input tokens. + +## Write guard + +`--guard` merges a `PreToolUse` hook into `.codex/hooks.json` (matcher: +`^Bash$|^apply_patch$|^Edit$|^Write$`, with a 30s timeout and status message). Dispatches must pass +`--dangerously-bypass-hook-trust` so the vetted project-local hook actually runs — the generated +recipes add it whenever the run was armed. The hook invokes the hidden `guard-codex` subcommand +(**stable on-disk contract — never rename**), which blocks via Codex's +`{ "decision": "block", "reason": "..." }` stdout shape and stays silent to allow. Teardown prunes +`.codex/` when restoring the original config leaves it empty (`guard_hook_cleanup_dir`). + +## Running inside Codex itself + +`eval-magic run --harness codex` from a Codex session writes `.agents/skills` (and, with +`--guard`, `.codex/hooks.json`). Those project-local Codex config paths are protected by Codex's +default workspace-write sandbox, so the runner may need approval/escalation or an external +terminal invocation. That approval is Codex's own permission boundary, not something eval-magic +bypasses. diff --git a/docs/harness-parity.md b/docs/harness-parity.md deleted file mode 100644 index 45a4c42..0000000 --- a/docs/harness-parity.md +++ /dev/null @@ -1,137 +0,0 @@ -# Eval-Magic Harness Parity Check - -You are an agent running inside one of the eval runner's supported harnesses. This file walks you through auditing **how completely your harness is wired into the eval runner** and prepping to close one gap. - -This file covers the **`eval-magic` runner only** — the infrastructure in `eval-magic` that dispatches, records, and grades skill evals. - -Read the file end-to-end before acting. Steps 4a and 4b are the source of truth for what "eval-magic parity" means today — when the `HarnessAdapter` trait gains a method, those steps are updated and this file stays evergreen. - -## The parity surface - -Every dispatch now rides a single mechanism: each task is delivered through a one-shot harness CLI subprocess (`claude -p`, `codex exec`), one subprocess per task, writing its events transcript to disk. **Codex** and **Claude Code** are the references. Whether an agent session or a human drives the runbook doesn't change how a single task reaches the harness — see the README's [How dispatch works](../README.md#how-dispatch-works) section. - -Parity is therefore a single axis: **harness-adapter feature parity.** Each harness plugs into the runner through one impl of the **`HarnessAdapter`** trait in `src/adapters/harness.rs`, resolved by `adapter_for(harness)`. The trait's methods *are* the feature surface: skill-list rendering, transcript parsing, staged-skills dir, plan-mode profile, the write-guard hook, and the CLI dispatch guidance. A harness reaches parity by implementing the trait methods the CLI dispatch path consumes. `capabilities_for(harness)` (`src/core/capabilities.rs`) carries the narrow run-option capabilities the generic `run` preflight validates. - -A harness can wire the dispatch path yet leave some adapter methods at their stub/default (lower fidelity). Step 4 audits how complete each is. - ---- - -## Step 1 — Identify your harness - -Name the harness you are running in. You almost certainly already know — confirm by checking: - -- Your invocation context and working directory -- The tool names available to you in this session -- Any session-start context block injected at the top of the conversation - -The intended supported harnesses are: **Claude Code, Codex CLI, OpenCode**. - -If the harness you are running in is not in that list, stop and ask the user before continuing. - ---- - -## Step 2 — Read the reference materials - -Read these in order. Paths are relative to the repository root. - -| Source | What to look for | -|--------|------------------| -| `src/core/capabilities.rs` | `HarnessRunCapabilities` and `capabilities_for` — the focused run-option capabilities the generic `run` preflight validates per harness | -| `src/adapters/harness.rs` | The `HarnessAdapter` trait (the feature surface), the three impls, and `adapter_for`. The reference impls are `ClaudeCodeAdapter` and `CodexAdapter` — read the one closest to your harness | -| `src/adapters/claude_stream_json.rs` and `src/adapters/codex_transcript.rs` | The reference CLI events parsers. `parse_cli_events*` translates each harness's `outputs/-events.jsonl` into the same `ToolInvocation` list / `TranscriptSummary` every downstream stage consumes (`claude_stream_json.rs` parses `claude -p` stream-json; Codex parses its `codex-events.jsonl`). A second harness translates its transcript shape into the same types | -| `eval-magic --help` and the README's [Environment parity](../README.md#environment-parity) / [Harnesses](../README.md#harnesses) sections | The cross-harness breadcrumbs and the flag-by-flag reference. Treat the breadcrumbs as starting points, not specifications | - -Do not skim. The parity report you produce in Step 4 is only as good as the reference you internalized here. - ---- - -## Step 3 — Discover your harness's existing surface area - -Enumerate, using ordinary file search, what already exists for your harness. Do not rely on memory — search the working tree. Useful heuristics: - -- Your harness's arm in `adapter_for` and its `HarnessAdapter` impl in `src/adapters/harness.rs` -- The harness name anywhere inside `src/` (especially `src/adapters/`, `src/core/context.rs`) and `profiles/` -- A per-harness section in the README, or tests exercising the runner for the harness (`tests/`, e.g. `tests/run/codex.rs`, `tests/run/opencode.rs`) - -Record every path you find. You will reference them in Step 4. - ---- - -## Step 4a — Audit CLI-dispatch parity - -Confirm your harness's CLI dispatch path runs end-to-end. It consumes these `HarnessAdapter` methods: `parse_cli_events` / `parse_cli_events_full` (the events parser), `cli_events_filename` (the per-task transcript file the CLI writes), `cli_model_flag` (the harness-native model-selection flag, when supported), `cli_next_steps` (the post-`run` dispatch guidance), `cli_manifest_section` (the dispatch-manifest recipe), and `cli_judge_next_steps` (the post-`grade` / post-`ingest` judge dispatch recipe). - -A harness reaches dispatch parity when its path runs end-to-end: dispatch guidance is emitted, each task's events transcript is found and parsed, and `record-runs` / `fill-transcripts` assemble records. The README's [How dispatch works](../README.md#how-dispatch-works) matrix tracks current support. - -## Step 4b — Audit harness-adapter feature parity - -For each `HarnessAdapter` method below, compare your harness's impl against the reference. Methods are described by what they *do* so they survive renames; when the trait changes, this list is updated and the rest of the file still applies. - -| Adapter capability | Trait method(s) | Reference behavior | -|--------------------|-----------------|--------------------| -| Realistic eval environment (skill staging) | `skills_dir`, `render_available_skills_block`, `rewrites_frontmatter_name`, `advertises_staged_slug_name`, `skill_surface_phrase`, `skill_unresolved_phrase` | Stage skills under the harness's project-local dir and render the discoverable-skills block in the harness's **native** presentation, so a dispatch reads like a real session in that harness, not an eval. Claude Code: `.claude/skills/` + `The following skills are available for use with the Skill tool:`. The `--bootstrap` `` wrapper and the slug-disambiguation framing are shared in `src/cli/run/dispatch.rs` | -| Skill-eval transcript adapter | `parse_cli_events`, `parse_cli_events_full` | Translate the harness's `outputs/-events.jsonl` into the same `ToolInvocation` list and `TranscriptSummary` (final message, tool invocations, deduped usage/timing) every downstream stage consumes. Claude Code parses `claude -p` stream-json (`claude_stream_json.rs`); Codex parses its `codex-events.jsonl`. A new harness implements this pair for its own events shape | -| Skill-eval auto-record (run/timing assembly) | (consumes `parse_cli_events_full` + `cli_events_filename`) | `src/pipeline/record_runs.rs` assembles each task's `run.json` + `timing.json` from disk: carry-over fields from `dispatch.json`, `final_message` from `outputs/final-message.md` (falling back to the transcript's final text — the primary path for `claude -p`, which has no `--output-last-message`), and tool invocations/tokens/duration from the parsed transcript. A harness closes this gap by supplying the transcript its parser consumes (the portable fallback — hand-authored records against `run-record.schema.json` — is unchanged) | -| CLI model selection | `cli_model_flag`, `cli_next_steps`, `cli_manifest_section`, `cli_judge_next_steps` | For one-shot CLI dispatch, `run --agent-model` is rendered into the agent command recipe and `run --judge-model` becomes the default model in `judge-tasks.json`; assertion-level `llm_judge.model` remains the most specific override. Codex is the reference: `cli_model_flag` returns `-m`, agent recipes render `codex --ask-for-approval never exec ... -m `, and judge recipes read each task's resolved `model` and pass `-m "$model"` only when present | -| Eval subagent write enforcement | `install_guard` | Opt-in `--guard` stages a pre-tool hook (`src/sandbox/`) that *denies* subagent writes/installs outside the eval sandbox while dispatches run. Portable fallback for every harness: the `eval-magic detect-stray-writes` post-pass (`src/pipeline/detect_stray_writes.rs`) flags out-of-bounds writes from the parsed transcript after the fact | -| Eval plan-mode operating context | `render_plan_mode_context` | Opt-in `--plan-mode` injects the shared `profiles/shared/plan-mode.md` (embedded at compile time) as a `` operating-context layer in every dispatch — identical text for every harness | -| Harness-details operator guide | (docs, not a trait method) | The README's per-harness section, e.g. [Claude Code](../README.md#claude-code) | - -**Note on the transcript adapter (raised bar).** Baseline eval suites use `transcript_check` assertions — deterministic regex checks against a run's tool invocations (e.g. "a test command ran", "the sibling skill was loaded"). These only grade when `parse_cli_events` is implemented for your harness. A harness without it still functions: those assertions grade as *unverifiable* and the `llm_judge` assertions carry the substantive measurement. But adapter richness is an explicit parity target, not optional polish — implementing or enriching `parse_cli_events*` lets more of a baseline suite grade mechanically. Treat it as a goal to aim at, not a box already checked. - -**Note on write enforcement (parity goal).** Eval subagents are instructed to write only inside their `outputs/` dir, but nothing in the portable contract *enforces* it — a misbehaving subagent can edit the real repo or install packages, silently tainting the run. Two layers address this: the portable `detect-stray-writes` post-pass (available to every harness, since it works off the same parsed transcript) and an opt-in harness-native `install_guard` that stages a pre-tool hook to *block* the write before it happens. Claude Code and Codex both wire this through their `PreToolUse` hook surfaces — `claude -p` loads the same project-local `settings.local.json` hook from the env cwd each dispatch runs in. **Harness-level tool enforcement is an explicit parity goal, not optional polish.** A harness that can express a pre-tool guard (a hook, a permission rule, a sandboxed cwd) should wire `install_guard`; until then, `detect-stray-writes` is the honest fallback. - -**Note on plan-mode fidelity (residual parity goal).** `--plan-mode` injects a shared, harness-agnostic plan-mode procedure as operating context, the closest a harness's eval runner can get to reproducing the wild failure where a real plan mode makes loading a skill feel redundant. It is **not** the real mode: it is still text the dispatched subagent reads, not a state the harness places it under, so a pass remains necessary-not-sufficient (the seeding ceiling is explained in the [`slow-powers`](https://github.com/slowdini/slow-powers) `evaluating-skills` skill). A harness that can dispatch an eval subagent *into* its own plan/research mode would close this gap; `--plan-mode` (a profile + renderer) is the approximation every harness can reach in the meantime. - -Surface your findings inline using this template: - -``` -## Eval-Magic Parity Report: - -### CLI-dispatch parity -- Dispatch path wired end-to-end? - -### Harness-adapter feature parity -- **Skill staging + native skill block** — ✅ Implemented / ⚠️ Partial / ❌ Missing / N/A - - Where: "> - - Gap: - -(... one block per adapter capability ...) - -## Summary -- Strongest area: -- Highest-leverage gap: — -- Suggested next gap to close this session: -``` - -Status meanings: - -- **✅ Implemented** — fully wired; works the same way the reference's does, using whatever native primitive the harness provides -- **⚠️ Partial** — some scaffolding exists (e.g. the trait method is a stub or returns the default) but the capability isn't end-to-end functional -- **❌ Missing** — no implementation; users of this harness do not get this capability -- **N/A** — the capability doesn't translate. State why - -The agent reports inline by default. If the user asks for a persistent artifact, write the report to `docs/parity-reports/-evals.md` (create the directory if missing). - ---- - -## Step 5 — Pick a gap and prep to close it - -Surface the report to the user and propose **one or two** gaps worth closing this session. Bias toward the smallest gap with the highest user impact — typically a `parse_cli_events` impl or an operator-guide section, not a wholesale runner rework. - -Once the user picks a gap: - -1. Re-read the reference impl for that capability in detail (the matching `HarnessAdapter` method on `ClaudeCodeAdapter` or `CodexAdapter`, plus what it delegates to). Note the *shape* — inputs, outputs, side effects — separately from the *harness-specific mechanism* it uses. -2. **Consult your harness's own documentation, MCP servers, or built-in references** before proposing harness-specific changes. Do not guess at hook schemas, transcript formats, or native tool names. If a docs-fetch server is available, prefer it over your training data — assume your knowledge of the harness may be stale. -3. Propose an adaptation that copies the reference's shape while using your harness's native conventions — i.e. fill in your harness's `HarnessAdapter` method. State explicitly what you are copying and what you are adapting. -4. Confirm with the user before writing code. - ---- - -## Guardrails - -- **Cross-harness compatibility is enforced.** A change for your harness MUST NOT break or degrade any other harness. Keep harness-specific behavior behind your `HarnessAdapter` impl; generic dispatch code goes through the trait. -- **One problem per PR.** A parity-closing PR should wire one capability for one harness. -- **Do not fabricate features that don't exist in any harness yet.** Parity means "catch up to a reference adapter," not "invent something new." -- **Do not guess at harness-specific details.** If your harness's docs don't confirm something, ask the user before proceeding. -- **Keep this file evergreen.** If the `HarnessAdapter` trait gains a method, update Step 4a / 4b here in the same PR. diff --git a/docs/opencode-notes.md b/docs/opencode-notes.md new file mode 100644 index 0000000..06742b4 --- /dev/null +++ b/docs/opencode-notes.md @@ -0,0 +1,56 @@ +# OpenCode — harness implementation notes + +> **Audience:** developers working on eval-magic's OpenCode support. Runtime usage lives in the +> README, `--help`, and the generated `RUNBOOK.md`; the enhancement model is in +> [progressive-enhancements.md](progressive-enhancements.md). + +## Code map + +Everything OpenCode-specific lives under `src/adapters/opencode/`: + +| File | What's in it | +|------|--------------| +| `mod.rs` | `OpenCodeAdapter` — the trait impl, plus the slug/naming rules | +| `session.rs` | native `` XML block | + +## What's wired + +Native staging only: `--harness opencode` stages under `.opencode/skills/`, rewrites the staged +skill-under-test's frontmatter `name:` to a sanitized slug, and renders the `` +XML block in dispatch prompts. Everything else rides the trait's enhancement defaults: + +- **No dispatch recipes** — `cli_next_steps` prints manual `opencode run` guidance instead of a + copy-pasteable template. +- **No transcript ingest** — `cli_events_filename` is `None`, so the ingest pipeline never calls + the (defaulted, erroring) parsers; `transcript_check` grades as unverifiable and the + `__skill_invoked` meta-check uses the LLM-judge fallback. +- **No model flag** — `--agent-model` / `--judge-model` are recorded as provenance only. +- **No write guard** — `--guard` is rejected in the `run` preflight + (`run_capabilities().supports_guard` is false); `detect-stray-writes` is the audit fallback. + +## Naming rules + +OpenCode skill names must be 1–64 characters, lowercase alphanumeric with single-hyphen separators +(no leading/trailing/consecutive hyphens), and match the containing directory name. `staged_slug` +sanitizes the generated slug while preserving the `slow-powers-eval-` cleanup prefix (truncating +the skill portion if the combination exceeds 64 chars); `validate_stage_name` applies the same +rules to `--stage-name` overrides. Sibling skills stage at their natural names and must already +satisfy the rules. + +## Known inconsistency + +The staged skill's frontmatter is rewritten to the slug (`rewrites_frontmatter_name` true) yet the +available-skills block advertises the *natural* name (`advertises_staged_slug_name` false) — +tracked for a separate fix. + +## Wiring the next enhancements + +- **Transcript ingest:** candidate sources are `opencode run --format json` and `opencode export`. + Implement `parse_cli_events` / `parse_cli_events_full` in a new `transcript.rs` and set + `cli_events_filename`; check whether the stream exposes a deterministic skill event before + leaving `transcript_surfaces_skill_invocation` at its default. +- **Dispatch recipes:** an `opencode run` command template in a new `cli.rs`, wired through + `cli_next_steps` / `cli_manifest_section` / `cli_judge_next_steps`. +- **Write guard:** needs an OpenCode pre-tool hook surface. Flip + `run_capabilities().supports_guard` and `guard_armed_message` together — an invariant test in + `src/adapters/harness.rs` enforces the lockstep. diff --git a/docs/progressive-enhancements.md b/docs/progressive-enhancements.md new file mode 100644 index 0000000..960f4f4 --- /dev/null +++ b/docs/progressive-enhancements.md @@ -0,0 +1,160 @@ +# Harness progressive enhancements + +> **Audience:** developers working on the eval-magic codebase. The README and `eval-magic --help` +> are the user-facing docs; this file explains how harness support is structured in code and what +> wiring more of it buys. Per-harness implementation notes live in [claude-notes.md](claude-notes.md), +> [codex-notes.md](codex-notes.md), and [opencode-notes.md](opencode-notes.md). + +Harness compatibility is not a parity checklist to audit — it is **a minimal baseline every harness +satisfies, plus optional enhancements** a harness's adapter opts into. A missing enhancement +degrades fidelity, never correctness: every enhancement has a documented fallback. + +## One dispatch mechanism + +Every eval test and judge is dispatched the same way: through the harness's one-shot CLI, one +subprocess per task, each `cd`'d into its `(group, condition)` env and writing its output to disk. +There is no other mode. The **generated artifacts are the runtime source of truth** for how to +dispatch: `run` writes `RUNBOOK.md` and `dispatch-manifest.md` carrying the exact per-task recipe +for the selected harness (rendered by the adapter's dispatch-recipe methods) — hand-maintained docs +never carry command recipes. + +## The baseline contract + +A harness qualifies at baseline with no harness-specific code beyond naming itself: + +1. **A headless exec command** — some way to invoke the harness with a prompt from a chosen cwd and + let it run to completion. +2. **A recoverable final message** — the agent writes `outputs/final-message.md` (the dispatch + prompt asks for this), or the transcript parser recovers it where one exists. +3. **`--no-stage` when native staging isn't wired** — each `SKILL.md` is inlined into its dispatch + prompt instead of staged for native discovery. + +That baseline already yields a working eval: `llm_judge` assertions grade every behavior, and the +`detect-stray-writes` post-pass (folded into `ingest`) audits out-of-bounds writes from whatever +run records exist. Run records without a transcript parser are assembled by hand per +`schema/run-record.schema.json`. + +In trait terms the baseline is two methods: `label()` and `skills_dir()`. Everything else on +`HarnessAdapter` has a default — either a working generic fallback or an `Unsupported` error naming +the enhancement it belongs to. + +## Where this lives in code + +- `src/adapters/harness.rs` — the `HarnessAdapter` trait, tiered into baseline and enhancement + sections; `adapter_for()` is the single place a concrete harness is named. +- `src/adapters//` — everything specific to one harness: the adapter impl, session + renderers, transcript parsers, dispatch-recipe rendering, guard hooks. +- `run_capabilities()` — the narrow table the `run` preflight uses to accept or reject run options + (`--guard`, `--bootstrap`/`--stage-name` with `--no-stage`) per harness. + +## The enhancements + +Each enhancement is a group of trait methods with defaults. Wire them together; invariant tests in +`harness.rs` catch the combinations that must move in lockstep. + +### Transcript parser + +*Why harness-specific:* every harness persists a different event stream (Claude Code's `-p` +stream-json, Codex's `item.completed` JSONL) — parsing is real per-harness code, not a mapping. + +*What it unlocks:* `transcript_check` assertions, token/cost/duration capture, automatic +`run.json`/`timing.json` assembly by `ingest`, and — where the transcript exposes a skill-tool +event — a deterministic `__skill_invoked` meta-check. + +*Fallback:* `transcript_check` grades as *unverifiable*, `llm_judge` carries the grading (bias +suites toward `llm_judge` for such a harness), tokens/duration go unrecorded, records are +hand-assembled, and the meta-check uses the LLM-judge fallback. + +*Trait methods:* `cli_events_filename` (gate: `None` means the ingest pipeline never calls the +parsers), `parse_cli_events`, `parse_cli_events_full`, `transcript_surfaces_skill_invocation`. + +### Native skill staging + skills block + +*Why harness-specific:* each harness has its own project-local discovery dir and its own way of +surfacing discoverable skills to a session (Claude Code's Skill-tool list, Codex's `## Skills` +markdown, OpenCode's `` XML), and some constrain skill naming. + +*What it unlocks:* environment parity — the staged skill is discovered the way a real install +would discover it, instead of being pasted into the prompt. + +*Fallback:* `--no-stage` inlines each `SKILL.md` into its dispatch prompt. + +*Trait methods:* `skills_dir` semantics, `staged_slug`, `validate_stage_name`, +`rewrites_frontmatter_name`, `advertises_staged_slug_name`, `render_available_skills_block`, +`skill_surface_phrase`, `skill_unresolved_phrase`, `config_dir_names`. + +### Model flag + +*Why harness-specific:* the CLI flag (and its position in the command) differs per harness. + +*What it unlocks:* `--agent-model` / `--judge-model` actually select models in the generated +recipes; judge tasks resolve a per-task model. + +*Fallback:* the models are recorded as provenance in `conditions.json` only; dispatches run on the +harness's default model. + +*Trait methods:* `cli_model_flag` (consumed by the harness's recipe renderers). + +### Write guard + +*Why harness-specific:* the guard arms a *native pre-tool hook* — hook config location, matcher +syntax, trust model, and deny-verdict shape are all harness-native (Claude Code's +`settings.local.json` + `hookSpecificOutput`, Codex's `hooks.json` + `{"decision": "block"}`). + +*What it unlocks:* out-of-bounds writes are *blocked before they happen* instead of detected +afterwards. + +*Fallback:* `detect-stray-writes` audits after the fact. (It also flags **live-source reads** — an +arm whose subagent read the live skill source instead of its staged copy, which contaminates the +arm; fatal in revision mode, where the `old_skill` arm then sees new-skill content.) + +*Trait methods:* `install_guard`, `guard_armed_message`, `guard_hook_cleanup_dir`, plus +`run_capabilities().supports_guard` (invariant-tested to stay in lockstep with the banner). The +hidden `guard` / `guard-codex` subcommands are the hook entry points — their names are a stable +on-disk contract. Shared marker/manifest/teardown machinery lives in `src/sandbox/`. + +### Plan-mode context + +*Why harness-specific in principle:* a harness could inject a real native plan mode. + +*What the default does:* wraps the shared `profiles/shared/plan-mode.md` procedure in a +`` block — an approximation that is the same for every harness today, since plan +modes can't be reproduced exactly in a one-shot dispatch anyway. + +*Trait methods:* `render_plan_mode_context`. + +### Dispatch recipes + +*Why harness-specific:* the copy-pasteable command template is the harness's CLI. + +*What it unlocks:* `RUNBOOK.md`, `dispatch-manifest.md`, and the post-`run`/post-`ingest` handoffs +carry exact per-task commands (including parallel and judge variants). + +*Fallback:* the generic handoff text; the operator constructs dispatch commands themselves. + +*Trait methods:* `cli_next_steps`, `cli_manifest_section`, `cli_judge_next_steps`. + +## Current support + +The **Harnesses table in the README is the source of truth** for which harness has which +enhancement — keep it in sync with the adapters when wiring or dropping one. + +## Adding a new harness + +1. Add the variant to `Harness` in `src/core/context.rs` (it derives `clap::ValueEnum`). +2. Create `src/adapters//mod.rs` with the adapter struct implementing `label()` + + `skills_dir()`, and register it in `adapter_for()` (`src/adapters/harness.rs`). +3. Create `docs/-notes.md` with the implementation notes discovered along the way. +4. Add the harness to the README support table (all enhancements ❌ at baseline). +5. Wire enhancements in leverage order — dispatch recipes and transcript parser first (they carry + the most fidelity), then staging, model flag, guard — updating the table as each lands. + +## Guardrails + +- **Cross-harness compatibility is enforced.** A change for one harness must not regress another; + the cross-harness tests in `src/adapters/harness.rs` and the per-harness integration tests under + `tests/run/` are the floor. +- **One enhancement per PR.** Wiring a harness happens one capability at a time. +- **Don't guess harness details.** CLI flags, hook shapes, and event vocabularies come from the + harness's own documentation or observed output — record what you verified in the harness's notes + file. diff --git a/src/adapters/claude_code/plugin_shadow.rs b/src/adapters/claude_code/plugin_shadow.rs index 9286e5d..ca63240 100644 --- a/src/adapters/claude_code/plugin_shadow.rs +++ b/src/adapters/claude_code/plugin_shadow.rs @@ -14,7 +14,7 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -const ISOLATION_DOC: &str = "README.md → Claude Code → \"Isolating from installed plugins\""; +const ISOLATION_DOC: &str = "docs/claude-notes.md → \"Isolating from installed plugins\""; /// A staged skill that is also discoverable from the live environment. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -241,9 +241,23 @@ pub fn format_shadow_banner(report: &PluginShadowReport) -> String { " skill-absent. The runner cannot strip an installed plugin from the dispatch." .to_string(), ); - lines.push(format!( - " Isolate each dispatch's Claude config — see {ISOLATION_DOC}." - )); + lines.push(" Isolate each dispatch's Claude config, one of these ways:".to_string()); + lines.push( + " 1. Drop user-scope plugins, keep auth: add `--setting-sources project,local`" + .to_string(), + ); + lines.push(" to each dispatch.".to_string()); + lines.push( + " 2. Disable the specific plugin: set `\"enabledPlugins\": { \"@\":" + .to_string(), + ); + lines.push(" false }` in a settings source the dispatch loads.".to_string()); + lines.push(" 3. Clean config dir (strips everything): run each dispatch under".to_string()); + lines.push( + " `CLAUDE_CONFIG_DIR=\"$(mktemp -d)\"` — auth may need `ANTHROPIC_API_KEY`." + .to_string(), + ); + lines.push(format!(" Full mechanics: {ISOLATION_DOC}.")); lines.join("\n") } @@ -468,4 +482,38 @@ mod tests { assert!(banner.contains("slow-powers@slowdini")); assert!(banner.to_lowercase().contains("isolat")); } + + #[test] + fn banner_carries_the_three_isolation_recipes_inline() { + // The remediation options live in the banner itself — shown exactly when + // the contamination is detected — not in a README section the operator + // has to go find. + let banner = format_shadow_banner(&sample_report()); + assert!( + banner.contains("--setting-sources project,local"), + "banner names the setting-sources option: {banner}" + ); + assert!( + banner.contains("enabledPlugins"), + "banner names the per-plugin disable option: {banner}" + ); + assert!( + banner.contains("CLAUDE_CONFIG_DIR"), + "banner names the clean-config-dir option: {banner}" + ); + assert!( + banner.contains("docs/claude-notes.md"), + "banner points at the harness dev notes for the full mechanics: {banner}" + ); + } + + #[test] + fn validity_warnings_point_at_a_doc_that_exists() { + let warnings = shadow_validity_warnings(&sample_report()); + assert!( + warnings[0].contains("docs/claude-notes.md"), + "warning points at the harness dev notes: {}", + warnings[0] + ); + } } From cb09d61f1106671655f44e8f1803a18a99f64211 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Mon, 6 Jul 2026 20:44:47 -0400 Subject: [PATCH 09/45] fix(sandbox): cover every harness config dir in guard tamper rule and stray-write lookbehind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard's Bash tamper-protection rule denied create/copy/move verbs only for paths under .claude, so a Codex-guarded run could tamper with its own hook config (mkdir -p .codex, cp evil.json .codex/hooks.json) without the Bash classifier objecting. detect-stray-writes' staging-dir lookbehind likewise recognized only .claude/.agents, misclassifying OpenCode staged copies (.opencode/skills/...) as live-source reads. Both are now built from a new adapters::all_config_dir_names() — the sorted, deduplicated union of HarnessAdapter::config_dir_names() across Harness::ALL — so the deny surface and staging vocabulary grow automatically with any future adapter. Staging's is_harness_config_dir collapses onto the same helper. This deliberately broadens the guard's deny surface cross-harness: a Claude-guarded run now also denies creates under .codex/.agents/.opencode (union semantics; the guard has no harness identity at classify time). Reads of the protected dirs stay allowed, and creates scoped to a run's allowed roots still pass. Closes #130 Co-Authored-By: Claude Fable 5 --- src/adapters/harness.rs | 32 +++++++++++- src/adapters/mod.rs | 2 +- src/cli/run/staging/mod.rs | 7 +-- src/pipeline/detect_stray_writes.rs | 43 +++++++++++++++-- src/sandbox/decide.rs | 75 +++++++++++++++++++++++++++++ src/sandbox/policy.rs | 60 +++++++++++++++++------ 6 files changed, 193 insertions(+), 26 deletions(-) diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 7f47fe6..f30acc0 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -24,6 +24,7 @@ use std::io; use std::path::{Path, PathBuf}; +use std::sync::LazyLock; use std::time::Duration; use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocation}; @@ -64,7 +65,10 @@ pub trait HarnessAdapter { /// The project-local config dir names this harness reads or the adapter /// writes (e.g. `.claude`). Staging excludes every harness's config dirs /// when copying a skill's sibling assets, so a stray checked-in config dir - /// never rides into a staged env. List the parent of + /// never rides into a staged env. Via [`all_config_dir_names`] this list + /// also feeds the guard's Bash tamper rule and detect-stray-writes' + /// staging-dir lookbehind, so adding a dir here automatically grows the + /// write-guard's deny surface. List the parent of /// [`skills_dir`](Self::skills_dir) plus any hook/config dirs the adapter /// writes. fn config_dir_names(&self) -> &'static [&'static str] { @@ -321,10 +325,36 @@ pub fn adapter_for(harness: Harness) -> &'static dyn HarnessAdapter { } } +/// The union of every harness's project-local config dir names (sorted, +/// deduplicated): the dirs harness-agnostic code must treat as protected — +/// staging's sibling-asset filter, the guard's Bash tamper rule, and +/// detect-stray-writes' staging-dir lookbehind. +pub fn all_config_dir_names() -> &'static [&'static str] { + static NAMES: LazyLock> = LazyLock::new(|| { + let mut names: Vec<&'static str> = Harness::ALL + .iter() + .flat_map(|&h| adapter_for(h).config_dir_names()) + .copied() + .collect(); + names.sort_unstable(); + names.dedup(); + names + }); + &NAMES +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn all_config_dir_names_unions_every_adapter() { + assert_eq!( + all_config_dir_names(), + [".agents", ".claude", ".codex", ".opencode"] + ); + } + #[test] fn labels_match_kebab_case_identifiers() { assert_eq!(adapter_for(Harness::ClaudeCode).label(), "claude-code"); diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index aa3c38c..1c92e73 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -16,7 +16,7 @@ pub mod transcript; pub use harness::{ CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, RUNBOOK_TEMPLATE, - adapter_for, + adapter_for, all_config_dir_names, }; pub use transcript::TranscriptSummary; diff --git a/src/cli/run/staging/mod.rs b/src/cli/run/staging/mod.rs index d84a6cc..88b260b 100644 --- a/src/cli/run/staging/mod.rs +++ b/src/cli/run/staging/mod.rs @@ -15,7 +15,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use regex::Regex; use serde::{Deserialize, Serialize}; -use crate::adapters::adapter_for; +use crate::adapters::{adapter_for, all_config_dir_names}; use crate::core::Harness; use crate::pipeline::io::now_iso8601; use crate::workspace::SNAPSHOT_META; @@ -116,10 +116,7 @@ pub(crate) fn skills_dir_for_harness(repo_root: &Path, harness: Harness) -> Path /// skill's sibling assets — regardless of the active harness — so a checked-in /// config dir never rides into a staged env. fn is_harness_config_dir(name: &str) -> bool { - Harness::ALL - .iter() - .flat_map(|&h| adapter_for(h).config_dir_names()) - .any(|&dir| dir == name) + all_config_dir_names().contains(&name) } /// Rewrite (or insert) the `name:` frontmatter field so a Codex-staged skill's diff --git a/src/pipeline/detect_stray_writes.rs b/src/pipeline/detect_stray_writes.rs index a1a5c57..b969d62 100644 --- a/src/pipeline/detect_stray_writes.rs +++ b/src/pipeline/detect_stray_writes.rs @@ -14,6 +14,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use crate::adapters::all_config_dir_names; use crate::core::{ConditionsRecord, RunRecord, ToolInvocation}; use crate::pipeline::error::PipelineError; use crate::pipeline::io::{now_iso8601, write_json}; @@ -142,9 +143,10 @@ fn is_trailing_boundary(b: u8) -> bool { } /// True if `command` references `rel` as a bare path token — bounded as a path -/// segment and **not** prefixed by a `.claude`/`.agents` staging dir. The -/// `regex` crate has no lookbehind, so each occurrence is scanned directly for -/// the boundary + preceding-segment conditions. +/// segment and **not** prefixed by any harness config dir +/// (`adapters::all_config_dir_names`). The `regex` crate has no lookbehind, so +/// each occurrence is scanned directly for the boundary + preceding-segment +/// conditions. fn references_bare_rel(command: &str, rel: &str) -> bool { if rel.is_empty() { return false; @@ -160,7 +162,9 @@ fn references_bare_rel(command: &str, rel: &str) -> bool { // including) that char must not end with a staging-dir prefix. let lookbehind_ok = start == 0 || { let before = &command[..start - 1]; - !before.ends_with(".claude") && !before.ends_with(".agents") + !all_config_dir_names() + .iter() + .any(|dir| before.ends_with(dir)) }; let trailing_ok = end == command.len() || is_trailing_boundary(bytes[end]); @@ -582,7 +586,22 @@ mod tests { repo(), ); assert_eq!(f.warnings.len(), 1); - assert!(f.warnings[0].reason.to_lowercase().contains(".claude")); + assert!(f.warnings[0].reason.to_lowercase().contains("config dir")); + } + + #[test] + fn creating_a_path_under_dot_codex_is_a_warning() { + let f = detect_stray_writes( + &[inv( + "Bash", + json!({"command": "cp evil.json .codex/hooks.json"}), + 0, + )], + OUTPUTS, + repo(), + ); + assert_eq!(f.warnings.len(), 1); + assert!(f.warnings[0].reason.to_lowercase().contains("config dir")); } #[test] @@ -742,6 +761,20 @@ mod tests { assert!(f.is_empty()); } + #[test] + fn a_bash_referencing_a_staged_copy_under_dot_opencode_skills_is_not_flagged() { + let f = detect_live_source_reads( + &[inv( + "Bash", + json!({"command": "cat .opencode/skills/mr-review/SKILL.md"}), + 0, + )], + live(), + repo(), + ); + assert!(f.is_empty()); + } + #[test] fn unrelated_reads_and_commands_are_not_flagged() { let f = detect_live_source_reads( diff --git a/src/sandbox/decide.rs b/src/sandbox/decide.rs index ba417a2..0dab7c2 100644 --- a/src/sandbox/decide.rs +++ b/src/sandbox/decide.rs @@ -362,6 +362,81 @@ mod tests { assert!(d.allow); } + #[test] + fn denies_bash_that_creates_a_path_under_dot_codex_via_non_redirect_verb() { + assert!( + !decide_now( + "Bash", + json!({ "command": "mkdir -p .codex/foo" }), + Some(&marker()) + ) + .allow + ); + assert!( + !decide_now( + "Bash", + json!({ "command": "cp evil.json .codex/hooks.json" }), + Some(&marker()) + ) + .allow + ); + } + + #[test] + fn denies_bash_that_creates_a_path_under_dot_agents_via_non_redirect_verb() { + assert!( + !decide_now( + "Bash", + json!({ "command": "mkdir -p .agents/foo" }), + Some(&marker()) + ) + .allow + ); + } + + #[test] + fn denies_bash_that_creates_a_path_under_dot_opencode_via_non_redirect_verb() { + assert!( + !decide_now( + "Bash", + json!({ "command": "touch .opencode/opencode.json" }), + Some(&marker()) + ) + .allow + ); + } + + #[test] + fn still_allows_reads_of_other_harness_config_dirs_with_no_create_verb() { + for command in [ + "cat .codex/hooks.json", + "ls .agents", + "cat .opencode/skills/x/SKILL.md", + ] { + assert!( + decide_now("Bash", json!({ "command": command }), Some(&marker())).allow, + "{command} should stay allowed" + ); + } + } + + #[test] + fn allows_a_create_scoped_to_a_codex_skills_staging_root() { + let codex_marker = GuardMarker { + allowed_roots: Some(vec![ + "/work/.eval-magic".to_string(), + "/work/.agents/skills".to_string(), + ]), + ..marker() + }; + let d = decide_now( + "Bash", + json!({ "command": "mkdir -p /work/.agents/skills/staged-x" }), + Some(&codex_marker), + ); + assert!(d.allow); + } + #[test] fn does_not_flag_a_skills_prefixed_dir_as_a_bare_skills_write() { // A `skills`-prefixed path that is NOT an allowed root: the bare-`skills/` diff --git a/src/sandbox/policy.rs b/src/sandbox/policy.rs index 02e9d0a..60405c8 100644 --- a/src/sandbox/policy.rs +++ b/src/sandbox/policy.rs @@ -27,43 +27,53 @@ pub fn is_write_tool(tool_name: &str) -> bool { /// Compiled once. The patterns are known-valid, so a compile failure here is a /// programmer error and panics. static BASH_MUTATION_PATTERNS: LazyLock> = LazyLock::new(|| { + let config_dirs = crate::adapters::all_config_dir_names() + .iter() + .map(|d| regex::escape(d)) + .collect::>() + .join("|"); [ ( - r"\b(npm|pnpm|yarn|bun)\s+(install|add|ci|i)\b", + r"\b(npm|pnpm|yarn|bun)\s+(install|add|ci|i)\b".to_string(), "package install/add", ), - (r"\bpip3?\s+install\b", "pip install"), - (r"\bsed\s+-i\b", "in-place file edit (sed -i)"), + (r"\bpip3?\s+install\b".to_string(), "pip install"), + (r"\bsed\s+-i\b".to_string(), "in-place file edit (sed -i)"), ( - r"\bgit\s+(commit|add|push|checkout|reset|restore|merge|rebase)\b", + r"\bgit\s+(commit|add|push|checkout|reset|restore|merge|rebase)\b".to_string(), "git mutation", ), ( - r"\bgit\s+worktree\s+add\b", + r"\bgit\s+worktree\s+add\b".to_string(), "git worktree add (working tree outside the sandbox)", ), - // A create/copy/move/link verb whose operand is a path under `.claude` — - // catches stray writes to the harness config dir that aren't a `>` - // redirect (caught below). Read-only verbs (`cat`, `ls`) aren't listed, - // so inspecting `.claude` stays allowed. + // A create/copy/move/link verb whose operand is a path under any + // harness config dir (`adapters::all_config_dir_names`) — catches + // stray writes to a config dir that aren't a `>` redirect (caught + // below). Read-only verbs (`cat`, `ls`) aren't listed, so inspecting + // the dirs stays allowed. ( - r"\b(cp|mv|mkdir|touch|ln|rsync|install)\b[^|;&\n]*\.claude(/|\b)", - "path under .claude", + format!(r"\b(cp|mv|mkdir|touch|ln|rsync|install)\b[^|;&\n]*({config_dirs})(/|\b)"), + "path under a harness config dir", ), // The same create verbs whose operand is a top-level `skills/` directory — // catches a bare `skills/` left in the cwd. `skills-data` and other // `skills`-prefixed names are excluded by the trailing `/`, whitespace, or // end-of-string boundary. ( - r#"\b(cp|mv|mkdir|touch|ln|rsync)\b[^|;&\n]*[\s'"=/]\.{0,2}/?skills(/|\s|$)"#, + r#"\b(cp|mv|mkdir|touch|ln|rsync)\b[^|;&\n]*[\s'"=/]\.{0,2}/?skills(/|\s|$)"# + .to_string(), "creates a bare skills/ dir", ), - (r"(^|\s)(>>?|tee)\s", "output redirection to a file"), + ( + r"(^|\s)(>>?|tee)\s".to_string(), + "output redirection to a file", + ), ] .into_iter() .map(|(re, reason)| { ( - Regex::new(re) + Regex::new(&re) .unwrap_or_else(|e| panic!("bundled bash pattern {re:?} is invalid: {e}")), reason, ) @@ -283,6 +293,28 @@ mod tests { ); } + #[test] + fn classify_bash_flags_creates_under_every_harness_config_dir_but_allows_reads() { + for dir in crate::adapters::all_config_dir_names() { + assert_eq!( + classify_bash(&format!("mkdir -p {dir}/x"), &[]), + Some("path under a harness config dir"), + "mkdir under {dir} should be flagged" + ); + assert_eq!( + classify_bash(&format!("cp evil.json {dir}/hooks.json"), &[]), + Some("path under a harness config dir"), + "cp into {dir} should be flagged" + ); + assert_eq!( + classify_bash(&format!("cat {dir}/settings.json"), &[]), + None, + "read of {dir} should stay allowed" + ); + assert_eq!(classify_bash(&format!("ls {dir}"), &[]), None); + } + } + #[test] fn classify_bash_allows_scoped_and_readonly_commands() { // Textually references an allowed root → scoped → allowed. From 932dbeb23a0a2ece88ad49cd1c4cc7d00887f60f Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Wed, 8 Jul 2026 18:47:59 -0400 Subject: [PATCH 10/45] refactor(adapters): de-static the HarnessAdapter trait surface Change the seven declarative getters on HarnessAdapter (label, config_dir_names, skill_surface_phrase, skill_unresolved_phrase, cli_events_filename, cli_model_flag, guard_armed_message) from &'static returns to owned types (String, Vec, Option), so descriptor-backed adapters built from runtime-loaded config can implement the trait without leaking. Prep for the descriptor registry (#122 phase 2); no behavior change, adapter_for() keeps its shape. all_config_dir_names() drops its LazyLock cache and computes the sorted, deduplicated union per call; detect_live_source_reads hoists the list out of the per-occurrence lookbehind loop. Closes #132 Co-Authored-By: Claude Fable 5 --- src/adapters/claude_code/mod.rs | 44 ++++++++++++++++------------- src/adapters/codex/mod.rs | 41 +++++++++++++++------------ src/adapters/harness.rs | 43 +++++++++++++--------------- src/adapters/opencode/mod.rs | 16 +++++------ src/cli/run/runbook.rs | 2 +- src/cli/run/staging/mod.rs | 2 +- src/cli/run/util.rs | 2 +- src/pipeline/detect_stray_writes.rs | 17 ++++++----- src/pipeline/record_runs.rs | 2 +- 9 files changed, 86 insertions(+), 83 deletions(-) diff --git a/src/adapters/claude_code/mod.rs b/src/adapters/claude_code/mod.rs index 11efa9f..8a7d48f 100644 --- a/src/adapters/claude_code/mod.rs +++ b/src/adapters/claude_code/mod.rs @@ -30,8 +30,8 @@ use stream_json::{parse_claude_stream_json, parse_claude_stream_json_full}; pub struct ClaudeCodeAdapter; impl HarnessAdapter for ClaudeCodeAdapter { - fn label(&self) -> &'static str { - "claude-code" + fn label(&self) -> String { + "claude-code".to_string() } fn skills_dir(&self, repo_root: &Path) -> PathBuf { repo_root.join(".claude").join("skills") @@ -43,28 +43,28 @@ impl HarnessAdapter for ClaudeCodeAdapter { supports_stage_name_with_no_stage: true, } } - fn config_dir_names(&self) -> &'static [&'static str] { - &[".claude"] + fn config_dir_names(&self) -> Vec { + vec![".claude".to_string()] } fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { render_available_skills_block(skills) } - fn skill_surface_phrase(&self) -> &'static str { - "via the Skill tool" + fn skill_surface_phrase(&self) -> String { + "via the Skill tool".to_string() } - fn skill_unresolved_phrase(&self) -> &'static str { - "If the Skill tool cannot resolve that identifier" + fn skill_unresolved_phrase(&self) -> String { + "If the Skill tool cannot resolve that identifier".to_string() } - fn cli_events_filename(&self) -> Option<&'static str> { - Some("claude-events.jsonl") + fn cli_events_filename(&self) -> Option { + Some("claude-events.jsonl".to_string()) } - fn cli_model_flag(&self) -> Option<&'static str> { - Some("--model") + fn cli_model_flag(&self) -> Option { + Some("--model".to_string()) } fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { format!( "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness claude-code`.", - claude_exec_command_template(self.cli_model_flag(), ctx.agent_model), + claude_exec_command_template(self.cli_model_flag().as_deref(), ctx.agent_model), target_args = ctx.target_args, iteration = ctx.iteration ) @@ -76,13 +76,13 @@ impl HarnessAdapter for ClaudeCodeAdapter { "Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `) -> Option { Some(claude_judge_dispatch_recipe( - self.cli_model_flag(), + self.cli_model_flag().as_deref(), ctx.iteration_dir, )) } @@ -109,9 +109,10 @@ impl HarnessAdapter for ClaudeCodeAdapter { ) -> io::Result { guard::install_guard(stage_root, guard_exe, ttl) } - fn guard_armed_message(&self) -> Option<&'static str> { + fn guard_armed_message(&self) -> Option { Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches.\n Each `claude -p` dispatch loads the hook from the env cwd it runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", + "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches.\n Each `claude -p` dispatch loads the hook from the env cwd it runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard" + .to_string(), ) } } @@ -124,8 +125,11 @@ mod tests { #[test] fn claude_adapter_advertises_cli_events_file_and_model_flag() { let a = adapter_for(Harness::ClaudeCode); - assert_eq!(a.cli_events_filename(), Some("claude-events.jsonl")); - assert_eq!(a.cli_model_flag(), Some("--model")); + assert_eq!( + a.cli_events_filename().as_deref(), + Some("claude-events.jsonl") + ); + assert_eq!(a.cli_model_flag().as_deref(), Some("--model")); } #[test] diff --git a/src/adapters/codex/mod.rs b/src/adapters/codex/mod.rs index 574ed78..e178394 100644 --- a/src/adapters/codex/mod.rs +++ b/src/adapters/codex/mod.rs @@ -27,8 +27,8 @@ use transcript::{parse_codex_events, parse_codex_events_full}; pub struct CodexAdapter; impl HarnessAdapter for CodexAdapter { - fn label(&self) -> &'static str { - "codex" + fn label(&self) -> String { + "codex".to_string() } fn skills_dir(&self, repo_root: &Path) -> PathBuf { repo_root.join(".agents").join("skills") @@ -40,8 +40,8 @@ impl HarnessAdapter for CodexAdapter { supports_stage_name_with_no_stage: false, } } - fn config_dir_names(&self) -> &'static [&'static str] { - &[".agents", ".codex"] + fn config_dir_names(&self) -> Vec { + vec![".agents".to_string(), ".codex".to_string()] } fn rewrites_frontmatter_name(&self) -> bool { true @@ -52,22 +52,26 @@ impl HarnessAdapter for CodexAdapter { fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { render_codex_available_skills_block(skills) } - fn skill_surface_phrase(&self) -> &'static str { - "as a Codex skill" + fn skill_surface_phrase(&self) -> String { + "as a Codex skill".to_string() } - fn skill_unresolved_phrase(&self) -> &'static str { - "If it does not load as a Codex skill" + fn skill_unresolved_phrase(&self) -> String { + "If it does not load as a Codex skill".to_string() } - fn cli_events_filename(&self) -> Option<&'static str> { - Some("codex-events.jsonl") + fn cli_events_filename(&self) -> Option { + Some("codex-events.jsonl".to_string()) } - fn cli_model_flag(&self) -> Option<&'static str> { - Some("-m") + fn cli_model_flag(&self) -> Option { + Some("-m".to_string()) } fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { format!( "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness codex`.", - codex_exec_command_template(self.cli_model_flag(), ctx.guard, ctx.agent_model), + codex_exec_command_template( + self.cli_model_flag().as_deref(), + ctx.guard, + ctx.agent_model + ), target_args = ctx.target_args, iteration = ctx.iteration ) @@ -79,13 +83,13 @@ impl HarnessAdapter for CodexAdapter { "Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `) -> Option { Some(codex_judge_dispatch_recipe( - self.cli_model_flag(), + self.cli_model_flag().as_deref(), ctx.guard, ctx.iteration_dir, )) @@ -121,9 +125,10 @@ impl HarnessAdapter for CodexAdapter { fn guard_hook_cleanup_dir(&self, stage_root: &Path) -> Option { Some(guard::hook_cleanup_dir(stage_root)) } - fn guard_armed_message(&self) -> Option<&'static str> { + fn guard_armed_message(&self) -> Option { Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .codex/hooks.json\n and will block writes/installs outside the eval sandbox during Codex dispatches.\n Dispatch with codex --ask-for-approval never exec --dangerously-bypass-hook-trust so the vetted eval hook runs.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard", + "\n🛡 Write guard armed: a PreToolUse hook is staged in .codex/hooks.json\n and will block writes/installs outside the eval sandbox during Codex dispatches.\n Dispatch with codex --ask-for-approval never exec --dangerously-bypass-hook-trust so the vetted eval hook runs.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard" + .to_string(), ) } } diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index f30acc0..56c02b6 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -24,7 +24,6 @@ use std::io; use std::path::{Path, PathBuf}; -use std::sync::LazyLock; use std::time::Duration; use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocation}; @@ -39,7 +38,7 @@ pub trait HarnessAdapter { /// **Baseline.** The kebab-case identifier used in CLI flags, /// `dispatch.json`, and the staged `conditions.json`. - fn label(&self) -> &'static str; + fn label(&self) -> String; /// **Baseline.** The project-local directory staged skills live under for /// this harness. Under `--no-stage` nothing is staged into it, so a @@ -71,8 +70,8 @@ pub trait HarnessAdapter { /// write-guard's deny surface. List the parent of /// [`skills_dir`](Self::skills_dir) plus any hook/config dirs the adapter /// writes. - fn config_dir_names(&self) -> &'static [&'static str] { - &[] + fn config_dir_names(&self) -> Vec { + Vec::new() } // ── Enhancement: native skill staging (defaulted) ──────────────────────── @@ -138,15 +137,15 @@ pub trait HarnessAdapter { /// **Enhancement: native staging.** How a staged skill is described as /// discoverable in the neutral slug-disambiguation line (e.g. "via the /// Skill tool"). - fn skill_surface_phrase(&self) -> &'static str { - "as a discoverable skill" + fn skill_surface_phrase(&self) -> String { + "as a discoverable skill".to_string() } /// **Enhancement: native staging.** The lead-in for the fallback "read the /// skill from ``" instruction when the staged identifier can't be /// resolved. - fn skill_unresolved_phrase(&self) -> &'static str { - "If the staged skill cannot be resolved" + fn skill_unresolved_phrase(&self) -> String { + "If the staged skill cannot be resolved".to_string() } // ── Enhancement: transcript parser (defaulted) ─────────────────────────── @@ -159,7 +158,7 @@ pub trait HarnessAdapter { /// `outputs/` dir) this harness's one-shot CLI writes the captured /// transcript to. `None` when no transcript ingest is wired — the ingest /// pipeline then never calls the parsers below. - fn cli_events_filename(&self) -> Option<&'static str> { + fn cli_events_filename(&self) -> Option { None } @@ -204,7 +203,7 @@ pub trait HarnessAdapter { /// **Enhancement: model flag.** The native model-selection flag accepted /// by this harness's CLI. `None` means no model-selection support is /// wired. - fn cli_model_flag(&self) -> Option<&'static str> { + fn cli_model_flag(&self) -> Option { None } @@ -234,7 +233,7 @@ pub trait HarnessAdapter { /// to remove it. `None` for a harness with no write guard (its /// [`install_guard`](Self::install_guard) errors), in which case no banner /// is printed. - fn guard_armed_message(&self) -> Option<&'static str> { + fn guard_armed_message(&self) -> Option { None } @@ -329,18 +328,14 @@ pub fn adapter_for(harness: Harness) -> &'static dyn HarnessAdapter { /// deduplicated): the dirs harness-agnostic code must treat as protected — /// staging's sibling-asset filter, the guard's Bash tamper rule, and /// detect-stray-writes' staging-dir lookbehind. -pub fn all_config_dir_names() -> &'static [&'static str] { - static NAMES: LazyLock> = LazyLock::new(|| { - let mut names: Vec<&'static str> = Harness::ALL - .iter() - .flat_map(|&h| adapter_for(h).config_dir_names()) - .copied() - .collect(); - names.sort_unstable(); - names.dedup(); - names - }); - &NAMES +pub fn all_config_dir_names() -> Vec { + let mut names: Vec = Harness::ALL + .iter() + .flat_map(|&h| adapter_for(h).config_dir_names()) + .collect(); + names.sort_unstable(); + names.dedup(); + names } #[cfg(test)] @@ -496,7 +491,7 @@ mod tests { .to_string_lossy() .into_owned(); assert!( - a.config_dir_names().contains(&top.as_str()), + a.config_dir_names().contains(&top), "{} config_dir_names {:?} misses {top}", a.label(), a.config_dir_names() diff --git a/src/adapters/opencode/mod.rs b/src/adapters/opencode/mod.rs index e8b2423..a97f243 100644 --- a/src/adapters/opencode/mod.rs +++ b/src/adapters/opencode/mod.rs @@ -17,14 +17,14 @@ use session::render_opencode_available_skills_block; pub struct OpenCodeAdapter; impl HarnessAdapter for OpenCodeAdapter { - fn label(&self) -> &'static str { - "opencode" + fn label(&self) -> String { + "opencode".to_string() } fn skills_dir(&self, repo_root: &Path) -> PathBuf { repo_root.join(".opencode").join("skills") } - fn config_dir_names(&self) -> &'static [&'static str] { - &[".opencode"] + fn config_dir_names(&self) -> Vec { + vec![".opencode".to_string()] } // OpenCode's constrained naming rules need a sanitized slug; the generated // slug and any --stage-name override are validated against the same rules. @@ -52,11 +52,11 @@ impl HarnessAdapter for OpenCodeAdapter { fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { render_opencode_available_skills_block(skills) } - fn skill_surface_phrase(&self) -> &'static str { - "as an OpenCode skill" + fn skill_surface_phrase(&self) -> String { + "as an OpenCode skill".to_string() } - fn skill_unresolved_phrase(&self) -> &'static str { - "If it does not load as an OpenCode skill" + fn skill_unresolved_phrase(&self) -> String { + "If it does not load as an OpenCode skill".to_string() } fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { let model_note = if ctx.agent_model.is_some() { diff --git a/src/cli/run/runbook.rs b/src/cli/run/runbook.rs index aaa35e9..d8b6c6e 100644 --- a/src/cli/run/runbook.rs +++ b/src/cli/run/runbook.rs @@ -97,7 +97,7 @@ pub(crate) fn build_runbook(ctx: &RunbookContext) -> String { ctx.target_args, ctx.iteration ); let teardown_cmd = format!("eval-magic teardown{} --harness {label}", ctx.target_args); - vars.push(("HARNESS", label)); + vars.push(("HARNESS", &label)); vars.push(("DISPATCH_RECIPE", &dispatch_recipe)); vars.push(("JUDGE_RECIPE", &judge_recipe)); vars.push(("FINALIZE_CMD", &finalize_cmd)); diff --git a/src/cli/run/staging/mod.rs b/src/cli/run/staging/mod.rs index 88b260b..da950f0 100644 --- a/src/cli/run/staging/mod.rs +++ b/src/cli/run/staging/mod.rs @@ -116,7 +116,7 @@ pub(crate) fn skills_dir_for_harness(repo_root: &Path, harness: Harness) -> Path /// skill's sibling assets — regardless of the active harness — so a checked-in /// config dir never rides into a staged env. fn is_harness_config_dir(name: &str) -> bool { - all_config_dir_names().contains(&name) + all_config_dir_names().iter().any(|d| d == name) } /// Rewrite (or insert) the `name:` frontmatter field so a Codex-staged skill's diff --git a/src/cli/run/util.rs b/src/cli/run/util.rs index e675e79..6a3cd0f 100644 --- a/src/cli/run/util.rs +++ b/src/cli/run/util.rs @@ -130,7 +130,7 @@ pub(crate) fn mode_str(mode: Mode) -> &'static str { } } -pub(crate) fn harness_label(harness: Harness) -> &'static str { +pub(crate) fn harness_label(harness: Harness) -> String { adapter_for(harness).label() } diff --git a/src/pipeline/detect_stray_writes.rs b/src/pipeline/detect_stray_writes.rs index b969d62..90880d4 100644 --- a/src/pipeline/detect_stray_writes.rs +++ b/src/pipeline/detect_stray_writes.rs @@ -143,11 +143,11 @@ fn is_trailing_boundary(b: u8) -> bool { } /// True if `command` references `rel` as a bare path token — bounded as a path -/// segment and **not** prefixed by any harness config dir -/// (`adapters::all_config_dir_names`). The `regex` crate has no lookbehind, so -/// each occurrence is scanned directly for the boundary + preceding-segment -/// conditions. -fn references_bare_rel(command: &str, rel: &str) -> bool { +/// segment and **not** prefixed by any harness config dir (`config_dirs`, the +/// caller-supplied `adapters::all_config_dir_names()` list). The `regex` crate +/// has no lookbehind, so each occurrence is scanned directly for the boundary + +/// preceding-segment conditions. +fn references_bare_rel(command: &str, rel: &str, config_dirs: &[String]) -> bool { if rel.is_empty() { return false; } @@ -162,9 +162,7 @@ fn references_bare_rel(command: &str, rel: &str) -> bool { // including) that char must not end with a staging-dir prefix. let lookbehind_ok = start == 0 || { let before = &command[..start - 1]; - !all_config_dir_names() - .iter() - .any(|dir| before.ends_with(dir)) + !config_dirs.iter().any(|dir| before.ends_with(dir.as_str())) }; let trailing_ok = end == command.len() || is_trailing_boundary(bytes[end]); @@ -189,6 +187,7 @@ pub fn detect_live_source_reads( let live_dir_str = live_dir.to_string_lossy(); let rel = path_relative(repo_root, &live_dir); let rel_usable = !rel.starts_with(".."); + let config_dirs = all_config_dir_names(); for inv in invocations { if READ_TOOLS.contains(&inv.name.as_str()) { @@ -209,7 +208,7 @@ pub fn detect_live_source_reads( if SHELL_TOOLS.contains(&inv.name.as_str()) { let command = command_of(inv); if command.contains(live_dir_str.as_ref()) - || (rel_usable && references_bare_rel(command, &rel)) + || (rel_usable && references_bare_rel(command, &rel, &config_dirs)) { findings.push(StrayFinding { tool: inv.name.clone(), diff --git a/src/pipeline/record_runs.rs b/src/pipeline/record_runs.rs index f1b0661..36fe63c 100644 --- a/src/pipeline/record_runs.rs +++ b/src/pipeline/record_runs.rs @@ -78,7 +78,7 @@ impl RecordRunsResult { }; let file = adapter_for(harness) .cli_events_filename() - .unwrap_or("the events file"); + .unwrap_or_else(|| "the events file".to_string()); let cause = format!("expected `outputs/{file}` was not found"); Some(format!( "{lead} — {cause}; tool_invocations/tokens/duration are empty, so transcript_check \ From 4836b0a0828c723d43e7e7c81909ef982f79a359 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Thu, 9 Jul 2026 22:41:45 -0400 Subject: [PATCH 11/45] refactor(adapters): extract shared JSONL reader The read-lines-and-skip-malformed loop was copy-pasted three times (claude_code/transcript.rs, codex/transcript.rs, and inlined in stream_json.rs). Replace all three with one read_jsonl helper in the shared adapters::transcript module. The stream-json full parse reads each line as a Value once and shapes records from it, so the terminal result event no longer needs a second from_str on the raw line. Behavior-identical: blank/whitespace-only and malformed lines are skipped exactly as before (whitespace-only lines previously failed the parse and were dropped; now they're filtered explicitly). Part of #133. Co-Authored-By: Claude Fable 5 --- src/adapters/claude_code/stream_json.rs | 24 ++++++------ src/adapters/claude_code/transcript.rs | 28 ++++---------- src/adapters/codex/transcript.rs | 21 ++--------- src/adapters/transcript.rs | 50 +++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 52 deletions(-) diff --git a/src/adapters/claude_code/stream_json.rs b/src/adapters/claude_code/stream_json.rs index 4615f40..2bbe13d 100644 --- a/src/adapters/claude_code/stream_json.rs +++ b/src/adapters/claude_code/stream_json.rs @@ -11,17 +11,18 @@ //! any other non-message events are ignored (they don't deserialize into an //! assistant/user record, so the shared extractor skips them). -use std::fs; use std::io; use std::path::Path; +use serde::Deserialize; +use serde_json::Value; + use crate::core::ToolInvocation; use crate::adapters::TranscriptSummary; +use crate::adapters::transcript::read_jsonl; -use super::transcript::{ - TranscriptRecord, UsageRecord, extract_invocations, last_assistant_text, read_records, -}; +use super::transcript::{TranscriptRecord, UsageRecord, extract_invocations, last_assistant_text}; /// The terminal `{"type":"result", …}` event of a `-p` stream-json run. #[derive(Debug, serde::Deserialize)] @@ -39,7 +40,7 @@ struct ResultEvent { /// Parse the event stream into ordered tool invocations. Reuses the shared /// extractor: non-message events deserialize into records the extractor skips. pub fn parse_claude_stream_json(path: &Path) -> io::Result> { - Ok(extract_invocations(&read_records(path)?)) + Ok(extract_invocations(&read_jsonl::(path)?)) } /// Parse the event stream into a full [`TranscriptSummary`]. Final text, @@ -47,19 +48,16 @@ pub fn parse_claude_stream_json(path: &Path) -> io::Result> /// missing or errored `result` the final text falls back to the last assistant /// message's text, and duration/tokens fall back to `None`. pub fn parse_claude_stream_json_full(path: &Path) -> io::Result { - let raw = fs::read_to_string(path)?; + let values = read_jsonl::(path)?; let mut records: Vec = Vec::new(); let mut result_event: Option = None; - for line in raw.split('\n') { - if line.trim().is_empty() { - continue; - } - // Skip malformed lines rather than failing the whole parse. - let Ok(record) = serde_json::from_str::(line) else { + for value in &values { + // Skip lines that don't shape into records rather than failing the parse. + let Ok(record) = TranscriptRecord::deserialize(value) else { continue; }; if record.record_type.as_deref() == Some("result") { - result_event = serde_json::from_str::(line).ok(); + result_event = ResultEvent::deserialize(value).ok(); } records.push(record); } diff --git a/src/adapters/claude_code/transcript.rs b/src/adapters/claude_code/transcript.rs index 0413349..b77a764 100644 --- a/src/adapters/claude_code/transcript.rs +++ b/src/adapters/claude_code/transcript.rs @@ -11,9 +11,6 @@ use crate::core::ToolInvocation; use serde::Deserialize; use serde_json::Value; use std::collections::HashMap; -use std::fs; -use std::io; -use std::path::Path; #[derive(Debug, Deserialize)] pub(crate) struct UsageRecord { @@ -80,21 +77,6 @@ fn stringify_result(content: Option<&Value>) -> String { } } -pub(crate) fn read_records(jsonl_path: &Path) -> io::Result> { - let raw = fs::read_to_string(jsonl_path)?; - let mut records = Vec::new(); - for line in raw.split('\n') { - if line.is_empty() { - continue; - } - // Skip malformed lines rather than failing the whole parse. - if let Ok(rec) = serde_json::from_str::(line) { - records.push(rec); - } - } - Ok(records) -} - pub(crate) fn extract_invocations(records: &[TranscriptRecord]) -> Vec { let mut invocations: Vec = Vec::new(); let mut index_by_id: HashMap = HashMap::new(); @@ -167,6 +149,7 @@ pub(crate) fn last_assistant_text(records: &[TranscriptRecord]) -> Option Vec { - extract_invocations(&read_records(path).unwrap()) + extract_invocations(&read_jsonl::(path).unwrap()) } #[test] @@ -295,7 +278,7 @@ mod tests { ], ); assert_eq!( - last_assistant_text(&read_records(&path).unwrap()), + last_assistant_text(&read_jsonl::(&path).unwrap()), Some("All tests pass.\nWrapping up.".into()) ); } @@ -308,6 +291,9 @@ mod tests { &path, &[json!({"type": "user", "message": {"role": "user", "content": "hi"}})], ); - assert_eq!(last_assistant_text(&read_records(&path).unwrap()), None); + assert_eq!( + last_assistant_text(&read_jsonl::(&path).unwrap()), + None + ); } } diff --git a/src/adapters/codex/transcript.rs b/src/adapters/codex/transcript.rs index 1150dbc..dc64ca4 100644 --- a/src/adapters/codex/transcript.rs +++ b/src/adapters/codex/transcript.rs @@ -7,30 +7,15 @@ //! accounting (excludes cached input tokens). use crate::adapters::TranscriptSummary; +use crate::adapters::transcript::read_jsonl; use crate::core::ToolInvocation; use serde_json::{Map, Value}; -use std::fs; use std::io; use std::path::Path; const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "plan_update"]; const ARG_OMIT_KEYS: [&str; 6] = ["id", "type", "status", "output", "result", "error"]; -fn read_events(jsonl_path: &Path) -> io::Result> { - let raw = fs::read_to_string(jsonl_path)?; - let mut out = Vec::new(); - for line in raw.split('\n') { - if line.is_empty() { - continue; - } - // Skip malformed lines rather than failing the whole parse. - if let Ok(v) = serde_json::from_str::(line) { - out.push(v); - } - } - Ok(out) -} - fn stringify_value(v: &Value) -> String { match v { Value::String(s) => s.clone(), @@ -91,7 +76,7 @@ fn extract_invocations(records: &[Value]) -> Vec { /// Parse a Codex event stream into ordered tool invocations. pub fn parse_codex_events(jsonl_path: &Path) -> io::Result> { - Ok(extract_invocations(&read_events(jsonl_path)?)) + Ok(extract_invocations(&read_jsonl::(jsonl_path)?)) } fn parse_millis(s: &str) -> Option { @@ -102,7 +87,7 @@ fn parse_millis(s: &str) -> Option { /// Parse a Codex event stream into a full [`TranscriptSummary`]. pub fn parse_codex_events_full(jsonl_path: &Path) -> io::Result { - let records = read_events(jsonl_path)?; + let records = read_jsonl::(jsonl_path)?; let mut first_ts: Option = None; let mut last_ts: Option = None; diff --git a/src/adapters/transcript.rs b/src/adapters/transcript.rs index c31b78b..e359d2b 100644 --- a/src/adapters/transcript.rs +++ b/src/adapters/transcript.rs @@ -5,9 +5,24 @@ //! harness's raw record types. use serde::{Deserialize, Serialize}; +use std::fs; +use std::io; +use std::path::Path; use crate::core::ToolInvocation; +/// Read a JSONL file, deserializing each non-blank line as `T` and silently +/// skipping malformed lines (a partial transcript still yields its parseable +/// records). +pub(crate) fn read_jsonl(path: &Path) -> io::Result> { + let raw = fs::read_to_string(path)?; + Ok(raw + .split('\n') + .filter(|line| !line.trim().is_empty()) + .filter_map(|line| serde_json::from_str::(line).ok()) + .collect()) +} + /// A transcript boiled down to the artifacts the pipeline needs. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct TranscriptSummary { @@ -20,3 +35,38 @@ pub struct TranscriptSummary { /// Concatenated text blocks of the last assistant message. pub final_text: Option, } + +#[cfg(test)] +mod tests { + use super::read_jsonl; + use serde_json::Value; + use std::fs; + use tempfile::TempDir; + + #[test] + fn read_jsonl_skips_malformed_lines() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("events.jsonl"); + fs::write(&path, "{\"a\":1}\nnot json\n{\"a\":2}\n").unwrap(); + let values: Vec = read_jsonl(&path).unwrap(); + assert_eq!(values.len(), 2); + assert_eq!(values[0]["a"], 1); + assert_eq!(values[1]["a"], 2); + } + + #[test] + fn read_jsonl_skips_blank_and_whitespace_only_lines() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("events.jsonl"); + fs::write(&path, "{\"a\":1}\n\n \n\t\n{\"a\":2}\n").unwrap(); + let values: Vec = read_jsonl(&path).unwrap(); + assert_eq!(values.len(), 2); + } + + #[test] + fn read_jsonl_errors_on_a_missing_file() { + let dir = TempDir::new().unwrap(); + let missing = dir.path().join("absent.jsonl"); + assert!(read_jsonl::(&missing).is_err()); + } +} From f225fa8db71ce1b02c2ade0393b138956f0ea13f Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Thu, 9 Jul 2026 22:48:12 -0400 Subject: [PATCH 12/45] refactor(adapters): fold plugin-shadow preflight behind the trait The `if ctx.harness == Harness::ClaudeCode` preflight branch in orchestrate/build.rs was the last variant-specific behavior branch outside adapter_for() and the guard CLI entry points. Replace it with a defaulted HarnessAdapter::detect_shadowed_skills enhancement method (None by default; Claude Code overrides via a thin env-reading wrapper over the new shadow_preflight fn, which owns the report-or-silence mapping and is unit-tested with an explicit config dir). The harness-neutral report types and formatters (PluginShadowReport, ShadowSource, format_shadow_banner, shadow_validity_warnings) move to a shared adapters::skill_shadow module; detection stays Claude-specific in claude_code/plugin_shadow.rs. detect_plugin_shadows/config_dir_from_env/ resolve_config_dir lose their crate-root re-export (no consumer outside claude_code/ remains). Ride-along: the staging Default impls use Harness::default() instead of naming ClaudeCode. Documents the new enhancement in docs/progressive-enhancements.md. Part of #133. Co-Authored-By: Claude Fable 5 --- docs/progressive-enhancements.md | 16 ++ src/adapters/claude_code/mod.rs | 12 ++ src/adapters/claude_code/plugin_shadow.rs | 192 +++------------------- src/adapters/harness.rs | 29 ++++ src/adapters/mod.rs | 9 +- src/adapters/skill_shadow.rs | 191 +++++++++++++++++++++ src/cli/run/orchestrate/build.rs | 44 +++-- src/cli/run/staging/mod.rs | 4 +- 8 files changed, 299 insertions(+), 198 deletions(-) create mode 100644 src/adapters/skill_shadow.rs diff --git a/docs/progressive-enhancements.md b/docs/progressive-enhancements.md index 960f4f4..2137819 100644 --- a/docs/progressive-enhancements.md +++ b/docs/progressive-enhancements.md @@ -113,6 +113,22 @@ arm; fatal in revision mode, where the `old_skill` arm then sees new-skill conte hidden `guard` / `guard-codex` subcommands are the hook entry points — their names are a stable on-disk contract. Shared marker/manifest/teardown machinery lives in `src/sandbox/`. +### Shadow preflight + +*Why harness-specific:* what "discoverable from the live environment" means is harness-native — +Claude Code dispatches load the operator's enabled plugins and global skills dir, so a staged +skill name colliding with one of those contaminates the with/without comparison. Other harnesses +load nothing global today. + +*What it unlocks:* a build-time contamination warning (banner + `plugin-shadow.json` in the +iteration dir), which `aggregate` folds into `benchmark.json` validity warnings. + +*Fallback:* no preflight — the run proceeds with no shadow report, exactly right for a harness +whose dispatches load nothing beyond the staged env. + +*Trait methods:* `detect_shadowed_skills` (returns the harness-neutral `PluginShadowReport` from +`src/adapters/skill_shadow.rs`; detection itself stays in the harness's module tree). + ### Plan-mode context *Why harness-specific in principle:* a harness could inject a real native plan mode. diff --git a/src/adapters/claude_code/mod.rs b/src/adapters/claude_code/mod.rs index 8a7d48f..3909cdf 100644 --- a/src/adapters/claude_code/mod.rs +++ b/src/adapters/claude_code/mod.rs @@ -21,6 +21,7 @@ use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; +use super::skill_shadow::PluginShadowReport; use cli::{ claude_exec_command_template, claude_judge_dispatch_recipe, claude_parallel_dispatch_recipe, }; @@ -95,6 +96,17 @@ impl HarnessAdapter for ClaudeCodeAdapter { ctx.iteration_dir, )) } + fn detect_shadowed_skills( + &self, + scan_root: &Path, + staged_skill_names: &[&str], + ) -> Option { + plugin_shadow::shadow_preflight( + &plugin_shadow::config_dir_from_env(), + scan_root, + staged_skill_names, + ) + } fn parse_cli_events(&self, path: &Path) -> io::Result> { parse_claude_stream_json(path) } diff --git a/src/adapters/claude_code/plugin_shadow.rs b/src/adapters/claude_code/plugin_shadow.rs index ca63240..ea410bc 100644 --- a/src/adapters/claude_code/plugin_shadow.rs +++ b/src/adapters/claude_code/plugin_shadow.rs @@ -6,53 +6,16 @@ //! name collides with one of those, both copies are discoverable and the //! with/without comparison is contaminated. The runner cannot strip an installed //! plugin from a dispatch, so this module only *detects and reports* the overlap, -//! reading declared settings as a best-effort proxy. +//! reading declared settings as a best-effort proxy. The report/banner shapes are +//! harness-neutral and live in [`skill_shadow`](crate::adapters::skill_shadow). +use serde::Deserialize; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -const ISOLATION_DOC: &str = "docs/claude-notes.md → \"Isolating from installed plugins\""; - -/// A staged skill that is also discoverable from the live environment. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(tag = "kind", rename_all = "kebab-case")] -pub enum ShadowSource { - Plugin { - plugin: String, - skill_name: String, - path: String, - }, - GlobalSkill { - skill_name: String, - path: String, - }, -} - -impl ShadowSource { - fn skill_name(&self) -> &str { - match self { - ShadowSource::Plugin { skill_name, .. } => skill_name, - ShadowSource::GlobalSkill { skill_name, .. } => skill_name, - } - } - - fn source_label(&self) -> String { - match self { - ShadowSource::Plugin { plugin, .. } => format!("enabled plugin '{plugin}'"), - ShadowSource::GlobalSkill { .. } => "the global skills dir".to_string(), - } - } -} - -/// The detector's findings for a run. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct PluginShadowReport { - pub config_dir: String, - pub shadowed: Vec, -} +use crate::adapters::skill_shadow::{PluginShadowReport, ShadowSource}; /// The Claude Code config dir: a non-empty `CLAUDE_CONFIG_DIR` override (passed /// in), else `~/.claude`. @@ -198,67 +161,16 @@ pub fn detect_plugin_shadows( } } -/// One `validity_warnings` line per shadowed skill (for benchmark.json). -pub fn shadow_validity_warnings(report: &PluginShadowReport) -> Vec { - report - .shadowed - .iter() - .map(|s| { - format!( - "staged skill '{}' is also provided by {} — each claude -p dispatch could discover \ - both copies, so with/without results may be contaminated. Isolate each dispatch's \ - Claude config (see {}).", - s.skill_name(), - s.source_label(), - ISOLATION_DOC - ) - }) - .collect() -} - -/// Build-time banner for the runner. Empty string when nothing is shadowed. -pub fn format_shadow_banner(report: &PluginShadowReport) -> String { - if report.shadowed.is_empty() { - return String::new(); - } - let mut lines = vec![ - String::new(), - "⚠ Plugin-shadow warning: skills staged for this eval are ALSO discoverable".to_string(), - " from your live environment:".to_string(), - ]; - for s in &report.shadowed { - lines.push(format!(" • {} — {}", s.skill_name(), s.source_label())); - } - lines.push( - " Each `claude -p` dispatch loads your user/global plugins and skills, so".to_string(), - ); - lines.push(" both the staged copy and the installed copy are discoverable — the".to_string()); - lines.push( - " with/without comparison may be contaminated and the control arm is not truly" - .to_string(), - ); - lines.push( - " skill-absent. The runner cannot strip an installed plugin from the dispatch." - .to_string(), - ); - lines.push(" Isolate each dispatch's Claude config, one of these ways:".to_string()); - lines.push( - " 1. Drop user-scope plugins, keep auth: add `--setting-sources project,local`" - .to_string(), - ); - lines.push(" to each dispatch.".to_string()); - lines.push( - " 2. Disable the specific plugin: set `\"enabledPlugins\": { \"@\":" - .to_string(), - ); - lines.push(" false }` in a settings source the dispatch loads.".to_string()); - lines.push(" 3. Clean config dir (strips everything): run each dispatch under".to_string()); - lines.push( - " `CLAUDE_CONFIG_DIR=\"$(mktemp -d)\"` — auth may need `ANTHROPIC_API_KEY`." - .to_string(), - ); - lines.push(format!(" Full mechanics: {ISOLATION_DOC}.")); - lines.join("\n") +/// The build-time preflight: `Some(report)` only when a staged name is +/// shadowed. `config_dir` is passed in so the env-reading caller (the adapter's +/// `detect_shadowed_skills`) stays a thin wrapper. +pub fn shadow_preflight( + config_dir: &Path, + cwd: &Path, + staged_skill_names: &[&str], +) -> Option { + let report = detect_plugin_shadows(config_dir, cwd, staged_skill_names); + (!report.shadowed.is_empty()).then_some(report) } #[cfg(test)] @@ -446,74 +358,22 @@ mod tests { assert_eq!(report.config_dir, config.to_string_lossy()); } - fn sample_report() -> PluginShadowReport { - PluginShadowReport { - config_dir: "/x".into(), - shadowed: vec![ShadowSource::Plugin { - plugin: "slow-powers@slowdini".into(), - skill_name: "verification-before-completion".into(), - path: "/p".into(), - }], - } - } - - #[test] - fn validity_warnings_name_skill_plugin_and_contamination() { - let warnings = shadow_validity_warnings(&sample_report()); - assert_eq!(warnings.len(), 1); - assert!(warnings[0].contains("verification-before-completion")); - assert!(warnings[0].contains("slow-powers@slowdini")); - assert!(warnings[0].to_lowercase().contains("contaminat")); - } - #[test] - fn banner_is_empty_when_nothing_shadowed() { - let empty = PluginShadowReport { - config_dir: "/x".into(), - shadowed: vec![], - }; - assert_eq!(format_shadow_banner(&empty), ""); - } - - #[test] - fn banner_lists_shadowed_skills_and_points_at_isolation_docs() { - let banner = format_shadow_banner(&sample_report()); - assert!(banner.contains("verification-before-completion")); - assert!(banner.contains("slow-powers@slowdini")); - assert!(banner.to_lowercase().contains("isolat")); - } - - #[test] - fn banner_carries_the_three_isolation_recipes_inline() { - // The remediation options live in the banner itself — shown exactly when - // the contamination is detected — not in a README section the operator - // has to go find. - let banner = format_shadow_banner(&sample_report()); - assert!( - banner.contains("--setting-sources project,local"), - "banner names the setting-sources option: {banner}" - ); - assert!( - banner.contains("enabledPlugins"), - "banner names the per-plugin disable option: {banner}" - ); - assert!( - banner.contains("CLAUDE_CONFIG_DIR"), - "banner names the clean-config-dir option: {banner}" - ); - assert!( - banner.contains("docs/claude-notes.md"), - "banner points at the harness dev notes for the full mechanics: {banner}" - ); + fn preflight_is_none_when_nothing_is_shadowed() { + let tmp = TempDir::new().unwrap(); + let (config, cwd) = fresh(&tmp); + assert_eq!(shadow_preflight(&config, &cwd, &["mine"]), None); } #[test] - fn validity_warnings_point_at_a_doc_that_exists() { - let warnings = shadow_validity_warnings(&sample_report()); - assert!( - warnings[0].contains("docs/claude-notes.md"), - "warning points at the harness dev notes: {}", - warnings[0] + fn preflight_returns_the_report_when_a_skill_is_shadowed() { + let tmp = TempDir::new().unwrap(); + let (config, cwd) = fresh(&tmp); + write_file( + &config.join("skills").join("my-skill").join("SKILL.md"), + "---\nname: my-skill\n---\n", ); + let report = shadow_preflight(&config, &cwd, &["my-skill"]).expect("skill is shadowed"); + assert_eq!(report.shadowed.len(), 1); } } diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 56c02b6..6e1538f 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -29,6 +29,7 @@ use std::time::Duration; use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; +use super::skill_shadow::PluginShadowReport; /// The behavior that varies by harness. Generic dispatch code depends on this /// trait, never on a concrete harness variant. See the module docs for the @@ -245,6 +246,24 @@ pub trait HarnessAdapter { None } + // ── Enhancement: shadow preflight (defaulted) ──────────────────────────── + // Fallback without it: no preflight — the run proceeds with no shadow + // report, exactly as for a harness whose dispatches load nothing global. + + /// **Enhancement: shadow preflight.** Detect staged skill names that are + /// also discoverable from the operator's live environment (e.g. Claude + /// Code's enabled plugins or global skills dir), which contaminates the + /// with/without comparison. `scan_root` is a real staged env root — its + /// project-local settings participate in detection. `None` when the + /// harness has no shadow preflight (the default) or nothing is shadowed. + fn detect_shadowed_skills( + &self, + _scan_root: &Path, + _staged_skill_names: &[&str], + ) -> Option { + None + } + // ── Enhancement: plan-mode context (defaulted) ─────────────────────────── /// **Enhancement: plan-mode context.** Wrap a plan-mode profile as an @@ -350,6 +369,16 @@ mod tests { ); } + #[test] + fn detect_shadowed_skills_defaults_to_none_for_harnesses_without_a_preflight() { + for h in [Harness::Codex, Harness::OpenCode] { + assert_eq!( + adapter_for(h).detect_shadowed_skills(Path::new("/nonexistent"), &["any-skill"]), + None + ); + } + } + #[test] fn labels_match_kebab_case_identifiers() { assert_eq!(adapter_for(Harness::ClaudeCode).label(), "claude-code"); diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 1c92e73..216b84a 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -12,15 +12,14 @@ mod cli_command; pub mod codex; pub mod harness; pub mod opencode; +pub mod skill_shadow; pub mod transcript; pub use harness::{ CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, RUNBOOK_TEMPLATE, adapter_for, all_config_dir_names, }; -pub use transcript::TranscriptSummary; - -pub use claude_code::plugin_shadow::{ - PluginShadowReport, ShadowSource, config_dir_from_env, detect_plugin_shadows, - format_shadow_banner, resolve_config_dir, shadow_validity_warnings, +pub use skill_shadow::{ + PluginShadowReport, ShadowSource, format_shadow_banner, shadow_validity_warnings, }; +pub use transcript::TranscriptSummary; diff --git a/src/adapters/skill_shadow.rs b/src/adapters/skill_shadow.rs new file mode 100644 index 0000000..034c3e6 --- /dev/null +++ b/src/adapters/skill_shadow.rs @@ -0,0 +1,191 @@ +//! Harness-neutral skill-shadow report types and formatters. +//! +//! A *shadow* is a staged skill name that is also discoverable from the +//! operator's live environment, contaminating the with/without comparison. +//! Detection is harness-specific (each adapter's +//! [`detect_shadowed_skills`](crate::adapters::HarnessAdapter::detect_shadowed_skills) +//! decides what "discoverable" means); the report shape and its renderings are +//! shared. The remediation text is Claude-flavored because only the Claude Code +//! adapter produces reports today — promote the formatters to adapter methods +//! when a second harness wires this preflight. + +use serde::{Deserialize, Serialize}; + +const ISOLATION_DOC: &str = "docs/claude-notes.md → \"Isolating from installed plugins\""; + +/// A staged skill that is also discoverable from the live environment. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case")] +pub enum ShadowSource { + Plugin { + plugin: String, + skill_name: String, + path: String, + }, + GlobalSkill { + skill_name: String, + path: String, + }, +} + +impl ShadowSource { + pub(crate) fn skill_name(&self) -> &str { + match self { + ShadowSource::Plugin { skill_name, .. } => skill_name, + ShadowSource::GlobalSkill { skill_name, .. } => skill_name, + } + } + + fn source_label(&self) -> String { + match self { + ShadowSource::Plugin { plugin, .. } => format!("enabled plugin '{plugin}'"), + ShadowSource::GlobalSkill { .. } => "the global skills dir".to_string(), + } + } +} + +/// The detector's findings for a run. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct PluginShadowReport { + pub config_dir: String, + pub shadowed: Vec, +} + +/// One `validity_warnings` line per shadowed skill (for benchmark.json). +pub fn shadow_validity_warnings(report: &PluginShadowReport) -> Vec { + report + .shadowed + .iter() + .map(|s| { + format!( + "staged skill '{}' is also provided by {} — each claude -p dispatch could discover \ + both copies, so with/without results may be contaminated. Isolate each dispatch's \ + Claude config (see {}).", + s.skill_name(), + s.source_label(), + ISOLATION_DOC + ) + }) + .collect() +} + +/// Build-time banner for the runner. Empty string when nothing is shadowed. +pub fn format_shadow_banner(report: &PluginShadowReport) -> String { + if report.shadowed.is_empty() { + return String::new(); + } + let mut lines = vec![ + String::new(), + "⚠ Plugin-shadow warning: skills staged for this eval are ALSO discoverable".to_string(), + " from your live environment:".to_string(), + ]; + for s in &report.shadowed { + lines.push(format!(" • {} — {}", s.skill_name(), s.source_label())); + } + lines.push( + " Each `claude -p` dispatch loads your user/global plugins and skills, so".to_string(), + ); + lines.push(" both the staged copy and the installed copy are discoverable — the".to_string()); + lines.push( + " with/without comparison may be contaminated and the control arm is not truly" + .to_string(), + ); + lines.push( + " skill-absent. The runner cannot strip an installed plugin from the dispatch." + .to_string(), + ); + lines.push(" Isolate each dispatch's Claude config, one of these ways:".to_string()); + lines.push( + " 1. Drop user-scope plugins, keep auth: add `--setting-sources project,local`" + .to_string(), + ); + lines.push(" to each dispatch.".to_string()); + lines.push( + " 2. Disable the specific plugin: set `\"enabledPlugins\": { \"@\":" + .to_string(), + ); + lines.push(" false }` in a settings source the dispatch loads.".to_string()); + lines.push(" 3. Clean config dir (strips everything): run each dispatch under".to_string()); + lines.push( + " `CLAUDE_CONFIG_DIR=\"$(mktemp -d)\"` — auth may need `ANTHROPIC_API_KEY`." + .to_string(), + ); + lines.push(format!(" Full mechanics: {ISOLATION_DOC}.")); + lines.join("\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample_report() -> PluginShadowReport { + PluginShadowReport { + config_dir: "/x".into(), + shadowed: vec![ShadowSource::Plugin { + plugin: "slow-powers@slowdini".into(), + skill_name: "verification-before-completion".into(), + path: "/p".into(), + }], + } + } + + #[test] + fn validity_warnings_name_skill_plugin_and_contamination() { + let warnings = shadow_validity_warnings(&sample_report()); + assert_eq!(warnings.len(), 1); + assert!(warnings[0].contains("verification-before-completion")); + assert!(warnings[0].contains("slow-powers@slowdini")); + assert!(warnings[0].to_lowercase().contains("contaminat")); + } + + #[test] + fn banner_is_empty_when_nothing_shadowed() { + let empty = PluginShadowReport { + config_dir: "/x".into(), + shadowed: vec![], + }; + assert_eq!(format_shadow_banner(&empty), ""); + } + + #[test] + fn banner_lists_shadowed_skills_and_points_at_isolation_docs() { + let banner = format_shadow_banner(&sample_report()); + assert!(banner.contains("verification-before-completion")); + assert!(banner.contains("slow-powers@slowdini")); + assert!(banner.to_lowercase().contains("isolat")); + } + + #[test] + fn banner_carries_the_three_isolation_recipes_inline() { + // The remediation options live in the banner itself — shown exactly when + // the contamination is detected — not in a README section the operator + // has to go find. + let banner = format_shadow_banner(&sample_report()); + assert!( + banner.contains("--setting-sources project,local"), + "banner names the setting-sources option: {banner}" + ); + assert!( + banner.contains("enabledPlugins"), + "banner names the per-plugin disable option: {banner}" + ); + assert!( + banner.contains("CLAUDE_CONFIG_DIR"), + "banner names the clean-config-dir option: {banner}" + ); + assert!( + banner.contains("docs/claude-notes.md"), + "banner points at the harness dev notes for the full mechanics: {banner}" + ); + } + + #[test] + fn validity_warnings_point_at_a_doc_that_exists() { + let warnings = shadow_validity_warnings(&sample_report()); + assert!( + warnings[0].contains("docs/claude-notes.md"), + "warning points at the harness dev notes: {}", + warnings[0] + ); + } +} diff --git a/src/cli/run/orchestrate/build.rs b/src/cli/run/orchestrate/build.rs index a9618a1..d956512 100644 --- a/src/cli/run/orchestrate/build.rs +++ b/src/cli/run/orchestrate/build.rs @@ -1,7 +1,7 @@ //! Phases 3 & 4 — build every `(eval, condition)` dispatch task and write //! `conditions.json` / `dispatch-manifest.md` / per-task prompts / `dispatch.json` -//! ([`write_dispatch`]), then arm the opt-in write guard and run the plugin-shadow -//! preflight ([`post_build`]). +//! ([`write_dispatch`]), then arm the opt-in write guard and run the harness's +//! shadow preflight ([`post_build`]). use std::collections::HashMap; use std::fs; @@ -9,10 +9,8 @@ use std::path::Path; use serde_json::{Value, json}; -use crate::adapters::{ - adapter_for, config_dir_from_env, detect_plugin_shadows, format_shadow_banner, -}; -use crate::core::{AvailableSkill, ConditionEntry, ConditionsRecord, Harness, RunContext}; +use crate::adapters::{adapter_for, format_shadow_banner}; +use crate::core::{AvailableSkill, ConditionEntry, ConditionsRecord, RunContext}; use crate::pipeline::io::now_iso8601; use super::super::dispatch::{ @@ -309,8 +307,8 @@ pub(super) fn write_dispatch( Ok(tasks.len()) } -/// Post-build side effects: arm the opt-in write guard and run the Claude Code -/// plugin-shadow preflight. +/// Post-build side effects: arm the opt-in write guard and run the harness's +/// shadow preflight. pub(super) fn post_build( ctx: &RunContext, opts: &RunOptions, @@ -354,23 +352,19 @@ pub(super) fn post_build( eprintln!("{notice}"); } - // Plugin-shadow preflight (Claude Code): a staged skill name also discoverable - // from an enabled plugin or the global skills dir contaminates the run. Scan the - // first staged env, not `ctx.stage_root` — only the per-`(group, condition)` - // envs are created, so the project-local `.claude/settings.json` enabledPlugins - // the scan reads must come from a real staged env. - if ctx.harness == Harness::ClaudeCode { - let mut names: Vec<&str> = vec![ctx.skill_name.as_str()]; - names.extend(ctx.sibling_skill_names.iter().map(String::as_str)); - let scan_root = targets - .first() - .map(|t| t.root.as_path()) - .unwrap_or(ctx.stage_root.as_path()); - let report = detect_plugin_shadows(&config_dir_from_env(), scan_root, &names); - if !report.shadowed.is_empty() { - write_json(&r.iteration_dir.join("plugin-shadow.json"), &report)?; - eprintln!("{}", format_shadow_banner(&report)); - } + // Shadow preflight: a staged skill name also discoverable from the operator's + // live environment contaminates the run. Scan the first staged env, not + // `ctx.stage_root` — only the per-`(group, condition)` envs are created, so + // the project-local settings the scan reads must come from a real staged env. + let mut names: Vec<&str> = vec![ctx.skill_name.as_str()]; + names.extend(ctx.sibling_skill_names.iter().map(String::as_str)); + let scan_root = targets + .first() + .map(|t| t.root.as_path()) + .unwrap_or(ctx.stage_root.as_path()); + if let Some(report) = adapter_for(ctx.harness).detect_shadowed_skills(scan_root, &names) { + write_json(&r.iteration_dir.join("plugin-shadow.json"), &report)?; + eprintln!("{}", format_shadow_banner(&report)); } Ok(()) } diff --git a/src/cli/run/staging/mod.rs b/src/cli/run/staging/mod.rs index da950f0..f461ca3 100644 --- a/src/cli/run/staging/mod.rs +++ b/src/cli/run/staging/mod.rs @@ -81,7 +81,7 @@ impl Default for StageSkillOpts<'_> { repo_root: Path::new(""), assets_dir: None, stage_name_override: None, - harness: Harness::ClaudeCode, + harness: Harness::default(), } } } @@ -101,7 +101,7 @@ impl Default for StageSiblingOpts<'_> { skill_under_test: "", skills_source_dir: Path::new(""), repo_root: Path::new(""), - harness: Harness::ClaudeCode, + harness: Harness::default(), } } } From f1923f5eb28a5819ad378cb00e41da7415cde381 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Thu, 9 Jul 2026 22:53:39 -0400 Subject: [PATCH 13/45] refactor(adapters): adapter-supplied tool vocabulary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WRITE_TOOLS / SHELL_TOOLS / READ_TOOLS and the inline apply_patch / file_change / command_execution names in src/sandbox/ and src/pipeline/ hardcoded per-harness tool naming in generic code. Each adapter now declares a ToolVocabulary (write/patch/shell/read tool names across its guard-hook and transcript surfaces); consumers match against the cross-harness union via all_tool_vocabulary(), mirroring all_config_dir_names() but LazyLock-cached since the guard arbiter consults it per hooked tool call. The guard arbiter's deny messages interpolate {tool_name} instead of naming apply_patch/Bash — byte-identical for every payload the current hook matchers deliver. A hypothetical file_change/command_execution guard payload would now be classified instead of falling through; unreachable under both current HOOK_MATCHERs, so behavior-preserving. Union semantics at guard time were already the effective behavior since both harness hooks funnel into one decide(). Removes the pub WRITE_TOOLS const from the sandbox surface (data moves into the Claude adapter; internal-only in practice). New cross-harness invariant tests: exact union contents, HOOK_MATCHER names must be declared in the vocabulary, transcript parsers imply stray-writes vocabulary, and role lists stay disjoint. Part of #133. Co-Authored-By: Claude Fable 5 --- docs/progressive-enhancements.md | 13 ++- src/adapters/claude_code/guard.rs | 2 +- src/adapters/claude_code/mod.rs | 14 ++- src/adapters/codex/guard.rs | 2 +- src/adapters/codex/mod.rs | 14 ++- src/adapters/harness.rs | 146 ++++++++++++++++++++++++++++ src/adapters/mod.rs | 2 +- src/pipeline/detect_stray_writes.rs | 28 +++--- src/sandbox/decide.rs | 20 ++-- src/sandbox/mod.rs | 2 +- src/sandbox/policy.rs | 56 +++++++++-- 11 files changed, 256 insertions(+), 43 deletions(-) diff --git a/docs/progressive-enhancements.md b/docs/progressive-enhancements.md index 2137819..3d6db60 100644 --- a/docs/progressive-enhancements.md +++ b/docs/progressive-enhancements.md @@ -67,6 +67,8 @@ hand-assembled, and the meta-check uses the LLM-judge fallback. *Trait methods:* `cli_events_filename` (gate: `None` means the ingest pipeline never calls the parsers), `parse_cli_events`, `parse_cli_events_full`, `transcript_surfaces_skill_invocation`. +The tool names the parser emits must be declared in `tool_vocabulary` (see the write-guard +enhancement) or `detect-stray-writes` audits nothing for the harness. ### Native skill staging + skills block @@ -108,10 +110,13 @@ afterwards. arm whose subagent read the live skill source instead of its staged copy, which contaminates the arm; fatal in revision mode, where the `old_skill` arm then sees new-skill content.) -*Trait methods:* `install_guard`, `guard_armed_message`, `guard_hook_cleanup_dir`, plus -`run_capabilities().supports_guard` (invariant-tested to stay in lockstep with the banner). The -hidden `guard` / `guard-codex` subcommands are the hook entry points — their names are a stable -on-disk contract. Shared marker/manifest/teardown machinery lives in `src/sandbox/`. +*Trait methods:* `install_guard`, `guard_armed_message`, `guard_hook_cleanup_dir`, +`tool_vocabulary`, plus `run_capabilities().supports_guard` (invariant-tested to stay in lockstep +with the banner). The guard arbiter and `detect-stray-writes` classify tool names against the +cross-harness vocabulary union (`all_tool_vocabulary`), so wiring a guard or transcript parser +without declaring the harness's tool names trips the invariant tests in `harness.rs`. The hidden +`guard` / `guard-codex` subcommands are the hook entry points — their names are a stable on-disk +contract. Shared marker/manifest/teardown machinery lives in `src/sandbox/`. ### Shadow preflight diff --git a/src/adapters/claude_code/guard.rs b/src/adapters/claude_code/guard.rs index a261bb2..4dd87d4 100644 --- a/src/adapters/claude_code/guard.rs +++ b/src/adapters/claude_code/guard.rs @@ -20,7 +20,7 @@ use crate::sandbox::install::{ use crate::sandbox::{now_ms, parse_tool_call}; /// Tool names the Claude Code PreToolUse hook fires on. -const HOOK_MATCHER: &str = "Write|Edit|MultiEdit|NotebookEdit|Bash"; +pub(crate) const HOOK_MATCHER: &str = "Write|Edit|MultiEdit|NotebookEdit|Bash"; /// Arm the write guard using Claude Code's project-local hook surface. Returns /// the staged marker path. diff --git a/src/adapters/claude_code/mod.rs b/src/adapters/claude_code/mod.rs index 3909cdf..7258703 100644 --- a/src/adapters/claude_code/mod.rs +++ b/src/adapters/claude_code/mod.rs @@ -20,7 +20,9 @@ use std::time::Duration; use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; -use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; +use super::harness::{ + CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, ToolVocabulary, +}; use super::skill_shadow::PluginShadowReport; use cli::{ claude_exec_command_template, claude_judge_dispatch_recipe, claude_parallel_dispatch_recipe, @@ -47,6 +49,16 @@ impl HarnessAdapter for ClaudeCodeAdapter { fn config_dir_names(&self) -> Vec { vec![".claude".to_string()] } + fn tool_vocabulary(&self) -> ToolVocabulary { + ToolVocabulary { + write_tools: ["Edit", "MultiEdit", "NotebookEdit", "Write"] + .map(String::from) + .to_vec(), + patch_tools: Vec::new(), + shell_tools: vec!["Bash".to_string()], + read_tools: ["Glob", "Grep", "Read"].map(String::from).to_vec(), + } + } fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { render_available_skills_block(skills) } diff --git a/src/adapters/codex/guard.rs b/src/adapters/codex/guard.rs index 05a3c4f..f4b6e82 100644 --- a/src/adapters/codex/guard.rs +++ b/src/adapters/codex/guard.rs @@ -20,7 +20,7 @@ use crate::sandbox::install::{ use crate::sandbox::{now_ms, parse_tool_call}; /// Tool names the Codex PreToolUse hook fires on. -const HOOK_MATCHER: &str = "^Bash$|^apply_patch$|^Edit$|^Write$"; +pub(crate) const HOOK_MATCHER: &str = "^Bash$|^apply_patch$|^Edit$|^Write$"; /// Arm the write guard using Codex's project-local hook surface. Returns the /// staged marker path. diff --git a/src/adapters/codex/mod.rs b/src/adapters/codex/mod.rs index e178394..408f1f2 100644 --- a/src/adapters/codex/mod.rs +++ b/src/adapters/codex/mod.rs @@ -17,7 +17,9 @@ use std::time::Duration; use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; -use super::harness::{CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter}; +use super::harness::{ + CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, ToolVocabulary, +}; use cli::{ codex_exec_command_template, codex_judge_dispatch_recipe, codex_parallel_dispatch_recipe, }; @@ -43,6 +45,16 @@ impl HarnessAdapter for CodexAdapter { fn config_dir_names(&self) -> Vec { vec![".agents".to_string(), ".codex".to_string()] } + fn tool_vocabulary(&self) -> ToolVocabulary { + ToolVocabulary { + // Guard hook payloads carry Claude-style names (see guard::HOOK_MATCHER); + // parsed transcripts carry Codex item types (file_change, command_execution). + write_tools: ["Edit", "Write", "file_change"].map(String::from).to_vec(), + patch_tools: vec!["apply_patch".to_string()], + shell_tools: ["Bash", "command_execution"].map(String::from).to_vec(), + read_tools: Vec::new(), + } + } fn rewrites_frontmatter_name(&self) -> bool { true } diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 6e1538f..c535adf 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -24,6 +24,7 @@ use std::io; use std::path::{Path, PathBuf}; +use std::sync::LazyLock; use std::time::Duration; use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocation}; @@ -31,6 +32,21 @@ use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocatio use super::TranscriptSummary; use super::skill_shadow::PluginShadowReport; +/// One harness's tool-name vocabulary: every name its guard hook payloads or +/// transcript parser can produce, grouped by role. Consumers match against the +/// union across all harnesses ([`all_tool_vocabulary`]). +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ToolVocabulary { + /// Tools that write the filesystem with a single target path argument. + pub write_tools: Vec, + /// apply_patch-style tools whose payload carries multiple patch targets. + pub patch_tools: Vec, + /// Shell-execution tools carrying a `command` argument. + pub shell_tools: Vec, + /// Read-only tools carrying a target path argument. + pub read_tools: Vec, +} + /// The behavior that varies by harness. Generic dispatch code depends on this /// trait, never on a concrete harness variant. See the module docs for the /// baseline-vs-enhancement contract. @@ -75,6 +91,16 @@ pub trait HarnessAdapter { Vec::new() } + /// The tool names this harness's guard hook payloads and parsed transcripts + /// use, grouped by role. Via [`all_tool_vocabulary`] this feeds the guard + /// arbiter's tool classification and detect-stray-writes' invocation audit, + /// so list every name this harness's surfaces produce — even names another + /// harness also uses; the union dedups. Default empty: a harness with no + /// guard and no transcript parser contributes nothing. + fn tool_vocabulary(&self) -> ToolVocabulary { + ToolVocabulary::default() + } + // ── Enhancement: native skill staging (defaulted) ──────────────────────── // Fallback without it: `--no-stage` inlines each SKILL.md into its // dispatch prompt instead of staging files for native discovery. @@ -357,6 +383,33 @@ pub fn all_config_dir_names() -> Vec { names } +/// The union of every harness's tool vocabulary (each list sorted, +/// deduplicated). Computed once behind a `LazyLock` — the guard arbiter +/// consults it on every hooked tool call. +pub fn all_tool_vocabulary() -> &'static ToolVocabulary { + static ALL: LazyLock = LazyLock::new(|| { + let mut union = ToolVocabulary::default(); + for &h in Harness::ALL.iter() { + let vocab = adapter_for(h).tool_vocabulary(); + union.write_tools.extend(vocab.write_tools); + union.patch_tools.extend(vocab.patch_tools); + union.shell_tools.extend(vocab.shell_tools); + union.read_tools.extend(vocab.read_tools); + } + for list in [ + &mut union.write_tools, + &mut union.patch_tools, + &mut union.shell_tools, + &mut union.read_tools, + ] { + list.sort_unstable(); + list.dedup(); + } + union + }); + &ALL +} + #[cfg(test)] mod tests { use super::*; @@ -369,6 +422,99 @@ mod tests { ); } + #[test] + fn all_tool_vocabulary_unions_every_adapter() { + let vocab = all_tool_vocabulary(); + assert_eq!( + vocab.write_tools, + ["Edit", "MultiEdit", "NotebookEdit", "Write", "file_change"] + ); + assert_eq!(vocab.patch_tools, ["apply_patch"]); + assert_eq!(vocab.shell_tools, ["Bash", "command_execution"]); + assert_eq!(vocab.read_tools, ["Glob", "Grep", "Read"]); + } + + #[test] + fn guard_hook_matcher_names_are_declared_in_the_vocabulary() { + // Every tool name a harness's guard hook can deliver must be declared in + // its vocabulary, or the arbiter would silently wave the tool through. + let matchers = [ + ( + Harness::ClaudeCode, + crate::adapters::claude_code::guard::HOOK_MATCHER, + ), + (Harness::Codex, crate::adapters::codex::guard::HOOK_MATCHER), + ]; + let guard_wired = Harness::ALL + .iter() + .filter(|&&h| adapter_for(h).run_capabilities().supports_guard) + .count(); + assert_eq!( + guard_wired, + matchers.len(), + "every guard-wired harness's HOOK_MATCHER must be checked here" + ); + for (h, matcher) in matchers { + let vocab = adapter_for(h).tool_vocabulary(); + for token in matcher.split('|') { + let name = token.trim_matches(|c| c == '^' || c == '$'); + let declared = vocab + .write_tools + .iter() + .chain(vocab.patch_tools.iter()) + .chain(vocab.shell_tools.iter()) + .any(|t| t == name); + assert!( + declared, + "{h:?} hook matcher tool {name} is missing from its tool_vocabulary" + ); + } + } + } + + #[test] + fn transcript_parser_implies_stray_writes_vocabulary() { + // A harness with a transcript parser but no declared tool names would + // make detect-stray-writes silently audit nothing for it. + for &h in Harness::ALL.iter() { + let adapter = adapter_for(h); + if adapter.cli_events_filename().is_none() { + continue; + } + let vocab = adapter.tool_vocabulary(); + assert!( + !vocab.write_tools.is_empty(), + "{h:?} parses transcripts but declares no write tools" + ); + assert!( + !vocab.shell_tools.is_empty(), + "{h:?} parses transcripts but declares no shell tools" + ); + } + } + + #[test] + fn tool_vocabulary_roles_are_disjoint_per_harness() { + for &h in Harness::ALL.iter() { + let vocab = adapter_for(h).tool_vocabulary(); + let mut all: Vec<&String> = vocab + .write_tools + .iter() + .chain(vocab.patch_tools.iter()) + .chain(vocab.shell_tools.iter()) + .chain(vocab.read_tools.iter()) + .collect(); + let total = all.len(); + all.sort_unstable(); + all.dedup(); + assert_eq!( + total, + all.len(), + "{h:?} declares a tool name in more than one vocabulary role" + ); + } + } + #[test] fn detect_shadowed_skills_defaults_to_none_for_harnesses_without_a_preflight() { for h in [Harness::Codex, Harness::OpenCode] { diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 216b84a..c78d46b 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -17,7 +17,7 @@ pub mod transcript; pub use harness::{ CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, RUNBOOK_TEMPLATE, - adapter_for, all_config_dir_names, + ToolVocabulary, adapter_for, all_config_dir_names, all_tool_vocabulary, }; pub use skill_shadow::{ PluginShadowReport, ShadowSource, format_shadow_banner, shadow_validity_warnings, diff --git a/src/pipeline/detect_stray_writes.rs b/src/pipeline/detect_stray_writes.rs index 90880d4..a6e44c7 100644 --- a/src/pipeline/detect_stray_writes.rs +++ b/src/pipeline/detect_stray_writes.rs @@ -3,8 +3,8 @@ //! Classifies a run's tool //! invocations against its allowed outputs dir: //! -//! - **violations**: file-write tools (Write/Edit/MultiEdit/NotebookEdit/Codex -//! `file_change`) whose target path resolves outside the outputs dir. +//! - **violations**: file-write tools (per the adapters' cross-harness +//! vocabulary union) whose target path resolves outside the outputs dir. //! - **warnings**: shell commands matching a mutating pattern that don't //! reference the outputs dir (via the sandbox `classify_bash` policy). //! - **live_source_reads**: read tools / shell commands that touched the live @@ -14,22 +14,18 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; -use crate::adapters::all_config_dir_names; +use crate::adapters::{all_config_dir_names, all_tool_vocabulary}; use crate::core::{ConditionsRecord, RunRecord, ToolInvocation}; use crate::pipeline::error::PipelineError; use crate::pipeline::io::{now_iso8601, write_json}; use crate::pipeline::slots::{run_key, run_slots}; -use crate::sandbox::{WRITE_TOOLS, classify_bash, is_under, path_arg}; +use crate::sandbox::{classify_bash, is_shell_tool, is_under, is_write_tool, path_arg}; use crate::validation::{SchemaName, validate_against_schema}; -/// Shell-execution tools across harnesses. -const SHELL_TOOLS: [&str; 2] = ["Bash", "command_execution"]; -/// Read-only tools that carry a target path argument. -const READ_TOOLS: [&str; 3] = ["Read", "Glob", "Grep"]; - -/// A file-write tool: a sandbox write tool, or Codex's `file_change`. -fn is_file_write_tool(name: &str) -> bool { - WRITE_TOOLS.contains(&name) || name == "file_change" +/// A read-only tool carrying a target path argument, in any harness's +/// vocabulary. +fn is_read_tool(name: &str) -> bool { + all_tool_vocabulary().read_tools.iter().any(|t| t == name) } const LIVE_SOURCE_REASON: &str = @@ -74,7 +70,7 @@ pub fn detect_stray_writes( let mut findings = RunFindings::default(); for inv in invocations { - if is_file_write_tool(&inv.name) { + if is_write_tool(&inv.name) { if let Some(p) = inv.args.as_ref().and_then(path_arg) && !is_under(p, outputs_dir, repo_root) { @@ -89,7 +85,7 @@ pub fn detect_stray_writes( continue; } - if SHELL_TOOLS.contains(&inv.name.as_str()) { + if is_shell_tool(&inv.name) { let command = command_of(inv); if let Some(reason) = classify_bash(command, std::slice::from_ref(&outputs_dir.to_string())) @@ -190,7 +186,7 @@ pub fn detect_live_source_reads( let config_dirs = all_config_dir_names(); for inv in invocations { - if READ_TOOLS.contains(&inv.name.as_str()) { + if is_read_tool(&inv.name) { if let Some(p) = inv.args.as_ref().and_then(path_arg) && is_under(p, &live_dir_str, repo_root) { @@ -205,7 +201,7 @@ pub fn detect_live_source_reads( continue; } - if SHELL_TOOLS.contains(&inv.name.as_str()) { + if is_shell_tool(&inv.name) { let command = command_of(inv); if command.contains(live_dir_str.as_ref()) || (rel_usable && references_bare_rel(command, &rel, &config_dirs)) diff --git a/src/sandbox/decide.rs b/src/sandbox/decide.rs index 0dab7c2..c036a33 100644 --- a/src/sandbox/decide.rs +++ b/src/sandbox/decide.rs @@ -10,7 +10,10 @@ use chrono::DateTime; use serde::Deserialize; use serde_json::Value; -use super::policy::{apply_patch_paths, classify_bash, is_under_any, is_write_tool, path_arg}; +use super::policy::{ + apply_patch_paths, classify_bash, is_patch_tool, is_shell_tool, is_under_any, is_write_tool, + path_arg, +}; /// The staged marker file that arms the guard. The guard is a no-op unless this /// file exists, is active, and has not expired — so a crashed run that never tore @@ -101,31 +104,30 @@ pub fn decide( return GuardDecision::allow(); } - if tool_name == "apply_patch" { + if is_patch_tool(tool_name) { let paths = apply_patch_paths(tool_input); if paths.is_empty() { - return GuardDecision::deny( - "eval guard: blocked apply_patch because no patch target path could be determined" - .to_string(), - ); + return GuardDecision::deny(format!( + "eval guard: blocked {tool_name} because no patch target path could be determined" + )); } if let Some(path) = paths.iter().find(|p| !is_under_any(p, &roots, &repo_root)) { return GuardDecision::deny(format!( - "eval guard: apply_patch target {path} is outside the eval sandbox (allowed: {})", + "eval guard: {tool_name} target {path} is outside the eval sandbox (allowed: {})", roots.join(", ") )); } return GuardDecision::allow(); } - if tool_name == "Bash" { + if is_shell_tool(tool_name) { let command = tool_input .get("command") .and_then(Value::as_str) .unwrap_or(""); if let Some(reason) = classify_bash(command, &roots) { return GuardDecision::deny(format!( - "eval guard: blocked Bash ({reason}) — runs outside the eval sandbox" + "eval guard: blocked {tool_name} ({reason}) — runs outside the eval sandbox" )); } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 66152e1..f5266e6 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -18,7 +18,7 @@ pub(crate) use guard::parse_tool_call; pub use guard::read_marker; pub(crate) use install::guard_is_armed; pub use install::{GUARD_MANIFEST, GUARD_MARKER, teardown_guard}; -pub use policy::{WRITE_TOOLS, classify_bash, is_under, is_under_any, is_write_tool, path_arg}; +pub use policy::{classify_bash, is_shell_tool, is_under, is_under_any, is_write_tool, path_arg}; use std::time::{SystemTime, UNIX_EPOCH}; diff --git a/src/sandbox/policy.rs b/src/sandbox/policy.rs index 60405c8..4a5f8b3 100644 --- a/src/sandbox/policy.rs +++ b/src/sandbox/policy.rs @@ -3,7 +3,8 @@ //! Stateless classifiers shared by the armed guard ([`super::decide`]) and //! `pipeline::detect-stray-writes`: which tools write, which Bash commands //! mutate state outside a sandbox, and whether a path falls under an allowed -//! root. +//! root. Tool names come from the adapters' cross-harness vocabulary union +//! ([`all_tool_vocabulary`]), so no harness's tool naming is hardcoded here. use std::path::Path; use std::sync::LazyLock; @@ -11,12 +12,33 @@ use std::sync::LazyLock; use regex::Regex; use serde_json::Value; -/// Tools that mutate the filesystem and carry a target path argument. -pub const WRITE_TOOLS: [&str; 4] = ["Write", "Edit", "MultiEdit", "NotebookEdit"]; +use crate::adapters::all_tool_vocabulary; -/// True for a tool name that writes the filesystem with a path argument. +/// True for a tool name that writes the filesystem with a single target path +/// argument, in any harness's vocabulary. pub fn is_write_tool(tool_name: &str) -> bool { - WRITE_TOOLS.contains(&tool_name) + all_tool_vocabulary() + .write_tools + .iter() + .any(|t| t == tool_name) +} + +/// True for an apply_patch-style tool whose payload carries patch targets +/// (extracted with [`apply_patch_paths`]), in any harness's vocabulary. +pub fn is_patch_tool(tool_name: &str) -> bool { + all_tool_vocabulary() + .patch_tools + .iter() + .any(|t| t == tool_name) +} + +/// True for a shell-execution tool carrying a `command` argument, in any +/// harness's vocabulary. +pub fn is_shell_tool(tool_name: &str) -> bool { + all_tool_vocabulary() + .shell_tools + .iter() + .any(|t| t == tool_name) } /// Bash command patterns that mutate state outside an eval's sandbox. Heuristics @@ -206,15 +228,33 @@ mod tests { } #[test] - fn is_write_tool_matches_the_four_write_tools() { - for t in ["Write", "Edit", "MultiEdit", "NotebookEdit"] { + fn is_write_tool_matches_every_harness_write_tool() { + for t in ["Write", "Edit", "MultiEdit", "NotebookEdit", "file_change"] { assert!(is_write_tool(t), "{t} should be a write tool"); } - for t in ["Read", "Bash", "Grep", ""] { + for t in ["Read", "Bash", "Grep", "apply_patch", ""] { assert!(!is_write_tool(t), "{t} should not be a write tool"); } } + #[test] + fn is_patch_tool_matches_apply_patch_style_tools_only() { + assert!(is_patch_tool("apply_patch")); + for t in ["Write", "Bash", "file_change", ""] { + assert!(!is_patch_tool(t), "{t} should not be a patch tool"); + } + } + + #[test] + fn is_shell_tool_matches_every_harness_shell_tool() { + for t in ["Bash", "command_execution"] { + assert!(is_shell_tool(t), "{t} should be a shell tool"); + } + for t in ["Write", "apply_patch", ""] { + assert!(!is_shell_tool(t), "{t} should not be a shell tool"); + } + } + #[test] fn path_arg_prefers_file_path_then_notebook_then_path() { assert_eq!(path_arg(&json!({ "file_path": "/a" })), Some("/a")); From 05a41a7c0c97da68cb62d70eca8040d552523988 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Thu, 9 Jul 2026 22:57:49 -0400 Subject: [PATCH 14/45] refactor(adapters): render judge recipes from one structural template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude's judge recipe spliced an optional $model_arg shell var while Codex duplicated its entire multi-line command in an if/else — two shell idioms for the same optional-model rule, blocking the recipes from templating as descriptor data. Both now render through one render_judge_dispatch_recipe template in adapters::cli_command, parameterized by the harness command line (containing $model_arg), the model flag, and the events/stderr capture prefix. The judge-worker prompt sentence now lives in one place. Claude output is byte-identical (locked by a new full-string snapshot test). Codex output is behavior-identical but not byte-identical: the if/else pair collapses to the $model_arg splice, -m "$model" (quoted) becomes -m $model inside the unquoted expansion (matching the Claude recipe's long-standing form; model IDs contain no whitespace), and command indentation drops from 6 to 4 spaces. Part of #133. Co-Authored-By: Claude Fable 5 --- src/adapters/claude_code/cli.rs | 59 ++++++++++++++++++--------------- src/adapters/cli_command.rs | 36 ++++++++++++++++++++ src/adapters/codex/cli.rs | 52 ++++++++--------------------- tests/cli/grade_models.rs | 2 +- 4 files changed, 84 insertions(+), 65 deletions(-) diff --git a/src/adapters/claude_code/cli.rs b/src/adapters/claude_code/cli.rs index 88d7056..5a356a2 100644 --- a/src/adapters/claude_code/cli.rs +++ b/src/adapters/claude_code/cli.rs @@ -8,7 +8,7 @@ //! written to a file. `, judge_cwd: &Path) -> String { - let model_flag = model_flag.unwrap_or("--model"); let cwd = judge_cwd.display(); - [ - "Dispatch each judge task from judge-tasks.json with:".to_string(), - String::new(), - "```bash".to_string(), - "JOBS=${JOBS:-4}".to_string(), - "jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // \"\")] | @tsv + \"\\u0000\"' judge-tasks.json | \\".to_string(), - " xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(), - " prompt_path=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(), - " response_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(), - " model=\"$(printf \"%s\" \"$1\" | cut -f3)\"".to_string(), - " response_base=\"${response_path%.json}\"".to_string(), - " mkdir -p \"$(dirname \"$response_path\")\"".to_string(), - " model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"".to_string() - + model_flag - + " $model\"", - format!( + render_judge_dispatch_recipe( + &format!( " cd \"{cwd}\" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \\" ), - " \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(), - " \"$response_base.claude-events.jsonl\" \\".to_string(), - " 2> \"$response_base.claude-stderr.log\"".to_string(), - " ' sh {}".to_string(), - "```".to_string(), - ] - .join("\n") + model_flag.unwrap_or("--model"), + "claude", + ) } #[cfg(test)] @@ -143,4 +123,31 @@ mod tests { assert!(!recipe.contains(""), "{recipe}"); assert!(recipe.contains("cd \"/work/iter-1\" &&"), "{recipe}"); } + + #[test] + fn judge_recipe_snapshot_is_stable() { + // Full-string pin: the judge recipe renders through the shared + // template, and this locks the Claude output byte-for-byte. + let recipe = claude_judge_dispatch_recipe(Some("--model"), Path::new("/work/iter-1")); + let expected = r#"Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="--model $model" + cd "/work/iter-1" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.claude-events.jsonl" \ + 2> "$response_base.claude-stderr.log" + ' sh {} +```"#; + assert_eq!(recipe, expected); + } } diff --git a/src/adapters/cli_command.rs b/src/adapters/cli_command.rs index a09793e..fc8a963 100644 --- a/src/adapters/cli_command.rs +++ b/src/adapters/cli_command.rs @@ -24,6 +24,42 @@ pub(crate) fn render_cli_model_arg(flag: Option<&str>, model: Option<&str>) -> S format!(" {flag} {}", shell_quote_arg(model)) } +/// Render the shared judge-dispatch recipe: the jq/xargs scaffold over +/// `judge-tasks.json` with the harness command line spliced in. +/// +/// `command_line` must reference `$model_arg` (empty when the task declares no +/// model, ` ` otherwise) and end with ` \`; `model_flag` fills the +/// `model_arg` assignment; `capture_prefix` names the per-task +/// `$response_base.-events.jsonl` / `.-stderr.log` captures. +pub(crate) fn render_judge_dispatch_recipe( + command_line: &str, + model_flag: &str, + capture_prefix: &str, +) -> String { + [ + "Dispatch each judge task from judge-tasks.json with:".to_string(), + String::new(), + "```bash".to_string(), + "JOBS=${JOBS:-4}".to_string(), + "jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // \"\")] | @tsv + \"\\u0000\"' judge-tasks.json | \\".to_string(), + " xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(), + " prompt_path=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(), + " response_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(), + " model=\"$(printf \"%s\" \"$1\" | cut -f3)\"".to_string(), + " response_base=\"${response_path%.json}\"".to_string(), + " mkdir -p \"$(dirname \"$response_path\")\"".to_string(), + format!(" model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"{model_flag} $model\""), + command_line.to_string(), + " \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(), + " \"$response_base.{capture_prefix}-events.jsonl\" \\"), + format!(" 2> \"$response_base.{capture_prefix}-stderr.log\""), + " ' sh {}".to_string(), + "```".to_string(), + ] + .join("\n") +} + #[cfg(test)] mod tests { use super::{render_cli_model_arg, shell_quote_arg}; diff --git a/src/adapters/codex/cli.rs b/src/adapters/codex/cli.rs index 860ec1b..53ab68e 100644 --- a/src/adapters/codex/cli.rs +++ b/src/adapters/codex/cli.rs @@ -1,6 +1,6 @@ //! Codex CLI command rendering (`codex exec`) for dispatch guidance. -use crate::adapters::cli_command::render_cli_model_arg; +use crate::adapters::cli_command::{render_cli_model_arg, render_judge_dispatch_recipe}; use std::path::Path; /// Copy/pasteable Codex dispatch command template. Stdin is detached so a @@ -73,41 +73,14 @@ pub(crate) fn codex_judge_dispatch_recipe( } else { "" }; - let model_flag = model_flag.unwrap_or("-m"); let cwd = judge_cwd.display(); - [ - "Dispatch each judge task from judge-tasks.json with:".to_string(), - String::new(), - "```bash".to_string(), - "JOBS=${JOBS:-4}".to_string(), - "jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // \"\")] | @tsv + \"\\u0000\"' judge-tasks.json | \\".to_string(), - " xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(), - " prompt_path=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(), - " response_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(), - " model=\"$(printf \"%s\" \"$1\" | cut -f3)\"".to_string(), - " response_base=\"${response_path%.json}\"".to_string(), - " mkdir -p \"$(dirname \"$response_path\")\"".to_string(), - " if [ -n \"$model\" ]; then".to_string(), - format!( - " codex --ask-for-approval never exec --cd \"{cwd}\" --sandbox workspace-write{hook_trust} {model_flag} \"$model\" --json \\" - ), - " \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(), - " \"$response_base.codex-events.jsonl\" \\".to_string(), - " 2> \"$response_base.codex-stderr.log\"".to_string(), - " else".to_string(), - format!( - " codex --ask-for-approval never exec --cd \"{cwd}\" --sandbox workspace-write{hook_trust} --json \\" + render_judge_dispatch_recipe( + &format!( + " codex --ask-for-approval never exec --cd \"{cwd}\" --sandbox workspace-write{hook_trust} $model_arg --json \\" ), - " \"Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers.\" \\".to_string(), - " \"$response_base.codex-events.jsonl\" \\".to_string(), - " 2> \"$response_base.codex-stderr.log\"".to_string(), - " fi".to_string(), - " ' sh {}".to_string(), - "```".to_string(), - ] - .join("\n") + model_flag.unwrap_or("-m"), + "codex", + ) } #[cfg(test)] @@ -144,18 +117,21 @@ mod tests { fn judge_recipe_places_approval_policy_before_exec() { let recipe = codex_judge_dispatch_recipe(Some("-m"), true, Path::new("/work/iter-1")); + // One command shape: the optional model flag is spliced via $model_arg + // (same structure as the Claude judge recipe), not an if/else pair. assert!( recipe.contains( - " codex --ask-for-approval never exec --cd \"/work/iter-1\" --sandbox workspace-write --dangerously-bypass-hook-trust -m \"$model\" --json \\" + " codex --ask-for-approval never exec --cd \"/work/iter-1\" --sandbox workspace-write --dangerously-bypass-hook-trust $model_arg --json \\" ), "{recipe}" ); assert!( - recipe.contains( - " codex --ask-for-approval never exec --cd \"/work/iter-1\" --sandbox workspace-write --dangerously-bypass-hook-trust --json \\" - ), + recipe.contains(" model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"-m $model\""), "{recipe}" ); + assert!(!recipe.contains("if [ -n"), "{recipe}"); + assert!(recipe.contains("codex-events.jsonl"), "{recipe}"); + assert!(recipe.contains("codex-stderr.log"), "{recipe}"); assert!(!recipe.contains(""), "{recipe}"); } } diff --git a/tests/cli/grade_models.rs b/tests/cli/grade_models.rs index 6c21e3c..da883ca 100644 --- a/tests/cli/grade_models.rs +++ b/tests/cli/grade_models.rs @@ -87,7 +87,7 @@ fn grade_defaults_judge_tasks_to_recorded_judge_model() { .success(); let stdout = String::from_utf8(assert.get_output().stdout.clone()).unwrap(); assert!(stdout.contains("codex --ask-for-approval never exec")); - assert!(stdout.contains("-m \"$model\"")); + assert!(stdout.contains("model_arg=\"-m $model\"")); let tasks: serde_json::Value = serde_json::from_str(&fs::read_to_string(iteration_dir.join("judge-tasks.json")).unwrap()) From b2c1a2199fdaf0e48d0a174d045383a3d72c69ac Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Thu, 9 Jul 2026 23:02:07 -0400 Subject: [PATCH 15/45] refactor(run): derive Default for the remaining harness-naming impls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same cleanup as the staging Default impls: StepParams and DispatchTaskOpts named Harness::ClaudeCode as a field default. ClaudeCode is #[default] and every other field already uses its type's default, so both manual impls collapse into #[derive(Default)] (clippy derivable_impls) — and the variant-specific acceptance grep for #133 comes back clean. Part of #133. Co-Authored-By: Claude Fable 5 --- src/cli/run/dispatch.rs | 27 +-------------------------- src/cli/run/steps.rs | 14 +------------- 2 files changed, 2 insertions(+), 39 deletions(-) diff --git a/src/cli/run/dispatch.rs b/src/cli/run/dispatch.rs index 131b0dd..4ac0704 100644 --- a/src/cli/run/dispatch.rs +++ b/src/cli/run/dispatch.rs @@ -52,7 +52,7 @@ pub struct DispatchTask { } /// Inputs to [`build_dispatch_task`]. `harness` defaults to Claude Code. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct DispatchTaskOpts<'a> { pub eval_id: &'a str, pub condition: &'a str, @@ -85,31 +85,6 @@ pub struct DispatchTaskOpts<'a> { pub eval_root: Option<&'a str>, } -impl Default for DispatchTaskOpts<'_> { - fn default() -> Self { - Self { - eval_id: "", - condition: "", - skill_path: None, - staged_skill_slug: None, - staged_skill_path: None, - user_prompt: "", - fixtures: Vec::new(), - outputs_dir: "", - cond_dir: "", - bootstrap_content: None, - plan_mode_content: None, - skill_name: "", - available_skills: Vec::new(), - harness: Harness::ClaudeCode, - run_tag: None, - run_index: None, - group: None, - eval_root: None, - } - } -} - fn render_available_skills_block_for_harness( harness: Harness, skills: &[AvailableSkill], diff --git a/src/cli/run/steps.rs b/src/cli/run/steps.rs index cd910d5..368ecab 100644 --- a/src/cli/run/steps.rs +++ b/src/cli/run/steps.rs @@ -41,7 +41,7 @@ pub struct StepCommand { } /// Resolved inputs shared by every step of a chain. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default)] pub struct StepParams<'a> { pub skill_dir: Option<&'a str>, pub skill: Option<&'a str>, @@ -50,18 +50,6 @@ pub struct StepParams<'a> { pub workspace_dir: Option<&'a str>, } -impl Default for StepParams<'_> { - fn default() -> Self { - Self { - skill_dir: None, - skill: None, - iteration: 0, - harness: Harness::ClaudeCode, - workspace_dir: None, - } - } -} - impl StepParams<'_> { fn step(&self, label: &'static str, kind: StepKind) -> StepCommand { StepCommand { From bf7048cf973532000a07d814c234282a11e6bf33 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Fri, 10 Jul 2026 00:10:12 -0400 Subject: [PATCH 16/45] test(golden): pin generated harness artifacts as committed fixtures Capture RUNBOOK.md, dispatch-manifest.md, dispatch prompts, skills blocks, judge recipes, and opencode next-steps per harness as byte-exact golden fixtures rendered from fixed contexts, ahead of the descriptor refactor (#134). Re-bless with GOLDEN_BLESS=1 cargo test golden_. Co-Authored-By: Claude Fable 5 --- .gitattributes | 2 + src/cli/run/golden_tests.rs | 226 ++++++++++++++++++ src/cli/run/mod.rs | 2 + .../claude-code/dispatch-prompt.golden.txt | 32 +++ .../golden/claude-code/judge-recipe.golden.md | 19 ++ .../claude-code/manifest-nomodel.golden.md | 113 +++++++++ tests/golden/claude-code/manifest.golden.md | 113 +++++++++ tests/golden/claude-code/runbook.golden.md | 65 +++++ .../claude-code/skills-block.golden.txt | 4 + tests/golden/codex/dispatch-prompt.golden.txt | 32 +++ .../codex/judge-recipe-noguard.golden.md | 19 ++ tests/golden/codex/judge-recipe.golden.md | 19 ++ tests/golden/codex/manifest-noguard.golden.md | 115 +++++++++ tests/golden/codex/manifest.golden.md | 115 +++++++++ tests/golden/codex/runbook.golden.md | 66 +++++ tests/golden/codex/skills-block.golden.txt | 4 + .../opencode/dispatch-prompt.golden.txt | 38 +++ tests/golden/opencode/manifest.golden.md | 87 +++++++ .../opencode/next-steps-model.golden.txt | 2 + .../opencode/next-steps-nomodel.golden.txt | 2 + tests/golden/opencode/runbook.golden.md | 41 ++++ tests/golden/opencode/skills-block.golden.txt | 10 + 22 files changed, 1126 insertions(+) create mode 100644 .gitattributes create mode 100644 src/cli/run/golden_tests.rs create mode 100644 tests/golden/claude-code/dispatch-prompt.golden.txt create mode 100644 tests/golden/claude-code/judge-recipe.golden.md create mode 100644 tests/golden/claude-code/manifest-nomodel.golden.md create mode 100644 tests/golden/claude-code/manifest.golden.md create mode 100644 tests/golden/claude-code/runbook.golden.md create mode 100644 tests/golden/claude-code/skills-block.golden.txt create mode 100644 tests/golden/codex/dispatch-prompt.golden.txt create mode 100644 tests/golden/codex/judge-recipe-noguard.golden.md create mode 100644 tests/golden/codex/judge-recipe.golden.md create mode 100644 tests/golden/codex/manifest-noguard.golden.md create mode 100644 tests/golden/codex/manifest.golden.md create mode 100644 tests/golden/codex/runbook.golden.md create mode 100644 tests/golden/codex/skills-block.golden.txt create mode 100644 tests/golden/opencode/dispatch-prompt.golden.txt create mode 100644 tests/golden/opencode/manifest.golden.md create mode 100644 tests/golden/opencode/next-steps-model.golden.txt create mode 100644 tests/golden/opencode/next-steps-nomodel.golden.txt create mode 100644 tests/golden/opencode/runbook.golden.md create mode 100644 tests/golden/opencode/skills-block.golden.txt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..974ce5f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Golden fixtures are byte-exact — never EOL-normalize them. +tests/golden/** -text diff --git a/src/cli/run/golden_tests.rs b/src/cli/run/golden_tests.rs new file mode 100644 index 0000000..cd6350b --- /dev/null +++ b/src/cli/run/golden_tests.rs @@ -0,0 +1,226 @@ +//! Golden-fixture tests pinning the generated harness artifacts byte-for-byte. +//! +//! Every fixture under `tests/golden//` is rendered from a fixed, +//! deterministic context (no tempdirs, no clocks) so the comparison is exact. +//! The fixtures were captured before the descriptor refactor (#134) and guard +//! byte-identity through it and any later descriptor edit. +//! +//! To regenerate after an intentional output change: +//! `GOLDEN_BLESS=1 cargo test golden_` — then review the fixture diff. + +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::adapters::{CliDispatchContext, CliJudgeContext, adapter_for}; +use crate::core::{AvailableSkill, Harness, Mode}; + +use super::dispatch::{DispatchTaskOpts, ManifestContext, build_dispatch_task, build_manifest}; +use super::runbook::{RunbookContext, build_runbook}; + +fn golden_path(rel: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/golden") + .join(rel) +} + +fn assert_golden(rel: &str, actual: &str) { + let path = golden_path(rel); + if std::env::var_os("GOLDEN_BLESS").is_some_and(|v| v == "1") { + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, actual).unwrap(); + } + let expected = fs::read_to_string(&path).unwrap_or_else(|_| { + panic!( + "golden fixture {} is missing — run `GOLDEN_BLESS=1 cargo test golden_` and commit the fixtures", + path.display() + ) + }); + assert_eq!( + actual, expected, + "output diverged from tests/golden/{rel}; if the change is intentional, \ + re-bless with `GOLDEN_BLESS=1 cargo test golden_` and review the fixture diff" + ); +} + +fn fixed_skills() -> Vec { + vec![ + AvailableSkill { + name: "widget-skill".to_string(), + path: "/work/staged/widget-skill/SKILL.md".to_string(), + description: "Builds widgets the house way.".to_string(), + }, + AvailableSkill { + name: "aux-helper".to_string(), + path: "/work/staged/aux-helper/SKILL.md".to_string(), + description: "Assists with auxiliary chores.".to_string(), + }, + ] +} + +fn staged_task(harness: Harness) -> DispatchTaskOpts<'static> { + DispatchTaskOpts { + eval_id: "demo-eval", + condition: "with_skill", + staged_skill_path: Some("/work/staged/widget-skill/SKILL.md"), + user_prompt: "Build me a widget.", + fixtures: vec!["/work/fixtures/input.txt".to_string()], + outputs_dir: "/work/outputs", + cond_dir: "/work/cond", + bootstrap_content: Some("Session guidelines: be concise."), + plan_mode_content: Some("PLAN STEP"), + skill_name: "widget-skill", + available_skills: fixed_skills(), + harness, + ..Default::default() + } +} + +fn bare_task(harness: Harness) -> DispatchTaskOpts<'static> { + DispatchTaskOpts { + eval_id: "demo-eval", + condition: "without_skill", + user_prompt: "Build me a widget.", + fixtures: vec!["/work/fixtures/input.txt".to_string()], + outputs_dir: "/work/outputs-b", + cond_dir: "/work/cond-b", + skill_name: "widget-skill", + harness, + ..Default::default() + } +} + +fn render_manifest(harness: Harness, guard: bool, agent_model: Option<&str>) -> String { + let slug = + adapter_for(harness).staged_slug("slow-powers-eval-", 2, "with_skill", "widget-skill"); + let mut staged = staged_task(harness); + staged.staged_skill_slug = Some(&slug); + let tasks = vec![ + build_dispatch_task(&staged).unwrap(), + build_dispatch_task(&bare_task(harness)).unwrap(), + ]; + build_manifest( + "widget-skill", + Mode::Revision, + Some("iteration-1"), + 2, + "2026-01-01T00:00:00Z", + &tasks, + ManifestContext { + harness, + guard, + agent_model, + }, + ) +} + +#[test] +fn golden_runbook_per_harness() { + for harness in Harness::ALL { + let label = adapter_for(harness).label(); + let dir = PathBuf::from("/work/.eval-magic/widget-skill/iteration-2"); + let book = build_runbook(&RunbookContext { + harness, + skill_name: "widget-skill", + iteration: 2, + iteration_dir: &dir, + mode: Mode::Revision, + cond_a: "old_skill", + cond_b: "new_skill", + num_tasks: 6, + target_args: " --skill-dir /tmp/skills --skill widget-skill", + guard: true, + agent_model: Some("model-x"), + }); + assert_golden(&format!("{label}/runbook.golden.md"), &book); + } +} + +#[test] +fn golden_manifest_per_harness() { + for harness in Harness::ALL { + let label = adapter_for(harness).label(); + let manifest = render_manifest(harness, true, Some("model-x")); + assert_golden(&format!("{label}/manifest.golden.md"), &manifest); + } +} + +#[test] +fn golden_manifest_codex_without_guard() { + // Pins the hook-trust conditional: no --dangerously-bypass-hook-trust. + let manifest = render_manifest(Harness::Codex, false, Some("model-x")); + assert_golden("codex/manifest-noguard.golden.md", &manifest); +} + +#[test] +fn golden_manifest_claude_without_model() { + // Pins the empty model-arg rendering. + let manifest = render_manifest(Harness::ClaudeCode, true, None); + assert_golden("claude-code/manifest-nomodel.golden.md", &manifest); +} + +#[test] +fn golden_dispatch_prompt_per_harness() { + for harness in Harness::ALL { + let label = adapter_for(harness).label(); + let slug = + adapter_for(harness).staged_slug("slow-powers-eval-", 2, "with_skill", "widget-skill"); + let mut opts = staged_task(harness); + opts.staged_skill_slug = Some(&slug); + let task = build_dispatch_task(&opts).unwrap(); + assert_golden( + &format!("{label}/dispatch-prompt.golden.txt"), + &task.dispatch_prompt, + ); + } +} + +#[test] +fn golden_skills_block_per_harness() { + for harness in Harness::ALL { + let label = adapter_for(harness).label(); + let block = adapter_for(harness).render_available_skills_block(&fixed_skills()); + assert_golden(&format!("{label}/skills-block.golden.txt"), &block); + } +} + +#[test] +fn golden_judge_recipe_claude_and_codex() { + for (harness, guard, rel) in [ + ( + Harness::ClaudeCode, + true, + "claude-code/judge-recipe.golden.md", + ), + (Harness::Codex, true, "codex/judge-recipe.golden.md"), + // Pins the hook-trust conditional in the judge command line. + ( + Harness::Codex, + false, + "codex/judge-recipe-noguard.golden.md", + ), + ] { + let recipe = adapter_for(harness) + .cli_judge_next_steps(CliJudgeContext { + guard, + iteration_dir: Path::new("/work/iter-1"), + }) + .expect("judge recipe is wired for this harness"); + assert_golden(rel, &recipe); + } +} + +#[test] +fn golden_opencode_next_steps_with_and_without_model() { + for (agent_model, rel) in [ + (Some("model-x"), "opencode/next-steps-model.golden.txt"), + (None, "opencode/next-steps-nomodel.golden.txt"), + ] { + let steps = adapter_for(Harness::OpenCode).cli_next_steps(CliDispatchContext { + guard: false, + target_args: " --skill-dir /tmp/skills --skill widget-skill", + iteration: 2, + agent_model, + }); + assert_golden(rel, &steps); + } +} diff --git a/src/cli/run/mod.rs b/src/cli/run/mod.rs index 5aebf6a..9604aef 100644 --- a/src/cli/run/mod.rs +++ b/src/cli/run/mod.rs @@ -17,6 +17,8 @@ use serde::Serialize; pub mod dispatch; pub mod fixtures; +#[cfg(test)] +mod golden_tests; pub mod grouping; pub mod orchestrate; pub mod runbook; diff --git a/tests/golden/claude-code/dispatch-prompt.golden.txt b/tests/golden/claude-code/dispatch-prompt.golden.txt new file mode 100644 index 0000000..19c177e --- /dev/null +++ b/tests/golden/claude-code/dispatch-prompt.golden.txt @@ -0,0 +1,32 @@ + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + +The following skills are available for use with the Skill tool: + +- aux-helper: Assists with auxiliary chores. +- widget-skill: Builds widgets the house way. + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable via the Skill tool. If you invoke it, use that identifier. +If the Skill tool cannot resolve that identifier, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. \ No newline at end of file diff --git a/tests/golden/claude-code/judge-recipe.golden.md b/tests/golden/claude-code/judge-recipe.golden.md new file mode 100644 index 0000000..97c7364 --- /dev/null +++ b/tests/golden/claude-code/judge-recipe.golden.md @@ -0,0 +1,19 @@ +Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="--model $model" + cd "/work/iter-1" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.claude-events.jsonl" \ + 2> "$response_base.claude-stderr.log" + ' sh {} +``` \ No newline at end of file diff --git a/tests/golden/claude-code/manifest-nomodel.golden.md b/tests/golden/claude-code/manifest-nomodel.golden.md new file mode 100644 index 0000000..60de6e2 --- /dev/null +++ b/tests/golden/claude-code/manifest-nomodel.golden.md @@ -0,0 +1,113 @@ +# Dispatch manifest — widget-skill iteration-2 + +Mode: revision (baseline: iteration-1) +Generated: 2026-01-01T00:00:00Z +Total dispatches: 2 + +## How to use this manifest + +In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. + +After all dispatches (Claude Code): + +Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with ` && claude -p --output-format stream-json --verbose --permission-mode acceptEdits \ + "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ + /claude-events.jsonl \ + 2> /claude-stderr.log +``` + +Parallel dispatch from this iteration directory: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + "\u0000"' dispatch.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + eval_root="$(printf "%s" "$1" | cut -f1)" + prompt_path="$(printf "%s" "$1" | cut -f2)" + outputs_dir="$(printf "%s" "$1" | cut -f3)" + mkdir -p "$outputs_dir" + cd "$eval_root" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ + "$outputs_dir/claude-events.jsonl" \ + 2> "$outputs_dir/claude-stderr.log" + ' sh {} +``` + +Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/claude-events.jsonl`. + +After all dispatches: + +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. + +On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. + +## Dispatches +### demo-eval / with_skill + +- run.json: /work/cond/run.json +- timing.json: /work/cond/timing.json + +``` + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + +The following skills are available for use with the Skill tool: + +- aux-helper: Assists with auxiliary chores. +- widget-skill: Builds widgets the house way. + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable via the Skill tool. If you invoke it, use that identifier. +If the Skill tool cannot resolve that identifier, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` + +### demo-eval / without_skill + +- run.json: /work/cond-b/run.json +- timing.json: /work/cond-b/timing.json + +``` +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +No skill is loaded. Respond as you naturally would. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs-b + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` diff --git a/tests/golden/claude-code/manifest.golden.md b/tests/golden/claude-code/manifest.golden.md new file mode 100644 index 0000000..9700276 --- /dev/null +++ b/tests/golden/claude-code/manifest.golden.md @@ -0,0 +1,113 @@ +# Dispatch manifest — widget-skill iteration-2 + +Mode: revision (baseline: iteration-1) +Generated: 2026-01-01T00:00:00Z +Total dispatches: 2 + +## How to use this manifest + +In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. + +After all dispatches (Claude Code): + +Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with ` && claude -p --output-format stream-json --verbose --permission-mode acceptEdits --model model-x \ + "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ + /claude-events.jsonl \ + 2> /claude-stderr.log +``` + +Parallel dispatch from this iteration directory: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + "\u0000"' dispatch.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + eval_root="$(printf "%s" "$1" | cut -f1)" + prompt_path="$(printf "%s" "$1" | cut -f2)" + outputs_dir="$(printf "%s" "$1" | cut -f3)" + mkdir -p "$outputs_dir" + cd "$eval_root" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits --model model-x \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ + "$outputs_dir/claude-events.jsonl" \ + 2> "$outputs_dir/claude-stderr.log" + ' sh {} +``` + +Then run `eval-magic ingest --harness claude-code`; ingest reads each task's `outputs/claude-events.jsonl`. + +After all dispatches: + +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. + +On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. + +## Dispatches +### demo-eval / with_skill + +- run.json: /work/cond/run.json +- timing.json: /work/cond/timing.json + +``` + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + +The following skills are available for use with the Skill tool: + +- aux-helper: Assists with auxiliary chores. +- widget-skill: Builds widgets the house way. + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable via the Skill tool. If you invoke it, use that identifier. +If the Skill tool cannot resolve that identifier, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` + +### demo-eval / without_skill + +- run.json: /work/cond-b/run.json +- timing.json: /work/cond-b/timing.json + +``` +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +No skill is loaded. Respond as you naturally would. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs-b + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` diff --git a/tests/golden/claude-code/runbook.golden.md b/tests/golden/claude-code/runbook.golden.md new file mode 100644 index 0000000..927197d --- /dev/null +++ b/tests/golden/claude-code/runbook.golden.md @@ -0,0 +1,65 @@ +# Eval run — widget-skill (iteration 2, claude-code) + +This runbook is for a human driving the run from a terminal. Work from this iteration directory +and copy-paste each step. The workspace is self-contained — you should not need the surrounding +repo. + +- **Skill under test:** widget-skill +- **Mode:** revision — comparing `old_skill` vs `new_skill` +- **Dispatches:** 6 (the `tasks[]` array in `/work/.eval-magic/widget-skill/iteration-2/dispatch.json`) + +## 1. Dispatch the eval agents, then ingest + +Next: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with: +cd && claude -p --output-format stream-json --verbose --permission-mode acceptEdits --model model-x \ + "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ + /claude-events.jsonl \ + 2> /claude-stderr.log +Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness claude-code`. + +`ingest` records each run, backfills transcripts, scans for stray writes, and grades every +mechanical assertion. It then prints any `llm_judge` tasks it could not grade itself. + +## 2. Dispatch the judge agents, then finalize +Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="--model $model" + cd "/work/.eval-magic/widget-skill/iteration-2" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.claude-events.jsonl" \ + 2> "$response_base.claude-stderr.log" + ' sh {} +``` + +Then merge the verdicts and aggregate: + +``` +eval-magic finalize --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness claude-code +``` + +## 3. Read the result + +`finalize` writes the cross-condition benchmark to: + +``` +/work/.eval-magic/widget-skill/iteration-2/benchmark.json +``` + +Read it for the per-condition pass rates and the `old_skill` − `new_skill` deltas. + +## 4. Tear down + +``` +eval-magic teardown --skill-dir /tmp/skills --skill widget-skill --harness claude-code +``` diff --git a/tests/golden/claude-code/skills-block.golden.txt b/tests/golden/claude-code/skills-block.golden.txt new file mode 100644 index 0000000..d83867d --- /dev/null +++ b/tests/golden/claude-code/skills-block.golden.txt @@ -0,0 +1,4 @@ +The following skills are available for use with the Skill tool: + +- aux-helper: Assists with auxiliary chores. +- widget-skill: Builds widgets the house way. \ No newline at end of file diff --git a/tests/golden/codex/dispatch-prompt.golden.txt b/tests/golden/codex/dispatch-prompt.golden.txt new file mode 100644 index 0000000..d3b0ef8 --- /dev/null +++ b/tests/golden/codex/dispatch-prompt.golden.txt @@ -0,0 +1,32 @@ + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + +## Skills + +- aux-helper: Assists with auxiliary chores. (file: /work/staged/aux-helper/SKILL.md) +- widget-skill: Builds widgets the house way. (file: /work/staged/widget-skill/SKILL.md) + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable as a Codex skill. If you invoke it, use that identifier. +If it does not load as a Codex skill, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. \ No newline at end of file diff --git a/tests/golden/codex/judge-recipe-noguard.golden.md b/tests/golden/codex/judge-recipe-noguard.golden.md new file mode 100644 index 0000000..dc2e2a5 --- /dev/null +++ b/tests/golden/codex/judge-recipe-noguard.golden.md @@ -0,0 +1,19 @@ +Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="-m $model" + codex --ask-for-approval never exec --cd "/work/iter-1" --sandbox workspace-write $model_arg --json \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.codex-events.jsonl" \ + 2> "$response_base.codex-stderr.log" + ' sh {} +``` \ No newline at end of file diff --git a/tests/golden/codex/judge-recipe.golden.md b/tests/golden/codex/judge-recipe.golden.md new file mode 100644 index 0000000..476c51a --- /dev/null +++ b/tests/golden/codex/judge-recipe.golden.md @@ -0,0 +1,19 @@ +Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="-m $model" + codex --ask-for-approval never exec --cd "/work/iter-1" --sandbox workspace-write --dangerously-bypass-hook-trust $model_arg --json \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.codex-events.jsonl" \ + 2> "$response_base.codex-stderr.log" + ' sh {} +``` \ No newline at end of file diff --git a/tests/golden/codex/manifest-noguard.golden.md b/tests/golden/codex/manifest-noguard.golden.md new file mode 100644 index 0000000..1de7120 --- /dev/null +++ b/tests/golden/codex/manifest-noguard.golden.md @@ -0,0 +1,115 @@ +# Dispatch manifest — widget-skill iteration-2 + +Mode: revision (baseline: iteration-1) +Generated: 2026-01-01T00:00:00Z +Total dispatches: 2 + +## How to use this manifest + +In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. + +After all dispatches (Codex): + +Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with ` --sandbox workspace-write -m model-x --json \ + --output-last-message /final-message.md \ + "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ + /codex-events.jsonl \ + 2> /codex-stderr.log +``` + +Parallel dispatch from this iteration directory: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + "\u0000"' dispatch.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + eval_root="$(printf "%s" "$1" | cut -f1)" + prompt_path="$(printf "%s" "$1" | cut -f2)" + outputs_dir="$(printf "%s" "$1" | cut -f3)" + mkdir -p "$outputs_dir" + codex --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write -m model-x --json \ + --output-last-message "$outputs_dir/final-message.md" \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ + "$outputs_dir/codex-events.jsonl" \ + 2> "$outputs_dir/codex-stderr.log" + ' sh {} +``` + +Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/codex-events.jsonl`. + +After all dispatches: + +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. + +On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. + +## Dispatches +### demo-eval / with_skill + +- run.json: /work/cond/run.json +- timing.json: /work/cond/timing.json + +``` + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + +## Skills + +- aux-helper: Assists with auxiliary chores. (file: /work/staged/aux-helper/SKILL.md) +- widget-skill: Builds widgets the house way. (file: /work/staged/widget-skill/SKILL.md) + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable as a Codex skill. If you invoke it, use that identifier. +If it does not load as a Codex skill, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` + +### demo-eval / without_skill + +- run.json: /work/cond-b/run.json +- timing.json: /work/cond-b/timing.json + +``` +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +No skill is loaded. Respond as you naturally would. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs-b + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` diff --git a/tests/golden/codex/manifest.golden.md b/tests/golden/codex/manifest.golden.md new file mode 100644 index 0000000..264f8bc --- /dev/null +++ b/tests/golden/codex/manifest.golden.md @@ -0,0 +1,115 @@ +# Dispatch manifest — widget-skill iteration-2 + +Mode: revision (baseline: iteration-1) +Generated: 2026-01-01T00:00:00Z +Total dispatches: 2 + +## How to use this manifest + +In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. + +After all dispatches (Codex): + +Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with ` --sandbox workspace-write --dangerously-bypass-hook-trust -m model-x --json \ + --output-last-message /final-message.md \ + "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ + /codex-events.jsonl \ + 2> /codex-stderr.log +``` + +Parallel dispatch from this iteration directory: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + "\u0000"' dispatch.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + eval_root="$(printf "%s" "$1" | cut -f1)" + prompt_path="$(printf "%s" "$1" | cut -f2)" + outputs_dir="$(printf "%s" "$1" | cut -f3)" + mkdir -p "$outputs_dir" + codex --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write --dangerously-bypass-hook-trust -m model-x --json \ + --output-last-message "$outputs_dir/final-message.md" \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ + "$outputs_dir/codex-events.jsonl" \ + 2> "$outputs_dir/codex-stderr.log" + ' sh {} +``` + +Then run `eval-magic ingest --harness codex`; Codex transcript ingest reads each task's `outputs/codex-events.jsonl`. + +After all dispatches: + +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. + +On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. + +## Dispatches +### demo-eval / with_skill + +- run.json: /work/cond/run.json +- timing.json: /work/cond/timing.json + +``` + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + +## Skills + +- aux-helper: Assists with auxiliary chores. (file: /work/staged/aux-helper/SKILL.md) +- widget-skill: Builds widgets the house way. (file: /work/staged/widget-skill/SKILL.md) + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with_skill__widget-skill` and is discoverable as a Codex skill. If you invoke it, use that identifier. +If it does not load as a Codex skill, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` + +### demo-eval / without_skill + +- run.json: /work/cond-b/run.json +- timing.json: /work/cond-b/timing.json + +``` +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +No skill is loaded. Respond as you naturally would. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs-b + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` diff --git a/tests/golden/codex/runbook.golden.md b/tests/golden/codex/runbook.golden.md new file mode 100644 index 0000000..f31464d --- /dev/null +++ b/tests/golden/codex/runbook.golden.md @@ -0,0 +1,66 @@ +# Eval run — widget-skill (iteration 2, codex) + +This runbook is for a human driving the run from a terminal. Work from this iteration directory +and copy-paste each step. The workspace is self-contained — you should not need the surrounding +repo. + +- **Skill under test:** widget-skill +- **Mode:** revision — comparing `old_skill` vs `new_skill` +- **Dispatches:** 6 (the `tasks[]` array in `/work/.eval-magic/widget-skill/iteration-2/dispatch.json`) + +## 1. Dispatch the eval agents, then ingest + +Next: iterate the tasks[] array in dispatch.json and dispatch each task with: +codex --ask-for-approval never exec --cd --sandbox workspace-write --dangerously-bypass-hook-trust -m model-x --json \ + --output-last-message /final-message.md \ + "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ + /codex-events.jsonl \ + 2> /codex-stderr.log +Then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex`. + +`ingest` records each run, backfills transcripts, scans for stray writes, and grades every +mechanical assertion. It then prints any `llm_judge` tasks it could not grade itself. + +## 2. Dispatch the judge agents, then finalize +Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="-m $model" + codex --ask-for-approval never exec --cd "/work/.eval-magic/widget-skill/iteration-2" --sandbox workspace-write --dangerously-bypass-hook-trust $model_arg --json \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.codex-events.jsonl" \ + 2> "$response_base.codex-stderr.log" + ' sh {} +``` + +Then merge the verdicts and aggregate: + +``` +eval-magic finalize --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness codex +``` + +## 3. Read the result + +`finalize` writes the cross-condition benchmark to: + +``` +/work/.eval-magic/widget-skill/iteration-2/benchmark.json +``` + +Read it for the per-condition pass rates and the `old_skill` − `new_skill` deltas. + +## 4. Tear down + +``` +eval-magic teardown --skill-dir /tmp/skills --skill widget-skill --harness codex +``` diff --git a/tests/golden/codex/skills-block.golden.txt b/tests/golden/codex/skills-block.golden.txt new file mode 100644 index 0000000..d894ed2 --- /dev/null +++ b/tests/golden/codex/skills-block.golden.txt @@ -0,0 +1,4 @@ +## Skills + +- aux-helper: Assists with auxiliary chores. (file: /work/staged/aux-helper/SKILL.md) +- widget-skill: Builds widgets the house way. (file: /work/staged/widget-skill/SKILL.md) \ No newline at end of file diff --git a/tests/golden/opencode/dispatch-prompt.golden.txt b/tests/golden/opencode/dispatch-prompt.golden.txt new file mode 100644 index 0000000..ec430d0 --- /dev/null +++ b/tests/golden/opencode/dispatch-prompt.golden.txt @@ -0,0 +1,38 @@ + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + + + + aux-helper + Assists with auxiliary chores. + + + widget-skill + Builds widgets the house way. + + + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with-skill-widget-skill` and is discoverable as an OpenCode skill. If you invoke it, use that identifier. +If it does not load as an OpenCode skill, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. \ No newline at end of file diff --git a/tests/golden/opencode/manifest.golden.md b/tests/golden/opencode/manifest.golden.md new file mode 100644 index 0000000..823a9af --- /dev/null +++ b/tests/golden/opencode/manifest.golden.md @@ -0,0 +1,87 @@ +# Dispatch manifest — widget-skill iteration-2 + +Mode: revision (baseline: iteration-1) +Generated: 2026-01-01T00:00:00Z +Total dispatches: 2 + +## How to use this manifest + +In an agent session, read `dispatch.json` (sibling of this file) instead of this manifest. Each task has a `dispatch_prompt_path` field pointing at the file that holds the full prompt — dispatch the task with a short "read this file and follow it" instruction rather than inlining the prompt — plus exact paths for `run.json` and `timing.json`. + +After all dispatches: + +1. Run `eval-magic ingest --harness ` — a fixed-order chain of record-runs (assembles every task's `run.json` from `dispatch.json` + the task's own `outputs/final-message.md` + the events file the harness CLI wrote under `outputs/`, and backfills `timing.json` with transcript-derived tokens/duration; never clobbers an existing record), fill-transcripts, detect-stray-writes, and grade. Optional higher-fidelity timing: write `{ "total_tokens": , "duration_ms": , "source": "completion-event" }` from the task completion event to `timing.json` right after a dispatch — completion-event numbers always win over the backfill. +2. Dispatch the judge tasks ingest lists, then run `eval-magic finalize` for the benchmark. + +On a harness without persisted transcripts, instead write each task's `run.json` (matching `skills/evaluating-skills/schema/run-record.schema.json`, enforced at runtime by grade/fill-transcripts/detect-stray-writes) and `timing.json` by hand when its subagent returns: carry over `eval_id`, `condition`, `skill_path` (`null` on the without_skill arm), `prompt`, and `files` from the task; populate `final_message` from the subagent's reply; leave `tool_invocations` as `[]`; capture `total_tokens`/`duration_ms` from the task completion event immediately — they may not be persisted anywhere else. + +## Dispatches +### demo-eval / with_skill + +- run.json: /work/cond/run.json +- timing.json: /work/cond/timing.json + +``` + +The following guidelines were loaded at session start by the slow-powers plugin +(equivalent to the SessionStart hook firing in a real user's environment): + +Session guidelines: be concise. + + + + aux-helper + Assists with auxiliary chores. + + + widget-skill + Builds widgets the house way. + + + + +PLAN STEP + + +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +The `widget-skill` skill is registered under the identifier `slow-powers-eval-2-with-skill-widget-skill` and is discoverable as an OpenCode skill. If you invoke it, use that identifier. +If it does not load as an OpenCode skill, read the skill from `/work/staged/widget-skill/SKILL.md` instead. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` + +### demo-eval / without_skill + +- run.json: /work/cond-b/run.json +- timing.json: /work/cond-b/timing.json + +``` +You are executing a single test case for a skill evaluation framework. +Treat this as a real user request — do NOT optimize behavior for the eval. + +No skill is loaded. Respond as you naturally would. + +Available fixture files: + - /work/fixtures/input.txt +Output directory: /work/outputs-b + +Instructions: +- Write any files you produce into the output directory. +- After completing the task, write your final user-facing response to /work/outputs-b/final-message.md. +- Do not write outside the output directory. + +User request: +Build me a widget. +``` diff --git a/tests/golden/opencode/next-steps-model.golden.txt b/tests/golden/opencode/next-steps-model.golden.txt new file mode 100644 index 0000000..4138e23 --- /dev/null +++ b/tests/golden/opencode/next-steps-model.golden.txt @@ -0,0 +1,2 @@ + +Next: iterate the tasks[] array in dispatch.json and dispatch each task with `opencode run`. Model selection was recorded as provenance, but the OpenCode adapter has no CLI model flag wired yet. OpenCode transcript ingest is not yet wired, so assemble each task's `run.json`/`timing.json` manually (or capture `opencode run --format json` / `opencode export` output), then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. \ No newline at end of file diff --git a/tests/golden/opencode/next-steps-nomodel.golden.txt b/tests/golden/opencode/next-steps-nomodel.golden.txt new file mode 100644 index 0000000..5cb9924 --- /dev/null +++ b/tests/golden/opencode/next-steps-nomodel.golden.txt @@ -0,0 +1,2 @@ + +Next: iterate the tasks[] array in dispatch.json and dispatch each task with `opencode run`. OpenCode transcript ingest is not yet wired, so assemble each task's `run.json`/`timing.json` manually (or capture `opencode run --format json` / `opencode export` output), then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. \ No newline at end of file diff --git a/tests/golden/opencode/runbook.golden.md b/tests/golden/opencode/runbook.golden.md new file mode 100644 index 0000000..23f25df --- /dev/null +++ b/tests/golden/opencode/runbook.golden.md @@ -0,0 +1,41 @@ +# Eval run — widget-skill (iteration 2, opencode) + +This runbook is for a human driving the run from a terminal. Work from this iteration directory +and copy-paste each step. The workspace is self-contained — you should not need the surrounding +repo. + +- **Skill under test:** widget-skill +- **Mode:** revision — comparing `old_skill` vs `new_skill` +- **Dispatches:** 6 (the `tasks[]` array in `/work/.eval-magic/widget-skill/iteration-2/dispatch.json`) + +## 1. Dispatch the eval agents, then ingest + +Next: iterate the tasks[] array in dispatch.json and dispatch each task with `opencode run`. Model selection was recorded as provenance, but the OpenCode adapter has no CLI model flag wired yet. OpenCode transcript ingest is not yet wired, so assemble each task's `run.json`/`timing.json` manually (or capture `opencode run --format json` / `opencode export` output), then run `ingest --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode`. + +`ingest` records each run, backfills transcripts, scans for stray writes, and grades every +mechanical assertion. It then prints any `llm_judge` tasks it could not grade itself. + +## 2. Dispatch the judge agents, then finalize +Dispatch each judge task `ingest` listed through the same harness CLI, capturing its transcript output, then finalize. + +Then merge the verdicts and aggregate: + +``` +eval-magic finalize --skill-dir /tmp/skills --skill widget-skill --iteration 2 --harness opencode +``` + +## 3. Read the result + +`finalize` writes the cross-condition benchmark to: + +``` +/work/.eval-magic/widget-skill/iteration-2/benchmark.json +``` + +Read it for the per-condition pass rates and the `old_skill` − `new_skill` deltas. + +## 4. Tear down + +``` +eval-magic teardown --skill-dir /tmp/skills --skill widget-skill --harness opencode +``` diff --git a/tests/golden/opencode/skills-block.golden.txt b/tests/golden/opencode/skills-block.golden.txt new file mode 100644 index 0000000..99fd1e5 --- /dev/null +++ b/tests/golden/opencode/skills-block.golden.txt @@ -0,0 +1,10 @@ + + + aux-helper + Assists with auxiliary chores. + + + widget-skill + Builds widgets the house way. + + \ No newline at end of file From 6259f9b9648cf9a6c65d9f8fe208645d9e080f0f Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Fri, 10 Jul 2026 00:21:11 -0400 Subject: [PATCH 17/45] feat(descriptors): harness descriptor schema, TOML loading, and load-time validation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add schema/harness-descriptor.schema.json (embedded as SchemaName::HarnessDescriptor), the three built-in descriptor files under harnesses/, and src/adapters/descriptor.rs: serde model, TOML -> JSON -> schema-gate -> typed-struct loading, single-brace template substitution, and cross-field invariant validation with actionable messages — the load-time form of the old cross-harness adapter tests (guard/banner lockstep, slug/naming-rule agreement, config-dir coverage, matcher/ vocabulary coverage, role disjointness, dispatch-template contracts). Code-backed features become named capabilities (src/adapters/ capabilities.rs): transcript parsers, guard engines, the opencode slug generator, and the claude-plugins shadow preflight. Nothing is wired to adapter_for() yet; the descriptor path is exercised by its own tests (a negative test per invariant plus an embedded-files load test). The packaged-file test pins the new shipped files. Part of #134. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 53 ++ Cargo.toml | 3 + harnesses/claude-code.toml | 96 +++ harnesses/codex.toml | 98 +++ harnesses/opencode.toml | 32 + schema/harness-descriptor.schema.json | 172 ++++++ src/adapters/capabilities.rs | 81 +++ src/adapters/descriptor.rs | 817 ++++++++++++++++++++++++++ src/adapters/mod.rs | 2 + src/adapters/opencode/mod.rs | 10 +- src/validation/schema.rs | 8 +- tests/cli/package.rs | 4 + 12 files changed, 1373 insertions(+), 3 deletions(-) create mode 100644 harnesses/claude-code.toml create mode 100644 harnesses/codex.toml create mode 100644 harnesses/opencode.toml create mode 100644 schema/harness-descriptor.schema.json create mode 100644 src/adapters/capabilities.rs create mode 100644 src/adapters/descriptor.rs diff --git a/Cargo.lock b/Cargo.lock index a34445e..851c6b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -289,6 +289,7 @@ dependencies = [ "serde_json", "tempfile", "thiserror", + "toml", "walkdir", ] @@ -938,6 +939,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "smallvec" version = "1.15.1" @@ -1027,6 +1037,37 @@ dependencies = [ "zerovec", ] +[[package]] +name = "toml" +version = "0.9.12+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" +dependencies = [ + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "winnow 0.7.15", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow 1.0.3", +] + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -1170,6 +1211,18 @@ dependencies = [ "windows-link", ] +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" + +[[package]] +name = "winnow" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" + [[package]] name = "wit-bindgen" version = "0.57.1" diff --git a/Cargo.toml b/Cargo.toml index 4e593a8..7e10e34 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,9 @@ regex = { version = "1.12.3", default-features = false, features = ["std", "perf serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["preserve_order"] } thiserror = "2.0.18" +# Harness descriptor files (harnesses/*.toml). Parse-only: descriptors are +# embedded and read, never written, so `display` is dropped. +toml = { version = "0.9", default-features = false, features = ["parse", "serde"] } walkdir = "2.5.0" [dev-dependencies] diff --git a/harnesses/claude-code.toml b/harnesses/claude-code.toml new file mode 100644 index 0000000..12d0612 --- /dev/null +++ b/harnesses/claude-code.toml @@ -0,0 +1,96 @@ +# Claude Code harness descriptor — the default harness. +# +# Command templates and banners use TOML *literal* strings ('...' / '''...''') +# so shell line-continuation backslashes survive verbatim. The banner starts +# with a blank line (a leading newline in the rendered message) and every +# template's exact whitespace is pinned by the golden tests under tests/golden/. + +label = "claude-code" +skills_dir = ".claude/skills" +config_dirs = [".claude"] + +[run] +supports_guard = true +supports_bootstrap_with_no_stage = true +supports_stage_name_with_no_stage = true + +[tools] +write = ["Edit", "MultiEdit", "NotebookEdit", "Write"] +patch = [] +shell = ["Bash"] +read = ["Glob", "Grep", "Read"] + +[staging] +slug_template = "{prefix}{iteration}-{condition}__{skill_name}" +rewrites_frontmatter_name = false +advertises_staged_slug_name = false +surface_phrase = "via the Skill tool" +unresolved_phrase = "If the Skill tool cannot resolve that identifier" + +[skills_block] +header = "The following skills are available for use with the Skill tool:\n" +item = "\n- {name}: {description}" + +[transcript] +events_filename = "claude-events.jsonl" +parser = "claude-stream-json" +surfaces_skill_invocation = true + +[model] +flag = "--model" + +[guard] +engine = "claude-hooks" +armed_message = ''' + +🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json + and will block writes/installs outside the eval sandbox during dispatches. + Each `claude -p` dispatch loads the hook from the env cwd it runs in. + It auto-expires in 6h and is removed on the next run; to remove it now: + eval-magic teardown-guard''' + +[shadow] +preflight = "claude-plugins" + +[dispatch] +capture_prefix = "claude" +next_steps_template = ''' + +Next: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with: +{exec_command} +Then run `ingest{target_args} --iteration {iteration} --harness claude-code`.''' +# `--output-format stream-json` requires `--verbose` in -p mode; there is no +# --cd flag, so the dispatch runs from the env dir; and there is no +# --output-last-message, so the final message is recovered from the stream-json +# `result` event. ` && claude -p --output-format stream-json --verbose --permission-mode acceptEdits{model_arg} \ + "Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary." \ + /claude-events.jsonl \ + 2> /claude-stderr.log''' +parallel_command_template = ''' + cd "$eval_root" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits{model_arg} \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary." \ + "$outputs_dir/claude-events.jsonl" \ + 2> "$outputs_dir/claude-stderr.log"''' +judge_command_template = ' cd "{cwd}" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \' +manifest_template = ''' +After all dispatches (Claude Code): + +Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with ` --sandbox workspace-write{guard_args}{model_arg} --json \ + --output-last-message /final-message.md \ + "Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md." \ + /codex-events.jsonl \ + 2> /codex-stderr.log''' +parallel_command_template = ''' + codex --ask-for-approval never exec --cd "$eval_root" --sandbox workspace-write{guard_args}{model_arg} --json \ + --output-last-message "$outputs_dir/final-message.md" \ + "Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md." \ + "$outputs_dir/codex-events.jsonl" \ + 2> "$outputs_dir/codex-stderr.log"''' +judge_command_template = ' codex --ask-for-approval never exec --cd "{cwd}" --sandbox workspace-write{guard_args} $model_arg --json \' +manifest_template = ''' +After all dispatches (Codex): + +Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `-events.jsonl)." + }, + "guard_args": { + "type": "string", + "description": "Extra args spliced into command templates at {guard_args} when the run is guarded (leading space included)." + }, + "model_note": { + "type": "string", + "description": "Sentence spliced into next_steps_template at {model_note} when an agent model was declared." + }, + "next_steps_template": { + "type": "string", + "description": "Post-run handoff text; {exec_command}/{target_args}/{iteration}/{model_note} placeholders." + }, + "exec_template": { + "type": "string", + "description": "Copy/pasteable single-dispatch command; -style angle placeholders are prose for the human." + }, + "parallel_command_template": { + "type": "string", + "description": "Per-task command block spliced into the shared jq/xargs parallel scaffold." + }, + "judge_command_template": { + "type": "string", + "description": "Judge command line spliced into the shared judge recipe; must reference $model_arg and {cwd} and end with a shell line continuation." + }, + "manifest_template": { + "type": "string", + "description": "The dispatch-manifest harness section; {exec_command}/{parallel_recipe} placeholders; ends with exactly one newline." + } + } + } + }, + "definitions": { + "tool_list": { + "type": "array", + "items": { "type": "string", "minLength": 1 } + } + } +} diff --git a/src/adapters/capabilities.rs b/src/adapters/capabilities.rs new file mode 100644 index 0000000..701219a --- /dev/null +++ b/src/adapters/capabilities.rs @@ -0,0 +1,81 @@ +//! Named code capabilities a harness descriptor references. +//! +//! Everything a descriptor cannot express as data — transcript stitching, +//! guard hook installation, slug sanitization, plugin-shadow scanning — lives +//! behind one of these closed enums. A descriptor opts in by naming the +//! capability (`parser = "codex-items"`, `engine = "claude-hooks"`); a harness +//! whose stream or hooks are compatible with an existing capability gets the +//! full feature from configuration alone. +//! +//! The enums deserialize from the kebab-case capability names the +//! `harness-descriptor` schema also enumerates, so an unknown name fails the +//! schema gate with a listed-allowed-values message before ever reaching Rust. + +use serde::Deserialize; + +/// Transcript parsers: turn a captured CLI events file into tool invocations +/// and a [`super::TranscriptSummary`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum TranscriptParser { + /// `claude -p --output-format stream-json` events. + ClaudeStreamJson, + /// `codex exec --json` `item.completed` events. + CodexItems, +} + +/// Write-guard engines: install a PreToolUse-style hook under the staged env. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum GuardEngine { + /// Hook merged into `.claude/settings.local.json`. + ClaudeHooks, + /// Hook merged into `.codex/hooks.json`. + CodexHooks, +} + +impl GuardEngine { + /// The tool-name matcher the engine's hook registers for. Exposed so + /// descriptor validation can prove every hooked tool is declared in the + /// descriptor's `[tools]` vocabulary. + pub(crate) fn hook_matcher(self) -> &'static str { + match self { + GuardEngine::ClaudeHooks => super::claude_code::guard::HOOK_MATCHER, + GuardEngine::CodexHooks => super::codex::guard::HOOK_MATCHER, + } + } +} + +/// Staged-slug generators, for harnesses whose naming rules need +/// sanitization/truncation beyond a format string. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SlugCapability { + /// OpenCode's lowercase-alphanumeric-single-hyphen names, length-capped. + Opencode, +} + +impl SlugCapability { + /// Generate the staged slug for one `(iteration, condition, skill)` cell. + pub(crate) fn staged_slug( + self, + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, + ) -> String { + match self { + SlugCapability::Opencode => { + super::opencode::opencode_slug(prefix, iteration, condition, skill_name) + } + } + } +} + +/// Shadow preflights: detect installed skills that shadow a staged slug. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ShadowPreflight { + /// Claude Code plugin/skill scan rooted at the user config dir. + ClaudePlugins, +} diff --git a/src/adapters/descriptor.rs b/src/adapters/descriptor.rs new file mode 100644 index 0000000..18b3769 --- /dev/null +++ b/src/adapters/descriptor.rs @@ -0,0 +1,817 @@ +//! Harness descriptor files: the data half of a harness adapter. +//! +//! A descriptor is a TOML file carrying every declarative value a harness +//! adapter exposes (label, dirs, capability booleans, phrases, templates, +//! banner prose) plus references to *named capabilities* — the code-backed +//! features in [`super::capabilities`] (transcript parsers, guard engines, +//! slug generation, shadow preflight). +//! +//! Loading is schema-gated: the TOML transcodes to JSON and must satisfy the +//! bundled `schema/harness-descriptor.schema.json`, then the cross-field +//! invariants in [`validate_descriptor`] — the load-time form of the old +//! cross-harness adapter tests, so user-supplied descriptor files inherit the +//! same checks. + +use regex::Regex; +use serde::Deserialize; + +use crate::validation::{SchemaName, ValidationError, validate_against_schema}; + +use super::capabilities::{GuardEngine, ShadowPreflight, SlugCapability, TranscriptParser}; + +/// The three built-in harness descriptors, embedded like the schemas: a +/// `(source path, TOML text)` pair per harness, in `Harness::ALL` order. +pub const EMBEDDED_DESCRIPTORS: [(&str, &str); 3] = [ + ( + "harnesses/claude-code.toml", + include_str!("../../harnesses/claude-code.toml"), + ), + ( + "harnesses/codex.toml", + include_str!("../../harnesses/codex.toml"), + ), + ( + "harnesses/opencode.toml", + include_str!("../../harnesses/opencode.toml"), + ), +]; + +/// A parsed, schema-checked, invariant-checked harness descriptor. +/// +/// Field docs live in `schema/harness-descriptor.schema.json` (the schema gate +/// and this struct are kept honest against each other by +/// [`validate_against_schema`]'s deserialize step). +#[derive(Debug, Clone, Deserialize)] +pub struct HarnessDescriptor { + pub label: String, + pub skills_dir: String, + #[serde(default)] + pub config_dirs: Vec, + #[serde(default)] + pub run: RunSection, + #[serde(default)] + pub tools: ToolsSection, + #[serde(default)] + pub staging: StagingSection, + pub skills_block: Option, + pub transcript: Option, + pub model: Option, + pub guard: Option, + pub shadow: Option, + #[serde(default)] + pub dispatch: DispatchSection, +} + +/// Run-option capabilities. The `Default` mirrors the baseline every harness +/// gets without opting in: no guard, bootstrap/stage-name allowed unstaged. +#[derive(Debug, Clone, Deserialize)] +pub struct RunSection { + #[serde(default)] + pub supports_guard: bool, + #[serde(default = "default_true")] + pub supports_bootstrap_with_no_stage: bool, + #[serde(default = "default_true")] + pub supports_stage_name_with_no_stage: bool, +} + +impl Default for RunSection { + fn default() -> Self { + RunSection { + supports_guard: false, + supports_bootstrap_with_no_stage: true, + supports_stage_name_with_no_stage: true, + } + } +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct ToolsSection { + #[serde(default)] + pub write: Vec, + #[serde(default)] + pub patch: Vec, + #[serde(default)] + pub shell: Vec, + #[serde(default)] + pub read: Vec, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct StagingSection { + pub slug_template: Option, + pub slug_capability: Option, + pub stage_name_pattern: Option, + pub stage_name_max_len: Option, + pub stage_name_invalid_message: Option, + #[serde(default)] + pub rewrites_frontmatter_name: bool, + #[serde(default)] + pub advertises_staged_slug_name: bool, + pub surface_phrase: Option, + pub unresolved_phrase: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct SkillsBlockSection { + pub header: String, + pub item: String, + #[serde(default)] + pub footer: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TranscriptSection { + pub events_filename: String, + pub parser: TranscriptParser, + #[serde(default = "default_true")] + pub surfaces_skill_invocation: bool, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ModelSection { + pub flag: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct GuardSection { + pub engine: GuardEngine, + pub armed_message: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ShadowSection { + pub preflight: ShadowPreflight, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct DispatchSection { + pub capture_prefix: Option, + pub guard_args: Option, + pub model_note: Option, + pub next_steps_template: Option, + pub exec_template: Option, + pub parallel_command_template: Option, + pub judge_command_template: Option, + pub manifest_template: Option, +} + +fn default_true() -> bool { + true +} + +/// A descriptor that failed to load. Every variant carries the descriptor's +/// source path so the message is actionable on its own. +#[derive(Debug, thiserror::Error)] +pub enum DescriptorError { + #[error("{path}: invalid TOML: {message}")] + Toml { path: String, message: String }, + #[error(transparent)] + Validation(#[from] ValidationError), + #[error("{path}: {message}")] + Invariant { path: String, message: String }, +} + +/// Parse, schema-check, and invariant-check one descriptor. `source` names the +/// descriptor in error messages (its file path). +pub fn load_descriptor(toml_src: &str, source: &str) -> Result { + let value: serde_json::Value = toml::from_str(toml_src).map_err(|e| DescriptorError::Toml { + path: source.to_string(), + message: e.to_string(), + })?; + let descriptor: HarnessDescriptor = + validate_against_schema(SchemaName::HarnessDescriptor, &value, source)?; + validate_descriptor(&descriptor, source)?; + Ok(descriptor) +} + +/// Substitute single-brace `{token}` placeholders in `template`. +/// +/// Single left-to-right pass over the original template: substituted values +/// are never re-scanned, and unknown or unterminated tokens (shell text like +/// `${JOBS:-4}` or `-I{}`) pass through verbatim. +pub(crate) fn subst(template: &str, vars: &[(&str, &str)]) -> String { + let mut out = String::with_capacity(template.len()); + let mut rest = template; + while let Some(start) = rest.find('{') { + out.push_str(&rest[..start]); + let after = &rest[start + 1..]; + let Some(end) = after.find('}') else { + out.push('{'); + rest = after; + continue; + }; + match vars.iter().find(|(k, _)| *k == &after[..end]) { + Some((_, value)) => { + out.push_str(value); + rest = &after[end + 1..]; + } + None => { + out.push('{'); + rest = after; + } + } + } + out.push_str(rest); + out +} + +/// The staged-slug shape every descriptor must satisfy, and the default when +/// neither a template nor a capability is declared. +const SLUG_PLACEHOLDERS: [&str; 4] = ["{prefix}", "{iteration}", "{condition}", "{skill_name}"]; +pub(crate) const DEFAULT_SLUG_TEMPLATE: &str = "{prefix}{iteration}-{condition}__{skill_name}"; + +/// Render the staged slug for one `(iteration, condition, skill)` cell from +/// the descriptor's slug capability or template (or the default template). +pub(crate) fn render_staged_slug( + staging: &StagingSection, + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, +) -> String { + if let Some(capability) = staging.slug_capability { + return capability.staged_slug(prefix, iteration, condition, skill_name); + } + let iteration = iteration.to_string(); + subst( + staging + .slug_template + .as_deref() + .unwrap_or(DEFAULT_SLUG_TEMPLATE), + &[ + ("prefix", prefix), + ("iteration", &iteration), + ("condition", condition), + ("skill_name", skill_name), + ], + ) +} + +/// Check `name` against the descriptor's declarative stage-name rules, +/// returning the rejection message on violation. +pub(crate) fn stage_name_error( + staging: &StagingSection, + regex: Option<&Regex>, + name: &str, +) -> Option { + let len_ok = staging + .stage_name_max_len + .is_none_or(|max| name.len() <= max); + let pattern_ok = regex.is_none_or(|r| r.is_match(name)); + if len_ok && pattern_ok { + return None; + } + Some(match &staging.stage_name_invalid_message { + Some(message) => subst(message, &[("name", name)]), + None => format!("stage name \"{name}\" violates the descriptor's naming rules"), + }) +} + +/// Cross-field invariants — the load-time form of the old cross-harness +/// adapter tests. Returns the first violation with an actionable message. +fn validate_descriptor(d: &HarnessDescriptor, source: &str) -> Result<(), DescriptorError> { + let fail = |message: String| { + Err(DescriptorError::Invariant { + path: source.to_string(), + message, + }) + }; + + // Guard capability and post-arm banner move in lockstep: `--guard` gates + // on the capability, and an armed guard the user is never told about (or a + // banner with no guard behind it) misleads the dispatch session. + if d.run.supports_guard != d.guard.is_some() { + return fail(format!( + "run.supports_guard is {} but the [guard] table is {}; the guard capability and \ + the armed banner must move in lockstep — declare both or neither", + d.run.supports_guard, + if d.guard.is_some() { + "present" + } else { + "absent" + }, + )); + } + + // Slug shape: one source of truth, all four placeholders, and the + // generated slug must satisfy the descriptor's own naming rules. + if d.staging.slug_template.is_some() && d.staging.slug_capability.is_some() { + return fail( + "declare either staging.slug_template or staging.slug_capability, not both".into(), + ); + } + if let Some(template) = &d.staging.slug_template { + for placeholder in SLUG_PLACEHOLDERS { + if !template.contains(placeholder) { + return fail(format!( + "staging.slug_template must contain {placeholder} — cleanup prefix-scans \ + and per-cell uniqueness rely on all four placeholders" + )); + } + } + } + let stage_regex = match &d.staging.stage_name_pattern { + Some(pattern) => match Regex::new(pattern) { + Ok(regex) => Some(regex), + Err(e) => { + return fail(format!("staging.stage_name_pattern does not compile: {e}")); + } + }, + None => None, + }; + let sample = render_staged_slug(&d.staging, "slow-powers-eval-", 2, "with_skill", "my-skill"); + if !sample.starts_with("slow-powers-eval-") { + return fail(format!( + "the staged slug must preserve the prefix (cleanup prefix-scans rely on it); \ + sample slug: \"{sample}\"" + )); + } + if let Some(message) = stage_name_error(&d.staging, stage_regex.as_ref(), &sample) { + return fail(format!( + "the staged slug \"{sample}\" fails its own stage-name rules ({message}); \ + align staging.slug_template/slug_capability with the naming rules" + )); + } + + // The skills dir must live under a declared config dir, or staging's + // sibling-asset filter would copy a checked-in copy into staged envs. + let top = d.skills_dir.split('/').next().unwrap_or_default(); + if !d.config_dirs.iter().any(|dir| dir == top) { + return fail(format!( + "config_dirs {:?} misses \"{top}\", the parent of skills_dir — staging's \ + sibling-asset filter and the guard tamper rules key off config_dirs", + d.config_dirs + )); + } + + // Every tool the guard engine hooks must be declared in the vocabulary, + // or the write-guard arbiter would silently wave it through. + if let Some(guard) = &d.guard { + let vocabulary: Vec<&str> = d + .tools + .write + .iter() + .chain(&d.tools.patch) + .chain(&d.tools.shell) + .map(String::as_str) + .collect(); + for token in guard.engine.hook_matcher().split('|') { + let token = token.trim_matches(['^', '$']); + if !vocabulary.contains(&token) { + return fail(format!( + "the guard engine hooks tool \"{token}\" but [tools] does not declare it \ + in write/patch/shell — the write-guard arbiter would not recognize it" + )); + } + } + } + + // A transcript parser without a write/shell vocabulary makes the + // stray-writes audit a silent no-op. + if d.transcript.is_some() && (d.tools.write.is_empty() || d.tools.shell.is_empty()) { + return fail( + "[transcript] is declared but [tools] write/shell are empty; \ + detect-stray-writes would audit nothing — declare the harness's tool names" + .into(), + ); + } + + // Tool roles are disjoint: one name in two roles would double-classify + // invocations in the stray-writes audit. + let mut seen: Vec<&str> = Vec::new(); + for name in d + .tools + .write + .iter() + .chain(&d.tools.patch) + .chain(&d.tools.shell) + .chain(&d.tools.read) + { + if seen.contains(&name.as_str()) { + return fail(format!( + "tool \"{name}\" appears in more than one [tools] role — \ + write/patch/shell/read must be disjoint" + )); + } + seen.push(name); + } + + // The judge command line splices into the shared judge recipe; its + // contract (see cli_command::render_judge_dispatch_recipe) is checkable + // here rather than at render time. + if let Some(judge) = &d.dispatch.judge_command_template { + if d.model.is_none() { + return fail( + "dispatch.judge_command_template requires model.flag — the judge recipe \ + splices \"$model_arg\" from each task's model via the model flag" + .into(), + ); + } + if d.dispatch.capture_prefix.is_none() { + return fail( + "dispatch.judge_command_template requires dispatch.capture_prefix — it names \ + the per-task $response_base capture files" + .into(), + ); + } + if !judge.contains("$model_arg") { + return fail( + "dispatch.judge_command_template must reference $model_arg (empty when a task \ + declares no model)" + .into(), + ); + } + if !judge.contains("{cwd}") { + return fail( + "dispatch.judge_command_template must contain {cwd} — judges run from the \ + iteration dir" + .into(), + ); + } + if !judge.ends_with(" \\") { + return fail( + "dispatch.judge_command_template must end with a shell line continuation \ + (\" \\\") so the recipe's prompt line follows it" + .into(), + ); + } + } + + // Placeholders must have a backing field, or the template renders with the + // token left in (the artifact tests' `!contains(\"{{\")` rule, at load time). + let dispatch = &d.dispatch; + let pairings: [(&Option, &str, &str, bool); 7] = [ + ( + &dispatch.next_steps_template, + "next_steps_template", + "{exec_command}", + dispatch.exec_template.is_some(), + ), + ( + &dispatch.next_steps_template, + "next_steps_template", + "{model_note}", + dispatch.model_note.is_some(), + ), + ( + &dispatch.manifest_template, + "manifest_template", + "{exec_command}", + dispatch.exec_template.is_some(), + ), + ( + &dispatch.manifest_template, + "manifest_template", + "{parallel_recipe}", + dispatch.parallel_command_template.is_some(), + ), + ( + &dispatch.exec_template, + "exec_template", + "{guard_args}", + dispatch.guard_args.is_some(), + ), + ( + &dispatch.parallel_command_template, + "parallel_command_template", + "{guard_args}", + dispatch.guard_args.is_some(), + ), + ( + &dispatch.judge_command_template, + "judge_command_template", + "{guard_args}", + dispatch.guard_args.is_some(), + ), + ]; + for (template, template_name, placeholder, backed) in pairings { + if template.as_deref().is_some_and(|t| t.contains(placeholder)) && !backed { + return fail(format!( + "dispatch.{template_name} references {placeholder} but the field that fills \ + it is not set" + )); + } + } + + // The manifest template is spliced as `split('\n')` lines; exactly one + // trailing newline reproduces the section's closing blank line. + if let Some(manifest) = &d.dispatch.manifest_template + && (!manifest.ends_with('\n') || manifest.ends_with("\n\n")) + { + return fail( + "dispatch.manifest_template must end with exactly one trailing newline — it \ + becomes the manifest section's closing blank line" + .into(), + ); + } + + // A skills-block item that never names the skill renders an unusable list. + if let Some(block) = &d.skills_block + && !block.item.contains("{name}") + { + return fail("skills_block.item must contain {name}".into()); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn load(toml_src: &str) -> Result { + load_descriptor(toml_src, "test.toml") + } + + fn err_of(toml_src: &str) -> String { + load(toml_src) + .expect_err("descriptor should be rejected") + .to_string() + } + + const MINIMAL: &str = r#" +label = "demo" +skills_dir = ".demo/skills" +config_dirs = [".demo"] +"#; + + /// A guard-wired descriptor whose tool vocabulary covers the claude-hooks + /// matcher; the base for guard/matcher mutation tests. + const GUARDED: &str = r#" +label = "demo" +skills_dir = ".demo/skills" +config_dirs = [".demo"] + +[run] +supports_guard = true + +[tools] +write = ["Edit", "MultiEdit", "NotebookEdit", "Write"] +shell = ["Bash"] + +[guard] +engine = "claude-hooks" +armed_message = "guard armed" +"#; + + #[test] + fn minimal_descriptor_loads() { + let d = load(MINIMAL).unwrap(); + assert_eq!(d.label, "demo"); + assert_eq!(d.skills_dir, ".demo/skills"); + assert_eq!(d.config_dirs, vec![".demo".to_string()]); + // Section defaults match the trait defaults. + assert!(!d.run.supports_guard); + assert!(d.run.supports_bootstrap_with_no_stage); + assert!(d.run.supports_stage_name_with_no_stage); + assert!(!d.staging.rewrites_frontmatter_name); + assert!(d.guard.is_none()); + assert!(d.transcript.is_none()); + } + + #[test] + fn guarded_descriptor_loads() { + let d = load(GUARDED).unwrap(); + assert!(d.run.supports_guard); + assert!(d.guard.is_some()); + } + + #[test] + fn embedded_descriptors_load_and_validate() { + for (source, toml_src) in EMBEDDED_DESCRIPTORS { + let d = load_descriptor(toml_src, source) + .unwrap_or_else(|e| panic!("embedded descriptor {source} is invalid: {e}")); + assert!(!d.label.is_empty()); + } + } + + #[test] + fn rejects_invalid_toml_syntax() { + let err = err_of("label = "); + assert!(err.contains("test.toml"), "{err}"); + assert!(err.contains("invalid TOML"), "{err}"); + } + + #[test] + fn rejects_unknown_top_level_field() { + let err = err_of(&format!("{MINIMAL}\nmystery = true\n")); + assert!(err.contains("harness-descriptor schema"), "{err}"); + } + + #[test] + fn rejects_non_kebab_case_label() { + let err = err_of( + r#" +label = "Not_Kebab" +skills_dir = ".demo/skills" +config_dirs = [".demo"] +"#, + ); + assert!(err.contains("harness-descriptor schema"), "{err}"); + } + + #[test] + fn rejects_guard_support_without_guard_table() { + let err = err_of( + r#" +label = "demo" +skills_dir = ".demo/skills" +config_dirs = [".demo"] + +[run] +supports_guard = true +"#, + ); + assert!(err.contains("run.supports_guard"), "{err}"); + assert!(err.contains("lockstep"), "{err}"); + } + + #[test] + fn rejects_guard_table_without_guard_support() { + let err = err_of( + r#" +label = "demo" +skills_dir = ".demo/skills" +config_dirs = [".demo"] + +[tools] +write = ["Edit", "MultiEdit", "NotebookEdit", "Write"] +shell = ["Bash"] + +[guard] +engine = "claude-hooks" +armed_message = "guard armed" +"#, + ); + assert!(err.contains("run.supports_guard"), "{err}"); + assert!(err.contains("lockstep"), "{err}"); + } + + #[test] + fn rejects_unknown_guard_engine_name() { + let err = err_of(&GUARDED.replace("claude-hooks", "mystery-hooks")); + assert!(err.contains("harness-descriptor schema"), "{err}"); + } + + #[test] + fn rejects_slug_template_missing_a_placeholder() { + let err = err_of(&format!( + "{MINIMAL}\n[staging]\nslug_template = \"{{prefix}}{{iteration}}-{{condition}}\"\n" + )); + assert!(err.contains("staging.slug_template"), "{err}"); + assert!(err.contains("{skill_name}"), "{err}"); + } + + #[test] + fn rejects_slug_template_and_capability_together() { + let err = err_of(&format!( + "{MINIMAL}\n[staging]\nslug_template = \"{{prefix}}{{iteration}}-{{condition}}__{{skill_name}}\"\nslug_capability = \"opencode\"\n" + )); + assert!(err.contains("not both"), "{err}"); + } + + #[test] + fn rejects_slug_that_fails_its_own_stage_name_rules() { + // The default slug shape emits `__`, which the single-hyphen pattern + // rejects — the staged-slug↔naming-rules invariant. + let err = err_of(&format!( + "{MINIMAL}\n[staging]\nslug_template = \"{{prefix}}{{iteration}}-{{condition}}__{{skill_name}}\"\nstage_name_pattern = \"^[a-z0-9]+(-[a-z0-9]+)*$\"\nstage_name_max_len = 64\n" + )); + assert!(err.contains("stage-name rules"), "{err}"); + } + + #[test] + fn rejects_config_dirs_missing_skills_dir_parent() { + let err = err_of( + r#" +label = "demo" +skills_dir = ".demo/skills" +config_dirs = [".other"] +"#, + ); + assert!(err.contains("parent of skills_dir"), "{err}"); + assert!(err.contains(".demo"), "{err}"); + } + + #[test] + fn rejects_guard_matcher_tool_missing_from_vocabulary() { + // claude-hooks matches Write|Edit|MultiEdit|NotebookEdit|Bash; drop + // Bash from the shell vocabulary and the arbiter would wave it through. + let err = err_of(&GUARDED.replace("shell = [\"Bash\"]", "shell = [\"Shell\"]")); + assert!(err.contains("Bash"), "{err}"); + assert!(err.contains("[tools]"), "{err}"); + } + + #[test] + fn rejects_transcript_without_write_and_shell_tools() { + let err = err_of(&format!( + "{MINIMAL}\n[transcript]\nevents_filename = \"demo-events.jsonl\"\nparser = \"codex-items\"\n" + )); + assert!(err.contains("detect-stray-writes"), "{err}"); + } + + #[test] + fn rejects_tool_declared_in_more_than_one_role() { + let err = err_of(&format!( + "{MINIMAL}\n[tools]\nwrite = [\"Edit\"]\nshell = [\"Edit\"]\n" + )); + assert!(err.contains("more than one [tools] role"), "{err}"); + assert!(err.contains("Edit"), "{err}"); + } + + #[test] + fn rejects_judge_template_without_model_flag() { + let err = err_of(&format!( + "{MINIMAL}\n[dispatch]\ncapture_prefix = \"demo\"\njudge_command_template = ' demo --cd \"{{cwd}}\" $model_arg \\'\n" + )); + assert!(err.contains("model.flag"), "{err}"); + } + + #[test] + fn rejects_judge_template_violating_the_recipe_contract() { + for (template, needle) in [ + ("' demo --cd \"{cwd}\" \\'", "$model_arg"), + ("' demo $model_arg \\'", "{cwd}"), + ("' demo --cd \"{cwd}\" $model_arg'", "line continuation"), + ] { + let err = err_of(&format!( + "{MINIMAL}\n[model]\nflag = \"-m\"\n\n[dispatch]\ncapture_prefix = \"demo\"\njudge_command_template = {template}\n" + )); + assert!(err.contains(needle), "expected {needle} in: {err}"); + } + } + + #[test] + fn rejects_judge_template_without_capture_prefix() { + let err = err_of(&format!( + "{MINIMAL}\n[model]\nflag = \"-m\"\n\n[dispatch]\njudge_command_template = ' demo --cd \"{{cwd}}\" $model_arg \\'\n" + )); + assert!(err.contains("capture_prefix"), "{err}"); + } + + #[test] + fn rejects_template_placeholders_without_backing_fields() { + for (dispatch_body, needle) in [ + ( + "next_steps_template = \"do {exec_command} now\"", + "{exec_command}", + ), + ( + "next_steps_template = \"go.{model_note} then\"", + "{model_note}", + ), + ("exec_template = \"demo{guard_args} run\"", "{guard_args}"), + ( + "exec_template = \"demo run\"\nmanifest_template = \"use:\\n{exec_command}\\n{parallel_recipe}\\n\"", + "{parallel_recipe}", + ), + ] { + let err = err_of(&format!("{MINIMAL}\n[dispatch]\n{dispatch_body}\n")); + assert!(err.contains(needle), "expected {needle} in: {err}"); + } + } + + #[test] + fn rejects_manifest_template_without_single_trailing_newline() { + for manifest in [ + "\"use:\\n{exec_command}\"", + "\"use:\\n{exec_command}\\n\\n\"", + ] { + let err = err_of(&format!( + "{MINIMAL}\n[dispatch]\nexec_template = \"demo run\"\nmanifest_template = {manifest}\n" + )); + assert!(err.contains("exactly one trailing newline"), "{err}"); + } + } + + #[test] + fn rejects_skills_block_item_without_name_placeholder() { + let err = err_of(&format!( + "{MINIMAL}\n[skills_block]\nheader = \"Skills:\"\nitem = \"- {{description}}\"\n" + )); + assert!(err.contains("{name}"), "{err}"); + } + + #[test] + fn rejects_stage_name_pattern_that_does_not_compile() { + let err = err_of(&format!( + "{MINIMAL}\n[staging]\nstage_name_pattern = \"[unclosed\"\n" + )); + assert!(err.contains("does not compile"), "{err}"); + } + + #[test] + fn subst_replaces_tokens_and_passes_unknown_through() { + let out = subst( + "run {exec} with ${JOBS:-4} and -I{} on {exec}", + &[("exec", "demo-cmd")], + ); + assert_eq!(out, "run demo-cmd with ${JOBS:-4} and -I{} on demo-cmd"); + } + + #[test] + fn subst_does_not_rescan_substituted_values() { + let out = subst("{a} {b}", &[("a", "holds-{b}"), ("b", "second")]); + assert_eq!(out, "holds-{b} second"); + } +} diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index c78d46b..9e8541d 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -7,9 +7,11 @@ //! parsers, dispatch-recipe rendering, and write-guard hooks. Generic code //! resolves an adapter with [`adapter_for`] and calls the trait. +pub mod capabilities; pub mod claude_code; mod cli_command; pub mod codex; +pub mod descriptor; pub mod harness; pub mod opencode; pub mod skill_shadow; diff --git a/src/adapters/opencode/mod.rs b/src/adapters/opencode/mod.rs index a97f243..e95e915 100644 --- a/src/adapters/opencode/mod.rs +++ b/src/adapters/opencode/mod.rs @@ -130,8 +130,14 @@ fn sanitize_opencode_name(name: &str) -> String { /// Build a slug that is valid for OpenCode's skill directory + frontmatter name /// constraints. `prefix` is the conspicuous staged-skill prefix, preserved so -/// cleanup prefix-scans still find it. -fn opencode_slug(prefix: &str, iteration: u32, condition: &str, skill_name: &str) -> String { +/// cleanup prefix-scans still find it. `pub(crate)` because the `opencode` +/// named slug capability dispatches here. +pub(crate) fn opencode_slug( + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, +) -> String { let condition = sanitize_opencode_name(condition); let skill = sanitize_opencode_name(skill_name); let base = format!("{prefix}{iteration}-{condition}-{skill}"); diff --git a/src/validation/schema.rs b/src/validation/schema.rs index c98a407..62a516c 100644 --- a/src/validation/schema.rs +++ b/src/validation/schema.rs @@ -24,17 +24,19 @@ pub enum SchemaName { StrayWrites, Benchmark, JudgeTasks, + HarnessDescriptor, } impl SchemaName { /// Every schema, for building the validator cache. - const ALL: [SchemaName; 6] = [ + const ALL: [SchemaName; 7] = [ SchemaName::RunRecord, SchemaName::Evals, SchemaName::Grading, SchemaName::StrayWrites, SchemaName::Benchmark, SchemaName::JudgeTasks, + SchemaName::HarnessDescriptor, ]; /// The schema's kebab-case name, as used in error messages and the on-disk @@ -47,6 +49,7 @@ impl SchemaName { SchemaName::StrayWrites => "stray-writes", SchemaName::Benchmark => "benchmark", SchemaName::JudgeTasks => "judge-tasks", + SchemaName::HarnessDescriptor => "harness-descriptor", } } @@ -59,6 +62,9 @@ impl SchemaName { SchemaName::StrayWrites => include_str!("../../schema/stray-writes.schema.json"), SchemaName::Benchmark => include_str!("../../schema/benchmark.schema.json"), SchemaName::JudgeTasks => include_str!("../../schema/judge-tasks.schema.json"), + SchemaName::HarnessDescriptor => { + include_str!("../../schema/harness-descriptor.schema.json") + } } } } diff --git a/tests/cli/package.rs b/tests/cli/package.rs index a4a1dca..e355e44 100644 --- a/tests/cli/package.rs +++ b/tests/cli/package.rs @@ -73,7 +73,11 @@ fn cargo_package_excludes_repo_local_authoring_files() { "LICENSE", "README.md", "schema/evals.schema.json", + "schema/harness-descriptor.schema.json", "profiles/shared/plan-mode.md", + "harnesses/claude-code.toml", + "harnesses/codex.toml", + "harnesses/opencode.toml", "src/main.rs", ] { assert!( From 151bf439eb4ad7967c7a39ec4cd61c1429221c80 Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Fri, 10 Jul 2026 00:26:38 -0400 Subject: [PATCH 18/45] feat(descriptors): generic DescriptorAdapter over the embedded descriptors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add src/adapters/descriptor_adapter.rs — one HarnessAdapter implementation reading every declarative value from a validated HarnessDescriptor and dispatching code-backed features through the named capabilities (which gain their dispatch methods here: parse/parse_full, install_guard/hook_cleanup_dir, detect). The skills-block render loop moves to a generic renderer (src/adapters/skills_block.rs) that the trait default also uses, and cli_command.rs gains the shared jq/xargs parallel scaffold (render_parallel_dispatch_recipe), mirroring the judge-recipe split. adapter_for() still returns the legacy hand-written adapters; a transitional equivalence test pins the descriptor-backed adapter against them across every observable method and the guard x model context matrix (manifest sections compared as joined text — the consumer joins with '\n', so element granularity is not observable). Deleted with the legacy impls in the cutover commit. Part of #134. Co-Authored-By: Claude Fable 5 --- src/adapters/capabilities.rs | 77 +++++ src/adapters/cli_command.rs | 22 ++ src/adapters/descriptor_adapter.rs | 534 +++++++++++++++++++++++++++++ src/adapters/harness.rs | 16 +- src/adapters/mod.rs | 2 + src/adapters/skills_block.rs | 87 +++++ 6 files changed, 728 insertions(+), 10 deletions(-) create mode 100644 src/adapters/descriptor_adapter.rs create mode 100644 src/adapters/skills_block.rs diff --git a/src/adapters/capabilities.rs b/src/adapters/capabilities.rs index 701219a..2449b8e 100644 --- a/src/adapters/capabilities.rs +++ b/src/adapters/capabilities.rs @@ -11,8 +11,17 @@ //! `harness-descriptor` schema also enumerates, so an unknown name fails the //! schema gate with a listed-allowed-values message before ever reaching Rust. +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + use serde::Deserialize; +use crate::core::ToolInvocation; + +use super::TranscriptSummary; +use super::skill_shadow::PluginShadowReport; + /// Transcript parsers: turn a captured CLI events file into tool invocations /// and a [`super::TranscriptSummary`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] @@ -24,6 +33,28 @@ pub enum TranscriptParser { CodexItems, } +impl TranscriptParser { + /// Parse the captured events file into ordered tool invocations. + pub(crate) fn parse(self, path: &Path) -> io::Result> { + match self { + TranscriptParser::ClaudeStreamJson => { + super::claude_code::stream_json::parse_claude_stream_json(path) + } + TranscriptParser::CodexItems => super::codex::transcript::parse_codex_events(path), + } + } + + /// The full-summary counterpart of [`parse`](Self::parse). + pub(crate) fn parse_full(self, path: &Path) -> io::Result { + match self { + TranscriptParser::ClaudeStreamJson => { + super::claude_code::stream_json::parse_claude_stream_json_full(path) + } + TranscriptParser::CodexItems => super::codex::transcript::parse_codex_events_full(path), + } + } +} + /// Write-guard engines: install a PreToolUse-style hook under the staged env. #[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -44,6 +75,34 @@ impl GuardEngine { GuardEngine::CodexHooks => super::codex::guard::HOOK_MATCHER, } } + + /// Arm the write guard under `stage_root`, returning the staged marker + /// path. + pub(crate) fn install_guard( + self, + stage_root: &Path, + guard_exe: &Path, + ttl: Option, + ) -> io::Result { + match self { + GuardEngine::ClaudeHooks => { + super::claude_code::guard::install_guard(stage_root, guard_exe, ttl) + } + GuardEngine::CodexHooks => { + super::codex::guard::install_guard(stage_root, guard_exe, ttl) + } + } + } + + /// A hook-config dir the install created outside the skills dir, which + /// teardown prunes when restoring the original config leaves it empty. + pub(crate) fn hook_cleanup_dir(self, stage_root: &Path) -> Option { + match self { + // The Claude hook lives in .claude/, which staging already owns. + GuardEngine::ClaudeHooks => None, + GuardEngine::CodexHooks => Some(super::codex::guard::hook_cleanup_dir(stage_root)), + } + } } /// Staged-slug generators, for harnesses whose naming rules need @@ -79,3 +138,21 @@ pub enum ShadowPreflight { /// Claude Code plugin/skill scan rooted at the user config dir. ClaudePlugins, } + +impl ShadowPreflight { + /// Detect staged skill names that are also discoverable from the + /// operator's live environment. `None` when nothing is shadowed. + pub(crate) fn detect( + self, + scan_root: &Path, + staged_skill_names: &[&str], + ) -> Option { + match self { + ShadowPreflight::ClaudePlugins => super::claude_code::plugin_shadow::shadow_preflight( + &super::claude_code::plugin_shadow::config_dir_from_env(), + scan_root, + staged_skill_names, + ), + } + } +} diff --git a/src/adapters/cli_command.rs b/src/adapters/cli_command.rs index fc8a963..b633e16 100644 --- a/src/adapters/cli_command.rs +++ b/src/adapters/cli_command.rs @@ -24,6 +24,28 @@ pub(crate) fn render_cli_model_arg(flag: Option<&str>, model: Option<&str>) -> S format!(" {flag} {}", shell_quote_arg(model)) } +/// Render the shared parallel-dispatch recipe: the jq/xargs scaffold over +/// `dispatch.json` tasks with the harness's per-task command block spliced in. +/// +/// `command_block` is the (possibly multi-line) command run per task inside +/// the `sh -c` body; it references `$eval_root` / `$prompt_path` / +/// `$outputs_dir` and joins as one element, so the output is line-identical +/// to the scaffold-with-block form. +pub(crate) fn render_parallel_dispatch_recipe(command_block: &str) -> String { + [ + "JOBS=${JOBS:-4}", + "jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + \"\\u0000\"' dispatch.json | \\", + " xargs -0 -P \"$JOBS\" -I{} sh -c '", + " eval_root=\"$(printf \"%s\" \"$1\" | cut -f1)\"", + " prompt_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"", + " outputs_dir=\"$(printf \"%s\" \"$1\" | cut -f3)\"", + " mkdir -p \"$outputs_dir\"", + command_block, + " ' sh {}", + ] + .join("\n") +} + /// Render the shared judge-dispatch recipe: the jq/xargs scaffold over /// `judge-tasks.json` with the harness command line spliced in. /// diff --git a/src/adapters/descriptor_adapter.rs b/src/adapters/descriptor_adapter.rs new file mode 100644 index 0000000..9c7e102 --- /dev/null +++ b/src/adapters/descriptor_adapter.rs @@ -0,0 +1,534 @@ +//! The generic descriptor-backed [`HarnessAdapter`]: one implementation +//! serving every harness, reading declarative values from a validated +//! [`HarnessDescriptor`] and dispatching code-backed features through the +//! named capabilities in [`super::capabilities`]. + +use std::io; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use regex::Regex; + +use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; + +use super::TranscriptSummary; +use super::cli_command::{ + render_cli_model_arg, render_judge_dispatch_recipe, render_parallel_dispatch_recipe, +}; +use super::descriptor::{HarnessDescriptor, render_staged_slug, stage_name_error, subst}; +use super::harness::{ + CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, ToolVocabulary, +}; +use super::skill_shadow::PluginShadowReport; +use super::skills_block::{DEFAULT_HEADER, DEFAULT_ITEM, render_skills_block}; + +/// A [`HarnessAdapter`] backed by a validated [`HarnessDescriptor`]. +pub struct DescriptorAdapter { + descriptor: HarnessDescriptor, + /// Compiled `staging.stage_name_pattern`; validated to compile at load + /// time, compiled once here. + stage_name_regex: Option, +} + +impl DescriptorAdapter { + /// Wrap a descriptor that already passed + /// [`load_descriptor`](super::descriptor::load_descriptor) (which proves + /// the stage-name pattern compiles). + pub fn from_descriptor(descriptor: HarnessDescriptor) -> Self { + let stage_name_regex = descriptor + .staging + .stage_name_pattern + .as_deref() + .map(|pattern| { + Regex::new(pattern).expect("stage_name_pattern is validated at descriptor load") + }); + DescriptorAdapter { + descriptor, + stage_name_regex, + } + } + + /// The single-dispatch command: the exec template with `{model_arg}` / + /// `{guard_args}` filled for this run. Empty when no template is wired. + fn exec_command(&self, guard: bool, agent_model: Option<&str>) -> String { + let Some(template) = &self.descriptor.dispatch.exec_template else { + return String::new(); + }; + let model_arg = render_cli_model_arg(self.model_flag(), agent_model); + subst( + template, + &[ + ("model_arg", &model_arg), + ("guard_args", self.guard_args(guard)), + ], + ) + } + + /// The `{guard_args}` value for this run: the descriptor's fragment when + /// the guard is armed, empty otherwise. + fn guard_args(&self, guard: bool) -> &str { + if guard { + self.descriptor.dispatch.guard_args.as_deref().unwrap_or("") + } else { + "" + } + } + + fn model_flag(&self) -> Option<&str> { + self.descriptor.model.as_ref().map(|m| m.flag.as_str()) + } +} + +impl HarnessAdapter for DescriptorAdapter { + fn label(&self) -> String { + self.descriptor.label.clone() + } + + fn skills_dir(&self, repo_root: &Path) -> PathBuf { + self.descriptor + .skills_dir + .split('/') + .fold(repo_root.to_path_buf(), |path, segment| path.join(segment)) + } + + fn run_capabilities(&self) -> HarnessRunCapabilities { + HarnessRunCapabilities { + supports_guard: self.descriptor.run.supports_guard, + supports_bootstrap_with_no_stage: self.descriptor.run.supports_bootstrap_with_no_stage, + supports_stage_name_with_no_stage: self + .descriptor + .run + .supports_stage_name_with_no_stage, + } + } + + fn config_dir_names(&self) -> Vec { + self.descriptor.config_dirs.clone() + } + + fn tool_vocabulary(&self) -> ToolVocabulary { + ToolVocabulary { + write_tools: self.descriptor.tools.write.clone(), + patch_tools: self.descriptor.tools.patch.clone(), + shell_tools: self.descriptor.tools.shell.clone(), + read_tools: self.descriptor.tools.read.clone(), + } + } + + fn staged_slug( + &self, + prefix: &str, + iteration: u32, + condition: &str, + skill_name: &str, + ) -> String { + render_staged_slug( + &self.descriptor.staging, + prefix, + iteration, + condition, + skill_name, + ) + } + + fn validate_stage_name(&self, name: &str) -> Result<(), String> { + match stage_name_error( + &self.descriptor.staging, + self.stage_name_regex.as_ref(), + name, + ) { + Some(message) => Err(message), + None => Ok(()), + } + } + + fn rewrites_frontmatter_name(&self) -> bool { + self.descriptor.staging.rewrites_frontmatter_name + } + + fn advertises_staged_slug_name(&self) -> bool { + self.descriptor.staging.advertises_staged_slug_name + } + + fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { + match &self.descriptor.skills_block { + Some(block) => render_skills_block(&block.header, &block.item, &block.footer, skills), + None => render_skills_block(DEFAULT_HEADER, DEFAULT_ITEM, "", skills), + } + } + + fn skill_surface_phrase(&self) -> String { + self.descriptor + .staging + .surface_phrase + .clone() + .unwrap_or_else(|| "as a discoverable skill".to_string()) + } + + fn skill_unresolved_phrase(&self) -> String { + self.descriptor + .staging + .unresolved_phrase + .clone() + .unwrap_or_else(|| "If the staged skill cannot be resolved".to_string()) + } + + fn cli_events_filename(&self) -> Option { + self.descriptor + .transcript + .as_ref() + .map(|t| t.events_filename.clone()) + } + + fn parse_cli_events(&self, path: &Path) -> io::Result> { + match &self.descriptor.transcript { + Some(transcript) => transcript.parser.parse(path), + None => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "transcript ingest is not wired for the {} harness", + self.label() + ), + )), + } + } + + fn parse_cli_events_full(&self, path: &Path) -> io::Result { + match &self.descriptor.transcript { + Some(transcript) => transcript.parser.parse_full(path), + None => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!( + "transcript ingest is not wired for the {} harness", + self.label() + ), + )), + } + } + + fn transcript_surfaces_skill_invocation(&self) -> bool { + self.descriptor + .transcript + .as_ref() + .is_none_or(|t| t.surfaces_skill_invocation) + } + + fn cli_model_flag(&self) -> Option { + self.model_flag().map(str::to_string) + } + + fn install_guard( + &self, + stage_root: &Path, + guard_exe: &Path, + ttl: Option, + ) -> io::Result { + match &self.descriptor.guard { + Some(guard) => guard.engine.install_guard(stage_root, guard_exe, ttl), + None => Err(io::Error::new( + io::ErrorKind::Unsupported, + format!("--guard is not supported for the {} harness", self.label()), + )), + } + } + + fn guard_armed_message(&self) -> Option { + self.descriptor + .guard + .as_ref() + .map(|g| g.armed_message.clone()) + } + + fn guard_hook_cleanup_dir(&self, stage_root: &Path) -> Option { + self.descriptor + .guard + .as_ref() + .and_then(|g| g.engine.hook_cleanup_dir(stage_root)) + } + + fn detect_shadowed_skills( + &self, + scan_root: &Path, + staged_skill_names: &[&str], + ) -> Option { + self.descriptor + .shadow + .as_ref() + .and_then(|s| s.preflight.detect(scan_root, staged_skill_names)) + } + + fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { + let Some(template) = &self.descriptor.dispatch.next_steps_template else { + return String::new(); + }; + let exec_command = self.exec_command(ctx.guard, ctx.agent_model); + let iteration = ctx.iteration.to_string(); + let model_note = if ctx.agent_model.is_some() { + self.descriptor.dispatch.model_note.as_deref().unwrap_or("") + } else { + "" + }; + subst( + template, + &[ + ("exec_command", &exec_command), + ("target_args", ctx.target_args), + ("iteration", &iteration), + ("model_note", model_note), + ], + ) + } + + fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { + let template = self.descriptor.dispatch.manifest_template.as_ref()?; + let exec_command = self.exec_command(ctx.guard, ctx.agent_model); + let parallel_recipe = match &self.descriptor.dispatch.parallel_command_template { + Some(block_template) => { + let model_arg = render_cli_model_arg(self.model_flag(), ctx.agent_model); + render_parallel_dispatch_recipe(&subst( + block_template, + &[ + ("model_arg", &model_arg), + ("guard_args", self.guard_args(ctx.guard)), + ], + )) + } + None => String::new(), + }; + Some( + subst( + template, + &[ + ("exec_command", &exec_command), + ("parallel_recipe", ¶llel_recipe), + ], + ) + .split('\n') + .map(String::from) + .collect(), + ) + } + + fn cli_judge_next_steps(&self, ctx: CliJudgeContext<'_>) -> Option { + let template = self.descriptor.dispatch.judge_command_template.as_ref()?; + let cwd = ctx.iteration_dir.display().to_string(); + let command_line = subst( + template, + &[("cwd", &cwd), ("guard_args", self.guard_args(ctx.guard))], + ); + Some(render_judge_dispatch_recipe( + &command_line, + // Both guaranteed by descriptor validation when the template is set. + self.model_flag().unwrap_or_default(), + self.descriptor + .dispatch + .capture_prefix + .as_deref() + .unwrap_or_default(), + )) + } +} + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + use super::*; + use crate::adapters::descriptor::{EMBEDDED_DESCRIPTORS, load_descriptor}; + use crate::adapters::harness::{ + CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, adapter_for, + }; + use crate::core::{AvailableSkill, Harness}; + + fn descriptor_adapter(harness: Harness) -> DescriptorAdapter { + let index = Harness::ALL + .iter() + .position(|&h| h == harness) + .expect("harness is built in"); + let (source, toml_src) = EMBEDDED_DESCRIPTORS[index]; + DescriptorAdapter::from_descriptor( + load_descriptor(toml_src, source) + .unwrap_or_else(|e| panic!("embedded descriptor {source} is invalid: {e}")), + ) + } + + fn sample_skills() -> Vec { + vec![ + AvailableSkill { + name: "zeta-skill".to_string(), + path: "/x/zeta-skill/SKILL.md".to_string(), + description: "Does zeta things.".to_string(), + }, + AvailableSkill { + name: "alpha-skill".to_string(), + path: "/x/alpha-skill/SKILL.md".to_string(), + description: "Does alpha things.".to_string(), + }, + ] + } + + /// Transitional equivalence pin for the descriptor cutover: every + /// observable method of the descriptor-backed adapter matches the legacy + /// hand-written adapter, across the guard × model context matrix. Deleted + /// with the legacy adapters once `adapter_for` routes through the + /// registry. + #[test] + fn descriptor_adapter_matches_legacy_adapter_observable_surface() { + let root = Path::new("/repo"); + let stage_root = Path::new("/stage"); + let iteration_dir = PathBuf::from("/work/iter-1"); + + for harness in Harness::ALL { + let legacy = adapter_for(harness); + let new = descriptor_adapter(harness); + + assert_eq!(new.label(), legacy.label()); + assert_eq!(new.skills_dir(root), legacy.skills_dir(root)); + assert_eq!(new.run_capabilities(), legacy.run_capabilities()); + assert_eq!(new.config_dir_names(), legacy.config_dir_names()); + assert_eq!(new.tool_vocabulary(), legacy.tool_vocabulary()); + assert_eq!( + new.rewrites_frontmatter_name(), + legacy.rewrites_frontmatter_name() + ); + assert_eq!( + new.advertises_staged_slug_name(), + legacy.advertises_staged_slug_name() + ); + assert_eq!(new.skill_surface_phrase(), legacy.skill_surface_phrase()); + assert_eq!( + new.skill_unresolved_phrase(), + legacy.skill_unresolved_phrase() + ); + assert_eq!(new.cli_events_filename(), legacy.cli_events_filename()); + assert_eq!(new.cli_model_flag(), legacy.cli_model_flag()); + assert_eq!(new.guard_armed_message(), legacy.guard_armed_message()); + assert_eq!( + new.transcript_surfaces_skill_invocation(), + legacy.transcript_surfaces_skill_invocation() + ); + assert_eq!( + new.guard_hook_cleanup_dir(stage_root), + legacy.guard_hook_cleanup_dir(stage_root) + ); + assert_eq!( + new.render_plan_mode_context(" PLAN\n"), + legacy.render_plan_mode_context(" PLAN\n") + ); + assert_eq!(new.render_plan_mode_context(" "), ""); + + // Staging: generated slugs and naming-rule verdicts. + for (prefix, iteration, condition, skill) in [ + ("slow-powers-eval-", 2, "with_skill", "my-skill"), + ("slow-powers-eval-", 7, "no_skill", "Very_Loud.Skill"), + ( + "slow-powers-eval-", + 1, + "with_skill", + "a-skill-name-clearly-engineered-to-overflow-opencodes-limit-of-sixty-four", + ), + ] { + assert_eq!( + new.staged_slug(prefix, iteration, condition, skill), + legacy.staged_slug(prefix, iteration, condition, skill), + "{harness:?} slug for {condition}/{skill}" + ); + } + for name in [ + "valid-name", + "slow-powers-eval-2-with-skill-my-skill", + "Invalid_Name", + "double--hyphen", + "-leading", + ] { + assert_eq!( + new.validate_stage_name(name), + legacy.validate_stage_name(name), + "{harness:?} stage-name verdict for {name}" + ); + } + + // Skills block: native shape, sortedness, and the empty guard. + assert_eq!( + new.render_available_skills_block(&sample_skills()), + legacy.render_available_skills_block(&sample_skills()) + ); + assert_eq!(new.render_available_skills_block(&[]), ""); + + // Dispatch recipes across the guard × model matrix. + for guard in [false, true] { + for agent_model in [None, Some("model-x")] { + let ctx = CliDispatchContext { + guard, + target_args: " --skill-dir /tmp/skills --skill widget-skill", + iteration: 2, + agent_model, + }; + assert_eq!( + new.cli_next_steps(ctx), + legacy.cli_next_steps(CliDispatchContext { + guard, + target_args: " --skill-dir /tmp/skills --skill widget-skill", + iteration: 2, + agent_model, + }), + "{harness:?} next steps guard={guard} model={agent_model:?}" + ); + // The manifest consumer joins the lines with '\n' + // (build_manifest), so equivalence holds on the joined + // text — the descriptor path splits multi-line commands + // into more elements than the legacy hand-built vec. + assert_eq!( + new.cli_manifest_section(CliManifestContext { guard, agent_model }) + .map(|lines| lines.join("\n")), + legacy + .cli_manifest_section(CliManifestContext { guard, agent_model }) + .map(|lines| lines.join("\n")), + "{harness:?} manifest guard={guard} model={agent_model:?}" + ); + } + assert_eq!( + new.cli_judge_next_steps(CliJudgeContext { + guard, + iteration_dir: &iteration_dir, + }), + legacy.cli_judge_next_steps(CliJudgeContext { + guard, + iteration_dir: &iteration_dir, + }), + "{harness:?} judge recipe guard={guard}" + ); + } + + // Unsupported-enhancement errors carry the same kind and message. + if new.cli_events_filename().is_none() { + let new_err = new.parse_cli_events(Path::new("/nope")).unwrap_err(); + let legacy_err = legacy.parse_cli_events(Path::new("/nope")).unwrap_err(); + assert_eq!(new_err.kind(), legacy_err.kind()); + assert_eq!(new_err.to_string(), legacy_err.to_string()); + let new_err = new.parse_cli_events_full(Path::new("/nope")).unwrap_err(); + let legacy_err = legacy + .parse_cli_events_full(Path::new("/nope")) + .unwrap_err(); + assert_eq!(new_err.to_string(), legacy_err.to_string()); + } + if !new.run_capabilities().supports_guard { + let new_err = new + .install_guard(stage_root, Path::new("/bin/guard"), None) + .unwrap_err(); + let legacy_err = legacy + .install_guard(stage_root, Path::new("/bin/guard"), None) + .unwrap_err(); + assert_eq!(new_err.kind(), legacy_err.kind()); + assert_eq!(new_err.to_string(), legacy_err.to_string()); + } + + // Shadow preflight: only Claude Code wires one; on a bare scan + // root both sides agree (None for the others by construction). + if harness != Harness::ClaudeCode { + assert!(new.detect_shadowed_skills(root, &["x"]).is_none()); + assert!(legacy.detect_shadowed_skills(root, &["x"]).is_none()); + } + } + } +} diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index c535adf..26c1ada 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -149,16 +149,12 @@ pub trait HarnessAdapter { /// list, Codex's `## Skills`, OpenCode's `` XML). The /// default is a neutral bulleted list. fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { - if skills.is_empty() { - return String::new(); - } - let mut sorted: Vec<&AvailableSkill> = skills.iter().collect(); - sorted.sort_by(|a, b| a.name.cmp(&b.name)); - let mut out = String::from("The following skills are available in this session:\n"); - for s in sorted { - out.push_str(&format!("\n- {}: {}", s.name, s.description)); - } - out + super::skills_block::render_skills_block( + super::skills_block::DEFAULT_HEADER, + super::skills_block::DEFAULT_ITEM, + "", + skills, + ) } /// **Enhancement: native staging.** How a staged skill is described as diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 9e8541d..be9b980 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -12,9 +12,11 @@ pub mod claude_code; mod cli_command; pub mod codex; pub mod descriptor; +pub mod descriptor_adapter; pub mod harness; pub mod opencode; pub mod skill_shadow; +mod skills_block; pub mod transcript; pub use harness::{ diff --git a/src/adapters/skills_block.rs b/src/adapters/skills_block.rs new file mode 100644 index 0000000..9c48242 --- /dev/null +++ b/src/adapters/skills_block.rs @@ -0,0 +1,87 @@ +//! The generic available-skills block renderer. +//! +//! Every harness surfaces discoverable skills as header + one item per skill +//! (+ optional footer); only the strings differ, so they live in each harness +//! descriptor's `[skills_block]` and this renderer supplies the shared shape: +//! empty list renders empty, skills sort by name, and the item template's +//! `{name}`/`{description}`/`{path}` placeholders fill per skill. + +use crate::core::AvailableSkill; + +use super::descriptor::subst; + +/// The neutral shape used when a descriptor declares no `[skills_block]` — +/// the trait's documented default. +pub(crate) const DEFAULT_HEADER: &str = "The following skills are available in this session:\n"; +pub(crate) const DEFAULT_ITEM: &str = "\n- {name}: {description}"; + +/// Render the block, or an empty string when no skills are staged (the caller +/// omits the block entirely in that case). +pub(crate) fn render_skills_block( + header: &str, + item: &str, + footer: &str, + skills: &[AvailableSkill], +) -> String { + if skills.is_empty() { + return String::new(); + } + let mut sorted: Vec<&AvailableSkill> = skills.iter().collect(); + sorted.sort_by(|a, b| a.name.cmp(&b.name)); + let mut out = String::from(header); + for skill in sorted { + out.push_str(&subst( + item, + &[ + ("name", &skill.name), + ("description", &skill.description), + ("path", &skill.path), + ], + )); + } + out.push_str(footer); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn skill(name: &str, description: &str) -> AvailableSkill { + AvailableSkill { + name: name.into(), + path: format!("/x/{name}/SKILL.md"), + description: description.into(), + } + } + + #[test] + fn renders_header_items_and_footer_sorted_by_name() { + let block = render_skills_block( + "", + "\n- {name}: {description} ({path})", + "\n", + &[skill("zebra", "z"), skill("alpha", "a")], + ); + assert_eq!( + block, + "\n- alpha: a (/x/alpha/SKILL.md)\n- zebra: z (/x/zebra/SKILL.md)\n" + ); + } + + #[test] + fn empty_list_renders_empty_string_even_with_templates() { + assert_eq!(render_skills_block("H", "\n- {name}", "F", &[]), ""); + } + + #[test] + fn braces_in_a_description_are_not_reexpanded() { + let block = render_skills_block( + "", + "\n- {name}: {description}", + "", + &[skill("a", "uses {name} literally")], + ); + assert_eq!(block, "\n- a: uses {name} literally"); + } +} From 45ebdf36076bd361a6c13012af4094111b5b12cf Mon Sep 17 00:00:00 2001 From: Max Haarhaus Date: Fri, 10 Jul 2026 03:02:13 -0400 Subject: [PATCH 19/45] refactor(adapters): route adapter_for through the descriptor registry adapter_for() now resolves each Harness variant to a descriptor-backed DescriptorAdapter held in a LazyLock registry built from the embedded harnesses/*.toml files (with a label<->variant lockstep check; a bad bundled descriptor panics with its own actionable message, mirroring the bundled-schema panics). The hand-written adapter impls dissolve: claude_code/{cli,session}.rs, codex/{cli,session}.rs, and opencode/session.rs are deleted, and each harness module tree keeps only its code capabilities (parsers, guards, plugin shadow, the opencode slug fns). The cross-method invariant tests in harness.rs are superseded by load-time descriptor validation; the per-value pins stay, and the deleted cli/session test coverage moves to descriptor_adapter.rs (including the byte-exact Claude judge-recipe snapshot). The transitional old-vs-new equivalence test is dropped with the legacy impls. Golden fixtures show zero diff: RUNBOOK.md, dispatch-manifest.md, dispatch prompts, skills blocks, and judge recipes are byte-identical through the descriptor path. Closes #134. Co-Authored-By: Claude Fable 5 --- src/adapters/claude_code/cli.rs | 153 ------------- src/adapters/claude_code/mod.rs | 136 +---------- src/adapters/claude_code/session.rs | 62 ----- src/adapters/codex/cli.rs | 137 ----------- src/adapters/codex/mod.rs | 145 +----------- src/adapters/codex/session.rs | 63 ------ src/adapters/descriptor_adapter.rs | 340 +++++++++++++--------------- src/adapters/harness.rs | 180 ++++----------- src/adapters/opencode/mod.rs | 78 +------ src/adapters/opencode/session.rs | 69 ------ 10 files changed, 212 insertions(+), 1151 deletions(-) delete mode 100644 src/adapters/claude_code/cli.rs delete mode 100644 src/adapters/claude_code/session.rs delete mode 100644 src/adapters/codex/cli.rs delete mode 100644 src/adapters/codex/session.rs delete mode 100644 src/adapters/opencode/session.rs diff --git a/src/adapters/claude_code/cli.rs b/src/adapters/claude_code/cli.rs deleted file mode 100644 index 5a356a2..0000000 --- a/src/adapters/claude_code/cli.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! Claude Code `claude -p` command rendering for CLI dispatch guidance. -//! -//! Differences from the Codex recipe, all forced by the `claude` CLI: -//! `--output-format stream-json` requires `--verbose` in `-p` mode; there is no -//! `--cd` flag, so the dispatch runs from the env dir (`cd &&`); -//! and there is no `--output-last-message`, so the final message is recovered -//! from the stream-json `result` event by the transcript adapter rather than -//! written to a file. `, - agent_model: Option<&str>, -) -> String { - let model_arg = render_cli_model_arg(model_flag, agent_model); - [ - format!( - "cd && claude -p --output-format stream-json --verbose --permission-mode acceptEdits{model_arg} \\" - ), - " \"Read the file at and follow its instructions exactly. When you finish, make your final response your closing summary.\" \\".to_string(), - " /claude-events.jsonl \\".to_string(), - " 2> /claude-stderr.log".to_string(), - ] - .join("\n") -} - -/// Parallel dispatch recipe over `dispatch.json` tasks, one `claude -p` per task. -pub(crate) fn claude_parallel_dispatch_recipe( - model_flag: Option<&str>, - agent_model: Option<&str>, -) -> String { - let model_arg = render_cli_model_arg(model_flag, agent_model); - [ - "JOBS=${JOBS:-4}".to_string(), - "jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + \"\\u0000\"' dispatch.json | \\".to_string(), - " xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(), - " eval_root=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(), - " prompt_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(), - " outputs_dir=\"$(printf \"%s\" \"$1\" | cut -f3)\"".to_string(), - " mkdir -p \"$outputs_dir\"".to_string(), - format!( - " cd \"$eval_root\" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits{model_arg} \\" - ), - " \"Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response your closing summary.\" \\".to_string(), - " \"$outputs_dir/claude-events.jsonl\" \\".to_string(), - " 2> \"$outputs_dir/claude-stderr.log\"".to_string(), - " ' sh {}".to_string(), - ] - .join("\n") -} - -/// Judge dispatch recipe over `judge-tasks.json`, one `claude -p` per task. -/// -/// Judges run from `judge_cwd` (the iteration dir) — a common ancestor of every -/// judge prompt, verdict `response_path`, and agent `outputs_dir`, and a dir with -/// no write-guard hook. -pub(crate) fn claude_judge_dispatch_recipe(model_flag: Option<&str>, judge_cwd: &Path) -> String { - let cwd = judge_cwd.display(); - render_judge_dispatch_recipe( - &format!( - " cd \"{cwd}\" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \\" - ), - model_flag.unwrap_or("--model"), - "claude", - ) -} - -#[cfg(test)] -mod tests { - use super::{ - claude_exec_command_template, claude_judge_dispatch_recipe, claude_parallel_dispatch_recipe, - }; - use std::path::Path; - - #[test] - fn exec_template_carries_required_stream_json_flags() { - let cmd = claude_exec_command_template(Some("--model"), None); - assert!(cmd.contains("claude -p"), "{cmd}"); - assert!(cmd.contains("--output-format stream-json"), "{cmd}"); - // stream-json requires --verbose in -p mode. - assert!(cmd.contains("--verbose"), "{cmd}"); - assert!(cmd.contains("--permission-mode acceptEdits"), "{cmd}"); - assert!(cmd.contains("> /claude-events.jsonl"), "{cmd}"); - assert!(cmd.contains("2> /claude-stderr.log"), "{cmd}"); - assert!(cmd.contains(""), "{cmd}"); - assert!(cmd.contains(""), "{cmd}"); - // claude has no --output-last-message; final text comes from the result event. - assert!(!cmd.contains("--output-last-message"), "{cmd}"); - assert!(!cmd.contains("final-message.md"), "{cmd}"); - } - - #[test] - fn exec_template_includes_model_only_when_declared() { - let with = claude_exec_command_template(Some("--model"), Some("opus")); - assert!(with.contains("--model opus"), "{with}"); - let without = claude_exec_command_template(Some("--model"), None); - assert!(!without.contains("--model "), "{without}"); - } - - #[test] - fn parallel_recipe_drives_claude_p_per_task() { - let recipe = claude_parallel_dispatch_recipe(Some("--model"), Some("sonnet")); - assert!(recipe.contains("claude -p"), "{recipe}"); - assert!(recipe.contains("claude-events.jsonl"), "{recipe}"); - assert!(recipe.contains("dispatch.json"), "{recipe}"); - assert!(recipe.contains("--model sonnet"), "{recipe}"); - } - - #[test] - fn judge_recipe_runs_from_iteration_dir() { - let recipe = claude_judge_dispatch_recipe(Some("--model"), Path::new("/work/iter-1")); - assert!(recipe.contains("claude -p"), "{recipe}"); - assert!(recipe.contains("judge-tasks.json"), "{recipe}"); - assert!(recipe.contains("response_path"), "{recipe}"); - assert!(!recipe.contains(""), "{recipe}"); - assert!(recipe.contains("cd \"/work/iter-1\" &&"), "{recipe}"); - } - - #[test] - fn judge_recipe_snapshot_is_stable() { - // Full-string pin: the judge recipe renders through the shared - // template, and this locks the Claude output byte-for-byte. - let recipe = claude_judge_dispatch_recipe(Some("--model"), Path::new("/work/iter-1")); - let expected = r#"Dispatch each judge task from judge-tasks.json with: - -```bash -JOBS=${JOBS:-4} -jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ - xargs -0 -P "$JOBS" -I{} sh -c ' - prompt_path="$(printf "%s" "$1" | cut -f1)" - response_path="$(printf "%s" "$1" | cut -f2)" - model="$(printf "%s" "$1" | cut -f3)" - response_base="${response_path%.json}" - mkdir -p "$(dirname "$response_path")" - model_arg=""; [ -n "$model" ] && model_arg="--model $model" - cd "/work/iter-1" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \ - "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ - "$response_base.claude-events.jsonl" \ - 2> "$response_base.claude-stderr.log" - ' sh {} -```"#; - assert_eq!(recipe, expected); - } -} diff --git a/src/adapters/claude_code/mod.rs b/src/adapters/claude_code/mod.rs index 7258703..3e855de 100644 --- a/src/adapters/claude_code/mod.rs +++ b/src/adapters/claude_code/mod.rs @@ -1,146 +1,16 @@ //! Claude Code harness support — the default harness. //! -//! Everything Claude-Code-specific lives in this module tree: the adapter impl -//! (this file), dispatch-recipe rendering ([`cli`]), the native skills block -//! ([`session`]), `claude -p` stream-json transcript parsing ([`stream_json`] + +//! The declarative half of this harness lives in `harnesses/claude-code.toml`; +//! this module tree keeps only the code-backed capabilities the descriptor +//! references: `claude -p` stream-json transcript parsing ([`stream_json`] + //! [`transcript`]), plugin-shadow detection ([`plugin_shadow`]), and the //! write-guard hook ([`guard`]). -mod cli; pub(crate) mod guard; pub mod plugin_shadow; -pub mod session; pub mod stream_json; pub mod transcript; -use std::io; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; - -use super::TranscriptSummary; -use super::harness::{ - CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, ToolVocabulary, -}; -use super::skill_shadow::PluginShadowReport; -use cli::{ - claude_exec_command_template, claude_judge_dispatch_recipe, claude_parallel_dispatch_recipe, -}; -use session::render_available_skills_block; -use stream_json::{parse_claude_stream_json, parse_claude_stream_json_full}; - -pub struct ClaudeCodeAdapter; - -impl HarnessAdapter for ClaudeCodeAdapter { - fn label(&self) -> String { - "claude-code".to_string() - } - fn skills_dir(&self, repo_root: &Path) -> PathBuf { - repo_root.join(".claude").join("skills") - } - fn run_capabilities(&self) -> HarnessRunCapabilities { - HarnessRunCapabilities { - supports_guard: true, - supports_bootstrap_with_no_stage: true, - supports_stage_name_with_no_stage: true, - } - } - fn config_dir_names(&self) -> Vec { - vec![".claude".to_string()] - } - fn tool_vocabulary(&self) -> ToolVocabulary { - ToolVocabulary { - write_tools: ["Edit", "MultiEdit", "NotebookEdit", "Write"] - .map(String::from) - .to_vec(), - patch_tools: Vec::new(), - shell_tools: vec!["Bash".to_string()], - read_tools: ["Glob", "Grep", "Read"].map(String::from).to_vec(), - } - } - fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { - render_available_skills_block(skills) - } - fn skill_surface_phrase(&self) -> String { - "via the Skill tool".to_string() - } - fn skill_unresolved_phrase(&self) -> String { - "If the Skill tool cannot resolve that identifier".to_string() - } - fn cli_events_filename(&self) -> Option { - Some("claude-events.jsonl".to_string()) - } - fn cli_model_flag(&self) -> Option { - Some("--model".to_string()) - } - fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { - format!( - "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task (from the env dir — `claude` has no --cd flag) with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness claude-code`.", - claude_exec_command_template(self.cli_model_flag().as_deref(), ctx.agent_model), - target_args = ctx.target_args, - iteration = ctx.iteration - ) - } - fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { - Some(vec![ - "After all dispatches (Claude Code):".to_string(), - String::new(), - "Run one fresh `claude -p` per task from the env dir (`cd ` — `claude` has no --cd flag). `--output-format stream-json` requires `--verbose`; detach stdin with `) -> Option { - Some(claude_judge_dispatch_recipe( - self.cli_model_flag().as_deref(), - ctx.iteration_dir, - )) - } - fn detect_shadowed_skills( - &self, - scan_root: &Path, - staged_skill_names: &[&str], - ) -> Option { - plugin_shadow::shadow_preflight( - &plugin_shadow::config_dir_from_env(), - scan_root, - staged_skill_names, - ) - } - fn parse_cli_events(&self, path: &Path) -> io::Result> { - parse_claude_stream_json(path) - } - fn parse_cli_events_full(&self, path: &Path) -> io::Result { - parse_claude_stream_json_full(path) - } - fn install_guard( - &self, - stage_root: &Path, - guard_exe: &Path, - ttl: Option, - ) -> io::Result { - guard::install_guard(stage_root, guard_exe, ttl) - } - fn guard_armed_message(&self) -> Option { - Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .claude/settings.local.json\n and will block writes/installs outside the eval sandbox during dispatches.\n Each `claude -p` dispatch loads the hook from the env cwd it runs in.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard" - .to_string(), - ) - } -} - #[cfg(test)] mod tests { use crate::adapters::adapter_for; diff --git a/src/adapters/claude_code/session.rs b/src/adapters/claude_code/session.rs deleted file mode 100644 index 37d0534..0000000 --- a/src/adapters/claude_code/session.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Claude Code-specific rendering of session-start context. -//! -//! The available-skills reminder is -//! a *harness-specific* surface: Claude Code presents discoverable skills to an -//! agent as "The following skills are available for use with the Skill tool:" -//! followed by `- name: description` bullets. It lives in an adapter rather than -//! the harness-agnostic orchestrator so a new harness adds its own renderer -//! alongside. - -use crate::core::AvailableSkill; - -/// Render the list of discoverable skills the way a real Claude Code session -/// surfaces them, so an eval dispatch mirrors a genuine session rather than -/// announcing itself as an eval. Returns an empty string when no skills are -/// staged (the caller omits the block entirely in that case). -pub fn render_available_skills_block(skills: &[AvailableSkill]) -> String { - if skills.is_empty() { - return String::new(); - } - let mut sorted: Vec<&AvailableSkill> = skills.iter().collect(); - sorted.sort_by(|a, b| a.name.cmp(&b.name)); - let mut out = String::from("The following skills are available for use with the Skill tool:\n"); - for s in sorted { - out.push_str(&format!("\n- {}: {}", s.name, s.description)); - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::AvailableSkill; - - fn skill(name: &str, description: &str) -> AvailableSkill { - AvailableSkill { - name: name.into(), - path: format!("/x/{name}/SKILL.md"), - description: description.into(), - } - } - - #[test] - fn uses_harness_native_header_and_one_bullet_per_skill() { - let block = render_available_skills_block(&[skill("foo", "the foo skill")]); - assert!(block.contains("The following skills are available for use with the Skill tool:")); - assert!(block.contains("- foo: the foo skill")); - // The eval-flavored wording and custom format must be gone. - assert!(!block.contains("staged and discoverable")); - assert!(!block.contains("*Trigger:*")); - } - - #[test] - fn sorts_skills_by_name() { - let block = render_available_skills_block(&[skill("zebra", "z"), skill("alpha", "a")]); - assert!(block.find("- alpha:").unwrap() < block.find("- zebra:").unwrap()); - } - - #[test] - fn empty_list_renders_empty_string() { - assert_eq!(render_available_skills_block(&[]), ""); - } -} diff --git a/src/adapters/codex/cli.rs b/src/adapters/codex/cli.rs deleted file mode 100644 index 53ab68e..0000000 --- a/src/adapters/codex/cli.rs +++ /dev/null @@ -1,137 +0,0 @@ -//! Codex CLI command rendering (`codex exec`) for dispatch guidance. - -use crate::adapters::cli_command::{render_cli_model_arg, render_judge_dispatch_recipe}; -use std::path::Path; - -/// Copy/pasteable Codex dispatch command template. Stdin is detached so a -/// surrounding `xargs`/pipe cannot be treated as extra prompt context. -pub(crate) fn codex_exec_command_template( - model_flag: Option<&str>, - guard: bool, - agent_model: Option<&str>, -) -> String { - let hook_trust = if guard { - " --dangerously-bypass-hook-trust" - } else { - "" - }; - let model_arg = render_cli_model_arg(model_flag, agent_model); - [ - format!( - "codex --ask-for-approval never exec --cd --sandbox workspace-write{hook_trust}{model_arg} --json \\" - ), - " --output-last-message /final-message.md \\".to_string(), - " \"Read the file at and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to /final-message.md.\" \\".to_string(), - " /codex-events.jsonl \\".to_string(), - " 2> /codex-stderr.log".to_string(), - ] - .join("\n") -} - -pub(crate) fn codex_parallel_dispatch_recipe( - model_flag: Option<&str>, - guard: bool, - agent_model: Option<&str>, -) -> String { - let hook_trust = if guard { - " --dangerously-bypass-hook-trust" - } else { - "" - }; - let model_arg = render_cli_model_arg(model_flag, agent_model); - [ - "JOBS=${JOBS:-4}".to_string(), - "jq -j '.tasks[] | [.eval_root, .dispatch_prompt_path, .outputs_dir] | @tsv + \"\\u0000\"' dispatch.json | \\".to_string(), - " xargs -0 -P \"$JOBS\" -I{} sh -c '".to_string(), - " eval_root=\"$(printf \"%s\" \"$1\" | cut -f1)\"".to_string(), - " prompt_path=\"$(printf \"%s\" \"$1\" | cut -f2)\"".to_string(), - " outputs_dir=\"$(printf \"%s\" \"$1\" | cut -f3)\"".to_string(), - " mkdir -p \"$outputs_dir\"".to_string(), - format!( - " codex --ask-for-approval never exec --cd \"$eval_root\" --sandbox workspace-write{hook_trust}{model_arg} --json \\" - ), - " --output-last-message \"$outputs_dir/final-message.md\" \\".to_string(), - " \"Read the file at $prompt_path and follow its instructions exactly. When you finish, make your final response exactly the same text you wrote to $outputs_dir/final-message.md.\" \\".to_string(), - " \"$outputs_dir/codex-events.jsonl\" \\".to_string(), - " 2> \"$outputs_dir/codex-stderr.log\"".to_string(), - " ' sh {}".to_string(), - ] - .join("\n") -} - -/// Judges run from `judge_cwd` (the iteration dir) — a common ancestor of every -/// judge prompt, verdict `response_path`, and agent `outputs_dir`. -pub(crate) fn codex_judge_dispatch_recipe( - model_flag: Option<&str>, - guard: bool, - judge_cwd: &Path, -) -> String { - let hook_trust = if guard { - " --dangerously-bypass-hook-trust" - } else { - "" - }; - let cwd = judge_cwd.display(); - render_judge_dispatch_recipe( - &format!( - " codex --ask-for-approval never exec --cd \"{cwd}\" --sandbox workspace-write{hook_trust} $model_arg --json \\" - ), - model_flag.unwrap_or("-m"), - "codex", - ) -} - -#[cfg(test)] -mod tests { - use super::{ - codex_exec_command_template, codex_judge_dispatch_recipe, codex_parallel_dispatch_recipe, - }; - use std::path::Path; - - #[test] - fn exec_template_places_approval_policy_before_exec() { - let cmd = codex_exec_command_template(Some("-m"), true, Some("gpt-5-mini")); - let first_line = cmd.lines().next().unwrap(); - - assert_eq!( - first_line, - "codex --ask-for-approval never exec --cd --sandbox workspace-write --dangerously-bypass-hook-trust -m gpt-5-mini --json \\" - ); - } - - #[test] - fn parallel_recipe_places_approval_policy_before_exec() { - let recipe = codex_parallel_dispatch_recipe(Some("-m"), true, Some("gpt-5-mini")); - - assert!( - recipe.contains( - " codex --ask-for-approval never exec --cd \"$eval_root\" --sandbox workspace-write --dangerously-bypass-hook-trust -m gpt-5-mini --json \\" - ), - "{recipe}" - ); - } - - #[test] - fn judge_recipe_places_approval_policy_before_exec() { - let recipe = codex_judge_dispatch_recipe(Some("-m"), true, Path::new("/work/iter-1")); - - // One command shape: the optional model flag is spliced via $model_arg - // (same structure as the Claude judge recipe), not an if/else pair. - assert!( - recipe.contains( - " codex --ask-for-approval never exec --cd \"/work/iter-1\" --sandbox workspace-write --dangerously-bypass-hook-trust $model_arg --json \\" - ), - "{recipe}" - ); - assert!( - recipe.contains(" model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"-m $model\""), - "{recipe}" - ); - assert!(!recipe.contains("if [ -n"), "{recipe}"); - assert!(recipe.contains("codex-events.jsonl"), "{recipe}"); - assert!(recipe.contains("codex-stderr.log"), "{recipe}"); - assert!(!recipe.contains(""), "{recipe}"); - } -} diff --git a/src/adapters/codex/mod.rs b/src/adapters/codex/mod.rs index 408f1f2..d6e8fc7 100644 --- a/src/adapters/codex/mod.rs +++ b/src/adapters/codex/mod.rs @@ -1,150 +1,13 @@ //! Codex harness support. //! -//! Everything Codex-specific lives in this module tree: the adapter impl (this -//! file), `codex exec` dispatch-recipe rendering ([`cli`]), the `## Skills` -//! block ([`session`]), `item.completed` event-stream parsing ([`transcript`]), -//! and the write-guard hook ([`guard`]). +//! The declarative half of this harness lives in `harnesses/codex.toml`; this +//! module tree keeps only the code-backed capabilities the descriptor +//! references: `item.completed` event-stream parsing ([`transcript`]) and the +//! write-guard hook ([`guard`]). -mod cli; pub(crate) mod guard; -pub mod session; pub mod transcript; -use std::io; -use std::path::{Path, PathBuf}; -use std::time::Duration; - -use crate::core::{AvailableSkill, HarnessRunCapabilities, ToolInvocation}; - -use super::TranscriptSummary; -use super::harness::{ - CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, ToolVocabulary, -}; -use cli::{ - codex_exec_command_template, codex_judge_dispatch_recipe, codex_parallel_dispatch_recipe, -}; -use session::render_codex_available_skills_block; -use transcript::{parse_codex_events, parse_codex_events_full}; - -pub struct CodexAdapter; - -impl HarnessAdapter for CodexAdapter { - fn label(&self) -> String { - "codex".to_string() - } - fn skills_dir(&self, repo_root: &Path) -> PathBuf { - repo_root.join(".agents").join("skills") - } - fn run_capabilities(&self) -> HarnessRunCapabilities { - HarnessRunCapabilities { - supports_guard: true, - supports_bootstrap_with_no_stage: false, - supports_stage_name_with_no_stage: false, - } - } - fn config_dir_names(&self) -> Vec { - vec![".agents".to_string(), ".codex".to_string()] - } - fn tool_vocabulary(&self) -> ToolVocabulary { - ToolVocabulary { - // Guard hook payloads carry Claude-style names (see guard::HOOK_MATCHER); - // parsed transcripts carry Codex item types (file_change, command_execution). - write_tools: ["Edit", "Write", "file_change"].map(String::from).to_vec(), - patch_tools: vec!["apply_patch".to_string()], - shell_tools: ["Bash", "command_execution"].map(String::from).to_vec(), - read_tools: Vec::new(), - } - } - fn rewrites_frontmatter_name(&self) -> bool { - true - } - fn advertises_staged_slug_name(&self) -> bool { - true - } - fn render_available_skills_block(&self, skills: &[AvailableSkill]) -> String { - render_codex_available_skills_block(skills) - } - fn skill_surface_phrase(&self) -> String { - "as a Codex skill".to_string() - } - fn skill_unresolved_phrase(&self) -> String { - "If it does not load as a Codex skill".to_string() - } - fn cli_events_filename(&self) -> Option { - Some("codex-events.jsonl".to_string()) - } - fn cli_model_flag(&self) -> Option { - Some("-m".to_string()) - } - fn cli_next_steps(&self, ctx: CliDispatchContext<'_>) -> String { - format!( - "\nNext: iterate the tasks[] array in dispatch.json and dispatch each task with:\n{}\nThen run `ingest{target_args} --iteration {iteration} --harness codex`.", - codex_exec_command_template( - self.cli_model_flag().as_deref(), - ctx.guard, - ctx.agent_model - ), - target_args = ctx.target_args, - iteration = ctx.iteration - ) - } - fn cli_manifest_section(&self, ctx: CliManifestContext<'_>) -> Option> { - Some(vec![ - "After all dispatches (Codex):".to_string(), - String::new(), - "Run one fresh `codex --ask-for-approval never exec --json` per task. Detach stdin with `) -> Option { - Some(codex_judge_dispatch_recipe( - self.cli_model_flag().as_deref(), - ctx.guard, - ctx.iteration_dir, - )) - } - fn parse_cli_events(&self, path: &Path) -> io::Result> { - parse_codex_events(path) - } - fn parse_cli_events_full(&self, path: &Path) -> io::Result { - parse_codex_events_full(path) - } - // Codex's JSONL exposes no deterministic skill-tool event, so the - // `__skill_invoked` meta-check uses the LLM-judge fallback. - fn transcript_surfaces_skill_invocation(&self) -> bool { - false - } - fn install_guard( - &self, - stage_root: &Path, - guard_exe: &Path, - ttl: Option, - ) -> io::Result { - guard::install_guard(stage_root, guard_exe, ttl) - } - fn guard_hook_cleanup_dir(&self, stage_root: &Path) -> Option { - Some(guard::hook_cleanup_dir(stage_root)) - } - fn guard_armed_message(&self) -> Option { - Some( - "\n🛡 Write guard armed: a PreToolUse hook is staged in .codex/hooks.json\n and will block writes/installs outside the eval sandbox during Codex dispatches.\n Dispatch with codex --ask-for-approval never exec --dangerously-bypass-hook-trust so the vetted eval hook runs.\n It auto-expires in 6h and is removed on the next run; to remove it now:\n eval-magic teardown-guard" - .to_string(), - ) - } -} - #[cfg(test)] mod tests { use crate::adapters::adapter_for; diff --git a/src/adapters/codex/session.rs b/src/adapters/codex/session.rs deleted file mode 100644 index e0761a6..0000000 --- a/src/adapters/codex/session.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Codex-specific rendering of the skills surface. -//! -//! Codex exposes skills with a name, -//! description, and file path in its initial skills list. Kept separate from -//! Claude Code's Skill-tool wording so dispatch prompts mirror the harness being -//! evaluated. - -use crate::core::AvailableSkill; - -/// Render the discoverable skills the way Codex surfaces them in its initial -/// skills list: a `## Skills` heading followed by `- name: description (file: -/// path)` bullets. Returns an empty string when no skills are staged. -pub fn render_codex_available_skills_block(skills: &[AvailableSkill]) -> String { - if skills.is_empty() { - return String::new(); - } - let mut sorted: Vec<&AvailableSkill> = skills.iter().collect(); - sorted.sort_by(|a, b| a.name.cmp(&b.name)); - let mut out = String::from("## Skills\n"); - for s in sorted { - out.push_str(&format!( - "\n- {}: {} (file: {})", - s.name, s.description, s.path - )); - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::core::AvailableSkill; - - fn skill(name: &str, description: &str) -> AvailableSkill { - AvailableSkill { - name: name.into(), - path: format!("/x/{name}/SKILL.md"), - description: description.into(), - } - } - - #[test] - fn renders_codex_surface_with_name_description_and_file_path() { - let block = - render_codex_available_skills_block(&[skill("mr-review", "review merge requests")]); - assert!(block.contains("## Skills")); - assert!(block.contains("- mr-review: review merge requests")); - assert!(block.contains("(file: /x/mr-review/SKILL.md)")); - assert!(!block.contains("The following skills are available for use with the Skill tool:")); - } - - #[test] - fn sorts_skills_by_name() { - let block = - render_codex_available_skills_block(&[skill("zebra", "z"), skill("alpha", "a")]); - assert!(block.find("- alpha:").unwrap() < block.find("- zebra:").unwrap()); - } - - #[test] - fn empty_list_renders_empty_string() { - assert_eq!(render_codex_available_skills_block(&[]), ""); - } -} diff --git a/src/adapters/descriptor_adapter.rs b/src/adapters/descriptor_adapter.rs index 9c7e102..d60fb7a 100644 --- a/src/adapters/descriptor_adapter.rs +++ b/src/adapters/descriptor_adapter.rs @@ -331,204 +331,170 @@ impl HarnessAdapter for DescriptorAdapter { #[cfg(test)] mod tests { - use std::path::{Path, PathBuf}; + use std::path::Path; - use super::*; - use crate::adapters::descriptor::{EMBEDDED_DESCRIPTORS, load_descriptor}; - use crate::adapters::harness::{ - CliDispatchContext, CliJudgeContext, CliManifestContext, HarnessAdapter, adapter_for, - }; + use crate::adapters::harness::{CliDispatchContext, CliJudgeContext, adapter_for}; use crate::core::{AvailableSkill, Harness}; - fn descriptor_adapter(harness: Harness) -> DescriptorAdapter { - let index = Harness::ALL - .iter() - .position(|&h| h == harness) - .expect("harness is built in"); - let (source, toml_src) = EMBEDDED_DESCRIPTORS[index]; - DescriptorAdapter::from_descriptor( - load_descriptor(toml_src, source) - .unwrap_or_else(|e| panic!("embedded descriptor {source} is invalid: {e}")), - ) + fn skill(name: &str, description: &str) -> AvailableSkill { + AvailableSkill { + name: name.into(), + path: format!("/x/{name}/SKILL.md"), + description: description.into(), + } + } + + fn next_steps(harness: Harness, agent_model: Option<&str>) -> String { + adapter_for(harness).cli_next_steps(CliDispatchContext { + guard: harness == Harness::Codex, + target_args: " --skill-dir /tmp/skills --skill widget-skill", + iteration: 2, + agent_model, + }) + } + + #[test] + fn exec_recipe_includes_model_only_when_declared() { + let with = next_steps(Harness::ClaudeCode, Some("opus")); + assert!(with.contains("--model opus"), "{with}"); + let without = next_steps(Harness::ClaudeCode, None); + assert!(!without.contains("--model "), "{without}"); + } + + #[test] + fn codex_recipes_gate_hook_trust_on_guard() { + let guarded = next_steps(Harness::Codex, Some("gpt-5-mini")); + assert!( + guarded.contains( + "codex --ask-for-approval never exec --cd --sandbox workspace-write --dangerously-bypass-hook-trust -m gpt-5-mini --json \\" + ), + "{guarded}" + ); + let unguarded = adapter_for(Harness::Codex).cli_next_steps(CliDispatchContext { + guard: false, + target_args: "", + iteration: 2, + agent_model: None, + }); + assert!( + !unguarded.contains("--dangerously-bypass-hook-trust"), + "{unguarded}" + ); + } + + #[test] + fn codex_judge_recipe_splices_model_arg_in_one_command_shape() { + let recipe = adapter_for(Harness::Codex) + .cli_judge_next_steps(CliJudgeContext { + guard: true, + iteration_dir: Path::new("/work/iter-1"), + }) + .expect("codex judge recipe is wired"); + // One command shape: the optional model flag is spliced via $model_arg + // (same structure as the Claude judge recipe), not an if/else pair. + assert!( + recipe.contains( + " codex --ask-for-approval never exec --cd \"/work/iter-1\" --sandbox workspace-write --dangerously-bypass-hook-trust $model_arg --json \\" + ), + "{recipe}" + ); + assert!( + recipe.contains(" model_arg=\"\"; [ -n \"$model\" ] && model_arg=\"-m $model\""), + "{recipe}" + ); + assert!(!recipe.contains("if [ -n"), "{recipe}"); } - fn sample_skills() -> Vec { - vec![ - AvailableSkill { - name: "zeta-skill".to_string(), - path: "/x/zeta-skill/SKILL.md".to_string(), - description: "Does zeta things.".to_string(), - }, - AvailableSkill { - name: "alpha-skill".to_string(), - path: "/x/alpha-skill/SKILL.md".to_string(), - description: "Does alpha things.".to_string(), - }, - ] - } - - /// Transitional equivalence pin for the descriptor cutover: every - /// observable method of the descriptor-backed adapter matches the legacy - /// hand-written adapter, across the guard × model context matrix. Deleted - /// with the legacy adapters once `adapter_for` routes through the - /// registry. #[test] - fn descriptor_adapter_matches_legacy_adapter_observable_surface() { - let root = Path::new("/repo"); - let stage_root = Path::new("/stage"); - let iteration_dir = PathBuf::from("/work/iter-1"); + fn claude_judge_recipe_snapshot_is_stable() { + // Full-string pin carried over from the pre-descriptor adapter: locks + // the Claude judge recipe byte-for-byte through the descriptor path. + let recipe = adapter_for(Harness::ClaudeCode) + .cli_judge_next_steps(CliJudgeContext { + guard: false, + iteration_dir: Path::new("/work/iter-1"), + }) + .expect("claude judge recipe is wired"); + let expected = r#"Dispatch each judge task from judge-tasks.json with: + +```bash +JOBS=${JOBS:-4} +jq -j '.tasks[] | [.dispatch_prompt_path, .response_path, (.model // "")] | @tsv + "\u0000"' judge-tasks.json | \ + xargs -0 -P "$JOBS" -I{} sh -c ' + prompt_path="$(printf "%s" "$1" | cut -f1)" + response_path="$(printf "%s" "$1" | cut -f2)" + model="$(printf "%s" "$1" | cut -f3)" + response_base="${response_path%.json}" + mkdir -p "$(dirname "$response_path")" + model_arg=""; [ -n "$model" ] && model_arg="--model $model" + cd "/work/iter-1" && claude -p --output-format stream-json --verbose --permission-mode acceptEdits $model_arg \ + "Read the file at $prompt_path and follow it exactly. You are a judge worker only: write the JSON verdict to $response_path, then reply with one sentence. Do not run eval-magic. Do not dispatch other judge tasks. Do not wait for other workers." \ + "$response_base.claude-events.jsonl" \ + 2> "$response_base.claude-stderr.log" + ' sh {} +```"#; + assert_eq!(recipe, expected); + } + + #[test] + fn skills_blocks_render_each_harness_native_shape() { + let skills = vec![skill("zebra", "z skill"), skill("alpha", "a skill")]; + + let claude = adapter_for(Harness::ClaudeCode).render_available_skills_block(&skills); + assert!( + claude.starts_with("The following skills are available for use with the Skill tool:"), + "{claude}" + ); + assert!(claude.contains("\n- alpha: a skill"), "{claude}"); + let codex = adapter_for(Harness::Codex).render_available_skills_block(&skills); + assert!(codex.starts_with("## Skills"), "{codex}"); + assert!( + codex.contains("- alpha: a skill (file: /x/alpha/SKILL.md)"), + "{codex}" + ); + + let opencode = adapter_for(Harness::OpenCode).render_available_skills_block(&skills); + assert!(opencode.starts_with(""), "{opencode}"); + assert!(opencode.ends_with("\n"), "{opencode}"); + assert!(opencode.contains("alpha"), "{opencode}"); + assert!( + opencode.contains("a skill"), + "{opencode}" + ); + + // Sorted by name in every shape, and empty renders empty. for harness in Harness::ALL { - let legacy = adapter_for(harness); - let new = descriptor_adapter(harness); - - assert_eq!(new.label(), legacy.label()); - assert_eq!(new.skills_dir(root), legacy.skills_dir(root)); - assert_eq!(new.run_capabilities(), legacy.run_capabilities()); - assert_eq!(new.config_dir_names(), legacy.config_dir_names()); - assert_eq!(new.tool_vocabulary(), legacy.tool_vocabulary()); - assert_eq!( - new.rewrites_frontmatter_name(), - legacy.rewrites_frontmatter_name() - ); - assert_eq!( - new.advertises_staged_slug_name(), - legacy.advertises_staged_slug_name() - ); - assert_eq!(new.skill_surface_phrase(), legacy.skill_surface_phrase()); - assert_eq!( - new.skill_unresolved_phrase(), - legacy.skill_unresolved_phrase() + let block = adapter_for(harness).render_available_skills_block(&skills); + assert!( + block.find("alpha").unwrap() < block.find("zebra").unwrap(), + "{harness:?} sorts by name: {block}" ); - assert_eq!(new.cli_events_filename(), legacy.cli_events_filename()); - assert_eq!(new.cli_model_flag(), legacy.cli_model_flag()); - assert_eq!(new.guard_armed_message(), legacy.guard_armed_message()); - assert_eq!( - new.transcript_surfaces_skill_invocation(), - legacy.transcript_surfaces_skill_invocation() - ); - assert_eq!( - new.guard_hook_cleanup_dir(stage_root), - legacy.guard_hook_cleanup_dir(stage_root) - ); - assert_eq!( - new.render_plan_mode_context(" PLAN\n"), - legacy.render_plan_mode_context(" PLAN\n") - ); - assert_eq!(new.render_plan_mode_context(" "), ""); - - // Staging: generated slugs and naming-rule verdicts. - for (prefix, iteration, condition, skill) in [ - ("slow-powers-eval-", 2, "with_skill", "my-skill"), - ("slow-powers-eval-", 7, "no_skill", "Very_Loud.Skill"), - ( - "slow-powers-eval-", - 1, - "with_skill", - "a-skill-name-clearly-engineered-to-overflow-opencodes-limit-of-sixty-four", - ), - ] { - assert_eq!( - new.staged_slug(prefix, iteration, condition, skill), - legacy.staged_slug(prefix, iteration, condition, skill), - "{harness:?} slug for {condition}/{skill}" - ); - } - for name in [ - "valid-name", - "slow-powers-eval-2-with-skill-my-skill", - "Invalid_Name", - "double--hyphen", - "-leading", - ] { - assert_eq!( - new.validate_stage_name(name), - legacy.validate_stage_name(name), - "{harness:?} stage-name verdict for {name}" - ); - } + assert_eq!(adapter_for(harness).render_available_skills_block(&[]), ""); + } + } - // Skills block: native shape, sortedness, and the empty guard. - assert_eq!( - new.render_available_skills_block(&sample_skills()), - legacy.render_available_skills_block(&sample_skills()) + #[test] + fn opencode_stage_name_rules_match_the_old_validator() { + let adapter = adapter_for(Harness::OpenCode); + assert!(adapter.validate_stage_name("valid-name-2").is_ok()); + for invalid in [ + "Invalid_Name", + "double--hyphen", + "-leading", + "trailing-", + "", + ] { + let err = adapter + .validate_stage_name(invalid) + .expect_err("name should be rejected"); + assert!( + err.contains(&format!("OpenCode skill name \"{invalid}\" is invalid")), + "{err}" ); - assert_eq!(new.render_available_skills_block(&[]), ""); - - // Dispatch recipes across the guard × model matrix. - for guard in [false, true] { - for agent_model in [None, Some("model-x")] { - let ctx = CliDispatchContext { - guard, - target_args: " --skill-dir /tmp/skills --skill widget-skill", - iteration: 2, - agent_model, - }; - assert_eq!( - new.cli_next_steps(ctx), - legacy.cli_next_steps(CliDispatchContext { - guard, - target_args: " --skill-dir /tmp/skills --skill widget-skill", - iteration: 2, - agent_model, - }), - "{harness:?} next steps guard={guard} model={agent_model:?}" - ); - // The manifest consumer joins the lines with '\n' - // (build_manifest), so equivalence holds on the joined - // text — the descriptor path splits multi-line commands - // into more elements than the legacy hand-built vec. - assert_eq!( - new.cli_manifest_section(CliManifestContext { guard, agent_model }) - .map(|lines| lines.join("\n")), - legacy - .cli_manifest_section(CliManifestContext { guard, agent_model }) - .map(|lines| lines.join("\n")), - "{harness:?} manifest guard={guard} model={agent_model:?}" - ); - } - assert_eq!( - new.cli_judge_next_steps(CliJudgeContext { - guard, - iteration_dir: &iteration_dir, - }), - legacy.cli_judge_next_steps(CliJudgeContext { - guard, - iteration_dir: &iteration_dir, - }), - "{harness:?} judge recipe guard={guard}" - ); - } - - // Unsupported-enhancement errors carry the same kind and message. - if new.cli_events_filename().is_none() { - let new_err = new.parse_cli_events(Path::new("/nope")).unwrap_err(); - let legacy_err = legacy.parse_cli_events(Path::new("/nope")).unwrap_err(); - assert_eq!(new_err.kind(), legacy_err.kind()); - assert_eq!(new_err.to_string(), legacy_err.to_string()); - let new_err = new.parse_cli_events_full(Path::new("/nope")).unwrap_err(); - let legacy_err = legacy - .parse_cli_events_full(Path::new("/nope")) - .unwrap_err(); - assert_eq!(new_err.to_string(), legacy_err.to_string()); - } - if !new.run_capabilities().supports_guard { - let new_err = new - .install_guard(stage_root, Path::new("/bin/guard"), None) - .unwrap_err(); - let legacy_err = legacy - .install_guard(stage_root, Path::new("/bin/guard"), None) - .unwrap_err(); - assert_eq!(new_err.kind(), legacy_err.kind()); - assert_eq!(new_err.to_string(), legacy_err.to_string()); - } - - // Shadow preflight: only Claude Code wires one; on a bare scan - // root both sides agree (None for the others by construction). - if harness != Harness::ClaudeCode { - assert!(new.detect_shadowed_skills(root, &["x"]).is_none()); - assert!(legacy.detect_shadowed_skills(root, &["x"]).is_none()); - } } + assert!(adapter.validate_stage_name(&"a".repeat(64)).is_ok()); + assert!(adapter.validate_stage_name(&"a".repeat(65)).is_err()); } } diff --git a/src/adapters/harness.rs b/src/adapters/harness.rs index 26c1ada..eec839c 100644 --- a/src/adapters/harness.rs +++ b/src/adapters/harness.rs @@ -30,6 +30,8 @@ use std::time::Duration; use crate::core::{AvailableSkill, Harness, HarnessRunCapabilities, ToolInvocation}; use super::TranscriptSummary; +use super::descriptor::{EMBEDDED_DESCRIPTORS, load_descriptor}; +use super::descriptor_adapter::DescriptorAdapter; use super::skill_shadow::PluginShadowReport; /// One harness's tool-name vocabulary: every name its guard hook payloads or @@ -354,14 +356,50 @@ pub struct CliJudgeContext<'a> { pub iteration_dir: &'a Path, } +/// The built-in adapters, one descriptor-backed adapter per [`Harness`] +/// variant, loaded once from the embedded descriptor files. +struct Registry { + claude_code: DescriptorAdapter, + codex: DescriptorAdapter, + opencode: DescriptorAdapter, +} + +/// Built on first `adapter_for` call. The embedded descriptors are bundled and +/// known-valid, so a load failure here is a programmer error (a bad descriptor +/// edit) and panics — mirroring the bundled-schema panics in +/// `validation::schema` — with the descriptor's own actionable message. +static REGISTRY: LazyLock = LazyLock::new(|| Registry { + claude_code: build_embedded(Harness::ClaudeCode, EMBEDDED_DESCRIPTORS[0]), + codex: build_embedded(Harness::Codex, EMBEDDED_DESCRIPTORS[1]), + opencode: build_embedded(Harness::OpenCode, EMBEDDED_DESCRIPTORS[2]), +}); + +fn build_embedded(harness: Harness, (source, toml_src): (&str, &str)) -> DescriptorAdapter { + let descriptor = load_descriptor(toml_src, source) + .unwrap_or_else(|e| panic!("bundled harness descriptor is invalid: {e}")); + // Label ↔ variant lockstep: the descriptor's label must be the variant's + // serde/CLI identifier, or `--harness