diff --git a/Cargo.lock b/Cargo.lock index 851c6b0..149d274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1047,6 +1047,7 @@ dependencies = [ "serde_spanned", "toml_datetime", "toml_parser", + "toml_writer", "winnow 0.7.15", ] @@ -1068,6 +1069,12 @@ dependencies = [ "winnow 1.0.3", ] +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "unicode-general-category" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 7e10e34..0e74a50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,9 +44,10 @@ regex = { version = "1.12.3", default-features = false, features = ["std", "perf serde = { version = "1.0.228", features = ["derive"] } serde_json = { version = "1.0.150", features = ["preserve_order"] } thiserror = "2.0.18" -# Harness descriptor files (harnesses/*.toml). Parse-only: descriptors are -# embedded and read, never written, so `display` is dropped. -toml = { version = "0.9", default-features = false, features = ["parse", "serde"] } +# Harness descriptor files (harnesses/*.toml). `display` serializes the +# resolved (layer-merged) descriptor back to authorable TOML for +# `harness show`. +toml = { version = "0.9", default-features = false, features = ["parse", "serde", "display"] } walkdir = "2.5.0" [dev-dependencies] diff --git a/README.md b/README.md index e220990..4217faa 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,8 @@ For the `without_skill` / baseline condition, the dispatch reflects "this skill Every artifact follows a JSON Schema in [`schema/`](schema/), so a run record means the same thing regardless of which harness produced it. Harness support is **a minimal baseline plus progressive enhancements**: any harness with a headless one-shot CLI can run evals at baseline (`--no-stage` inlines the skill into each dispatch prompt, `llm_judge` grades, `detect-stray-writes` audits), and each enhancement a harness's adapter wires raises fidelity from there. What each enhancement is, why it needs harness-specific support, and what its fallback is are documented in [docs/progressive-enhancements.md](docs/progressive-enhancements.md). +**Bring your own harness:** a harness eval-magic has never seen is added with a TOML descriptor file — no code. Descriptors layer (embedded built-ins → `~/.config/eval-magic/harnesses/` → `.eval-magic/harnesses/` → `--harness-file`) with field-level merge, so a project can also retune a single field of a built-in. `eval-magic harness list|show|lint` inspect and validate the registry; the authoring guide is [docs/byoh.md](docs/byoh.md). + ### How dispatch works Every eval test and judge is dispatched the same way: through the harness's one-shot CLI (`claude -p`, `codex exec`), one subprocess per task, each `cd`'d into its `(group, condition)` env and writing its events transcript to disk. `run` prepares the envs and `RUNBOOK.md`; from there an **agent session** can drive the loop (reading the runbook and shelling out each recipe) or a **human** can follow the same runbook by hand. The generated `RUNBOOK.md` / `dispatch-manifest.md` carry the exact per-task recipes for the selected harness — they, not this README, are the runtime reference for dispatch commands. @@ -272,7 +274,7 @@ This table is the source of truth for per-harness enhancement support: ¹ `run --harness opencode` stages skills and emits native dispatch prompts, but prints manual `opencode run` guidance instead of a copy-pasteable recipe. -A missing enhancement degrades fidelity, never correctness — every column has a fallback: without native staging, `--no-stage` inlines each `SKILL.md` into its dispatch prompt; without transcript ingest, `transcript_check` assertions grade as unverifiable and `llm_judge` carries the grading (tokens and duration go unrecorded); without a model flag, `--agent-model` / `--judge-model` are recorded as provenance only; without a write guard, `--guard` is rejected and `detect-stray-writes` audits after the fact. +A missing enhancement degrades fidelity, never correctness — every column has a fallback, and the `run` preflight warns naming it: without native staging, `--no-stage` inlines each `SKILL.md` into its dispatch prompt; without transcript ingest, `transcript_check` assertions grade as unverifiable and `llm_judge` carries the grading (tokens and duration go unrecorded); without a model flag, `--agent-model` / `--judge-model` are recorded as provenance only; without a write guard, `--guard` continues unguarded and `detect-stray-writes` audits after the fact. Per-harness implementation notes for developers wiring features live in [docs/claude-notes.md](docs/claude-notes.md), [docs/codex-notes.md](docs/codex-notes.md), and [docs/opencode-notes.md](docs/opencode-notes.md). @@ -281,6 +283,7 @@ Per-harness implementation notes for developers wiring features live in [docs/cl | Where | What's in it | |-------|--------------| | `eval-magic --help` / `eval-magic --help` | The flag-by-flag reference: every subcommand and flag, worked examples, the `--skill-dir` model, the skill-invocation meta-check | +| [docs/byoh.md](docs/byoh.md) | Bring your own harness: authoring a descriptor file for an unknown harness, layering/merge rules, named capabilities, and the `harness list`/`show`/`lint` workflow | | [docs/progressive-enhancements.md](docs/progressive-enhancements.md) | Development doc: the harness baseline-vs-enhancement contract — what each enhancement unlocks, why it needs harness-specific code, and its fallback | | [docs/claude-notes.md](docs/claude-notes.md) / [docs/codex-notes.md](docs/codex-notes.md) / [docs/opencode-notes.md](docs/opencode-notes.md) | Development docs: per-harness implementation notes for working on eval-magic's harness support | | [GitHub issues](https://github.com/slowdini/eval-magic/issues) | Planned features and known limitations | diff --git a/docs/byoh.md b/docs/byoh.md new file mode 100644 index 0000000..e762709 --- /dev/null +++ b/docs/byoh.md @@ -0,0 +1,173 @@ +# Bring your own harness + +> **Audience:** anyone (human or agent) pointing eval-magic at a harness it has never seen. No Rust +> required — a harness declares its capabilities in a TOML descriptor file, and every capability it +> *doesn't* declare degrades to a documented fallback with a warning, never a rejection. The +> baseline-vs-enhancement contract behind this is +> [progressive-enhancements.md](progressive-enhancements.md). + +## The five-minute version + +From inside a project that dispatches through `cool-custom-harness`: + +```sh +mkdir -p .eval-magic/harnesses +cat > .eval-magic/harnesses/cool-custom-harness.toml <<'EOF' +label = "cool-custom-harness" + +[dispatch] +exec_template = ''' +cool-cli run --cd {model_arg} \ + "Read the file at and follow its instructions exactly." \ + > /final-message.md''' +EOF + +eval-magic harness lint .eval-magic/harnesses/cool-custom-harness.toml +eval-magic harness list +eval-magic run --harness cool-custom-harness +``` + +That run is complete: `run` warns about each enhancement the descriptor doesn't declare (naming the +fallback that carries it), builds `dispatch.json` / `RUNBOOK.md` / `dispatch-manifest.md` with your +exec recipe, and after you dispatch the tasks, `ingest` assembles records from each task's +`outputs/final-message.md`, runs the `detect-stray-writes` audit, and hands grading to `llm_judge`. + +For a one-off descriptor that shouldn't live in the project, pass it directly — its label becomes +the invocation's default harness, so `--harness` can be omitted: + +```sh +eval-magic run --harness-file ./cool-custom-harness.toml +``` + +## The minimum viable descriptor + +`label` alone loads: + +```toml +label = "cool-custom-harness" +``` + +That is a **baseline** harness: staging is unavailable (runs fall back to `--no-stage`, inlining +each `SKILL.md` into its dispatch prompt), and dispatch guidance is generic. Add an +`[dispatch] exec_template` and the generated `RUNBOOK.md` / `dispatch-manifest.md` carry a +copy-pasteable per-task recipe instead. + +### The exec-template contract + +`exec_template` is the one-shot command that dispatches a single task. eval-magic substitutes +`{model_arg}` (the harness's model flag + requested model, empty unless `[model].flag` is declared) +and `{guard_args}` (guard-only arguments, built-ins only); the ``, +``, and `` placeholders are filled per task by whoever dispatches +(the runbook explains this). Two hard requirements: + +1. **Run from the task's `eval_root`** — each task's per-`(group, condition)` env dir is the + subprocess cwd. +2. **Recover the final message** — the agent's final reply must land in + `outputs/final-message.md`. If your CLI can't write it directly (like the example above + redirecting stdout), capture it yourself before running `ingest`. + +## Layering and field-level merge + +Descriptor files stack in precedence order; **later layers override individual fields** of an +earlier descriptor with the same `label` (tables merge key-by-key; scalars and arrays replace +wholesale — never whole-file shadowing). A new `label` defines a new harness. + +1. **Embedded built-ins** — `claude-code`, `codex`, `opencode` (bundled in the binary). +2. **User-global** — `/harnesses/*.toml`, where the config root is + `$EVAL_MAGIC_CONFIG_DIR` (empty value disables the layer), else `$XDG_CONFIG_HOME/eval-magic`, + else `~/.config/eval-magic`. +3. **Project-local** — `/.eval-magic/harnesses/*.toml`. +4. **`--harness-file `** — a one-off top layer; when `--harness` is omitted its label is the + invocation's default harness. + +Files within one directory apply in filename order; one file per label per layer (a duplicate is +skipped with a warning). So a project can retune a single built-in field without restating the +descriptor: + +```toml +# .eval-magic/harnesses/claude-code.toml — override one field of a built-in +label = "claude-code" + +[model] +flag = "--model-x" +``` + +`eval-magic harness show claude-code` prints the resolved post-merge descriptor (as authorable +TOML, headed by the contributing files) — the fastest way to see what a layer actually changed. +Two caveats: the merge can't *delete* (an override can replace a field's value but cannot remove +an embedded table), and a guarded built-in's output isn't copy-paste-safe as a user layer until +you drop its `[guard]` table and `run.supports_guard` (see the guard restriction below). + +**Broken discovered files never brick the CLI** — they are skipped with a warning pointing at +`eval-magic harness lint `. A broken `--harness-file` is a hard error (you named it +explicitly). + +## Field reference + +The authoritative field-by-field reference is the bundled schema, +[`schema/harness-descriptor.schema.json`](../schema/harness-descriptor.schema.json); the semantics +and fallbacks per enhancement are in +[progressive-enhancements.md](progressive-enhancements.md). The short map: + +| Table | Declares | Fallback when absent | +|-------|----------|----------------------| +| (top level) | `label` (required), `skills_dir`, `config_dirs` | no `skills_dir` ⇒ forced `--no-stage`, SKILL.md inlined | +| `[dispatch]` | exec/parallel/judge/next-steps/manifest templates | generic handoff text; with only `exec_template`, generic recipes are built around it | +| `[transcript]` | `events_filename` + `parser` (a named capability) | `transcript_check` grades unverifiable, `llm_judge` carries grading, tokens/duration unrecorded | +| `[model]` | `flag` | `--agent-model`/`--judge-model` recorded as provenance only | +| `[staging]` + `[skills_block]` | slug/naming rules, skills-block format | `--no-stage` inlining | +| `[tools]` | tool-name vocabulary by role | required alongside `[transcript]` (the stray-writes audit classifies by it) | +| `[shadow]` | `preflight` (named capability) | no shadow report — correct for harnesses that load nothing global | +| `[guard]` | **built-ins only** — see below | `detect-stray-writes` audits after the fact | + +### Named capabilities: real code for free + +Everything that is genuinely code — transcript stitching, guard hooks, slug sanitization, shadow +scanning — is a **named capability** a descriptor references. If your harness emits a compatible +stream, you get the full feature from configuration alone: + +- `transcript.parser = "claude-stream-json"` — Claude Code `-p --output-format stream-json` events. +- `transcript.parser = "codex-items"` — Codex `item.started`/`item.completed` JSONL. +- `staging.slug_capability = "opencode"` — OpenCode's sanitizing slug rules. +- `shadow.preflight = "claude-plugins"` — the Claude plugin/global-skills shadow scan. + +For example, a harness that logs Codex-compatible item JSONL gets full transcript ingest — parsed +tool invocations, `transcript_check` grading, the works — with: + +```toml +[tools] +write = ["file_change"] +shell = ["command_execution"] + +[transcript] +events_filename = "cool-events.jsonl" +parser = "codex-items" +``` + +An unknown capability name fails the schema gate listing the allowed values. + +### The guard restriction + +User-supplied descriptors may **not** declare `[guard]` or set `run.supports_guard = true`: the +write guard installs native hook config into dispatch environments and stays restricted to +built-in descriptors until the guard engine is opened up (fail-open safety). Unguarded runs fall +back to the `detect-stray-writes` audit. A project-local overlay of a guarded built-in is fine — +the restriction applies to the user file's own content, and the embedded guard merges through +underneath it. + +## The workflow + +```sh +eval-magic harness lint # full per-file report: TOML + schema, user-layer + # restrictions, cross-field invariants (merged onto the + # registered harness with the same label, if any) +eval-magic harness lint # strictly re-lint every discovered layer of a name — + # surfaces files that registry init skipped with a warning +eval-magic harness list # every registered harness: layers + declared enhancements +eval-magic harness show # the resolved post-merge descriptor as authorable TOML +``` + +Then `run --harness ` and read the warnings: each one names the fallback carrying that part +of the run, which doubles as your wiring roadmap — declare the enhancement and the warning +disappears. (A `harness lint --probe` live dispatch check — rendering the exec template with a +trivial prompt and verifying final-message recovery — is planned; see the tracking issue.) diff --git a/docs/progressive-enhancements.md b/docs/progressive-enhancements.md index d34e49c..25acf8f 100644 --- a/docs/progressive-enhancements.md +++ b/docs/progressive-enhancements.md @@ -3,7 +3,9 @@ > **Audience:** developers working on the eval-magic codebase. The README and `eval-magic --help` > are the user-facing docs; this file explains how harness support is structured in code and what > wiring more of it buys. Per-harness implementation notes live in [claude-notes.md](claude-notes.md), -> [codex-notes.md](codex-notes.md), and [opencode-notes.md](opencode-notes.md). +> [codex-notes.md](codex-notes.md), and [opencode-notes.md](opencode-notes.md). Pointing eval-magic +> at a harness it doesn't know — user-supplied descriptor files, layering, `harness +> list`/`show`/`lint` — is the user-facing [byoh.md](byoh.md). Harness compatibility is not a parity checklist to audit — it is **a minimal baseline every harness satisfies, plus optional enhancements** a harness's adapter opts into. A missing enhancement @@ -34,9 +36,12 @@ That baseline already yields a working eval: `llm_judge` assertions grade every run records exist. Run records without a transcript parser are assembled by hand per `schema/run-record.schema.json`. -In descriptor terms the baseline is two required fields: `label` and `skills_dir`. Everything else -in a harness descriptor is optional — an absent field or table gets a working generic fallback or -an `Unsupported` error naming the enhancement it belongs to. +In descriptor terms the baseline is one required field: `label`. Everything else in a harness +descriptor is optional — an absent field or table gets a working generic fallback, and the `run` +preflight *warns* naming that fallback rather than rejecting (a harness without `skills_dir` +forces `--no-stage`; `--guard` without a declared guard continues unguarded behind the +`detect-stray-writes` audit; requested models without a model flag are recorded as provenance +only). Only genuinely contradictory flag combinations stay errors. ## Where this lives in code @@ -44,20 +49,30 @@ an `Unsupported` error naming the enhancement it belongs to. declarative value (labels, dirs, capability booleans, phrases, command templates, banner prose). Descriptors are schema-gated (`schema/harness-descriptor.schema.json`) and invariant-checked at load time. +- `src/adapters/registry.rs` — the label-keyed registry: `init_registry` loads every **layer** + (embedded → user-global → project-local → `--harness-file`) with field-level merge per label, + and `adapter_for()` resolves each `Harness` handle to its descriptor-backed adapter — still the + single dispatch point. Broken discovered files skip with a warning; the layer model itself is + documented in [byoh.md](byoh.md). +- `src/adapters/descriptor/layers.rs` — layer discovery (config-root resolution, per-directory + scans, the `--harness-file` top layer) and the user-layer restrictions (no `[guard]`). - `src/adapters/harness.rs` — the `HarnessAdapter` trait, tiered into baseline and enhancement - sections; `adapter_for()` resolves each `Harness` variant to a descriptor-backed adapter in the - embedded registry — still the single place a concrete harness is named. -- `src/adapters/descriptor.rs` — the descriptor model, TOML loading, and `validate_descriptor` - (the load-time invariants); `src/adapters/descriptor_adapter.rs` — the one generic - `DescriptorAdapter` implementing the trait from a descriptor. + sections. +- `src/adapters/descriptor.rs` — the descriptor model, the per-file schema gate + (`parse_descriptor_value`), field-level merge (`merge_descriptor_value`), and + `finalize_descriptor` (the load-time invariants, with merged-file provenance in every message); + `src/adapters/descriptor_adapter.rs` — the one generic `DescriptorAdapter` implementing the + trait from a descriptor. - `src/adapters/capabilities.rs` — the **named capabilities**: closed enums a descriptor references by kebab-case name for everything that is real code (transcript parsers, guard engines, slug generation, shadow preflight). - `src/adapters//` — only the code behind those capabilities: transcript parsers, guard hooks, the plugin-shadow scan, the OpenCode slug sanitizer. -- `run_capabilities()` (descriptor table `[run]`) — the narrow table the `run` preflight uses to - accept or reject run options (`--guard`, `--bootstrap`/`--stage-name` with `--no-stage`) per - harness. +- `run_capabilities()` (descriptor table `[run]`) + `harness_run_preflight()` + (`src/cli/run/util.rs`) — the `run` preflight: undeclared enhancements warn naming their + fallback and adjust the options (`--guard` forced off, missing `skills_dir` forces + `--no-stage`); only contradictory flag combinations (`--bootstrap`/`--stage-name` where the + descriptor declares them incompatible with `--no-stage`) reject. ## The enhancements @@ -141,7 +156,9 @@ The guard arbiter and `detect-stray-writes` classify tool names against the cros vocabulary union (`all_tool_vocabulary`), so wiring a guard or transcript parser without declaring the harness's tool names is rejected at descriptor load. The hidden `guard` / `guard-codex` subcommands are the hook entry points — their names are a stable on-disk contract. Shared -marker/manifest/teardown machinery lives in `src/sandbox/`. +marker/manifest/teardown machinery lives in `src/sandbox/`. **User-supplied descriptors may not +declare `[guard]`** (fail-open safety — the engine installs native hook config); the restriction +lifts when the guard engine is opened up. ### Shadow preflight @@ -195,18 +212,20 @@ enhancement — keep it in sync with the adapters when wiring or dropping one. ## Adding a new harness -1. Add the variant to `Harness` in `src/core/context.rs` (it derives `clap::ValueEnum`). -2. Create `harnesses/