feat(evals): eval harness + hot-swappable prompt optimizer — 2.7.1

- devsper/evals/: EvalCase, EvalDataset (JSONL), EvalRunner (async,
  bounded concurrency), EvalSummary, MetricFn. Built-in metrics:
  exact_match, contains, regex_match, word_overlap, non_empty, llm_judge.
- openevals_metric() adapter: get_metric("openevals:correctness|groundedness")
  wraps OpenEvals LLM-as-judge evaluators; falls back to llm_judge.
- PromptOptimizerBackend ABC (noop/dspy/gepa) — hot-swappable via
  DEVSPER_PROMPT_OPTIMIZER env or [prompt_optimizer] provider config.
- DSPy backend: BootstrapFewShot / MIPROv2 / BootstrapFewShotWithRandomSearch.
- GEPA backend: evolutionary optimizer; built-in fallback loop works
  without the gepa package.
- devsper eval CLI: eval stub, eval run (--optimize, --optimizer),
  eval results.
- [prompt_optimizer] and [evals] config sections in devsperConfigModel.
- get_role_config() auto-loads .devsper/optimized_prompts/{role}.json.
- devsper[dspy], devsper[gepa], devsper[openevals], devsper[evals] extras.
- 44 new tests (26 unit + 18 integration, 2 live-gated).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: rithulkamesh

CHANGELOG.md

@@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [2.7.1] — 2026-04-07
+
+### Added
+
+- **Eval harness** — New `devsper/evals/` module: `EvalCase`, `EvalDataset` (JSONL-backed), `EvalRunner` (async, bounded concurrency), `EvalSummary`, and `MetricFn` protocol. Built-in metrics: `exact_match`, `contains`, `regex_match`, `word_overlap`, `non_empty`, `llm_judge(model=)`.
+- **Prompt optimizer abstraction** — `PromptOptimizerBackend` ABC mirroring the `MemoryBackend` / `LLMBackend` hot-swap pattern. Active backend resolved from `DEVSPER_PROMPT_OPTIMIZER` env var → `[prompt_optimizer] provider` config → default `"noop"`.
+- **DSPy backend** (`devsper[dspy]`) — `DSPyBackend` compiles few-shot examples into an optimized system prompt using `BootstrapFewShot`, `MIPROv2`, or `BootstrapFewShotWithRandomSearch`. Configured via `[prompt_optimizer] dspy_optimizer`, `max_demos`, `num_candidates`.
+- **GEPA backend** (`devsper[gepa]`) — `GEPABackend` runs an evolutionary prompt optimization loop. Uses the `gepa` library when installed; falls back to a built-in LLM-driven mutation loop so it works without any extra dependencies.
+- **OpenEvals metric adapter** (`devsper[openevals]`) — `openevals_metric(name)` wraps any OpenEvals LLM-as-judge evaluator as a `MetricFn`. Prebuilt names: `correctness`, `conciseness`, `groundedness`, `relevance`. Accessible via `get_metric("openevals:correctness")`. Falls back to built-in `llm_judge` when the package is absent.
+- **`devsper eval` CLI** — Three subcommands: `eval stub` (generate JSONL stub datasets per role), `eval run` (score a dataset, optional `--optimize` flag, `--optimizer dspy|gepa`), `eval results` (list persisted result files).
+- **Config sections** — `[prompt_optimizer]` and `[evals]` added to `devsperConfigModel`.
+- **Optimized prompt persistence** — `EvalRunner` saves results to `.devsper/optimized_prompts/{role}.json`. `get_role_config()` auto-loads these on the next run.
+- **`devsper[evals]`, `devsper[dspy]`, `devsper[gepa]`, `devsper[openevals]` extras.**
+- **pytest `live` marker** — Gates tests that require real API keys.
+
+### Changed
+
+- **`get_metric(name)`** extended: now accepts `"openevals:<evaluator>"` in addition to built-in names.
+- **`get_role_config()`** — Checks `.devsper/optimized_prompts/{role}.json` and uses the optimized prefix if present.
+
 ## [2.7.0] — 2026-04-07
 
 ### Added

devsper/agents/roles.py

@@ -105,10 +105,34 @@ class RoleConfig:
 
 
 def get_role_config(role: str | None) -> RoleConfig:
-    """Return config for the given role, or default if unknown/None."""
-    if role and role in ROLE_CONFIGS:
-        return ROLE_CONFIGS[role]
-    return ROLE_CONFIGS[DEFAULT_ROLE]
+    """Return config for the given role, or default if unknown/None.
+
+    If an optimized prompt exists at .devsper/optimized_prompts/{role}.json
+    (written by EvalRunner after prompt optimization), it is used in place of
+    the hardcoded prompt_prefix.
+    """
+    base = ROLE_CONFIGS.get(role or DEFAULT_ROLE, ROLE_CONFIGS[DEFAULT_ROLE]) if role else ROLE_CONFIGS[DEFAULT_ROLE]
+    optimized = _load_optimized_prompt(base.name)
+    if optimized:
+        from dataclasses import replace
+        return replace(base, prompt_prefix=optimized)
+    return base
+
+
+def _load_optimized_prompt(role: str) -> str | None:
+    """Load persisted optimized prompt for a role, if it exists."""
+    import json
+    from pathlib import Path
+
+    p = Path(".devsper/optimized_prompts") / f"{role}.json"
+    if not p.exists():
+        return None
+    try:
+        data = json.loads(p.read_text())
+        prompt = data.get("prompt_prefix", "")
+        return prompt if prompt else None
+    except Exception:
+        return None
 
 
 # Keywords to infer role from task description

devsper/cli/main.py

@@ -145,7 +145,7 @@ def _run_platform_once(task: str, args: object) -> tuple[int, dict, float]:
     api = _platform_api_builder()
     if not api.enabled():
         print(
-            "Platform routing is not configured. Run `devsper platform connect` first.",
+            "Cloud routing is not configured. Run `devsper platform connect` first.",
             file=sys.stderr,
         )
         return 2, {}, 0.0
@@ -2721,6 +2721,164 @@ def _run_version(args: object) -> int:
     return 0
 
 
+def _run_eval(args: object) -> int:
+    """Eval harness: run dataset, score results, optionally optimize prompts."""
+    import asyncio
+    import json
+    from pathlib import Path
+
+    eval_cmd = getattr(args, "eval_cmd", None)
+
+    if eval_cmd == "stub" or eval_cmd is None and not hasattr(args, "dataset"):
+        # Generate stub dataset
+        from devsper.evals.dataset import EvalDataset
+
+        role = getattr(args, "role", "general")
+        n = getattr(args, "n", 5)
+        out = getattr(args, "out", None)
+        dataset = EvalDataset.stub(role=role, n=n)
+        if out:
+            dataset.save(out)
+            print(f"Stub dataset ({len(dataset)} cases) written to {out}")
+        else:
+            for case in dataset:
+                print(json.dumps(case.to_dict()))
+        return 0
+
+    if eval_cmd == "results":
+        from devsper.config import get_config
+
+        try:
+            results_dir = Path(getattr(args, "dir", None) or get_config().evals.results_dir)
+        except Exception:
+            results_dir = Path(".devsper/eval_results")
+        if not results_dir.exists():
+            print(f"No results found in {results_dir}")
+            return 0
+        files = sorted(results_dir.glob("*.json"), key=lambda p: p.stat().st_mtime, reverse=True)
+        if not files:
+            print("No eval result files found.")
+            return 0
+        for f in files[:20]:
+            try:
+                data = json.loads(f.read_text())
+                print(
+                    f"  {f.name}  role={data.get('role', '?')}  "
+                    f"pass_rate={data.get('pass_rate', '?')}  "
+                    f"mean_score={data.get('mean_score', '?')}"
+                )
+            except Exception:
+                print(f"  {f.name}")
+        return 0
+
+    # eval_cmd == "run"
+    from devsper.evals.dataset import EvalDataset
+    from devsper.evals.metrics import get_metric
+    from devsper.evals.runner import EvalRunner
+    from devsper.config import get_config
+
+    try:
+        cfg = get_config()
+    except Exception:
+        from devsper.config.schema import devsperConfigModel
+        cfg = devsperConfigModel()
+
+    dataset_path = getattr(args, "dataset", None)
+    if not dataset_path:
+        print("Error: --dataset is required for 'eval run'")
+        return 1
+
+    dataset = EvalDataset.load(dataset_path)
+    role = getattr(args, "role", None)
+    metric_name = getattr(args, "metric", None) or cfg.evals.default_metric
+    threshold = getattr(args, "threshold", None) or cfg.evals.pass_threshold
+    concurrency = getattr(args, "concurrency", None) or cfg.evals.concurrency
+    do_optimize = getattr(args, "optimize", False)
+    optimizer_override = getattr(args, "optimizer", None)
+    out_path = getattr(args, "out", None)
+
+    metric = get_metric(metric_name)
+
+    # Build optimizer if requested
+    optimizer = None
+    if do_optimize:
+        from devsper.prompt_optimizer.factory import get_prompt_optimizer, reset_prompt_optimizer
+
+        if optimizer_override:
+            import os
+            os.environ["DEVSPER_PROMPT_OPTIMIZER"] = optimizer_override
+            reset_prompt_optimizer()
+        optimizer = get_prompt_optimizer(cfg)
+
+    # Build a minimal agent for evaluation
+    from devsper.agents.agent import Agent
+
+    agent = Agent(model_name=cfg.models.worker, use_tools=False)
+
+    runner = EvalRunner(
+        agent=agent,
+        metric=metric,
+        pass_threshold=threshold,
+        concurrency=concurrency,
+        optimize_after=do_optimize,
+        optimizer=optimizer,
+    )
+
+    try:
+        summary = asyncio.run(runner.run_async(dataset, role=role))
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        summary = loop.run_until_complete(runner.run_async(dataset, role=role))
+
+    # Print summary
+    try:
+        from rich.console import Console
+        from rich.table import Table
+
+        console = Console()
+        console.print(
+            f"\n[bold]Eval Results[/bold]  role=[cyan]{summary.role}[/cyan]  "
+            f"metric=[cyan]{summary.metric_name}[/cyan]  "
+            f"optimizer=[cyan]{summary.optimizer_backend}[/cyan]"
+        )
+        console.print(
+            f"  Passed: [green]{summary.passed}[/green]/{summary.total}  "
+            f"Pass rate: [bold]{summary.pass_rate:.1%}[/bold]  "
+            f"Mean score: [bold]{summary.mean_score:.3f}[/bold]\n"
+        )
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("ID", style="dim")
+        table.add_column("Task", max_width=40)
+        table.add_column("Score")
+        table.add_column("Pass")
+        for r in summary.results:
+            color = "green" if r.passed else "red"
+            table.add_row(
+                r.case.id,
+                r.case.task[:40],
+                f"{r.score:.2f}",
+                f"[{color}]{'✓' if r.passed else '✗'}[/{color}]",
+            )
+        console.print(table)
+    except ImportError:
+        print(f"\nEval: role={summary.role} metric={summary.metric_name}")
+        print(f"  {summary.passed}/{summary.total} passed ({summary.pass_rate:.1%})")
+        print(f"  Mean score: {summary.mean_score:.3f}")
+
+    # Persist results
+    results_dir = Path(cfg.evals.results_dir)
+    results_dir.mkdir(parents=True, exist_ok=True)
+    ts = __import__("datetime").datetime.now().strftime("%Y%m%d_%H%M%S")
+    result_file = results_dir / f"eval_{summary.role}_{ts}.json"
+    result_file.write_text(summary.to_json())
+    print(f"\nResults saved to {result_file}")
+
+    if out_path:
+        Path(out_path).write_text(summary.to_json())
+
+    return 0 if summary.pass_rate >= threshold else 1
+
+
 def _run_health(args: object) -> int:
     """Run health checks. Exit 0 if healthy, 1 otherwise. Print ✓/✗ per check."""
     import asyncio
@@ -2811,7 +2969,7 @@ def _run_upgrade(args: object) -> int:
 
 
 def _run_cloud_dispatch(args: object) -> int:
-    """Devsper Platform (cloud): login, run, status, logs."""
+    """Devsper Cloud: login, run, status, logs."""
     cmd = getattr(args, "cloud_cmd", None)
     if not cmd:
         return 0
@@ -4082,6 +4240,64 @@ def main() -> int:
     )
     observe_parser.set_defaults(func=lambda a: _run_observe(a.port, a.db))
 
+    eval_parser = subparsers.add_parser(
+        "eval",
+        help="Eval harness and prompt optimization",
+        description="Run evals against a JSONL dataset and optionally optimize prompts.",
+        epilog="""
+Examples:
+  devsper eval run --dataset evals.jsonl --metric contains
+  devsper eval run --dataset evals.jsonl --role research --optimize --optimizer dspy
+  devsper eval stub --role research --out evals.jsonl
+  devsper eval results
+""",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    eval_sub = eval_parser.add_subparsers(dest="eval_cmd", help="Subcommand")
+
+    eval_run_p = eval_sub.add_parser("run", help="Run eval dataset")
+    eval_run_p.add_argument("--dataset", required=True, help="Path to JSONL dataset")
+    eval_run_p.add_argument("--role", default=None, help="Filter to this agent role")
+    eval_run_p.add_argument(
+        "--metric",
+        default=None,
+        help="Metric name: exact_match | contains | regex_match | word_overlap | llm_judge (default: config)",
+    )
+    eval_run_p.add_argument(
+        "--threshold", type=float, default=None, help="Pass threshold (default: config)"
+    )
+    eval_run_p.add_argument(
+        "--optimize",
+        action="store_true",
+        help="Run prompt optimization after eval using the configured optimizer",
+    )
+    eval_run_p.add_argument(
+        "--optimizer",
+        default=None,
+        help="Override optimizer backend: noop | dspy | gepa",
+    )
+    eval_run_p.add_argument(
+        "--concurrency", type=int, default=None, help="Parallel eval cases"
+    )
+    eval_run_p.add_argument("--out", default=None, help="Save JSON results to this path")
+    eval_run_p.set_defaults(eval_cmd="run")
+
+    eval_stub_p = eval_sub.add_parser("stub", help="Generate a stub dataset")
+    eval_stub_p.add_argument(
+        "--role", default="general", help="Agent role (research/code/analysis/general)"
+    )
+    eval_stub_p.add_argument("-n", type=int, default=5, help="Number of examples")
+    eval_stub_p.add_argument(
+        "--out", default=None, help="Output JSONL path (default: prints to stdout)"
+    )
+    eval_stub_p.set_defaults(eval_cmd="stub")
+
+    eval_results_p = eval_sub.add_parser("results", help="List recent eval result files")
+    eval_results_p.add_argument("--dir", default=None, help="Results directory")
+    eval_results_p.set_defaults(eval_cmd="results")
+
+    eval_parser.set_defaults(func=_run_eval)
+
     health_parser = subparsers.add_parser(
         "health",
         help="Health and readiness check",

devsper/config/schema.py

@@ -309,6 +309,32 @@ class HitlConfig(BaseModel):
     policies: list[HitlPolicyConfig] = Field(default_factory=list)
 
 
+class PromptOptimizerConfig(BaseModel):
+    """v2.7: Prompt optimization backend config.
+
+    provider: "noop" | "dspy" | "gepa"  (override with DEVSPER_PROMPT_OPTIMIZER env var)
+    """
+
+    provider: str = "noop"
+    # DSPy-specific
+    dspy_optimizer: str = "bootstrap"  # "bootstrap" | "mipro" | "bootstrap_random"
+    max_demos: int = 4
+    num_candidates: int = 10           # MIPROv2 only
+    # GEPA-specific
+    population_size: int = 5
+    n_iterations: int = 10
+
+
+class EvalConfig(BaseModel):
+    """v2.7: Eval harness config."""
+
+    dataset_dir: str = ".devsper/evals"   # default location for JSONL datasets
+    results_dir: str = ".devsper/eval_results"
+    pass_threshold: float = 0.5
+    concurrency: int = 4
+    default_metric: str = "contains"      # metric name from devsper.evals.metrics
+
+
 class devsperConfigModel(BaseModel):
     """Full resolved configuration with Pydantic validation."""
 
@@ -333,6 +359,8 @@ class devsperConfigModel(BaseModel):
     sandbox: SandboxConfig = Field(default_factory=SandboxConfig)
     compliance: ComplianceConfig = Field(default_factory=ComplianceConfig)
     hitl: HitlConfig = Field(default_factory=HitlConfig)
+    prompt_optimizer: PromptOptimizerConfig = Field(default_factory=PromptOptimizerConfig)
+    evals: EvalConfig = Field(default_factory=EvalConfig)
 
     # Backward-compat aliases (property-style access from old devsperConfig)
     @property

devsper/evals/__init__.py

@@ -0,0 +1,19 @@
+"""
+devsper evals — eval harness and prompt optimization integration.
+"""
+
+from devsper.evals.types import EvalCase, EvalResult, EvalSummary, MetricFn
+from devsper.evals.dataset import EvalDataset
+from devsper.evals.metrics import get_metric, BUILTIN_METRICS
+from devsper.evals.runner import EvalRunner
+
+__all__ = [
+    "EvalCase",
+    "EvalResult",
+    "EvalSummary",
+    "EvalDataset",
+    "EvalRunner",
+    "MetricFn",
+    "get_metric",
+    "BUILTIN_METRICS",
+]