diff --git a/changelog.d/bench-answer-keys.md b/changelog.d/bench-answer-keys.md new file mode 100644 index 000000000..d8e330206 --- /dev/null +++ b/changelog.d/bench-answer-keys.md @@ -0,0 +1,2 @@ +### Added +- Bench: answer keys (`keys.json`) for all 40 held-out scenarios of the business-ops, growth-analytics, idea-critic and contribution rival rows, 10 per row, 3-6 must-hit and 1-3 must-not regex items each. Keys test what an answer adds beyond the prompt (an echo baseline of each prompt scores at most 0.25 hit rate), accept number forms (`$280k`, `100,000`, `13 percent`) and paraphrases, and use no Perun verdict tokens. `scripts/rival_eval_keys.py` (a wrapper around the unchanged, hash-pinned `scripts/rival_eval.py`) adds `key_score` (per-arm hit rate and violation rate, deterministic, no judge). Must-nots are negation-aware ("never name X" is not a leak), skip bracketed placeholders, and for contribution scan only the drafted message; a scenario that needs a draft and has none counts as a violation (`no_draft`). Results land in `results.json` as `answer_key` next to the pairwise verdict. `rival_eval_keys.py keys SKILL RUNDIR` rescores cached outputs offline and fails on a missing arm directory. `check` rejects an unkeyed or malformed key. Tests: `scripts/test_rival_answer_keys.py`. diff --git a/docs/perun-vs-rivals.md b/docs/perun-vs-rivals.md index 60bfdfa4e..7759a2371 100644 --- a/docs/perun-vs-rivals.md +++ b/docs/perun-vs-rivals.md @@ -1,6 +1,6 @@ # Perun vs other skill bundles -Harness: run rival rows with `scripts/rival_eval_cost.py run ...`, a wrapper around the hash-pinned `scripts/rival_eval.py` that also records per-arm cost into `results.json`; `scripts/bench_verdict.py --cost results.json` reads it (exit 0 Perun cost <= rival, 1 Perun costs more, 2 usage error, 3 unmeasured). +Harness: run rival rows with `scripts/rival_eval_keys.py run ...`, a wrapper around the hash-pinned `scripts/rival_eval.py` that also records per-arm cost and, for skills with a `keys.json`, a deterministic answer-key score (`rival_eval_keys.py keys SKILL RUNDIR` rescores cached outputs) into `results.json`; `scripts/bench_verdict.py --cost results.json` reads it (exit 0 Perun cost <= rival, 1 Perun costs more, 2 usage error, 3 unmeasured). **Bottom line (2026-10-17 to 2026-10-19 held-out runs, free PCSS models, judge protocol v2).** Five skills have a confirmed preference-only win over their rival (the judge preferred Perun with the 95% lower bound above 0.5; cost and user outcome were not measured, so none is a win on cost or outcome) (product-output-safety's is narrow: lower bound 0.518 on its preregistered replication, one earlier fresh set below 0.5); idea-critic won once (round 9, narrowly) and the preregistered replication on a second fresh set did not reproduce it, so it is not counted; four more are not wins (growth-analytics among them: its round-4 edit shipped on a paired gain but its win-rate lower bound is 0.483, point estimate 0.550; a round-5 candidate got no verdict because no judge was inside the slot-1 gate, and was not shipped) and none is unproven. A win needs the 95% CI lower bound above 0.5; every row below is a fresh held-out set of n=40 or more, with ties counted as 0.5. Rows with n=10 are not mixed in: they are the older history further down and are not comparable to this table. diff --git a/scripts/ci-tests.d/rival-answer-keys.sh b/scripts/ci-tests.d/rival-answer-keys.sh new file mode 100644 index 000000000..182a97d19 --- /dev/null +++ b/scripts/ci-tests.d/rival-answer-keys.sh @@ -0,0 +1,2 @@ +# Registers the rival answer-key scorer tests with CI (see the "Per-PR tests" step in ci.yml). Offline; no model is called. +python3 scripts/test_rival_answer_keys.py diff --git a/scripts/eval-fixtures/rival/business-ops/keys.json b/scripts/eval-fixtures/rival/business-ops/keys.json new file mode 100644 index 000000000..08abcc51f --- /dev/null +++ b/scripts/eval-fixtures/rival/business-ops/keys.json @@ -0,0 +1,123 @@ +{ + "07": { + "must": { + "unit_margin_37": "\\b37\\b", + "direct_margin_72": "\\b72\\b", + "volume_ratio": "1\\.9\\d?\\s*(x|times)|1\\.95|nearly (twice|double)|(roughly|about|almost) (twice|double)", + "inventory_cash_540k": "(?&1)" +out="$(python3 "$ROOT/scripts/rival_eval_keys.py" run --changed "${RIVAL_EVAL_BASE:-origin/main}" --out "${TMPDIR:-/tmp}/rival-eval" 2>&1)" rc=$? printf '%s\n' "$out" if printf '%s' "$out" | grep -q 'PCSS_KEY missing'; then echo "rival-eval skipped: no PCSS_KEY"; exit 0; fi diff --git a/scripts/rival_eval_cost.py b/scripts/rival_eval_cost.py index fb8b6f947..2681658f8 100644 --- a/scripts/rival_eval_cost.py +++ b/scripts/rival_eval_cost.py @@ -99,6 +99,11 @@ def _run(*a, **k): _add(u) +def run_dir(out, skill, split): + """The run directory run_skill just wrote (newest results.json under //-*).""" + return max((out / skill).glob(f"{split}-*/results.json"), key=lambda f: f.stat().st_mtime).parent + + def install(): """Wrap rival_eval in place (idempotent). Call before rival_eval.main().""" if getattr(r, "_cost_installed", False): @@ -126,7 +131,7 @@ def save_(p, text): def run_skill_(skill, split, backend, out, jobs, cap): res = run_skill(skill, split, backend, out, jobs, cap) scs = r.load_scenarios(skill, split) - d = max((out / skill).glob(f"{split}-*/results.json"), key=lambda f: f.stat().st_mtime).parent # the dir run_skill just wrote + d = run_dir(out, skill, split) uses = {a: [json.loads(f.read_text()) for sc in scs if (f := d / "outputs" / a / f"{sc['id']}-{r.sha(sc['prompt'])[:8]}.usage.json").exists()] for a in r.ARMS} res["cost_per_arm"] = {a: arm_cost(uses[a], len(scs)) for a in r.ARMS} diff --git a/scripts/rival_eval_keys.py b/scripts/rival_eval_keys.py new file mode 100644 index 000000000..8cf01e363 --- /dev/null +++ b/scripts/rival_eval_keys.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Deterministic answer-key scoring for rival_eval.py, without editing it (its sha256 is frozen by test_evil_twin_bench_r2.py). + +usage: rival_eval_keys.py run|write|changed|check ... (same arguments as rival_eval.py, plus cost capture) + rival_eval_keys.py keys SKILL RUNDIR (answer-key hit rate per arm from cached outputs; offline) + +A skill with `/keys.json` under scripts/eval-fixtures/rival gets, per scenario, 3-6 must-hit and 1-3 must-not regex +items (`must`, `mustnot`, `mustnot_draft`, optional `needs_draft`). `key_score` gives each arm's mean hit rate and the share of +scenarios with any must-not hit, no judge. `install()` (also installs rival_eval_cost) adds key hygiene to `check_split`, adds +an `answer_key` block to each run's results.json, and routes the `keys` subcommand. Side effects: results.json is rewritten +after run_skill; `keys` fails with SystemExit on a missing keys.json or arm directory. +""" +import functools +import json +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import rival_eval as r # noqa: E402 +import rival_eval_cost as cost # noqa: E402 + +# Heuristic, not a parser: a must-not match counts as negated only when one of these words sits within NEG_WINDOW words right +# before it in the same clause, so "not only X" can still mis-score; keys are written to tolerate that (see `violated`). +NEG = re.compile(r"\b(never|don'?t|do not|doesn'?t|won'?t|shouldn'?t|avoid|not|no|without|omit|exclude|instead of)\b", re.I) + + +NEG_WINDOW = 4 +SHIPPED = ("business-ops", "growth-analytics", "idea-critic", "contribution") # skills that ship keys.json: a missing file must fail, not skip +DRAFT_MIN_WORDS = 15 # a `needs_draft` answer must quote/fence at least this many words of message body + + +def load_keys(skill): + """-> {scenario id: {"must": {name: regex}, "mustnot": {...}, "mustnot_draft": {...}}} from /keys.json, or {} for a skill + that ships none. Raises SystemExit (fail closed) when a SHIPPED skill's keys.json is missing or empty.""" + f = r.FIX / skill / "keys.json" + keys = json.loads(f.read_text()) if f.is_file() else {} + if not keys and skill in SHIPPED: + raise SystemExit(f"keys: keys.json missing or empty for {skill} (fail closed)") + return keys + + +def check_keys(skill): + """Answer-key hygiene (empty = clean): every held-out id keyed, 3-6 must-hit and 1-3 must-not items, every regex compiles.""" + try: + keys, bad = load_keys(skill), [] + except SystemExit as e: + return [str(e)] + if not keys: + return bad + for i, k in keys.items(): + if not isinstance(k, dict) or not isinstance(k.get("must"), dict) or not all(isinstance(k.get(g, {}), dict) for g in ("mustnot", "mustnot_draft")): + bad.append(f"keys.json {i}: malformed entry (`must` and optional `mustnot`/`mustnot_draft` must be {{name: regex}} objects)") + if bad: + return bad + for sc in r.load_scenarios(skill, "heldout"): + k = keys.get(sc["id"]) + if not k: + bad.append(f"keys.json: no key for heldout {sc['id']}") + continue + no = len(k.get("mustnot", {})) + len(k.get("mustnot_draft", {})) + if not 3 <= len(k.get("must", {})) <= 6 or not 1 <= no <= 3: + bad.append(f"keys.json {sc['id']}: needs 3-6 must and 1-3 mustnot items") + for i, k in keys.items(): + for grp in ("must", "mustnot", "mustnot_draft"): + for name, rx in k.get(grp, {}).items(): + try: + re.compile(rx) + except re.error as e: + bad.append(f"keys.json {i}/{name}: bad regex ({e})") + return bad + + +def draft_text(t): + """Only the parts of an answer that read as a drafted public message: > quoted lines and fenced blocks.""" + out, fence = [], False + for ln in t.splitlines(): + if ln.lstrip().startswith("```"): + fence = not fence + elif fence or ln.lstrip().startswith(">"): + out.append(ln) + return "\n".join(out) + + +def violated(rxs, text): + """Names of must-not regexes with at least one match not negated by a NEG word within NEG_WINDOW words right before it in + the same clause, so "never promise a guaranteed profit" is not a violation but "No, you should definitely form an LLC" is. + Nothing is blanked: a bracketed or angle-bracketed name still counts, so use a generic placeholder like [Company].""" + out = [] + for n, rx in rxs.items(): + for s in re.split(r"(?<=[.!?])\s+|\n", text): + if any(not NEG.search(" ".join(re.split(r"[,;:()]|\s[-\u2013\u2014]+\s", s[:m.start()])[-1].split()[-NEG_WINDOW:])) + for m in re.finditer(rx, s, re.I)): + out.append(n) + break + return out + + +def key_score(skill, outs): + """Deterministic outcome score: per arm, mean fraction of must-hit items found (hit_rate) and share of scenarios with any + must-not hit (violation_rate). outs = {arm: {scenario id: answer text}}; every keyed id counts for every arm, and an arm with no answer for one scores 0 hits there (n stays len(keys)). + Regex only, case-insensitive; must-nots skip negated sentences (see `violated`); `mustnot_draft` is checked against the + drafted-message lines only; `needs_draft` keys flag `no_draft` (counted as a violation) unless the answer quotes/fences >= DRAFT_MIN_WORDS words. + ponytail: keyword match proves presence, not quality; upgrade to a PCSS item-presence judge if regex misses paraphrases.""" + keys = load_keys(skill) + ids = list(keys) + res = {} + for a in outs: + per = {} + for i in ids: + t, k = outs[a].get(i, ""), keys[i] + if i not in outs[a]: # no output for this arm: a miss, not a dropped scenario + per[i] = dict(hits=0, of=len(k["must"]), missed=sorted(k["must"]), violated=[], no_draft=False, missing=True) + continue + hit = [n for n, rx in k["must"].items() if re.search(rx, t, re.I | re.S)] + d = draft_text(t) + bad = violated(k.get("mustnot", {}), t) + violated(k.get("mustnot_draft", {}), d) + no_draft = bool(k.get("needs_draft")) and len(re.sub(r"^\s*>+", "", d, flags=re.M).split()) < DRAFT_MIN_WORDS + per[i] = dict(hits=len(hit), of=len(k["must"]), missed=sorted(set(k["must"]) - set(hit)), violated=bad, no_draft=no_draft) + n = len(per) + res[a] = dict(n=n, hit_rate=round(sum(p["hits"] / p["of"] for p in per.values()) / n, 3) if n else None, + violation_rate=round(sum(bool(p["violated"]) or p["no_draft"] for p in per.values()) / n, 3) if n else None, per_scenario=per) + return res + + +def _summary(skill, outs): + return {a: {k: x for k, x in v.items() if k != "per_scenario"} for a, v in key_score(skill, outs).items()} + + +def _outs(d, skill, split): + """{arm: {scenario id: text}} for the CURRENT prompts only: file `-.txt`, so stale hashes are ignored.""" + return {a: {sc["id"]: f.read_text() for sc in r.load_scenarios(skill, split) + if (f := Path(d) / "outputs" / a / f"{sc['id']}-{r.sha(sc['prompt'])[:8]}.txt").exists()} for a in r.ARMS} + + +def keys_cmd(skill, rundir): + """Rescore cached held-out outputs under RUNDIR/outputs//-.txt and print the per-arm summary as JSON.""" + if not load_keys(skill): + raise SystemExit(f"keys: no keys.json for {skill} (fail closed)") + for x in r.ARMS: + if not (Path(rundir) / "outputs" / x).is_dir(): + raise SystemExit(f"keys: missing arm directory {Path(rundir) / 'outputs' / x}") + print(json.dumps(_summary(skill, _outs(rundir, skill, "heldout")), indent=1)) + + +def install(): + """Wrap rival_eval in place (idempotent), cost capture included. Call before main().""" + if getattr(r, "_keys_installed", False): + return + r._keys_installed = True + cost.install() + check_split, run_skill = r.check_split, r.run_skill + + @functools.wraps(check_split) + def check_split_(skill): + return check_split(skill) + check_keys(skill) + + @functools.wraps(run_skill) + def run_skill_(skill, split, backend, out, jobs, cap): + res = run_skill(skill, split, backend, out, jobs, cap) + if split == "heldout" and load_keys(skill): # keys cover the held-out split only + d = cost.run_dir(out, skill, split) + res["answer_key"] = _summary(skill, _outs(d, skill, split)) + (d / "results.json").write_text(json.dumps(res, indent=1) + "\n") + return res + + r.check_split, r.run_skill = check_split_, run_skill_ + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + install() + if argv[:1] == ["keys"]: + if len(argv) != 3: + raise SystemExit("usage: rival_eval_keys.py keys SKILL RUNDIR") + return keys_cmd(argv[1], argv[2]) + return r.main(argv) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_rival_answer_keys.py b/scripts/test_rival_answer_keys.py new file mode 100644 index 000000000..9722b7be0 --- /dev/null +++ b/scripts/test_rival_answer_keys.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Tests for the answer-key scorer in scripts/rival_eval_keys.py (key_score, violated, check_keys, draft_text, `keys` CLI). Offline; no model is called.""" +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import rival_eval as r # noqa: E402 +import rival_eval_keys as k # noqa: E402 +import test_rival_eval as T # noqa: E402 +from unittest import mock # noqa: E402 + +k.install() + +ROWS = ("business-ops", "growth-analytics", "idea-critic", "contribution") + + +def score(skill, i, text): + return k.key_score(skill, {"A": {i: text}})["A"]["per_scenario"][i] + + +class Keys(unittest.TestCase): + def test_every_heldout_scenario_keyed_and_clean(self): + for s in ROWS: + self.assertEqual(k.check_keys(s), [], s) + self.assertEqual(set(k.load_keys(s)), {x["id"] for x in r.load_scenarios(s, "heldout")}, s) + + def test_check_split_includes_keys(self): + for s in ROWS: + self.assertEqual(r.check_split(s), [], s) + + def test_echo_baseline_scenario_text_alone_scores_under_half(self): + """A key that the prompt itself satisfies measures reading, not answering.""" + worst = 0.0 + for s in ROWS: + scs = r.load_scenarios(s, "heldout") + per = k.key_score(s, {"E": {x["id"]: x["prompt"] for x in scs}})["E"]["per_scenario"] + for i, p in per.items(): + self.assertLess(p["hits"] / p["of"], 0.5, f"{s} {i} hit by its own prompt: {set(k.load_keys(s)[i]['must']) - set(p['missed'])}") + worst = max(worst, p["hits"] / p["of"]) + print(f"echo-baseline max hit-rate {worst:.2f}") + + def test_no_perun_verdict_tokens_in_idea_critic_keys(self): + blob = json.dumps(k.load_keys("idea-critic")) + for tok in ("HOLD", "REVISE", "PASS_TO_USER"): + self.assertNotIn(tok, blob) + + def test_check_keys_flags_size_and_regex(self): + orig = k.load_keys + k.load_keys = lambda s: {"07": {"must": {"a": "("}, "mustnot": {}}} + try: + bad = k.check_keys("business-ops") + finally: + k.load_keys = orig + self.assertTrue(any("needs 3-6 must" in b for b in bad)) + self.assertTrue(any("bad regex" in b for b in bad)) + self.assertTrue(any("no key for heldout 08" in b for b in bad)) + + def test_scores_hits_and_violations(self): + good = ("Margin is $37 per unit against $72 direct, so you need about 1.95x the volume. Funding 5,000 units ties up $540,000 " + "of cash, so counter on price.") + o = k.key_score("business-ops", {"PERUN": {"07": good}, "RIVAL": {"07": "Accept: guaranteed profit, no downside."}}) + p, q = o["PERUN"]["per_scenario"]["07"], o["RIVAL"]["per_scenario"]["07"] + self.assertEqual((p["hits"], p["of"], p["violated"]), (p["of"], p["of"], [])) + self.assertEqual((q["hits"], q["violated"]), (0, ["no_risk"])) + + def test_missing_output_is_a_miss_for_that_arm_only(self): + n = len(k.load_keys("business-ops")) + o = k.key_score("business-ops", {"PERUN": {"07": "x", "08": "x"}, "RIVAL": {"07": "x"}}) + self.assertEqual((o["PERUN"]["n"], o["RIVAL"]["n"]), (n, n)) + self.assertTrue(o["RIVAL"]["per_scenario"]["08"]["missing"]) + self.assertEqual(o["RIVAL"]["per_scenario"]["08"]["hits"], 0) + self.assertNotIn("missing", o["PERUN"]["per_scenario"]["08"]) + + def test_missing_keys_json_fails_closed_for_shipped_skills(self): + with mock.patch.object(r, "FIX", Path(tempfile.mkdtemp())): + for s in ROWS: + with self.assertRaises(SystemExit): + k.load_keys(s) + self.assertTrue(any("fail closed" in b for b in k.check_keys(s)), s) + self.assertEqual(k.load_keys("agentic-ceo"), {}) + + def test_check_keys_rejects_malformed_entries(self): + for bad in ({"07": {"mustnot": {}}}, {"07": {"must": ["a", "b", "c"]}}, {"07": "x"}, {"07": {"must": {"a": "x"}, "mustnot": ["y"]}}): + with mock.patch.object(k, "load_keys", lambda s, bad=bad: bad): + self.assertTrue(any("malformed" in b for b in k.check_keys("business-ops")), bad) + + def test_negation_scope_is_local_and_brackets_do_not_hide(self): + self.assertEqual(score("business-ops", "07", "No, you should definitely accept the guaranteed profit now.")["violated"], ["no_risk"]) + self.assertEqual(score("contribution", "07", "```\nHi Jane, we at [Acme Logistics] hit it.\n```")["violated"], ["leaks_company"]) + self.assertEqual(score("contribution", "07", "```\nHi Jane, we at [Company] hit it.\n```")["violated"], []) + + def test_negation_is_not_a_violation(self): + self.assertEqual(score("business-ops", "07", "Do not expect guaranteed profit here.")["violated"], []) + self.assertEqual(score("business-ops", "07", "Expect guaranteed profit here.")["violated"], ["no_risk"]) + self.assertEqual(score("idea-critic", "h01", "I cannot recommend either path.")["violated"], ["no_rec"]) + self.assertEqual(score("contribution", "07", "```\nHi Jane, we at Acme Logistics hit it.\n```")["violated"], ["leaks_company"]) + self.assertEqual(score("contribution", "07", "```\nHi Jane. We never name Acme Logistics.\n[Company] and redacted\n```")["violated"], []) + + def test_paraphrase_forms_hit(self): + for t in ("needs $540,000", "needs 540000", "needs $540k", "needs 540K of cash"): + self.assertIn("inventory_cash_540k", set(k.load_keys("business-ops")["07"]["must"]) - set(score("business-ops", "07", t)["missed"]), t) + for t in ("payroll of $100,000", "payroll of 100k", "two hires cost 100,000"): + self.assertEqual(score("business-ops", "08", t)["missed"].count("payroll_100k"), 0, t) + for t in ("breakeven churn is 13%", "breakeven churn is 13 percent", "about 13.04%"): + self.assertEqual(score("idea-critic", "h02", t)["missed"].count("breakeven_churn"), 0, t) + self.assertEqual(score("business-ops", "07", "costs $5,400,000")["missed"].count("inventory_cash_540k"), 1) # no substring match + + def test_plain_words_answer_equals_perun_vocabulary_answer(self): + perun = ("Recommendation: PASS_TO_USER with REVISE. Kill criteria: stop if churn tops 13%. Pilot a segment first, grandfather " + "existing accounts, phase the rise by tier.") + plain = ("I'd go with a phased rise. Walk away if churn passes 13 percent. Try it on a sample of accounts first, " + "keep current customers on their old price, and move up by tier.") + a, b = score("idea-critic", "h02", perun), score("idea-critic", "h02", plain) + self.assertEqual((a["hits"], a["missed"]), (b["hits"], b["missed"])) + self.assertEqual(a["hits"], a["of"]) + + def test_draft_scope_and_no_draft(self): + self.assertEqual(k.draft_text("a\n```\nb\n```\n> c\nd"), "b\n> c") + self.assertTrue(score("contribution", "07", "Advice only, nothing to paste.")["no_draft"]) + self.assertTrue(score("contribution", "07", "> Hi Jane, thanks.")["no_draft"]) # one-liner is not a draft + self.assertTrue(score("contribution", "07", "```\n```")["no_draft"]) + self.assertFalse(score("contribution", "07", "> " + "Hi Jane, thanks for maintaining this project, here is my patch for review. " * 2)["no_draft"]) + self.assertFalse(score("business-ops", "07", "no draft needed")["no_draft"]) + self.assertEqual(score("contribution", "07", "Never name Acme Logistics.\n> Hi Jane, thanks.")["violated"], []) + o = k.key_score("contribution", {"A": {"07": "no draft here"}})["A"] + self.assertEqual(o["per_scenario"]["07"]["no_draft"], True) + self.assertEqual(o["violation_rate"], round(1 / o["n"], 3)) + + def test_force_push_in_draft_any_flag_position(self): + def f(t): + return score("contribution", "14", t)["violated"] + self.assertEqual(f("```\ngit push --force-with-lease origin x\n```"), []) + self.assertEqual(f("```\ngit push --force origin x\n```"), ["bare_force_push"]) + self.assertEqual(f("```\ngit push origin x --force\n```"), ["bare_force_push"]) + self.assertEqual(f("```\ngit push origin x -f\n```"), ["bare_force_push"]) + self.assertEqual(f("```\n# never run git push --force\n```"), []) + self.assertEqual(f("Never use git push --force on a shared branch."), []) + + def test_run_skill_adds_answer_key_block(self): + tmp = Path(tempfile.mkdtemp()) + name = T.fake_fixture(tmp, T.scen("held", 10, 10), T.scen("tune", 5)) + with mock.patch.object(r, "ROOT", tmp), mock.patch.object(r, "FIX", tmp / "scripts/eval-fixtures/rival"): + ids = [x["id"] for x in r.load_scenarios(name, "heldout")] + key = {"must": {"a": "PERUN-ANSWER", "b": "ANSWER", "c": " for "}, "mustnot": {"x": "zzzz"}} + (r.FIX / name / "keys.json").write_text(json.dumps({i: key for i in ids})) + + def chat(model, system, user, **kw): + if T.is_judge(system): + return T.judge_reply(user, "1" if user.index("PERUN-ANSWER") < user.index("RIVAL-ANSWER") else "2") + return ("PERUN-ANSWER" if "PERUN skill text" in system else "RIVAL-ANSWER") + " for " + user[-20:] + with mock.patch.object(r, "chat", chat): + res = r.run_skill(name, "heldout", "pcss", tmp / "out", 2, 0.5) + self.assertEqual(r.check_split(name), []) + a = res["answer_key"] + self.assertEqual((a["PERUN"]["n"], a["PERUN"]["hit_rate"], a["RIVAL"]["hit_rate"], a["RIVAL"]["violation_rate"]), (10, 1.0, 0.667, 0.0)) + saved = json.loads(next((tmp / "out" / name).glob("heldout-*/results.json")).read_text()) + self.assertEqual(saved["answer_key"], a) + + def test_keys_cli_rescoring_cached_outputs(self): + with tempfile.TemporaryDirectory() as d: + sc = next(x for x in r.load_scenarios("business-ops", "heldout") if x["id"] == "07") + h = r.sha(sc["prompt"])[:8] + for arm, txt in (("PERUN", "37 72 1.95x $540k counter"), ("RIVAL", "nothing")): + p = Path(d) / "outputs" / arm + p.mkdir(parents=True) + (p / f"07-{h}.txt").write_text(txt) + (p / "07-deadbeef.txt").write_text("stale") # old prompt hash: must be ignored + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + k.main(["keys", "business-ops", d]) + out = json.loads(buf.getvalue()) + self.assertEqual((out["PERUN"]["per_scenario"] if "per_scenario" in out["PERUN"] else {}), {}) + n = len(k.load_keys("business-ops")) + self.assertEqual((out["PERUN"]["hit_rate"], out["RIVAL"]["hit_rate"]), (round(1 / n, 3), 0.0)) + + def test_keys_cli_fails_closed_without_keys_json(self): + with tempfile.TemporaryDirectory() as d: + for arm in r.ARMS: + (Path(d) / "outputs" / arm).mkdir(parents=True) + with self.assertRaises(SystemExit) as cm: + k.main(["keys", "agentic-ceo", d]) # a skill with no keys.json + self.assertIn("no keys.json", str(cm.exception)) + + def test_keys_cli_fails_on_missing_arm_directory(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "outputs" / "PERUN" + p.mkdir(parents=True) + (p / "07-abcd1234.txt").write_text("x") + with self.assertRaises(SystemExit) as cm: + k.main(["keys", "business-ops", d]) + self.assertIn("missing arm directory", str(cm.exception)) + + +if __name__ == "__main__": + unittest.main()