From b4e45118c8291bceda24e5fa7db8fd98f86e4249 Mon Sep 17 00:00:00 2001 From: Remigiusz Antczak Date: Fri, 23 Oct 2026 22:33:43 +0200 Subject: [PATCH] Bench: answer keys and a deterministic scorer for four stalled rival rows Add keys.json (3-6 must-hit and 1-3 must-not regex items per scenario) for the 40 held-out scenarios of business-ops, growth-analytics, idea-critic and contribution. Keys test what an answer adds beyond its prompt (echo baseline max 0.25), accept number forms and paraphrases, and carry no Perun verdict tokens. rival_eval.py gains key_score (per-arm hit rate and violation rate, no judge), an answer_key block in results.json, a keys subcommand that rescores cached outputs offline and fails on a missing arm directory, and key hygiene in check. Must-nots are negation-aware and skip placeholders; contribution leak and force-push checks scan only the drafted message, and a missing draft is flagged no_draft. No paid call. Tests: 61/61 (test_rival_answer_keys 14, test_rival_eval 47) Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_014QNFFPWVcPUbeuvwM5fk2Z --- changelog.d/bench-answer-keys.md | 2 + scripts/ci-tests.d/rival-answer-keys.sh | 2 + .../rival/business-ops/keys.json | 123 ++++++++++++++++ .../rival/contribution/keys.json | 135 +++++++++++++++++ .../rival/growth-analytics/keys.json | 130 +++++++++++++++++ .../eval-fixtures/rival/idea-critic/keys.json | 128 ++++++++++++++++ scripts/rival_eval.py | 92 ++++++++++++ scripts/test_rival_answer_keys.py | 137 ++++++++++++++++++ 8 files changed, 749 insertions(+) create mode 100644 changelog.d/bench-answer-keys.md create mode 100644 scripts/ci-tests.d/rival-answer-keys.sh create mode 100644 scripts/eval-fixtures/rival/business-ops/keys.json create mode 100644 scripts/eval-fixtures/rival/contribution/keys.json create mode 100644 scripts/eval-fixtures/rival/growth-analytics/keys.json create mode 100644 scripts/eval-fixtures/rival/idea-critic/keys.json create mode 100644 scripts/test_rival_answer_keys.py diff --git a/changelog.d/bench-answer-keys.md b/changelog.d/bench-answer-keys.md new file mode 100644 index 000000000..803383517 --- /dev/null +++ b/changelog.d/bench-answer-keys.md @@ -0,0 +1,2 @@ +### Added +- Bench: answer keys (`keys.json`) for all 40 held-out scenarios of the business-ops, growth-analytics, idea-critic and contribution rival rows, 10 per row, 3-6 must-hit and 1-3 must-not regex items each. Keys test what an answer adds beyond the prompt (an echo baseline of each prompt scores at most 0.25 hit rate), accept number forms (`$280k`, `100,000`, `13 percent`) and paraphrases, and use no Perun verdict tokens. `scripts/rival_eval.py` gains `key_score` (per-arm hit rate and violation rate, deterministic, no judge). Must-nots are negation-aware ("never name X" is not a leak), skip bracketed placeholders, and for contribution scan only the drafted message; a scenario that needs a draft and has none counts as a violation (`no_draft`). Results land in `results.json` as `answer_key` next to the pairwise verdict. `rival_eval.py keys SKILL RUNDIR` rescores cached outputs offline and fails on a missing arm directory. `check` rejects an unkeyed or malformed key. Tests: `scripts/test_rival_answer_keys.py`. diff --git a/scripts/ci-tests.d/rival-answer-keys.sh b/scripts/ci-tests.d/rival-answer-keys.sh new file mode 100644 index 000000000..182a97d19 --- /dev/null +++ b/scripts/ci-tests.d/rival-answer-keys.sh @@ -0,0 +1,2 @@ +# Registers the rival answer-key scorer tests with CI (see the "Per-PR tests" step in ci.yml). Offline; no model is called. +python3 scripts/test_rival_answer_keys.py diff --git a/scripts/eval-fixtures/rival/business-ops/keys.json b/scripts/eval-fixtures/rival/business-ops/keys.json new file mode 100644 index 000000000..08abcc51f --- /dev/null +++ b/scripts/eval-fixtures/rival/business-ops/keys.json @@ -0,0 +1,123 @@ +{ + "07": { + "must": { + "unit_margin_37": "\\b37\\b", + "direct_margin_72": "\\b72\\b", + "volume_ratio": "1\\.9\\d?\\s*(x|times)|1\\.95|nearly (twice|double)|(roughly|about|almost) (twice|double)", + "inventory_cash_540k": "(? 0.5 for t in tsh): bad.append(f"heldout {h['id']} overlaps a tune prompt by more than half") + return bad + check_keys(skill) + + +def load_keys(skill): + """-> {scenario id: {"must": {name: regex}, "mustnot": {...}, "mustnot_draft": {...}}} from /keys.json, or {} without one.""" + f = FIX / skill / "keys.json" + return json.loads(f.read_text()) if f.is_file() else {} + + +def check_keys(skill): + """Answer-key hygiene (empty = clean): every held-out id keyed, 3-6 must-hit and 1-3 must-not items, every regex compiles.""" + keys, bad = load_keys(skill), [] + if not keys: + return bad + for sc in load_scenarios(skill, "heldout"): + k = keys.get(sc["id"]) + if not k: + bad.append(f"keys.json: no key for heldout {sc['id']}") + continue + no = len(k.get("mustnot", {})) + len(k.get("mustnot_draft", {})) + if not 3 <= len(k.get("must", {})) <= 6 or not 1 <= no <= 3: + bad.append(f"keys.json {sc['id']}: needs 3-6 must and 1-3 mustnot items") + for i, k in keys.items(): + for grp in ("must", "mustnot", "mustnot_draft"): + for name, rx in k.get(grp, {}).items(): + try: + re.compile(rx) + except re.error as e: + bad.append(f"keys.json {i}/{name}: bad regex ({e})") return bad +def draft_text(t): + """Only the parts of an answer that read as a drafted public message: > quoted lines and fenced blocks.""" + out, fence = [], False + for ln in t.splitlines(): + if ln.lstrip().startswith("```"): + fence = not fence + elif fence or ln.lstrip().startswith(">"): + out.append(ln) + return "\n".join(out) + + +NEG = re.compile(r"\b(never|don'?t|do not|doesn'?t|won'?t|shouldn'?t|avoid|not|no|without|omit|exclude|instead of)\b", re.I) + + +def violated(rxs, text): + """Names of must-not regexes that match inside a sentence with no negation word before the match, so "never promise a + guaranteed profit" is not a violation. [placeholder] and spans are blanked first.""" + text = re.sub(r"\[[^\]]*\]|<[^>]*>", " ", text) + out = [] + for n, rx in rxs.items(): + for s in re.split(r"(?<=[.!?])\s+|\n", text): + if any(not NEG.search(s[:m.start()]) for m in re.finditer(rx, s, re.I)): + out.append(n) + break + return out + + +def key_score(skill, outs): + """Deterministic outcome score: per arm, mean fraction of must-hit items found (hit_rate) and share of scenarios with any + must-not hit (violation_rate). outs = {arm: {scenario id: answer text}}; only ids with a key and every arm's answer count. + Regex only, case-insensitive; must-nots skip negated sentences (see `violated`); `mustnot_draft` is checked against the + drafted-message lines only; `needs_draft` keys flag `no_draft` (counted as a violation) when the answer has none. + ponytail: keyword match proves presence, not quality; upgrade to a PCSS item-presence judge if regex misses paraphrases.""" + keys = load_keys(skill) + ids = [i for i in keys if all(i in outs[a] for a in outs)] + res = {} + for a in outs: + per = {} + for i in ids: + t, k = outs[a][i], keys[i] + hit = [n for n, rx in k["must"].items() if re.search(rx, t, re.I | re.S)] + d = draft_text(t) + bad = violated(k.get("mustnot", {}), t) + violated(k.get("mustnot_draft", {}), d) + no_draft = bool(k.get("needs_draft")) and not d.strip() # asked for a drafted message, none in a > quote or code fence + per[i] = dict(hits=len(hit), of=len(k["must"]), missed=sorted(set(k["must"]) - set(hit)), violated=bad, no_draft=no_draft) + n = len(per) + res[a] = dict(n=n, hit_rate=round(sum(p["hits"] / p["of"] for p in per.values()) / n, 3) if n else None, + violation_rate=round(sum(bool(p["violated"]) or p["no_draft"] for p in per.values()) / n, 3) if n else None, per_scenario=per) + return res + + def arm_text(skill, arm): cfg = config(skill) if arm == "PERUN": @@ -386,6 +467,8 @@ def both(sc, model=model): spend_usd=round(st.spent, 4), per_scenario=sc_scores, errors=errs, stopped=stopped, perun_sha256=sha(systems["PERUN"])[:16], rival_sha256=sha(systems["RIVAL"])[:16], bootstrap="10000 resamples, seed 1, percentile, over scenarios") + if load_keys(skill): + res["answer_key"] = {a: {k: x for k, x in r.items() if k != "per_scenario"} for a, r in key_score(skill, outs).items()} if v: lo, hi = boot(v) res.update(win_rate=round(sum(v) / len(v), 3), ci95=[round(lo, 3), round(hi, 3)]) @@ -471,6 +554,9 @@ def main(argv=None): w.add_argument("--force", action="store_true", help="overwrite a frozen held-out set") c = sub.add_parser("changed") c.add_argument("base", nargs="?", default="origin/main") + q = sub.add_parser("keys", help="rescore cached outputs against the skill's answer key (offline, no model call)") + q.add_argument("skill") + q.add_argument("dir", help="a run dir holding outputs//-.txt") k = sub.add_parser("check") k.add_argument("skills", nargs="*") a = ap.parse_args(argv) @@ -478,6 +564,12 @@ def main(argv=None): return write_set(a.skill, a.split, a.force) if a.cmd == "changed": return print("\n".join(changed_skills(a.base))) + if a.cmd == "keys": + for x in ARMS: + if not (Path(a.dir) / "outputs" / x).is_dir(): + raise SystemExit(f"keys: missing arm directory {Path(a.dir) / 'outputs' / x}") + outs = {x: {f.stem.rsplit("-", 1)[0]: f.read_text() for f in (Path(a.dir) / "outputs" / x).glob("*.txt")} for x in ARMS} + return print(json.dumps({x: {k: v for k, v in r.items() if k != "per_scenario"} for x, r in key_score(a.skill, outs).items()}, indent=1)) if a.cmd == "check": rc = 0 for s in a.skills or skills(): diff --git a/scripts/test_rival_answer_keys.py b/scripts/test_rival_answer_keys.py new file mode 100644 index 000000000..7388edf0f --- /dev/null +++ b/scripts/test_rival_answer_keys.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Tests for the answer-key scorer in scripts/rival_eval.py (key_score, violated, check_keys, draft_text, `keys` CLI). Offline; no model is called.""" +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import rival_eval as r # noqa: E402 + +ROWS = ("business-ops", "growth-analytics", "idea-critic", "contribution") + + +def score(skill, i, text): + return r.key_score(skill, {"A": {i: text}})["A"]["per_scenario"][i] + + +class Keys(unittest.TestCase): + def test_every_heldout_scenario_keyed_and_clean(self): + for s in ROWS: + self.assertEqual(r.check_keys(s), [], s) + self.assertEqual(set(r.load_keys(s)), {x["id"] for x in r.load_scenarios(s, "heldout")}, s) + + def test_check_split_includes_keys(self): + for s in ROWS: + self.assertEqual(r.check_split(s), [], s) + + def test_echo_baseline_scenario_text_alone_scores_under_half(self): + """A key that the prompt itself satisfies measures reading, not answering.""" + worst = 0.0 + for s in ROWS: + scs = r.load_scenarios(s, "heldout") + per = r.key_score(s, {"E": {x["id"]: x["prompt"] for x in scs}})["E"]["per_scenario"] + for i, p in per.items(): + self.assertLess(p["hits"] / p["of"], 0.5, f"{s} {i} hit by its own prompt: {set(r.load_keys(s)[i]['must']) - set(p['missed'])}") + worst = max(worst, p["hits"] / p["of"]) + print(f"echo-baseline max hit-rate {worst:.2f}") + + def test_no_perun_verdict_tokens_in_idea_critic_keys(self): + blob = json.dumps(r.load_keys("idea-critic")) + for tok in ("HOLD", "REVISE", "PASS_TO_USER"): + self.assertNotIn(tok, blob) + + def test_check_keys_flags_size_and_regex(self): + orig = r.load_keys + r.load_keys = lambda s: {"07": {"must": {"a": "("}, "mustnot": {}}} + try: + bad = r.check_keys("business-ops") + finally: + r.load_keys = orig + self.assertTrue(any("needs 3-6 must" in b for b in bad)) + self.assertTrue(any("bad regex" in b for b in bad)) + self.assertTrue(any("no key for heldout 08" in b for b in bad)) + + def test_scores_hits_and_violations(self): + good = ("Margin is $37 per unit against $72 direct, so you need about 1.95x the volume. Funding 5,000 units ties up $540,000 " + "of cash, so counter on price.") + o = r.key_score("business-ops", {"PERUN": {"07": good}, "RIVAL": {"07": "Accept: guaranteed profit, no downside."}}) + self.assertEqual((o["PERUN"]["hit_rate"], o["PERUN"]["violation_rate"]), (1.0, 0.0)) + self.assertEqual((o["RIVAL"]["hit_rate"], o["RIVAL"]["violation_rate"]), (0.0, 1.0)) + self.assertEqual(o["RIVAL"]["per_scenario"]["07"]["violated"], ["no_risk"]) + + def test_only_commonly_answered_ids_count(self): + o = r.key_score("business-ops", {"PERUN": {"07": "x", "08": "x"}, "RIVAL": {"07": "x"}}) + self.assertEqual((o["PERUN"]["n"], o["RIVAL"]["n"]), (1, 1)) + + def test_negation_is_not_a_violation(self): + self.assertEqual(score("business-ops", "07", "Do not expect guaranteed profit here.")["violated"], []) + self.assertEqual(score("business-ops", "07", "Expect guaranteed profit here.")["violated"], ["no_risk"]) + self.assertEqual(score("idea-critic", "h01", "I cannot recommend either path.")["violated"], ["no_rec"]) + self.assertEqual(score("contribution", "07", "```\nHi Jane, we at Acme Logistics hit it.\n```")["violated"], ["leaks_company"]) + self.assertEqual(score("contribution", "07", "```\nHi Jane. We never name Acme Logistics.\n[Acme Logistics] and redacted\n```")["violated"], []) + + def test_paraphrase_forms_hit(self): + for t in ("needs $540,000", "needs 540000", "needs $540k", "needs 540K of cash"): + self.assertIn("inventory_cash_540k", set(r.load_keys("business-ops")["07"]["must"]) - set(score("business-ops", "07", t)["missed"]), t) + for t in ("payroll of $100,000", "payroll of 100k", "two hires cost 100,000"): + self.assertEqual(score("business-ops", "08", t)["missed"].count("payroll_100k"), 0, t) + for t in ("breakeven churn is 13%", "breakeven churn is 13 percent", "about 13.04%"): + self.assertEqual(score("idea-critic", "h02", t)["missed"].count("breakeven_churn"), 0, t) + self.assertEqual(score("business-ops", "07", "costs $5,400,000")["missed"].count("inventory_cash_540k"), 1) # no substring match + + def test_plain_words_answer_equals_perun_vocabulary_answer(self): + perun = ("Recommendation: PASS_TO_USER with REVISE. Kill criteria: stop if churn tops 13%. Pilot a segment first, grandfather " + "existing accounts, phase the rise by tier.") + plain = ("I'd go with a phased rise. Walk away if churn passes 13 percent. Try it on a sample of accounts first, " + "keep current customers on their old price, and move up by tier.") + a, b = score("idea-critic", "h02", perun), score("idea-critic", "h02", plain) + self.assertEqual((a["hits"], a["missed"]), (b["hits"], b["missed"])) + self.assertEqual(a["hits"], a["of"]) + + def test_draft_scope_and_no_draft(self): + self.assertEqual(r.draft_text("a\n```\nb\n```\n> c\nd"), "b\n> c") + self.assertTrue(score("contribution", "07", "Advice only, nothing to paste.")["no_draft"]) + self.assertFalse(score("contribution", "07", "> Hi Jane, thanks.")["no_draft"]) + self.assertFalse(score("business-ops", "07", "no draft needed")["no_draft"]) + self.assertEqual(score("contribution", "07", "Never name Acme Logistics.\n> Hi Jane, thanks.")["violated"], []) + o = r.key_score("contribution", {"A": {"07": "no draft here"}})["A"] + self.assertEqual(o["violation_rate"], 1.0) + + def test_force_push_in_draft_any_flag_position(self): + def f(t): + return score("contribution", "14", t)["violated"] + self.assertEqual(f("```\ngit push --force-with-lease origin x\n```"), []) + self.assertEqual(f("```\ngit push --force origin x\n```"), ["bare_force_push"]) + self.assertEqual(f("```\ngit push origin x --force\n```"), ["bare_force_push"]) + self.assertEqual(f("```\ngit push origin x -f\n```"), ["bare_force_push"]) + self.assertEqual(f("```\n# never run git push --force\n```"), []) + self.assertEqual(f("Never use git push --force on a shared branch."), []) + + def test_keys_cli_rescoring_cached_outputs(self): + with tempfile.TemporaryDirectory() as d: + for arm, txt in (("PERUN", "37 72 1.95x $540k counter"), ("RIVAL", "nothing")): + p = Path(d) / "outputs" / arm + p.mkdir(parents=True) + (p / "07-abcd1234.txt").write_text(txt) + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + r.main(["keys", "business-ops", d]) + out = json.loads(buf.getvalue()) + self.assertEqual((out["PERUN"]["hit_rate"], out["RIVAL"]["hit_rate"]), (1.0, 0.0)) + + def test_keys_cli_fails_on_missing_arm_directory(self): + with tempfile.TemporaryDirectory() as d: + p = Path(d) / "outputs" / "PERUN" + p.mkdir(parents=True) + (p / "07-abcd1234.txt").write_text("x") + with self.assertRaises(SystemExit) as cm: + r.main(["keys", "business-ops", d]) + self.assertIn("missing arm directory", str(cm.exception)) + + +if __name__ == "__main__": + unittest.main()