bench: answer keys and outcome scorer for four rival rows

changelog.d/bench-answer-keys.md

@@ -0,0 +1,2 @@
+### Added
+- Bench: answer keys (`keys.json`) for all 40 held-out scenarios of the business-ops, growth-analytics, idea-critic and contribution rival rows, 10 per row, 3-6 must-hit and 1-3 must-not regex items each. Keys test what an answer adds beyond the prompt (an echo baseline of each prompt scores at most 0.25 hit rate), accept number forms (`$280k`, `100,000`, `13 percent`) and paraphrases, and use no Perun verdict tokens. `scripts/rival_eval.py` gains `key_score` (per-arm hit rate and violation rate, deterministic, no judge). Must-nots are negation-aware ("never name X" is not a leak), skip bracketed placeholders, and for contribution scan only the drafted message; a scenario that needs a draft and has none counts as a violation (`no_draft`). Results land in `results.json` as `answer_key` next to the pairwise verdict. `rival_eval.py keys SKILL RUNDIR` rescores cached outputs offline and fails on a missing arm directory. `check` rejects an unkeyed or malformed key. Tests: `scripts/test_rival_answer_keys.py`.

scripts/ci-tests.d/rival-answer-keys.sh

@@ -0,0 +1,2 @@
+# Registers the rival answer-key scorer tests with CI (see the "Per-PR tests" step in ci.yml). Offline; no model is called.
+python3 scripts/test_rival_answer_keys.py

scripts/eval-fixtures/rival/business-ops/keys.json

@@ -0,0 +1,123 @@
+{
+ "07": {
+  "must": {
+   "unit_margin_37": "\\b37\\b",
+   "direct_margin_72": "\\b72\\b",
+   "volume_ratio": "1\\.9\\d?\\s*(x|times)|1\\.95|nearly (twice|double)|(roughly|about|almost) (twice|double)",
+   "inventory_cash_540k": "(?<![\\d,.])[$\u00a3\u20ac]?540,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?540k\\b",
+   "counter_offer": "counter|negotiat|minimum order|tiered|price floor|volume tier"
+  },
+  "mustnot": {
+   "commits_followon": "20,?000 (units )?(are|is|will be) (committed|guaranteed|certain)",
+   "no_risk": "guaranteed profit|no downside|risk[- ]free"
+  }
+ },
+ "08": {
+  "must": {
+   "liability_shield": "veil|liability shield|separate legal|asset protection",
+   "state_registration": "foreign (qualification|registration)|nexus|register(ed)? in|state (tax|filing|registration)",
+   "payroll_100k": "(?<![\\d,.])[$\u00a3\u20ac]?100,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?100k\\b",
+   "pro_threshold": "(CPA|tax (professional|advisor|adviser)|accountant|attorney|lawyer|solicitor|counsel).{0,120}(\\$\\s?\\d|\\d{2,3}k|\\d+ (employees|hires)|first (hire|employee)|payroll)|(\\$\\s?\\d|\\d{2,3}k|\\d+ (employees|hires)|first (hire|employee)|payroll).{0,120}(CPA|tax (professional|advisor|adviser)|accountant|attorney|lawyer|solicitor|counsel)"
+  },
+  "mustnot": {
+   "definitive_entity": "you (should|must) (definitely )?(form|elect|choose) an? (S-?corp|LLC) (now|immediately|today)"
+  }
+ },
+ "09": {
+  "must": {
+   "heavy_200": "\\b200\\b",
+   "revenue_at_risk": "(?<![\\d,.])[$\u00a3\u20ac]?60,?200(?!\\d|,\\d)|(?<![\\d,.])[$\u00a3\u20ac]?100,?200(?!\\d|,\\d)|\\b301\\b|\\b501\\b",
+   "churned_30": "\\b30 (heavy|high|power)?[- ]?(usage )?(accounts|customers|users)",
+   "breakeven": "break-?even",
+   "softer_alternative": "overage|spend cap|usage alert|committed[- ]use|bill[- ]shock (alert|protection)|prepaid"
+  },
+  "mustnot": {
+   "claims_churn_fixed": "capping (will|would) (eliminate|solve|fix) (the )?churn",
+   "guarantee": "guarantee[sd]? (retention|lower churn)"
+  }
+ },
+ "10": {
+  "must": {
+   "emi": "\\bEMI\\b",
+   "value_10k": "(?<![\\d,.])[$\u00a3\u20ac]?10,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?10k\\b",
+   "salary_gap_20k": "(?<![\\d,.])[$\u00a3\u20ac]?20,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?20k\\b",
+   "hmrc": "HMRC",
+   "adviser_before_grant": "(solicitor|lawyer|advis[eo]r).{0,80}before (you )?(grant|issu|sign|offer)|before (you )?(grant|issu|sign|offer).{0,80}(solicitor|lawyer|advis[eo]r)"
+  },
+  "mustnot": {
+   "definitive_tax": "(will|would) (pay|owe) no tax|guaranteed (tax|relief)|tax[- ]free for (sure|certain)"
+  }
+ },
+ "11": {
+  "must": {
+   "payers_2000": "\\b2,?000 (paying|paid|subscribers|payers|users|customers)",
+   "mrr_20k": "19,?9\\d\\d|\\b20k|\\$20,?000",
+   "costs_19k": "(?<![\\d,.])[$\u00a3\u20ac]?19,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?19k\\b",
+   "control_test": "A/B|control group|holdout|split test",
+   "cannibalization": "cannibal|downgrade|trade down|substitut"
+  },
+  "mustnot": {
+   "asserts_gain": "will (definitely )?increase (total )?revenue",
+   "invented_elasticity": "elasticity (is|of) -?\\d"
+  }
+ },
+ "12": {
+  "must": {
+   "special_category": "special[- ]categor|article 9|explicit consent",
+   "dpia": "DPIA|data protection impact",
+   "lawyer_trigger": "(CPA|tax (professional|advisor|adviser)|accountant|attorney|lawyer|solicitor|counsel).{0,100}(when|once|before|if you|threshold|trigger)|(when|once|before|if you|threshold|trigger).{0,100}(CPA|tax (professional|advisor|adviser)|accountant|attorney|lawyer|solicitor|counsel)|\\bDPO\\b",
+   "total_users_15k": "(?<![\\d,.])[$\u00a3\u20ac]?15,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?15k\\b"
+  },
+  "mustnot": {
+   "no_lawyer": "no (privacy )?(lawyer|counsel) (is )?needed|you are (fully )?(exempt|compliant)"
+  }
+ },
+ "13": {
+  "must": {
+   "target_fee": "(?<![\\d,.])[$\u00a3\u20ac]?24,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?24k\\b|(?<![\\d,.])[$\u00a3\u20ac]?23,?400(?!\\d|,\\d)",
+   "concentration": "concentration|both clients|two clients|single point",
+   "downside_floor": "floor|base fee|retainer|collar|minimum fee",
+   "effective_rate": "\\b15[34]\\b",
+   "value_evidence": "baseline|outcome|ROI|case stud"
+  },
+  "mustnot": {
+   "risk_free": "risk[- ]free|no risk|guarantee[sd]? (a )?(20|higher)"
+  }
+ },
+ "14": {
+  "must": {
+   "cac_50": "\\$50\\b",
+   "payback": "\\b6\\.[23]\\d?\\b",
+   "ltv_margin": "\\b(160|144)\\b",
+   "ratio": "\\b(3\\.2|3\\.1|3\\.0|2\\.9|2\\.88)\\b",
+   "m2_decline": "\\b10 ?(%|percent)",
+   "nrr": "contraction|expansion|net revenue retention|\\bNRR\\b|gross (revenue )?churn"
+  },
+  "mustnot": {
+   "unmargined_ltv": "LTV (is|=|of|equals) \\$?200\\b"
+  }
+ },
+ "15": {
+  "must": {
+   "weekly_5k": "((?<![\\d,.])[$\u00a3\u20ac]?5,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?5k\\b) ?(per|a|/|each) ?week|\\$5k",
+   "exceeds_grant": "(?<![\\d,.])[$\u00a3\u20ac]?65,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?65k\\b|(?<![\\d,.])[$\u00a3\u20ac]?130,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?130k\\b|(?<![\\d,.])[$\u00a3\u20ac]?260,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?260k\\b",
+   "net_negative": "net (negative|loss)|grant (does not|doesn't|won't) cover|exceeds the grant|more than the grant|costs more than",
+   "hours_25": "\\b25 (hours|hrs|h)\\b"
+  },
+  "mustnot": {
+   "free_money": "(pure|free) (money|profit)|grant (is|will be) (clearly )?worth it"
+  }
+ },
+ "16": {
+  "must": {
+   "vietnam_180k": "(?<![\\d,.])[$\u00a3\u20ac]?180,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?180k\\b",
+   "volume_not_revenue": "take rate|commission|GMV",
+   "minimum_performance": "minimum|quota|milestone|performance (target|threshold)",
+   "exit_clause": "terminat|revert|lose exclusivity|exit",
+   "total_1_3m": "1\\.3 million|(?<![\\d,.])[$\u00a3\u20ac]?1,?300,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?1300k\\b|\\$1\\.3m"
+  },
+  "mustnot": {
+   "volume_as_revenue": "1\\.3 million (in )?revenue|\\$?500,?000 (in )?revenue"
+  }
+ }
+}

scripts/eval-fixtures/rival/contribution/keys.json

@@ -0,0 +1,135 @@
+{
+ "07": {
+  "must": {
+   "numbered_plan": "PR ?1.{0,200}PR ?2.{0,200}PR ?3|first PR.{0,200}second PR.{0,200}third|(#1|\\b1\\)).{0,150}(#2|\\b2\\)).{0,150}(#3|\\b3\\))",
+   "each_builds": "(each|every) (PR|pull request|step|commit).{0,60}(builds?|compiles?|passes|green|CI)",
+   "dependency": "depends on|stacked|based on|merge (in )?order|follow-?up",
+   "generic_rename": "(rename|placeholder|neutral|generic) (names?|identifiers?|variables?|constants?)",
+   "tone": "thank|appreciate"
+  },
+  "mustnot_draft": {
+   "leaks_company": "Acme Logistics|warehouse code",
+   "leaks_schedule": "deploy(ment)? schedule"
+  },
+  "needs_draft": true
+ },
+ "08": {
+  "must": {
+   "advisory": "security advisories?|GitHub's private|SECURITY\\.md|security@|PGP|encrypted",
+   "no_payload": "without (the )?(payload|exploit|proof)|(do not|don't|will not|won't) (include|attach|share) (the )?(payload|exploit|proof)|(payload|exploit).{0,30}(upon|on) request|withhold|omit",
+   "timeline": "90[- ]day|\\b(30|45|60|90) days|disclosure (deadline|window|timeline)",
+   "escalation": "CERT|CVE|MITRE|coordination (center|service)"
+  },
+  "mustnot": {
+   "public_issue_advice": "(should|recommend|go ahead and) (open|file) a public issue"
+  },
+  "mustnot_draft": {
+   "leaks_company": "Globe Finance"
+  },
+  "needs_draft": true
+ },
+ "09": {
+  "must": {
+   "retry_request": "(re-?run|retry|restart) (the )?(job|check|workflow|CI)|manual retry|runner (memory|config|resources?|size)",
+   "log_evidence": "link to (the )?(log|run)|logs? (attached|below|linked)|CI (log|run) (link|url)",
+   "unrelated": "unrelated to (my|the) (change|diff|patch)|(change|diff|patch) (doesn't|does not) (touch|affect)",
+   "passes_locally": "(passes|pass(ed)?|green) (locally|on my machine)",
+   "gracious": "happy to (help|adjust|test)|thanks for (maintaining|your time)|appreciate"
+  },
+  "mustnot_draft": {
+   "leaks_fork": "Zenith|internal (fork|CI|setup)"
+  },
+  "needs_draft": true
+ },
+ "10": {
+  "must": {
+   "acknowledge": "understand (the|your) (preference|goal|design|reasoning)|appreciate",
+   "joint_benchmark": "(shared|joint|public|reproducible|common) (benchmark|test|harness)",
+   "middle_ground": "middle ground|compromise|hybrid|behind a flag|opt-in",
+   "synthetic": "synthetic|generic workload|public (data|dataset)|representative",
+   "interim": "interim|temporary|stopgap|in the meantime|backport"
+  },
+  "mustnot_draft": {
+   "leaks_numbers": "BlueWave|15 ?%|fifteen percent"
+  },
+  "needs_draft": true
+ },
+ "11": {
+  "must": {
+   "no_api_change": "(no|without) (any )?(public )?API (change|changes|break)|backward[- ]compat|opt-in|off by default|behind a flag",
+   "rfc_first": "(before|prior to) (sending|opening|writing) (a|any) (PR|pull request|code)|\\bRFC\\b|design discussion|gauge interest",
+   "small_scope": "small|minimal|self-contained|isolated|separate (module|file|header)|maintenance (cost|burden)",
+   "will_maintain": "(I|we)('| wi)ll (maintain|own|keep|add tests)|happy to (maintain|own)|commit(ted)? to maintain",
+   "generic_case": "(any|every|all) (application|user|client)s?|(general|generic)[- ]purpose|(common|typical) (use case|workload)"
+  },
+  "mustnot_draft": {
+   "leaks_business": "Apex|latency target"
+  },
+  "needs_draft": true
+ },
+ "12": {
+  "must": {
+   "preferences": "if you have (a )?(preferred|preference)|prefer(red)? (test|commit|style)",
+   "small_patch": "single (commit|file|change)|minimal (patch|diff|fix)|small (patch|diff|fix)",
+   "follow_existing": "follow(ing)? (the )?existing|match(es|ing)? (the )?(existing|surrounding)",
+   "fallback": "(if|in case) (you('re| are)|this is) (no longer|not) (maintain|accept|review)|(happy|glad) to (maintain|fork|help co-?maintain)|no pressure",
+   "test_included": "(regression|unit) test.{0,60}(included|attached|covers|reproduc)|added a (regression )?test"
+  },
+  "mustnot_draft": {
+   "leaks_internal": "Silent Hill|session (architecture|structure)|internal test"
+  },
+  "needs_draft": true
+ },
+ "13": {
+  "must": {
+   "ask": "(what|how) (do you|would you) (prefer|want)|could you (confirm|advise|point me)",
+   "maintainer_regenerates": "(maintainer|you|someone)( can| could| to| should)? (run|re-?generate|regenerate)|regenerat(e|ion) (on|during|at) (merge|release)",
+   "no_hand_edit": "(I|we) (have not|haven't|didn't|did not) (hand-?edit|edit)",
+   "changelog": "changelog (entry )?(will be|is) (added|generated) (by|at)|leave (the )?changelog|not (touching|editing) the changelog"
+  },
+  "mustnot_draft": {
+   "leaks_bypass": "DataCrunch|bypass|workaround"
+  },
+  "needs_draft": true
+ },
+ "14": {
+  "must": {
+   "interactive": "git rebase (-i|--interactive)",
+   "backup": "git branch (backup|bak)|backup branch|reflog",
+   "force_with_lease": "--force-with-lease",
+   "comments_persist": "(comments|review|conversation).{0,60}(stay|remain|persist|preserved|are not lost)",
+   "reword": "\\breword\\b|\\bfixup\\b|filter-repo|git commit --amend"
+  },
+  "mustnot_draft": {
+   "bare_force_push": "git push\\b[^\\n]*?(\\s-f\\b|\\s--force(?![-\\w]))",
+   "leaks_org": "Prism Solutions"
+  },
+  "needs_draft": true
+ },
+ "15": {
+  "must": {
+   "acknowledge": "thank(s| you) (for|to)|appreciate (the|everyone's) (work|effort)",
+   "root_cause": "(initiali[sz]ation|initial count|initial value).{0,80}(root cause|underlying|actual cause)",
+   "open": "(would|if) (you|the maintainers) (be open|like|prefer)|happy to (close|withdraw|fold)|not (trying|meaning) to (compete|pile)",
+   "test": "(regression|reproduc(tion|er)|failing) test",
+   "refs": "#\\d+|(PR|pull request) (#|number)"
+  },
+  "mustnot_draft": {
+   "leaks_setup": "Velocity Systems|stress test|modified (version|broker)|internal"
+  },
+  "needs_draft": true
+ },
+ "16": {
+  "must": {
+   "separate": "(i|we)('d| would|'ll| will) (split|send|open|submit|file)[^.]{0,60}separate|(docs?|documentation)[^.]{0,40}(in|as) (a )?(separate|second|follow-?up)|(two|2) (PRs|pull requests)",
+   "ordering": "(after|once) the fix (lands|merges|is merged)|follow-?up (PR|pull request)|docs? (PR )?(follows|after)",
+   "fictional": "(fictional|made-up|hypothetical|toy|contrived) (service|counter|cache|queue|scenario|worker)",
+   "concept": "happens-before|data race|mutex|atomic|synchroni[sz]ed|immutable",
+   "ask": "(open|file) an issue|ask (the )?maintainers|check (with|whether)"
+  },
+  "mustnot_draft": {
+   "leaks_product": "Nexus Analytics|proprietary"
+  },
+  "needs_draft": true
+ }
+}

scripts/eval-fixtures/rival/growth-analytics/keys.json

@@ -0,0 +1,130 @@
+{
+ "h01": {
+  "must": {
+   "account_level": "account[- ]level|team[- ]level|per[- ]account|renewal cohorts",
+   "corr_analysis": "correlat|cohort analysis|regress|backtest",
+   "leading_lagging": "leading (indicator|metric)|lagging",
+   "validation_plan": "validate|test the hypothesis|before (committing|adopting)"
+  },
+  "mustnot": {
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d",
+   "overclaims": "proves|definitively (predicts|drives)"
+  }
+ },
+ "h02": {
+  "must": {
+   "event_names": "\\b(paywall[ _](viewed|shown|hit|seen)|trial[ _](started|expired|converted|ended)|session[ _](started|completed)|app[ _](opened|installed))\\b",
+   "retention_days": "day[ -]?(1|7)\\b|\\bD1\\b|\\bD7\\b|day-?one|day-?seven",
+   "defer_list": "defer|later|can wait|nice[- ]to[- ]have|phase 2|not (critical|needed) (yet|at launch)",
+   "identity": "(user|device|anonymous)[ _]id|identity|identif"
+  },
+  "mustnot": {
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h03": {
+  "must": {
+   "annualized_36k": "(?<![\\d,.])[$\u00a3\u20ac]?36,?000(?!\\d|,\\d)|(?<![\\d.])[$\u00a3\u20ac]?36k\\b",
+   "relative_vs_absolute": "relative|absolute|percentage point",
+   "guardrail": "guardrail|counter[- ]?metric|secondary metric",
+   "cycle": "(full )?(weekly|business) cycle|novelty|seasonal|two weeks|14 days",
+   "practical": "minimum detectable|\\bMDE\\b|practical significance|business significance"
+  },
+  "mustnot": {
+   "ship_on_p_value": "statistically significant,? so (ship|roll|implement)",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h04": {
+  "must": {
+   "by_channel": "by (channel|source|campaign)|\\butm\\b|traffic source",
+   "weekly_cohorts": "weekly (signup )?cohorts?|cohorts? by (signup|week|channel)",
+   "activation_event": "activation (event|rate|metric)|aha moment|time[- ]to[- ]value|first[- ]key[- ]action",
+   "retention_curve": "day[ -]?[1-7]\\b|\\bD[17]\\b|retention curve|survival",
+   "mix_shift": "mix[- ]shift|simpson|holdout|compare .{0,40}(older|prior|previous) cohorts?"
+  },
+  "mustnot": {
+   "asserts_cause": "(is|are) (clearly|definitely) (the )?(low[- ]quality|cause)",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h05": {
+  "must": {
+   "always_valid": "alpha[- ]spending|group[- ]sequential|SPRT|Bayesian|always[- ]valid",
+   "weeks_to_3200": "(?<![\\d,.])[$\u00a3\u20ac]?3,?200(?!\\d|,\\d)",
+   "alpha": "type I\\b|\\balpha\\b",
+   "baseline": "baseline (activation|rate|conversion)|current activation",
+   "stopping_rule": "pre-?regist|stopping rule|decision rule|guardrail",
+   "mde": "\\bMDE\\b|minimum detectable|power analysis|statistical power"
+  },
+  "mustnot": {
+   "peek_and_stop": "stop (as soon as|when) (it('s| is) )?significant",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h06": {
+  "must": {
+   "workout_event": "workout[ _](logged|completed|saved|started)|log(ged)?[ _]workout",
+   "progress_event": "progress[ _](viewed|opened)|view(ed)?[ _]progress",
+   "share_event": "share[ _](tapped|clicked|started|completed|initiated)",
+   "schema_future": "schema|properties|parameters|extensible",
+   "remote_config": "remote config|feature flag|server-side|kill switch"
+  },
+  "mustnot": {
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h07": {
+  "must": {
+   "no_call": "neither (a )?win|not (a )?(win|loss)|underpowered|can't (conclude|claim)|cannot (conclude|claim)|do not (ship|claim)|don't (ship|claim)",
+   "srm": "\\bSRM\\b|sample ratio",
+   "bias_sources": "peeking|novelty|winner'?s curse|day[- ]of[- ]week|weekly seasonal|full week",
+   "validate": "re-?run|replicate|holdout|extend|repeat",
+   "balance_check": "balance check|pre-?experiment|A/A|covariate"
+  },
+  "mustnot": {
+   "calls_win": "ship (it|the treatment|the new algorithm) (now|immediately)|call(ed)? (it )?a win",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h08": {
+  "must": {
+   "conversion": "free[- ]to[- ]paid|conversion rate",
+   "act_to_paid": "(activated|activation).{0,80}(convert|paid|upgrade)",
+   "leading_lagging": "leading|lagging",
+   "seven_points": "\\b7 (percentage )?points?|\\b7 ?pp\\b",
+   "payback": "payback|\\bLTV\\b|\\bARPU\\b|expansion"
+  },
+  "mustnot": {
+   "names_tool": "\\b(Amplitude|Mixpanel|Heap|Pendo)\\b",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h09": {
+  "must": {
+   "mapping": "mapping (table|layer)|alias|crosswalk|view layer",
+   "deprecation": "deprecat|sunset",
+   "convention": "object[_ -]action|snake_case|verb[_ -]noun|naming standard",
+   "historical": "backfill|dual[- ]?(write|emit)|translat",
+   "governance": "schema registry|tracking plan|lint|CI check|review gate|owner"
+  },
+  "mustnot": {
+   "delete_old": "(delete|drop) (all )?(the )?(old|legacy|duplicate) (events|data)",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ },
+ "h10": {
+  "must": {
+   "uncertainty": "confidence interval|underpowered|power analysis",
+   "subgroup_risk": "multiple (comparison|testing)|post[- ]hoc|subgroup|Simpson|pre-?regist",
+   "revenue_metric": "revenue per (user|visitor)|ARPU|LTV|retention",
+   "follow_up": "confirm|replicat|follow-?up|holdout|second test",
+   "weighting": "weighted|mix|segment size|share of (users|revenue)"
+  },
+  "mustnot": {
+   "asserts_segments": "younger users are (clearly|definitely) more price[- ]sensitive",
+   "ship_segmented": "roll out (age[- ]based|segmented) pricing (immediately|now)",
+   "fabricated_benchmark": "industry (benchmark|standard|average)s? (of|is|are|for) ~?\\d"
+  }
+ }
+}