Merge pull request #201 from JiusiServe/codex/eval-ci-snapshot-20260927

Freeze archived review replay against live CI reads

Author: tzhouam

eval/dataset/REVIEWBOT_EVAL.md

@@ -42,6 +42,9 @@ their pinned historical open snapshot to the installed ReviewBot Direct
 pipeline. It requires `POST_MODE=shadow`, `REVIEW_CONTEXT_MODE=no_discussion`,
 and `post=False`, and rejects every non-GET GitHub request. ReviewBot's
 production rule remains open PRs only (drafts require an explicit request).
+The adapter disables live CI-status and branch-rule reads: those results were
+not frozen with the historical PRs, so newer ReviewBot releases receive
+`ci_evidence.read_status=unavailable` rather than today's post-merge checks.
 For a future live open-PR dataset, invoke `run_reviewbot_arm.py` directly
 without the replay flag.
 

eval/dataset/reviewbot_archived_replay.py

@@ -81,6 +81,19 @@ def historical_pull(pull, *, pr: int, head: str, base: str):
     )
 
 
+def disable_live_ci(github) -> None:
+    """Historical CI was not frozen with the PR; never read today's checks.
+
+    Installed ReviewBot releases that support a CI snapshot then mark it
+    unavailable. Older paired wheels do not need these methods, so the same
+    replay adapter remains compatible with both release generations.
+    """
+    for method in (
+        "list_commit_statuses", "list_check_runs", "list_required_status_checks",
+    ):
+        setattr(github, method, None)
+
+
 def shadow_candidate(body: str, inline_comments: list[dict]) -> str:
     """Include the inline payload that a COMMENT review would publish.
 
@@ -181,6 +194,7 @@ def list_files(number):
 
     github.get_pull = get_pull
     github.list_files = list_files
+    disable_live_ci(github)
     original_inline = publication.inline_review_comments
     captured: list[dict] | None = None
 

test/test_reviewbot_eval.py

@@ -101,6 +101,26 @@ class Pull:
             archive.historical_pull(stale, pr=4893, head="a" * 40, base="d" * 40)
 
 
+def test_archived_replay_cannot_read_live_ci():
+    class Client:
+        def list_commit_statuses(self, *_args, **_kwargs):
+            raise AssertionError("live CI must not be read")
+
+        def list_check_runs(self, *_args, **_kwargs):
+            raise AssertionError("live CI must not be read")
+
+        def list_required_status_checks(self, *_args, **_kwargs):
+            raise AssertionError("live CI must not be read")
+
+    client = Client()
+    archive.disable_live_ci(client)
+
+    for method in (
+        "list_commit_statuses", "list_check_runs", "list_required_status_checks",
+    ):
+        assert getattr(client, method) is None
+
+
 def test_shadow_candidate_includes_the_posted_inline_payload():
     body = "## Omni ReviewBot review\n\nSummary\n\nSee inline comments below.\n"
     comment = {