From f2a8c07bfc59ec6b58197bfa1071d48ce8f32578 Mon Sep 17 00:00:00 2001
From: tzhouam <tzhouam@connect.ust.hk>
Date: Sun, 27 Sep 2026 02:40:59 +0800
Subject: [PATCH] Keep archived review replay independent of live CI

---
 eval/dataset/REVIEWBOT_EVAL.md            |  3 +++
 eval/dataset/reviewbot_archived_replay.py | 14 ++++++++++++++
 test/test_reviewbot_eval.py               | 20 ++++++++++++++++++++
 3 files changed, 37 insertions(+)

diff --git a/eval/dataset/REVIEWBOT_EVAL.md b/eval/dataset/REVIEWBOT_EVAL.md
index 3c4d1f5a..aabd5af7 100644
--- a/eval/dataset/REVIEWBOT_EVAL.md
+++ b/eval/dataset/REVIEWBOT_EVAL.md
@@ -42,6 +42,9 @@ their pinned historical open snapshot to the installed ReviewBot Direct
 pipeline. It requires `POST_MODE=shadow`, `REVIEW_CONTEXT_MODE=no_discussion`,
 and `post=False`, and rejects every non-GET GitHub request. ReviewBot's
 production rule remains open PRs only (drafts require an explicit request).
+The adapter disables live CI-status and branch-rule reads: those results were
+not frozen with the historical PRs, so newer ReviewBot releases receive
+`ci_evidence.read_status=unavailable` rather than today's post-merge checks.
 For a future live open-PR dataset, invoke `run_reviewbot_arm.py` directly
 without the replay flag.
 
diff --git a/eval/dataset/reviewbot_archived_replay.py b/eval/dataset/reviewbot_archived_replay.py
index 7d075190..a25d1cc3 100644
--- a/eval/dataset/reviewbot_archived_replay.py
+++ b/eval/dataset/reviewbot_archived_replay.py
@@ -81,6 +81,19 @@ def historical_pull(pull, *, pr: int, head: str, base: str):
     )
 
 
+def disable_live_ci(github) -> None:
+    """Historical CI was not frozen with the PR; never read today's checks.
+
+    Installed ReviewBot releases that support a CI snapshot then mark it
+    unavailable. Older paired wheels do not need these methods, so the same
+    replay adapter remains compatible with both release generations.
+    """
+    for method in (
+        "list_commit_statuses", "list_check_runs", "list_required_status_checks",
+    ):
+        setattr(github, method, None)
+
+
 def shadow_candidate(body: str, inline_comments: list[dict]) -> str:
     """Include the inline payload that a COMMENT review would publish.
 
@@ -181,6 +194,7 @@ def list_files(number):
 
     github.get_pull = get_pull
     github.list_files = list_files
+    disable_live_ci(github)
     original_inline = publication.inline_review_comments
     captured: list[dict] | None = None
 
diff --git a/test/test_reviewbot_eval.py b/test/test_reviewbot_eval.py
index 08b72b8b..79d4703f 100644
--- a/test/test_reviewbot_eval.py
+++ b/test/test_reviewbot_eval.py
@@ -101,6 +101,26 @@ class Pull:
             archive.historical_pull(stale, pr=4893, head="a" * 40, base="d" * 40)
 
 
+def test_archived_replay_cannot_read_live_ci():
+    class Client:
+        def list_commit_statuses(self, *_args, **_kwargs):
+            raise AssertionError("live CI must not be read")
+
+        def list_check_runs(self, *_args, **_kwargs):
+            raise AssertionError("live CI must not be read")
+
+        def list_required_status_checks(self, *_args, **_kwargs):
+            raise AssertionError("live CI must not be read")
+
+    client = Client()
+    archive.disable_live_ci(client)
+
+    for method in (
+        "list_commit_statuses", "list_check_runs", "list_required_status_checks",
+    ):
+        assert getattr(client, method) is None
+
+
 def test_shadow_candidate_includes_the_posted_inline_payload():
     body = "## Omni ReviewBot review\n\nSummary\n\nSee inline comments below.\n"
     comment = {