diff --git a/eval/dataset/arms/copilot_v20cb_train/manifest.json b/eval/dataset/arms/copilot_v20cb_train/manifest.json new file mode 100644 index 000000000..cbf76cf65 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/manifest.json @@ -0,0 +1,34 @@ +{ + "arm": "copilot_v20cb_train", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "train" + ], + "kinds": [ + "pr_review" + ], + "n_items": 10, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "no_discussion", + "resolved": { + "moa_when": "off", + "moa_eligible_full_depth_pr": false, + "llm_mixture_members": [ + "mimo-v2.5", + "qwen3.6-plus" + ], + "strict_backend": "cursor", + "strict_backend_model": "composer-2.5", + "review_lens_backends": {}, + "agent_model": "deepseek-v4-pro[1m]", + "review_planner_model": "deepseek-v4-flash", + "review_promotion_model": "deepseek-v4-flash" + } +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4804.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4804.cost.json new file mode 100644 index 000000000..083e6cd7f --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4804.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1003.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4804/run-20260817-194336-a281f3", + "recorded_at": "2026-08-17T20:00:17", + "attempts": [ + { + "invocation_id": "inv-c3405671", + "outcome": "run:run-20260817-194336-a281f3", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-194336-a281f3", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-194336-a281f3", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.5684, + "minutes": 16.69, + "ci_minutes": 0, + "input_tokens": 1690401, + "output_tokens": 101793, + "tool_calls": 466, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.7052, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.39, + "gate": 1.64, + "review": 995.42, + "report": 0.06 + }, + "time_to_first_result_s": 1001.5 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4804.trace.json.gz", + "trace_bytes": 63929 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4804.md b/eval/dataset/arms/copilot_v20cb_train/pr4804.md new file mode 100644 index 000000000..b75cdc955 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4804.md @@ -0,0 +1,81 @@ +# Run report + +- checkout: PR-TIME TREE (head 5d3e53170aae) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 4 finding(s) below | +| Security | no finding reported | +| Docs / comments | 2 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 5 finding(s) below | + +**Validated:** +- [resolved] Abort slot leak — GPU generation runner now calls on_requests_finished; residual: NPU generation runner still missing same hook (npu_generation_model_runner.py:48-57 vs gpu_generation_model_runner.py:81-82). +- [resolved] talker2codec_raw put_req_chunk — emitted_any reads transfer_manager.put_req_chunk incremented by omni_connector_model_runner_mixin.py:1001-1002 on send. +- [resolved] Abort slot-leak concern addressed on CUDA generation runner; residual unchecked on NPU generation runner +- [resolved] NPU twin — npu_ar_model_runner already calls `on_requests_finished`; gpu_generation change mirrors pattern; NPU CI red unlikely introduced here +- [claim-verified] patch.py:1244-1251 — only caller of compute_seed_tts_wer_metrics; stereo env vars handled upstream in _pcm_s16le_to_seed_tts_wer_bytes +- [claim-refuted] PR body 'Fix some bugs: TODO' — incomplete; no enumerated bug-fix list at head. + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:~478` [major] — In `_decode_streaming_batch`, when all `codec_stream_slots` are busy and a streaming request's payload carries `finished=True` before `session.acquire()` succeeds, the code warns and calls `_finish_stream_request(..., None)`, which pops `_stream_pending_codes` without decoding (lines 477-484, 561). That request is omitted from `outputs`, so `audios[i]` stays `_empty_audio()` (lines 243, 360-362) while `forward()` still returns `OmniOutput`. With `codec_stream_slots: 8` and stage-1 `max_num_seqs: 8` in moss_tts_local.yaml, this is reachable under full concurrent load when slots are held across forwards. Prefer admission control, blocking wait, or a client-visible error over silent audio loss. (evidence: modeling_moss_tts_codec.py:477-484 `if finished: logger.warning(...); self._finish_stream_request(request_id, session, None); continue` — finish while starved drops codes and skips outputs; modeling_moss_tts_codec.py:243 `audios: list[torch.Tensor] = [empty] * num_req` and modeling_moss_tts_codec.py:360-362 only overwrite `audios[i]` from `_decode_streaming_batch` outputs; modeling_moss_tts_codec.py:561 `self._stream_pending_codes.pop(request_id, None)` clears buffered codes without decode; moss_tts_local.yaml:21 `codec_stream_slots: 8` and moss_tts_local.yaml:42 `max_num_seqs: 8`.) + +```suggestion +if finished: + raise RuntimeError( + f"MOSS codec stream request {request_id} finished before a codec " + f"stream slot became available; refusing to drop buffered codes." + ) +``` + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~48` [major] — This PR wires `on_requests_finished` into `gpu_generation_model_runner._update_request_states` so `MossTTSCodecDecoder` releases codec stream slots on abort/disconnect; `npu_generation_model_runner._update_request_states` still only removes batch entries and never calls the hook, so MOSS codec stream slots and pending code buffers leak on NPU when requests finish outside the terminal streaming payload. Mirror the GPU hook before removing finished requests. (evidence: Added by this PR: gpu_generation_model_runner.py:81-82 `if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): self.model.on_requests_finished(scheduler_output.finished_req_ids)`. Unchanged NPU gap at npu_generation_model_runner.py:48-51 `def _update_request_states(self, scheduler_output: SchedulerOutput): # remove requests for req_id in scheduler_output.finished_req_ids: self.input_batch.remove_request(req_id)` — no `on_requests_finished` call. Sibling AR runners already invoke the hook: gpu_ar_model_runner.py:942-943 and npu_ar_model_runner.py:419-420. Consumer needing cleanup: modeling_moss_tts_codec.py:564-571 `def on_requests_finished(...) ... to avoid leaking stream slots and buffered codes.`) + +Proposed change (not a ready patch): def _update_request_states(self, scheduler_output: SchedulerOutput): + if scheduler_output.finished_req_ids and hasattr(self.model, "on_requests_finished"): + self.model.on_requests_finished(scheduler_output.finished_req_ids) + for req_id in scheduler_output.finished_req_ids: + +`vllm_omni/benchmarks/patch/patch.py:1250` [major] — In `async_request_openai_audio_speech`, the `except` around `_pcm_s16le_to_seed_tts_wer_bytes` stores raw `pcm_capture` (48 kHz stereo for Local-v1.5) but still leaves `success=True`. `compute_seed_tts_wer_metrics` then treats those bytes as 24 kHz mono (`_pcm_s16le_to_f32_16k` default), producing plausible-but-wrong WER. Mark the sample failed instead of storing mislabeled PCM. (evidence: patch.py:1249-1251 `except Exception as ex: logger.warning(...); output.tts_output_pcm_bytes = bytes(pcm_capture)` — fallback stores raw streamed PCM; patch.py:1260 `output.success = True` — success set regardless of normalization failure; patch.py:117-119 documents MOSS-TTS-Local-Transformer-v1.5 streams 48 kHz stereo PCM; seed_tts_eval.py:598 `wav_16k = _pcm_s16le_to_f32_16k(pcm)` — consumer assumes default 24000 Hz mono (seed_tts_eval.py:176 `pcm_sample_rate: int = 24000`).) + +Proposed change (not a ready patch): except Exception as ex: + logger.error("Seed-TTS WER PCM normalization failed: %s", ex) + output.tts_output_pcm_bytes = None + output.success = False + output.error = f"pcm_normalization_failed: {ex}" + +`vllm_omni/platforms/npu/worker/npu_generation_model_runner.py:~783` [major] — PR makes MossTTSCodecDecoder.forward require seq_token_counts; GPU gpu_generation_model_runner._dummy_run supplies it (line 799) but NPU npu_generation_model_runner._dummy_run (780–783) still omits it while passing padded input_ids, so Moss codec warmup/profile on NPU raises RuntimeError. (evidence: modeling_moss_tts_codec.py:260-265 `token_counts = self._normalize_seq_token_counts(kwargs.get("seq_token_counts"))` then `raise RuntimeError("MossTTS codec requires seq_token_counts; otherwise concatenated codec tokens cannot be split per request.")`; gpu_generation_model_runner.py:799 `model_kwargs["seq_token_counts"] = [int(num_tokens_padded)]`; unchanged NPU sibling at npu_generation_model_runner.py:780-783 only sets runtime_additional_information with no seq_token_counts assignment.) + +Proposed change (not a ready patch): if hasattr(self.model, "get_dummy_runtime_additional_information"): + runtime_addi = self.model.get_dummy_runtime_additional_information(num_reqs) + model_kwargs["runtime_additional_information"] = runtime_addi + model_kwargs["seq_token_counts"] = [int(num_tokens_padded)] + +`vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1462` [major] — Local talker preprocess does `max_new_frames = info_dict.get('max_new_frames', [-1])[0]` without scalar normalization; a bare int raises TypeError on prefill. MossTTSDelayTalkerForGeneration (lines 471-477) still normalizes list/tuple/int — mirror that here. In-repo serving wraps the value as a one-element list, so this is latent for those paths but a regression from the prior Local preprocess block. (evidence: modeling_moss_tts_talker.py:1462 `max_new_frames = info_dict.get("max_new_frames", [-1])[0]` — subscripts without scalar guard; int input raises TypeError. modeling_moss_tts_talker.py:471-477 (unchanged Delay talker preprocess) `max_new_frames_req = info_dict.get("max_new_frames")` then list/tuple unwrap and `int(...)` with TypeError fallback. serving_speech.py:1993-1994 `params["max_new_frames"] = [request.max_new_tokens]` — in-repo online path passes a list, not bare int.) + +Proposed change (not a ready patch): max_new_frames_req = info_dict.get("max_new_frames") + if isinstance(max_new_frames_req, (list, tuple)) and max_new_frames_req: + max_new_frames = max_new_frames_req[0] + else: + max_new_frames = max_new_frames_req if max_new_frames_req is not None else -1 + +`vllm_omni/model_executor/stage_input_processors/moss_tts.py:225` [minor] — moss_tts_local (and moss_tts_realtime) route Stage 0 through new talker2codec_raw_async_chunk with initial_codec_chunk_frames, codec_streaming meta, and a finish sentinel (code_flat_numel=0), but no tests reference this processor—only tts_local_seed renames in test_omni_gpu_model_runner.py. Add CPU core_model tests for first-chunk threshold, steady chunks, and finish-sentinel behavior. (evidence: pipeline.py:94 `async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2codec_raw_async_chunk")` under MOSS_TTS_LOCAL_PIPELINE; moss_tts.py:225 `def talker2codec_raw_async_chunk(`; moss_tts.py:271-272 `initial_chunk_frames = int(cfg.get("initial_codec_chunk_frames") or 0)`; moss_tts.py:300-301 `code_flat_numel=0,` / `codec_streaming=True,`; repo grep: zero matches for talker2codec_raw_async_chunk or talker2codec under tests/; test_omni_gpu_model_runner.py:278 `extra_args={"tts_local_seed": 42},`) + +`vllm_omni/entrypoints/openai/serving_speech.py:3715` [minor] — This PR removes the qwen3_tts-only gate and writes tts_local_seed into stage-0 extra_args whenever request.seed is set. gpu_model_runner._talker_mtp_forward consumes tts_local_seed for every talker_mtp model (qwen3_tts, fish_speech, moss_tts_local, qwen3_omni), so HTTP seed now drives talker-local sampling for backends beyond the MOSS-local path without an explicit per-backend contract. Scope the write to consumers (e.g. qwen3_tts, moss_tts_local) or document the cross-backend contract. (evidence: serving_speech.py:3707 `if request.seed is not None and sampling_params_list:` — no _tts_model_type guard; serving_speech.py:3715 `stage0_params.extra_args["tts_local_seed"] = request.seed` — unconditional write. Unchanged consumer in PR-time tree: gpu_model_runner.py:1826 `seed = extra_args.get("tts_local_seed")` inside _talker_mtp_forward, which runs only when the model exposes talker_mtp (grep: qwen3_tts, fish_speech, moss_tts_local, qwen3_omni).) + +```suggestion +if self._tts_model_type in ("qwen3_tts", "moss_tts_local"): + if stage0_params.extra_args is None: + stage0_params.extra_args = {} + stage0_params.extra_args["tts_local_seed"] = request.seed +``` + +**Additional observations (beyond the comment budget):** +- `general` [minor] Abort slot-leak cleanup is wired on the CUDA generation runner (`gpu_generation_model_runner._update_request_states`); the NPU generation runner still lacks the matching `on_requests_finished` hook. +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_codec.py:729` [minor] `_build_codec` always instantiates vendored v2 via `MossAudioTokenizerV2Config.from_pretrained` with no checkpoint-version gate; legacy is only reached on exception. Both config classes share `model_type='moss-audio-tokenizer'` and strip legacy `model_type` from JSON, so config load can succeed for either layout and mismatch surfaces only in strict `load_weights`. Consider gating on architecture-discriminating fields (sampling_rate/downsample_rate/number_channels or weight key patterns), not `model_type` alone. +- `vllm_omni/model_executor/models/moss_tts/modeling_moss_tts_talker.py:1505` [minor] MOSS Local per-frame generation now runs in `talker_mtp` and mutates per-request `req_infos`; `_talker_mtp_forward` only forwards `req_infos` when `talker_mtp_accepts_req_infos` is true (production Moss Local sets it). Existing `CaptureTalkerMTP` tests omit both the flag and a `req_infos` parameter, so this runner↔talker contract is untested. +- `tests/model_executor/models/moss_tts/test_per_request_codes.py:71` [minor] `test_per_request_codes.py` guards MOSS-TTS per-request codec routing but has no `pytestmark`; Buildkite runs `tests/model_executor` with `-m 'core_model and cpu'`, so these tests are deselected. Add `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]` (see `test_talker_logits_processor.py`). + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4804.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4804.trace.json.gz new file mode 100644 index 000000000..877e7369b Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4804.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4817.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4817.cost.json new file mode 100644 index 000000000..fe1e7f0f6 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4817.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 542.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4817/run-20260817-194337-778499", + "recorded_at": "2026-08-17T19:52:37", + "attempts": [ + { + "invocation_id": "inv-580b823f", + "outcome": "run:run-20260817-194337-778499", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-194337-778499", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-194337-778499", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1656, + "minutes": 8.99, + "ci_minutes": 0, + "input_tokens": 402875, + "output_tokens": 51647, + "tool_calls": 172, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3637, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.72, + "gate": 2.24, + "review": 532.45, + "report": 0.11 + }, + "time_to_first_result_s": 539.46 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4817.trace.json.gz", + "trace_bytes": 19765 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4817.md b/eval/dataset/arms/copilot_v20cb_train/pr4817.md new file mode 100644 index 000000000..d8d67c9e7 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4817.md @@ -0,0 +1,47 @@ +# Run report + +- checkout: PR-TIME TREE (head fc423e5a9e86) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 4 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] stale `_is_blackwell` name — renamed `_is_quack_capable`; no remaining `_is_blackwell` references in repo +- [resolved] Multi-GPU heterogeneity via get_device_capability() without device_id — pre-existing at base quack_fp8.py:22; residual: homogeneous-GPU assumption unchanged +- [claim-verified] PR env override preserved: quack_fp8.py:38-40 checks env before capability; tests cover truthy on sm_120 and falsy on sm_100 +- [claim-verified] file_at_base main had `_is_blackwell()` with `>= 10`; diff is a one-line predicate tighten plus rename +- [claim-refuted] PR test-plan CI coverage: test_quack_fp8_gate.py lacks pytestmark; .buildkite/test-ready.yml:29 deselects unmarked tests under `-m 'core_model and cpu'` +- [validated] vllm_omni/patch.py:414 — sole `install_quack_fp8_patch()` caller respects `quack_enabled()` at import time; sm_120 fix prevents patch installation + +`tests/quantization/test_quack_fp8_gate.py:18` [major] — This PR adds tests/quantization/test_quack_fp8_gate.py to regression-guard the _is_quack_capable()/quack_enabled() gate change (get_device_capability()[0] >= 10 → == 10), but the module has no pytestmark. Buildkite L1/L2 'Simple · Other Test' runs pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor (.buildkite/test-ready.yml:29), which collects tests/quantization/ but deselects unmarked modules — these gate tests never execute in merge CI and the sm_120 regression can re-land silently. Add pytestmark = [pytest.mark.core_model, pytest.mark.cpu] (as in tests/diffusion/quantization/test_mxfp8_config.py:22, minus diffusion). (evidence: tests/quantization/test_quack_fp8_gate.py:18 `import pytest` — tests/quantization/test_quack_fp8_gate.py:19 `import torch` — tests/quantization/test_quack_fp8_gate.py:21 `from vllm_omni.quantization import quack_fp8` (no pytestmark in module); unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:29 `- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; unchanged by this diff, present in PR-time tree: tests/diffusion/quantization/test_mxfp8_config.py:22 `pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]`) + +```suggestion +import pytest +import torch + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +from vllm_omni.quantization import quack_fp8 +``` + +`docs/user_guide/quantization/fp8.md:~33` [minor] — The diff rewrites the quack auto-enable paragraph (~lines 47–55) to datacenter Blackwell only (`sm_100`/`sm_101`/`sm_103`, cc `10.x`), matching the `== 10` gate, but the section intro at line 33 still opens with "On Blackwell (SM 100+)", implying quack install/benefit applies to all Blackwell including sm_120/sm_121 where quack is no longer auto-enabled. Narrow the intro to datacenter Blackwell (cc 10.x) so it matches the default-off behavior documented below. (evidence: unchanged by this diff, present in the PR-time tree: docs/user_guide/quantization/fp8.md:33 `On Blackwell (SM 100+), vLLM runs FP8 linears through the FlashInfer kernel, which`; docs/user_guide/quantization/fp8.md:47-48 `It is enabled automatically once installed (no flag needed) and is **datacenter Blackwell only** (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`, e.g.`; docs/user_guide/quantization/fp8.md:51-52 `workstation/consumer Blackwell (`sm_120` / `sm_121`, compute capability `12.x`, e.g. RTX PRO 6000 / RTX 50-series) `tcgen05` is absent — so quack is **not** auto-enabled`) + +```suggestion +On datacenter Blackwell (SM 100/101/103, compute capability 10.x), vLLM runs FP8 linears through the FlashInfer kernel, which +``` + +`general` [minor] — sm_120/121 serving protected: `_is_quack_capable()` requires cc major==10 (quack_fp8.py:32), so `quack_enabled()` is False by default and `install_quack_fp8_patch()` no-ops (quack_fp8.py:102-103; patch.py:414) — per-GEMM quack→FlashInfer wrapper never installed. Residual: `warmup_quack_fp8()` gates only on `_load_quack()` (quack_fp8.py:136), not serving; fp8.md updated for sm_120/121. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`; vllm_omni/quantization/quack_fp8.py:102-103 `if not quack_enabled(): return`; vllm_omni/patch.py:414 `install_quack_fp8_patch()`; vllm_omni/quantization/quack_fp8.py:117-124 per-GEMM `quack_scaled_fp8_mm` try/except fallback to `original(...)` only when patch installed) + +`quack_fp8.py:32` [minor] — '>= 10 matched cc 12.x' regression — quack_fp8.py:32 `== 10` fixes it; residual: future cc 11.x parts with tcgen05 would also be gated off by the `== 10` check (no sm_110a mention in this PR). (evidence: unchanged-by-this-diff, present at PR base: vllm_omni/quantization/quack_fp8.py:21 `return torch.cuda.get_device_capability()[0] >= 10`; PR head: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`; tests/quantization/test_quack_fp8_gate.py:13 `Regression guard for the ``>= 10`` gate that matched cc ``12.x`` too.`; tests/quantization/test_quack_fp8_gate.py:37 `((12, 0), False), # sm_120 workstation Blackwell — the regression`) + +`quack_fp8.py:32` [minor] — Multi-GPU heterogeneity: `_is_quack_capable()` still calls `torch.cuda.get_device_capability()` without `device_id` (pre-existing at base `quack_fp8.py:22`; PR-time head `quack_fp8.py:32`). This PR only tightens the major-version gate (`== 10` vs `>= 10`); homogeneous-GPU assumption is unchanged. (evidence: unchanged by this diff in probe shape, present in PR-time tree: vllm_omni/quantization/quack_fp8.py:32 `return torch.cuda.get_device_capability()[0] == 10`; pre-existing at base quack_fp8.py:22 `return torch.cuda.get_device_capability()[0] >= 10`) + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4817.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4817.trace.json.gz new file mode 100644 index 000000000..fea365dc7 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4817.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4859.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4859.cost.json new file mode 100644 index 000000000..1ac932ec7 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4859.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1027.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4859/run-20260817-195957-8a6842", + "recorded_at": "2026-08-17T20:17:04", + "attempts": [ + { + "invocation_id": "inv-c1d79ba2", + "outcome": "run:run-20260817-195957-8a6842", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-195957-8a6842", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-195957-8a6842", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4271, + "minutes": 17.12, + "ci_minutes": 0, + "input_tokens": 1044022, + "output_tokens": 132044, + "tool_calls": 438, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6547, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.51, + "gate": 2.48, + "review": 1020.02, + "report": 0.06 + }, + "time_to_first_result_s": 1027.07 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4859.trace.json.gz", + "trace_bytes": 27155 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4859.md b/eval/dataset/arms/copilot_v20cb_train/pr4859.md new file mode 100644 index 000000000..f4421fecf --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4859.md @@ -0,0 +1,69 @@ +# Run report + +- checkout: PR-TIME TREE (head 6ff4ce59ed87) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] flash dialect omission already expected in test line 99 `{'风格':'calm'}`; residual: dense `language` field no longer mapped—undocumented in test +- [claim-verified] PR Ming MoE deploy + 44100 sample rate: constants.py SAMPLE_RATE=44100, test plan env matches model +- [claim-verified] PR test plan exports VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 for Ming MoE benchmark run +- [claim-refuted] PR 1.83% WER with VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 at HEAD: patch stores 24k pcm, eval uses env 44100 +- [claim-refuted] PR 1.83% WER with VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 — calc ratio 16000/44100÷(16000/24000)=0.544 mis-decodes 24 kHz normalized PCM +- [validated] patch.py:1215-1218 stores 24 kHz mono in tts_output_pcm_bytes before seed_tts_eval consumes it + +### 1. patch.py already normalizes `tts_output_pcm_bytes` to 24 kHz mono via `_pcm_s16le_to_seed_tts_wer_bytes` before WER consumes it (118-133, 1215-1218). + +`vllm_omni/benchmarks/data_modules/seed_tts_eval.py:~616` [major] + +WER eval at seed_tts_eval.py:616 calls `_pcm_s16le_to_f32_16k(pcm)` without rate/channels, so it reads `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE`/`CHANNELS` from `stream_pcm_format_from_env()` (188-193). With `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100`, eval mis-decodes normalized 24 kHz bytes as 44100 Hz (~0.544× duration), corrupting ASR and invalidating reported WER. Pass explicit `pcm_sample_rate=24000, channels=1` at the call site. + +> unchanged by this diff, present in PR-time tree: patch.py:118 `"""Normalize streamed raw PCM to the 24 kHz mono PCM used by Seed-TTS WER."""`; patch.py:127-131 `if int(sample_rate) != 24000 and pcm_f32.size:` / `resampler = AudioResampler(target_sr=24000)`; patch.py:1215-1218 `output.tts_output_pcm_bytes = _pcm_s16le_to_seed_tts_wer_bytes(bytes(pcm_capture), sample_rate=sample_rate, channels=channels)`; seed_tts_eval.py:616 `wav_16k = _pcm_s16le_to_f32_16k(pcm)`; seed_tts_eval.py:188-193 `if pcm_sample_rate is None or channels is None:` / `env_sample_rate, env_channels = stream_pcm_format_from_env()` + +```suggestion +wav_16k = _pcm_s16le_to_f32_16k(pcm, pcm_sample_rate=24000, channels=1) +``` + +### 2. Clients following those examples silently lose dialect control; `test_ming_shared_modules.py` only asserts non-Auto `language` is ignored, not that dialect […] + +`vllm_omni/entrypoints/openai/serving_speech.py:2627` [major] + +This diff removes `include_language` and the `request.language` → `instruction_dict['方言']` mapping from `_parse_ming_instruction_fields` / `_parse_ming_instruction`, but `examples/online_serving/text_to_speech/ming_tts/README.md:101` still documents `language` as Ming dialect control and `openai_speech_client.py:103-104` maps `--dialect` to `payload['language']`. + +> vllm_omni/entrypoints/openai/serving_speech.py:2627 `instruction_dict: dict[str, Any] = {}` — no subsequent `request.language` → `方言` assignment in `_parse_ming_instruction_fields`; vllm_omni/entrypoints/openai/serving_speech.py:2649-2652 `return self._parse_ming_instruction_fields(request, include_voice=True, plain_text_passthrough=True,)` — `include_language` removed; unchanged by this diff, present in PR-time tree: examples/online_serving/text_to_speech/ming_tts/openai_speech_client.py:103-104 `if args.dialect:` / `payload["language"] = args.dialect`; unchanged by this diff, present in PR-time tree: examples/online_serving/text_to_speech/ming_tts/README.md:101 `| \`language\` | Ming \`方言\` control |` + +```suggestion +if request.language not in (None, "", "Auto"): + instruction_dict["方言"] = request.language +``` + +### 3. audio_vae.py:351 `encode_latent` now samples `mean + std * torch.randn_like(mean)` (fixes mean-only vs VAE/HF); + +`general` [minor] + +> unchanged by this diff, present in PR-time tree: audio_vae.py:139 `self.fc3 = nn.Linear(config.hidden_size, latent_dim * 2)`; main baseline audio_vae.py:349-350 `mean = torch.chunk(h, 2, dim=1)[0]` / `latent = mean.transpose(1, 2)`; PR head audio_vae.py:349-351 `mean, scale = torch.chunk(h, 2, dim=1)` / `std = F.softplus(scale) + 1e-4` / `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)`; prompt_encoder.py:137 `latents, _ = encoder.encode_latent(waveform, waveform_length)` + +### 4. This diff switches `encode_latent` to stochastic reparameterization (audio_vae.py:348-350), but no unit test asserts this path—`test_ming_shared_modules.py` […] + +`vllm_omni/model_executor/models/common/ming/audio_vae.py:351` [minor] + +Consider a CPU test with seeded `torch.randn_like` mock proving mean/scale chunking and softplus std match HF semantics, runnable in merge CI's `tests/model_executor -m 'core_model and cpu'` lane. + +> audio_vae.py:348-350 `mean, scale = torch.chunk(h, 2, dim=1)` / `std = F.softplus(scale) + 1e-4` / `latent = (mean + std * torch.randn_like(mean)).transpose(1, 2)` — stochastic latent sampling introduced by this diff; unchanged by this diff, present in PR-time tree: tests/model_executor/models/ming_tts/test_ming_shared_modules.py:30-35 `def test_ming_tts_audio_vae_uses_common_config():` / `cfg = AudioVAEConfig(sample_rate=16000, patch_size=-1)` / `assert cfg.sample_rate == 16000` / `assert cfg.patch_size == -1` — config-only smoke, no encode_latent call; grep `encode_latent` under tests/ → no matches + +**Additional observations (beyond the comment budget):** +- `general` [minor] Residual organization: bench-only `VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE` / `VLLM_OMNI_BENCH_AUDIO_CHANNELS` now live in `vllm_omni/metrics/definitions.py` (lines 287-288) beside Prometheus/bench metric stems, after unifying callers on `stream_pcm_format_from_env()`. +- `general` [minor] Flash dialect omission is already asserted at test_ming_shared_modules.py:99 (`assert flash_fields == {"风格": "calm"}` with `language="粤语"`). Residual gap: this PR drops dense `language`→`方言` mapping in serving_speech.py but dense tests only cover `language=None`/`Auto`, so the behavior change is undocumented. +- `vllm_omni/model_executor/models/ming_tts/patch_emission.py:130` [minor] This diff changes Ming stop-head gating from min_stop_step+1 to min_stop_step+2 in _validate_ming_decode_window (patch_emission.py:130) and _resolve_ming_stop_decision (patch_emission.py:151). With default STOP_HEAD_MIN_STEPS=3 (constants.py:51), min_required_decode_steps becomes 5 instead of 4 for every Ming-TTS audio decode request on the ming_tts_llm.py path; tighter max_decode_steps configs now fail validation, and stop-head cannot fire until (step+1)>=5 (patch_emission.py:159)—one step later than before. +- `vllm_omni/metrics/definitions.py:317` [minor] Bench `VLLM_OMNI_BENCH_AUDIO_*` env parsing moved from patch-local `_audio_pcm_format` into `metrics/definitions.py::stream_pcm_format_from_env` (317–337), next to runtime `resolve_audio_sample_rate`. Only bench callers use it today, but placing bench capture knobs in a module server code already imports for runtime audio metrics weakens the prior patch-local boundary; a `benchmarks/audio_format.py` helper would keep bench env overrides out of the shared metrics surface. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4859.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4859.trace.json.gz new file mode 100644 index 000000000..6005e70cc Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4859.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4870.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4870.cost.json new file mode 100644 index 000000000..7fd56a342 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4870.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 619.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4870/run-20260817-194336-3b412c", + "recorded_at": "2026-08-17T19:53:54", + "attempts": [ + { + "invocation_id": "inv-45ac559d", + "outcome": "run:run-20260817-194336-3b412c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-194336-3b412c", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-194336-3b412c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.196, + "minutes": 10.29, + "ci_minutes": 0, + "input_tokens": 473039, + "output_tokens": 62100, + "tool_calls": 317, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.409, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.07, + "gate": 1.58, + "review": 611.8, + "report": 0.13 + }, + "time_to_first_result_s": 617.5 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4870.trace.json.gz", + "trace_bytes": 23207 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4870.md b/eval/dataset/arms/copilot_v20cb_train/pr4870.md new file mode 100644 index 000000000..44087f28b --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4870.md @@ -0,0 +1,60 @@ +# Run report + +- checkout: PR-TIME TREE (head 231b3ddb97c1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Non-qwen3 async_chunk=False streaming scoping — test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3 at :3473; residual: moss_tts/higgs/indextts2 async_chunk=False paths not individually tested +- [claim-verified] PR E2E H20 table — not re-run here; issue #4851 non-streaming concurrent `no_async_chunk` aligns with worker fix primary path +- [claim-verified] Issue #4851 non-streaming failures addressed by worker split; serving hunk is streaming-only adjunct +- [claim-verified] PR body scoping claim — grep shows `coerce_param_message_types` only changed in serving_speech; serving_chat/realtime_connection/omni_base callers untouched +- [claim-verified] Issue #4851 failing cases live in test_qwen3_tts_*_expansion.py (full_model marker), not merge-gate core_model e2e +- [validated] vllm_omni/worker/gpu_ar_model_runner.py:1734 — mm split length now `int(scheduler_output.total_num_scheduled_tokens)`; matches `to_payload_element` contract in vllm_omni/utils/mm_outputs.py:131 when batched mm dim-0 equals scheduled tokens + +`vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:~905` [minor] — GPU now splits multimodal payloads with mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) (gpu_ar_model_runner.py:1734). NPU still uses seq_len = hidden_states.shape[0] at sample_tokens (:905) and passes it to to_payload_element at :1144, so tail-only Qwen3-TTS batches on Ascend can mis-slice codes.audio. Port the GPU mm_seq_len pattern inside needs_pooler_payload when that path matters. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:905 `seq_len = hidden_states.shape[0]`; vllm_omni/platforms/npu/worker/npu_ar_model_runner.py:1144 `seq_len=seq_len,`; changed in this PR: vllm_omni/worker/gpu_ar_model_runner.py:1734 `mm_seq_len = int(scheduler_output.total_num_scheduled_tokens)`; vllm_omni/worker/gpu_ar_model_runner.py:1790 `seq_len=mm_seq_len,`; vllm_omni/utils/mm_outputs.py:131 `if seq_len is not None and isinstance(element, torch.Tensor) and element.shape[0] == seq_len:`; vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:311 `self.requires_full_prefix_cached_hidden_states = False`) + +Proposed change (not a ready patch): mm_seq_len = int(scheduler_output.total_num_scheduled_tokens) + # ... later in pooler loop ... + mm_payload[mm_key] = to_payload_element( + element=mm_val, + idx=idx, + start=start, + end=end, + pass_lists_through=False, + seq_len=mm_seq_len, + ) + +`vllm_omni/entrypoints/openai/serving_speech.py:3507` [minor] — Qwen3 full-payload streaming is gated at serving_speech.py:3507 with getattr(self.model_config, "async_chunk", True), but OmniModelConfig.async_chunk defaults to False (model.py:121), the audio path in the same file uses getattr(..., False) at 3823, and workers default to False. If async_chunk is absent, the True fallback leaves DELTA coercion enabled instead of FINAL_ONLY. Align the fallback to False. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/config/model.py:121 `async_chunk: bool = False`; vllm_omni/entrypoints/openai/serving_speech.py:3507 `async_chunk = getattr(self.model_config, "async_chunk", True)`; vllm_omni/entrypoints/openai/serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; vllm_omni/entrypoints/openai/serving_speech.py:3823 `async_chunk = bool(getattr(self.engine_client.model_config, "async_chunk", False))`; vllm_omni/worker/gpu_ar_model_runner.py:293 `self._async_chunk = getattr(self.model_config, "async_chunk", False)`) + +```suggestion +async_chunk = getattr(self.model_config, "async_chunk", False) +``` + +`serving_speech.py:3508` [minor] — Broad `async_chunk=False` FINAL_ONLY gate narrowed to qwen3_tts only (serving_speech.py:3508); residual checked: other async_chunk=False TTS families still use DELTA via voxtral fixture. (evidence: [resolved] Broad `async_chunk=False` FINAL_ONLY gate narrowed to qwen3_tts only (serving_speech.py:3508); residual checked: other async_chunk=False TTS families still use DELTA via voxtral fixture.) + +`general` [minor] — Non-qwen3 async_chunk=False streaming scoping is covered only by voxtral at test_serving_speech.py:3473; moss_tts_nano, higgs_audio_v2, and indextts2 deploy with async_chunk=false but have no dedicated delta-coercion tests. (evidence: tests/entrypoints/openai_api/test_serving_speech.py:3473 `def test_prepare_speech_generation_no_async_chunk_stream_keeps_delta_for_non_qwen3(` — sole non-qwen3 proxy uses `voxtral_server`; unchanged by this diff, present in PR-time tree: serving_speech.py:3508 `qwen3_full_payload = self._tts_model_type == "qwen3_tts" and not bool(async_chunk)`; unchanged by this diff, present in PR-time tree: vllm_omni/deploy/moss_tts_nano.yaml:14 `async_chunk: false`, vllm_omni/deploy/higgs_audio_v2.yaml:14 `async_chunk: false`, vllm_omni/deploy/indextts2.yaml:1 `async_chunk: false` — no matching per-model tests in test_serving_speech.py) + +`tests/e2e/online_serving/test_qwen3_tts_customvoice.py:~49` [minor] — Merge-gate E2E (`test-ready.yml:333`) runs `test_qwen3_tts_customvoice.py`, which only parametrizes `async_chunk` (line 49). The `--no-async-chunk` path lives in `test_qwen3_tts_customvoice_expansion.py` (`full_model`, lines 52–57). This PR fixes no-async-chunk streaming via unit tests only; merge-gate will not catch a `no_async_chunk` serving regression — add a `pytest.param` with `--no-async-chunk` to the core_model file or wire expansion into an appropriate CI lane. (evidence: unchanged by this diff, present in PR-time tree: tests/e2e/online_serving/test_qwen3_tts_customvoice.py:49 `id="async_chunk",`; tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py:55 `server_args=["--trust-remote-code", "--no-async-chunk"],` and :57 `id="no_async_chunk",`; .buildkite/test-ready.yml:333 `pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`) + +```suggestion +pytest.param( + OmniServerParams( + model=MODEL, + stage_config_path=_STAGE_CONFIG, + server_args=["--trust-remote-code", "--no-async-chunk"], + ), + id="no_async_chunk", + ), +``` + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4870.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4870.trace.json.gz new file mode 100644 index 000000000..c4e8d6d2b Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4870.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4923.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4923.cost.json new file mode 100644 index 000000000..446a779de --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4923.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1064.8, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4923/run-20260817-194335-f20184", + "recorded_at": "2026-08-17T20:01:19", + "attempts": [ + { + "invocation_id": "inv-953210a5", + "outcome": "run:run-20260817-194335-f20184", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-194335-f20184", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-194335-f20184", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.333, + "minutes": 17.74, + "ci_minutes": 0, + "input_tokens": 777569, + "output_tokens": 111852, + "tool_calls": 442, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6232, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.04, + "gate": 1.77, + "review": 1058.3, + "report": 0.07 + }, + "time_to_first_result_s": 1064.15 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4923.trace.json.gz", + "trace_bytes": 28373 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4923.md b/eval/dataset/arms/copilot_v20cb_train/pr4923.md new file mode 100644 index 000000000..d356531e5 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4923.md @@ -0,0 +1,59 @@ +# Run report + +- checkout: PR-TIME TREE (head 3b2d87c9c3bc) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] async output + prefix caching mutual exclusion — enable_prefix_caching:false at qwen3_tts.yaml:46 allows async path per gpu_ar_model_runner.py:1540-1541; residual: re-enabling prefix cache silently drops async with no test +- [claim-verified] PR root-cause (nested graphs + missing outer wrap + per-row generator scalar fallback) matches live runner/model code +- [claim-verified] PR 39ms→9ms claim is 4.33x; arithmetic consistent, runtime unverified in this step +- [claim-refuted] PR implies e2e alone validates graph layout — online e2e is functional smoke only; no assertion on CUDAGraphWrapper capture or nested-graph avoidance +- [validated] qwen3_tts_code_predictor_vllm.py:39 use_cuda_graphs=current_omni_platform.is_npu() matches qwen3_omni_moe_code_predictor_mtp.py:21 — avoids nested inner graphs under outer talker_mtp FULL wrap +- [validated] gpu_ar_model_runner.py:1537-1561 async omni output requires async_scheduling, omni_prefix_cache is None, async_chunk, and eager_omni_postprocess_before_async_output + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:346` [minor] — This diff gates talker_mtp_accepts_per_row_generators (qwen3_tts_talker.py:346–350) by re-reading cudagraph_mode.has_full_cudagraphs(), duplicating the runner’s wrap decision in _init_talker_mtp (gpu_model_runner.py:205–211). The in-file TODO(#4923) acknowledges runner ownership. Consolidate or add a single shared predicate so model and runner cannot drift and silently flip batched vs scalar MTP. (evidence: qwen3_tts_talker.py:346 `cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None)`; qwen3_tts_talker.py:347-350 `talker_mtp_graph_wrapped = (self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs())` / `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; unchanged by this diff, present in PR-time tree: gpu_model_runner.py:209 `if cudagraph_mode.has_full_cudagraphs() and (has_separate_talker or talker_mtp_graph_safe):`; qwen3_tts_talker.py:337-339 `# TODO(#4923): the model should not read ``cudagraph_mode`` — it is an upstream (runner) concern, and the runner already re-derives the same wrap decision in ``_init_talker_mtp``.`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:350` [minor] — This diff gates `talker_mtp_accepts_per_row_generators` off under `cudagraph_mode.has_full_cudagraphs()` (talker.py:350), and the model comments (334–335) state per-request `tts_local_seed` is not reproducible in that mode. Serving still injects `tts_local_seed` from `request.seed`/deploy default (serving_speech.py:3694, 3706), and `test_prepare_speech_generation_qwen3_default_seed_sets_tts_local_seed` only asserts it is set in `extra_args` (3365), not that residual codec sampling is seed-stable on the default GPU bundled deploy (`qwen3_tts.yaml` runs talker cudagraph by default). Consider documenting this limitation in serving/API guidance or warning when seeds are accepted but ignored under full cudagraphs. (evidence: unchanged by this diff, present in PR-time tree: qwen3_tts_talker.py:334-335 `# Consequence: under full cudagraphs, per-request ``tts_local_seed`` is` / `# not reproducible (matching Qwen3-Omni, which has no per-row seeding);`; qwen3_tts_talker.py:350 `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; serving_speech.py:3694 `stage0_params.extra_args["tts_local_seed"] = request.seed`; serving_speech.py:3706 `stage0_params.extra_args.setdefault("tts_local_seed", int(default_seed))`; test_serving_speech.py:3365 `assert stage0_params.extra_args["tts_local_seed"] == 42`; qwen3_tts.yaml:11 `# * Stage 0 does not set enforce_eager — talker runs cudagraph by default.`) + +`vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:~346` [minor] — This diff gates talker_mtp_accepts_per_row_generators on cudagraph_mode.has_full_cudagraphs() at talker __init__ (qwen3_tts_talker.py:346–350), but no talker unit test exercises the True/False branches: existing talker tests use __new__ and skip __init__, and the only tests/ reference to the flag manually injects True on a runner mock (test_omni_gpu_model_runner.py:374). Add a lightweight init test mocking compilation_config.cudagraph_mode to assert FULL mode sets the flag False and eager/None leaves it True. (evidence: qwen3_tts_talker.py:346 `cudagraph_mode = getattr(vllm_config.compilation_config, "cudagraph_mode", None)`; qwen3_tts_talker.py:347-350 `talker_mtp_graph_wrapped = ( self.talker_mtp_graph_safe and cudagraph_mode is not None and cudagraph_mode.has_full_cudagraphs() )` / `self.talker_mtp_accepts_per_row_generators = not talker_mtp_graph_wrapped`; test_qwen3_tts_talker_preprocess.py:28-35 `Skips ``__init__`` and ``load_weights``` / `model = Qwen3TTSTalkerForConditionalGeneration.__new__(Qwen3TTSTalkerForConditionalGeneration)`; test_omni_gpu_model_runner.py:374 `talker_mtp_accepts_per_row_generators=True,` (manually injected on runner.model SimpleNamespace, not from talker __init__)) + +```suggestion +# in tests/model_executor/models/qwen3_tts/test_qwen3_tts_talker_cudagraph_flags.py +from types import SimpleNamespace + +def test_talker_disables_per_row_generators_under_full_cudagraphs(mocker): + mode = SimpleNamespace(has_full_cudagraphs=lambda: True) + vllm_config = SimpleNamespace(compilation_config=SimpleNamespace(cudagraph_mode=mode)) + # patch Qwen3Model/ParallelLMHead/etc. to no-ops, then: + talker = Qwen3TTSTalkerForConditionalGeneration(vllm_config=vllm_config) + assert talker.talker_mtp_graph_safe is True + assert talker.talker_mtp_accepts_per_row_generators is False +``` + +`general` [minor] — Async vs prefix-cache mutual exclusion: this PR sets enable_prefix_caching:false (qwen3_tts.yaml:46) and use_async_omni_output=True (qwen3_tts_talker.py:351), so omni_prefix_cache stays None and async output can activate. Residual: re-enabling enable_prefix_caching initializes omni_prefix_cache (gpu_model_runner.py:169) and silently disables async output via gpu_ar_model_runner.py:1540-1541 — no warning is emitted. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/worker/gpu_ar_model_runner.py:1540 `if self.omni_prefix_cache is not None:`; vllm_omni/worker/gpu_ar_model_runner.py:1541 `return False`; vllm_omni/worker/gpu_model_runner.py:169 `if self.cache_config.enable_prefix_caching:`; PR diff: vllm_omni/deploy/qwen3_tts.yaml:46 `enable_prefix_caching: false`; PR diff: vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py:351 `self.use_async_omni_output = True`) + +`general` [minor] — Nested GPU code-predictor CUDA graphs vs talker_mtp FULL wrap — fixed by `use_cuda_graphs=current_omni_platform.is_npu()` (GPU False). Residual minor: `vllm_omni/deploy/qwen3_tts_high_concurrency.yaml:27` still sets `code_predictor_prefix_graphs: true`; gating at `qwen3_code_predictor.py:477` disables prefix graphs when `use_cuda_graphs` is False, so the yaml key is a no-op on GPU. (evidence: qwen3_tts_code_predictor_vllm.py:39 `use_cuda_graphs=current_omni_platform.is_npu(),` — PR diff disables GPU code-predictor graphs; unchanged by diff, present in PR-time tree: qwen3_code_predictor.py:477 `self._prefix_graphs_enabled = prefix_graphs_requested and wrapper_config.use_cuda_graphs and not is_npu` — prefix graphs require use_cuda_graphs; unchanged by diff: qwen3_tts_high_concurrency.yaml:27 `code_predictor_prefix_graphs: true` — yaml still requests prefix graphs) + +`docs/user_guide/examples/online_serving/text_to_speech.md:~334` [minor] — This diff raises bundled `qwen3_tts.yaml` stage-0/1 `max_num_seqs` from 10 to 64 for throughput, but `docs/user_guide/examples/online_serving/text_to_speech.md:334` and `docs/serving/speech_api.md:667` still document the old default of 10. Refresh both docs to state 64 is the new default and retain CustomVoice/VoiceDesign TTFA guidance (`max_num_seqs: 1`) from the yaml comment. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/deploy/qwen3_tts.yaml:38 `max_num_seqs: 64`; vllm_omni/deploy/qwen3_tts.yaml:70 `max_num_seqs: 64`; docs/user_guide/examples/online_serving/text_to_speech.md:334 `The bundled \`qwen3_tts.yaml\` ships stage 1 (Code2Wav) at \`max_num_seqs: 10\`, tuned for Base voice cloning:`; docs/serving/speech_api.md:667 `benchmark before reducing it below \`10\`.`) + +```suggestion +The bundled `qwen3_tts.yaml` ships stage 1 (Code2Wav) at `max_num_seqs: 64` for throughput; for CustomVoice/VoiceDesign TTFA, override to `max_num_seqs: 1`. +``` + +**Additional observations (beyond the comment budget):** +- `tests/model_executor/models/qwen3_tts/test_code_predictor_dtype.py:481` [minor] PR sets code-predictor `use_cuda_graphs=current_omni_platform.is_npu()` (False on GPU), but `test_tts_config` still instantiates `CodePredictorWrapperConfig` with `use_cuda_graphs=True` and asserts True—it never exercises the wrapper constructor, so the test passes while encoding the old TTS default. Rewrite it to construct `Qwen3TTSTalkerCodePredictorForConditionalGenerationVLLM` with mocked `is_npu()` False/True and assert `_wrapper_config.use_cuda_graphs` follows the split. +- `tests/worker/test_omni_gpu_model_runner.py:356` [minor] This PR's perf win is one batched `_talker_mtp_forward` per step when `talker_mtp` is graph-wrapped (`isinstance(..., get_graph_wrapper_cls())` at gpu_model_runner.py:1808) and `talker_mtp_accepts_per_row_generators=False` with no per-row seeds—the scalar fallback at :1851-1876 only fires when seeds are present. Extend `tests/worker/test_omni_gpu_model_runner.py` with a case using a fake graph wrapper and `accepts=False` without `tts_local_seed` to assert a single `batch_size==N` call; existing tests only cover scalar fallback with seeds (:317-354) and seeded batching when accepts=True (:356-393). +- `tests/e2e/offline_inference/test_qwen3_tts_customvoice.py:38` [minor] This PR changes the production full-cudagraph + async-output talker path, but merge-gate offline Qwen3-TTS tests (`test_qwen3_tts_customvoice.py` / `test_qwen3_tts_base.py`) build temp YAMLs with `enforce_eager: True` on both stages, so AMD offline `advanced_model` runs never exercise nested-graph fixes. Online E2E already uses prod `qwen3_tts.yaml` without `enforce_eager`; consider adding one offline or worker-level test that keeps cudagraphs enabled. +- `vllm_omni/deploy/qwen3_tts.yaml:98` [minor] NPU `platforms` adds `cudagraph_mode: PIECEWISE` for stages 0 and 1 (lines 98–102) without explaining why. With `talker_mtp_graph_safe=True` and `use_cuda_graphs=current_omni_platform.is_npu()`, default FULL would make `_init_talker_mtp` outer-wrap `talker_mtp` while the code predictor still captures inner NPU graphs—the nested-graph failure this PR addresses on GPU. Mirror qwen3_omni_moe.yaml:81–87: stage 0 blocks outer FULL talker_mtp wrap; stage 1 keeps Code2Wav inner decode graphs capture-safe. + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4923.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4923.trace.json.gz new file mode 100644 index 000000000..733a6abf9 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4923.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4926.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4926.cost.json new file mode 100644 index 000000000..58edc5fb0 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4926.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1118.2, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4926/run-20260817-195356-94d3bc", + "recorded_at": "2026-08-17T20:12:32", + "attempts": [ + { + "invocation_id": "inv-5dbb20c1", + "outcome": "run:run-20260817-195356-94d3bc", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-195356-94d3bc", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-195356-94d3bc", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3938, + "minutes": 18.61, + "ci_minutes": 0, + "input_tokens": 1007978, + "output_tokens": 110565, + "tool_calls": 384, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6665, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 7.17, + "gate": 2.03, + "review": 1107.01, + "report": 0.12 + }, + "time_to_first_result_s": 1116.32 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4926.trace.json.gz", + "trace_bytes": 34546 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4926.md b/eval/dataset/arms/copilot_v20cb_train/pr4926.md new file mode 100644 index 000000000..daaa1fa87 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4926.md @@ -0,0 +1,135 @@ +# Run report + +- checkout: PR-TIME TREE (head 35aa741e1962) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 7 finding(s) below | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Hopper pre-9 FA3_HUB→FLASH_ATTN_HUB — cuda/platform.py:140-148; residual: Blackwell sm_10x/sm_12x still passes gate +- [resolved] FA3 platform gate added (deeb67af) — test asserts pre-Hopper→FLASH_ATTN_HUB; residual: FA3 forward never CI-run on L4 (major<9 skip at test:92) +- [resolved] piecewise varlen fallback in _flash_wrapper (332da8f2) — no test drives full_attn_spans through FlashAttentionHubImpl +- [resolved] registry KeyError on non-CUDA explicit hub selection — NPU platform.py:110-118 now remaps hub names to FLASH_ATTN before DiffusionAttentionBackendEnum[backend_upper]; residual: no parametrized ROCm/NPU/XPU/MUSA platform tests (kept rocm:91) +- [claim-verified] PR unit-test claim: test_kernels_hub.py marked @pytest.mark.core_model @pytest.mark.cpu collects in .buildkite/test-ready.yml L1/L2. +- [claim-refuted] PR/docs 'perfect numerical consistency' — test compares Hub vs SDPA at 1e-2, no diffusers-hub parity + +### 1. Each diffusion Attention layer builds its own impl via layer.py:95 self.attention = self.attn_impl_cls(...), so a multi-layer DiT repeats Hub kernel resolution […] + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:~96` [major] + +Both FlashAttentionHubImpl and FlashAttention3HubImpl call get_kernel() in __init__ (flash_attn_hub.py:96-103, 275-282). Local FLASH_ATTN binds flash_attn_func once at fa.py import. Cache the hub module per process and assign only function pointers per impl. + +> flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)` — inside FlashAttentionHubImpl.__init__; unchanged by this diff, present in PR-time tree: layer.py:95 `self.attention = self.attn_impl_cls(` — one impl per Attention layer; unchanged by this diff, present in PR-time tree: fa.py:26 `flash_attn_func = None` with module-level import chain (lines 35-68) — local FLASH_ATTN loads once at import, not per-layer __init__ + +```suggestion +@functools.lru_cache(maxsize=1) +def _load_flash_attn2_hub_module(): + from kernels import get_kernel + for version in (1, 2, None): + try: + return get_kernel("kernels-community/flash-attn2", version=version) if version else get_kernel("kernels-community/flash-attn2") + except Exception: + continue + raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub") + +# in __init__: +hub_module = _load_flash_attn2_hub_module() +``` + +### 2. `test_kernels_hub_execution` (`@hardware_test(res={'cuda': 'L4'})`, `@pytest.mark.core_model`) is not collected by PR CUDA diffusion CI […] + +`tests/diffusion/attention/test_kernels_hub.py:58` [major] + +tests/diffusion/*.py -m 'core_model and cuda'`, which excludes `tests/diffusion/attention/test_kernels_hub.py`. The CPU fallback test in the same file (`@pytest.mark.cpu`) is collected by line 13 (`tests/diffusion -m 'core_model and cpu'`). Add a CI job for `tests/diffusion/attention/` if the Hub kernel forward path should be gated in PR CI. + +> tests/diffusion/attention/test_kernels_hub.py:56-58 `@pytest.mark.core_model` / `@hardware_test(res={'cuda': 'L4'}, num_cards=1)` / `def test_kernels_hub_execution():`; unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:145 `timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"` (glob limits collection to top-level `tests/diffusion/*.py`, not `attention/`); .buildkite/test-ready.yml:13 `timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu'` (recursive `tests/diffusion` collects CPU-marked subdirectory tests); tests/helpers/mark.py:12 `test_platform_detail = pytest.mark.cuda` (hardware_test adds cuda mark, so execution test matches cuda filter but wrong path scope excludes the file). + +Proposed change (not a ready patch): # In .buildkite/test-ready.yml Diffusion group, add e.g.: +- label: "Diffusion · Attention Test" + commands: + - timeout 15m pytest -sv tests/diffusion/attention/ -m 'core_model and cuda and L4' --run-level "core_model" + +### 3. `FlashAttentionHubImpl.__init__` then calls `get_kernel` with no fallback to local FLASH_ATTN—the final default `get_kernel` at flash_attn_hub.py:103 is […] + +`vllm_omni/platforms/cuda/platform.py:~126` [major] + +CUDA platform selection only probes `importlib.import_module('kernels')` and falls back on ImportError (cuda/platform.py:125-138), but once `kernels` imports the Hub backend is selected. Docs promise graceful fallback to `FLASH_ATTN` when kernels is unavailable (attention_backends.md:231), so air-gapped or Hub-fetch failures with `kernels` installed can hard-fail at model construction instead of re-routing. + +> cuda/platform.py:125-128 `try: importlib.import_module("kernels")` / `except ImportError:` — fallback only when the package cannot be imported; flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")` — final Hub load attempt is not wrapped in except; flash_attn_hub.py:109 `raise RuntimeError("Failed to load flash-attn2 kernel from HuggingFace Hub: no functions found")`; docs/user_guide/diffusion/attention_backends.md:231 `If the kernels library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (FLASH_ATTN).` + +Proposed change (not a ready patch): Wrap get_kernel in the shared loader with ImportError/RuntimeError handling that logs and returns None, and in platform selection (or impl init) re-resolve to FLASH_ATTN when loader returns None. + +### 4. Extend the flash-like backend check to include hub backends so masked LTX-2 runs with --diffusion-attention-backend FLASH_ATTN_HUB receive correctly shaped 2D […] + +`vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:~397` [major] + +This PR adds FLASH_ATTN_HUB/FLASH_ATTN_3_HUB backends whose masked path asserts a 2D padding mask (flash_attn_hub.py:141), but LTX-2 still converts masks only when get_name().upper() == 'FLASH_ATTN' (ltx2_transformer.py:397). + +> unchanged by this diff, present in PR-time tree: ltx2_transformer.py:397-398 `if attn.attn.attn_backend.get_name().upper() == "FLASH_ATTN":` / `attention_mask = self._to_padding_mask(attention_mask)`; flash_attn_hub.py:141 `assert attention_mask.ndim == 2, "attention_mask must be 2D, (batch_size, seq_len)"` + +Proposed change (not a ready patch): flash_like = attn.attn.attn_backend.get_name().upper() in ( + "FLASH_ATTN", "FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB" + ) + if flash_like: + attention_mask = self._to_padding_mask(attention_mask) + +### 5. Users who explicitly select a hub backend on the diffusers adapter path hit `case _` (line 247) and silently fall back to diffusers `native` SDPA. + +`vllm_omni/diffusion/models/diffusers_adapter/pipeline_diffusers_adapter.py:~247` [major] + +This PR registers `FLASH_ATTN_HUB` and `FLASH_ATTN_3_HUB` in the diffusion backend registry, but `DiffusersAdapterPipeline._set_attention_backend` still matches only `FLASH_ATTN|None`, `SAGE_ATTN`, `ASCEND`, and `TORCH_SDPA` (`pipeline_diffusers_adapter.py:216-246`). Add hub-specific cases that extend the existing `flash_hub`/`_flash_3_hub` attempt chains. + +> unchanged by this diff, present in PR-time tree: pipeline_diffusers_adapter.py:217 `case "FLASH_ATTN" | None:` — only explicit backend cases before catch-all are SAGE_ATTN, ASCEND, TORCH_SDPA; no FLASH_ATTN_HUB/FLASH_ATTN_3_HUB branch. pipeline_diffusers_adapter.py:247-249 `case _:` / `logger.warning(f"Invalid attention backend: {attention_backend_config}. Falling back to SDPA.")` / `attention_backend_attempts.append("native")` — hub enum values fall through here. pipeline_diffusers_adapter.py:230-235 `"_flash_3_hub", ... "flash_hub", "flash_varlen_hub"` — hub diffusers backends are attempted only under the FLASH_ATTN|None branch. + +Proposed change (not a ready patch): case "FLASH_ATTN_3_HUB": + attention_backend_attempts.extend(["_flash_3_hub", "_flash_3_varlen_hub", "_flash_3", "_flash_varlen_3"]) + case "FLASH_ATTN_HUB": + attention_backend_attempts.extend(["flash_hub", "flash_varlen_hub", "flash", "flash_varlen"]) + +### 6. This PR adds `test_kernels_hub_execution` with dense unmasked forwards only (`AttentionMetadata(attn_mask=None)`, test_kernels_hub.py:75), pins the execution […] + +`tests/diffusion/attention/test_kernels_hub.py:58` [minor] + +Consider adding masked/piecewise hub unit tests and a Hopper `hardware_test` row for `FLASH_ATTN_3_HUB` if FA3 hub execution is part of the stated parity goal. + +> tests/diffusion/attention/test_kernels_hub.py:57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` — execution test is scheduled on L4; tests/diffusion/attention/test_kernels_hub.py:75 `attn_metadata_sdpa = AttentionMetadata(attn_mask=None)` — only unmasked dense metadata is exercised; tests/diffusion/attention/test_kernels_hub.py:92-93 `if major < 9:` / `pytest.skip("FLASH_ATTN_3_HUB execution requires Hopper-class GPU (compute capability >= 9.0)")` — FA3 hub forward is skipped on L4; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/attention/backends/flash_attn_hub.py:189 `if full_attn_spans is not None:` and flash_attn_hub.py:205 `if attention_mask is not None and torch.any(~attention_mask):` — masked/piecewise hub paths exist but have no matching coverage in tests/diffusion/attention/test_kernels_hub.py + +### 7. Hub kernel loading uses a version fallback chain (version=1 → version=2 → unpinned default) in both FlashAttentionHubImpl (:96-103) and FlashAttention3HubImpl […] + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:96` [minor] + +Different replicas can therefore resolve different Hub artifacts when earlier versions fail, undermining the documented train↔rollout consistency goal. Pin one Hub revision (constant or env var) and fail loudly instead of cascading. + +> flash_attn_hub.py:96 `hub_module = get_kernel("kernels-community/flash-attn2", version=1)`; flash_attn_hub.py:100 `hub_module = get_kernel("kernels-community/flash-attn2", version=2)`; flash_attn_hub.py:103 `hub_module = get_kernel("kernels-community/flash-attn2")`; flash_attn_hub.py:275 `hub_module = get_kernel("kernels-community/flash-attn3", version=1)`; flash_attn_hub.py:279 `hub_module = get_kernel("kernels-community/flash-attn3", version=2)`; flash_attn_hub.py:282 `hub_module = get_kernel("kernels-community/flash-attn3")`; attention_backends.md:217-218 `To achieve perfect numerical consistency between **training** ... and **serving/rollout** ... This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.` + +```suggestion +HUB_KERNEL_VERSION = int(os.environ.get("VLLM_OMNI_HUB_KERNEL_VERSION", "1")) +hub_module = get_kernel("kernels-community/flash-attn2", version=HUB_KERNEL_VERSION) +``` + +### 8. At cuda/platform.py:156 this PR enables `FLASH_ATTN_3_HUB` whenever `compute_capability.major >= 9`, which includes Blackwell (major 10/11/12), while […] + +`vllm_omni/platforms/cuda/platform.py:~156` [minor] + +On the same path, `SAGE_ATTN_3` is gated at major >= 10 (cuda/platform.py:159). Restrict FA3 hub to Hopper (`major == 9`) or document and test Blackwell behavior explicitly. + +> vllm_omni/platforms/cuda/platform.py:156 `fa3_hub_supported = compute_capability is not None and compute_capability.major >= 9`; docs/user_guide/diffusion/attention_backends.md:24 `| \`FLASH_ATTN_3_HUB\` | FlashAttention 3 from HuggingFace \`kernels\` library. CUDA Hopper (sm_90+) only; falls back to \`FLASH_ATTN_HUB\` on older GPUs. |`; docs/user_guide/diffusion/attention_backends.md:231 `On CUDA GPUs below Hopper (compute capability < 9.0), \`FLASH_ATTN_3_HUB\` falls back to \`FLASH_ATTN_HUB\`.`; unchanged by this diff, present in PR-time tree: vllm_omni/platforms/cuda/platform.py:159 `sage_attn3_supported = compute_capability is not None and compute_capability.major >= 10` + +```suggestion +fa3_hub_supported = compute_capability is not None and compute_capability.major == 9 +``` + +**Additional observations (beyond the comment budget):** +- `docs/user_guide/diffusion/attention_backends.md:217` [minor] Docs claim 'perfect numerical consistency' between Diffusers Hub-kernel training and vLLM-Omni serving (attention_backends.md:217) and tell users `pip install kernels==0.14.1` (attention_backends.md:228), but the added test only checks Hub output within 1e-2 of local SDPA (test_kernels_hub.py:86-88), not Diffusers parity, and pyproject pins `kernels` under dev extras only (pyproject.toml:70). Tone down the claim or document actual parity evidence and the supported install path. +- `docs/user_guide/diffusion/attention_backends.md:231` [minor] When the `kernels` package is missing, `FLASH_ATTN_3_HUB` is remapped to local `FLASH_ATTN` (FA2), not a local FA3 backend (none is registered). Docs line 231 names `FLASH_ATTN` but 'corresponding local backend implementations' may still imply FA3 parity for FA3-hub users. Consider: 'If `kernels` is not installed, both `FLASH_ATTN_HUB` and `FLASH_ATTN_3_HUB` fall back to local `FLASH_ATTN` (FlashAttention 2).' Pre-Hopper `FLASH_ATTN_3_HUB` → `FLASH_ATTN_HUB` is already documented on the same line and implemented at cuda/platform.py:140-147. +- `flash_attn_hub.py:116` [minor] piecewise varlen fallback — flash_attn_hub.py:116-126 `_flash_wrapper` falls back to `_run_varlen_dense` when `attn_func` is None; `forward_cuda` piecewise path (189-203) binds `self.flash_attn_func` into that wrapper. Residual: no hub-backend test for `full_attn_spans`/piecewise. + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4926.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4926.trace.json.gz new file mode 100644 index 000000000..3363a165c Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4926.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4950.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4950.cost.json new file mode 100644 index 000000000..22e0e1e0c --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4950.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 561.7, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4950/run-20260817-200120-91c12b", + "recorded_at": "2026-08-17T20:10:41", + "attempts": [ + { + "invocation_id": "inv-38c7f121", + "outcome": "run:run-20260817-200120-91c12b", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-200120-91c12b", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-200120-91c12b", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0631, + "minutes": 9.35, + "ci_minutes": 0, + "input_tokens": 156054, + "output_tokens": 19028, + "tool_calls": 116, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3209, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.21, + "gate": 1.69, + "review": 555.03, + "report": 0.06 + }, + "time_to_first_result_s": 560.97 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4950.trace.json.gz", + "trace_bytes": 9445 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4950.md b/eval/dataset/arms/copilot_v20cb_train/pr4950.md new file mode 100644 index 000000000..d089a42ba --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4950.md @@ -0,0 +1,48 @@ +# Run report + +- checkout: PR-TIME TREE (head 48d4fd8a5e98) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | no finding reported | +| Security | no finding reported | +| Docs / comments | 3 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Old recipe claimed text+audio in `choices[0]` — head now documents separate choices; residual: Summary still says "one shot" without mentioning two choices +- [claim-verified] PR silent-audio note — without `use_tts_template`, stage processor omits `tts_token_ids` (minicpmo_4_5_omni.py:104-118); talker skips synthesis (minicpmo_4_5_omni_tts.py:406-409) +- [validated] vllm_omni/entrypoints/openai/serving_chat.py:373,381 — chat template path reads `request.chat_template_kwargs` via `_effective_chat_template_kwargs`, not nested `extra_body` +- [validated] vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py:91-108 — 4.5 TTS span uses token IDs 151703/151704; slice built only when `<|tts_bos|>` present +- [sweep] diff_stat: only `recipes/OpenBMB/MiniCPM-o-4_5.md` changed (+19/-8); no code/tests touched +- [sweep] CI gate `buildkite/vllm-omni-npu-ci` — diff is markdown-only; no NPU configs/yaml/code intersection at this head + +### 1. Users copying that pattern into raw HTTP (curl) will not pass use_tts_template and can get silent audio. + +`examples/online_serving/minicpmo/README.md:~58` [minor] + +This PR fixes the recipe curl example to use top-level chat_template_kwargs, but examples/online_serving/minicpmo/README.md still documents only extra_body.chat_template_kwargs without the curl-vs-SDK distinction. + +> unchanged by this diff, present in PR-time tree: examples/online_serving/minicpmo/README.md:58 `extra_body.chat_template_kwargs.use_tts_template=True`, which appends`; unchanged: vllm_omni/entrypoints/openai/serving_chat.py:437-440 `# [NOTE] When sending request via openai client Python library,` / `# `extra_body` is flattented and merged into the payload's root.` / `# When sending raw request with curl, no flattening happens.`; PR diff fixes recipe: recipes/OpenBMB/MiniCPM-o-4_5.md:116 `"chat_template_kwargs": {"use_tts_template": true}` at request root for curl. + +```suggestion +- **TTS trigger**: for the OpenAI Python SDK (as in `gradio_demo.py`), pass + `extra_body={"chat_template_kwargs": {"use_tts_template": True}}` — the + client merges `extra_body` into the request root. For **curl**, put + `"chat_template_kwargs": {"use_tts_template": true}` at the request root + (nested `extra_body.chat_template_kwargs` is ignored). +``` + +### 2. Gate report shows `buildkite/vllm-omni-npu-ci` red at head; this diff touches only `recipes/OpenBMB/MiniCPM-o-4_5.md` (docs-only, no NPU configs or code). + +`recipes/OpenBMB/MiniCPM-o-4_5.md:104` [nit] + +Please confirm the failure is pre-existing on main rather than introduced here. + +> diff_stat at head 48d4fd8a5e98: `recipes/OpenBMB/MiniCPM-o-4_5.md | 27 +++++++++++++++++++--------` (1 file changed, 19 insertions, 8 deletions); recipes/OpenBMB/MiniCPM-o-4_5.md:104 `**Text + speech in one response** (the headline 4.5 feature). The TTS`; grep NPU in changed file: (no matches) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4950.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4950.trace.json.gz new file mode 100644 index 000000000..318c75388 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4950.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4970.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4970.cost.json new file mode 100644 index 000000000..bd91c61cb --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4970.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 309.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4970/run-20260817-200018-77ac7c", + "recorded_at": "2026-08-17T20:05:27", + "attempts": [ + { + "invocation_id": "inv-c2416cdd", + "outcome": "run:run-20260817-200018-77ac7c", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-200018-77ac7c", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-200018-77ac7c", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.0372, + "minutes": 5.14, + "ci_minutes": 0, + "input_tokens": 90318, + "output_tokens": 11672, + "tool_calls": 57, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.1989, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.04, + "gate": 1.57, + "review": 302.82, + "report": 0.06 + }, + "time_to_first_result_s": 308.48 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4970.trace.json.gz", + "trace_bytes": 8714 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4970.md b/eval/dataset/arms/copilot_v20cb_train/pr4970.md new file mode 100644 index 000000000..f29175ca9 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4970.md @@ -0,0 +1,32 @@ +# Run report + +- checkout: PR-TIME TREE (head 4992e10b7367) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 1 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | no finding reported | + +**Validated:** +- [claim-verified] Nightly perf launches with model name only, no request seed — tests/dfx/perf/tests/test_tts.json `server_params` has no `seed` field +- [claim-verified] Nightly WER/quality phase disabled — test_tts.json quality cells have `"enabled": false` +- [claim-verified] VoxCPM2 #4963 remainder explicitly out of scope — PR description tracks separate bisect +- [validated] serving_speech.py:3696-3706 — Qwen3 gate copies deploy `stage0_params.seed` into `extra_args['tts_local_seed']` only when `default_seed is not None` +- [validated] qwen3_tts_high_concurrency.yaml — sibling default profile already omits deploy-level seed; PR aligns default with perf profile +- [validated] test_omni_gpu_model_runner.py:250-269 — `sampling_params.seed` without `tts_local_seed` does not pass a generator to talker MTP + +### 1. This PR removes deploy-level `seed: 42` from `vllm_omni/deploy/qwen3_tts.yaml`. + +`.buildkite/test-amd-ready.yaml:~173` [minor] + +The AMD/NPU Qwen3-TTS e2e steps (`.buildkite/test-amd-ready.yaml:173`, `.buildkite/test-npu-ready.yaml:21`) run `test_qwen3_tts_customvoice.py`, which loads that deploy YAML at server startup. Can you confirm whether the current `buildkite/vllm-omni-amd-ci` / `buildkite/vllm-omni-npu-ci` failures are pre-existing on main or tied to dropping the fixed seed? + +> unchanged by this diff, present in PR-time tree: tests/e2e/online_serving/test_qwen3_tts_customvoice.py:20-22 `_STAGE_CONFIG = modify_stage_config(get_deploy_config_path("qwen3_tts.yaml"),`; unchanged by this diff, present in PR-time tree: .buildkite/test-amd-ready.yaml:173 `- timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model"`; unchanged by this diff, present in PR-time tree: .buildkite/test-npu-ready.yaml:21 `- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'`; unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:328 `- vllm_omni/deploy/qwen3_tts.yaml`; changed by this PR: vllm_omni/deploy/qwen3_tts.yaml stage-0 `default_sampling_params` has `min_tokens: 2` and `repetition_penalty: 1.05` with no `seed` key (diff removes `- seed: 42`). + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4970.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4970.trace.json.gz new file mode 100644 index 000000000..431bebc99 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4970.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4977.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr4977.cost.json new file mode 100644 index 000000000..3fe2d81bb --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4977.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 810.1, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr4977/run-20260817-195238-4a4b46", + "recorded_at": "2026-08-17T20:06:07", + "attempts": [ + { + "invocation_id": "inv-48d5500e", + "outcome": "run:run-20260817-195238-4a4b46", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-195238-4a4b46", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-195238-4a4b46", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2107, + "minutes": 13.47, + "ci_minutes": 0, + "input_tokens": 522175, + "output_tokens": 63414, + "tool_calls": 219, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4844, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 7.74, + "gate": 2.36, + "review": 797.97, + "report": 0.06 + }, + "time_to_first_result_s": 808.19 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4977.trace.json.gz", + "trace_bytes": 18788 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4977.md b/eval/dataset/arms/copilot_v20cb_train/pr4977.md new file mode 100644 index 000000000..521d171dc --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr4977.md @@ -0,0 +1,117 @@ +# Run report + +- checkout: PR-TIME TREE (head 9947f41430aa) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] PR-body `trust_remote_code=True` claim — removed at head `9947f414`; calls are `get_kernel(repo_id, version=version)` / `get_kernel(repo_id)` (`flash_attn_hub.py:34-35`); residual checked: `pyproject.toml:70` test pin `kernels<0.15.0` +- [resolved] `trust_remote_code` dropped at 9947f414 — matches `get_kernel(repo_id, version=version)` signature; pyproject pins `kernels<0.15.0` +- [claim-verified] Commit 528804e8 — documents hundreds of Hub requests per rollout worker and 429 rate limits motivating cache. +- [claim-verified] PR test `test_kernels_hub_platform_fallback` — cpu+core_model; runs via test-ready.yml tests/diffusion recursive job. +- [claim-refuted] PR test plan cites test_kernels_hub_platform_fallback but it only calls CudaOmniPlatform.get_diffusion_attn_backend_cls; never imports flash_attn_hub or _get_hub_module +- [validated] layer.py:95-103 — each `Attention` layer constructs `attn_impl_cls()` independently; per-layer hub load was the regression multiplier. + +### 1. This diff adds process-wide `_hub_modules` caching in `_get_hub_module()` (`flash_attn_hub.py:42-46`), used from both hub impl `__init__` paths (`:119` […] + +`tests/diffusion/attention/test_kernels_hub.py:~19` [minor] + +Add a CPU `@pytest.mark.core_model` test that clears `_hub_modules`, monkeypatches `kernels.get_kernel` with a counter returning a stub module with `flash_attn_func`, builds two `FlashAttentionHubImpl` instances, and asserts exactly one `get_kernel` call for `kernels-community/flash-attn2`. + +> flash_attn_hub.py:22 `_hub_modules: dict[str, object] = {}`; flash_attn_hub.py:42-46 `def _get_hub_module(repo_id: str):` / `with _hub_lock:` / `if repo_id not in _hub_modules:` / `_hub_modules[repo_id] = _load_hub_module(repo_id)` / `return _hub_modules[repo_id]`; flash_attn_hub.py:119 `hub_module = _get_hub_module("kernels-community/flash-attn2")`; flash_attn_hub.py:284 `hub_module = _get_hub_module("kernels-community/flash-attn3")`; tests/diffusion/attention/test_kernels_hub.py:19 `def test_kernels_hub_platform_fallback(monkeypatch: pytest.MonkeyPatch):`; tests/diffusion/attention/test_kernels_hub.py:41-46 `backend_path = CudaOmniPlatform.get_diffusion_attn_backend_cls("FLASH_ATTN_HUB", head_size=64)` / `assert backend_path == DiffusionAttentionBackendEnum.FLASH_ATTN.get_path()` — platform routing only, no `FlashAttentionHubImpl` construction or `get_kernel` call counting + +```suggestion +@pytest.mark.core_model +@pytest.mark.cpu +def test_hub_kernel_loaded_once(monkeypatch: pytest.MonkeyPatch): + import types + import vllm_omni.diffusion.attention.backends.flash_attn_hub as hub + from vllm_omni.diffusion.attention.backends.flash_attn_hub import FlashAttentionHubImpl + + hub._hub_modules.clear() + calls: list[tuple[str, int | None]] = [] + + def fake_get_kernel(repo_id: str, version: int | None = None): + calls.append((repo_id, version)) + return types.SimpleNamespace( + flash_attn_func=lambda q, k, v, **kw: q, + flash_attn_varlen_func=None, + ) + + kernels_mod = types.ModuleType("kernels") + kernels_mod.get_kernel = fake_get_kernel + monkeypatch.setitem(__import__("sys").modules, "kernels", kernels_mod) + + FlashAttentionHubImpl(num_heads=8, head_size=64, softmax_scale=0.125, causal=False) + FlashAttentionHubImpl(num_heads=8, head_size=64, softmax_scale=0.125, causal=False) + assert [c for c in calls if c[0] == "kernels-community/flash-attn2"] == [("kernels-community/flash-attn2", 1)] +``` + +### 2. If `get_kernel()` returns a module missing both symbols, that object is stored in `_hub_modules` even though `__init__` raises; + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:45` [minor] + +Caching in `_get_hub_module` (flash_attn_hub.py:45) happens before `FlashAttentionHubImpl.__init__` validates `flash_attn_func`/`flash_attn_varlen_func` (lines 120-123). Pre-PR each layer called `get_kernel` independently. Validate required entrypoints inside `_load_hub_module` before caching, or defer the `_hub_modules` insert until after symbol validation. + +> flash_attn_hub.py:45 `_hub_modules[repo_id] = _load_hub_module(repo_id)` — unconditional cache insert inside `_get_hub_module`; flash_attn_hub.py:31-35 `for version in (1, 2, None): ... return get_kernel(repo_id, version=version)` / `return get_kernel(repo_id)` — returns first successful load with no symbol check; flash_attn_hub.py:119-123 `hub_module = _get_hub_module("kernels-community/flash-attn2")` then `if self.flash_attn_func is None and self.flash_attn_varlen_func is None: raise RuntimeError(...)` — validation runs after cache population; flash_attn_hub.py:44-46 `if repo_id not in _hub_modules:` skipped on later calls, so cached bad module is reused without retry + +```suggestion +def _load_hub_module(repo_id: str): + from kernels import get_kernel + + logger.info("Loading %s kernel from HuggingFace Hub...", repo_id) + last_error = None + for version in (1, 2, None): + try: + module = get_kernel(repo_id, version=version) if version is not None else get_kernel(repo_id) + if getattr(module, "flash_attn_func", None) is None and getattr(module, "flash_attn_varlen_func", None) is None: + raise RuntimeError(f"Hub kernel {repo_id!r} exposes no flash_attn functions") + return module + except Exception as exc: + logger.info("Failed to load %s version %s: %s", repo_id, version, exc) + last_error = exc + raise RuntimeError(f"Failed to load HuggingFace Hub kernel {repo_id!r}") from last_error +``` + +### 3. Not needed for trusted `kernels-community/flash-attn2|3` publishers. + +`general` [minor] + +[resolved] trust_remote_code — at 9947f414, `flash_attn_hub._load_hub_module` calls `get_kernel(repo_id, version=...)` / `get_kernel(repo_id)` with no `trust_remote_code`; consistent with kernels v0.13.x API (no such parameter). + +> flash_attn_hub.py:34 `return get_kernel(repo_id, version=version)`; flash_attn_hub.py:35 `return get_kernel(repo_id)` — no trust_remote_code kwarg at PR head. Unchanged external contract for 0.13.x (not in diff): Hugging Face kernels v0.13.0 docs list `get_kernel(repo_id, revision=None, version=None, backend=None, user_agent=None)` with no `trust_remote_code` parameter. + +### 4. The CPU lane (test-ready.yml:13) recursively collects `tests/diffusion` but only executes `test_kernels_hub_platform_fallback`, which mocks kernels away and […] + +`.buildkite/test-ready.yml:~145` [minor] + +This diff caches hub loads in `_get_hub_module()` (flash_attn_hub.py:42-46), but PR-ready CUDA CI skips `tests/diffusion/attention/`: test-ready.yml:145 runs only top-level `tests/diffusion/*.py` with `core_model and cuda`, so `test_kernels_hub_execution` (CUDA/L4 hub forward) never runs. Extend a PR-ready CUDA lane to `tests/diffusion/attention/` or add a CPU cache-dedup test to the recursive CPU job. + +> unchanged by this diff, present in PR-time tree: .buildkite/test-ready.yml:145 `- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"`; .buildkite/test-ready.yml:13 `- "timeout 20m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' ..."`; tests/diffusion/attention/test_kernels_hub.py:56-57 `@hardware_test(res={"cuda": "L4"}, num_cards=1)` / `def test_kernels_hub_execution():`; flash_attn_hub.py:42-46 `def _get_hub_module(repo_id: str):` / `with _hub_lock:` / `if repo_id not in _hub_modules:` / `_hub_modules[repo_id] = _load_hub_module(repo_id)`; grep of `.buildkite` for `diffusion/attention`: no matches. + +### 5. The shipped head no longer passes `trust_remote_code` to `get_kernel` (removed in 9947f414 because kernels 0.13.x rejects it), but the PR Purpose still claims […] + +`vllm_omni/diffusion/attention/backends/flash_attn_hub.py:26` [minor] + +Please update the PR body to match the final loading contract. + +> vllm_omni/diffusion/attention/backends/flash_attn_hub.py:33 `return get_kernel(repo_id, version=version)`; vllm_omni/diffusion/attention/backends/flash_attn_hub.py:34 `return get_kernel(repo_id)` — no trust_remote_code kwarg at PR head. Commit 9947f414 message: 'kernels 0.13.x get_kernel() does not accept trust_remote_code'. PR #4977 body (live): 'adding `trust_remote_code=True` for more reliable Hub access'. + +### 6. Unchanged `NPUOmniPlatform.get_diffusion_attn_backend_cls` (`npu/platform.py:110-116`) rewrites `FLASH_ATTN_HUB`/`FLASH_ATTN_3_HUB` to local `FLASH_ATTN` and […] + +`vllm_omni/platforms/npu/platform.py:~110` [nit] + +NPU CI is red at head, but this PR only changes `flash_attn_hub.py`. Please confirm whether the NPU CI failure predates this PR. + +> unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/platform.py:110 `if backend_upper in ("FLASH_ATTN_HUB", "FLASH_ATTN_3_HUB"):`; vllm_omni/platforms/npu/platform.py:114 `backend_upper = "FLASH_ATTN"`; vllm_omni/platforms/npu/platform.py:116 `return backend.get_path()`; vllm_omni/diffusion/attention/backends/registry.py:59 `FLASH_ATTN = "vllm_omni.diffusion.attention.backends.flash_attn.FlashAttentionBackend"` vs :65-66 hub paths pointing to `flash_attn_hub` + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_train/pr4977.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr4977.trace.json.gz new file mode 100644 index 000000000..45c627ff5 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr4977.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_train/pr5009.cost.json b/eval/dataset/arms/copilot_v20cb_train/pr5009.cost.json new file mode 100644 index 000000000..42246eb4c --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr5009.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 982.0, + "split": "train", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train/runs/pr5009/run-20260817-194335-7cd55f", + "recorded_at": "2026-08-17T19:59:56", + "attempts": [ + { + "invocation_id": "inv-f13e3ed4", + "outcome": "run:run-20260817-194335-7cd55f", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260817-194335-7cd55f", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260817-194335-7cd55f", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.3022, + "minutes": 16.35, + "ci_minutes": 0, + "input_tokens": 716117, + "output_tokens": 98945, + "tool_calls": 451, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.5837, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 4.69, + "gate": 1.62, + "review": 974.56, + "report": 0.06 + }, + "time_to_first_result_s": 980.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr5009.trace.json.gz", + "trace_bytes": 23900 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_train/pr5009.md b/eval/dataset/arms/copilot_v20cb_train/pr5009.md new file mode 100644 index 000000000..8e1d5a9be --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_train/pr5009.md @@ -0,0 +1,57 @@ +# Run report + +- checkout: PR-TIME TREE (head 7314f7e8102a) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 3 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Qwen-Image Part 2 (`nn.RMSNorm`) bypasses IR op priority for those layers — Part 1 fleet benefit targets other diffusion models still on vllm `RMSNorm` (wan, flux, sd3, etc.) +- [resolved] Cosmos3 native-only path preserved — quote `priority_kwargs["rms_norm"] = ["native"]` at pipeline_cosmos3.py:620; residual: no other model overrides besides Cosmos3. +- [resolved] Part 2 makes Qwen norms IR-independent — residual: Part 1 still affects other diffusion models on vllm RMSNorm, not Qwen transformer norms at head. +- [resolved] Stale consumers of OLD CUDA inductor→native default — no tests/docs grep hits outside rocm/xpu platform siblings +- [resolved] Part 2 no in-PR unit test — L4 accuracy/perf nightly is the guard, but L2/L3 merge gate cannot catch SSIM/PSNR or vllm RMSNorm reversion +- [claim-verified] PR claim per-model native overrides exist — only `Cosmos3OmniDiffusersPipeline` in `_DIFFUSION_IR_OP_PRIORITY_FUNCS` (registry.py:545-549) + +`vllm_omni/platforms/cuda/platform.py:253` [minor] — The CUDA hunk at `platform.py:253` removes the inductor-active `native`-only branch so every CUDA diffusion worker inherits `["vllm_c", "native"]` via `diffusion_worker._resolve_ir_op_priority` (line 175), yet 14 diffusion model files still import vLLM `RMSNorm` (flux, wan, sd3, ltx2, omnigen2, etc.) while this PR only switches Qwen-Image to `nn.RMSNorm` and adds platform CPU tests. Note in the test plan whether representative inheritors (e.g. Wan/Flux) were checked post-change or file follow-up if scope stays Qwen-only. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]`; vllm_omni/diffusion/worker/diffusion_worker.py:175 `ir_op_priority = current_omni_platform.get_default_ir_op_priority(vllm_config)`; vllm_omni/diffusion/registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func"` (sole per-model IR-priority override); vllm_omni/diffusion/models/flux2/flux2_transformer.py:18 `from vllm.model_executor.layers.layernorm import RMSNorm`; vllm_omni/diffusion/models/ltx2/ltx2_transformer.py:35 `from vllm.model_executor.layers.layernorm import RMSNorm`; vllm_omni/diffusion/models/omnigen2/omnigen2_transformer.py:15 `from vllm.model_executor.layers.layernorm import RMSNorm`) + +`vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:557` [minor] — Part 2 at `qwen_image_transformer.py:557` replaces vLLM `RMSNorm` with `nn.RMSNorm` for Qwen-Image attention norms with no new merge-gate unit test; merge-gate `tests/e2e/online_serving/test_qwen_image.py` is smoke-only (request send, no pixel asserts), and SSIM/PSNR recovery is validated only by L4 nightly `tests/e2e/accuracy/test_qwen_image.py` (`full_model`, H100). Consider a CPU unit test that instantiates `QwenImageCrossAttention` and asserts `norm_q`/`norm_k`/`norm_added_*` are `nn.RMSNorm` instances. (evidence: qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; unchanged on main (PR-time tree): qwen_image_transformer.py@main `self.norm_q = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; merge-gate smoke only: tests/e2e/online_serving/test_qwen_image.py:80 `openai_client.send_diffusion_request(request_config)`; L4 SSIM/PSNR: tests/e2e/accuracy/test_qwen_image.py:204-211 `assert_similarity(..., ssim_threshold=SSIM_THRESHOLD, psnr_threshold=PSNR_THRESHOLD)` with tests/e2e/accuracy/test_qwen_image.py:20 `pytestmark = [pytest.mark.full_model, pytest.mark.diffusion]` and tests/e2e/accuracy/test_qwen_image.py:196 `@hardware_test(res={'cuda': 'H100'}, num_cards=1)`) + +```suggestion +# tests/diffusion/models/qwen_image/test_qwen_image_rms_norm.py +import torch.nn as nn +from vllm_omni.diffusion.models.qwen_image.qwen_image_transformer import QwenImageCrossAttention + +def test_qwen_image_cross_attention_uses_torch_rms_norm(): + attn = QwenImageCrossAttention(dim=64, num_heads=4, head_dim=16, added_kv_proj_dim=64) + assert isinstance(attn.norm_q, nn.RMSNorm) + assert isinstance(attn.norm_k, nn.RMSNorm) + assert isinstance(attn.norm_added_q, nn.RMSNorm) +``` + +`general` [minor] — Qwen-Image Part 2 (`nn.RMSNorm` in qwen_image_transformer.py) bypasses `kernel_config.ir_op_priority` for those norms; Part 1's `vllm_c`-first RMSNorm priority benefits other diffusion models still on `vllm.model_executor.layers.layernorm.RMSNorm` (flux, sd3, flux2, etc.). (evidence: qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; unchanged by this diff, present in PR-time tree: flux_transformer.py:19 `from vllm.model_executor.layers.layernorm import RMSNorm`; unchanged by this diff, present in PR-time tree: sd3_transformer.py:137 `self.norm_q = RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; platform.py:253 `default = ["vllm_c", "native"]`; diffusion_worker.py:290 `vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(self.od_config, vllm_config)`) + +`pipeline_cosmos3.py:620` [minor] — Cosmos3 native-only RMS-norm path is preserved via model hook at pipeline_cosmos3.py:620-621; Cosmos3 is the only registered diffusion ir_op_priority override (registry.py:549). (evidence: unchanged by this diff, present in the PR-time tree: pipeline_cosmos3.py:620 `priority_kwargs["rms_norm"] = ["native"]`; pipeline_cosmos3.py:621 `priority_kwargs["fused_add_rms_norm"] = ["native"]`; registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func",` — sole entry in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`; PR diff changes platform default at cuda/platform.py:253 to `default = ["vllm_c", "native"]`) + +`general` [minor] — Part 2 makes Qwen Image transformer norms IR-independent (nn.RMSNorm). Residual scope: Part 1’s CudaOmniPlatform rms_norm priority still applies to other diffusion models that use vllm.model_executor.layers.layernorm.RMSNorm (e.g. FLUX), not Qwen norms at this PR head. (evidence: qwen_image_transformer.py:557 `self.norm_q = nn.RMSNorm(head_dim, eps=eps) if qk_norm else nn.Identity()`; qwen_image_transformer.py:1009 `self.txt_norm = nn.RMSNorm(joint_attention_dim, eps=1e-6)` — no vllm layernorm RMSNorm import in this file. platform.py:253 `default = ["vllm_c", "native"]`; platform.py:262 `return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm)`. diffusion_worker.py:290 `vllm_config.kernel_config.ir_op_priority = _resolve_ir_op_priority(self.od_config, vllm_config)`. unchanged by this diff, present in PR-time tree: flux_transformer.py:19 `from vllm.model_executor.layers.layernorm import RMSNorm`; flux_transformer.py:150 `self.norm_q = RMSNorm(dim_head, eps=eps)`) + +`test_cosmos3_pipeline.py:762` [minor] — Cosmos3 IR-op override under new default — tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py:762-771 seeds platform default `["vllm_c", "native"]` and asserts Cosmos3 merge still yields `["native"]`; residual: only Cosmos3 is registered in `_DIFFUSION_IR_OP_PRIORITY_FUNCS`. (evidence: ['tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py:763 `rms_norm=["vllm_c", "native"]`', 'tests/diffusion/models/cosmos3/test_cosmos3_pipeline.py:770 `assert merged.rms_norm == ["native"]`', 'vllm_omni/diffusion/registry.py:549 `"Cosmos3OmniDiffusersPipeline": "get_cosmos3_ir_op_priority_func"`', 'unchanged by this diff, present in PR-time tree: vllm_omni/platforms/cuda/platform.py:253 `default = ["vllm_c", "native"]`', 'unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/cosmos3/pipeline_cosmos3.py:620 `priority_kwargs["rms_norm"] = ["native"]`']) + +`tests/platforms/test_cuda_ir_op_priority.py:~127` [minor] — This PR adds `tests/platforms/test_cuda_ir_op_priority.py`, but `docs/contributing/ci/tests_style.md:123` maps `vllm_omni/platforms/` to indirect coverage with no `tests/platforms/` tree, and AMD `test-amd-ready.yaml` Simple lanes only pytest `tests/diffusion` and `tests/model_executor`, so this regression never runs on AMD CI. `tests/diffusion/test_diffusion_plugin_hooks.py:127` tests `_resolve_ir_op_priority` with a mocked platform default but not actual priority lists. Relocate beside the diffusion platform tests or update the CI layout docs in the same PR. (evidence: unchanged by this diff, present in PR-time tree: docs/contributing/ci/tests_style.md:123 `├── platforms/ → (no tests/platforms/, e2e and stage_configs provide indirect coverage)`; unchanged by this diff, present in PR-time tree: .buildkite/test-amd-ready.yaml:20 `- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`; unchanged by this diff, present in PR-time tree: tests/diffusion/test_diffusion_plugin_hooks.py:135 `mock_platform.get_default_ir_op_priority.return_value = default_priority` — mocks platform default, does not assert `CudaOmniPlatform` priority lists; added by this PR: tests/platforms/test_cuda_ir_op_priority.py exercises `CudaOmniPlatform.get_default_ir_op_priority` directly.) + +```suggestion +# Relocate to tests/diffusion/test_cuda_ir_op_priority.py (same contents, delete tests/platforms/ directory) +``` + +**Additional observations (beyond the comment budget):** +- `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py:1009` [minor] This hunk switches `txt_norm` from vllm `RMSNorm` to `nn.RMSNorm` on the text-entry boundary (applied every forward at 1140 before `txt_in` at 1141), but the `#2728` comment at 1011–1012 only documents `img_in`/`txt_in`. Extend that boundary comment to include `txt_norm`; if you add a CrossAttention `nn.RMSNorm` unit test for the 557 hunk, also assert `type(model.txt_norm) is nn.RMSNorm` — L4 `test_qwen_image_matches_diffusers` covers e2e parity, not this layer choice. + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_train/pr5009.trace.json.gz b/eval/dataset/arms/copilot_v20cb_train/pr5009.trace.json.gz new file mode 100644 index 000000000..30187d8d3 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_train/pr5009.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_val/manifest.json b/eval/dataset/arms/copilot_v20cb_val/manifest.json new file mode 100644 index 000000000..0c9a7b021 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/manifest.json @@ -0,0 +1,34 @@ +{ + "arm": "copilot_v20cb_val", + "engine": "infermatrix-copilot CLI (shipped pipeline, pr-review@6 adaptive-depth ensemble; issue-answer dry-run)", + "llm": "per .env (DeepSeek-routed)", + "dataset": "vllm_omni_dataset.yaml", + "splits": [ + "val" + ], + "kinds": [ + "pr_review" + ], + "n_items": 5, + "trace": { + "AGENT_TRACE": "(default)", + "AGENT_TRACE_IO": "(default)", + "AGENT_TRACE_IO_FULL": "(default)" + }, + "moa_when": "off", + "pr_context_mode": "no_discussion", + "resolved": { + "moa_when": "off", + "moa_eligible_full_depth_pr": false, + "llm_mixture_members": [ + "mimo-v2.5", + "qwen3.6-plus" + ], + "strict_backend": "cursor", + "strict_backend_model": "composer-2.5", + "review_lens_backends": {}, + "agent_model": "deepseek-v4-pro[1m]", + "review_planner_model": "deepseek-v4-flash", + "review_promotion_model": "deepseek-v4-flash" + } +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4810.cost.json b/eval/dataset/arms/copilot_v20cb_val/pr4810.cost.json new file mode 100644 index 000000000..f4a7aa632 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4810.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 1242.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val/runs/pr4810/run-20260818-112615-3b1202", + "recorded_at": "2026-08-18T11:46:55", + "attempts": [ + { + "invocation_id": "inv-15337e06", + "outcome": "run:run-20260818-112615-3b1202", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260818-112615-3b1202", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260818-112615-3b1202", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.379, + "minutes": 20.67, + "ci_minutes": 0, + "input_tokens": 853208, + "output_tokens": 135160, + "tool_calls": 523, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.6942, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.92, + "gate": 1.83, + "review": 1234.11, + "report": 0.06 + }, + "time_to_first_result_s": 1239.92 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4810.trace.json.gz", + "trace_bytes": 25474 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4810.md b/eval/dataset/arms/copilot_v20cb_val/pr4810.md new file mode 100644 index 000000000..0948641a2 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4810.md @@ -0,0 +1,64 @@ +# Run report + +- checkout: PR-TIME TREE (head f9665cc223c8) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 6 finding(s) below | +| Security | no finding reported | +| Docs / comments | 1 finding(s) below | +| Behavior / compatibility | no finding reported | +| Correctness | 1 finding(s) below | + +**Validated:** +- [resolved] Author PR note to remove tests once validated — tests at head are present and should be kept as regression guard +- [resolved] mimo mapper vs `model.` rewrite ordering — concern checked; production `model.*` names remain correct after mapper-then-double +- [claim-verified] PR removes manual get_cache_scale from hunyuan_image3.py:225-227, mammoth_moda2/mimo_audio add mapper at :487/:1157, qwen2_old drops guarded branch +- [claim-verified] pyproject.toml markers define core_model/cpu; dependency get_cache_scale_mapper assumed from vLLM ≥0.23 per PR body (repo rebased past #4286) +- [claim-refuted] Issue #4809 class fix is incomplete: `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls `self.quant_config.get_cache_scale(name)` +- [claim-refuted] Static `_STALE_API_FILES` guard incomplete — author's own main-branch failure log included `hunyuan_image3_transformer.py` but final PR omits it while file still calls removed API. + +`vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:~2238` [major] — This PR removes the stale `get_cache_scale` walrus branch from AR `hunyuan_image3.py` and documents mapper delegation to the outer `AutoWeightsLoader`, but the diffusion twin still calls the removed API inside `HunyuanImage3Model.load_weights`. That is the only remaining production `.get_cache_scale(` site at head; `HunyuanImage3Pipeline.load_weights` still enters it via `AutoWeightsLoader`, so quantized Hunyuan Image3 diffusion loads can still raise `AttributeError` on vLLM ≥0.23. Mirror the AR fix here or close it under tracked #4806. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 `if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`; changed in this PR: vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py:225-226 `# KV-cache scales are renamed to .attn.{k,v,q}_scale by the` / `# quant_config mapper in the outer AutoWeightsLoader.`; unchanged pipeline entry: vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py:458-462 `loader = AutoWeightsLoader(` / ` self,` / ` skip_prefixes=skip_prefixes,` / `)` / `return loader.load_weights(weights)`) + +```suggestion +# KV-cache scales are renamed to .attn.{k,v,q}_scale by the + # quant_config mapper in the outer AutoWeightsLoader. + is_found = False +``` + +`vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py:~486` [minor] — MammothModa2Qwen2ForCausalLM.load_weights prepends quant_config.get_cache_scale_mapper().apply(weights) (mammoth_moda2.py:486-489), consistent with the PR's direct-loader test classification. On the AR serving path, MammothModa2ForConditionalGeneration.load_weights calls self.ar.load_weights (mammoth_moda2.py:810-814); MammothModa2ARForConditionalGeneration does not override load_weights and sets self.language_model to MammothModa2Qwen2ForCausalLM (mammoth_moda2.py:572-577), so it inherits Qwen2_5_VLForConditionalGeneration's AutoWeightsLoader entry, which already applies get_cache_scale_mapper() before delegating to child load_weights. Hunyuan's inner loader only documents outer mapping (hunyuan_image3.py:225-226). Please confirm mapper idempotency on already-mapped scale names or align Mammoth with the Hunyuan comment-only pattern on the AR-delegated path. (evidence: mammoth_moda2.py:486 `if self.quant_config is not None:`; mammoth_moda2.py:487 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()`; mammoth_moda2.py:489 `weights = cache_scale_mapper.apply(weights)`; mammoth_moda2.py:572 `self.language_model = init_vllm_registered_model(`; mammoth_moda2.py:810 `def load_weights(self, weights):`; mammoth_moda2.py:814 `loaded = self.ar.load_weights(weights)`; hunyuan_image3.py:225 `# KV-cache scales are renamed to .attn.{k,v,q}_scale by the`; hunyuan_image3.py:226 `# quant_config mapper in the outer AutoWeightsLoader.`; unchanged by this diff, present in installed vllm dependency: qwen2_5_vl.py:2028 `loader = AutoWeightsLoader(self)`; utils.py:414 `mapper |= quant_config.get_cache_scale_mapper()`; utils.py:418 `weights = mapper.apply(weights)`; utils.py:331 `loaded_params = module_load_weights(weights)`) + +```suggestion +# KV-cache scales are renamed by quant_config.get_cache_scale_mapper() + # in the outer AutoWeightsLoader on MammothModa2ARForConditionalGeneration. +``` + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:15` [minor] — `_STALE_API_FILES` (lines 15–20) only guards four `model_executor` loaders; `vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238` still calls the removed `get_cache_scale` API and is not in the list. Add that path once fixed, or document with e.g. `TODO(#4806)` that diffusion is out of scope so the grep test is not read as full-repo coverage. (evidence: tests/model_executor/models/test_kv_cache_scale_mapper.py:15 `_STALE_API_FILES = [` — lists only hunyuan_image3.py, mammoth_moda2.py, mimo_audio_llm.py, qwen2_old.py under model_executor; unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py:2238 `if self.quant_config is not None and (scale_name := self.quant_config.get_cache_scale(name)):`) + +```suggestion +_STALE_API_FILES = [ + "vllm_omni/model_executor/models/hunyuan_image3/hunyuan_image3.py", + "vllm_omni/model_executor/models/mammoth_moda2/mammoth_moda2.py", + "vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py", + "vllm_omni/model_executor/models/qwen2_5_omni/qwen2_old.py", + # TODO(#4806): add vllm_omni/diffusion/models/hunyuan_image3/hunyuan_image3_transformer.py +] +``` + +`general` [minor] — Mammoth/MiMo: old `get_cache_scale` branch removed and replaced with `get_cache_scale_mapper().apply(weights)`. Residual (Mammoth AR only): `MammothModa2ARForConditionalGeneration` subclasses `Qwen2_5_VLForConditionalGeneration` and inherits its `AutoWeightsLoader` path, so the inner mapper in `MammothModa2Qwen2ForCausalLM.load_weights` is redundant on that path and no-op on already-mapped names. (evidence: mammoth_moda2.py:540 `class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):`; mammoth_moda2.py:810-814 `loaded = self.ar.load_weights(weights)`; mammoth_moda2.py:486-489 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()` / `weights = cache_scale_mapper.apply(weights)`; mammoth_moda2.py:572-576 `self.language_model = init_vllm_registered_model(..., architectures=["MammothModa2Qwen2ForCausalLM"])`; hunyuan_image3.py:225-226 `# quant_config mapper in the outer AutoWeightsLoader.`; unchanged by this diff, present in upstream vLLM: Qwen2_5_VLForConditionalGeneration.load_weights uses `loader = AutoWeightsLoader(self)` and AutoWeightsLoader.load_weights merges `quant_config.get_cache_scale_mapper()` then `weights = mapper.apply(weights)` before `_load_module`.) + +`general` [minor] — When MammothModa2Qwen2ForCausalLM.load_weights is reached via MammothModa2ARForConditionalGeneration → Qwen2_5_VL AutoWeightsLoader, get_cache_scale_mapper runs twice (outer AutoWeightsLoader then inner load_weights). Harmless for standard .attn.{k,v,q}_scale names because upstream regexes skip already-mapped suffixes; tests only cover direct MammothModa2Qwen2ForCausalLM.load_weights. (evidence: mammoth_moda2.py:540 `class MammothModa2ARForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):` — no load_weights override; inherits upstream AutoWeightsLoader path. mammoth_moda2.py:572-577 `self.language_model = init_vllm_registered_model(... architectures=["MammothModa2Qwen2ForCausalLM"],)` — child loader is MammothModa2Qwen2ForCausalLM.load_weights. mammoth_moda2.py:486-489 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()` / `weights = cache_scale_mapper.apply(weights)` — inner mapper apply. qwen2_5_vl.py:2021-2023 `loader = AutoWeightsLoader(self)` / `return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)` — outer AutoWeightsLoader entry. utils.py:411-415 `mapper |= quant_config.get_cache_scale_mapper()` / `weights = mapper.apply(weights)` — outer mapper apply before child delegation. test_kv_cache_scale_mapper.py:122-123 parametrizes `MammothModa2Qwen2ForCausalLM` in `test_direct_custom_loaders_apply_cache_scale_mapper`, not MammothModa2ARForConditionalGeneration.) + +`tests/model_executor/models/test_kv_cache_scale_mapper.py:~144` [minor] — MiMo's parametrized case in `test_direct_custom_loaders_apply_cache_scale_mapper` should feed a `model.`-prefixed cache-scale source name. Production weights are routed from `mimo_audio.py` only when `k.startswith("model.")`, and `mimo_audio_llm.load_weights` then prepends another `model.` before parameter lookup; the current unprefixed `_SOURCE_SCALE_NAME` skips that naming path. (evidence: unchanged by this diff, present in the PR-time tree: tests/model_executor/models/test_kv_cache_scale_mapper.py:22 `_SOURCE_SCALE_NAME = "layers.0.self_attn.k_proj.output_scale"`; tests/model_executor/models/test_kv_cache_scale_mapper.py:144 `loaded = load_weights(fake_model, [(_SOURCE_SCALE_NAME, torch.ones(()))])`; unchanged by this diff, present in the PR-time tree: vllm_omni/model_executor/models/mimo_audio/mimo_audio.py:708 `k.startswith("model.")`; vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1157 `cache_scale_mapper = self.quant_config.get_cache_scale_mapper()`; vllm_omni/model_executor/models/mimo_audio/mimo_audio_llm.py:1162-1163 `if name.startswith("model."): name = "model." + name`) + +```suggestion +loaded = load_weights( + fake_model, + [("model." + _SOURCE_SCALE_NAME, torch.ones(()))], + ) +``` + +**Verdict:** FOLLOW-UP REQUIRED (post-merge) diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4810.trace.json.gz b/eval/dataset/arms/copilot_v20cb_val/pr4810.trace.json.gz new file mode 100644 index 000000000..d6aca57e5 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_val/pr4810.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4816.cost.json b/eval/dataset/arms/copilot_v20cb_val/pr4816.cost.json new file mode 100644 index 000000000..521ee183a --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4816.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 511.4, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val/runs/pr4816/run-20260818-112614-acb085", + "recorded_at": "2026-08-18T11:34:44", + "attempts": [ + { + "invocation_id": "inv-7ed02169", + "outcome": "run:run-20260818-112614-acb085", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260818-112614-acb085", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260818-112614-acb085", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.1716, + "minutes": 8.49, + "ci_minutes": 0, + "input_tokens": 374847, + "output_tokens": 63988, + "tool_calls": 270, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.354, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.39, + "gate": 1.52, + "review": 504.25, + "report": 0.06 + }, + "time_to_first_result_s": 509.25 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4816.trace.json.gz", + "trace_bytes": 15754 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4816.md b/eval/dataset/arms/copilot_v20cb_val/pr4816.md new file mode 100644 index 000000000..583a1c281 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4816.md @@ -0,0 +1,49 @@ +# Run report + +- checkout: PR-TIME TREE (head e1dac23ac02c) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 2 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 1 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Stale-consumer concern: all 7 production sites + 3 test mock sites updated; checked tests/, examples/, docs/ — nothing left using old key. +- [resolved] Pure-diffusion speech error residual: `serving_tokenization=None` at api_server.py:754 while `openai_serving_speech.for_diffusion` exists — `base()` still returns None on caught ValueError; pre-existing, not introduced by rename. +- [resolved] Stale attribute name concern: fully addressed by rename; no in-repo survivors. +- [claim-verified] PR body: upstream `instrumentator.basic.tokenization()` reads `request.app.state.serving_tokenization` — confirmed at api_server.py:67 import of `base` and :376 `base(raw_request).create_error_response(...)`. +- [claim-verified] PR scope is only rename alignment; diff_stat shows 2 files, 9 insertions, 9 deletions +- [claim-verified] PR description misalignment with upstream vLLM #46022: rename matches upstream `instrumentator.basic.tokenization()` contract. + +`tests/entrypoints/openai_api/test_serving_speech.py:~2382` [minor] — This PR renames `openai_serving_tokenization` to `serving_tokenization` in `api_server.py` (init at 754/1017; `getattr` fallbacks at 1157/1248/1286/1329) and updates three handler-missing tests to set `raw_request.app.state.serving_tokenization`, but `upload_voice` `ValueError` errors go through `_create_speech_error_json_response` → `base(raw_request).create_error_response` (api_server.py:376, called from :1483). `test_api_server_upload_voice_value_error_returns_400` and sibling voice-error tests call `_patch_api_server_base`, which patches `api_server_module.base` (test_serving_speech.py:2352) and would still pass if init reverted the state attribute name. Add one sibling test that sets `raw_request.app.state.serving_tokenization` (no `base` mock) and asserts `upload_voice` `ValueError` returns HTTP 400 via the real `base()` → state chain. (evidence: test_serving_speech.py:2352 `mocker.patch.object(api_server_module, "base", return_value=fake_base)`; test_serving_speech.py:2382 `def test_api_server_upload_voice_value_error_returns_400(mocker: MockerFixture):` and :2383 `_patch_api_server_base(mocker)`; api_server.py:376 `err = base(raw_request).create_error_response(`; api_server.py:1483 `return _create_speech_error_json_response(raw_request, str(e))`; api_server.py:1017 `state.serving_tokenization = ServingTokenization(`; test_serving_speech.py:2533 `raw_request.app.state.serving_tokenization = fake_base` (also :2546, :2615 for the three updated handler-missing tests)) + +```suggestion +def test_api_server_upload_voice_value_error_uses_serving_tokenization_state(mocker: MockerFixture): + fake_tokenization = mocker.MagicMock() + fake_tokenization.create_error_response.side_effect = lambda message, **kw: ErrorResponse( + error=ErrorInfo(message=message, type=kw.get("err_type", "BadRequestError"), code=400) + ) + handler = mocker.MagicMock() + handler.upload_voice = mocker.AsyncMock(side_effect=ValueError("Could not decode audio file")) + raw_request = _make_api_server_request(handler) + raw_request.app.state.serving_tokenization = fake_tokenization + response = asyncio.run( + api_server_module.upload_voice( + raw_request, audio_sample=mocker.MagicMock(), speaker_embedding=None, + consent="c", name="probe", + ) + ) + _assert_openai_error_response(response, status_code=400, message="Could not decode audio file") +``` + +`vllm_omni/entrypoints/openai/api_server.py:~754` [nit] — Gate report shows `buildkite/vllm-omni-amd-ci`, `buildkite/vllm-omni-intel-ci`, and `buildkite/vllm-omni-npu-ci` failing at head, while this diff only touches `vllm_omni/entrypoints/openai/api_server.py` and `tests/entrypoints/openai_api/test_serving_speech.py` with no platform/kernel/config changes. Please confirm whether those CI failures are pre-existing on main and unrelated to this rename. (evidence: gate_report: 3 failing Buildkite checks; diff_stat: only tests/entrypoints/openai_api/test_serving_speech.py and vllm_omni/entrypoints/openai/api_server.py changed.) + +`vllm_omni/entrypoints/openai/api_server.py:~754` [nit] — Pure-diffusion init still sets `state.serving_tokenization = None` (api_server.py:754) while `OmniOpenAIServingSpeech.for_diffusion` remains registered (784). Voice routes that use `_create_speech_error_json_response` → `base(raw_request).create_error_response` (376-380) will raise when `serving_tokenization` is None—especially upload_voice/delete_voice exception handlers (1483-1490, 1542-1547). Pre-existing; not introduced by the rename; fine to defer a follow-up to guard that path or unify with the explicit `serving_tokenization` checks used in create_speech (1248-1252). (evidence: api_server.py:754 `state.serving_tokenization = None`; api_server.py:784 `state.openai_serving_speech = OmniOpenAIServingSpeech.for_diffusion(`; api_server.py:376-380 `err = base(raw_request).create_error_response(`; api_server.py:1483 `return _create_speech_error_json_response(raw_request, str(e))`; unchanged by this diff, upstream vllm `instrumentator/basic.py`: `def base(request: Request) -> ServingTokenization:` / `return request.app.state.serving_tokenization`) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4816.trace.json.gz b/eval/dataset/arms/copilot_v20cb_val/pr4816.trace.json.gz new file mode 100644 index 000000000..1ae62de7f Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_val/pr4816.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4825.cost.json b/eval/dataset/arms/copilot_v20cb_val/pr4825.cost.json new file mode 100644 index 000000000..a9ce8f134 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4825.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 314.5, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val/runs/pr4825/run-20260818-112613-1ed514", + "recorded_at": "2026-08-18T11:31:27", + "attempts": [ + { + "invocation_id": "inv-9b01556e", + "outcome": "run:run-20260818-112613-1ed514", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260818-112613-1ed514", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260818-112613-1ed514", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2414, + "minutes": 5.22, + "ci_minutes": 0, + "input_tokens": 606149, + "output_tokens": 70690, + "tool_calls": 262, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.2936, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.42, + "gate": 1.81, + "review": 308.07, + "report": 0.06 + }, + "time_to_first_result_s": 313.35 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4825.trace.json.gz", + "trace_bytes": 18373 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4825.md b/eval/dataset/arms/copilot_v20cb_val/pr4825.md new file mode 100644 index 000000000..7c8853147 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4825.md @@ -0,0 +1,55 @@ +# Run report + +- checkout: PR-TIME TREE (head ac3a2ce5a9b9) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | 2 finding(s) below | +| Correctness | no finding reported | + +**Validated:** +- [resolved] Concern: to_out.0 checkpoint normalization — removed at ac3a2ce5 (`weights_mapper=None` manager.py:309); residual: SDXL UNet uses `nn.ModuleList` paths ending in `to_out.0` (sdxl_unet.py:151-178) without load_weights remap unlike flux/qwen transformers +- [claim-verified] PR body: base-identical SDXL+LoRA output — mechanism matches zero layer replacements under `unet` while adapter load/activate still succeeded +- [claim-verified] PR manual SDXL test (stabilityai/stable-diffusion-xl-base-1.0 + cosmic-tf/sdxl_finetuned_8styles) — no automated SDXL LoRA test in repo (grep tests for sdxl/SDXL: no matches) +- [claim-verified] PR manual test model stabilityai/stable-diffusion-xl-base-1.0 matches StableDiffusionXLPipeline registry entry +- [claim-verified] PR body manual SDXL+cosmic-tf LoRA images show base vs LoRA divergence — author evidence only, not CI +- [validated] vllm_omni/diffusion/lora/manager.py:374-381 — adds `unet` to `default_components`; `StableDiffusionXLPipeline` defines `self.unet` at pipeline_sdxl.py:110 + +`vllm_omni/diffusion/lora/manager.py:372` [minor] — This diff adds `"unet"` to `default_components` (manager.py:372-381) so SDXL-style pipelines scan the denoiser, but PR head ac3a2ce5 still leaves `weights_mapper=None` in `_load_adapter` (manager.py:309) after reverting the companion `WeightsMapper(orig_to_new_substr={".to_out.0.": ".to_out."})` and its unit test. SDXL UNet registers attention output projections as `to_out.0` inside `nn.ModuleList` (sdxl_unet.py:151-155, 175-179), and unlike flux2-style models it does not normalize `.to_out.0.` in `load_weights`. Adapters that target/wrap `…to_out.0` may therefore still fail to bind checkpoint keys at load/activate time. Please confirm the PR-tested adapter does not depend on that mapper, restore it with the reverted test, or document Q/K/V-only scope and track `to_out.0` normalization as follow-up. (evidence: unchanged by this diff, present in PR-time tree: manager.py:309 `weights_mapper=None,`; manager.py:372-380 `# NOTE: SDXL-style pipelines expose the denoiser as ``unet``. ... "unet",`; sdxl_unet.py:151-154 `self.to_out = nn.ModuleList([ RowParallelLinear(inner_dim, dim, bias=True), ])`; commit ac3a2ce5 reverted `WeightsMapper(orig_to_new_substr={".to_out.0.": ".to_out."})` back to `weights_mapper=None` and deleted `test_lora_manager_load_adapter_uses_to_out_normalization_mapper`.) + +Proposed change (not a ready patch): weights_mapper = WeightsMapper( + orig_to_new_substr={ + ".to_out.0.": ".to_out.", + } + ) + + lora_model = LoRAModel.from_local_checkpoint( + lora_path, + expected_lora_modules=self._expected_lora_modules, + peft_helper=peft_helper, + lora_model_id=lora_request.lora_int_id, + device="cpu", + dtype=self.dtype, + model_vocab_size=None, + tensorizer_config_dict=lora_request.tensorizer_config_dict, + weights_mapper=weights_mapper, + ) + +`tests/diffusion/lora/test_lora_manager.py:580` [minor] — This diff adds `test_lora_manager_discovers_unet_component` (test_lora_manager.py:580-619), which monkeypatches `from_layer_diffusion`/`replace_submodule` and calls `_replace_layers_with_lora` on a fake `pipeline.unet` tree with `_FakeLinearBase` only—proving the new `"unet"` entry routes replacement, not that a real SDXL UNet `QKVParallelLinear`/`RowParallelLinear` is wrapped and activated through `add_adapter` → `_load_adapter` → `_activate_adapter`. Unlike bagel/sensenova e2e (`test_bagel_expansion.py`, `test_sensenova_u1_expansion.py`) and the Z-Image case in `test_diffusion_lora.py`, there is no in-repo SDXL LoRA regression; consider a CPU-level test with real vLLM parallel linear types (or an offline/L4 image-diff test for `stabilityai/stable-diffusion-xl-base-1.0`) so SDXL target-module matching regressions are caught in CI. (evidence: tests/diffusion/lora/test_lora_manager.py:593 `monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion)`; :604 `pipeline.unet.down_block.proj = _FakeLinearBase()`; :613-614 `peft_helper = type("_PH", (), {"r": 1})()` / `manager._replace_layers_with_lora(peft_helper)` — no `add_adapter`/`_activate_adapter` call. Unchanged in PR-time tree: vllm_omni/diffusion/models/sdxl/sdxl_unet.py:145 `self.to_qkv = QKVParallelLinear(`; :153 `RowParallelLinear(inner_dim, dim, bias=True),`. Deeper LoRA coverage elsewhere: tests/diffusion/lora/test_diffusion_lora.py:100 `# Real LoRA E2E: generate again with a real on-disk PEFT adapter`. No SDXL tests: `grep -ri 'sdxl|SDXL' tests/` returned zero matches.) + +`tests/diffusion/lora/test_lora_manager.py:~19` [nit] — This diff adds tests in tests/diffusion/lora/test_lora_manager.py (pytestmark core_model+cpu at line 19). AMD blocking CI runs `pytest -sv tests/diffusion -m 'core_model and cpu'` (.buildkite/test-amd-merge.yml:21), so these tests are in scope. If buildkite/vllm-omni-amd-ci is red at head, please confirm whether failures are pre-existing on main or introduced here (NPU ready config does not run this diffusion CPU suite). (evidence: tests/diffusion/lora/test_lora_manager.py:19 `pytestmark = [pytest.mark.core_model, pytest.mark.cpu]`; unchanged by this diff, present in PR-time tree: .buildkite/test-amd-merge.yml:21 `- "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"`) + +`vllm_omni/diffusion/lora/manager.py:380` [nit] — This diff globally adds "unet" to default_components (manager.py:375-381). At head, only StableDiffusionXLPipeline exposes top-level pipeline.unet (pipeline_sdxl.py:110), so blast radius is SDXL-only today. SDXL already declares _dit_modules=["unet"] (pipeline_sdxl.py:46), which the LoRA manager does not consult; the manager's opt-in hook is _lora_components (manager.py:382), currently used by sensenova_u1 (pipeline_sensenova_u1.py:543). Consider _lora_components = ["unet"] on SDXL if you want to avoid globally scanning every pipeline that happens to expose a top-level unet attribute. (evidence: manager.py:375-382 `default_components = ("transformer", "transformer_2", "dit", "bagel", "unet",)` and `extra_components = tuple(getattr(self.pipeline, "_lora_components", ()) or ())`; unchanged by this diff, present in PR-time tree: pipeline_sdxl.py:46 `_dit_modules: list[str] = ["unet"]`; pipeline_sdxl.py:110 `self.unet = SDXLUNet2DConditionModel(od_config=od_config)` (sole pipeline*.py top-level self.unet); pipeline_sensenova_u1.py:543 `_lora_components: ClassVar[list[str]] = ["language_model"]`; no `_dit_modules` references under vllm_omni/diffusion/lora/.) + +```suggestion +extra_components = tuple(getattr(self.pipeline, "_lora_components", ()) or ()) + # On StableDiffusionXLPipeline add: _lora_components: ClassVar[list[str]] = ["unet"] +``` + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4825.trace.json.gz b/eval/dataset/arms/copilot_v20cb_val/pr4825.trace.json.gz new file mode 100644 index 000000000..a9f71ebd4 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_val/pr4825.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4837.cost.json b/eval/dataset/arms/copilot_v20cb_val/pr4837.cost.json new file mode 100644 index 000000000..5f26aa24e --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4837.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 411.3, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val/runs/pr4837/run-20260818-112613-1811ae", + "recorded_at": "2026-08-18T11:33:04", + "attempts": [ + { + "invocation_id": "inv-8f5cb48c", + "outcome": "run:run-20260818-112613-1811ae", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260818-112613-1811ae", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260818-112613-1811ae", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.2377, + "minutes": 6.85, + "ci_minutes": 0, + "input_tokens": 551181, + "output_tokens": 80834, + "tool_calls": 356, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.3401, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.96, + "gate": 1.61, + "review": 405.06, + "report": 0.08 + }, + "time_to_first_result_s": 410.68 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4837.trace.json.gz", + "trace_bytes": 17402 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4837.md b/eval/dataset/arms/copilot_v20cb_val/pr4837.md new file mode 100644 index 000000000..ac3e10b28 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4837.md @@ -0,0 +1,57 @@ +# Run report + +- checkout: PR-TIME TREE (head 78cc4dcbb161) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 3 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 2 finding(s) below | + +**Validated:** +- [resolved] Ming outputs[0] indexing — pre-PR pipeline_ming_imagegen.py used `output = outputs[0]` + hasattr chain; ZImagePipeline.forward returns `DiffusionOutput` directly (pipeline_z_image.py:720); PR correctly uses `outputs.output` and returns full `outputs` to keep `stage_durations` +- [resolved] Concern that unwrap only on update path was intentional — old `already_submitted and len==1` gated unwrap to `submit_update` only; residual checked: `async_chunk` prewarm for Ming uses `async_chunk: false` (ming_flash_omni_image.yaml:19) so initial-submit path is the live failure mode +- [claim-verified] PR body cites #4079 regression — commit c45ac74d introduced request-level batching and stage_pool list-prompt rejection; `already_submitted and len==1` unwrap predates this PR on main +- [claim-verified] Blast radius: other list-returning diffusion bridges (`hunyuan_image3.ar2diffusion` at hunyuan_image3.py:222) with single element also benefit; `glm_image.ar2diffusion` returns bare `dict` (glm_image.py:220) and is unaffected +- [claim-refuted] PR manual serve/curl plan — no CI job loads `vllm_omni/deploy/ming_flash_omni_image.yaml`; e2e uses `ci/ming_flash_omni_thinker_only.yaml` +- [validated] orchestrator.py:1290-1291 — removing `already_submitted` restores unwrap on initial submit; `thinker2imagegen` returns `[{"prompt": "", "extra": ...}]` (ming_flash_omni.py:435) and `submit_initial` raises on `list` (stage_pool.py:951-955) + +`vllm_omni/engine/orchestrator.py:1290` [minor] — This hunk drops the `already_submitted` guard so one-element bridge lists unwrap before the first `submit_initial` (orchestrator.py:1290–1291). Ming `thinker2imagegen` returns `[{"prompt": "", "extra": extra}]` (ming_flash_omni.py:435); `StagePool.submit_initial` rejects list prompts (stage_pool.py:951–955), which was the #4839 failure mode when unwrap only ran on updates. Existing `test_orchestrator_kv_sender_info.py` cases leave `custom_process_input_func=None` and never drive a singleton bridge list through this path—add an L2 regression that stubs the bridge to return a one-element list and asserts `diffusion_stage.calls[0]["prompt"]` is the inner dict. (evidence: orchestrator.py:1290 `if len(diffusion_prompt) == 1:`; orchestrator.py:1291 `diffusion_prompt = diffusion_prompt[0]`; orchestrator.py:1297–1298 `else: await next_pool.submit_initial(`; ming_flash_omni.py:435 `return [{"prompt": "", "extra": extra}]`; stage_pool.py:951–955 `if isinstance(request, list): raise ValueError(`; tests/engine/test_orchestrator_kv_sender_info.py:31 `custom_process_input_func = None`) + +```suggestion +def test_forward_to_diffusion_unwraps_singleton_bridge_list_on_initial_submit(): + orchestrator = object.__new__(Orchestrator) + bridge_output = [{"prompt": "", "extra": {"thinker_hidden_states": object()}}] + diffusion_stage = _DummyDiffusionStage(engine_input_source=[0]) + diffusion_stage.custom_process_input_func = lambda *_a, **_k: bridge_output + sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151}) + diffusion_pool = StagePool(1, diffusion_stage) + orchestrator.num_stages = 2 + orchestrator.stage_pools = [sender_pool, diffusion_pool] + orchestrator._cfg_tracker = CfgCompanionTracker() + params = OmniDiffusionSamplingParams() + req_state = OrchestratorRequestState( + request_id="req-bridge", + prompt={"prompt": "draw a cat"}, + sampling_params_list=[SamplingParams(max_tokens=4), params], + final_stage_id=1, + ) + output = SimpleNamespace(request_id="req-bridge", finished=True) + asyncio.run(Orchestrator._forward_to_next_stage(orchestrator, "req-bridge", 0, output, req_state)) + assert len(diffusion_stage.calls) == 1 + assert diffusion_stage.calls[0]["prompt"] is bridge_output[0] + assert not isinstance(diffusion_stage.calls[0]["prompt"], list) +``` + +`ming_flash_omni_image.yaml:19` [minor] — Resolved concern that unwrap-on-update-only was intentional: old `already_submitted and len(diffusion_prompt) == 1` gated unwrap to submit_update; Ming deploy config sets `async_chunk: false` (vllm_omni/deploy/ming_flash_omni_image.yaml:19), so async_chunk prewarm never runs and the first diffusion handoff uses submit_initial — that is the live failure mode this PR fixes. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/deploy/ming_flash_omni_image.yaml:19 `async_chunk: false`; orchestrator.py:486-487 `if self.async_chunk and stage_id == 0 and final_stage_id > 0:` / `await self._prewarm_async_chunk_stages(request_id, prompt, req_state)`; orchestrator.py:1290-1298 `if len(diffusion_prompt) == 1:` / `diffusion_prompt = diffusion_prompt[0]` then `if already_submitted:` / `await next_pool.submit_update(...)` / `else:` / `await next_pool.submit_initial(...)`; PR diff removes `already_submitted and` from the unwrap guard at orchestrator.py ~1290) + +`vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:475` [minor] — This hunk changes MingImagePipeline.forward to read `raw = outputs.output` (line 465), validate it is a torch.Tensor, and `return outputs` (line 475) instead of re-wrapping `DiffusionOutput(output=raw)` — preserving parent fields such as `stage_durations` from ZImagePipeline.forward (line 720). The diff adds no automated test for this return contract; only config-registry wiring cites MingImagePipeline (e.g. tests/test_config_factory.py). Add a mocked unit/L2 test that patches ZImagePipeline.forward to return a DiffusionOutput with `stage_durations` and asserts MingImagePipeline.forward returns that object unchanged. (evidence: unchanged by this diff, present in PR-time tree: vllm_omni/diffusion/models/z_image/pipeline_z_image.py:720 `return DiffusionOutput(output=image, stage_durations=stage_durations)`; vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:465 `raw = outputs.output`; vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:475 `return outputs`; tests/test_config_factory.py:1344 `assert s.model_arch == "MingImagePipeline"` (config wiring only, no forward() test)) + +`vllm_omni/engine/orchestrator.py:1290` [nit] — This diff only touches `vllm_omni/engine/orchestrator.py` (line 1290: unwrap single-element `diffusion_prompt`) and `pipeline_ming_imagegen.py` (return parent `DiffusionOutput` from `super().forward`). No AMD/NPU/platform files are in the diff — please confirm `buildkite/vllm-omni-amd-ci` / `buildkite/vllm-omni-npu-ci` failures at head are already failing on main. (evidence: merge-base diff_stat (head 78cc4dcbb161): only `vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py` (+3/-9) and `vllm_omni/engine/orchestrator.py` (+1/-1). vllm_omni/engine/orchestrator.py:1290 `if len(diffusion_prompt) == 1:`. vllm_omni/diffusion/models/ming_flash_omni/pipeline_ming_imagegen.py:475 `return outputs` (parent `DiffusionOutput`; unchanged by this diff vs prior `return DiffusionOutput(output=raw)`).) + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4837.trace.json.gz b/eval/dataset/arms/copilot_v20cb_val/pr4837.trace.json.gz new file mode 100644 index 000000000..432d4f933 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_val/pr4837.trace.json.gz differ diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4893.cost.json b/eval/dataset/arms/copilot_v20cb_val/pr4893.cost.json new file mode 100644 index 000000000..41d6a90c9 --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4893.cost.json @@ -0,0 +1,94 @@ +{ + "wall_s": 558.1, + "split": "val", + "rc": 0, + "run_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val/runs/pr4893/run-20260818-112614-797d28", + "recorded_at": "2026-08-18T11:35:30", + "attempts": [ + { + "invocation_id": "inv-4c457881", + "outcome": "run:run-20260818-112614-797d28", + "rc": 0, + "pre_run": null + }, + { + "run_dir": "run-20260818-112614-797d28", + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace" + } + ], + "attempt_usd_total": 0.0, + "llm_calls": 0, + "input_tokens": 0, + "output_tokens": 0, + "usd": 0.0, + "source": "no-trace", + "metrics": { + "schema": 1, + "run": "run-20260818-112614-797d28", + "task_kind": "pr_review", + "status": "done", + "quality": { + "q": null, + "weight_coverage": 0.0, + "partial": true, + "abstained": false, + "components": { + "recall_w": null, + "precision_v": null, + "useful": null, + "calib": null, + "decision": null + } + }, + "cost": { + "usd": 0.4043, + "minutes": 9.28, + "ci_minutes": 0, + "input_tokens": 1063917, + "output_tokens": 106365, + "tool_calls": 347, + "usd_ref": 1.0, + "min_ref": 10.0, + "cost_index": 1.4745, + "cost_partial": false, + "unpriced_calls": 0, + "source": "events", + "llm_calls": null, + "by_role": null, + "cache_read_tokens": null, + "cache_creation_tokens": null + }, + "timings": { + "steps": { + "fetch": 3.99, + "gate": 1.81, + "review": 550.57, + "report": 0.07 + }, + "time_to_first_result_s": 556.45 + }, + "risk": { + "incidents": { + "catastrophic": 0, + "severe": 0, + "moderate": 0, + "minor": 0 + }, + "safety_multiplier": 1.0 + }, + "catq": null, + "tus": null, + "signals": { + "escalations": 0, + "pushes": 0, + "posted": 0, + "steps_completed": 4 + } + }, + "trace": "pr4893.trace.json.gz", + "trace_bytes": 26178 +} \ No newline at end of file diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4893.md b/eval/dataset/arms/copilot_v20cb_val/pr4893.md new file mode 100644 index 000000000..b039bcfcc --- /dev/null +++ b/eval/dataset/arms/copilot_v20cb_val/pr4893.md @@ -0,0 +1,88 @@ +# Run report + +- checkout: PR-TIME TREE (head 0128660809e1) — the tree matches the diff exactly; repo-wide greps DO reflect PR-time state + +## review_text + +**Scan:** +| Category | Result | +|---|---| +| Tests / verification | 9 finding(s) below | +| Security | no finding reported | +| Docs / comments | no finding reported | +| Behavior / compatibility | no finding reported | +| Correctness | 3 finding(s) below | + +**Validated:** +- [resolved] Concern: aliasing vllm_parallel_state._PCP = _SP — fixed at parallel_state.py:859-864 with init_vllm_model_parallel_group; residual checked: TP/EP also migrated to vLLM coordinators +- [resolved] Concern: cfg_parallel_size>1 guard blocked tp2_sp2 remap — removed at parallel_state.py:882; residual: cfg=1 yields cfg-dp world_size=1 so dp_metadata correctly skipped via hunyuan_fused_moe.py:38-39 +- [claim-verified] Fixes #4885 layouts tp2_cfgp2 and tp2_sp2: perf JSONs at tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json and test_hunyuan_image_tp2_sp2.json; PR reports remote 4-GPU smoke for both +- [claim-verified] PR '11 passed' matches two touched test files with 11 total test functions +- [claim-verified] PR 11 passed — not re-run here; tests are CPU mocks only +- [claim-verified] PR test plan cites 11 passed for target pytest files; not re-run in this read-only step + +`tests/diffusion/distributed/test_expert_parallel_layout.py:~100` [minor] — This PR removes the `cfg_parallel_size > 1` guard so vLLM `_PCP`/`_TP`/`_DP`/`_EP` coordinators are built via `init_vllm_model_parallel_group` whenever `use_moe_parallel_mapping` is true, including tp2_sp2 with `cfg_parallel_size=1`. `test_moe_ep_maps_diffusion_sp_cfg_dp_to_vllm_groups` (line 100) only initializes with `cfg_parallel_size=2` and `world_size=32`. Add a focused sibling test with `tensor_parallel_size=2`, `sequence_parallel_size=2`, `ulysses_degree=2`, `cfg_parallel_size=1`, `world_size=4` asserting `vllm_parallel_state._PCP is not parallel_state._SP`, vLLM coordinators have `device_communicator`, and `vllm_group_names == ["vllm_pcp", "vllm_tp", "vllm_dp", "vllm_ep"]`. (evidence: unchanged by this diff, present in PR-time tree: parallel_state.py:868 `if use_moe_parallel_mapping:`; parallel_state.py:869 `vllm_parallel_state._TP = init_vllm_model_parallel_group(`; parallel_state.py:882 `if use_moe_parallel_mapping:`; parallel_state.py:883 `vllm_parallel_state._DP = init_vllm_model_parallel_group(`; parallel_state.py has zero matches for `cfg_parallel_size > 1`; test_expert_parallel_layout.py:42 `world_size = 32`; test_expert_parallel_layout.py:106 `cfg_parallel_size=2,`; test_expert_parallel_layout.py has zero matches for `cfg_parallel_size=1`) + +```suggestion +@pytest.mark.cpu +@pytest.mark.core_model +def test_moe_ep_sp2_cfg1_builds_vllm_coordinators(monkeypatch): + """tp2_sp2 needs vLLM coordinators even when cfg_parallel_size==1.""" + # reuse _FakeGroup + fake_init_vllm_model_parallel_group from sibling test + parallel_state.initialize_model_parallel( + tensor_parallel_size=2, + sequence_parallel_size=2, + ulysses_degree=2, + ring_degree=1, + pipeline_parallel_size=1, + cfg_parallel_size=1, + data_parallel_size=1, + enable_expert_parallel=True, + backend="gloo", + ) + assert parallel_state.vllm_parallel_state._PCP is not parallel_state._SP + assert parallel_state.vllm_parallel_state._PCP.device_communicator is not None +``` + +`tests/diffusion/models/hunyuan_image3/test_hunyuan_fused_moe.py:~74` [minor] — The diff registers `_num_tokens_pre_hook` on the FusedMoE runner (hunyuan_fused_moe.py:85-93) to call `_set_forward_context_dp_metadata(hidden_states.shape[0])` before dispatch, but `test_sets_dp_metadata_from_vllm_dp_group` invokes `_set_forward_context_dp_metadata(1024)` directly at test_hunyuan_fused_moe.py:74. Consider adding a test that instantiates `HunyuanFusedMoEDefault` with a mocked `FusedMoE` runner and asserts the registered pre-hook triggers `all_gather_object` and populates `forward_context.dp_metadata`—the path vLLM 0.24 all2all dispatch actually hits. (evidence: test_hunyuan_fused_moe.py:74 `hunyuan_moe._set_forward_context_dp_metadata(1024)` — direct helper call, not via pre-hook; hunyuan_fused_moe.py:91 `_set_forward_context_dp_metadata(hidden_states.shape[0])` inside `_num_tokens_pre_hook`; hunyuan_fused_moe.py:93 `moe_runner.register_forward_pre_hook(_num_tokens_pre_hook, with_kwargs=True)` — production wiring; tests/ grep: no `register_forward_pre_hook` references outside source) + +```suggestion +def test_pre_hook_sets_dp_metadata_before_runner_forward(self, mocker): + import vllm_omni.diffusion.models.hunyuan_image3.hunyuan_fused_moe as hunyuan_moe + mock_runner = mocker.MagicMock() + mocker.patch("vllm.model_executor.layers.fused_moe.FusedMoE", return_value=mock_runner) + ... + impl = hunyuan_moe.HunyuanFusedMoEDefault(prefix="") + # invoke captured pre_hook with hidden_states shape [1024, H] + mock_all_gather.assert_called_once() +``` + +`vllm_omni/diffusion/distributed/parallel_state.py:526` [minor] — This diff adds `init_vllm_model_parallel_group`, which forwards `use_device_communicator=True` to `vllm_parallel_state.init_model_parallel_group` (parallel_state.py:520-526), but `test_expert_parallel_layout.py` monkeypatches the entire wrapper with `fake_init_vllm_model_parallel_group` (test_expert_parallel_layout.py:62-76, 92), so CI cannot catch regressions that drop that kwarg. Add a unit test patching `vllm_parallel_state.init_model_parallel_group` and asserting `use_device_communicator=True` is forwarded. (evidence: parallel_state.py:526 `use_device_communicator=True,` — wrapper forwards kwarg to upstream init; unchanged by this diff, present in PR-time tree: test_expert_parallel_layout.py:92 `monkeypatch.setattr(parallel_state, "init_vllm_model_parallel_group", fake_init_vllm_model_parallel_group)` — bypasses wrapper; test_expert_parallel_layout.py:72 `device_communicator=object(),` — fake injects communicator directly instead of relying on forwarding) + +```suggestion +def test_init_vllm_model_parallel_group_forwards_device_communicator(mocker): + mock_init = mocker.patch.object( + parallel_state.vllm_parallel_state, "init_model_parallel_group", return_value=mocker.MagicMock() + ) + parallel_state.init_vllm_model_parallel_group([[0, 1]], 0, "gloo", "tp") + mock_init.assert_called_once_with( + group_ranks=[[0, 1]], local_rank=0, backend="gloo", + group_name="tp", use_device_communicator=True, + ) +``` + +`parallel_state.py:859` [minor] — [resolved] MoE mapping no longer aliases vllm_parallel_state._PCP to diffusion _SP; parallel_state.py:859-864 builds PCP via init_vllm_model_parallel_group, and TP/DP/EP are likewise migrated under use_moe_parallel_mapping (869/885/904). (evidence: parallel_state.py:859 `vllm_parallel_state._PCP = init_vllm_model_parallel_group(` — dedicated vLLM coordinator instead of `_SP` aliasing; parallel_state.py:869 `vllm_parallel_state._TP = init_vllm_model_parallel_group(`; parallel_state.py:904 `vllm_parallel_state._EP = init_vllm_model_parallel_group(`; test_expert_parallel_layout.py:112 `assert parallel_state.vllm_parallel_state._PCP is not parallel_state._SP`) + +`general` [minor] — At merge-base, vLLM `_DP` was replaced with a cfg-dp vLLM coordinator only when `use_moe_parallel_mapping and cfg_parallel_size > 1`; head always builds it under MoE mapping. Needed for tp2_sp2 (cfg=1, SP=2, data_parallel_size=1): diffusion `_DP` stays world_size 1, but PCP/TP/EP still require vLLM coordinators with `device_communicator`/`reduce_scatter`. (evidence: PR head parallel_state.py:882 `if use_moe_parallel_mapping:` — no `cfg_parallel_size > 1` guard. PR head parallel_state.py:885 `vllm_parallel_state._DP = init_vllm_model_parallel_group(` with `group_name="dp"`. PR head parallel_state.py:810-816 diffusion DP init and alias: `_DP = init_model_parallel_group(group_ranks=rank_generator.get_ranks("dp"), ...)` then `vllm_parallel_state._DP = _DP`. PR head parallel_state.py:853-864 / 868-873 / 902-908 MoE mapping builds vLLM PCP/TP/EP via `init_vllm_model_parallel_group`. PR diff minus hunk (merge-base): `if use_moe_parallel_mapping and cfg_parallel_size > 1:` guarded vLLM `_DP` replacement. tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json:10-12 `"tensor-parallel-size": 2`, `"ulysses-degree": 2`. vllm_omni/deploy/hunyuan_image3_dit.yaml:27 `cfg_parallel_size: 1`; data_parallel_size defaults to 1 at parallel_state.py:692.) + +`parallel_state.py:919` [minor] — destroy_model_parallel PCP/DP split cleanup — `if vllm_parallel_state._DP and vllm_parallel_state._DP is not _DP: destroy()` at parallel_state.py:919-921; PCP guard at 931-933; vLLM _TP/_EP destroy paths unchanged at 939-945 (evidence: parallel_state.py:919-921 `if vllm_parallel_state._DP and vllm_parallel_state._DP is not _DP:` / `vllm_parallel_state._DP.destroy()`; parallel_state.py:931-933 `if vllm_parallel_state._PCP and vllm_parallel_state._PCP is not _SP:` / `vllm_parallel_state._PCP.destroy()`; parallel_state.py:939-945 unchanged vLLM _TP/_EP: `if vllm_parallel_state._TP: vllm_parallel_state._TP.destroy()` and `if vllm_parallel_state._EP: vllm_parallel_state._EP.destroy()`) + +`vllm_omni/diffusion/distributed/parallel_state.py:515` [minor] — Gate report shows buildkite/vllm-omni-amd-ci and buildkite/vllm-omni-npu-ci red at PR head. Under use_moe_parallel_mapping, initialize_model_parallel now builds vLLM EP via init_vllm_model_parallel_group and exposes ranks through get_expert_parallel_group_ranks(), which NPU Hunyuan MC2 setup reads when enable_expert_parallel (platforms/npu/models/hunyuan_fused_moe.py:77-86). Can you confirm whether those lanes fail on main too or are introduced here? If NPU-specific, a cfg=1+SP EP layout smoke on NPU would close the loop. (evidence: parallel_state.py:515 `def init_vllm_model_parallel_group(`; parallel_state.py:904 `vllm_parallel_state._EP = init_vllm_model_parallel_group(`; parallel_state.py:306 `def get_expert_parallel_group_ranks() -> list[list[int]]:` and parallel_state.py:309 `return _EXPERT_PARALLEL_GROUP_RANKS`; unchanged by this diff, present in PR-time tree: vllm_omni/platforms/npu/models/hunyuan_fused_moe.py:77 `if vllm_config.parallel_config.enable_expert_parallel:` and hunyuan_fused_moe.py:85 `group_ranks=get_expert_parallel_group_ranks(),`) + +**Additional observations (beyond the comment budget):** +- `tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json:3` [minor] Hunyuan TP2+SP2 and TP2+CfgP2 launch paths are covered by perf recipes (`tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json`, `test_hunyuan_image_tp2_cfgp2.json`; both already set `skip-performance-assertion: true`) but neither is referenced in `.buildkite/test-nightly.yml`, unlike `test_qwen_image_vllm_omni.json`. Consider wiring a launch-only nightly job or stating these remain manual L4 runs. +- `.buildkite/test-amd-ready.yaml:19` [minor] Head shows red `buildkite/vllm-omni-amd-ci` and `buildkite/vllm-omni-npu-ci`. This PR changes `parallel_state.py` and Hunyuan MoE helpers with new/updated CPU tests under `tests/diffusion`, which are collected by the AMD gate at `.buildkite/test-amd-ready.yaml:19` (`pytest -sv tests/diffusion ... -m 'core_model and cpu'`). The NPU ready gate (`.buildkite/test-npu-ready.yaml:21`) only runs `test_qwen3_tts_customvoice.py`. Can you confirm whether the AMD failure is pre-existing on main or tied to this PR’s CPU diffusion tests? +- `tests/diffusion/distributed/test_expert_parallel_layout.py:124` [minor] The test fakes vLLM coordinators with reduce_scatter via fake_init_vllm_model_parallel_group while diffusion _SP uses fake_init_model_parallel_group without it. Lines 124-126 assert vLLM _PCP/_DP/_EP have reduce_scatter, but there is no complementary assert that diffusion _SP stays bare—add `assert not hasattr(parallel_state._SP, "reduce_scatter")` next to those checks to lock the separation described in parallel_state.py. +- `tests/diffusion/distributed/test_expert_parallel_layout.py:121` [minor] The fake wires `device_communicator` and `reduce_scatter` for every vLLM coordinator it creates (pcp/tp/dp/ep), and MoE production routes `_TP` through the same helper. Assertions at lines 121–126 cover `_PCP`, `_DP`, and `_EP` only—extend them to `vllm_parallel_state._TP` so a regression dropping `use_device_communicator` on the TP path is caught. + +**Verdict:** COMMENT diff --git a/eval/dataset/arms/copilot_v20cb_val/pr4893.trace.json.gz b/eval/dataset/arms/copilot_v20cb_val/pr4893.trace.json.gz new file mode 100644 index 000000000..5de3f14a4 Binary files /dev/null and b/eval/dataset/arms/copilot_v20cb_val/pr4893.trace.json.gz differ diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/JUDGE_REPORT.md b/eval/dataset/judgments/goal_v20cb_train_sonnet/JUDGE_REPORT.md new file mode 100644 index 000000000..f0fbe0f87 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/JUDGE_REPORT.md @@ -0,0 +1,51 @@ +# Judgment: copilot_v20cb_train vs claudecode_opus5 + +Judge: claude-sonnet-5 (blind, randomized order, 3 replicate(s) x 10 item(s) = 30 verdicts) + +## Wins + +- copilot_v20cb_train: 11 +- claudecode_opus5: 19 +- tie: 0 + +## Mean rubric scores + +| arm | actionability | gap_hit | precision | recall | +|---|---|---|---|---| +| claudecode_opus5 | 0.86 | 0.00 | 0.71 | 0.63 | +| copilot_v20cb_train | 0.75 | 0.00 | 0.76 | 0.63 | + +## Per-verdict detail + +| item.rep | winner | margin | rationale (head) | +|---|---|---|---| +| pr4804.r1 | claudecode_opus5 | clear | Both candidates miss linyueqian's cumulative/delta re-slice bug and the audio_tokenizer_v2.py:949 ceil/floor padding bug, but Y covers both of the two High-severity ground-truth findings substantively | +| pr4804.r2 | claudecode_opus5 | slight | Both candidates independently rediscover the two flagship ground-truth issues (slot lifecycle on abort/exhaustion, and v2-tokenizer-vs-legacy-checkpoint model_type collision) plus the WER PCM-fallback | +| pr4804.r3 | claudecode_opus5 | slight | X surfaces more of the ground truth's substantive open concerns, notably the reviewer-flagged [High] v2-tokenizer-misdetection bug (hsliuustc0106) as a top-billed major finding, plus the WER-fallback | +| pr4817.r1 | claudecode_opus5 | clear | Ground truth has no substantive concerns, so recall is vacuous for both. Both catch the key CI-blind-spot (missing pytestmark deselects the new test), but X verifies it with an actual local pytest run | +| pr4817.r2 | claudecode_opus5 | slight | Ground truth has no substantive reviewer concerns, so recall is trivially satisfied by both. Both independently catch the same critical finding (new test lacks pytestmark and is silently deselected by | +| pr4817.r3 | claudecode_opus5 | decisive | Ground truth carries no substantive concerns (bot spam, 'thanks', no inline comments), so recall is vacuous for both. Both candidates independently and correctly flag the real CI gap (new test file la | +| pr4859.r1 | copilot_v20cb_train | slight | X has broader nominal coverage and crisper remediation writing, but its 'High' finding on audio_vae.py:141 fabricates a forward-pass regression ('verified locally... max abs diff 0.083') that directly | +| pr4859.r2 | copilot_v20cb_train | slight | Both independently surface the same high-value bug the human reviewers missed (env-driven sample rate mis-decoding already-normalized 24kHz PCM in seed_tts_eval), and both hit the language/dialect-dro | +| pr4859.r3 | claudecode_opus5 | slight | Y covers more ground-truth concerns than X (audio_vae.py config-mutation risk and dropped env-var documentation are entirely absent from X but substantively addressed by Y), giving Y meaningfully high | +| pr4870.r1 | copilot_v20cb_train | slight | Both correctly recognize the diff already bakes in the qwen3_tts scoping fix and the seq_len removal, and both independently catch the async_chunk getattr default inconsistency (True vs False) that ec | +| pr4870.r2 | copilot_v20cb_train | clear | X explicitly cross-references the actual review thread's resolution status (confirming the Med gating-scope and Low fallback-robustness concerns were addressed) while surfacing a well-grounded new res | +| pr4870.r3 | copilot_v20cb_train | slight | Both candidates independently recall the ground-truth Low (async_chunk fallback robustness) concern and validate the already-resolved Med scoping fix, but neither surfaces amy-why-3459's model-placeme | +| pr4923.r1 | claudecode_opus5 | clear | Both candidates are technically detailed and well-evidenced, but Y engages more directly and more deeply with the PR's actual central controversy — the cudagraph_mode/talker_mtp_accepts_per_row_genera | +| pr4923.r2 | claudecode_opus5 | slight | Both reviews independently rediscover the two GT threads (model reading cudagraph_mode should live upstream; per-row seed breaks under full cudagraphs) plus the NPU compilation_config gap, and both ar | +| pr4923.r3 | claudecode_opus5 | clear | Both candidates ground findings in real code paths, but Y lands much closer to the actual reviewer thread: it independently reconstructs gcanlin's architectural objection to reading cudagraph_mode in | +| pr4926.r1 | copilot_v20cb_train | clear | Both cover the core live issues (Hub kernel version-pinning risk, FA3 Hopper/Blackwell gating nuance, per-layer kernel reload cost, thin test coverage), but Y grounds every claim in explicit file:line | +| pr4926.r2 | copilot_v20cb_train | slight | Both hit the two big confirmed bugs (missing SM90+ gate on FLASH_ATTN_3_HUB, and the varlen/func mismatch that piecewise_attn can crash on), and both flag the duplicated-code smell, the degenerate q=k | +| pr4926.r3 | claudecode_opus5 | clear | Both candidates go well beyond the ground truth's fairly light docs/test-marker complaints into real code-level bugs, and both independently converge on several genuine issues (uncached get_kernel() p | +| pr4950.r1 | claudecode_opus5 | clear | Ground truth has no substantive concerns (just LGTM approvals), so recall is trivially satisfied by both. X offers only two adjacent-file/process findings, one of which (README gap) is arguably underc | +| pr4950.r2 | claudecode_opus5 | slight | Ground truth is a rubber-stamp approval with zero concerns, so recall is trivially satisfied by both. X does deeper verification (traces the actual talker/serving_chat code path to dispute the PR's ow | +| pr4950.r3 | claudecode_opus5 | clear | Ground truth has no substantive concerns (just LGTM/approve), so recall is trivially satisfied by both. X goes much deeper, tracing the actual failure mode of missing use_tts_template through three fi | +| pr4970.r1 | copilot_v20cb_train | clear | Ground truth is thin (essentially 'LGTM' plus a request to split out the VoxCPM2 regression fix into a separate PR) for a trivial 2-line seed removal. X stays appropriately scoped, grounds every claim | +| pr4970.r2 | copilot_v20cb_train | clear | The only substantive ground-truth concern is Gaohan123's ask to split out the VoxCPM2 regression fix into a separate PR; X explicitly addresses this ('VoxCPM2 #4963 remainder explicitly out of scope — | +| pr4970.r3 | copilot_v20cb_train | slight | Ground truth is nearly empty (LGTM plus a tangential ask to split out a VoxCPM2 regression fix); Y explicitly validates that VoxCPM2/#4963 is out of scope and tracked separately, directly matching the | +| pr4977.r1 | claudecode_opus5 | slight | Both candidates independently surface the same underlying fact the ground truth's sole inline concern (trust_remote_code/kernels-version fragility) was already resolved by a later commit, and both fla | +| pr4977.r2 | claudecode_opus5 | slight | Both candidates correctly identify and resolve the single ground-truth concern (stale trust_remote_code claim, fixed at 9947f414) and add substantive, diff-grounded findings with file:line citations w | +| pr4977.r3 | copilot_v20cb_train | slight | Both candidates independently dug past the truncated diff and correctly identified/resolved the sole substantive ground-truth concern (trust_remote_code kwarg incompatible with pinned kernels<0.15.0), | +| pr5009.r1 | claudecode_opus5 | clear | Both candidates correctly identify the central ground-truth concern (the unconditional vllm_c default now applies globally to ~14 diffusion models validated only on Qwen-Image), but X backs it with re | +| pr5009.r2 | claudecode_opus5 | clear | The one concern still live at this diff's head is the unvalidated global default change affecting ~14 non-Qwen diffusion models — Y nails this with near-exact model list and even proposes the same fix | +| pr5009.r3 | claudecode_opus5 | clear | Y nails the ground truth's central, still-live concern almost verbatim: it names nearly the same affected-model list (flux, flux2, sd3, hunyuan_video_15, ltx2, omnigen2, wan2_2_s2v, ovis, longcat, ern | diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r1.json new file mode 100644 index 000000000..23f029cef --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.32, + "precision": 0.78, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.52, + "precision": 0.8, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates miss linyueqian's cumulative/delta re-slice bug and the audio_tokenizer_v2.py:949 ceil/floor padding bug, but Y covers both of the two High-severity ground-truth findings substantively (the stream-slot leak, explained with a deeper id-space mechanism, and the legacy-checkpoint-parses-as-v2 codec issue matching hsliuustc's report almost exactly), while X only surfaces the v2-codec issue as a buried 'minor' aside and frames the slot leak as mostly already resolved. Several findings converge independently between X and Y (max_new_frames TypeError at talker.py:1462, WER PCM fallback quality bug, NPU seq_token_counts gap), which is strong evidence both are grounded rather than fabricated, keeping precision comparable; X edges out slightly on directly diff-verifiable findings (the unconditional tts_local_seed write) and literal suggestion blocks. Both give concrete file/line findings with proposed fixes, but Y's broader and better-prioritized coverage of the real high-severity issues gives it the edge.", + "_cost_usd": 0.47207459999999996, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "05687f47874bfd8f79375846b8e0f01fa55ed0ec4b1489dfd7465a539d9e8e91", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "18986f2736f812d8c896b12152ebcbf1d5319e9e1f79042f5fa16f1dd6272542", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r2.json new file mode 100644 index 000000000..0e36fbb42 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.55, + "precision": 0.68, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.45, + "precision": 0.75, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Both candidates independently rediscover the two flagship ground-truth issues (slot lifecycle on abort/exhaustion, and v2-tokenizer-vs-legacy-checkpoint model_type collision) plus the WER PCM-fallback note, but X additionally flags the stale docstrings item from linyueqian's non-blocking notes, giving it slightly broader recall. Y compensates with a more rigorous verification methodology (explicit resolved/refuted claim-checking against the live tree), which likely yields a somewhat higher precision-to-claim ratio, but its narrower finding set and downgrading of the legacy-checkpoint issue to 'minor' (vs. ground truth's 'High') cost it recall. Both misses the two medium-severity confirmed bugs (cumulative/delta chunk_np re-slice, ceil-division length bug) and neither includes a labeled LATENT GAP CHECK section. Actionability is comparably strong on both sides with concrete file:line citations and suggested fixes.", + "_cost_usd": 0.3882611, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "05687f47874bfd8f79375846b8e0f01fa55ed0ec4b1489dfd7465a539d9e8e91", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "18986f2736f812d8c896b12152ebcbf1d5319e9e1f79042f5fa16f1dd6272542", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r3.json new file mode 100644 index 000000000..8be27cac0 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4804.r3.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.5, + "precision": 0.68, + "actionability": 0.92 + }, + "y": { + "recall": 0.42, + "precision": 0.78, + "actionability": 0.87 + }, + "winner": "X", + "margin": "slight", + "rationale": "X surfaces more of the ground truth's substantive open concerns, notably the reviewer-flagged [High] v2-tokenizer-misdetection bug (hsliuustc0106) as a top-billed major finding, plus the WER-fallback bogus-data issue and partial docstring-staleness coverage, giving it stronger recall despite some speculative claims (e.g. Realtime yaml slot mismatch, legacy-codec streaming API) that aren't verifiable from the shown diff and cost it some precision. Y is tighter and more rigorously self-validated — it correctly identifies the abort slot-leak as already fixed on GPU and pinpoints a precise residual NPU gap, and catches a diff-verifiable finding (the qwen3_tts gate removal on tts_local_seed) — earning higher precision, but it buries the High-severity tokenizer bug as a minor 'additional observation' and misses several other line-level concerns, softening its recall. Both are highly actionable with concrete file/line evidence and proposed fixes.", + "_cost_usd": 0.380124, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "05687f47874bfd8f79375846b8e0f01fa55ed0ec4b1489dfd7465a539d9e8e91", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "18986f2736f812d8c896b12152ebcbf1d5319e9e1f79042f5fa16f1dd6272542", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r1.json new file mode 100644 index 000000000..22e28157f --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.85, + "actionability": 0.95, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.75, + "actionability": 0.7, + "gap_hit": false + }, + "winner": "X", + "margin": "clear", + "rationale": "Ground truth has no substantive concerns, so recall is vacuous for both. Both catch the key CI-blind-spot (missing pytestmark deselects the new test), but X verifies it with an actual local pytest run showing '18 deselected'/'18 passed' and backs every claim with concrete patches (warmup_quack_fp8 gate bypass, sticky-disable fix, warning_once dedup bug), giving it sharper actionability. Y's findings are grounded but more repetitive/formulaic, and pads with an out-of-scope pre-existing multi-GPU note duplicated across two entries, diluting precision and usefulness.", + "_cost_usd": 0.1799461, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "5b735b7d69d020190d86617adfc062545c2c3b7cd1b9ca14a80d1c714e8448cf", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "de884ebeca54520b85a490b897397bd22bf362507bf9f7d4a833d74b390ae063", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r2.json new file mode 100644 index 000000000..76c9ca436 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.8, + "actionability": 0.92, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.85, + "actionability": 0.82, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Ground truth has no substantive reviewer concerns, so recall is trivially satisfied by both. Both independently catch the same critical finding (new test lacks pytestmark and is silently deselected by the CI marker filter) with strong, verifiable evidence. X goes further and surfaces a genuinely important, well-argued blocking bug — warmup_quack_fp8() checks only _load_quack() and bypasses quack_enabled(), so it can still fire on gated-off sm_120 hardware — with a concrete fix, whereas Y only mentions this as a buried 'residual' note in a catch-all finding rather than flagging it clearly. Y is slightly more rigorous about citing exact evidence quotes and correctly separating pre-existing vs. PR-introduced issues, while X's CI-status claim (specific red Buildkite checks) is less grounded in the diff itself, so precision is roughly comparable with a slight edge to Y; X's clearer, code-backed fixes give it a slight actionability edge overall.", + "_cost_usd": 0.10631750000000002, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "5b735b7d69d020190d86617adfc062545c2c3b7cd1b9ca14a80d1c714e8448cf", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "de884ebeca54520b85a490b897397bd22bf362507bf9f7d4a833d74b390ae063", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r3.json new file mode 100644 index 000000000..d4259cb9f --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4817.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.55, + "actionability": 0.6, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.85, + "actionability": 0.9, + "gap_hit": false + }, + "winner": "Y", + "margin": "decisive", + "rationale": "Ground truth carries no substantive concerns (bot spam, 'thanks', no inline comments), so recall is vacuous for both. Both candidates independently and correctly flag the real CI gap (new test file lacks pytestmark, so it's deselected by the core_model/cpu marker filter), but Y verifies it with an actual pytest run and X buries it among several redundant/low-value 'findings' (restating the PR's own fix as a finding, pre-existing multi-GPU heterogeneity noted twice) that dilute precision and make the audit-log-style output ('[claim-verified]', 'general') harder to act on. Y additionally surfaces a genuine blocking bug X only gestures at vaguely (warmup_quack_fp8 bypassing the gate entirely), a real logging/dedup defect, and a design fix for the catastrophic per-call retry amplifier, each with concrete patches and calibrated confidence (explicitly flagging its own unverified claims like the (10,3) test case).", + "_cost_usd": 0.2870977, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "5b735b7d69d020190d86617adfc062545c2c3b7cd1b9ca14a80d1c714e8448cf", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "de884ebeca54520b85a490b897397bd22bf362507bf9f7d4a833d74b390ae063", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r1.json new file mode 100644 index 000000000..94377cc48 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.65, + "precision": 0.55, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.55, + "precision": 0.85, + "actionability": 0.75, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "X has broader nominal coverage and crisper remediation writing, but its 'High' finding on audio_vae.py:141 fabricates a forward-pass regression ('verified locally... max abs diff 0.083') that directly contradicts ground truth: both amy-why-3459 and LHXuuu confirm the encoder's actual layer count is unchanged since it's constructed before the config mutation — a serious grounding failure for a top-billed finding. Y avoids that fabrication and its other findings (double sample-rate conversion, dialect/doc mismatch, env-var layering, patch_emission +2 semantics) are accurate and well-grounded, though it misses the audio_vae aggregator topic entirely and its suggested revert of the dialect removal is itself wrong per ground truth (LHXuuu confirms that removal was intentional). Neither includes a labeled LATENT GAP CHECK section.", + "_cost_usd": 0.5719895999999999, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "5ecfd2073e66a06e84102081f728cde910646f5a782a5d0fa1678b5597b2f837", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "264e453d996b4a5c483bf206cefb21e513e90d0d6bda5e49948434353154f4a5", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r2.json new file mode 100644 index 000000000..3fd743c1d --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.8, + "precision": 0.6, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.65, + "precision": 0.85, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "Both independently surface the same high-value bug the human reviewers missed (env-driven sample rate mis-decoding already-normalized 24kHz PCM in seed_tts_eval), and both hit the language/dialect-drop and definitions.py-placement threads from the real review. X has broader topical recall (also flags the audio_vae config-mutation concern that matches amy-why-3459's GT comment), but backs that finding with a fabricated-looking 'verified locally' transformers execution trace claiming the encoder is truncated — directly contradicted by LHXuuu's actual PR clarification that the encoder is unaffected, which is a serious grounding failure for a 'High' severity claim. Y omits that topic entirely (lower recall) but is more disciplined throughout, tagging claims as verified/refuted against cited code and avoiding unverifiable execution claims, giving it cleaner precision. Actionability is comparable (both give file:line-specific fixes; Y adds literal suggestion diffs).", + "_cost_usd": 0.41808960000000006, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "5ecfd2073e66a06e84102081f728cde910646f5a782a5d0fa1678b5597b2f837", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "264e453d996b4a5c483bf206cefb21e513e90d0d6bda5e49948434353154f4a5", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r3.json new file mode 100644 index 000000000..58f0fdea9 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4859.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.45, + "precision": 0.82, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.75, + "precision": 0.65, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "Y covers more ground-truth concerns than X (audio_vae.py config-mutation risk and dropped env-var documentation are entirely absent from X but substantively addressed by Y), giving Y meaningfully higher recall. However Y's headline audio_vae finding fabricates a specific 'verified locally' test result claiming the encoder is truncated, which is directly contradicted by the actual PR maintainer's explanation (encoder layer count is unchanged, matches vendor source) — a real precision hit, compounded by likely-invented PR-history citations (#2906/#4341). X's smaller finding set is more consistently grounded in the diff with concrete suggestion blocks. Both are highly actionable with specific file/line citations and concrete fixes, so the tiebreak favors Y's superior coverage despite its fabrication flaw.", + "_cost_usd": 0.4782999, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "5ecfd2073e66a06e84102081f728cde910646f5a782a5d0fa1678b5597b2f837", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "264e453d996b4a5c483bf206cefb21e513e90d0d6bda5e49948434353154f4a5", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r1.json new file mode 100644 index 000000000..6bd5ebf69 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.35, + "precision": 0.75, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.55, + "precision": 0.82, + "actionability": 0.78, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "Both correctly recognize the diff already bakes in the qwen3_tts scoping fix and the seq_len removal, and both independently catch the async_chunk getattr default inconsistency (True vs False) that echoes the ground-truth [Low] comment, plus the identical unfixed NPU bug. Y additionally reconstructs the spirit of the ground-truth [Med] blast-radius concern by naming the specific untested async_chunk=false models (moss_tts_nano, higgs_audio_v2, indextts2) still on DELTA coercion, and flags a real merge-gate E2E coverage gap with a concrete suggested patch — direct recall hits X doesn't match. X compensates with deeper, well-grounded original analysis (cudagraph padding root cause, list-vs-tensor consumer mismatch, WebSocket realtime path) that is highly actionable and precise but mostly orthogonal to what the human reviewers actually raised, so it helps precision/actionability more than recall. Neither candidate raises the multi-axis batch-splitting concern from the latent gap check.", + "_cost_usd": 0.2795591, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "604e00e0e0d031df52599006d904f2f4716e04cd3044d847250b5b62c11f189e", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "03d7423c3a91e45838083e2025e3dea932f1d9448ba923b1d6ca1a0373899ea1", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r2.json new file mode 100644 index 000000000..5324279cd --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.6, + "precision": 0.85, + "actionability": 0.9, + "gap_hit": false + }, + "y": { + "recall": 0.4, + "precision": 0.7, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "X", + "margin": "clear", + "rationale": "X explicitly cross-references the actual review thread's resolution status (confirming the Med gating-scope and Low fallback-robustness concerns were addressed) while surfacing a well-grounded new residual bug (getattr default True vs config default False) plus concrete, evidence-trailed test/CI coverage gaps with suggested patches. Y produces deeper but more speculative analysis (a padding root-cause hypothesis and an unverifiable 'AMD CI is red' claim not grounded in the diff) and never engages with the literal reviewer concerns, so it covers less of the actual ground truth despite comparable actionability. Neither candidate raises the multi-batch-axis splitting concern from the latent gap check.", + "_cost_usd": 0.4290276, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "604e00e0e0d031df52599006d904f2f4716e04cd3044d847250b5b62c11f189e", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "03d7423c3a91e45838083e2025e3dea932f1d9448ba923b1d6ca1a0373899ea1", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r3.json new file mode 100644 index 000000000..f453302d1 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4870.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.45, + "precision": 0.85, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.4, + "precision": 0.65, + "actionability": 0.8, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Both candidates independently recall the ground-truth Low (async_chunk fallback robustness) concern and validate the already-resolved Med scoping fix, but neither surfaces amy-why-3459's model-placement question or the multi-axis latent gap. X is exceptionally disciplined, backing every claim with explicit diff-changed vs. unchanged-tree evidence citations, yielding high trust and concrete, ready-to-apply suggestions (NPU port gap, CI merge-gate coverage gap, default-value inconsistency). Y is deeper and more insightful (list-audio consumer disagreement, WebSocket realtime path) but mixes in speculative or unverifiable claims (a confident 'AMD CI is red' assertion and an unsourced '0.228-cosine mystery' reference) with no evidence trail, which drags down its precision relative to X's rigor.", + "_cost_usd": 0.39816860000000004, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "604e00e0e0d031df52599006d904f2f4716e04cd3044d847250b5b62c11f189e", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "03d7423c3a91e45838083e2025e3dea932f1d9448ba923b1d6ca1a0373899ea1", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r1.json new file mode 100644 index 000000000..cfa796a1a --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.3, + "precision": 0.55, + "actionability": 0.75, + "gap_hit": false + }, + "y": { + "recall": 0.55, + "precision": 0.65, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates are technically detailed and well-evidenced, but Y engages more directly and more deeply with the PR's actual central controversy — the cudagraph_mode/talker_mtp_accepts_per_row_generators design that GT reviewers (gcanlin, R2-Y) flagged, going further by showing the seed-reproducibility fix is silently broken even at batch size 1 (CUDAGraphWrapper replay behavior) and that talker_mtp_graph_safe=True is unsound on NPU — effectively root-causing the Wallbreazzz NPU crash thread rather than just noting the yaml patch exists. X covers adjacent ground (duplication of the runner's wrap decision, seed-limitation documentation, NPU yaml lacking rationale) but stays at a more surface/consolidation level and spends much of its budget on test-coverage and doc-staleness findings not present in GT. Both are concrete and actionable with file/line citations and suggested diffs, giving Y a moderate edge on recall/precision and a slight edge on actionability.", + "_cost_usd": 0.3359803, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "eb58658a2a954e44f9ac3eb6b558b56bcc2e87a522facf8e32c7f106ac5c88c9", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "9da2d20c4c6f3ebd76201601229ebe30caa5b10534567d559eab79d6e55fdaba", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r2.json new file mode 100644 index 000000000..1821f1064 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.6, + "precision": 0.75, + "actionability": 0.9, + "gap_hit": false + }, + "y": { + "recall": 0.55, + "precision": 0.85, + "actionability": 0.75, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Both reviews independently rediscover the two GT threads (model reading cudagraph_mode should live upstream; per-row seed breaks under full cudagraphs) plus the NPU compilation_config gap, and both are well grounded with specific file:line citations. X edges ahead because its fix for the seed issue literally implements the follow-up refactor gcanlin asked for (move the gating into the runner, drop the model's cudagraph_mode read), and nearly every finding ships a concrete code diff, whereas Y is more conservative/precise but leans on 'consider documenting/adding a test' language more often. X's boldest claim (seed is discarded even at batch=1 due to CUDAGraphWrapper replay semantics) is plausible but unverified and slightly overreaches beyond what the human reviewer stated, which is the main drag on its precision relative to Y's more tightly evidenced claims. Y's unusual process scaffolding (Run report/Scan table/Validated bullets) adds bulk without adding findings, hurting its actionability density.", + "_cost_usd": 0.3445111, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "eb58658a2a954e44f9ac3eb6b558b56bcc2e87a522facf8e32c7f106ac5c88c9", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "9da2d20c4c6f3ebd76201601229ebe30caa5b10534567d559eab79d6e55fdaba", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r3.json new file mode 100644 index 000000000..39f21a4bb --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4923.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.35, + "precision": 0.75, + "actionability": 0.75, + "gap_hit": false + }, + "y": { + "recall": 0.62, + "precision": 0.85, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates ground findings in real code paths, but Y lands much closer to the actual reviewer thread: it independently reconstructs gcanlin's architectural objection to reading cudagraph_mode in modeling (comment 1, even proposing the exact runner-side fix that would let the TODO be deleted), precisely nails the seed-reproducibility regression gcanlin warned about (and sharpens it by showing CUDAGraphWrapper replay drops seeds at every batch size, not just >1), and connects talker_mtp_graph_safe=True being false on NPU to the real Wallbreazzz crash plus the failing NPU/AMD CI gates. X's findings (prefix-cache/async interaction, docs staleness, missing unit tests) are plausible but mostly tangential to what reviewers actually flagged, and it restates the same async/prefix-cache point twice, diluting precision. Y's comments also carry more concrete, root-cause-targeted code suggestions rather than generic 'add a test/doc' asks.", + "_cost_usd": 0.1927112, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "eb58658a2a954e44f9ac3eb6b558b56bcc2e87a522facf8e32c7f106ac5c88c9", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "9da2d20c4c6f3ebd76201601229ebe30caa5b10534567d559eab79d6e55fdaba", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r1.json new file mode 100644 index 000000000..07973ffe8 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r1.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.45, + "precision": 0.5, + "actionability": 0.75 + }, + "y": { + "recall": 0.55, + "precision": 0.7, + "actionability": 0.8 + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both cover the core live issues (Hub kernel version-pinning risk, FA3 Hopper/Blackwell gating nuance, per-layer kernel reload cost, thin test coverage), but Y grounds every claim in explicit file:line quotes tagged 'unchanged by this diff, present in PR-time tree' and even cross-references the actual fix commits (332da8f2, deeb67af) that match the ground-truth reviewer thread almost exactly, plus independently surfaces the CI-collection/marker concern that mirrors wtomin's real inline comment. X reads as more polished but repeatedly presents suspiciously precise 'evidence' (exact Hub commit SHAs, measured warm/cold load timings, 'simulated' capability sweeps) that could not plausibly be produced from a static diff review, which is a real fabrication risk and drags down precision despite the underlying architectural points being reasonable. Neither review includes a LATENT GAP CHECK section.", + "_cost_usd": 0.3618831, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "d1dbf8c3832158320545e603632c2cd18f949fddb250f2074402a1ca10ddb6df", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "0639263265995e00edcd06c6242ac0a6a3647a980509c282af885c86ce6930e0", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r2.json new file mode 100644 index 000000000..701b65157 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r2.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.45, + "precision": 0.6, + "actionability": 0.85 + }, + "y": { + "recall": 0.55, + "precision": 0.7, + "actionability": 0.75 + }, + "winner": "Y", + "margin": "slight", + "rationale": "Both hit the two big confirmed bugs (missing SM90+ gate on FLASH_ATTN_3_HUB, and the varlen/func mismatch that piecewise_attn can crash on), and both flag the duplicated-code smell, the degenerate q=k=v test tensors, and the overstated 'perfect numerical consistency' doc claim. Y's coverage is broader and more precisely grounded — it directly ties findings to the exact GT-confirmed fixes (citing the real follow-up commits and quoting SamitHuang's actual fix logic), catches a real gap that the platform only guards ImportError of `kernels` but not failures inside `get_kernel()` itself (contradicting the docs' 'falls back gracefully' claim), and surfaces two plausible downstream-consumer misses (ltx2_transformer, diffusers_adapter) that X doesn't mention. X is more readable and its suggestions are cleanly actionable, but it pads several findings with fabricated-looking 'verified' evidence (invented timing benchmarks, HF Hub query output, live CI-lane status) that reads as hallucinated theater rather than diff-grounded analysis, which hurts trust in an otherwise solid set of findings. Neither includes a literal 'LATENT GAP CHECK' section.", + "_cost_usd": 0.25007019999999996, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "d1dbf8c3832158320545e603632c2cd18f949fddb250f2074402a1ca10ddb6df", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "0639263265995e00edcd06c6242ac0a6a3647a980509c282af885c86ce6930e0", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r3.json new file mode 100644 index 000000000..28b68a55f --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4926.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.3, + "precision": 0.85, + "actionability": 0.75, + "gap_hit": false + }, + "y": { + "recall": 0.4, + "precision": 0.9, + "actionability": 0.9, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates go well beyond the ground truth's fairly light docs/test-marker complaints into real code-level bugs, and both independently converge on several genuine issues (uncached get_kernel() per attention layer, the unwrapped final get_kernel() call with no fallback, FA3-Hub's major>=9 gate wrongly admitting Blackwell) — this convergence lends credibility to both. Y more precisely recalls the ground-truth threads: it directly answers wtomin's 'what do version=1/2/default mean' question with concrete evidence (branch SHAs, actual warning text) rather than just proposing a pin, and it correctly identifies that RuixiangMa's flash_attn_func=None crash concern is only partially fixed (the piecewise path was patched but _forward_varlen_masked/_run_varlen_dense still call an unchecked varlen_func), whereas X marks that whole area '[resolved]' without noticing the masked-path gap still exists. Y's findings are also generally more verifiable/grounded (shows actual timing runs, simulated selector matrix, real repo version metadata) and read as a coherent, well-prioritized human review, while X's is comprehensive but its finding headlines are truncated/garbled ('[…]') and wrapped in denser, harder-to-parse scaffolding. X does add a few plausible unique findings (LTX-2 mask conversion, diffusers-adapter case fallthrough) that Y lacks, keeping the gap from being decisive.", + "_cost_usd": 0.4982763, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "d1dbf8c3832158320545e603632c2cd18f949fddb250f2074402a1ca10ddb6df", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "0639263265995e00edcd06c6242ac0a6a3647a980509c282af885c86ce6930e0", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r1.json new file mode 100644 index 000000000..b2ce97eca --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.75, + "actionability": 0.6, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.9, + "actionability": 0.9, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Ground truth has no substantive concerns (just LGTM approvals), so recall is trivially satisfied by both. X offers only two adjacent-file/process findings, one of which (README gap) is arguably undercut by the fact that file's example is SDK-based, not curl-based. Y digs much deeper into the diff's own new prose, catching a concrete inaccuracy ('silent audio' vs. the traced 500-error path) and an overbroad claim about extra_body that contradicts another recipe (GLM-Image), both with precise file:line evidence and exact suggested rewrites plus a verification command. Y's findings are more numerous, better grounded in the actual diff content, and more actionable, making it the stronger review despite being longer.", + "_cost_usd": 0.29245930000000003, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "24b99e8934db6b4f15191280cd7fef3e1ddd05553cde67db583199bd019e915b", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "5a2e689fc33efd67adbf119f1e4cf034841928ad70fea2d99f142a727ca8ea18", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r2.json new file mode 100644 index 000000000..2bb9f62c2 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.7, + "actionability": 0.9, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.75, + "actionability": 0.8, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Ground truth is a rubber-stamp approval with zero concerns, so recall is trivially satisfied by both. X does deeper verification (traces the actual talker/serving_chat code path to dispute the PR's own 'silent audio' wording with a specific, well-cited failure mode, and confirms the sibling README needs no fix by checking gradio_demo.py actually uses the SDK), yielding higher-value, well-grounded findings with concrete line refs and rewrites. Y is more conservative and equally well-cited but its main finding (README curl risk) is speculative and arguably contradicted by X's deeper check of the same file; both provide concrete, actionable suggestions and a reasonable CI-nit, but X's extra rigor edges it out despite the added length.", + "_cost_usd": 0.2610264, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "24b99e8934db6b4f15191280cd7fef3e1ddd05553cde67db583199bd019e915b", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "5a2e689fc33efd67adbf119f1e4cf034841928ad70fea2d99f142a727ca8ea18", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r3.json new file mode 100644 index 000000000..5ac7065a9 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4950.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.9, + "actionability": 0.95, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.75, + "actionability": 0.8, + "gap_hit": false + }, + "winner": "X", + "margin": "clear", + "rationale": "Ground truth has no substantive concerns (just LGTM/approve), so recall is trivially satisfied by both. X goes much deeper, tracing the actual failure mode of missing use_tts_template through three files to show the PR's own new text ('returns silent audio') is likely wrong (it's actually a 500 via a pydantic validation bug), a finding Y misses entirely and even implicitly endorses by marking the silent-audio claim 'claim-verified.' X also scopes the overbroad extra_body claim against a contradicting recipe (GLM-Image), adds a calibrated 'checked, no change needed' section, and gives concrete rewordings/suggestions throughout. Y is competent and grounded (README inconsistency, CI-unrelated note) but shallower, with only two findings and one loosely-supported aside ('Summary still says one shot') not visible in the given diff.", + "_cost_usd": 0.1859645, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "24b99e8934db6b4f15191280cd7fef3e1ddd05553cde67db583199bd019e915b", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "5a2e689fc33efd67adbf119f1e4cf034841928ad70fea2d99f142a727ca8ea18", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r1.json new file mode 100644 index 000000000..6143e0a03 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.55, + "precision": 0.85, + "actionability": 0.5, + "gap_hit": false + }, + "y": { + "recall": 0.05, + "precision": 0.35, + "actionability": 0.75, + "gap_hit": false + }, + "winner": "X", + "margin": "clear", + "rationale": "Ground truth is thin (essentially 'LGTM' plus a request to split out the VoxCPM2 regression fix into a separate PR) for a trivial 2-line seed removal. X stays appropriately scoped, grounds every claim in quoted diff/file evidence, and directly confirms the one real reviewer concern (VoxCPM2 tracked separately, out of scope). Y produces a much longer, more actionable-looking review with concrete patches, but it hinges on an enormous number of unverifiable hyper-specific claims (exact line numbers across unrelated vLLM internals, a fabricated 3-PR seed history, doc line citations) for a PR the actual reviewers rubber-stamped, and never engages the VoxCPM2 scoping concern at all — a major groundedness and recall risk despite its polished format.", + "_cost_usd": 0.2397207, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "39d8f83c76264f8f4b1d3eff4b8cc1b57c2258fa18392d4f278ded4fc414d472", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "410634f22ac0d9a55948ec0dfc9caa6557a8995e24766ff507dbe677d6dd37dc", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r2.json new file mode 100644 index 000000000..d02bf4cd2 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.8, + "precision": 0.85, + "actionability": 0.65, + "gap_hit": false + }, + "y": { + "recall": 0.15, + "precision": 0.35, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "X", + "margin": "clear", + "rationale": "The only substantive ground-truth concern is Gaohan123's ask to split out the VoxCPM2 regression fix into a separate PR; X explicitly addresses this ('VoxCPM2 #4963 remainder explicitly out of scope — PR description tracks separate bisect') while Y never mentions VoxCPM2 at all. X's other finding (CI gate question) is modest, well-scoped, and grounded with explicit file/line citations showing actual verification against the PR-time tree. Y is an impressively written but sprawling five-point review whose claims (exact internal line numbers across unrelated sampler files, specific docs content, precise throughput percentages, PR-history narratives) go far beyond what's verifiable from this trivial two-line diff and read as likely-fabricated or scope-creeping padding, which hurts precision much more than its high surface-level actionability (ready-to-paste code/test snippets) can offset.", + "_cost_usd": 0.11671860000000002, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "39d8f83c76264f8f4b1d3eff4b8cc1b57c2258fa18392d4f278ded4fc414d472", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "410634f22ac0d9a55948ec0dfc9caa6557a8995e24766ff507dbe677d6dd37dc", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r3.json new file mode 100644 index 000000000..9d5163749 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4970.r3.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.0, + "precision": 0.5, + "actionability": 0.85 + }, + "y": { + "recall": 0.55, + "precision": 0.7, + "actionability": 0.5 + }, + "winner": "Y", + "margin": "slight", + "rationale": "Ground truth is nearly empty (LGTM plus a tangential ask to split out a VoxCPM2 regression fix); Y explicitly validates that VoxCPM2/#4963 is out of scope and tracked separately, directly matching the one real reviewer concern, while X never mentions VoxCPM2 at all. Y's claims are also more visibly grounded (explicit [validated] tags with quoted PR-time-tree evidence) versus X's much larger volume of highly specific but unverifiable technical claims (exact FlashInfer/aiter line ranges, docs line numbers, invented follow-up items) that read as scope creep for a 2-line YAML diff the actual reviewers rubber-stamped. X is far more actionable with concrete code/comment suggestions, but that doesn't offset its miss on the only real ground-truth signal and its riskier precision.", + "_cost_usd": 0.248401, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "39d8f83c76264f8f4b1d3eff4b8cc1b57c2258fa18392d4f278ded4fc414d472", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "410634f22ac0d9a55948ec0dfc9caa6557a8995e24766ff507dbe677d6dd37dc", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r1.json new file mode 100644 index 000000000..f301880db --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r1.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.85, + "precision": 0.78, + "actionability": 0.85 + }, + "y": { + "recall": 0.85, + "precision": 0.85, + "actionability": 0.8 + }, + "winner": "Y", + "margin": "slight", + "rationale": "Both candidates independently surface the same underlying fact the ground truth's sole inline concern (trust_remote_code/kernels-version fragility) was already resolved by a later commit, and both flag the same secondary issues (stale PR body, uncovered test, possibly-unrelated red NPU CI check), so recall is essentially tied and full. X's strongest original catch is a genuine, diff-native correctness bug (a module missing required symbols gets cached before validation, so a bad load is 'sticky' and skips retries) and ships two ready-to-merge suggestion diffs, giving it a slight actionability edge. Y's findings are marginally more precise and cohesive: it demonstrates actual verification (stubbed kernels module, checked against installed kernels 0.16.0) and its dead-code/version-pin-mismatch finding directly deepens the exact 'kernels version compatibility' theme the human reviewer raised, whereas X's coverage of that theme mostly just reports it as already fixed. Neither has a LATENT GAP CHECK section.", + "_cost_usd": 0.2386651, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "bafccb74b5b06fb396b4f22cbb89466da222e65825b9f072c4cea6162dc21fe3", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "589a8ff13cb3893b76dd0d5220ad632d82f46cc0edd9083528ebb2466be7c8a7", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r2.json new file mode 100644 index 000000000..29ce893f4 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.9, + "precision": 0.88, + "actionability": 0.87, + "gap_hit": false + }, + "y": { + "recall": 0.9, + "precision": 0.82, + "actionability": 0.87, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Both candidates correctly identify and resolve the single ground-truth concern (stale trust_remote_code claim, fixed at 9947f414) and add substantive, diff-grounded findings with file:line citations well beyond the ground truth's sparse output. Y catches one genuinely unique, valuable bug X misses (cache poisoning when a hub module lacks required symbols, with a ready-to-apply fix), but pads the review by repeating the trust_remote_code point four separate times across sections and has visibly truncated/garbled headers ('[…]'), diluting signal density. X is more concise per point, covers comparable ground (missing test coverage, sticky-failure caching, stale docs, CI gate) plus a distinctive kernels-version dead-code catch, without redundancy. Actionability is comparable — Y offers two full code suggestion blocks, X offers precise prose fixes — so the tiebreak favors X's tighter, less repetitive signal.", + "_cost_usd": 0.2401201, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "bafccb74b5b06fb396b4f22cbb89466da222e65825b9f072c4cea6162dc21fe3", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "589a8ff13cb3893b76dd0d5220ad632d82f46cc0edd9083528ebb2466be7c8a7", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r3.json new file mode 100644 index 000000000..335c683d0 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr4977.r3.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.82, + "actionability": 0.9 + }, + "y": { + "recall": 1.0, + "precision": 0.72, + "actionability": 0.85 + }, + "winner": "X", + "margin": "slight", + "rationale": "Both candidates independently dug past the truncated diff and correctly identified/resolved the sole substantive ground-truth concern (trust_remote_code kwarg incompatible with pinned kernels<0.15.0), confirming it was fixed by a later commit — recall is equal and high. X's remaining findings (cache-before-validation bug, missing unit test, CI coverage gaps) are conservatively labeled with verification status and grounded in cited PR-tree state, while Y makes several bold but unverifiable claims presented as fact ('verified against the installed kernels 0.16.0' library internals, 'merged with this check failing' for NPU CI) that read as risky assertions rather than diff-grounded findings, pulling precision down. Both reviews are concrete and actionable with specific file:line citations and suggested fixes, giving X a slight overall edge.", + "_cost_usd": 0.13753030000000002, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "bafccb74b5b06fb396b4f22cbb89466da222e65825b9f072c4cea6162dc21fe3", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "589a8ff13cb3893b76dd0d5220ad632d82f46cc0edd9083528ebb2466be7c8a7", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r1.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r1.json new file mode 100644 index 000000000..71c2692a6 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.55, + "precision": 0.85, + "actionability": 0.9, + "gap_hit": false + }, + "y": { + "recall": 0.45, + "precision": 0.6, + "actionability": 0.65, + "gap_hit": false + }, + "winner": "X", + "margin": "clear", + "rationale": "Both candidates correctly identify the central ground-truth concern (the unconditional vllm_c default now applies globally to ~14 diffusion models validated only on Qwen-Image), but X backs it with real verification (ran the new tests, checked numerics/strides, traced the sole caller chain) and layers on genuinely new grounded findings (stale RoCm logic, missing Part-2 regression test, Part-1-becomes-a-no-op-for-Qwen nuance) with concrete suggested diffs. Y restates the same scope point roughly five times across 'Validated' bullets and separate 'general' findings, and several of its entries are purely confirmatory (Cosmos3 override still works) rather than actionable asks, diluting its practical value despite good citation discipline.", + "_cost_usd": 0.36509159999999996, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_train" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "479440490faf69f773aade9ad0338400860989209a85201d20696bbef7795244", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "97e45909dc2469db55da15b6e8d6a525549029c219aefca76123ec8b801fd1bf", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r2.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r2.json new file mode 100644 index 000000000..b8dec6e9c --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r2.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.32, + "precision": 0.7, + "actionability": 0.6 + }, + "y": { + "recall": 0.68, + "precision": 0.78, + "actionability": 0.82 + }, + "winner": "Y", + "margin": "clear", + "rationale": "The one concern still live at this diff's head is the unvalidated global default change affecting ~14 non-Qwen diffusion models — Y nails this with near-exact model list and even proposes the same fix mechanism (_DIFFUSION_IR_OP_PRIORITY_FUNCS/registry) that hsliuustc0106 suggested, and separately reconstructs the exact 'future upstream re-sync silently regresses' risk that motivated the added test. X gestures at the same blast-radius theme but more weakly ('note in test plan') and pads its comment budget with three near-duplicate restatements of the same Part1/Part2 interaction plus two Cosmos3 confirmations, diluting distinct signal. Both are richly grounded with specific file:line evidence beyond the visible diff and give concrete, actionable remedies, but Y's findings are more diverse and better matched to what the human reviewers actually asked for.", + "_cost_usd": 0.4653736, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "479440490faf69f773aade9ad0338400860989209a85201d20696bbef7795244", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "97e45909dc2469db55da15b6e8d6a525549029c219aefca76123ec8b801fd1bf", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r3.json b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r3.json new file mode 100644 index 000000000..8b5daa92d --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_train_sonnet/pr5009.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.35, + "precision": 0.55, + "actionability": 0.5, + "gap_hit": false + }, + "y": { + "recall": 0.78, + "precision": 0.85, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Y nails the ground truth's central, still-live concern almost verbatim: it names nearly the same affected-model list (flux, flux2, sd3, hunyuan_video_15, ltx2, omnigen2, wan2_2_s2v, ovis, longcat, ernie) that NumberWan later posted, and proposes the exact fix hsliuustc0106 asked for (scope via `_DIFFUSION_IR_OP_PRIORITY_FUNCS`/registry.py using Cosmos3 as precedent). It also independently surfaces that Part 2 removes all vLLM RMSNorm from the Qwen path, undercutting the PR's own P1 benchmark — a sharp, diff-grounded insight — and addresses the perf-comparison and regression-test asks with concrete, actionable follow-ups. X repeats essentially the same 'Qwen bypasses IR priority / other models untested' observation four times across 'Validated' bullets and 'general' findings without sharpening it into the scoping ask, and frames the blast radius as a 'fleet benefit' rather than the unvalidated risk ground truth flagged, diluting precision and actionability. Neither includes a LATENT GAP CHECK section.", + "_cost_usd": 0.33492220000000006, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_train", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_train", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_train", + "arm_a_sha256": "479440490faf69f773aade9ad0338400860989209a85201d20696bbef7795244", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "97e45909dc2469db55da15b6e8d6a525549029c219aefca76123ec8b801fd1bf", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/JUDGE_REPORT.md b/eval/dataset/judgments/goal_v20cb_val_sonnet/JUDGE_REPORT.md new file mode 100644 index 000000000..139dfcd62 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/JUDGE_REPORT.md @@ -0,0 +1,36 @@ +# Judgment: copilot_v20cb_val vs claudecode_opus5 + +Judge: claude-sonnet-5 (blind, randomized order, 3 replicate(s) x 5 item(s) = 15 verdicts) + +## Wins + +- copilot_v20cb_val: 9 +- claudecode_opus5: 6 +- tie: 0 + +## Mean rubric scores + +| arm | actionability | gap_hit | precision | recall | +|---|---|---|---|---| +| claudecode_opus5 | 0.82 | 0.23 | 0.67 | 0.67 | +| copilot_v20cb_val | 0.78 | 0.23 | 0.76 | 0.67 | + +## Per-verdict detail + +| item.rep | winner | margin | rationale (head) | +|---|---|---|---| +| pr4810.r1 | copilot_v20cb_val | slight | Both candidates independently rediscover the exact gap the human review missed — the diffusion-transformer hunyuan_image3_transformer.py:2238 still calling the removed get_cache_scale API — and both g | +| pr4810.r2 | claudecode_opus5 | clear | Both candidates independently surface the same latent gap (hunyuan_image3_transformer.py:2238 still calling the removed get_cache_scale API, later #4891), so gap_hit is true for both. Y additionally r | +| pr4810.r3 | claudecode_opus5 | clear | Both independently surface the exact latent gap (hunyuan_image3_transformer.py:2238 still calling the removed get_cache_scale API), with specific reachability chains through the pipeline/AutoWeightsLo | +| pr4816.r1 | claudecode_opus5 | slight | Ground truth has zero substantive concerns (a blank codex-limits notice and a plain 'lgtm' approval), so recall is trivially satisfied by both. Both candidates independently converge on the same real, | +| pr4816.r2 | copilot_v20cb_val | slight | Ground truth is empty (bot rate-limit notice plus a bare 'lgtm'), so recall is vacuous for both. Both candidates independently converge on the same real findings — the residual pure-diffusion serving_ | +| pr4816.r3 | copilot_v20cb_val | slight | Ground truth is an empty/mechanical PR (bot rate-limit notice + human 'lgtm'), so recall is vacuous for both. Both candidates independently surface the same real, diff-grounded core issue (tests mock | +| pr4825.r1 | copilot_v20cb_val | clear | The real GT thread centers on dsocek questioning whether the naming/mapping fix is comprehensive (suggesting driving it from packed_modules_mapping) and tthakkal responding by removing that companion | +| pr4825.r2 | copilot_v20cb_val | slight | Y's headline finding — the reverted to_out.0 WeightsMapper normalization at ac3a2ce5 — directly reconstructs the specific change tthakkal says he removed to 'keep this PR specific to sdxl support,' an | +| pr4825.r3 | copilot_v20cb_val | clear | The ground-truth thread's core substance is dsocek questioning a narrower naming-conflict workaround and tthakkal responding 'I will remove this change and keep this PR specific to sdxl support' — Y's | +| pr4837.r1 | copilot_v20cb_val | clear | Both candidates land the one real ground-truth concern (already_submitted removal is safe because both submit_initial and submit_update reject list prompts), with comparable actionability via concrete | +| pr4837.r2 | copilot_v20cb_val | decisive | Both candidates independently converge on the same core mechanism the ground-truth inline comment makes (submit_initial and submit_update both reject list prompts, so gating the unwrap on already_subm | +| pr4837.r3 | copilot_v20cb_val | clear | Both candidates correctly reconstruct the core ground-truth insight (already_submitted was gating unwrap even though both submit_initial and submit_update reject list prompts, so the distinction was m | +| pr4893.r1 | claudecode_opus5 | clear | The ground truth is thin (mostly off-topic PR chatter plus one inline ask to extend reduce_scatter test verification, which the diff already appears to satisfy); X engages that exact test region more | +| pr4893.r2 | claudecode_opus5 | clear | Ground truth is thin (essentially one concern: the new hasattr(reduce_scatter)/device_communicator assertions don't really verify the fix). Both X and Y independently converge on this exact gap (X's p | +| pr4893.r3 | claudecode_opus5 | clear | Ground truth is thin (one inline ask to verify reduce_scatter in the test), and X engages it most directly by showing those new hasattr assertions test the mock's own injected attributes rather than t | diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r1.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r1.json new file mode 100644 index 000000000..001adcc40 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.45, + "precision": 0.75, + "actionability": 0.85, + "gap_hit": true + }, + "y": { + "recall": 0.55, + "precision": 0.8, + "actionability": 0.85, + "gap_hit": true + }, + "winner": "Y", + "margin": "slight", + "rationale": "Both candidates independently rediscover the exact gap the human review missed — the diffusion-transformer hunyuan_image3_transformer.py:2238 still calling the removed get_cache_scale API — and both give concrete, actionable fixes. Y edges ahead on recall by explicitly endorsing keeping the regression test (mirroring the ground truth's '+1 on keeping the regression test') and traces a deeper Mammoth AR inheritance path issue, while grounding every claim in quoted file:line evidence. X's review is also well-verified (ran tests locally, cites vLLM internals) but its unverifiable claim about specific CI gates being red is an ungrounded risk, and it misses the qwen2_old getattr-cleanup nuance and the explicit test-retention endorsement that Y captures.", + "_cost_usd": 0.2801225, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "87e3c276c5df2682b6b29b2405606b24368d5ace131cf1c2c50e48e5a263f9e9", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "62ede4acd697d051f739a163828672d894ec4150b7b17c941bb4b33f4b81409e", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r2.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r2.json new file mode 100644 index 000000000..01b1961d6 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.35, + "precision": 0.7, + "actionability": 0.8, + "gap_hit": true + }, + "y": { + "recall": 0.55, + "precision": 0.85, + "actionability": 0.85, + "gap_hit": true + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates independently surface the same latent gap (hunyuan_image3_transformer.py:2238 still calling the removed get_cache_scale API, later #4891), so gap_hit is true for both. Y additionally reconstructs the reasoning a careful reviewer would need — it verifies default_weight_loader's scalar handling, maybe_remap_kv_scale_name's pass-through behavior, and AutoWeightsLoader's child-scanning — which closely mirrors and substantiates the ground-truth reviewer's 'correct migration' verdict, and its critique that the delegated-loader test never actually exercises AutoWeightsLoader (only the inner loader) directly echoes the human reviewer's concern that tests validate 'called' but not 'resolves against real registered params.' X's findings (double-mapper-apply speculation, stale test list, MiMo prefix) are grounded but narrower, more duplicative (two near-identical entries on the mammoth double-apply concern), and includes a specific tracking-issue number (#4806) that may be fabricated/incorrect against the real #4891. Both are reasonably actionable with concrete file/line suggestions, but Y's review is more comprehensive and better corroborates ground truth.", + "_cost_usd": 0.2485425, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_val", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "87e3c276c5df2682b6b29b2405606b24368d5ace131cf1c2c50e48e5a263f9e9", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "62ede4acd697d051f739a163828672d894ec4150b7b17c941bb4b33f4b81409e", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r3.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r3.json new file mode 100644 index 000000000..e27ce9d0f --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4810.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.4, + "precision": 0.75, + "actionability": 0.75, + "gap_hit": true + }, + "y": { + "recall": 0.6, + "precision": 0.8, + "actionability": 0.85, + "gap_hit": true + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both independently surface the exact latent gap (hunyuan_image3_transformer.py:2238 still calling the removed get_cache_scale API), with specific reachability chains through the pipeline/AutoWeightsLoader, so gap_hit is true for both. But Y goes further on the one substantive concern the human reviewer actually raised — that the tests use fakes and don't prove real-checkpoint/mapper fidelity — hitting it from five angles (CI not wired up, hardcoded allowlist, misleading test name, hand-rolled mapper diverging from the real WeightsMapper API, unrealistic MiMo naming) plus verifying the safe-removal claims against actual vLLM source (default_weight_loader, maybe_remap_kv_scale_name) and reporting an actual local test run. X's review is comparably well-grounded on the gap finding but padded with two near-duplicate 'general' findings about the same AR double-mapper nit, and covers the test-realism theme with only one finding versus Y's five, giving it materially lower recall against the ground-truth review's core critique. Both are concrete and file/line-anchored, but Y's clearer verdict-first structure and explicit fix snippets edge out X's denser, more redundant format on actionability.", + "_cost_usd": 0.12163370000000001, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_val", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "87e3c276c5df2682b6b29b2405606b24368d5ace131cf1c2c50e48e5a263f9e9", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "62ede4acd697d051f739a163828672d894ec4150b7b17c941bb4b33f4b81409e", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r1.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r1.json new file mode 100644 index 000000000..4a4036951 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.75, + "actionability": 0.9, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.8, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "X", + "margin": "slight", + "rationale": "Ground truth has zero substantive concerns (a blank codex-limits notice and a plain 'lgtm' approval), so recall is trivially satisfied by both. Both candidates independently converge on the same real, diff-grounded issues (untested `base()` coupling in the voice-error tests, the pure-diffusion `serving_tokenization=None` residual, and unverifiable CI gate failures), which is a strong precision signal, though X frames the pre-existing diffusion-mode gap as if the PR itself falls short rather than clearly labeling it pre-existing as Y does, and X's item 4 is openly speculative. X edges ahead on actionability by providing a concrete code fix for the helper function in addition to a test, whereas Y only proposes a test and duplicates its own finding between the Validated section and the nit list.", + "_cost_usd": 0.2584382, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "ea9b83a5feab151e43dd0c58f0d51c50187342f0ac98d60187ec7e9653f753b2", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "b9fe4c00140a73cf5cc08ceded6d03eb8b8faea45f6c3b50e6f9753b9a1a1a01", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r2.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r2.json new file mode 100644 index 000000000..3c280e180 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.72, + "actionability": 0.9, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.87, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "Ground truth is empty (bot rate-limit notice plus a bare 'lgtm'), so recall is vacuous for both. Both candidates independently converge on the same real findings — the residual pure-diffusion serving_tokenization=None path and the untested real-base() error path — which cross-validates those as genuine rather than fabricated. X is more actionable (ships a concrete helper-function fix and inline test code) but pads its review with many unverifiable external claims (upstream file/line citations, Docker pin details, a speculative sibling-rename sweep) and is internally inconsistent, labeling a residual issue 'blocking-ish' while still verdicting Approve. Y stays tighter to the diff, cites evidence lines for every claim, correctly identifies the diffusion-mode issue as pre-existing/nit rather than a regression, and its COMMENT verdict is better calibrated to what is otherwise a clean mechanical rename.", + "_cost_usd": 0.18368540000000003, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "ea9b83a5feab151e43dd0c58f0d51c50187342f0ac98d60187ec7e9653f753b2", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "b9fe4c00140a73cf5cc08ceded6d03eb8b8faea45f6c3b50e6f9753b9a1a1a01", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r3.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r3.json new file mode 100644 index 000000000..81d900405 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4816.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 1.0, + "precision": 0.55, + "actionability": 0.75, + "gap_hit": false + }, + "y": { + "recall": 1.0, + "precision": 0.8, + "actionability": 0.75, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "Ground truth is an empty/mechanical PR (bot rate-limit notice + human 'lgtm'), so recall is vacuous for both. Both candidates independently surface the same real, diff-grounded core issue (tests mock `base()` so they never exercise the real `serving_tokenization` state path) and the same pre-existing diffusion-mode None gap, with concrete file/line evidence and code suggestions. X goes further with speculative Dockerfile version-pin sleuthing and a 'follow-up' about sibling renames sourced from a locally-installed vLLM 0.26.0 that it admits doesn't match the pinned 0.24.0 — padding that isn't grounded in the diff and lowers precision, and its 'blocking-ish' framing overstates severity for a pre-existing, already-approved change. Y's findings stay tighter, correctly labels the diffusion issue as pre-existing/nit, and offers a ready-to-apply GitHub suggestion block, better matching the actual low-severity outcome.", + "_cost_usd": 0.1336847, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "ea9b83a5feab151e43dd0c58f0d51c50187342f0ac98d60187ec7e9653f753b2", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "b9fe4c00140a73cf5cc08ceded6d03eb8b8faea45f6c3b50e6f9753b9a1a1a01", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r1.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r1.json new file mode 100644 index 000000000..21732b221 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.25, + "precision": 0.6, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.45, + "precision": 0.75, + "actionability": 0.75, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "The real GT thread centers on dsocek questioning whether the naming/mapping fix is comprehensive (suggesting driving it from packed_modules_mapping) and tthakkal responding by removing that companion change to keep the PR SDXL-only. Y's top finding (the reverted to_out.0 WeightsMapper at manager.py:309, tied to commit ac3a2ce5) directly reconstructs this exact history and flags it as an open, unconfirmed gap — a strong recall/precision hit. X notices the same revert but confidently declares it 'correctly reverted, good call, don't reinstate,' which reads as a dismissal of the very concern the real reviewers left unresolved, and X also asserts '_lora_components has zero users in the entire repo,' which Y directly contradicts with a concrete counter-example (pipeline_sensenova_u1.py:543) — a clear factual error hurting X's precision. X is more conventionally formatted and has slightly crisper actionable suggestions across more findings, but several (FF-layer remap, text-encoder LoRA failure) are unverifiable speculative extensions beyond PR scope, while Y's sparser findings are each backed by explicit grep verification and quoted evidence. Neither includes a LATENT GAP CHECK section.", + "_cost_usd": 0.2944165, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "01af7bfbb2b84fa2ced47b92b26a0bcf159b5908df7daca668c6df10a2a71703", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "9201be5ea408a41833fb8e6040e5eb97d26044159a083e4d1896451ac3585494", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r2.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r2.json new file mode 100644 index 000000000..e8ed463fb --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.3, + "precision": 0.55, + "actionability": 0.8, + "gap_hit": false + }, + "y": { + "recall": 0.5, + "precision": 0.7, + "actionability": 0.7, + "gap_hit": false + }, + "winner": "Y", + "margin": "slight", + "rationale": "Y's headline finding — the reverted to_out.0 WeightsMapper normalization at ac3a2ce5 — directly reconstructs the specific change tthakkal says he removed to 'keep this PR specific to sdxl support,' and Y's _dit_modules/_lora_components suggestion echoes dsocek's ask to drive component detection from existing metadata rather than a hardcoded tuple; claims are hedged and evidence-quoted rather than asserted as fact. X's parallel _dit_modules point covers similar ground but the review asserts as fact that CI lanes are 'red' with no diff-grounded basis, and speculative findings (FF-layer renames, text-encoder hard-fail) extend well beyond verifiable PR scope, weakening precision despite strong code-level actionability.", + "_cost_usd": 0.1143382, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "01af7bfbb2b84fa2ced47b92b26a0bcf159b5908df7daca668c6df10a2a71703", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "9201be5ea408a41833fb8e6040e5eb97d26044159a083e4d1896451ac3585494", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r3.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r3.json new file mode 100644 index 000000000..d4e74ad0a --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4825.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.3, + "precision": 0.55, + "actionability": 0.8, + "gap_hit": false + }, + "y": { + "recall": 0.5, + "precision": 0.75, + "actionability": 0.7, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "The ground-truth thread's core substance is dsocek questioning a narrower naming-conflict workaround and tthakkal responding 'I will remove this change and keep this PR specific to sdxl support' — Y's top finding independently reconstructs exactly this (citing commit ac3a2ce5, the reverted WeightsMapper for to_out.0, and the deleted test), a precise recall hit with hedged, well-evidenced framing. X notices the same revert but asserts it was 'correctly reverted — good call, don't reinstate it,' which mischaracterizes a scope decision as a settled technical non-issue and anchors several other deep, unverifiable extrapolations (FF-layer weight drop, text-encoder hard-fail, CI-red claims) that the actual approving reviewers never raised, softening X's precision despite strong actionable code suggestions. Both lack a LATENT GAP CHECK section.", + "_cost_usd": 0.1553932, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "01af7bfbb2b84fa2ced47b92b26a0bcf159b5908df7daca668c6df10a2a71703", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "9201be5ea408a41833fb8e6040e5eb97d26044159a083e4d1896451ac3585494", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r1.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r1.json new file mode 100644 index 000000000..403c98326 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r1.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.85, + "precision": 0.55, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.85, + "precision": 0.78, + "actionability": 0.85, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates land the one real ground-truth concern (already_submitted removal is safe because both submit_initial and submit_update reject list prompts), with comparable actionability via concrete line refs and suggested fixes/tests. X's centerpiece 'blocker' — that super().forward() no longer binds because an unrelated prior PR (#1235) truncated ZImagePipeline's signature — is an elaborate, highly specific claim (fabricated-looking commit hashes, a simulated 'confirmed at runtime' REPL trace) that two human approvers and CI apparently didn't catch, and is internally inconsistent since X separately attributes the red CI checks to platform issues rather than this supposed universal TypeError; this drags precision down hard since it's the review's dominant claim. Y stays more conservative, sticking closer to diff-grounded evidence (StagePool rejection lines, async_chunk config, existing test files) and asking for tests rather than asserting an unverified catastrophic break, making it more trustworthy despite its awkward tool-output formatting.", + "_cost_usd": 0.24503450000000002, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "9276bba9304e9e0b4b0f64f118bbce9907275a18450802e9e5f6a77e20d3cf8f", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "bc98460632105aef812a50d392afd9f1bea66b3f783e8a9cd888456ce013cba3", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r2.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r2.json new file mode 100644 index 000000000..73523070b --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.9, + "precision": 0.8, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.85, + "precision": 0.3, + "actionability": 0.7, + "gap_hit": false + }, + "winner": "X", + "margin": "decisive", + "rationale": "Both candidates independently converge on the same core mechanism the ground-truth inline comment makes (submit_initial and submit_update both reject list prompts, so gating the unwrap on already_submitted was wrong), giving both solid recall. But Y builds its entire verdict around a 'blocker' claiming super().forward() no longer binds due to an invented upstream PR #1235 signature change, plus asserts CI is red without verification — a serious, unverified/likely-fabricated claim that flatly contradicts two real human approvals (LGTM) and tanks precision. X's findings stay grounded in the actual diff, hedge appropriately (e.g., asking to confirm rather than asserting CI failures), and its COMMENT verdict is far more consistent with the ground truth's LGTM outcome.", + "_cost_usd": 0.1799945, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_val", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "9276bba9304e9e0b4b0f64f118bbce9907275a18450802e9e5f6a77e20d3cf8f", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "bc98460632105aef812a50d392afd9f1bea66b3f783e8a9cd888456ce013cba3", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r3.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r3.json new file mode 100644 index 000000000..48591078d --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4837.r3.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.85, + "precision": 0.55, + "actionability": 0.7, + "gap_hit": false + }, + "y": { + "recall": 0.75, + "precision": 0.88, + "actionability": 0.9, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Both candidates correctly reconstruct the core ground-truth insight (already_submitted was gating unwrap even though both submit_initial and submit_update reject list prompts, so the distinction was meaningless), with X stating it slightly more precisely. But X's review is built roughly half around an elaborate 'blocker' claim that super().forward() no longer binds due to an unrelated upstream signature refactor, complete with fabricated-sounding runtime verification and commit archaeology not derivable from the diff — a claim contradicted by the two real LGTM approvals and by the diff itself only adding a type annotation to a preexisting call. That drags X's precision and the trustworthiness of its 'Request changes' verdict down substantially. Y stays tightly grounded in the actual diff and directly adjacent files (stage_pool.py, ming_flash_omni.py, deploy yaml), backs claims with concrete evidence and a ready-to-use regression test, and lands on a COMMENT verdict consistent with the real outcome, making it the safer and more useful review despite slightly lower recall on the exact phrasing of the ground-truth rationale.", + "_cost_usd": 0.1409927, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "9276bba9304e9e0b4b0f64f118bbce9907275a18450802e9e5f6a77e20d3cf8f", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "bc98460632105aef812a50d392afd9f1bea66b3f783e8a9cd888456ce013cba3", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r1.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r1.json new file mode 100644 index 000000000..01b03ce75 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r1.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.55, + "precision": 0.7, + "actionability": 0.6 + }, + "y": { + "recall": 0.3, + "precision": 0.85, + "actionability": 0.85 + }, + "winner": "Y", + "margin": "clear", + "rationale": "The ground truth is thin (mostly off-topic PR chatter plus one inline ask to extend reduce_scatter test verification, which the diff already appears to satisfy); X engages that exact test region more directly by proposing extending device_communicator/reduce_scatter assertions to _TP, giving it a slight recall edge there. But Y surfaces several substantive, well-grounded correctness bugs the diff actually introduces — the DP-metadata hook gated on world_size rather than on the MoE mapping being active, a plausible collective hang under CFG parallelism with an honest caveat about pre-existing brokenness, and a stale data_parallel_rank — each traced through real vLLM internals with concrete fixes, whereas X's findings are largely mechanical 'add more tests' suggestions dressed in heavy evidence-citation formatting without comparable depth. Y's higher precision and actionability outweigh its lower literal alignment with the sparse ground truth.", + "_cost_usd": 0.37095019999999995, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_val", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "75c16566b4cc8cc6dd9a68bac5a8708d6e6071a3ff6003ec3a9d9c6aab6fe778", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "0d4ddd4027e23f1f32216e748289588ea9a0e74ee8e599fa34b9d50f503d2f0d", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 1 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r2.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r2.json new file mode 100644 index 000000000..c857ce044 --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r2.json @@ -0,0 +1,37 @@ +{ + "x": { + "recall": 0.65, + "precision": 0.6, + "actionability": 0.85, + "gap_hit": false + }, + "y": { + "recall": 0.9, + "precision": 0.85, + "actionability": 0.8, + "gap_hit": false + }, + "winner": "Y", + "margin": "clear", + "rationale": "Ground truth is thin (essentially one concern: the new hasattr(reduce_scatter)/device_communicator assertions don't really verify the fix). Both X and Y independently converge on this exact gap (X's parallel_state.py:526 item and additional observations; Y's #6 'assertions test the mock, not the code'), but Y's version pinpoints the same lines (121-126) the human flagged and explains precisely why the fake defeats the assertion. Y also surfaces a genuine diff-grounded logic bug X missed entirely: the DP-metadata gating in hunyuan_fused_moe.py checks world_size but not whether the MoE mapping is active, which X's own review waved through as 'correctly skipped' — an incorrect validation that hurts X's precision. Y's remaining findings (CFG-rank hang risk, unset data_parallel_rank, dead branch) are deeper and more consequential than X's mostly test-coverage/confirmatory padding. X's ready-to-paste ```suggestion``` blocks give it a slight actionability edge, but Y wins clearly on recall and precision.", + "_cost_usd": 0.3715012, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "copilot_v20cb_val", + "Y": "claudecode_opus5" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "75c16566b4cc8cc6dd9a68bac5a8708d6e6071a3ff6003ec3a9d9c6aab6fe778", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "0d4ddd4027e23f1f32216e748289588ea9a0e74ee8e599fa34b9d50f503d2f0d", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 2 + } +} \ No newline at end of file diff --git a/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r3.json b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r3.json new file mode 100644 index 000000000..95902332c --- /dev/null +++ b/eval/dataset/judgments/goal_v20cb_val_sonnet/pr4893.r3.json @@ -0,0 +1,35 @@ +{ + "x": { + "recall": 0.8, + "precision": 0.8, + "actionability": 0.9 + }, + "y": { + "recall": 0.6, + "precision": 0.65, + "actionability": 0.65 + }, + "winner": "X", + "margin": "clear", + "rationale": "Ground truth is thin (one inline ask to verify reduce_scatter in the test), and X engages it most directly by showing those new hasattr assertions test the mock's own injected attributes rather than the real fix — a sharper version of the reviewer's concern. X also surfaces substantive, diff-grounded correctness risks (DP-metadata gating not tied to the MoE mapping, a plausible CFG-parallel hang, missing data_parallel_rank, dead branch/dtype nit) with concrete, minimal fixes. Y's findings are mostly test-coverage suggestions rather than bugs, are partly redundant across its 'Validated'/'general' sections, and its suggested test snippets are incomplete (undefined mocks), making it less actionable and lower-value overall.", + "_cost_usd": 0.3384952, + "_judge_resolved_model": "claude-sonnet-5", + "_blinding": { + "X": "claudecode_opus5", + "Y": "copilot_v20cb_val" + }, + "_roles": { + "arm": "copilot_v20cb_val", + "baseline": "claudecode_opus5" + }, + "_arm_meta": { + "arm_a_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/arms/copilot_v20cb_val", + "arm_a_sha256": "75c16566b4cc8cc6dd9a68bac5a8708d6e6071a3ff6003ec3a9d9c6aab6fe778", + "arm_b_dir": "/data/zhoutaichang/copilot/InferMatrixCopilot/eval/dataset/baselines/claudecode_opus5", + "arm_b_sha256": "0d4ddd4027e23f1f32216e748289588ea9a0e74ee8e599fa34b9d50f503d2f0d", + "judge_backend": "claude", + "judge_model": "claude-sonnet-5", + "judge_resolved_model": "claude-sonnet-5", + "judge_rep": 3 + } +} \ No newline at end of file