From 374f8abf28c5ef1d5b8650c0a0013818774584c1 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Wed, 8 Jul 2026 17:08:36 -0700 Subject: [PATCH 1/2] Improve dashboard failure diagnostics --- .../pull-request-dashboard/classification.py | 18 ++++- .../pull-request-dashboard/dashboard.py | 80 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/.github/scripts/pull-request-dashboard/classification.py b/.github/scripts/pull-request-dashboard/classification.py index 7d785b827..af7842aeb 100644 --- a/.github/scripts/pull-request-dashboard/classification.py +++ b/.github/scripts/pull-request-dashboard/classification.py @@ -214,13 +214,25 @@ def run_llm_for_thread(thread: dict[str, Any], model: str) -> dict[str, Any]: print_copilot_otel_file(otel_path) response_text = proc.stdout decision, valid_response = parse_thread_decision(response_text) + failed = proc.returncode != 0 or not valid_response record = { "thread_id": thread["thread_id"], "thread_kind": thread["thread_kind"], "_copilot_cli_call": True, - "failed": proc.returncode != 0 or not valid_response, + "failed": failed, "decision": decision, } + if failed: + reasons = [] + if proc.returncode != 0: + reasons.append(f"Copilot CLI exited with status {proc.returncode}") + if not valid_response: + reasons.append("Copilot CLI did not return a valid classification JSON object") + record["error"] = "; ".join(reasons) + if response_text.strip(): + record["response_text"] = response_text + if proc.stderr.strip(): + record["stderr"] = proc.stderr return record @@ -259,7 +271,7 @@ def cached_classification_record(record: dict[str, Any]) -> dict[str, Any]: return { k: v for k, v in record.items() - if k not in ("_copilot_cli_call", "error", "response_text", "usage") + if k not in ("_copilot_cli_call", "error", "response_text", "stderr", "usage") } @@ -295,6 +307,7 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> "thread_kind": thread["thread_kind"], "_copilot_cli_call": True, "failed": True, + "error": f"Copilot CLI timed out after {LLM_THREAD_TIMEOUT_SECONDS}s", "decision": {"thread_action": "unclear", "reason": "LLM timeout"}, } except Exception as e: @@ -307,6 +320,7 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> "thread_id": thread["thread_id"], "thread_kind": thread["thread_kind"], "failed": True, + "error": f"LLM failed: {e!r}", "decision": {"thread_action": "unclear", "reason": f"LLM failed: {e!r}"}, } classifications.append(record) diff --git a/.github/scripts/pull-request-dashboard/dashboard.py b/.github/scripts/pull-request-dashboard/dashboard.py index 282971c4b..3a838b85c 100644 --- a/.github/scripts/pull-request-dashboard/dashboard.py +++ b/.github/scripts/pull-request-dashboard/dashboard.py @@ -1045,6 +1045,85 @@ def select_backfill_prs( ) +def log_line_value(value: Any) -> str: + return " ".join(str(value or "").split()) + + +def log_multiline_value(label: str, value: Any) -> None: + text = str(value or "").strip() + if not text: + return + print(f" {label}:", file=sys.stderr) + print(f" --- BEGIN {label} ---", file=sys.stderr) + for line in text.splitlines(): + print(f" | {line}", file=sys.stderr) + print(f" --- END {label} ---", file=sys.stderr) + + +def log_failed_classification_diagnostics( + classification: dict[str, Any], + thread: dict[str, Any] | None, +) -> None: + decision = classification.get("decision") or {} + print( + " failed classification: " + f"thread_id={classification.get('thread_id') or ''} " + f"kind={classification.get('thread_kind') or ''} " + f"action={decision.get('thread_action') or ''} " + f"reason={log_line_value(decision.get('reason')) or ''}", + file=sys.stderr, + ) + if classification.get("error"): + print(f" error: {log_line_value(classification.get('error'))}", file=sys.stderr) + if thread: + comments = thread.get("comments") or [] + latest = comments[-1] if comments else {} + location = thread.get("path") or "" + if location and thread.get("line"): + location = f"{location}:{thread.get('line')}" + print( + " thread: " + f"location={location or ''} " + f"latest_actor={latest.get('actor') or ''} " + f"latest_role={latest.get('actor_role') or ''} " + f"latest_at={latest.get('timestamp') or ''}", + file=sys.stderr, + ) + if latest.get("body"): + print(f" latest_body: {log_line_value(latest.get('body'))}", file=sys.stderr) + for key in ("response_text", "stderr"): + if classification.get(key): + log_multiline_value(key, classification.get(key)) + + +def log_failed_result_diagnostics(results: dict[int, dict[str, Any]], failed_results: list[int]) -> None: + print("dashboard failure diagnostics:", file=sys.stderr) + for number in failed_results: + result = results.get(number) or {} + print( + f" PR #{number}: route={result.get('route') or ''} " + f"error={log_line_value(result.get('error')) or ''}", + file=sys.stderr, + ) + if result.get("pr_url"): + print(f" url: {result.get('pr_url')}", file=sys.stderr) + threads = { + thread.get("thread_id"): thread + for thread in (result.get("threads") or []) + if isinstance(thread, dict) and thread.get("thread_id") + } + failed_classifications = [ + classification + for classification in (result.get("classifications") or []) + if classification.get("failed") + ] + for classification in failed_classifications: + log_failed_classification_diagnostics( + classification, + threads.get(classification.get("thread_id")), + ) + + def render_and_save_dashboard( args: argparse.Namespace, prs: list[dict[str, Any]], @@ -1058,6 +1137,7 @@ def render_and_save_dashboard( if result.get("failed") ] if failed_results: + log_failed_result_diagnostics(results, failed_results) print( "dashboard refresh hit PR failure(s); refusing to publish failed state: " + ", ".join(f"#{number}" for number in failed_results), From 2c1348655be860450ae9483b0917f9284794c776 Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Thu, 9 Jul 2026 08:17:48 -0700 Subject: [PATCH 2/2] Address review comment: preserve timeout output --- .github/scripts/pull-request-dashboard/classification.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/scripts/pull-request-dashboard/classification.py b/.github/scripts/pull-request-dashboard/classification.py index af7842aeb..a25b27665 100644 --- a/.github/scripts/pull-request-dashboard/classification.py +++ b/.github/scripts/pull-request-dashboard/classification.py @@ -301,7 +301,7 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> continue try: record = run_llm_for_thread(thread, model) - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired as e: record = { "thread_id": thread["thread_id"], "thread_kind": thread["thread_kind"], @@ -310,6 +310,10 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> "error": f"Copilot CLI timed out after {LLM_THREAD_TIMEOUT_SECONDS}s", "decision": {"thread_action": "unclear", "reason": "LLM timeout"}, } + if e.stdout and e.stdout.strip(): + record["response_text"] = e.stdout + if e.stderr and e.stderr.strip(): + record["stderr"] = e.stderr except Exception as e: print( f" warning: thread {thread['thread_id']} on PR #{number} failed to classify:",