Improve dashboard failure diagnostics

.github/scripts/pull-request-dashboard/classification.py

@@ -214,13 +214,25 @@ def run_llm_for_thread(thread: dict[str, Any], model: str) -> dict[str, Any]:
         print_copilot_otel_file(otel_path)
     response_text = proc.stdout
     decision, valid_response = parse_thread_decision(response_text)
+    failed = proc.returncode != 0 or not valid_response
     record = {
         "thread_id": thread["thread_id"],
         "thread_kind": thread["thread_kind"],
         "_copilot_cli_call": True,
-        "failed": proc.returncode != 0 or not valid_response,
+        "failed": failed,
         "decision": decision,
     }
+    if failed:
+        reasons = []
+        if proc.returncode != 0:
+            reasons.append(f"Copilot CLI exited with status {proc.returncode}")
+        if not valid_response:
+            reasons.append("Copilot CLI did not return a valid classification JSON object")
+        record["error"] = "; ".join(reasons)
+        if response_text.strip():
+            record["response_text"] = response_text
+        if proc.stderr.strip():
+            record["stderr"] = proc.stderr
     return record
 
 
@@ -259,7 +271,7 @@ def cached_classification_record(record: dict[str, Any]) -> dict[str, Any]:
     return {
         k: v
         for k, v in record.items()
-        if k not in ("_copilot_cli_call", "error", "response_text", "usage")
+        if k not in ("_copilot_cli_call", "error", "response_text", "stderr", "usage")
     }
 
 
@@ -289,14 +301,19 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) ->
             continue
         try:
             record = run_llm_for_thread(thread, model)
-        except subprocess.TimeoutExpired:
+        except subprocess.TimeoutExpired as e:
             record = {
                 "thread_id": thread["thread_id"],
                 "thread_kind": thread["thread_kind"],
                 "_copilot_cli_call": True,
                 "failed": True,
+                "error": f"Copilot CLI timed out after {LLM_THREAD_TIMEOUT_SECONDS}s",
                 "decision": {"thread_action": "unclear", "reason": "LLM timeout"},
             }
+            if e.stdout and e.stdout.strip():
+                record["response_text"] = e.stdout
+            if e.stderr and e.stderr.strip():
+                record["stderr"] = e.stderr
         except Exception as e:
             print(
                 f"  warning: thread {thread['thread_id']} on PR #{number} failed to classify:",
@@ -307,6 +324,7 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) ->
                 "thread_id": thread["thread_id"],
                 "thread_kind": thread["thread_kind"],
                 "failed": True,
+                "error": f"LLM failed: {e!r}",
                 "decision": {"thread_action": "unclear", "reason": f"LLM failed: {e!r}"},
             }
         classifications.append(record)

.github/scripts/pull-request-dashboard/dashboard.py

@@ -1039,6 +1039,84 @@ def select_backfill_prs(
     )
 
 
+def log_line_value(value: Any) -> str:
+    return " ".join(str(value or "").split())
+
+
+def log_multiline_value(label: str, value: Any) -> None:
+    text = str(value or "").strip()
+    if not text:
+        return
+    print(f"      {label}:", file=sys.stderr)
+    print(f"      --- BEGIN {label} ---", file=sys.stderr)
+    for line in text.splitlines():
+        print(f"      | {line}", file=sys.stderr)
+    print(f"      --- END {label} ---", file=sys.stderr)
+
+
+def log_failed_classification_diagnostics(
+    classification: dict[str, Any],
+    thread: dict[str, Any] | None,
+) -> None:
+    decision = classification.get("decision") or {}
+    print(
+        "    failed classification: "
+        f"thread_id={classification.get('thread_id') or '<unknown>'} "
+        f"kind={classification.get('thread_kind') or '<unknown>'} "
+        f"action={decision.get('thread_action') or '<unknown>'} "
+        f"reason={log_line_value(decision.get('reason')) or '<none>'}",
+        file=sys.stderr,
+    )
+    if classification.get("error"):
+        print(f"      error: {log_line_value(classification.get('error'))}", file=sys.stderr)
+    if thread:
+        comments = thread.get("comments") or []
+        latest = comments[-1] if comments else {}
+        location = thread.get("path") or ""
+        if location and thread.get("line"):
+            location = f"{location}:{thread.get('line')}"
+        print(
+            "      thread: "
+            f"location={location or '<none>'} "
+            f"latest_actor={latest.get('actor') or '<unknown>'} "
+            f"latest_role={latest.get('actor_role') or '<unknown>'} "
+            f"latest_at={latest.get('timestamp') or '<unknown>'}",
+            file=sys.stderr,
+        )
+        if latest.get("body"):
+            print(f"      latest_body: {log_line_value(latest.get('body'))}", file=sys.stderr)
+    for key in ("response_text", "stderr"):
+        if classification.get(key):
+            log_multiline_value(key, classification.get(key))
+
+
+def log_failed_result_diagnostics(result: dict[str, Any]) -> None:
+    print("dashboard failure diagnostics:", file=sys.stderr)
+    number = result.get("pr_number") or "?"
+    print(
+        f"  PR #{number}: route={result.get('route') or '<unknown>'} "
+        f"error={log_line_value(result.get('error')) or '<none>'}",
+        file=sys.stderr,
+    )
+    if result.get("pr_url"):
+        print(f"    url: {result.get('pr_url')}", file=sys.stderr)
+    threads = {
+        thread.get("thread_id"): thread
+        for thread in (result.get("threads") or [])
+        if isinstance(thread, dict) and thread.get("thread_id")
+    }
+    failed_classifications = [
+        classification
+        for classification in (result.get("classifications") or [])
+        if classification.get("failed")
+    ]
+    for classification in failed_classifications:
+        log_failed_classification_diagnostics(
+            classification,
+            threads.get(classification.get("thread_id")),
+        )
+
+
 def has_failed_dashboard_result(result: dict[str, Any] | None) -> bool:
     return bool(result and result.get("failed"))
 
@@ -1047,6 +1125,7 @@ def reject_failed_dashboard_result(result: dict[str, Any] | None) -> bool:
     if result is None or not has_failed_dashboard_result(result):
         return False
     number = result.get("pr_number") or "?"
+    log_failed_result_diagnostics(result)
     print(
         f"dashboard refresh hit PR failure(s); refusing to publish failed state: #{number}",
         file=sys.stderr,