diff --git a/.github/scripts/pull-request-dashboard/classification.py b/.github/scripts/pull-request-dashboard/classification.py index 7d785b827..a25b27665 100644 --- a/.github/scripts/pull-request-dashboard/classification.py +++ b/.github/scripts/pull-request-dashboard/classification.py @@ -214,13 +214,25 @@ def run_llm_for_thread(thread: dict[str, Any], model: str) -> dict[str, Any]: print_copilot_otel_file(otel_path) response_text = proc.stdout decision, valid_response = parse_thread_decision(response_text) + failed = proc.returncode != 0 or not valid_response record = { "thread_id": thread["thread_id"], "thread_kind": thread["thread_kind"], "_copilot_cli_call": True, - "failed": proc.returncode != 0 or not valid_response, + "failed": failed, "decision": decision, } + if failed: + reasons = [] + if proc.returncode != 0: + reasons.append(f"Copilot CLI exited with status {proc.returncode}") + if not valid_response: + reasons.append("Copilot CLI did not return a valid classification JSON object") + record["error"] = "; ".join(reasons) + if response_text.strip(): + record["response_text"] = response_text + if proc.stderr.strip(): + record["stderr"] = proc.stderr return record @@ -259,7 +271,7 @@ def cached_classification_record(record: dict[str, Any]) -> dict[str, Any]: return { k: v for k, v in record.items() - if k not in ("_copilot_cli_call", "error", "response_text", "usage") + if k not in ("_copilot_cli_call", "error", "response_text", "stderr", "usage") } @@ -289,14 +301,19 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> continue try: record = run_llm_for_thread(thread, model) - except subprocess.TimeoutExpired: + except subprocess.TimeoutExpired as e: record = { "thread_id": thread["thread_id"], "thread_kind": thread["thread_kind"], "_copilot_cli_call": True, "failed": True, + "error": f"Copilot CLI timed out after {LLM_THREAD_TIMEOUT_SECONDS}s", "decision": {"thread_action": "unclear", "reason": "LLM timeout"}, } + if e.stdout and e.stdout.strip(): + record["response_text"] = e.stdout + if e.stderr and e.stderr.strip(): + record["stderr"] = e.stderr except Exception as e: print( f" warning: thread {thread['thread_id']} on PR #{number} failed to classify:", @@ -307,6 +324,7 @@ def classify_threads(number: int, threads: list[dict[str, Any]], model: str) -> "thread_id": thread["thread_id"], "thread_kind": thread["thread_kind"], "failed": True, + "error": f"LLM failed: {e!r}", "decision": {"thread_action": "unclear", "reason": f"LLM failed: {e!r}"}, } classifications.append(record) diff --git a/.github/scripts/pull-request-dashboard/dashboard.py b/.github/scripts/pull-request-dashboard/dashboard.py index b8a5f39af..20f5601dd 100644 --- a/.github/scripts/pull-request-dashboard/dashboard.py +++ b/.github/scripts/pull-request-dashboard/dashboard.py @@ -1039,6 +1039,84 @@ def select_backfill_prs( ) +def log_line_value(value: Any) -> str: + return " ".join(str(value or "").split()) + + +def log_multiline_value(label: str, value: Any) -> None: + text = str(value or "").strip() + if not text: + return + print(f" {label}:", file=sys.stderr) + print(f" --- BEGIN {label} ---", file=sys.stderr) + for line in text.splitlines(): + print(f" | {line}", file=sys.stderr) + print(f" --- END {label} ---", file=sys.stderr) + + +def log_failed_classification_diagnostics( + classification: dict[str, Any], + thread: dict[str, Any] | None, +) -> None: + decision = classification.get("decision") or {} + print( + " failed classification: " + f"thread_id={classification.get('thread_id') or ''} " + f"kind={classification.get('thread_kind') or ''} " + f"action={decision.get('thread_action') or ''} " + f"reason={log_line_value(decision.get('reason')) or ''}", + file=sys.stderr, + ) + if classification.get("error"): + print(f" error: {log_line_value(classification.get('error'))}", file=sys.stderr) + if thread: + comments = thread.get("comments") or [] + latest = comments[-1] if comments else {} + location = thread.get("path") or "" + if location and thread.get("line"): + location = f"{location}:{thread.get('line')}" + print( + " thread: " + f"location={location or ''} " + f"latest_actor={latest.get('actor') or ''} " + f"latest_role={latest.get('actor_role') or ''} " + f"latest_at={latest.get('timestamp') or ''}", + file=sys.stderr, + ) + if latest.get("body"): + print(f" latest_body: {log_line_value(latest.get('body'))}", file=sys.stderr) + for key in ("response_text", "stderr"): + if classification.get(key): + log_multiline_value(key, classification.get(key)) + + +def log_failed_result_diagnostics(result: dict[str, Any]) -> None: + print("dashboard failure diagnostics:", file=sys.stderr) + number = result.get("pr_number") or "?" + print( + f" PR #{number}: route={result.get('route') or ''} " + f"error={log_line_value(result.get('error')) or ''}", + file=sys.stderr, + ) + if result.get("pr_url"): + print(f" url: {result.get('pr_url')}", file=sys.stderr) + threads = { + thread.get("thread_id"): thread + for thread in (result.get("threads") or []) + if isinstance(thread, dict) and thread.get("thread_id") + } + failed_classifications = [ + classification + for classification in (result.get("classifications") or []) + if classification.get("failed") + ] + for classification in failed_classifications: + log_failed_classification_diagnostics( + classification, + threads.get(classification.get("thread_id")), + ) + + def has_failed_dashboard_result(result: dict[str, Any] | None) -> bool: return bool(result and result.get("failed")) @@ -1047,6 +1125,7 @@ def reject_failed_dashboard_result(result: dict[str, Any] | None) -> bool: if result is None or not has_failed_dashboard_result(result): return False number = result.get("pr_number") or "?" + log_failed_result_diagnostics(result) print( f"dashboard refresh hit PR failure(s); refusing to publish failed state: #{number}", file=sys.stderr,