feat(devops): add PR CI failure triage bot

.github/workflows/create_test_report.yml

@@ -4,6 +4,9 @@ on:
     workflows: ["tests 1", "tests 2", "tests others", "MCP"]
     types:
       - completed
+
+permissions: {}
+
 jobs:
   merge-reports:
     permissions:
@@ -13,6 +16,10 @@ jobs:
       contents: read    # This is required for actions/checkout to succeed
     if: ${{ github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'push' }}
     runs-on: ubuntu-latest
+    outputs:
+      pr_number: ${{ steps.pr.outputs.number }}
+      has_failures: ${{ steps.failures.outputs.has_failures }}
+      triage_allowed: ${{ steps.pr.outputs.triage_allowed }}
     env:
       MARKDOWN_OUTPUT_FILE: ${{ github.workspace }}/report.md
     steps:
@@ -36,6 +43,15 @@ jobs:
         NODE_OPTIONS: --max-old-space-size=8192
         WORKFLOW_RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }}
 
+    - name: Detect failures
+      id: failures
+      run: |
+        if [ -f "$MARKDOWN_OUTPUT_FILE" ] && grep -qE '\*\*[0-9]+ failed\*\*' "$MARKDOWN_OUTPUT_FILE"; then
+          echo "has_failures=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "has_failures=false" >> "$GITHUB_OUTPUT"
+        fi
+
     - name: Upload HTML report
       id: upload-report
       uses: actions/upload-artifact@v7
@@ -47,7 +63,7 @@ jobs:
     # The triggering workflow ran on a pull_request event, but workflow_run.pull_requests is
     # empty for PRs from forks, so resolve the number from the head ref instead. gh's head
     # filter wants "owner:branch" for forks but a bare branch for same-repo PRs.
-    - name: Resolve PR number
+    - name: Resolve PR number and triage allow-list
       if: ${{ github.event.workflow_run.event == 'pull_request' }}
       id: pr
       env:
@@ -57,6 +73,14 @@ jobs:
         NUMBER=$(gh pr view --repo "${{ github.repository }}" "$HEAD_REF" --json number --jq '.number' 2>/dev/null || true)
         echo "number=$NUMBER" >> "$GITHUB_OUTPUT"
 
+        AUTHOR=$(gh pr view --repo "${{ github.repository }}" "$HEAD_REF" --json author --jq '.author.login' 2>/dev/null || true)
+        ALLOWED="github-actions[bot] pavelfeldman yury-s dgozman Skn0tt dcrousso"
+        TRIAGE_ALLOWED=false
+        for a in $ALLOWED; do
+          if [ "$a" = "$AUTHOR" ]; then TRIAGE_ALLOWED=true; break; fi
+        done
+        echo "triage_allowed=$TRIAGE_ALLOWED" >> "$GITHUB_OUTPUT"
+
     - name: Post report comment to PR
       if: ${{ steps.pr.outputs.number }}
       uses: actions/github-script@v8
@@ -88,3 +112,98 @@ jobs:
             target_url: '${{ steps.upload-report.outputs.artifact-url }}',
             context: 'HTML Report (${{ github.event.workflow_run.name }})',
           });
+
+  triage:
+    needs: merge-reports
+    if: ${{ needs.merge-reports.outputs.has_failures == 'true' && needs.merge-reports.outputs.pr_number && needs.merge-reports.outputs.triage_allowed == 'true' }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      copilot-requests: write
+    outputs:
+      has_draft: ${{ steps.triage.outputs.has_draft }}
+      pr_number: ${{ needs.merge-reports.outputs.pr_number }}
+    env:
+      PR_NUMBER: ${{ needs.merge-reports.outputs.pr_number }}
+      GH_TOKEN: ${{ github.token }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+
+      - name: Install Copilot CLI
+        run: npm install -g @github/copilot
+
+      - name: Triage failures with Copilot CLI
+        id: triage
+        env:
+          COPILOT_GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p output
+          PROMPT=$(cat <<EOF
+          Come up with a verdict on the CI test failures on pull request #$PR_NUMBER of ${{ github.repository }}: are they likely caused by this PR? Follow the pr-ci-triage guide (.github/workflows/pr-ci-triage.md) for the analysis.
+
+          Write your verdict to output/triage.md as a PR comment, in the format the guide specifies and the playwright-bot-voice register (.claude/skills/playwright-bot-voice/SKILL.md).
+
+          Do not post anything yourself — a later workflow step posts output/triage.md.
+          EOF
+          )
+          copilot \
+            --allow-all-tools \
+            --allow-all-paths \
+            --no-ask-user \
+            --enable-all-github-mcp-tools \
+            --share=output/copilot-session.md \
+            --model claude-opus-4.8 \
+            --max-ai-credits 500 \
+            -p "$PROMPT"
+
+          if [ -s output/triage.md ]; then
+            echo "has_draft=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "has_draft=false" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Add session transcript to job summary
+        if: ${{ always() }}
+        run: |
+          {
+            echo "## CI triage session transcript (PR #$PR_NUMBER)"
+            echo ''
+            cat "output/copilot-session.md" 2>/dev/null || echo "(no transcript)"
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Upload output
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-triage-${{ needs.merge-reports.outputs.pr_number }}
+          path: output/triage.md
+          if-no-files-found: warn
+
+  post:
+    needs: triage
+    if: needs.triage.outputs.has_draft == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    env:
+      PR_NUMBER: ${{ needs.triage.outputs.pr_number }}
+      GH_TOKEN: ${{ github.token }}
+      WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - name: Download triage output
+        uses: actions/download-artifact@v4
+        with:
+          name: ci-triage-${{ needs.triage.outputs.pr_number }}
+          path: output
+
+      - name: Post triage comment
+        run: |
+          printf '\n\n<sub>Triaged by the Playwright bot - [agent run](%s)</sub>\n<!-- playwright-ci-triage -->\n' "$WORKFLOW_URL" >> output/triage.md
+          gh issue comment "$PR_NUMBER" --repo "${{ github.repository }}" --body-file output/triage.md
+

.github/workflows/pr-ci-triage.md

@@ -0,0 +1,50 @@
+# PR CI Failure Verdict
+
+Come up with a verdict on a PR's failing CI tests: **are they likely caused by this PR, or
+pre-existing flakes / infra noise?**
+We want a verdict for the failures. If you can trivially come up with a fix, you can also propose it.
+A merged report is posted on the PR by the `github-actions[bot]` comment, look at the most recent one.
+
+No failing tests (only flaky/interrupted) → nothing to triage. Group the same test failing across browsers as one story.
+
+**Hard rule for calling something a flake:** you must find the *same test* failing or flaking somewhere the PR can't be responsible for. Query the aggregated CI results via the `playwright-test-results` skill — look for the same `(project_name, file, test_title)` failing on other SHAs/PRs, or flipping verdict across runs. Divergence ("different tests, different browsers/OSes") and a plausible signature (timeout, element-not-found) are **not** enough on their own — flakes are often surprising and unrelated to the test's subject. No such evidence found → the failure is **uncertain**, not a flake.
+
+Some things to look into per failure (or group):
+
+- **Does the diff reach it?** The file under test, a feature it exercises, a shared helper it imports, or the product code path it asserts on. Network test vs. a docs-only PR → unrelated; click test vs. a PR rewriting input dispatch → suspicious.
+- **Has this test flaked before?** Go look (see the hard rule above): use the `playwright-test-results` skill to check the same test's history across runs. Run the skill's `update` step first — the shared snapshot lags a few hours, so the freshest runs (including this PR's own) may not be in it yet. This is the only thing that proves a flake.
+- **Flake/infra signature?** Timeouts, `Target closed`, browser launch/download errors, network hiccups, or the same test also flaking (passed on retry) in this report. A supporting hint, never proof on its own.
+- **Browser/platform divergence.** One browser only, in code the PR didn't touch for that engine → leans flake; explainable on every browser → leans caused-by-PR. A weak hint only.
+
+## Verdict format
+
+Lead with a single traffic-light headline so the signal is readable at a glance, then put the per-group detail in a collapsible `<details>`:
+
+- 🔴 **Red** — at least one failure is caused by this PR.
+- 🟡 **Yellow** — uncertain; you couldn't prove it either way.
+- 🟢 **Green** — no real failures, or all confirmed pre-existing flakes / infra. The PR is clear.
+
+The overall colour is the worst of the per-group calls (any red → red, else any yellow → yellow, else green).
+
+Inside the collapsible, make an overall assessment of the PR's impact on CI failures, then segment by each failure/group as **caused by this PR** (which change, why), **pre-existing flake / infra** (cite where else the same test failed/flaked), or **uncertain** (what you'd need to be sure).
+
+Example (a red verdict — one real failure plus flakes):
+
+```markdown
+## 🔴 One failure looks caused by this PR
+
+`page-request-gc.spec.ts:36` fails on Firefox because the fix here is Chromium-only.
+
+<details>
+<summary>Details</summary>
+
+**Caused by this PR**
+
+- `[firefox-page] › page/page-request-gc.spec.ts:36 › should collect element retained by locator hit-target interceptor after detach` — the test the PR adds. The retention fix lives in `crPage.ts::requestGC`, which is Chromium-specific, so the element is still retained under Firefox and `weakRef.deref()` isn't `undefined`. The test has no browser gate, so it runs and fails on Firefox. Gate it to Chromium, e.g. `test.skip(browserName === 'firefox')`.
+
+**Pre-existing flake / infra**
+
+- `[chromium] › mcp/annotate.spec.ts:230 › should capture annotations via show --annotate` (+ `:496`) — pre-existing flake. Across the test-results DB this test flips verdict: **failed 7 of 54 runs (13%), passed the other 47**, on SHAs unrelated to this PR. This PR only touches the hit-target interceptor and Chromium `requestGC`, which the MCP annotate flow doesn't exercise.
+
+</details>
+```

.github/workflows/triage.yml

@@ -41,14 +41,12 @@ jobs:
         id: triage
         env:
           COPILOT_GITHUB_TOKEN: ${{ github.token }}
-          WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           mkdir -p output
           PROMPT=$(cat <<EOF
           Triage https://github.com/${{ github.repository }}/issues/$ISSUE using the playwright-triage skill.
 
           Do not post anything yourself. Write the comment to output/triage.md and a later step posts it.
-          The link to this agent run is in \$WORKFLOW_URL.
           EOF
           )
           copilot \
@@ -93,6 +91,7 @@ jobs:
     env:
       ISSUE: ${{ github.event.issue.number || inputs.issue }}
       GH_TOKEN: ${{ github.token }}
+      WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
     steps:
       - name: Download triage output
         uses: actions/download-artifact@v4
@@ -102,5 +101,5 @@ jobs:
 
       - name: Post triage comment
         run: |
-          printf '\n\n<!-- playwright-ai-triage -->\n' >> output/triage.md
+          printf '\n\n<sub>Triaged by the Playwright bot - [agent run](%s)</sub>\n<!-- playwright-ai-triage -->\n' "$WORKFLOW_URL" >> output/triage.md
           gh issue comment "$ISSUE" --repo "${{ github.repository }}" --body-file output/triage.md