diff --git a/.github/workflows/create_test_report.yml b/.github/workflows/create_test_report.yml index 2473a050962e6..786f1dd50f159 100644 --- a/.github/workflows/create_test_report.yml +++ b/.github/workflows/create_test_report.yml @@ -4,6 +4,9 @@ on: workflows: ["tests 1", "tests 2", "tests others", "MCP"] types: - completed + +permissions: {} + jobs: merge-reports: permissions: @@ -13,6 +16,10 @@ jobs: contents: read # This is required for actions/checkout to succeed if: ${{ github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'push' }} runs-on: ubuntu-latest + outputs: + pr_number: ${{ steps.pr.outputs.number }} + has_failures: ${{ steps.failures.outputs.has_failures }} + triage_allowed: ${{ steps.pr.outputs.triage_allowed }} env: MARKDOWN_OUTPUT_FILE: ${{ github.workspace }}/report.md steps: @@ -36,6 +43,15 @@ jobs: NODE_OPTIONS: --max-old-space-size=8192 WORKFLOW_RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + - name: Detect failures + id: failures + run: | + if [ -f "$MARKDOWN_OUTPUT_FILE" ] && grep -qE '\*\*[0-9]+ failed\*\*' "$MARKDOWN_OUTPUT_FILE"; then + echo "has_failures=true" >> "$GITHUB_OUTPUT" + else + echo "has_failures=false" >> "$GITHUB_OUTPUT" + fi + - name: Upload HTML report id: upload-report uses: actions/upload-artifact@v7 @@ -47,7 +63,7 @@ jobs: # The triggering workflow ran on a pull_request event, but workflow_run.pull_requests is # empty for PRs from forks, so resolve the number from the head ref instead. gh's head # filter wants "owner:branch" for forks but a bare branch for same-repo PRs. - - name: Resolve PR number + - name: Resolve PR number and triage allow-list if: ${{ github.event.workflow_run.event == 'pull_request' }} id: pr env: @@ -57,6 +73,14 @@ jobs: NUMBER=$(gh pr view --repo "${{ github.repository }}" "$HEAD_REF" --json number --jq '.number' 2>/dev/null || true) echo "number=$NUMBER" >> "$GITHUB_OUTPUT" + AUTHOR=$(gh pr view --repo "${{ github.repository }}" "$HEAD_REF" --json author --jq '.author.login' 2>/dev/null || true) + ALLOWED="github-actions[bot] pavelfeldman yury-s dgozman Skn0tt dcrousso" + TRIAGE_ALLOWED=false + for a in $ALLOWED; do + if [ "$a" = "$AUTHOR" ]; then TRIAGE_ALLOWED=true; break; fi + done + echo "triage_allowed=$TRIAGE_ALLOWED" >> "$GITHUB_OUTPUT" + - name: Post report comment to PR if: ${{ steps.pr.outputs.number }} uses: actions/github-script@v8 @@ -88,3 +112,98 @@ jobs: target_url: '${{ steps.upload-report.outputs.artifact-url }}', context: 'HTML Report (${{ github.event.workflow_run.name }})', }); + + triage: + needs: merge-reports + if: ${{ needs.merge-reports.outputs.has_failures == 'true' && needs.merge-reports.outputs.pr_number && needs.merge-reports.outputs.triage_allowed == 'true' }} + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + copilot-requests: write + outputs: + has_draft: ${{ steps.triage.outputs.has_draft }} + pr_number: ${{ needs.merge-reports.outputs.pr_number }} + env: + PR_NUMBER: ${{ needs.merge-reports.outputs.pr_number }} + GH_TOKEN: ${{ github.token }} + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + + - name: Install Copilot CLI + run: npm install -g @github/copilot + + - name: Triage failures with Copilot CLI + id: triage + env: + COPILOT_GITHUB_TOKEN: ${{ github.token }} + run: | + mkdir -p output + PROMPT=$(cat <> "$GITHUB_OUTPUT" + else + echo "has_draft=false" >> "$GITHUB_OUTPUT" + fi + + - name: Add session transcript to job summary + if: ${{ always() }} + run: | + { + echo "## CI triage session transcript (PR #$PR_NUMBER)" + echo '' + cat "output/copilot-session.md" 2>/dev/null || echo "(no transcript)" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload output + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: ci-triage-${{ needs.merge-reports.outputs.pr_number }} + path: output/triage.md + if-no-files-found: warn + + post: + needs: triage + if: needs.triage.outputs.has_draft == 'true' + runs-on: ubuntu-latest + permissions: + issues: write + env: + PR_NUMBER: ${{ needs.triage.outputs.pr_number }} + GH_TOKEN: ${{ github.token }} + WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - name: Download triage output + uses: actions/download-artifact@v4 + with: + name: ci-triage-${{ needs.triage.outputs.pr_number }} + path: output + + - name: Post triage comment + run: | + printf '\n\nTriaged by the Playwright bot - [agent run](%s)\n\n' "$WORKFLOW_URL" >> output/triage.md + gh issue comment "$PR_NUMBER" --repo "${{ github.repository }}" --body-file output/triage.md + diff --git a/.github/workflows/pr-ci-triage.md b/.github/workflows/pr-ci-triage.md new file mode 100644 index 0000000000000..7ab6b783cb50b --- /dev/null +++ b/.github/workflows/pr-ci-triage.md @@ -0,0 +1,50 @@ +# PR CI Failure Verdict + +Come up with a verdict on a PR's failing CI tests: **are they likely caused by this PR, or +pre-existing flakes / infra noise?** +We want a verdict for the failures. If you can trivially come up with a fix, you can also propose it. +A merged report is posted on the PR by the `github-actions[bot]` comment, look at the most recent one. + +No failing tests (only flaky/interrupted) → nothing to triage. Group the same test failing across browsers as one story. + +**Hard rule for calling something a flake:** you must find the *same test* failing or flaking somewhere the PR can't be responsible for. Query the aggregated CI results via the `playwright-test-results` skill — look for the same `(project_name, file, test_title)` failing on other SHAs/PRs, or flipping verdict across runs. Divergence ("different tests, different browsers/OSes") and a plausible signature (timeout, element-not-found) are **not** enough on their own — flakes are often surprising and unrelated to the test's subject. No such evidence found → the failure is **uncertain**, not a flake. + +Some things to look into per failure (or group): + +- **Does the diff reach it?** The file under test, a feature it exercises, a shared helper it imports, or the product code path it asserts on. Network test vs. a docs-only PR → unrelated; click test vs. a PR rewriting input dispatch → suspicious. +- **Has this test flaked before?** Go look (see the hard rule above): use the `playwright-test-results` skill to check the same test's history across runs. Run the skill's `update` step first — the shared snapshot lags a few hours, so the freshest runs (including this PR's own) may not be in it yet. This is the only thing that proves a flake. +- **Flake/infra signature?** Timeouts, `Target closed`, browser launch/download errors, network hiccups, or the same test also flaking (passed on retry) in this report. A supporting hint, never proof on its own. +- **Browser/platform divergence.** One browser only, in code the PR didn't touch for that engine → leans flake; explainable on every browser → leans caused-by-PR. A weak hint only. + +## Verdict format + +Lead with a single traffic-light headline so the signal is readable at a glance, then put the per-group detail in a collapsible `
`: + +- 🔴 **Red** — at least one failure is caused by this PR. +- 🟡 **Yellow** — uncertain; you couldn't prove it either way. +- 🟢 **Green** — no real failures, or all confirmed pre-existing flakes / infra. The PR is clear. + +The overall colour is the worst of the per-group calls (any red → red, else any yellow → yellow, else green). + +Inside the collapsible, make an overall assessment of the PR's impact on CI failures, then segment by each failure/group as **caused by this PR** (which change, why), **pre-existing flake / infra** (cite where else the same test failed/flaked), or **uncertain** (what you'd need to be sure). + +Example (a red verdict — one real failure plus flakes): + +```markdown +## 🔴 One failure looks caused by this PR + +`page-request-gc.spec.ts:36` fails on Firefox because the fix here is Chromium-only. + +
+Details + +**Caused by this PR** + +- `[firefox-page] › page/page-request-gc.spec.ts:36 › should collect element retained by locator hit-target interceptor after detach` — the test the PR adds. The retention fix lives in `crPage.ts::requestGC`, which is Chromium-specific, so the element is still retained under Firefox and `weakRef.deref()` isn't `undefined`. The test has no browser gate, so it runs and fails on Firefox. Gate it to Chromium, e.g. `test.skip(browserName === 'firefox')`. + +**Pre-existing flake / infra** + +- `[chromium] › mcp/annotate.spec.ts:230 › should capture annotations via show --annotate` (+ `:496`) — pre-existing flake. Across the test-results DB this test flips verdict: **failed 7 of 54 runs (13%), passed the other 47**, on SHAs unrelated to this PR. This PR only touches the hit-target interceptor and Chromium `requestGC`, which the MCP annotate flow doesn't exercise. + +
+``` diff --git a/.github/workflows/triage.yml b/.github/workflows/triage.yml index 08f8584c688d7..c5b6d1f01c825 100644 --- a/.github/workflows/triage.yml +++ b/.github/workflows/triage.yml @@ -41,14 +41,12 @@ jobs: id: triage env: COPILOT_GITHUB_TOKEN: ${{ github.token }} - WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | mkdir -p output PROMPT=$(cat <\n' >> output/triage.md + printf '\n\nTriaged by the Playwright bot - [agent run](%s)\n\n' "$WORKFLOW_URL" >> output/triage.md gh issue comment "$ISSUE" --repo "${{ github.repository }}" --body-file output/triage.md