From 2eef1af67d17a2e4fe134ab69fd7c0b37fa58efc Mon Sep 17 00:00:00 2001 From: Nicholas Shirley Date: Mon, 11 May 2026 10:01:20 -0600 Subject: [PATCH] feat(tests): manually run functional tests on pr Because: - We want to reduce the ci credit usage from running functional tests on every pr push This pull request: - Gates PR functional tests behind a ci parameter - Adds a script to trigger tests locally for a given branch - Adds a Claude skill to help run the tests as well Closes: FXA-13697 --- .circleci/config.yml | 11 ++- .../skills/fxa-run-functional-tests/SKILL.md | 44 +++++++++ _scripts/approve-functional-tests.sh | 92 +++++++++++++++++++ 3 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 .claude/skills/fxa-run-functional-tests/SKILL.md create mode 100755 _scripts/approve-functional-tests.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 6a91ca30a7e..16e8f3315fd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -984,11 +984,21 @@ workflows: workflow: test_pull_request requires: - Build (PR) + # Functional tests are heavy and expensive, so they're gated behind this + # manual approval. Approve the "Approve Functional Tests (PR)" job in + # the CircleCI UI, use the script under _scripts, or use the Claude skill + # to run them against the PR's already-built workspace. + - approve-functional-tests: + name: Approve Functional Tests (PR) + type: approval + requires: + - Build (PR) - playwright-functional-tests: name: Firefox Functional Tests - Playwright (PR) workflow: test_pull_request requires: - Build (PR) + - Approve Functional Tests (PR) - on-complete: name: Tests Complete (PR) stage: Tests @@ -1002,7 +1012,6 @@ workflows: - Integration Test - Servers - Auth (PR) - Integration Test - Servers - Auth Scripts (PR) - Integration Test - Libraries (PR) - - Firefox Functional Tests - Playwright (PR) # Triggered remotely. See .circleci/README.md production_smoke_tests: diff --git a/.claude/skills/fxa-run-functional-tests/SKILL.md b/.claude/skills/fxa-run-functional-tests/SKILL.md new file mode 100644 index 00000000000..c66175d660f --- /dev/null +++ b/.claude/skills/fxa-run-functional-tests/SKILL.md @@ -0,0 +1,44 @@ +--- +name: fxa-run-functional-tests +description: Approves the on-hold "Approve Functional Tests (PR)" CircleCI job for the current PR branch, kicking off the gated Playwright functional tests. Requires CIRCLECI_TOKEN in the environment. +allowed-tools: Bash +user-invocable: true +context: inherit +--- + +# Run Functional Tests + +Functional (Playwright) tests for PRs are gated in CircleCI behind a manual approval job (`Approve Functional Tests (PR)` in the `test_pull_request` workflow). This skill approves that job for the current branch's latest pipeline so the tests start running against the already-built workspace. + +## Steps + +1. Determine the current branch: + ```sh + git rev-parse --abbrev-ref HEAD + ``` + If the branch is `main` or `HEAD` (detached), stop and tell the user functional tests are only gated on PR branches. + +2. Confirm a CircleCI token is set in the environment — the script reads `CIRCLECI_TOKEN`, falling back to `CIRCLECI_CLI_TOKEN`: + ```sh + [[ -n "${CIRCLECI_TOKEN:-${CIRCLECI_CLI_TOKEN:-}}" ]] && echo set || echo missing + ``` + If missing, instruct the user to export a personal API token (project tokens don't work for v2 approvals): + > Get one at https://app.circleci.com/settings/user/tokens, then `export CIRCLECI_TOKEN=...` and retry. + +3. Run the approval script from the repo root: + ```sh + ./_scripts/approve-functional-tests.sh + ``` + +4. Report the result to the user: + - On success, share the pipeline URL printed by the script. + - If the script prints "Functional tests not on hold yet" (either "workflow not visible" or "approval job not visible"), relay that and suggest waiting a moment for CI to expand the workflow, then retrying. + - If the script prints "Build (PR) hasn't finished yet", the gating build is still running — suggest waiting for it before retrying. + - If the script reports the approval was already given, share the pipeline URL so the user can watch progress. + - If the script errors with a SHA mismatch (latest pipeline is for an older commit), relay that — the user likely just pushed and CircleCI hasn't created the new pipeline yet. Suggest retrying in a few seconds. + +## Notes + +- The script targets the most recent pipeline on the branch and refuses to approve it if its commit doesn't match the local branch tip — that prevents accidentally approving a stale pipeline when a newer commit has just been pushed. +- The approval becomes available as soon as Build (PR) completes and the workflow expands, but if you try to approve before that happens you'll get a "Functional tests not on hold" message. Just wait a moment for CI to catch up and try again. +- Stale runs are cancelled automatically by the project-level "auto-cancel redundant workflows" setting in CircleCI. diff --git a/_scripts/approve-functional-tests.sh b/_scripts/approve-functional-tests.sh new file mode 100755 index 00000000000..64debe71e50 --- /dev/null +++ b/_scripts/approve-functional-tests.sh @@ -0,0 +1,92 @@ +#! /bin/bash +# +# Approve the on-hold "Approve Functional Tests (PR)" job for the latest +# CircleCI pipeline on a PR branch. +# +# Usage: CIRCLECI_TOKEN= ./_scripts/approve-functional-tests.sh [branch] +# +# Personal API token required (project tokens don't work for v2 approvals): +# https://app.circleci.com/settings/user/tokens. Falls back to CIRCLECI_CLI_TOKEN. + +set -euo pipefail + +branch="${1:-$(git rev-parse --abbrev-ref HEAD)}" +if [[ "${branch}" == "main" || "${branch}" == "HEAD" ]]; then + echo "ERROR: refusing to approve on '${branch}'." >&2 + exit 1 +fi + +token="${CIRCLECI_TOKEN:-${CIRCLECI_CLI_TOKEN:-}}" +: "${token:?set CIRCLECI_TOKEN (https://app.circleci.com/settings/user/tokens)}" + +slug="github/mozilla/fxa" +api="https://circleci.com/api/v2" +auth=(-H "Circle-Token: ${token}") + +pipeline=$(curl -fsS "${auth[@]}" --get \ + --data-urlencode "branch=${branch}" \ + "${api}/project/${slug}/pipeline") + +pipeline_id=$(echo "${pipeline}" | jq -r '.items[0].id // empty') +pipeline_number=$(echo "${pipeline}" | jq -r '.items[0].number // empty') +pipeline_revision=$(echo "${pipeline}" | jq -r '.items[0].vcs.revision // empty') + +if [[ -z "${pipeline_id}" ]]; then + echo "ERROR: no pipelines on '${branch}'. Push a commit first." >&2 + exit 1 +fi + +# Refuse to approve a stale pipeline. When no branch arg is passed, compare +# against HEAD so worktrees with the branch checked out elsewhere see their +# own working state rather than the other worktree's. +if [[ -n "${1:-}" ]]; then + local_revision=$(git rev-parse "refs/heads/${branch}" 2>/dev/null || true) +else + local_revision=$(git rev-parse HEAD 2>/dev/null || true) +fi +if [[ -n "${local_revision}" && "${pipeline_revision}" != "${local_revision}" ]]; then + echo "ERROR: latest pipeline is for ${pipeline_revision:0:7}, branch is at ${local_revision:0:7}." >&2 + echo "Wait for CI to pick up the new commit and retry." >&2 + exit 1 +fi + +workflow_id=$(curl -fsS "${auth[@]}" "${api}/pipeline/${pipeline_id}/workflow" \ + | jq -r '[.items[] | select(.name == "test_pull_request")] | .[0].id // empty') + +if [[ -z "${workflow_id}" ]]; then + echo "Functional tests not on hold yet (workflow not visible). Try again soon." >&2 + exit 1 +fi + +job=$(curl -fsS "${auth[@]}" "${api}/workflow/${workflow_id}/job" \ + | jq -c '[.items[] | select(.name == "Approve Functional Tests (PR)")] | .[0] // empty') + +if [[ -z "${job}" ]]; then + echo "Functional tests not on hold yet (approval job not visible). Try again soon." >&2 + exit 1 +fi + +status=$(echo "${job}" | jq -r '.status') +case "${status}" in + on_hold) ;; + success) + echo "Already approved on this pipeline." + echo "https://app.circleci.com/pipelines/${slug}/${pipeline_number}" + exit 0 + ;; + blocked) + echo "Build (PR) hasn't finished yet — wait and retry." >&2 + exit 1 + ;; + *) + echo "Approval job status is '${status}', not on hold." >&2 + exit 1 + ;; +esac + +approval_id=$(echo "${job}" | jq -r '.approval_request_id') +curl -fsS -X POST "${auth[@]}" \ + "${api}/workflow/${workflow_id}/approve/${approval_id}" >/dev/null + +echo "Approved functional tests on '${branch}'." +echo "https://app.circleci.com/pipelines/${slug}/${pipeline_number}"