From 907bd4e45bc64672c5447b4c2330bc2a9b25d3ac Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Fri, 31 Jul 2026 08:44:56 -0700 Subject: [PATCH 1/2] ci: route merge-queue builds to self-hosted runner tiers with hosted fallback Merge-queue entries default to the self-hosted scale sets (arc-heavy/ medium/light/bench-linux); PR events opt in via the ci:self-hosted label; push and dispatch stay on GitHub-hosted runners. Before routing, precheck probes runner availability and queue congestion (optional ARC_STATUS_TOKEN secret) and falls back to ubuntu-latest per tier, so the pipeline never stalls when the self-hosted fleet is down. The ARC_AVAILABLE repo variable is a manual kill switch for the first jobs that cannot probe before queueing. Step conditions become runner- agnostic (runner.os == 'Linux') so matrix entries work on any Linux runner label. Co-Authored-By: Claude Fable 5 --- .github/workflows/backport-checks.yml | 7 +- .github/workflows/benchmarks.yml | 48 +++++++- .github/workflows/build.yml | 64 +++++++---- .github/workflows/check-header.yml | 7 +- .github/workflows/precheck.yml | 152 +++++++++++++++++++++++++- .github/workflows/required-checks.yml | 10 +- 6 files changed, 259 insertions(+), 29 deletions(-) diff --git a/.github/workflows/backport-checks.yml b/.github/workflows/backport-checks.yml index 7288928ef3d..ea12197f1fa 100644 --- a/.github/workflows/backport-checks.yml +++ b/.github/workflows/backport-checks.yml @@ -41,6 +41,9 @@ on: - unlabeled permissions: + # actions: read is required by the called precheck.yml (backlog probe); + # a reusable workflow's permissions must be a subset of its caller's. + actions: read checks: write contents: read pull-requests: read @@ -62,7 +65,9 @@ jobs: apply-check: needs: precheck if: ${{ needs.precheck.outputs.backport_targets != '[]' }} - runs-on: ubuntu-latest + # Control-plane resilience: follow precheck's probed routing (label + # present AND the scale set has online runners); ubuntu-latest otherwise. + runs-on: ${{ needs.precheck.outputs.light_runner == 'arc-light-linux' && 'arc-light-linux' || 'ubuntu-latest' }} outputs: buildable: ${{ steps.check.outputs.buildable }} steps: diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index b07c146f897..f564ae56688 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -88,6 +88,8 @@ on: workflow_dispatch: permissions: + # actions: read backs the queued-job backlog probe for runner routing. + actions: read contents: write concurrency: @@ -101,9 +103,15 @@ jobs: # run). Lifted from required-checks.yml's precheck so the trigger # surface matches amber-integration exactly. name: Precheck - runs-on: ubuntu-latest + # Control-plane resilience: with the ci:self-hosted label this job + # runs on arc-light-linux so a jammed GitHub-hosted pool cannot stall + # the pipeline. Availability cannot be probed before this job runs, so + # the repo variable ARC_AVAILABLE is the kill switch: set it to + # 'false' when the cluster is down to route back to ubuntu-latest. + runs-on: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:self-hosted') && vars.ARC_AVAILABLE != 'false') && 'arc-light-linux' || 'ubuntu-latest' }} outputs: run_bench: ${{ steps.decide.outputs.run_bench }} + linux_runner: ${{ steps.decide.outputs.linux_runner }} steps: - name: Wait for Pull Request Labeler if: github.event_name == 'pull_request' @@ -132,6 +140,10 @@ jobs: - name: Decide whether to run bench id: decide uses: actions/github-script@v9 + env: + # Optional PAT (administration:read) for the runner availability + # probe; see precheck.yml. + ARC_STATUS_TOKEN: ${{ secrets.ARC_STATUS_TOKEN }} with: script: | const eventName = context.eventName; @@ -139,6 +151,7 @@ jobs: // push to main / workflow_dispatch always run. core.info(`event=${eventName} — running unconditionally`); core.setOutput("run_bench", "true"); + core.setOutput("linux_runner", "ubuntu-latest"); return; } // Re-fetch labels: the labeler may have just added some. @@ -168,12 +181,43 @@ jobs: : "No trigger label present; skipping bench." ); core.setOutput("run_bench", shouldRun ? "true" : "false"); + // ci:self-hosted routes the bench onto the dedicated + // arc-bench-linux runner. Availability-only fallback: benchmark + // numbers must come from consistent hardware, so a busy bench + // runner means WAIT (queue), never drift to GitHub-hosted — + // only a set with no online runner at all falls back. + let linuxRunner = labels.includes("ci:self-hosted") ? "arc-bench-linux" : "ubuntu-latest"; + if (linuxRunner === "arc-bench-linux" && process.env.ARC_STATUS_TOKEN) { + try { + const res = await fetch( + `https://api.github.com/repos/${context.repo.owner}/${context.repo.repo}/actions/runners?per_page=100`, + { + headers: { + authorization: `Bearer ${process.env.ARC_STATUS_TOKEN}`, + accept: "application/vnd.github+json", + }, + } + ); + if (!res.ok) throw new Error(`runner list HTTP ${res.status}`); + const online = (await res.json()).runners?.some( + (r) => r.status === "online" && r.labels.some((l) => l.name === "arc-bench-linux") + ); + if (!online) { + linuxRunner = "ubuntu-latest"; + core.warning("arc-bench-linux has no online runner; falling back to ubuntu-latest."); + } + } catch (e) { + core.warning(`Runner availability probe failed (${e.message}); trusting the label.`); + } + } + core.setOutput("linux_runner", linuxRunner); + core.info(`Linux runner: ${linuxRunner}`); bench: name: Bench needs: precheck if: ${{ needs.precheck.outputs.run_bench == 'true' }} - runs-on: ubuntu-latest + runs-on: ${{ needs.precheck.outputs.linux_runner }} env: JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a82cbfde95b..dda503bdf86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,24 @@ on: required: false type: string default: "" + # Runner labels for Linux jobs, three tiers (precheck decides via the + # ci:self-hosted PR label; GitHub-hosted ubuntu-latest otherwise). + # heavy: amber / amber-integration / frontend (8C/16G class; the + # Angular prod build OOMs under the medium tier's 8Gi limit). + heavy_runner: + required: false + type: string + default: "ubuntu-latest" + # medium: platform / platform-integration / pyamber (4C/8G class). + medium_runner: + required: false + type: string + default: "ubuntu-latest" + # light: infra / agent-service / pyright (2C/4G class). + light_runner: + required: false + type: string + default: "ubuntu-latest" run_frontend: required: false type: boolean @@ -88,11 +106,11 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: ["${{ inputs.heavy_runner }}", windows-latest, macos-latest] include: - os: macos-latest arch: arm64 - - os: ubuntu-latest + - os: "${{ inputs.heavy_runner }}" arch: x64 - os: windows-latest arch: x64 @@ -133,12 +151,12 @@ jobs: - name: Prod build run: yarn --cwd frontend run build:ci - name: Check bundled npm packages against per-module LICENSE-binary files - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' run: ./bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} npm frontend/dist/3rdpartylicenses.json - name: Run frontend unit tests run: yarn --cwd frontend run test:ci - name: Upload frontend coverage to Codecov - if: matrix.os == 'ubuntu-latest' && always() + if: runner.os == 'Linux' && always() uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -149,7 +167,7 @@ jobs: # vitest.config.ts adds a `junit` reporter that writes to junit.xml # in the working dir; the @angular/build:unit-test runner forwards # vitest config through unchanged. - if: matrix.os == 'ubuntu-latest' && !cancelled() + if: runner.os == 'Linux' && !cancelled() uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -159,14 +177,14 @@ jobs: disable_search: true fail_ci_if_error: false - name: Install Playwright Chromium - run: yarn --cwd frontend playwright install ${{ matrix.os == 'ubuntu-latest' && '--with-deps' || '' }} chromium + run: yarn --cwd frontend playwright install ${{ runner.os == 'Linux' && '--with-deps' || '' }} chromium - name: Run frontend browser-mode tests run: yarn --cwd frontend ng run gui:test-browser - name: Upload frontend browser-mode test results to Codecov # vitest.browser.config.ts emits junit-browser.xml (distinct from # the unit-test report). Same `frontend` flag — Codecov merges # multi-file uploads under one flag. - if: matrix.os == 'ubuntu-latest' && !cancelled() + if: runner.os == 'Linux' && !cancelled() uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -185,7 +203,7 @@ jobs: if: ${{ inputs.run_amber }} strategy: matrix: - os: [ubuntu-latest] + os: ["${{ inputs.heavy_runner }}"] java-version: [17] runs-on: ${{ matrix.os }} env: @@ -359,7 +377,7 @@ jobs: # the docker image, macOS uses brew + the upstream # aarch64-apple-darwin lakekeeper tarball. matrix: - os: [ubuntu-latest, macos-latest] + os: ["${{ inputs.heavy_runner }}", macos-latest] java-version: [17] runs-on: ${{ matrix.os }} env: @@ -657,10 +675,10 @@ jobs: # `amber` and runs in parallel (as platform-integration does vs # platform); the sbt compile is already warm from the integration-test # run above. ubuntu-only: the boot is pure-JVM and OS-independent. - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' run: sbt "WorkflowExecutionService/dist" - name: Unzip amber dist - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' run: | mkdir -p /tmp/dists unzip -q amber/target/universal/amber-*.zip -d /tmp/dists/ @@ -671,7 +689,7 @@ jobs: # working directory for a directory named `amber`; run from the checkout # root (the default) that resolves to ./amber and reads # amber/src/main/resources/web-config.yml (port 8080). - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' env: # Quiet boot logs, same wiring as the test steps above. Safe here: # smoke-boot's verdict is LISTEN-based, never log-scraping (#6332). @@ -688,7 +706,7 @@ jobs: # so no iceberg / S3 access. Config resolves via Utils.amberHomePath to # amber/src/main/resources/computing-unit-master-config.yml (port 8085). # Reuses the amber dist built + unzipped above for the texera-web boot. - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' env: # Quiet boot logs, same wiring as the test steps above. Safe here: # smoke-boot's verdict is LISTEN-based, never log-scraping (#6332). @@ -724,7 +742,7 @@ jobs: # cover every module in the amber job above, so this matrix skips them. if: ${{ inputs.run_platform }} name: ${{ format('platform{0} ({1})', inputs.job_name_suffix, matrix.service) }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.medium_runner }} strategy: fail-fast: false matrix: @@ -856,7 +874,7 @@ jobs: # alone. Unit tests + coverage stay in `platform`. See #6273. if: ${{ inputs.run_platform_integration }} name: ${{ format('platform-integration{0} ({1})', inputs.job_name_suffix, matrix.service) }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.medium_runner }} strategy: fail-fast: false matrix: @@ -988,7 +1006,7 @@ jobs: if: ${{ inputs.run_pyamber }} strategy: matrix: - os: [ubuntu-latest] + os: ["${{ inputs.medium_runner }}"] python-version: ["3.11", "3.12", "3.13"] runs-on: ${{ matrix.os }} services: @@ -1108,7 +1126,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: ["${{ inputs.light_runner }}", macos-latest] bun-version: ["1.3.3"] defaults: run: @@ -1130,12 +1148,12 @@ jobs: - name: Install production dependencies run: bun install --production --frozen-lockfile - name: Generate agent-service license manifest - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' run: | mkdir -p dist bun run bin/collect-licenses.ts > dist/3rdpartylicenses.json - name: Check bundled agent-service packages against per-module LICENSE-binary files - if: matrix.os == 'ubuntu-latest' + if: runner.os == 'Linux' run: ../bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} agent-npm dist/3rdpartylicenses.json - name: Install development dependencies run: bun install --frozen-lockfile @@ -1155,7 +1173,7 @@ jobs: TEXERA_SERVICE_LOG_LEVEL: ${{ runner.debug == '1' && 'DEBUG' || 'WARN' }} run: bun test --coverage --coverage-reporter=lcov --reporter=junit --reporter-outfile=junit.xml - name: Upload agent-service coverage to Codecov - if: matrix.os == 'ubuntu-latest' && always() + if: runner.os == 'Linux' && always() uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -1165,7 +1183,7 @@ jobs: - name: Upload agent-service test results to Codecov # Test Analytics ingestion. Runs on the same ubuntu leg that # uploads coverage. `!cancelled()` so test failures still upload. - if: matrix.os == 'ubuntu-latest' && !cancelled() + if: runner.os == 'Linux' && !cancelled() uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -1190,7 +1208,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: ["${{ inputs.light_runner }}", macos-latest] python-version: ["3.12"] steps: - name: Checkout Texera @@ -1249,7 +1267,7 @@ jobs: # stays a fast, docker-free job. if: ${{ inputs.run_pyright_language_service }} name: ${{ format('pyright-language-service{0}', inputs.job_name_suffix) }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.light_runner }} defaults: run: working-directory: pyright-language-service diff --git a/.github/workflows/check-header.yml b/.github/workflows/check-header.yml index 003747c6e1c..3d6c9b2fb91 100644 --- a/.github/workflows/check-header.yml +++ b/.github/workflows/check-header.yml @@ -27,7 +27,12 @@ on: jobs: test: name: Check License Headers - runs-on: ubuntu-latest + # Control-plane resilience: merge-queue entries (and labeled PRs) run + # this job on arc-light-linux so a jammed GitHub-hosted pool cannot + # stall the pipeline. Availability cannot be probed before this job runs, so + # the repo variable ARC_AVAILABLE is the kill switch: set it to + # 'false' when the cluster is down to route back to ubuntu-latest. + runs-on: ${{ ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:self-hosted'))) && vars.ARC_AVAILABLE != 'false') && 'arc-light-linux' || 'ubuntu-latest' }} steps: - uses: actions/checkout@v7 - uses: apache/skywalking-eyes@61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1 # v0.8.0 diff --git a/.github/workflows/precheck.yml b/.github/workflows/precheck.yml index 4ce1d240923..febbb3cd4a8 100644 --- a/.github/workflows/precheck.yml +++ b/.github/workflows/precheck.yml @@ -50,8 +50,19 @@ on: value: ${{ jobs.decide.outputs.run_pyright_language_service }} backport_targets: value: ${{ jobs.decide.outputs.backport_targets }} + # Runner labels for Linux build jobs, three tiers: the self-hosted + # scale sets when the PR carries the ci:self-hosted label, + # GitHub-hosted ubuntu-latest otherwise. + heavy_runner: + value: ${{ jobs.decide.outputs.heavy_runner }} + medium_runner: + value: ${{ jobs.decide.outputs.medium_runner }} + light_runner: + value: ${{ jobs.decide.outputs.light_runner }} permissions: + # actions: read backs the queued-job backlog probe for runner routing. + actions: read checks: read # contents: read backs the git-data commit lookups (tree SHAs) in the # merge-queue fast path. @@ -61,7 +72,12 @@ permissions: jobs: decide: name: Precheck - runs-on: ubuntu-latest + # Control-plane resilience: merge-queue entries (and labeled PRs) run + # this job on arc-light-linux so a jammed GitHub-hosted pool cannot + # stall the pipeline. Availability cannot be probed before this job runs, so + # the repo variable ARC_AVAILABLE is the kill switch: set it to + # 'false' when the cluster is down to route back to ubuntu-latest. + runs-on: ${{ ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:self-hosted'))) && vars.ARC_AVAILABLE != 'false') && 'arc-light-linux' || 'ubuntu-latest' }} outputs: run_frontend: ${{ steps.decide.outputs.run_frontend }} run_amber: ${{ steps.decide.outputs.run_amber }} @@ -73,6 +89,9 @@ jobs: run_infra: ${{ steps.decide.outputs.run_infra }} run_pyright_language_service: ${{ steps.decide.outputs.run_pyright_language_service }} backport_targets: ${{ steps.decide.outputs.backport_targets }} + heavy_runner: ${{ steps.decide.outputs.heavy_runner }} + medium_runner: ${{ steps.decide.outputs.medium_runner }} + light_runner: ${{ steps.decide.outputs.light_runner }} steps: - name: Wait for Pull Request Labeler if: github.event_name == 'pull_request' @@ -101,6 +120,11 @@ jobs: - name: Decide which jobs to run id: decide uses: actions/github-script@v9 + env: + # Fine-grained PAT with administration:read on this repo; used + # only to probe self-hosted runner availability. Optional — when + # absent the ci:self-hosted label is trusted without probing. + ARC_STATUS_TOKEN: ${{ secrets.ARC_STATUS_TOKEN }} with: script: | const eventName = context.eventName; @@ -314,6 +338,132 @@ jobs: runPyrightLanguageService = false; } + // Runner tiers: merge queue entries default to the + // self-hosted scale sets (the queue is the latency-critical + // serialized path most hurt by hosted-pool congestion); + // PR events opt in via the ci:self-hosted label. Everything + // else (push / dispatch) stays GitHub-hosted. Tiers: + // arc-heavy-linux for the amber/sbt stacks, arc-medium-linux + // for platform and pyamber, arc-light-linux for the small + // service jobs. + // + // Fallback probing, two levels, evaluated per tier: + // + // Level 1 (availability): a set with no online runner (cluster + // down, scale set removed) falls back to GitHub-hosted so a + // labeled PR never queues forever. Relies on each set keeping + // warm runners (minRunners > 0) — an idle scale-set with + // minRunners: 0 registers no runners and would look + // unavailable. + // + // Level 2 (congestion): predict queueing before it happens by + // comparing this round's demand (the stack decisions above + // tell us how many jobs each tier is about to receive) + // against idle supply (runner busy flags) plus the backlog + // of already-queued jobs waiting for the same label. Blind + // spot: k8s-side capacity (Pending pods, scale-up headroom) + // is invisible to the GitHub API. + // + // Runner listing needs ARC_STATUS_TOKEN (administration:read); + // without it the label is trusted as-is. Any probe error + // degrades to trusting the label. + const selfHosted = eventName === "merge_group" || labels.includes("ci:self-hosted"); + const tiers = { + heavy_runner: selfHosted ? "arc-heavy-linux" : "ubuntu-latest", + medium_runner: selfHosted ? "arc-medium-linux" : "ubuntu-latest", + light_runner: selfHosted ? "arc-light-linux" : "ubuntu-latest", + }; + // Per-tier job counts this round. PLATFORM_SERVICES mirrors the + // service matrix size in build.yml — keep in sync. + const PLATFORM_SERVICES = 6; + const demand = { + "arc-heavy-linux": + (runAmber ? 1 : 0) + (runAmberIntegration ? 1 : 0) + (runFrontend ? 1 : 0), + "arc-medium-linux": + (runPlatform ? PLATFORM_SERVICES : 0) + + (runPlatformIntegration ? PLATFORM_SERVICES : 0) + + (runPyamber ? 3 : 0), + "arc-light-linux": + (runInfra ? 1 : 0) + + (runAgentService ? 1 : 0) + + (runPyrightLanguageService ? 1 : 0), + }; + if (selfHosted && process.env.ARC_STATUS_TOKEN) { + try { + const res = await fetch( + `https://api.github.com/repos/${context.repo.owner}/${context.repo.repo}/actions/runners?per_page=100`, + { + headers: { + authorization: `Bearer ${process.env.ARC_STATUS_TOKEN}`, + accept: "application/vnd.github+json", + }, + } + ); + if (!res.ok) throw new Error(`runner list HTTP ${res.status}`); + const online = {}; + const idle = {}; + for (const r of (await res.json()).runners ?? []) { + if (r.status !== "online") continue; + for (const l of r.labels) { + online[l.name] = (online[l.name] ?? 0) + 1; + if (!r.busy) idle[l.name] = (idle[l.name] ?? 0) + 1; + } + } + // Backlog: queued jobs per label across currently queued + // runs (capped sample; uses the workflow GITHUB_TOKEN). + const backlog = {}; + try { + const { data: queued } = await github.rest.actions.listWorkflowRunsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + status: "queued", + per_page: 10, + }); + for (const run of queued.workflow_runs ?? []) { + const { data: jobsData } = await github.rest.actions.listJobsForWorkflowRun({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: run.id, + per_page: 100, + }); + for (const job of jobsData.jobs ?? []) { + if (job.status !== "queued") continue; + for (const l of job.labels) backlog[l] = (backlog[l] ?? 0) + 1; + } + } + } catch (e) { + core.warning(`Backlog probe failed (${e.message}); assuming no backlog.`); + } + for (const [output, scaleSet] of Object.entries({ ...tiers })) { + if (scaleSet === "ubuntu-latest") continue; + const nOnline = online[scaleSet] ?? 0; + const nIdle = idle[scaleSet] ?? 0; + const nBacklog = backlog[scaleSet] ?? 0; + const nDemand = demand[scaleSet] ?? 0; + let reason = null; + if (nOnline === 0) { + reason = "no online runner"; + } else if (nIdle === 0 && nBacklog >= 2) { + reason = `all ${nOnline} runners busy with ${nBacklog} jobs already queued`; + } else if (nDemand > nIdle + 2) { + reason = `demand ${nDemand} exceeds idle ${nIdle} + slack`; + } + if (reason) { + tiers[output] = "ubuntu-latest"; + core.warning(`${scaleSet}: ${reason}; falling back to ubuntu-latest.`); + } else { + core.info(`${scaleSet}: online=${nOnline} idle=${nIdle} backlog=${nBacklog} demand=${nDemand} — ok`); + } + } + } catch (e) { + core.warning(`Runner availability probe failed (${e.message}); trusting the label.`); + } + } else if (selfHosted) { + core.info("ARC_STATUS_TOKEN not set; trusting the ci:self-hosted label without probing."); + } + for (const [output, runner] of Object.entries(tiers)) core.setOutput(output, runner); + core.info(`Linux runners: heavy=${tiers.heavy_runner} medium=${tiers.medium_runner} light=${tiers.light_runner}`); + core.setOutput("run_frontend", runFrontend ? "true" : "false"); core.setOutput("run_amber", runAmber ? "true" : "false"); core.setOutput("run_amber_integration", runAmberIntegration ? "true" : "false"); diff --git a/.github/workflows/required-checks.yml b/.github/workflows/required-checks.yml index 1db8a52668e..7a1edd2160e 100644 --- a/.github/workflows/required-checks.yml +++ b/.github/workflows/required-checks.yml @@ -33,6 +33,9 @@ on: workflow_dispatch: permissions: + # actions: read is required by the called precheck.yml (backlog probe); + # a reusable workflow's permissions must be a subset of its caller's. + actions: read checks: write contents: read pull-requests: read @@ -56,6 +59,9 @@ jobs: needs: precheck uses: ./.github/workflows/build.yml with: + heavy_runner: ${{ needs.precheck.outputs.heavy_runner }} + medium_runner: ${{ needs.precheck.outputs.medium_runner }} + light_runner: ${{ needs.precheck.outputs.light_runner }} run_frontend: ${{ needs.precheck.outputs.run_frontend == 'true' }} run_amber: ${{ needs.precheck.outputs.run_amber == 'true' }} run_amber_integration: ${{ needs.precheck.outputs.run_amber_integration == 'true' }} @@ -72,7 +78,9 @@ jobs: name: Required Checks needs: [precheck, build] if: always() - runs-on: ubuntu-latest + # Control-plane resilience: follow precheck's probed routing (label + # present AND the scale set has online runners); ubuntu-latest otherwise. + runs-on: ${{ needs.precheck.outputs.light_runner == 'arc-light-linux' && 'arc-light-linux' || 'ubuntu-latest' }} steps: - name: Verify all required checks succeeded or were skipped run: | From a11dc3890d144ed62bc71c851d6a6604075590f5 Mon Sep 17 00:00:00 2001 From: Yicong Huang <17627829+Yicong-Huang@users.noreply.github.com> Date: Fri, 31 Jul 2026 09:52:44 -0700 Subject: [PATCH 2/2] ci: heartbeat-driven fleet health with automatic hosted fallback runner-heartbeat.yml runs a trivial job on the fleet every 15 minutes. precheck consults the last successful heartbeat (plain GITHUB_TOKEN) and the ARC_AVAILABLE variable, which the heartbeat watchdog flips automatically (GITHUB_TOKEN cannot write variables, so the watchdog uses AUTO_MERGE_TOKEN and degrades to log-only without it). Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmarks.yml | 6 ++ .github/workflows/precheck.yml | 40 +++++++++ .github/workflows/runner-heartbeat.yml | 111 +++++++++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 .github/workflows/runner-heartbeat.yml diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index f564ae56688..e81d21bb516 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -144,6 +144,8 @@ jobs: # Optional PAT (administration:read) for the runner availability # probe; see precheck.yml. ARC_STATUS_TOKEN: ${{ secrets.ARC_STATUS_TOKEN }} + # Kill switch mirrored by runner-heartbeat.yml's watchdog. + ARC_AVAILABLE: ${{ vars.ARC_AVAILABLE }} with: script: | const eventName = context.eventName; @@ -187,6 +189,10 @@ jobs: // runner means WAIT (queue), never drift to GitHub-hosted — // only a set with no online runner at all falls back. let linuxRunner = labels.includes("ci:self-hosted") ? "arc-bench-linux" : "ubuntu-latest"; + if (linuxRunner === "arc-bench-linux" && process.env.ARC_AVAILABLE === "false") { + linuxRunner = "ubuntu-latest"; + core.warning("ARC_AVAILABLE=false; bench falls back to ubuntu-latest."); + } if (linuxRunner === "arc-bench-linux" && process.env.ARC_STATUS_TOKEN) { try { const res = await fetch( diff --git a/.github/workflows/precheck.yml b/.github/workflows/precheck.yml index febbb3cd4a8..fa971601ac9 100644 --- a/.github/workflows/precheck.yml +++ b/.github/workflows/precheck.yml @@ -125,6 +125,9 @@ jobs: # only to probe self-hosted runner availability. Optional — when # absent the ci:self-hosted label is trusted without probing. ARC_STATUS_TOKEN: ${{ secrets.ARC_STATUS_TOKEN }} + # Manual/auto kill switch mirrored by runner-heartbeat.yml's + # watchdog; 'false' forces every Linux tier back to GitHub-hosted. + ARC_AVAILABLE: ${{ vars.ARC_AVAILABLE }} with: script: | const eventName = context.eventName; @@ -388,6 +391,43 @@ jobs: (runAgentService ? 1 : 0) + (runPyrightLanguageService ? 1 : 0), }; + // Kill switch: ARC_AVAILABLE=false (manual, or flipped by the + // runner-heartbeat watchdog) forces every tier to GitHub-hosted. + if (selfHosted && process.env.ARC_AVAILABLE === "false") { + for (const k of Object.keys(tiers)) tiers[k] = "ubuntu-latest"; + core.warning("ARC_AVAILABLE=false; routing all Linux tiers to GitHub-hosted runners."); + } + // Heartbeat fallback: runner-heartbeat.yml proves fleet liveness + // every 15 minutes. Stale last-success (with history present) + // means the fleet is down — fall back without needing any extra + // credential (plain GITHUB_TOKEN, actions: read). + if (selfHosted && tiers.light_runner !== "ubuntu-latest") { + try { + const STALE_MS = 35 * 60 * 1000; + const ok = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "runner-heartbeat.yml", + status: "success", + per_page: 1, + }); + const any = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "runner-heartbeat.yml", + per_page: 1, + }); + const hasHistory = (any.data.workflow_runs ?? []).length > 0; + const last = ok.data.workflow_runs?.[0]; + const lastOk = last ? new Date(last.updated_at).getTime() : 0; + if (hasHistory && Date.now() - lastOk > STALE_MS) { + for (const k of Object.keys(tiers)) tiers[k] = "ubuntu-latest"; + core.warning("Runner heartbeat is stale; routing all Linux tiers to GitHub-hosted runners."); + } + } catch (e) { + core.info(`Heartbeat check skipped (${e.message}).`); + } + } if (selfHosted && process.env.ARC_STATUS_TOKEN) { try { const res = await fetch( diff --git a/.github/workflows/runner-heartbeat.yml b/.github/workflows/runner-heartbeat.yml new file mode 100644 index 00000000000..6269163440e --- /dev/null +++ b/.github/workflows/runner-heartbeat.yml @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Liveness probe for the self-hosted runner fleet. `beat` runs a trivial +# job on the fleet every 15 minutes; a stale last-success means the fleet +# is down. Two consumers act on that signal: +# - precheck.yml queries the last successful heartbeat directly (plain +# GITHUB_TOKEN) and falls back to GitHub-hosted runners for the build +# tiers when it is stale. +# - `watchdog` mirrors the verdict into the ARC_AVAILABLE repository +# variable, which the first-hop runs-on expressions (unable to call +# APIs) consult. The workflow GITHUB_TOKEN cannot write variables +# ("Resource not accessible by integration"), so watchdog uses +# AUTO_MERGE_TOKEN (collaborator suffices for the Variables API) and +# degrades to log-only when the secret is absent. +name: Runner Heartbeat + +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + +permissions: + actions: read + +concurrency: + group: runner-heartbeat + # A beat stuck queueing on a dead fleet is superseded by the next tick + # instead of piling up. + cancel-in-progress: true + +jobs: + beat: + runs-on: arc-light-linux + timeout-minutes: 5 + steps: + - name: Prove the fleet is alive + run: echo "self-hosted fleet alive at $(date -u +%FT%TZ) on $RUNNER_NAME" + + watchdog: + runs-on: ubuntu-slim + timeout-minutes: 5 + steps: + - name: Mirror fleet health into ARC_AVAILABLE + uses: actions/github-script@v9 + env: + VAR_TOKEN: ${{ secrets.AUTO_MERGE_TOKEN }} + with: + script: | + const STALE_MS = 35 * 60 * 1000; + const { data } = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: "runner-heartbeat.yml", + status: "success", + per_page: 1, + }); + const last = data.workflow_runs?.[0]; + const lastOk = last ? new Date(last.updated_at).getTime() : 0; + const healthy = Date.now() - lastOk <= STALE_MS; + core.info( + `fleet ${healthy ? "healthy" : "STALE"}; last successful heartbeat: ` + + (lastOk ? new Date(lastOk).toISOString() : "never") + ); + if (!process.env.VAR_TOKEN) { + core.warning("AUTO_MERGE_TOKEN not set; cannot update ARC_AVAILABLE (log-only)."); + return; + } + const desired = healthy ? "true" : "false"; + const headers = { + authorization: `Bearer ${process.env.VAR_TOKEN}`, + accept: "application/vnd.github+json", + }; + const base = `https://api.github.com/repos/${context.repo.owner}/${context.repo.repo}/actions/variables`; + const cur = await fetch(`${base}/ARC_AVAILABLE`, { headers }); + if (cur.status === 404) { + const res = await fetch(base, { + method: "POST", + headers, + body: JSON.stringify({ name: "ARC_AVAILABLE", value: desired }), + }); + if (!res.ok) throw new Error(`create ARC_AVAILABLE: HTTP ${res.status}`); + core.notice(`ARC_AVAILABLE created as ${desired}`); + } else if (cur.ok) { + const { value } = await cur.json(); + if (value !== desired) { + const res = await fetch(`${base}/ARC_AVAILABLE`, { + method: "PATCH", + headers, + body: JSON.stringify({ name: "ARC_AVAILABLE", value: desired }), + }); + if (!res.ok) throw new Error(`update ARC_AVAILABLE: HTTP ${res.status}`); + core.notice(`ARC_AVAILABLE flipped to ${desired}`); + } + } else { + throw new Error(`read ARC_AVAILABLE: HTTP ${cur.status}`); + }