ci: route merge-queue builds to self-hosted runner tiers with hosted fallback

.github/workflows/backport-checks.yml

@@ -41,6 +41,9 @@ on:
       - unlabeled
 
 permissions:
+  # actions: read is required by the called precheck.yml (backlog probe);
+  # a reusable workflow's permissions must be a subset of its caller's.
+  actions: read
   checks: write
   contents: read
   pull-requests: read
@@ -62,7 +65,9 @@ jobs:
   apply-check:
     needs: precheck
     if: ${{ needs.precheck.outputs.backport_targets != '[]' }}
-    runs-on: ubuntu-latest
+    # Control-plane resilience: follow precheck's probed routing (label
+    # present AND the scale set has online runners); ubuntu-latest otherwise.
+    runs-on: ${{ needs.precheck.outputs.light_runner == 'arc-light-linux' && 'arc-light-linux' || 'ubuntu-latest' }}
     outputs:
       buildable: ${{ steps.check.outputs.buildable }}
     steps:

.github/workflows/benchmarks.yml

@@ -88,6 +88,8 @@ on:
   workflow_dispatch:
 
 permissions:
+  # actions: read backs the queued-job backlog probe for runner routing.
+  actions: read
   contents: write
 
 concurrency:
@@ -101,9 +103,15 @@ jobs:
     # run). Lifted from required-checks.yml's precheck so the trigger
     # surface matches amber-integration exactly.
     name: Precheck
-    runs-on: ubuntu-latest
+    # Control-plane resilience: with the ci:self-hosted label this job
+    # runs on arc-light-linux so a jammed GitHub-hosted pool cannot stall
+    # the pipeline. Availability cannot be probed before this job runs, so
+    # the repo variable ARC_AVAILABLE is the kill switch: set it to
+    # 'false' when the cluster is down to route back to ubuntu-latest.
+    runs-on: ${{ (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:self-hosted') && vars.ARC_AVAILABLE != 'false') && 'arc-light-linux' || 'ubuntu-latest' }}
     outputs:
       run_bench: ${{ steps.decide.outputs.run_bench }}
+      linux_runner: ${{ steps.decide.outputs.linux_runner }}
     steps:
       - name: Wait for Pull Request Labeler
         if: github.event_name == 'pull_request'
@@ -132,13 +140,20 @@ jobs:
       - name: Decide whether to run bench
         id: decide
         uses: actions/github-script@v9
+        env:
+          # Optional PAT (administration:read) for the runner availability
+          # probe; see precheck.yml.
+          ARC_STATUS_TOKEN: ${{ secrets.ARC_STATUS_TOKEN }}
+          # Kill switch mirrored by runner-heartbeat.yml's watchdog.
+          ARC_AVAILABLE: ${{ vars.ARC_AVAILABLE }}
         with:
           script: |
             const eventName = context.eventName;
             if (eventName !== "pull_request") {
               // push to main / workflow_dispatch always run.
               core.info(`event=${eventName} — running unconditionally`);
               core.setOutput("run_bench", "true");
+              core.setOutput("linux_runner", "ubuntu-latest");
               return;
             }
             // Re-fetch labels: the labeler may have just added some.
@@ -168,12 +183,47 @@ jobs:
                 : "No trigger label present; skipping bench."
             );
             core.setOutput("run_bench", shouldRun ? "true" : "false");
+            // ci:self-hosted routes the bench onto the dedicated
+            // arc-bench-linux runner. Availability-only fallback: benchmark
+            // numbers must come from consistent hardware, so a busy bench
+            // runner means WAIT (queue), never drift to GitHub-hosted —
+            // only a set with no online runner at all falls back.
+            let linuxRunner = labels.includes("ci:self-hosted") ? "arc-bench-linux" : "ubuntu-latest";
+            if (linuxRunner === "arc-bench-linux" && process.env.ARC_AVAILABLE === "false") {
+              linuxRunner = "ubuntu-latest";
+              core.warning("ARC_AVAILABLE=false; bench falls back to ubuntu-latest.");
+            }
+            if (linuxRunner === "arc-bench-linux" && process.env.ARC_STATUS_TOKEN) {
+              try {
+                const res = await fetch(
+                  `https://api.github.com/repos/${context.repo.owner}/${context.repo.repo}/actions/runners?per_page=100`,
+                  {
+                    headers: {
+                      authorization: `Bearer ${process.env.ARC_STATUS_TOKEN}`,
+                      accept: "application/vnd.github+json",
+                    },
+                  }
+                );
+                if (!res.ok) throw new Error(`runner list HTTP ${res.status}`);
+                const online = (await res.json()).runners?.some(
+                  (r) => r.status === "online" && r.labels.some((l) => l.name === "arc-bench-linux")
+                );
+                if (!online) {
+                  linuxRunner = "ubuntu-latest";
+                  core.warning("arc-bench-linux has no online runner; falling back to ubuntu-latest.");
+                }
+              } catch (e) {
+                core.warning(`Runner availability probe failed (${e.message}); trusting the label.`);
+              }
+            }
+            core.setOutput("linux_runner", linuxRunner);
+            core.info(`Linux runner: ${linuxRunner}`);
 
   bench:
     name: Bench
     needs: precheck
     if: ${{ needs.precheck.outputs.run_bench == 'true' }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ needs.precheck.outputs.linux_runner }}
     env:
       JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
       JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8

.github/workflows/build.yml

@@ -36,6 +36,24 @@ on:
         required: false
         type: string
         default: ""
+      # Runner labels for Linux jobs, three tiers (precheck decides via the
+      # ci:self-hosted PR label; GitHub-hosted ubuntu-latest otherwise).
+      # heavy: amber / amber-integration / frontend (8C/16G class; the
+      # Angular prod build OOMs under the medium tier's 8Gi limit).
+      heavy_runner:
+        required: false
+        type: string
+        default: "ubuntu-latest"
+      # medium: platform / platform-integration / pyamber (4C/8G class).
+      medium_runner:
+        required: false
+        type: string
+        default: "ubuntu-latest"
+      # light: infra / agent-service / pyright (2C/4G class).
+      light_runner:
+        required: false
+        type: string
+        default: "ubuntu-latest"
       run_frontend:
         required: false
         type: boolean
@@ -88,11 +106,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: ["${{ inputs.heavy_runner }}", windows-latest, macos-latest]
         include:
           - os: macos-latest
             arch: arm64
-          - os: ubuntu-latest
+          - os: "${{ inputs.heavy_runner }}"
             arch: x64
           - os: windows-latest
             arch: x64
@@ -133,12 +151,12 @@ jobs:
       - name: Prod build
         run: yarn --cwd frontend run build:ci
       - name: Check bundled npm packages against per-module LICENSE-binary files
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         run: ./bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} npm frontend/dist/3rdpartylicenses.json
       - name: Run frontend unit tests
         run: yarn --cwd frontend run test:ci
       - name: Upload frontend coverage to Codecov
-        if: matrix.os == 'ubuntu-latest' && always()
+        if: runner.os == 'Linux' && always()
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -149,7 +167,7 @@ jobs:
         # vitest.config.ts adds a `junit` reporter that writes to junit.xml
         # in the working dir; the @angular/build:unit-test runner forwards
         # vitest config through unchanged.
-        if: matrix.os == 'ubuntu-latest' && !cancelled()
+        if: runner.os == 'Linux' && !cancelled()
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -159,14 +177,14 @@ jobs:
           disable_search: true
           fail_ci_if_error: false
       - name: Install Playwright Chromium
-        run: yarn --cwd frontend playwright install ${{ matrix.os == 'ubuntu-latest' && '--with-deps' || '' }} chromium
+        run: yarn --cwd frontend playwright install ${{ runner.os == 'Linux' && '--with-deps' || '' }} chromium
       - name: Run frontend browser-mode tests
         run: yarn --cwd frontend ng run gui:test-browser
       - name: Upload frontend browser-mode test results to Codecov
         # vitest.browser.config.ts emits junit-browser.xml (distinct from
         # the unit-test report). Same `frontend` flag — Codecov merges
         # multi-file uploads under one flag.
-        if: matrix.os == 'ubuntu-latest' && !cancelled()
+        if: runner.os == 'Linux' && !cancelled()
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -185,7 +203,7 @@ jobs:
     if: ${{ inputs.run_amber }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: ["${{ inputs.heavy_runner }}"]
         java-version: [17]
     runs-on: ${{ matrix.os }}
     env:
@@ -359,7 +377,7 @@ jobs:
       # the docker image, macOS uses brew + the upstream
       # aarch64-apple-darwin lakekeeper tarball.
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: ["${{ inputs.heavy_runner }}", macos-latest]
         java-version: [17]
     runs-on: ${{ matrix.os }}
     env:
@@ -657,10 +675,10 @@ jobs:
         # `amber` and runs in parallel (as platform-integration does vs
         # platform); the sbt compile is already warm from the integration-test
         # run above. ubuntu-only: the boot is pure-JVM and OS-independent.
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         run: sbt "WorkflowExecutionService/dist"
       - name: Unzip amber dist
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         run: |
           mkdir -p /tmp/dists
           unzip -q amber/target/universal/amber-*.zip -d /tmp/dists/
@@ -671,7 +689,7 @@ jobs:
         # working directory for a directory named `amber`; run from the checkout
         # root (the default) that resolves to ./amber and reads
         # amber/src/main/resources/web-config.yml (port 8080).
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         env:
           # Quiet boot logs, same wiring as the test steps above. Safe here:
           # smoke-boot's verdict is LISTEN-based, never log-scraping (#6332).
@@ -688,7 +706,7 @@ jobs:
         # so no iceberg / S3 access. Config resolves via Utils.amberHomePath to
         # amber/src/main/resources/computing-unit-master-config.yml (port 8085).
         # Reuses the amber dist built + unzipped above for the texera-web boot.
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         env:
           # Quiet boot logs, same wiring as the test steps above. Safe here:
           # smoke-boot's verdict is LISTEN-based, never log-scraping (#6332).
@@ -724,7 +742,7 @@ jobs:
     # cover every module in the amber job above, so this matrix skips them.
     if: ${{ inputs.run_platform }}
     name: ${{ format('platform{0} ({1})', inputs.job_name_suffix, matrix.service) }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.medium_runner }}
     strategy:
       fail-fast: false
       matrix:
@@ -856,7 +874,7 @@ jobs:
     # alone. Unit tests + coverage stay in `platform`. See #6273.
     if: ${{ inputs.run_platform_integration }}
     name: ${{ format('platform-integration{0} ({1})', inputs.job_name_suffix, matrix.service) }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.medium_runner }}
     strategy:
       fail-fast: false
       matrix:
@@ -988,7 +1006,7 @@ jobs:
     if: ${{ inputs.run_pyamber }}
     strategy:
       matrix:
-        os: [ubuntu-latest]
+        os: ["${{ inputs.medium_runner }}"]
         python-version: ["3.11", "3.12", "3.13"]
     runs-on: ${{ matrix.os }}
     services:
@@ -1108,7 +1126,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: ["${{ inputs.light_runner }}", macos-latest]
         bun-version: ["1.3.3"]
     defaults:
       run:
@@ -1130,12 +1148,12 @@ jobs:
       - name: Install production dependencies
         run: bun install --production --frozen-lockfile
       - name: Generate agent-service license manifest
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         run: |
           mkdir -p dist
           bun run bin/collect-licenses.ts > dist/3rdpartylicenses.json
       - name: Check bundled agent-service packages against per-module LICENSE-binary files
-        if: matrix.os == 'ubuntu-latest'
+        if: runner.os == 'Linux'
         run: ../bin/licensing/check_binary_deps.py ${{ inputs.mode == 'PR' && '--ignore-transitive-version' || '' }} agent-npm dist/3rdpartylicenses.json
       - name: Install development dependencies
         run: bun install --frozen-lockfile
@@ -1155,7 +1173,7 @@ jobs:
           TEXERA_SERVICE_LOG_LEVEL: ${{ runner.debug == '1' && 'DEBUG' || 'WARN' }}
         run: bun test --coverage --coverage-reporter=lcov --reporter=junit --reporter-outfile=junit.xml
       - name: Upload agent-service coverage to Codecov
-        if: matrix.os == 'ubuntu-latest' && always()
+        if: runner.os == 'Linux' && always()
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -1165,7 +1183,7 @@ jobs:
       - name: Upload agent-service test results to Codecov
         # Test Analytics ingestion. Runs on the same ubuntu leg that
         # uploads coverage. `!cancelled()` so test failures still upload.
-        if: matrix.os == 'ubuntu-latest' && !cancelled()
+        if: runner.os == 'Linux' && !cancelled()
         uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
@@ -1190,7 +1208,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os: ["${{ inputs.light_runner }}", macos-latest]
         python-version: ["3.12"]
     steps:
       - name: Checkout Texera
@@ -1249,7 +1267,7 @@ jobs:
     # stays a fast, docker-free job.
     if: ${{ inputs.run_pyright_language_service }}
     name: ${{ format('pyright-language-service{0}', inputs.job_name_suffix) }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.light_runner }}
     defaults:
       run:
         working-directory: pyright-language-service

.github/workflows/check-header.yml

@@ -27,7 +27,12 @@ on:
 jobs:
   test:
     name: Check License Headers
-    runs-on: ubuntu-latest
+    # Control-plane resilience: merge-queue entries (and labeled PRs) run
+    # this job on arc-light-linux so a jammed GitHub-hosted pool cannot
+    # stall the pipeline. Availability cannot be probed before this job runs, so
+    # the repo variable ARC_AVAILABLE is the kill switch: set it to
+    # 'false' when the cluster is down to route back to ubuntu-latest.
+    runs-on: ${{ ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'ci:self-hosted'))) && vars.ARC_AVAILABLE != 'false') && 'arc-light-linux' || 'ubuntu-latest' }}
     steps:
       - uses: actions/checkout@v7
       - uses: apache/skywalking-eyes@61275cc80d0798a405cb070f7d3a8aaf7cf2c2c1 # v0.8.0