ci: add focused PR benchmark labels

.github/workflows/commit-metadata.yml

@@ -22,13 +22,13 @@ jobs:
         with:
           fetch-depth: 2
 
-      # v4 (Postgres) ingest -- REQUIRED (see bench.yml rationale). Empty records:
+      # v4 (Postgres) ingest -- REQUIRED (see develop-bench.yml rationale). Empty records:
       # post-ingest.py --postgres upserts the commit row only. Gated on the
       # ingest-role ARN var (the assume-role input that MUST exist for OIDC to
       # succeed).
       #
       # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only
-      # the uv binary, never the synced workspace (see bench.yml rationale).
+      # the uv binary, never the synced workspace (see develop-bench.yml rationale).
       - name: Install uv for v4 ingest
         if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6

.github/workflows/bench.yml → .github/workflows/develop-bench.yml

@@ -1,6 +1,6 @@
 # Runs after every commit to `develop` (or in other words, _after_ every pull request merges).
 
-name: Benchmarks
+name: Develop Benchmarks
 
 on:
   push:
@@ -66,7 +66,7 @@ jobs:
         run: |
           wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
-          echo "$PWD" >> $GITHUB_PATH
+          echo "$PWD" >> "$GITHUB_PATH"
 
       - uses: ./.github/actions/system-info
 
@@ -167,7 +167,7 @@ jobs:
           deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure
 
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "develop"

.github/workflows/nightly-bench.yml

@@ -18,7 +18,7 @@ permissions:
 
 jobs:
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "develop"

.github/workflows/pr-bench-compress.yml

@@ -0,0 +1,14 @@
+# Runs the compression benchmark for a pull request.
+
+name: PR Compression Benchmark
+
+on:
+  workflow_call: { }
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: compress-bench
+      benchmark_name: Compression

.github/workflows/bench-dispatch.yml → .github/workflows/pr-bench-dispatch.yml

@@ -2,7 +2,7 @@
 # This is a separate workflow so that non-benchmark label events don't create
 # phantom check suites that obscure in-progress benchmark runs on the PR.
 
-name: Benchmark Dispatch
+name: PR Benchmark Dispatch
 
 on:
   pull_request:
@@ -16,52 +16,70 @@ permissions:
   id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
-  remove-bench-label:
+  remove-ra-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark'
+    if: github.event.label.name == 'action/bench-ra'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark
+          labels: action/bench-ra
           fail_on_error: true
 
-  bench:
-    needs: remove-bench-label
-    uses: ./.github/workflows/bench-pr.yml
+  random-access-bench:
+    needs: remove-ra-label
+    uses: ./.github/workflows/pr-bench-ra.yml
+    secrets: inherit
+
+  remove-compress-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-compress'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-compress
+          fail_on_error: true
+
+  compression-bench:
+    needs: remove-compress-label
+    uses: ./.github/workflows/pr-bench-compress.yml
     secrets: inherit
 
   remove-sql-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-sql'
+    if: github.event.label.name == 'action/bench-sql'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-sql
+          labels: action/bench-sql
           fail_on_error: true
 
   sql-bench:
     needs: remove-sql-label
-    uses: ./.github/workflows/sql-pr.yml
+    uses: ./.github/workflows/pr-bench-sql.yml
     secrets: inherit
+    with:
+      matrix_preset: "pr"
 
-  remove-sql-full-label:
+  remove-sql-compact-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-sql-full'
+    if: github.event.label.name == 'action/bench-sql-compact'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-sql-full
+          labels: action/bench-sql-compact
           fail_on_error: true
 
-  sql-full-bench:
-    needs: remove-sql-full-label
-    uses: ./.github/workflows/sql-pr.yml
+  sql-compact-bench:
+    needs: remove-sql-compact-label
+    uses: ./.github/workflows/pr-bench-sql.yml
     secrets: inherit
     with:
-      matrix_preset: "pr-full"
+      matrix_preset: "pr-compact"

.github/workflows/pr-bench-ra.yml

@@ -0,0 +1,15 @@
+# Runs the random-access benchmark for a pull request.
+
+name: PR Random Access Benchmark
+
+on:
+  workflow_call: { }
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: random-access-bench
+      benchmark_name: Random Access
+      with_lance: true

.github/workflows/bench-pr.yml → .github/workflows/pr-bench-runner.yml

@@ -1,17 +1,26 @@
-# Runs all benchmarks once for a pull request.
-# Called from bench-dispatch.yml when the `action/benchmark` label is added.
+# Runs one non-SQL benchmark for a pull request.
 
-name: PR Benchmarks
+name: PR Benchmark Runner
 
 concurrency:
   # The group causes runs to queue instead of running in parallel.
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}
   # Don't cancel benchmarks that are already running, instead just queue them up.
   cancel-in-progress: false
 
 on:
-  workflow_call: { }
-  workflow_dispatch: { }
+  workflow_call:
+    inputs:
+      benchmark_id:
+        required: true
+        type: string
+      benchmark_name:
+        required: true
+        type: string
+      with_lance:
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -23,16 +32,8 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
+          && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
           || 'ubuntu-latest' }}
-    strategy:
-      matrix:
-        benchmark:
-          - id: random-access-bench
-            name: Random Access
-            build_args: "--features lance"
-          - id: compress-bench
-            name: Compression
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -52,29 +53,38 @@ jobs:
         run: |
           wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
-          echo "$PWD" >> $GITHUB_PATH
+          echo "$PWD" >> "$GITHUB_PATH"
 
       - uses: ./.github/actions/system-info
 
       - name: Build binary
+        if: inputs.with_lance == false
         shell: bash
         env:
           RUSTFLAGS: "-C target-cpu=native"
         run: |
-          cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings
+          cargo build --package ${{ inputs.benchmark_id }} --profile release_debug --features unstable_encodings
+
+      - name: Build binary with Lance
+        if: inputs.with_lance
+        shell: bash
+        env:
+          RUSTFLAGS: "-C target-cpu=native"
+        run: |
+          cargo build --package ${{ inputs.benchmark_id }} --profile release_debug --features lance,unstable_encodings
 
       - name: Setup Polar Signals
         if: github.event.pull_request.head.repo.fork == false
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ inputs.benchmark_id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
           profiling_frequency: 199
           extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
 
-      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
-        if: matrix.benchmark.id == 'random-access-bench'
+      - name: Run ${{ inputs.benchmark_name }} benchmark (per-combination)
+        if: inputs.benchmark_id == 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
@@ -83,15 +93,15 @@ jobs:
         run: |
           python3 scripts/random-access-split.py
 
-      - name: Run ${{ matrix.benchmark.name }} benchmark
-        if: matrix.benchmark.id != 'random-access-bench'
+      - name: Run ${{ inputs.benchmark_name }} benchmark
+        if: inputs.benchmark_id != 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
+          bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false
@@ -113,16 +123,16 @@ jobs:
           python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \
             > comment.md
-          cat comment.md >> $GITHUB_STEP_SUMMARY
+          cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR
         if: github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
 
       - name: Comment PR on failure
         if: failure() && github.event.pull_request.head.repo.fork == false
@@ -131,12 +141,5 @@ jobs:
           message: |
             # BENCHMARK FAILED
 
-            Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
-
-  sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
-    secrets: inherit
-    with:
-      mode: "pr"
-      matrix_preset: "pr-full"
+            Benchmark `${{ inputs.benchmark_name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}

.github/workflows/sql-pr.yml → .github/workflows/pr-bench-sql.yml

@@ -1,5 +1,5 @@
 # Runs SQL benchmarks once for a pull request.
-# Called from bench-dispatch.yml when SQL benchmark labels are added.
+# Called from pr-bench-dispatch.yml when SQL benchmark labels are added.
 
 name: PR SQL Benchmarks
 
@@ -26,6 +26,7 @@ on:
         default: "pr"
         options:
           - "pr"
+          - "pr-compact"
           - "pr-full"
 
 permissions:
@@ -35,7 +36,7 @@ permissions:
 
 jobs:
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "pr"

.github/workflows/sql-benchmarks.yml → .github/workflows/sql-bench-matrix.yml

@@ -1,4 +1,4 @@
-name: "SQL-related benchmarks"
+name: SQL Benchmark Matrix
 
 on:
   workflow_call:
@@ -227,7 +227,7 @@ jobs:
           bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json
 
       # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live
-      # benchmarks website (see bench.yml for the full rationale); a failure here fails the
+      # benchmarks website (see develop-bench.yml for the full rationale); a failure here fails the
       # job. Gated on inputs.mode == 'develop' + the ingest-role ARN var (the assume-role
       # input that MUST exist for OIDC to succeed). post-ingest.py mints the RDS IAM token
       # (boto3) from the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates