From fb5f8aac32def29931d699021d18f532be3824b2 Mon Sep 17 00:00:00 2001
From: Connor Tsui <connor.tsui20@gmail.com>
Date: Mon, 27 Jul 2026 12:40:19 -0400
Subject: [PATCH] ci: add focused PR benchmark labels

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
---
 .github/workflows/commit-metadata.yml         |  4 +-
 .../{bench.yml => develop-bench.yml}          |  6 +-
 .github/workflows/nightly-bench.yml           |  2 +-
 .github/workflows/pr-bench-compress.yml       | 20 ++++++
 ...nch-dispatch.yml => pr-bench-dispatch.yml} | 62 ++++++++++------
 ...bench-pr.yml => pr-bench-gpu-compress.yml} | 10 +--
 .github/workflows/pr-bench-random-access.yml  | 21 ++++++
 .../{bench-pr.yml => pr-bench-runner.yml}     | 71 +++++++++----------
 .../{sql-pr.yml => pr-bench-sql.yml}          |  5 +-
 ...ql-benchmarks.yml => sql-bench-matrix.yml} | 12 ++--
 .../bench_orchestrator/ci_matrix/catalog.py   | 16 +++++
 bench-orchestrator/tests/test_matrix.py       |  9 +++
 docs/developer-guide/benchmarking.md          | 15 ++--
 13 files changed, 167 insertions(+), 86 deletions(-)
 rename .github/workflows/{bench.yml => develop-bench.yml} (98%)
 create mode 100644 .github/workflows/pr-bench-compress.yml
 rename .github/workflows/{bench-dispatch.yml => pr-bench-dispatch.yml} (54%)
 rename .github/workflows/{gpu-compress-bench-pr.yml => pr-bench-gpu-compress.yml} (89%)
 create mode 100644 .github/workflows/pr-bench-random-access.yml
 rename .github/workflows/{bench-pr.yml => pr-bench-runner.yml} (70%)
 rename .github/workflows/{sql-pr.yml => pr-bench-sql.yml} (84%)
 rename .github/workflows/{sql-benchmarks.yml => sql-bench-matrix.yml} (95%)

diff --git a/.github/workflows/commit-metadata.yml b/.github/workflows/commit-metadata.yml
index c30b2194cd1..5c136028984 100644
--- a/.github/workflows/commit-metadata.yml
+++ b/.github/workflows/commit-metadata.yml
@@ -22,13 +22,13 @@ jobs:
         with:
           fetch-depth: 2
 
-      # v4 (Postgres) ingest -- REQUIRED (see bench.yml rationale). Empty records:
+      # v4 (Postgres) ingest -- REQUIRED (see develop-bench.yml rationale). Empty records:
       # post-ingest.py --postgres upserts the commit row only. Gated on the
       # ingest-role ARN var (the assume-role input that MUST exist for OIDC to
       # succeed).
       #
       # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only
-      # the uv binary, never the synced workspace (see bench.yml rationale).
+      # the uv binary, never the synced workspace (see develop-bench.yml rationale).
       - name: Install uv for v4 ingest
         if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
diff --git a/.github/workflows/bench.yml b/.github/workflows/develop-bench.yml
similarity index 98%
rename from .github/workflows/bench.yml
rename to .github/workflows/develop-bench.yml
index 1b9ad94c604..e29d715e9e7 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/develop-bench.yml
@@ -1,6 +1,6 @@
 # Runs after every commit to `develop` (or in other words, _after_ every pull request merges).
 
-name: Benchmarks
+name: Develop Benchmarks
 
 on:
   push:
@@ -66,7 +66,7 @@ jobs:
         run: |
           wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
-          echo "$PWD" >> $GITHUB_PATH
+          echo "$PWD" >> "$GITHUB_PATH"
 
       - uses: ./.github/actions/system-info
 
@@ -167,7 +167,7 @@ jobs:
           deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure
 
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "develop"
diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml
index 4564ef681ca..70c61d07f40 100644
--- a/.github/workflows/nightly-bench.yml
+++ b/.github/workflows/nightly-bench.yml
@@ -18,7 +18,7 @@ permissions:
 
 jobs:
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "develop"
diff --git a/.github/workflows/pr-bench-compress.yml b/.github/workflows/pr-bench-compress.yml
new file mode 100644
index 00000000000..6d7abd7e86f
--- /dev/null
+++ b/.github/workflows/pr-bench-compress.yml
@@ -0,0 +1,20 @@
+# Runs the compression benchmark for a pull request.
+
+name: PR Compression Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: compress-bench
+      benchmark_name: Compression
diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml
similarity index 54%
rename from .github/workflows/bench-dispatch.yml
rename to .github/workflows/pr-bench-dispatch.yml
index 92b3fafb96f..4e79c6fc5f5 100644
--- a/.github/workflows/bench-dispatch.yml
+++ b/.github/workflows/pr-bench-dispatch.yml
@@ -2,7 +2,7 @@
 # This is a separate workflow so that non-benchmark label events don't create
 # phantom check suites that obscure in-progress benchmark runs on the PR.
 
-name: Benchmark Dispatch
+name: PR Benchmark Dispatch
 
 on:
   pull_request:
@@ -16,68 +16,86 @@ permissions:
   id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
-  remove-bench-label:
+  remove-random-access-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark'
+    if: github.event.label.name == 'action/bench-random-access'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark
+          labels: action/bench-random-access
           fail_on_error: true
 
-  bench:
-    needs: remove-bench-label
-    uses: ./.github/workflows/bench-pr.yml
+  random-access-bench:
+    needs: remove-random-access-label
+    uses: ./.github/workflows/pr-bench-random-access.yml
     secrets: inherit
 
-  remove-gpu-compress-bench-label:
+  remove-compress-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-gpu-compress'
+    if: github.event.label.name == 'action/bench-compress'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-gpu-compress
+          labels: action/bench-compress
+          fail_on_error: true
+
+  compression-bench:
+    needs: remove-compress-label
+    uses: ./.github/workflows/pr-bench-compress.yml
+    secrets: inherit
+
+  remove-gpu-compress-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-gpu-compress'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-gpu-compress
           fail_on_error: true
 
   gpu-compress-bench:
-    needs: remove-gpu-compress-bench-label
-    uses: ./.github/workflows/gpu-compress-bench-pr.yml
+    needs: remove-gpu-compress-label
+    uses: ./.github/workflows/pr-bench-gpu-compress.yml
     secrets: inherit
 
   remove-sql-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-sql'
+    if: github.event.label.name == 'action/bench-sql'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-sql
+          labels: action/bench-sql
           fail_on_error: true
 
   sql-bench:
     needs: remove-sql-label
-    uses: ./.github/workflows/sql-pr.yml
+    uses: ./.github/workflows/pr-bench-sql.yml
     secrets: inherit
+    with:
+      matrix_preset: "pr"
 
-  remove-sql-full-label:
+  remove-sql-compact-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-sql-full'
+    if: github.event.label.name == 'action/bench-sql-compact'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-sql-full
+          labels: action/bench-sql-compact
           fail_on_error: true
 
-  sql-full-bench:
-    needs: remove-sql-full-label
-    uses: ./.github/workflows/sql-pr.yml
+  sql-compact-bench:
+    needs: remove-sql-compact-label
+    uses: ./.github/workflows/pr-bench-sql.yml
     secrets: inherit
     with:
-      matrix_preset: "pr-full"
+      matrix_preset: "pr-compact"
diff --git a/.github/workflows/gpu-compress-bench-pr.yml b/.github/workflows/pr-bench-gpu-compress.yml
similarity index 89%
rename from .github/workflows/gpu-compress-bench-pr.yml
rename to .github/workflows/pr-bench-gpu-compress.yml
index 58654974fdd..5993d796e97 100644
--- a/.github/workflows/gpu-compress-bench-pr.yml
+++ b/.github/workflows/pr-bench-gpu-compress.yml
@@ -1,10 +1,10 @@
 # Runs the GPU-enabled compression decompression benchmarks for a pull request.
-# Called from bench-dispatch.yml when the `action/benchmark-gpu-compress` label is added.
+# Called from pr-bench-dispatch.yml when the `action/bench-gpu-compress` label is added.
 
-name: GPU Compression Benchmarks
+name: PR GPU Compression Benchmark
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-gpu-compress
   cancel-in-progress: false
 
 on:
@@ -71,13 +71,13 @@ jobs:
           } > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
       - name: Comment PR
-        if: github.event.pull_request.head.repo.fork == false
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
           comment-tag: bench-pr-comment-gpu-compress
       - name: Comment PR on failure
-        if: failure() && github.event.pull_request.head.repo.fork == false
+        if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           message: |
diff --git a/.github/workflows/pr-bench-random-access.yml b/.github/workflows/pr-bench-random-access.yml
new file mode 100644
index 00000000000..112d4d8687d
--- /dev/null
+++ b/.github/workflows/pr-bench-random-access.yml
@@ -0,0 +1,21 @@
+# Runs the random-access benchmark for a pull request.
+
+name: PR Random Access Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: random-access-bench
+      benchmark_name: Random Access
+      with_lance: true
diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/pr-bench-runner.yml
similarity index 70%
rename from .github/workflows/bench-pr.yml
rename to .github/workflows/pr-bench-runner.yml
index c88afe3ff00..b8d190fa7af 100644
--- a/.github/workflows/bench-pr.yml
+++ b/.github/workflows/pr-bench-runner.yml
@@ -1,17 +1,26 @@
-# Runs all benchmarks once for a pull request.
-# Called from bench-dispatch.yml when the `action/benchmark` label is added.
+# Runs one non-SQL benchmark for a pull request.
 
-name: PR Benchmarks
+name: PR Benchmark Runner
 
 concurrency:
   # The group causes runs to queue instead of running in parallel.
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}
   # Don't cancel benchmarks that are already running, instead just queue them up.
   cancel-in-progress: false
 
 on:
-  workflow_call: { }
-  workflow_dispatch: { }
+  workflow_call:
+    inputs:
+      benchmark_id:
+        required: true
+        type: string
+      benchmark_name:
+        required: true
+        type: string
+      with_lance:
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -23,16 +32,8 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
+          && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
           || 'ubuntu-latest' }}
-    strategy:
-      matrix:
-        benchmark:
-          - id: random-access-bench
-            name: Random Access
-            build_args: "--features lance"
-          - id: compress-bench
-            name: Compression
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -40,7 +41,7 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
       - uses: ./.github/actions/setup-rust
@@ -52,7 +53,7 @@ jobs:
         run: |
           wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
-          echo "$PWD" >> $GITHUB_PATH
+          echo "$PWD" >> "$GITHUB_PATH"
 
       - uses: ./.github/actions/system-info
 
@@ -61,20 +62,21 @@ jobs:
         env:
           RUSTFLAGS: "-C target-cpu=native"
         run: |
-          cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings
+          cargo build --package ${{ inputs.benchmark_id }} --profile release_debug \
+            --features ${{ inputs.with_lance && 'lance,' || '' }}unstable_encodings
 
       - name: Setup Polar Signals
         if: github.event.pull_request.head.repo.fork == false
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ inputs.benchmark_id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
           profiling_frequency: 199
           extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
 
-      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
-        if: matrix.benchmark.id == 'random-access-bench'
+      - name: Run ${{ inputs.benchmark_name }} benchmark (per-combination)
+        if: inputs.benchmark_id == 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
@@ -83,15 +85,15 @@ jobs:
         run: |
           python3 scripts/random-access-split.py
 
-      - name: Run ${{ matrix.benchmark.name }} benchmark
-        if: matrix.benchmark.id != 'random-access-bench'
+      - name: Run ${{ inputs.benchmark_name }} benchmark
+        if: inputs.benchmark_id != 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json
+          bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false
@@ -113,30 +115,23 @@ jobs:
           python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \
             > comment.md
-          cat comment.md >> $GITHUB_STEP_SUMMARY
+          cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR
-        if: github.event.pull_request.head.repo.fork == false
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
 
       - name: Comment PR on failure
-        if: failure() && github.event.pull_request.head.repo.fork == false
+        if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           message: |
             # BENCHMARK FAILED
 
-            Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
-
-  sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
-    secrets: inherit
-    with:
-      mode: "pr"
-      matrix_preset: "pr-full"
+            Benchmark `${{ inputs.benchmark_name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/pr-bench-sql.yml
similarity index 84%
rename from .github/workflows/sql-pr.yml
rename to .github/workflows/pr-bench-sql.yml
index b14016030c0..63f1bd8f7f2 100644
--- a/.github/workflows/sql-pr.yml
+++ b/.github/workflows/pr-bench-sql.yml
@@ -1,5 +1,5 @@
 # Runs SQL benchmarks once for a pull request.
-# Called from bench-dispatch.yml when SQL benchmark labels are added.
+# Called from pr-bench-dispatch.yml when SQL benchmark labels are added.
 
 name: PR SQL Benchmarks
 
@@ -26,6 +26,7 @@ on:
         default: "pr"
         options:
           - "pr"
+          - "pr-compact"
           - "pr-full"
 
 permissions:
@@ -35,7 +36,7 @@ permissions:
 
 jobs:
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "pr"
diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-bench-matrix.yml
similarity index 95%
rename from .github/workflows/sql-benchmarks.yml
rename to .github/workflows/sql-bench-matrix.yml
index d19315c29f0..fc70a6d42d9 100644
--- a/.github/workflows/sql-benchmarks.yml
+++ b/.github/workflows/sql-bench-matrix.yml
@@ -1,4 +1,4 @@
-name: "SQL-related benchmarks"
+name: SQL Benchmark Matrix
 
 on:
   workflow_call:
@@ -207,8 +207,10 @@ jobs:
         with:
           file-path: comment.md
           # There is exactly one comment per comment-tag. If a comment with this tag already exists,
-          # this action will *update* the comment instead of posting a new comment.
-          comment-tag: bench-pr-comment-${{ matrix.id }}
+          # this action will *update* the comment instead of posting a new comment. The preset is
+          # part of the tag because presets overlap on benchmark ids (`pr` and `pr-compact` share
+          # nine) and can run concurrently, so a preset-less tag would let them clobber each other.
+          comment-tag: bench-pr-comment-${{ matrix.id }}-${{ inputs.matrix_preset }}
 
       - name: Comment PR on failure
         if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
@@ -218,7 +220,7 @@ jobs:
             # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨
 
             Benchmark `${{ matrix.name }}` (${{ inputs.matrix_preset }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ matrix.id }}
+          comment-tag: bench-pr-comment-${{ matrix.id }}-${{ inputs.matrix_preset }}
 
       - name: Upload Benchmark Results
         if: inputs.mode == 'develop'
@@ -227,7 +229,7 @@ jobs:
           bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json
 
       # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live
-      # benchmarks website (see bench.yml for the full rationale); a failure here fails the
+      # benchmarks website (see develop-bench.yml for the full rationale); a failure here fails the
       # job. Gated on inputs.mode == 'develop' + the ingest-role ARN var (the assume-role
       # input that MUST exist for OIDC to succeed). post-ingest.py mints the RDS IAM token
       # (boto3) from the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates
diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
index 55f6ce523f1..f7b915f7001 100644
--- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
+++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
@@ -14,6 +14,7 @@
 PRESETS = {
     "develop": "Every regular SQL benchmark at full target coverage.",
     "pr": "The quicker pull-request SQL benchmark matrix.",
+    "pr-compact": "Pull-request SQL benchmarks that only run Vortex Compact.",
     "pr-full": "Every regular SQL benchmark at full PR target coverage.",
     "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
 }
@@ -52,6 +53,8 @@
 DUCKDB_DEFAULT_TARGETS = DEFAULT_TARGETS.only(Engine.DUCKDB)
 DUCKDB_STANDARD_TARGETS = STANDARD_TARGETS.only(Engine.DUCKDB)
 DATAFUSION_VORTEX_TARGETS = df(Format.VORTEX)
+COMPACT_TARGETS = df(Format.VORTEX_COMPACT) | duck(Format.VORTEX_COMPACT)
+COMPACT_DUCKDB_TARGETS = duck(Format.VORTEX_COMPACT)
 
 DEFAULT = Coverage(DEFAULT_TARGETS)
 STANDARD = Coverage(STANDARD_TARGETS)
@@ -68,6 +71,8 @@
 DUCKDB_DEFAULT = Coverage(DUCKDB_DEFAULT_TARGETS)
 DUCKDB_STANDARD = Coverage(DUCKDB_STANDARD_TARGETS)
 DATAFUSION_VORTEX = Coverage(DATAFUSION_VORTEX_TARGETS)
+COMPACT = Coverage(COMPACT_TARGETS)
+COMPACT_DUCKDB = Coverage(COMPACT_DUCKDB_TARGETS)
 
 # Concrete benchmark cases
 
@@ -78,6 +83,7 @@
         name="Clickbench on NVME",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": FULL_LOCAL,
         },
@@ -88,6 +94,7 @@
         name="Clickbench Sorted on NVME",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": FULL_LOCAL,
         },
@@ -100,6 +107,7 @@
         iterations=10,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": FULL_PR,
             "develop": FULL_LOCAL,
         },
@@ -115,6 +123,7 @@
         remote_key="tpch/1.0",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -127,6 +136,7 @@
         iterations=10,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": FULL_PR,
             "develop": FULL_LOCAL,
         },
@@ -141,6 +151,7 @@
         local_dir="vortex-bench/data/tpch/10.0",
         remote_key="tpch/10.0",
         runs={
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -169,6 +180,7 @@
         scale_factor=1.0,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD_WITH_DUCKDB,
             "develop": STANDARD_WITH_DUCKDB,
         },
@@ -181,6 +193,7 @@
         local_dir="vortex-bench/data/statpopgen",
         runs={
             "pr": DUCKDB_DEFAULT,
+            "pr-compact": COMPACT_DUCKDB,
             "pr-full": DUCKDB_STANDARD,
             "develop": DUCKDB_STANDARD,
         },
@@ -192,6 +205,7 @@
         scale_factor=100,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -206,6 +220,7 @@
         remote_key="fineweb",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -227,6 +242,7 @@
         name="Appian on NVME",
         iterations=10,
         runs={
+            "pr-compact": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": STANDARD_WITH_DUCKDB,
         },
diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py
index dd19d12e07f..ed94961b3d1 100644
--- a/bench-orchestrator/tests/test_matrix.py
+++ b/bench-orchestrator/tests/test_matrix.py
@@ -33,6 +33,9 @@
     "appian-nvme",
     "vortex-queries",
 )
+COMPACT_IDS = tuple(
+    benchmark_id for benchmark_id in REGULAR_IDS if benchmark_id not in {"polarsignals", "vortex-queries"}
+)
 EXPECTED_IDS = {
     "develop": REGULAR_IDS,
     "pr": tuple(
@@ -40,6 +43,7 @@
         for benchmark_id in REGULAR_IDS
         if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"}
     ),
+    "pr-compact": COMPACT_IDS,
     "pr-full": REGULAR_IDS,
     "nightly": ("tpch-nvme", "tpch-s3"),
 }
@@ -68,6 +72,7 @@ def test_matrix_presets(preset: str, expected_ids: tuple[str, ...]) -> None:
 def test_pr_target_selection() -> None:
     develop = {entry["id"]: entry for entry in _entries("develop")}
     pr = {entry["id"]: entry for entry in _entries("pr")}
+    pr_compact = {entry["id"]: entry for entry in _entries("pr-compact")}
     pr_full = {entry["id"]: entry for entry in _entries("pr-full")}
 
     assert _targets(pr["tpch-nvme"]) == {
@@ -79,6 +84,10 @@ def test_pr_target_selection() -> None:
     assert ("datafusion", "lance") in _targets(develop["tpch-nvme"])
     assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values())
     assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"])
+    assert all(
+        all(target[1] == "vortex-compact" for target in _targets(entry)) and entry["data_formats"] == ["vortex-compact"]
+        for entry in pr_compact.values()
+    )
 
 
 def test_resolver_rejects_empty_targets() -> None:
diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md
index 1655d7f8a08..45695a4a720 100644
--- a/docs/developer-guide/benchmarking.md
+++ b/docs/developer-guide/benchmarking.md
@@ -210,14 +210,13 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman
 
 - **Post-commit** -- compression, random access, and SQL benchmarks run on every commit to
   `develop`, with results uploaded for historical tracking.
-- **PR benchmarks** -- triggered by the `action/benchmark` label. Results are compared against
-  the latest `develop` run and posted as a PR comment.
-- **GPU compression benchmarks** -- triggered by the `action/benchmark-gpu-compress` label. Runs
-  the allow-listed Vortex decompression cases on a GPU runner and posts the timings as a PR comment.
-- **SQL benchmarks** -- triggered by the `action/benchmark-sql` label. Runs the base SQL matrix,
-  which excludes Appian, TPC-H SF=10 on S3, `vortex-compact`, and `duckdb:duckdb`.
-- **Full SQL benchmarks** -- triggered by the `action/benchmark-sql-full` label. Runs the full
-  SQL matrix of suites, engines, formats, and storage backends (NVMe, S3).
+- **Random access** -- `action/bench-random-access` runs only the random-access benchmark.
+- **Compression** -- `action/bench-compress` runs only the compression benchmark.
+- **GPU compression** -- `action/bench-gpu-compress` runs the allow-listed Vortex decompression
+  cases on a GPU runner.
+- **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`.
+- **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which generates
+  and benchmarks only `vortex-compact`.
 
 All CI benchmarks run on dedicated instances with the `release_debug` profile and
 `-C target-cpu=native` to produce representative numbers.