diff --git a/.github/workflows/commit-metadata.yml b/.github/workflows/commit-metadata.yml index c30b2194cd1..5c136028984 100644 --- a/.github/workflows/commit-metadata.yml +++ b/.github/workflows/commit-metadata.yml @@ -22,13 +22,13 @@ jobs: with: fetch-depth: 2 - # v4 (Postgres) ingest -- REQUIRED (see bench.yml rationale). Empty records: + # v4 (Postgres) ingest -- REQUIRED (see develop-bench.yml rationale). Empty records: # post-ingest.py --postgres upserts the commit row only. Gated on the # ingest-role ARN var (the assume-role input that MUST exist for OIDC to # succeed). # # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only - # the uv binary, never the synced workspace (see bench.yml rationale). + # the uv binary, never the synced workspace (see develop-bench.yml rationale). - name: Install uv for v4 ingest if: vars.GH_BENCH_INGEST_ROLE_ARN != '' uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 diff --git a/.github/workflows/bench.yml b/.github/workflows/develop-bench.yml similarity index 98% rename from .github/workflows/bench.yml rename to .github/workflows/develop-bench.yml index 1b9ad94c604..e29d715e9e7 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/develop-bench.yml @@ -1,6 +1,6 @@ # Runs after every commit to `develop` (or in other words, _after_ every pull request merges). -name: Benchmarks +name: Develop Benchmarks on: push: @@ -66,7 +66,7 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info @@ -167,7 +167,7 @@ jobs: deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure sql: - uses: ./.github/workflows/sql-benchmarks.yml + uses: ./.github/workflows/sql-bench-matrix.yml secrets: inherit with: mode: "develop" diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index 4564ef681ca..70c61d07f40 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -18,7 +18,7 @@ permissions: jobs: sql: - uses: ./.github/workflows/sql-benchmarks.yml + uses: ./.github/workflows/sql-bench-matrix.yml secrets: inherit with: mode: "develop" diff --git a/.github/workflows/pr-bench-compress.yml b/.github/workflows/pr-bench-compress.yml new file mode 100644 index 00000000000..988a9266981 --- /dev/null +++ b/.github/workflows/pr-bench-compress.yml @@ -0,0 +1,14 @@ +# Runs the compression benchmark for a pull request. + +name: PR Compression Benchmark + +on: + workflow_call: { } + +jobs: + bench: + uses: ./.github/workflows/pr-bench-runner.yml + secrets: inherit + with: + benchmark_id: compress-bench + benchmark_name: Compression diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml similarity index 54% rename from .github/workflows/bench-dispatch.yml rename to .github/workflows/pr-bench-dispatch.yml index 6313e37c64a..c0ccb41adfb 100644 --- a/.github/workflows/bench-dispatch.yml +++ b/.github/workflows/pr-bench-dispatch.yml @@ -2,7 +2,7 @@ # This is a separate workflow so that non-benchmark label events don't create # phantom check suites that obscure in-progress benchmark runs on the PR. -name: Benchmark Dispatch +name: PR Benchmark Dispatch on: pull_request: @@ -16,52 +16,70 @@ permissions: id-token: write # enables AWS-GitHub OIDC jobs: - remove-bench-label: + remove-ra-label: runs-on: ubuntu-latest timeout-minutes: 10 - if: github.event.label.name == 'action/benchmark' + if: github.event.label.name == 'action/bench-ra' steps: - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' with: - labels: action/benchmark + labels: action/bench-ra fail_on_error: true - bench: - needs: remove-bench-label - uses: ./.github/workflows/bench-pr.yml + random-access-bench: + needs: remove-ra-label + uses: ./.github/workflows/pr-bench-ra.yml + secrets: inherit + + remove-compress-label: + runs-on: ubuntu-latest + timeout-minutes: 10 + if: github.event.label.name == 'action/bench-compress' + steps: + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' + with: + labels: action/bench-compress + fail_on_error: true + + compression-bench: + needs: remove-compress-label + uses: ./.github/workflows/pr-bench-compress.yml secrets: inherit remove-sql-label: runs-on: ubuntu-latest timeout-minutes: 10 - if: github.event.label.name == 'action/benchmark-sql' + if: github.event.label.name == 'action/bench-sql' steps: - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' with: - labels: action/benchmark-sql + labels: action/bench-sql fail_on_error: true sql-bench: needs: remove-sql-label - uses: ./.github/workflows/sql-pr.yml + uses: ./.github/workflows/pr-bench-sql.yml secrets: inherit + with: + matrix_preset: "pr" - remove-sql-full-label: + remove-sql-compact-label: runs-on: ubuntu-latest timeout-minutes: 10 - if: github.event.label.name == 'action/benchmark-sql-full' + if: github.event.label.name == 'action/bench-sql-compact' steps: - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' with: - labels: action/benchmark-sql-full + labels: action/bench-sql-compact fail_on_error: true - sql-full-bench: - needs: remove-sql-full-label - uses: ./.github/workflows/sql-pr.yml + sql-compact-bench: + needs: remove-sql-compact-label + uses: ./.github/workflows/pr-bench-sql.yml secrets: inherit with: - matrix_preset: "pr-full" + matrix_preset: "pr-compact" diff --git a/.github/workflows/pr-bench-ra.yml b/.github/workflows/pr-bench-ra.yml new file mode 100644 index 00000000000..fe24acc1104 --- /dev/null +++ b/.github/workflows/pr-bench-ra.yml @@ -0,0 +1,15 @@ +# Runs the random-access benchmark for a pull request. + +name: PR Random Access Benchmark + +on: + workflow_call: { } + +jobs: + bench: + uses: ./.github/workflows/pr-bench-runner.yml + secrets: inherit + with: + benchmark_id: random-access-bench + benchmark_name: Random Access + with_lance: true diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/pr-bench-runner.yml similarity index 72% rename from .github/workflows/bench-pr.yml rename to .github/workflows/pr-bench-runner.yml index c88afe3ff00..e40a3de7596 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/pr-bench-runner.yml @@ -1,17 +1,26 @@ -# Runs all benchmarks once for a pull request. -# Called from bench-dispatch.yml when the `action/benchmark` label is added. +# Runs one non-SQL benchmark for a pull request. -name: PR Benchmarks +name: PR Benchmark Runner concurrency: # The group causes runs to queue instead of running in parallel. - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }} # Don't cancel benchmarks that are already running, instead just queue them up. cancel-in-progress: false on: - workflow_call: { } - workflow_dispatch: { } + workflow_call: + inputs: + benchmark_id: + required: true + type: string + benchmark_name: + required: true + type: string + with_lance: + required: false + type: boolean + default: false permissions: contents: read @@ -23,16 +32,8 @@ jobs: timeout-minutes: 120 runs-on: >- ${{ github.repository == 'vortex-data/vortex' - && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') + && format('runs-on={0}/runner=bench-dedicated/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '') || 'ubuntu-latest' }} - strategy: - matrix: - benchmark: - - id: random-access-bench - name: Random Access - build_args: "--features lance" - - id: compress-bench - name: Compression steps: - uses: runs-on/action@v2 if: github.event.pull_request.head.repo.fork == false @@ -52,29 +53,38 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info - name: Build binary + if: inputs.with_lance == false shell: bash env: RUSTFLAGS: "-C target-cpu=native" run: | - cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + cargo build --package ${{ inputs.benchmark_id }} --profile release_debug --features unstable_encodings + + - name: Build binary with Lance + if: inputs.with_lance + shell: bash + env: + RUSTFLAGS: "-C target-cpu=native" + run: | + cargo build --package ${{ inputs.benchmark_id }} --profile release_debug --features lance,unstable_encodings - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ inputs.benchmark_id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) - if: matrix.benchmark.id == 'random-access-bench' + - name: Run ${{ inputs.benchmark_name }} benchmark (per-combination) + if: inputs.benchmark_id == 'random-access-bench' shell: bash env: RUST_BACKTRACE: full @@ -83,15 +93,15 @@ jobs: run: | python3 scripts/random-access-split.py - - name: Run ${{ matrix.benchmark.name }} benchmark - if: matrix.benchmark.id != 'random-access-bench' + - name: Run ${{ inputs.benchmark_name }} benchmark + if: inputs.benchmark_id != 'random-access-bench' shell: bash env: RUST_BACKTRACE: full VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" FLAT_LAYOUT_INLINE_ARRAY_NODE: "1" run: | - bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} -d gh-json -o results.json + bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json - name: Setup AWS CLI if: github.event.pull_request.head.repo.fork == false @@ -113,16 +123,16 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ + uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \ > comment.md - cat comment.md >> $GITHUB_STEP_SUMMARY + cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR if: github.event.pull_request.head.repo.fork == false uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 with: file-path: comment.md - comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} + comment-tag: bench-pr-comment-${{ inputs.benchmark_id }} - name: Comment PR on failure if: failure() && github.event.pull_request.head.repo.fork == false @@ -131,12 +141,5 @@ jobs: message: | # BENCHMARK FAILED - Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. - comment-tag: bench-pr-comment-${{ matrix.benchmark.id }} - - sql: - uses: ./.github/workflows/sql-benchmarks.yml - secrets: inherit - with: - mode: "pr" - matrix_preset: "pr-full" + Benchmark `${{ inputs.benchmark_name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + comment-tag: bench-pr-comment-${{ inputs.benchmark_id }} diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/pr-bench-sql.yml similarity index 84% rename from .github/workflows/sql-pr.yml rename to .github/workflows/pr-bench-sql.yml index b14016030c0..63f1bd8f7f2 100644 --- a/.github/workflows/sql-pr.yml +++ b/.github/workflows/pr-bench-sql.yml @@ -1,5 +1,5 @@ # Runs SQL benchmarks once for a pull request. -# Called from bench-dispatch.yml when SQL benchmark labels are added. +# Called from pr-bench-dispatch.yml when SQL benchmark labels are added. name: PR SQL Benchmarks @@ -26,6 +26,7 @@ on: default: "pr" options: - "pr" + - "pr-compact" - "pr-full" permissions: @@ -35,7 +36,7 @@ permissions: jobs: sql: - uses: ./.github/workflows/sql-benchmarks.yml + uses: ./.github/workflows/sql-bench-matrix.yml secrets: inherit with: mode: "pr" diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-bench-matrix.yml similarity index 99% rename from .github/workflows/sql-benchmarks.yml rename to .github/workflows/sql-bench-matrix.yml index d19315c29f0..67ebb4caff6 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-bench-matrix.yml @@ -1,4 +1,4 @@ -name: "SQL-related benchmarks" +name: SQL Benchmark Matrix on: workflow_call: @@ -227,7 +227,7 @@ jobs: bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live - # benchmarks website (see bench.yml for the full rationale); a failure here fails the + # benchmarks website (see develop-bench.yml for the full rationale); a failure here fails the # job. Gated on inputs.mode == 'develop' + the ingest-role ARN var (the assume-role # input that MUST exist for OIDC to succeed). post-ingest.py mints the RDS IAM token # (boto3) from the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py index 55f6ce523f1..1513b315673 100644 --- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py @@ -14,6 +14,7 @@ PRESETS = { "develop": "Every regular SQL benchmark at full target coverage.", "pr": "The quicker pull-request SQL benchmark matrix.", + "pr-compact": "Pull-request SQL benchmarks that only run Vortex Compact.", "pr-full": "Every regular SQL benchmark at full PR target coverage.", "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", } @@ -52,6 +53,8 @@ DUCKDB_DEFAULT_TARGETS = DEFAULT_TARGETS.only(Engine.DUCKDB) DUCKDB_STANDARD_TARGETS = STANDARD_TARGETS.only(Engine.DUCKDB) DATAFUSION_VORTEX_TARGETS = df(Format.VORTEX) +COMPACT_TARGETS = df(Format.VORTEX_COMPACT) | duck(Format.VORTEX_COMPACT) +COMPACT_DUCKDB_TARGETS = duck(Format.VORTEX_COMPACT) DEFAULT = Coverage(DEFAULT_TARGETS) STANDARD = Coverage(STANDARD_TARGETS) @@ -68,6 +71,8 @@ DUCKDB_DEFAULT = Coverage(DUCKDB_DEFAULT_TARGETS) DUCKDB_STANDARD = Coverage(DUCKDB_STANDARD_TARGETS) DATAFUSION_VORTEX = Coverage(DATAFUSION_VORTEX_TARGETS) +COMPACT = Coverage(COMPACT_TARGETS, data_formats=(Format.VORTEX_COMPACT,)) +COMPACT_DUCKDB = Coverage(COMPACT_DUCKDB_TARGETS, data_formats=(Format.VORTEX_COMPACT,)) # Concrete benchmark cases @@ -78,6 +83,7 @@ name="Clickbench on NVME", runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, "develop": FULL_LOCAL, }, @@ -88,6 +94,7 @@ name="Clickbench Sorted on NVME", runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, "develop": FULL_LOCAL, }, @@ -100,6 +107,7 @@ iterations=10, runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": FULL_PR, "develop": FULL_LOCAL, }, @@ -115,6 +123,7 @@ remote_key="tpch/1.0", runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": STANDARD, "develop": STANDARD, }, @@ -127,6 +136,7 @@ iterations=10, runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": FULL_PR, "develop": FULL_LOCAL, }, @@ -141,6 +151,7 @@ local_dir="vortex-bench/data/tpch/10.0", remote_key="tpch/10.0", runs={ + "pr-compact": COMPACT, "pr-full": STANDARD, "develop": STANDARD, }, @@ -169,6 +180,7 @@ scale_factor=1.0, runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": STANDARD_WITH_DUCKDB, "develop": STANDARD_WITH_DUCKDB, }, @@ -181,6 +193,7 @@ local_dir="vortex-bench/data/statpopgen", runs={ "pr": DUCKDB_DEFAULT, + "pr-compact": COMPACT_DUCKDB, "pr-full": DUCKDB_STANDARD, "develop": DUCKDB_STANDARD, }, @@ -192,6 +205,7 @@ scale_factor=100, runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": STANDARD, "develop": STANDARD, }, @@ -206,6 +220,7 @@ remote_key="fineweb", runs={ "pr": DEFAULT, + "pr-compact": COMPACT, "pr-full": STANDARD, "develop": STANDARD, }, @@ -227,6 +242,7 @@ name="Appian on NVME", iterations=10, runs={ + "pr-compact": COMPACT, "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, "develop": STANDARD_WITH_DUCKDB, }, diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py index dd19d12e07f..ed94961b3d1 100644 --- a/bench-orchestrator/tests/test_matrix.py +++ b/bench-orchestrator/tests/test_matrix.py @@ -33,6 +33,9 @@ "appian-nvme", "vortex-queries", ) +COMPACT_IDS = tuple( + benchmark_id for benchmark_id in REGULAR_IDS if benchmark_id not in {"polarsignals", "vortex-queries"} +) EXPECTED_IDS = { "develop": REGULAR_IDS, "pr": tuple( @@ -40,6 +43,7 @@ for benchmark_id in REGULAR_IDS if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"} ), + "pr-compact": COMPACT_IDS, "pr-full": REGULAR_IDS, "nightly": ("tpch-nvme", "tpch-s3"), } @@ -68,6 +72,7 @@ def test_matrix_presets(preset: str, expected_ids: tuple[str, ...]) -> None: def test_pr_target_selection() -> None: develop = {entry["id"]: entry for entry in _entries("develop")} pr = {entry["id"]: entry for entry in _entries("pr")} + pr_compact = {entry["id"]: entry for entry in _entries("pr-compact")} pr_full = {entry["id"]: entry for entry in _entries("pr-full")} assert _targets(pr["tpch-nvme"]) == { @@ -79,6 +84,10 @@ def test_pr_target_selection() -> None: assert ("datafusion", "lance") in _targets(develop["tpch-nvme"]) assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values()) assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"]) + assert all( + all(target[1] == "vortex-compact" for target in _targets(entry)) and entry["data_formats"] == ["vortex-compact"] + for entry in pr_compact.values() + ) def test_resolver_rejects_empty_targets() -> None: diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index dfa0fa1cd45..0a275523948 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -210,12 +210,11 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman - **Post-commit** -- compression, random access, and SQL benchmarks run on every commit to `develop`, with results uploaded for historical tracking. -- **PR benchmarks** -- triggered by the `action/benchmark` label. Results are compared against - the latest `develop` run and posted as a PR comment. -- **SQL benchmarks** -- triggered by the `action/benchmark-sql` label. Runs the base SQL matrix, - which excludes Appian, TPC-H SF=10 on S3, `vortex-compact`, and `duckdb:duckdb`. -- **Full SQL benchmarks** -- triggered by the `action/benchmark-sql-full` label. Runs the full - SQL matrix of suites, engines, formats, and storage backends (NVMe, S3). +- **Random access** -- `action/bench-ra` runs only the random-access benchmark. +- **Compression** -- `action/bench-compress` runs only the compression benchmark. +- **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`. +- **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which generates + and benchmarks only `vortex-compact`. All CI benchmarks run on dedicated instances with the `release_debug` profile and `-C target-cpu=native` to produce representative numbers.