diff --git a/.github/actions/upload-parca-debuginfo/action.yml b/.github/actions/upload-parca-debuginfo/action.yml new file mode 100644 index 00000000000..40369a8227e --- /dev/null +++ b/.github/actions/upload-parca-debuginfo/action.yml @@ -0,0 +1,117 @@ +name: "Pre-upload benchmark debuginfo to Polar Signals" +description: "Install parca-debuginfo and upload built benchmark binaries to Polar Signals Cloud" + +inputs: + paths: + description: "Newline-delimited paths to binaries whose debuginfo should be uploaded" + required: true + polarsignals-cloud-token: + description: "Polar Signals service account token" + required: true + project-id: + description: "Polar Signals Cloud project ID" + required: true + store-address: + description: "Parca-compatible debuginfo gRPC endpoint" + required: false + default: "grpc.polarsignals.com:443" + version: + description: "parca-debuginfo version to install" + required: false + default: "0.13.0" + force: + description: "Replace existing debuginfo for the same build ID" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Download and verify parca-debuginfo CLI + shell: bash + env: + VERSION: ${{ inputs.version }} + run: | + set -Eeu -o pipefail + + base_url="https://github.com/parca-dev/parca-debuginfo/releases/download/v${VERSION}" + asset="parca-debuginfo_${VERSION}_$(uname -s)_$(uname -m)" + checksums="$(mktemp)" + curl -fsSL "${base_url}/checksums.txt" -o "$checksums" + + mkdir -p "$RUNNER_TEMP/parca-debuginfo" + cd "$RUNNER_TEMP/parca-debuginfo" + + if curl -fsSLO "${base_url}/${asset}"; then + shasum --ignore-missing -a 256 --check "$checksums" + chmod +x "$asset" + mv "$asset" parca-debuginfo + else + archive="${asset}.tar.gz" + curl -fsSLO "${base_url}/${archive}" + shasum --ignore-missing -a 256 --check "$checksums" + tar -xzf "$archive" parca-debuginfo + chmod +x parca-debuginfo + fi + + echo "$PWD" >> "$GITHUB_PATH" + + - name: Upload benchmark binary debuginfo to Polar Signals + shell: bash + env: + PATHS: ${{ inputs.paths }} + POLARSIGNALS_CLOUD_TOKEN: ${{ inputs.polarsignals-cloud-token }} + PROJECT_ID: ${{ inputs.project-id }} + STORE_ADDRESS: ${{ inputs.store-address }} + FORCE: ${{ inputs.force }} + run: | + set -Eeu -o pipefail + + token_file="$(mktemp)" + upload_log="$(mktemp)" + trap 'rm -f "$token_file" "$upload_log"' EXIT + printf "%s" "$POLARSIGNALS_CLOUD_TOKEN" > "$token_file" + + uploaded=0 + while IFS= read -r binary; do + binary="${binary#"${binary%%[![:space:]]*}"}" + binary="${binary%"${binary##*[![:space:]]}"}" + [[ -z "$binary" ]] && continue + + if [[ ! -f "$binary" ]]; then + echo "::error::Debuginfo upload target does not exist: $binary" + exit 1 + fi + + cmd=( + parca-debuginfo + upload + "--store-address=$STORE_ADDRESS" + "--bearer-token-file=$token_file" + "--grpc-headers=projectID=$PROJECT_ID" + "$binary" + ) + if [[ "$FORCE" == "true" ]]; then + cmd+=(--force) + fi + + set +e + "${cmd[@]}" > "$upload_log" 2>&1 + status=$? + set -e + cat "$upload_log" + + if [[ "$status" -ne 0 ]]; then + if grep -Eiq "upload id mismatch|already exists|AlreadyExists|previous upload is still in-progress|not stale yet|only stale uploads can be retried" "$upload_log"; then + echo "::notice::Debuginfo upload already exists or is in progress for $binary; continuing" + else + exit "$status" + fi + fi + uploaded=$((uploaded + 1)) + done <<< "$PATHS" + + if [[ "$uploaded" -eq 0 ]]; then + echo "::error::No debuginfo upload targets were provided" + exit 1 + fi diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index c88afe3ff00..7782c83420e 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -52,26 +52,37 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info - name: Build binary shell: bash env: - RUSTFLAGS: "-C target-cpu=native" + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + - name: Pre-upload benchmark debuginfo to Polar Signals + if: github.event.pull_request.head.repo.fork == false + uses: ./.github/actions/upload-parca-debuginfo + with: + paths: | + target/release_debug/${{ matrix.benchmark.id }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.event.pull_request.head.sha }};benchmark=${{ matrix.benchmark.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + extra_args: "--debuginfo-strip=false" + parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) if: matrix.benchmark.id == 'random-access-bench' @@ -115,7 +126,7 @@ jobs: uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ > comment.md - cat comment.md >> $GITHUB_STEP_SUMMARY + cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR if: github.event.pull_request.head.repo.fork == false diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1b9ad94c604..dbb8c806789 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -66,25 +66,35 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info - name: Build binary shell: bash env: - RUSTFLAGS: "-C target-cpu=native" + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + - name: Pre-upload benchmark debuginfo to Polar Signals + uses: ./.github/actions/upload-parca-debuginfo + with: + paths: | + target/release_debug/${{ matrix.benchmark.id }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Setup Polar Signals uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ matrix.benchmark.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + extra_args: "--debuginfo-strip=false" + parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) if: matrix.benchmark.id == 'random-access-bench' diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index d19315c29f0..918762cea96 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -90,7 +90,7 @@ jobs: - name: Build binaries shell: bash env: - RUSTFLAGS: "-C target-cpu=native" + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) if [ "${{ inputs.mode }}" != "pr" ]; then @@ -98,6 +98,17 @@ jobs: fi cargo build "${packages[@]}" --profile release_debug --features unstable_encodings + - name: Pre-upload SQL benchmark debuginfo to Polar Signals + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + uses: ./.github/actions/upload-parca-debuginfo + with: + paths: | + target/release_debug/datafusion-bench + target/release_debug/duckdb-bench + ${{ inputs.mode != 'pr' && 'target/release_debug/lance-bench' || '' }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Generate data shell: bash env: @@ -131,10 +142,12 @@ jobs: uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }};benchmark=${{ matrix.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + job_name: "${{ matrix.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + extra_args: "--debuginfo-strip=false" + parca_agent_version: "0.49.0" - name: Run ${{ matrix.name }} benchmark if: matrix.remote_key == null || github.event.pull_request.head.repo.fork == true diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 20a42f902d1..dfb1aaa920e 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -20,7 +20,16 @@ class Engine(Enum): @property def binary_name(self) -> str: - """Return the cargo binary name for this engine when used to execute benchmarks.""" + """Return the benchmark executable name for this engine.""" + return { + Engine.DUCKDB: "duckdb-bench", + Engine.DATAFUSION: "datafusion-bench", + Engine.LANCE: "lance-bench", + }[self] + + @property + def package_name(self) -> str: + """Return the Cargo package name that provides this engine's benchmark binary.""" return { Engine.DUCKDB: "duckdb-bench", Engine.DATAFUSION: "datafusion-bench", diff --git a/bench-orchestrator/bench_orchestrator/runner/builder.py b/bench-orchestrator/bench_orchestrator/runner/builder.py index 070100f1660..47224f710c9 100644 --- a/bench-orchestrator/bench_orchestrator/runner/builder.py +++ b/bench-orchestrator/bench_orchestrator/runner/builder.py @@ -47,12 +47,15 @@ def build(self, backends: list[Engine]) -> dict[Engine, Path]: for backend in backends: binary_name = backend.binary_name + package_name = backend.package_name console.print(f"[blue]Building {binary_name}...[/blue]") cmd = [ "cargo", "build", - "-p", + "--package", + package_name, + "--bin", binary_name, "--profile", self.config.profile, diff --git a/bench-orchestrator/tests/test_config.py b/bench-orchestrator/tests/test_config.py index 12c48d21433..1652cd82934 100644 --- a/bench-orchestrator/tests/test_config.py +++ b/bench-orchestrator/tests/test_config.py @@ -123,3 +123,8 @@ def test_group_targets_by_backend_routes_lance_to_lance_binary() -> None: Engine.DUCKDB, ] assert groups[Engine.LANCE] == [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)] + + +def test_datafusion_package_and_binary_names_match() -> None: + assert Engine.DATAFUSION.package_name == "datafusion-bench" + assert Engine.DATAFUSION.binary_name == "datafusion-bench" diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index e7e18e6376e..306669754d8 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -19,6 +19,7 @@ ignored = ["vortex-cuda"] [[bin]] name = "datafusion-bench" +path = "src/main.rs" test = false [lib] diff --git a/benchmarks/datafusion-bench/src/tracer.rs b/benchmarks/datafusion-bench/src/tracer.rs index 43886993770..bd801ac7ff3 100644 --- a/benchmarks/datafusion-bench/src/tracer.rs +++ b/benchmarks/datafusion-bench/src/tracer.rs @@ -31,6 +31,7 @@ pub fn get_labelset_from_global() -> Labelset { pub fn set_labels(benchmark_name: String, query_idx: usize, format: Format) -> Labelset { let labels = vec![ + ("engine", "datafusion".to_string()), ("benchmark_name", benchmark_name), ("query_idx", query_idx.to_string()), ("format", format.to_string()), diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index 8b75750801f..c887ce621ba 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -199,6 +199,7 @@ fn main() -> anyhow::Result<()> { }, |ctx, query_idx, format, query| { set_global_labels(vec![ + ("engine", "duckdb".to_string()), ("format", format.to_string()), ("benchmark_name", benchmark_name.clone()), ("query_idx", query_idx.to_string()),