Make PolarSignals profiling data more visible

.github/actions/upload-parca-debuginfo/action.yml

@@ -0,0 +1,117 @@
+name: "Pre-upload benchmark debuginfo to Polar Signals"
+description: "Install parca-debuginfo and upload built benchmark binaries to Polar Signals Cloud"
+
+inputs:
+  paths:
+    description: "Newline-delimited paths to binaries whose debuginfo should be uploaded"
+    required: true
+  polarsignals-cloud-token:
+    description: "Polar Signals service account token"
+    required: true
+  project-id:
+    description: "Polar Signals Cloud project ID"
+    required: true
+  store-address:
+    description: "Parca-compatible debuginfo gRPC endpoint"
+    required: false
+    default: "grpc.polarsignals.com:443"
+  version:
+    description: "parca-debuginfo version to install"
+    required: false
+    default: "0.13.0"
+  force:
+    description: "Replace existing debuginfo for the same build ID"
+    required: false
+    default: "false"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Download and verify parca-debuginfo CLI
+      shell: bash
+      env:
+        VERSION: ${{ inputs.version }}
+      run: |
+        set -Eeu -o pipefail
+
+        base_url="https://github.com/parca-dev/parca-debuginfo/releases/download/v${VERSION}"
+        asset="parca-debuginfo_${VERSION}_$(uname -s)_$(uname -m)"
+        checksums="$(mktemp)"
+        curl -fsSL "${base_url}/checksums.txt" -o "$checksums"
+
+        mkdir -p "$RUNNER_TEMP/parca-debuginfo"
+        cd "$RUNNER_TEMP/parca-debuginfo"
+
+        if curl -fsSLO "${base_url}/${asset}"; then
+          shasum --ignore-missing -a 256 --check "$checksums"
+          chmod +x "$asset"
+          mv "$asset" parca-debuginfo
+        else
+          archive="${asset}.tar.gz"
+          curl -fsSLO "${base_url}/${archive}"
+          shasum --ignore-missing -a 256 --check "$checksums"
+          tar -xzf "$archive" parca-debuginfo
+          chmod +x parca-debuginfo
+        fi
+
+        echo "$PWD" >> "$GITHUB_PATH"
+
+    - name: Upload benchmark binary debuginfo to Polar Signals
+      shell: bash
+      env:
+        PATHS: ${{ inputs.paths }}
+        POLARSIGNALS_CLOUD_TOKEN: ${{ inputs.polarsignals-cloud-token }}
+        PROJECT_ID: ${{ inputs.project-id }}
+        STORE_ADDRESS: ${{ inputs.store-address }}
+        FORCE: ${{ inputs.force }}
+      run: |
+        set -Eeu -o pipefail
+
+        token_file="$(mktemp)"
+        upload_log="$(mktemp)"
+        trap 'rm -f "$token_file" "$upload_log"' EXIT
+        printf "%s" "$POLARSIGNALS_CLOUD_TOKEN" > "$token_file"
+
+        uploaded=0
+        while IFS= read -r binary; do
+          binary="${binary#"${binary%%[![:space:]]*}"}"
+          binary="${binary%"${binary##*[![:space:]]}"}"
+          [[ -z "$binary" ]] && continue
+
+          if [[ ! -f "$binary" ]]; then
+            echo "::error::Debuginfo upload target does not exist: $binary"
+            exit 1
+          fi
+
+          cmd=(
+            parca-debuginfo
+            upload
+            "--store-address=$STORE_ADDRESS"
+            "--bearer-token-file=$token_file"
+            "--grpc-headers=projectID=$PROJECT_ID"
+            "$binary"
+          )
+          if [[ "$FORCE" == "true" ]]; then
+            cmd+=(--force)
+          fi
+
+          set +e
+          "${cmd[@]}" > "$upload_log" 2>&1
+          status=$?
+          set -e
+          cat "$upload_log"
+
+          if [[ "$status" -ne 0 ]]; then
+            if grep -Eiq "upload id mismatch|already exists|AlreadyExists|previous upload is still in-progress|not stale yet|only stale uploads can be retried" "$upload_log"; then
+              echo "::notice::Debuginfo upload already exists or is in progress for $binary; continuing"
+            else
+              exit "$status"
+            fi
+          fi
+          uploaded=$((uploaded + 1))
+        done <<< "$PATHS"
+
+        if [[ "$uploaded" -eq 0 ]]; then
+          echo "::error::No debuginfo upload targets were provided"
+          exit 1
+        fi

.github/workflows/bench-pr.yml

@@ -52,26 +52,37 @@ jobs:
         run: |
           wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
-          echo "$PWD" >> $GITHUB_PATH
+          echo "$PWD" >> "$GITHUB_PATH"
 
       - uses: ./.github/actions/system-info
 
       - name: Build binary
         shell: bash
         env:
-          RUSTFLAGS: "-C target-cpu=native"
+          RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
         run: |
           cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings
 
+      - name: Pre-upload benchmark debuginfo to Polar Signals
+        if: github.event.pull_request.head.repo.fork == false
+        uses: ./.github/actions/upload-parca-debuginfo
+        with:
+          paths: |
+            target/release_debug/${{ matrix.benchmark.id }}
+          polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
+          project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
+
       - name: Setup Polar Signals
         if: github.event.pull_request.head.repo.fork == false
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.event.pull_request.head.sha }};benchmark=${{ matrix.benchmark.id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
+          job_name: "${{ matrix.benchmark.id }}"
           profiling_frequency: 199
-          extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
+          extra_args: "--debuginfo-strip=false"
+          parca_agent_version: "0.49.0"
 
       - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
         if: matrix.benchmark.id == 'random-access-bench'
@@ -115,7 +126,7 @@ jobs:
 
           uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
             > comment.md
-          cat comment.md >> $GITHUB_STEP_SUMMARY
+          cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR
         if: github.event.pull_request.head.repo.fork == false

.github/workflows/bench.yml

@@ -66,25 +66,35 @@ jobs:
         run: |
           wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb
           chmod +x duckdb
-          echo "$PWD" >> $GITHUB_PATH
+          echo "$PWD" >> "$GITHUB_PATH"
 
       - uses: ./.github/actions/system-info
 
       - name: Build binary
         shell: bash
         env:
-          RUSTFLAGS: "-C target-cpu=native"
+          RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
         run: |
           cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings
 
+      - name: Pre-upload benchmark debuginfo to Polar Signals
+        uses: ./.github/actions/upload-parca-debuginfo
+        with:
+          paths: |
+            target/release_debug/${{ matrix.benchmark.id }}
+          polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
+          project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
+
       - name: Setup Polar Signals
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ matrix.benchmark.id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
+          job_name: "${{ matrix.benchmark.id }}"
           profiling_frequency: 199
-          extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
+          extra_args: "--debuginfo-strip=false"
+          parca_agent_version: "0.49.0"
 
       - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
         if: matrix.benchmark.id == 'random-access-bench'

.github/workflows/sql-benchmarks.yml

@@ -90,14 +90,25 @@ jobs:
       - name: Build binaries
         shell: bash
         env:
-          RUSTFLAGS: "-C target-cpu=native"
+          RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
         run: |
           packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench)
           if [ "${{ inputs.mode }}" != "pr" ]; then
             packages+=(--bin lance-bench)
           fi
           cargo build "${packages[@]}" --profile release_debug --features unstable_encodings
 
+      - name: Pre-upload SQL benchmark debuginfo to Polar Signals
+        if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false
+        uses: ./.github/actions/upload-parca-debuginfo
+        with:
+          paths: |
+            target/release_debug/datafusion-bench
+            target/release_debug/duckdb-bench
+            ${{ inputs.mode != 'pr' && 'target/release_debug/lance-bench' || '' }}
+          polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
+          project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
+
       - name: Generate data
         shell: bash
         env:
@@ -131,10 +142,12 @@ jobs:
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }};benchmark=${{ matrix.id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
+          job_name: "${{ matrix.id }}"
           profiling_frequency: 199
-          extra_args: "--off-cpu-threshold=0.03"  # Personally tuned by @brancz
+          extra_args: "--debuginfo-strip=false"
+          parca_agent_version: "0.49.0"
 
       - name: Run ${{ matrix.name }} benchmark
         if: matrix.remote_key == null || github.event.pull_request.head.repo.fork == true

bench-orchestrator/bench_orchestrator/config.py

@@ -20,7 +20,16 @@ class Engine(Enum):
 
     @property
     def binary_name(self) -> str:
-        """Return the cargo binary name for this engine when used to execute benchmarks."""
+        """Return the benchmark executable name for this engine."""
+        return {
+            Engine.DUCKDB: "duckdb-bench",
+            Engine.DATAFUSION: "datafusion-bench",
+            Engine.LANCE: "lance-bench",
+        }[self]
+
+    @property
+    def package_name(self) -> str:
+        """Return the Cargo package name that provides this engine's benchmark binary."""
         return {
             Engine.DUCKDB: "duckdb-bench",
             Engine.DATAFUSION: "datafusion-bench",

bench-orchestrator/bench_orchestrator/runner/builder.py

@@ -47,12 +47,15 @@ def build(self, backends: list[Engine]) -> dict[Engine, Path]:
 
         for backend in backends:
             binary_name = backend.binary_name
+            package_name = backend.package_name
             console.print(f"[blue]Building {binary_name}...[/blue]")
 
             cmd = [
                 "cargo",
                 "build",
-                "-p",
+                "--package",
+                package_name,
+                "--bin",
                 binary_name,
                 "--profile",
                 self.config.profile,

bench-orchestrator/tests/test_config.py

@@ -123,3 +123,8 @@ def test_group_targets_by_backend_routes_lance_to_lance_binary() -> None:
         Engine.DUCKDB,
     ]
     assert groups[Engine.LANCE] == [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)]
+
+
+def test_datafusion_package_and_binary_names_match() -> None:
+    assert Engine.DATAFUSION.package_name == "datafusion-bench"
+    assert Engine.DATAFUSION.binary_name == "datafusion-bench"

benchmarks/datafusion-bench/Cargo.toml

@@ -19,6 +19,7 @@ ignored = ["vortex-cuda"]
 
 [[bin]]
 name = "datafusion-bench"
+path = "src/main.rs"
 test = false
 
 [lib]

benchmarks/datafusion-bench/src/tracer.rs

@@ -31,6 +31,7 @@ pub fn get_labelset_from_global() -> Labelset {
 
 pub fn set_labels(benchmark_name: String, query_idx: usize, format: Format) -> Labelset {
     let labels = vec![
+        ("engine", "datafusion".to_string()),
         ("benchmark_name", benchmark_name),
         ("query_idx", query_idx.to_string()),
         ("format", format.to_string()),

benchmarks/duckdb-bench/src/main.rs

@@ -199,6 +199,7 @@ fn main() -> anyhow::Result<()> {
         },
         |ctx, query_idx, format, query| {
             set_global_labels(vec![
+                ("engine", "duckdb".to_string()),
                 ("format", format.to_string()),
                 ("benchmark_name", benchmark_name.clone()),
                 ("query_idx", query_idx.to_string()),