From 78fe111747545ae7c3ac51171a4f3368a27f6ab3 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 13:17:48 +0100 Subject: [PATCH 01/11] ps profiles Signed-off-by: Adam Gutglick --- .github/workflows/bench-pr.yml | 10 ++ .github/workflows/sql-benchmarks.yml | 10 ++ scripts/compare-benchmark-jsons.py | 14 +++ scripts/polarsignals_url.py | 141 ++++++++++++++++++++++ scripts/tests/test_benchmark_reporting.py | 45 +++++++ 5 files changed, 220 insertions(+) create mode 100644 scripts/polarsignals_url.py diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index c88afe3ff00..79b627dcc89 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -63,6 +63,12 @@ jobs: run: | cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + - name: Capture Polar Signals start time + if: github.event.pull_request.head.repo.fork == false + shell: bash + run: | + echo "POLARSIGNALS_START_MS=$(python3 -c 'import time; print(int(time.time() * 1000))')" >> "$GITHUB_ENV" + - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 @@ -70,6 +76,7 @@ jobs: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz @@ -107,6 +114,9 @@ jobs: - name: Compare results shell: bash + env: + POLARSIGNALS_PROJECT_UUID: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + POLARSIGNALS_LABELS: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" run: | set -Eeu -o pipefail -x diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index d19315c29f0..3af909d302c 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -126,6 +126,12 @@ jobs: aws s3 rm --recursive "$REMOTE_STORAGE" aws s3 cp --recursive "${{ matrix.local_dir }}" "$REMOTE_STORAGE" + - name: Capture Polar Signals start time + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + shell: bash + run: | + echo "POLARSIGNALS_START_MS=$(python3 -c 'import time; print(int(time.time() * 1000))')" >> "$GITHUB_ENV" + - name: Setup Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 @@ -133,6 +139,7 @@ jobs: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + job_name: "${{ matrix.id }}" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz @@ -191,6 +198,9 @@ jobs: - name: Compare results if: inputs.mode == 'pr' shell: bash + env: + POLARSIGNALS_PROJECT_UUID: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + POLARSIGNALS_LABELS: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" run: | set -Eeu -o pipefail -x diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index d7cd34793db..78aa22987e9 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -22,6 +22,7 @@ import numpy as np import orjson import pandas as pd +from polarsignals_url import build_query_url_from_env # Analysis overview: # - Join base and PR benchmark rows on benchmark identity. @@ -809,6 +810,15 @@ def format_report_help() -> str: ) +def format_polarsignals_link() -> str | None: + """Render the Polar Signals profile link for the current benchmark run.""" + + url = build_query_url_from_env() + if url is None: + return None + return f"**Profile**: [Polar Signals]({url})" + + ENGINE_ORDER = { "vortex": 0, "datafusion": 1, @@ -894,6 +904,10 @@ def main() -> None: if engine_summary is not None: summary_fields.append(f"**Engines**: {engine_summary}") + profile_link = format_polarsignals_link() + if profile_link is not None: + summary_fields.append(profile_link) + if len(vortex_df) > 0: vortex_performance = format_performance( vortex_geo_mean_ratio, diff --git a/scripts/polarsignals_url.py b/scripts/polarsignals_url.py new file mode 100644 index 00000000000..79ac24f6066 --- /dev/null +++ b/scripts/polarsignals_url.py @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Build Polar Signals Cloud links for profiled CI benchmark runs.""" + +from __future__ import annotations + +import argparse +import os +import time +from urllib.parse import urlencode + +DEFAULT_CLOUD_HOSTNAME = "cloud.polarsignals.com" +DEFAULT_PROFILE_METRIC = "parca_agent:samples:count:cpu:nanoseconds:delta" +DEFAULT_RELATIVE_WINDOW = "relative:hour|6" + + +def parse_labels(labels: str) -> dict[str, str]: + """Parse the semicolon-delimited label format used by the profiling action.""" + + parsed = {} + for raw_label in labels.split(";"): + key, separator, value = raw_label.strip().partition("=") + if key and separator: + parsed[key.strip()] = value.strip() + return parsed + + +def _quote_label_value(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def label_selector(labels: dict[str, str]) -> str: + """Render labels as a Prometheus-style selector.""" + + if not labels: + return "" + selectors = [f'{key}="{_quote_label_value(value)}"' for key, value in labels.items()] + return "{" + ",".join(selectors) + "}" + + +def _timestamp_ms(value: str | None) -> int | None: + if not value: + return None + try: + return int(value) + except ValueError: + return None + + +def build_query_url( + project_uuid: str, + labels: dict[str, str], + *, + cloud_hostname: str = DEFAULT_CLOUD_HOSTNAME, + profile_metric: str = DEFAULT_PROFILE_METRIC, + start_ms: int | None = None, + end_ms: int | None = None, +) -> str: + """Build a Polar Signals Cloud query URL for one benchmark profile.""" + + expression = f"{profile_metric}{label_selector(labels)}" + params = { + "query_browser_mode": "simple", + "expression_a": expression, + "sum_by_a": "comm", + "selection_a": expression, + } + + if start_ms is not None: + if end_ms is None: + end_ms = int(time.time() * 1000) + duration_seconds = max(0, (end_ms - start_ms) // 1000) + params.update( + { + "step_count": str(min(max(duration_seconds // 10, 50), 500)), + "from_a": str(start_ms), + "to_a": str(end_ms), + "time_selection_a": f"absolute:{start_ms}-{end_ms}", + "merge_from_a": str(start_ms * 1_000_000), + "merge_to_a": str(end_ms * 1_000_000), + } + ) + else: + params.update( + { + "step_count": "50", + "time_selection_a": DEFAULT_RELATIVE_WINDOW, + } + ) + + return f"https://{cloud_hostname}/projects/{project_uuid}?{urlencode(params)}" + + +def build_query_url_from_env(*, now_ms: int | None = None) -> str | None: + """Build a profiling URL from POLARSIGNALS_* environment variables.""" + + project_uuid = os.environ.get("POLARSIGNALS_PROJECT_UUID") + raw_labels = os.environ.get("POLARSIGNALS_LABELS", "") + labels = parse_labels(raw_labels) + if not project_uuid or not labels: + return None + + start_ms = _timestamp_ms(os.environ.get("POLARSIGNALS_START_MS")) + end_ms = _timestamp_ms(os.environ.get("POLARSIGNALS_END_MS")) + if end_ms is None: + end_ms = now_ms + + return build_query_url( + project_uuid, + labels, + cloud_hostname=os.environ.get("POLARSIGNALS_CLOUD_HOSTNAME", DEFAULT_CLOUD_HOSTNAME), + profile_metric=os.environ.get("POLARSIGNALS_PROFILE_METRIC", DEFAULT_PROFILE_METRIC), + start_ms=start_ms, + end_ms=end_ms, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--github-output", + metavar="NAME", + help="also write the generated URL to this GitHub Actions output", + ) + args = parser.parse_args() + + url = build_query_url_from_env() + if url is None: + return + + print(url) + if args.github_output: + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as output: + print(f"{args.github_output}={url}", file=output) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 7925161e91e..8edb062628f 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -6,6 +6,7 @@ import subprocess import sys from pathlib import Path +from urllib.parse import parse_qs, urlparse import pandas as pd @@ -15,6 +16,7 @@ def load_compare_module(): + sys.path.insert(0, str(COMPARE_SCRIPT.parent)) spec = importlib.util.spec_from_file_location("compare_benchmark_jsons", COMPARE_SCRIPT) assert spec is not None module = importlib.util.module_from_spec(spec) @@ -157,6 +159,49 @@ def test_within_engine_analysis_uses_each_engines_own_parquet_control() -> None: assert compare.build_verdict(analyses["duckdb"])["impact"] == "+20.0%" +def test_polarsignals_link_uses_run_labels_and_absolute_time(monkeypatch) -> None: + compare = load_compare_module() + monkeypatch.setenv("POLARSIGNALS_PROJECT_UUID", "project-uuid") + monkeypatch.setenv( + "POLARSIGNALS_LABELS", + "branch=feature/profile-links;gh_run_id=123456;benchmark=tpch-nvme", + ) + monkeypatch.setenv("POLARSIGNALS_START_MS", "1000") + monkeypatch.setenv("POLARSIGNALS_END_MS", "61000") + + field = compare.format_polarsignals_link() + + assert field is not None + assert field.startswith("**Profile**: [Polar Signals](") + url = field.removeprefix("**Profile**: [Polar Signals](").removesuffix(")") + parsed = urlparse(url) + query = parse_qs(parsed.query) + + assert parsed.scheme == "https" + assert parsed.netloc == "cloud.polarsignals.com" + assert parsed.path == "/projects/project-uuid" + assert query["time_selection_a"] == ["absolute:1000-61000"] + assert query["from_a"] == ["1000"] + assert query["to_a"] == ["61000"] + assert query["merge_from_a"] == ["1000000000"] + assert query["merge_to_a"] == ["61000000000"] + expected_expression = ( + 'parca_agent:samples:count:cpu:nanoseconds:delta{branch="feature/profile-links",' + 'gh_run_id="123456",benchmark="tpch-nvme"}' + ) + assert query["expression_a"] == [expected_expression] + assert query["selection_a"] == query["expression_a"] + assert query["sum_by_a"] == ["comm"] + + +def test_polarsignals_link_is_omitted_without_labels(monkeypatch) -> None: + compare = load_compare_module() + monkeypatch.delenv("POLARSIGNALS_PROJECT_UUID", raising=False) + monkeypatch.delenv("POLARSIGNALS_LABELS", raising=False) + + assert compare.format_polarsignals_link() is None + + def file_size_record(commit: str, size: int) -> dict[str, object]: return file_size_record_for(commit, size, "tpch", "10", "vortex-file-compressed", "part-0.vortex") From 494222a0c6084aed21edc9aa0c2f15e89641196d Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 13:30:28 +0100 Subject: [PATCH 02/11] Simplify Signed-off-by: Adam Gutglick --- .github/workflows/bench-pr.yml | 11 +- .github/workflows/bench.yml | 3 +- .github/workflows/sql-benchmarks.yml | 11 +- benchmarks/datafusion-bench/src/tracer.rs | 1 + benchmarks/duckdb-bench/src/main.rs | 1 + scripts/compare-benchmark-jsons.py | 14 --- scripts/polarsignals_url.py | 141 ---------------------- scripts/tests/test_benchmark_reporting.py | 44 ------- 8 files changed, 6 insertions(+), 220 deletions(-) delete mode 100644 scripts/polarsignals_url.py diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 79b627dcc89..53273dc89ca 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -63,18 +63,12 @@ jobs: run: | cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - name: Capture Polar Signals start time - if: github.event.pull_request.head.repo.fork == false - shell: bash - run: | - echo "POLARSIGNALS_START_MS=$(python3 -c 'import time; print(int(time.time() * 1000))')" >> "$GITHUB_ENV" - - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.event.pull_request.head.sha }};benchmark=${{ matrix.benchmark.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 @@ -114,9 +108,6 @@ jobs: - name: Compare results shell: bash - env: - POLARSIGNALS_PROJECT_UUID: "e5d846e1-b54c-46e7-9174-8bf055a3af56" - POLARSIGNALS_LABELS: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" run: | set -Eeu -o pipefail -x diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1b9ad94c604..10052ead7f6 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -81,8 +81,9 @@ jobs: uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.benchmark.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ matrix.benchmark.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 3af909d302c..c9ccff7f273 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -126,18 +126,12 @@ jobs: aws s3 rm --recursive "$REMOTE_STORAGE" aws s3 cp --recursive "${{ matrix.local_dir }}" "$REMOTE_STORAGE" - - name: Capture Polar Signals start time - if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false - shell: bash - run: | - echo "POLARSIGNALS_START_MS=$(python3 -c 'import time; print(int(time.time() * 1000))')" >> "$GITHUB_ENV" - - name: Setup Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }} - labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" + labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }};benchmark=${{ matrix.id }}" project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.id }}" profiling_frequency: 199 @@ -198,9 +192,6 @@ jobs: - name: Compare results if: inputs.mode == 'pr' shell: bash - env: - POLARSIGNALS_PROJECT_UUID: "e5d846e1-b54c-46e7-9174-8bf055a3af56" - POLARSIGNALS_LABELS: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};benchmark=${{ matrix.id }}" run: | set -Eeu -o pipefail -x diff --git a/benchmarks/datafusion-bench/src/tracer.rs b/benchmarks/datafusion-bench/src/tracer.rs index 43886993770..bd801ac7ff3 100644 --- a/benchmarks/datafusion-bench/src/tracer.rs +++ b/benchmarks/datafusion-bench/src/tracer.rs @@ -31,6 +31,7 @@ pub fn get_labelset_from_global() -> Labelset { pub fn set_labels(benchmark_name: String, query_idx: usize, format: Format) -> Labelset { let labels = vec![ + ("engine", "datafusion".to_string()), ("benchmark_name", benchmark_name), ("query_idx", query_idx.to_string()), ("format", format.to_string()), diff --git a/benchmarks/duckdb-bench/src/main.rs b/benchmarks/duckdb-bench/src/main.rs index 8b75750801f..c887ce621ba 100644 --- a/benchmarks/duckdb-bench/src/main.rs +++ b/benchmarks/duckdb-bench/src/main.rs @@ -199,6 +199,7 @@ fn main() -> anyhow::Result<()> { }, |ctx, query_idx, format, query| { set_global_labels(vec![ + ("engine", "duckdb".to_string()), ("format", format.to_string()), ("benchmark_name", benchmark_name.clone()), ("query_idx", query_idx.to_string()), diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index 78aa22987e9..d7cd34793db 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -22,7 +22,6 @@ import numpy as np import orjson import pandas as pd -from polarsignals_url import build_query_url_from_env # Analysis overview: # - Join base and PR benchmark rows on benchmark identity. @@ -810,15 +809,6 @@ def format_report_help() -> str: ) -def format_polarsignals_link() -> str | None: - """Render the Polar Signals profile link for the current benchmark run.""" - - url = build_query_url_from_env() - if url is None: - return None - return f"**Profile**: [Polar Signals]({url})" - - ENGINE_ORDER = { "vortex": 0, "datafusion": 1, @@ -904,10 +894,6 @@ def main() -> None: if engine_summary is not None: summary_fields.append(f"**Engines**: {engine_summary}") - profile_link = format_polarsignals_link() - if profile_link is not None: - summary_fields.append(profile_link) - if len(vortex_df) > 0: vortex_performance = format_performance( vortex_geo_mean_ratio, diff --git a/scripts/polarsignals_url.py b/scripts/polarsignals_url.py deleted file mode 100644 index 79ac24f6066..00000000000 --- a/scripts/polarsignals_url.py +++ /dev/null @@ -1,141 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors - -"""Build Polar Signals Cloud links for profiled CI benchmark runs.""" - -from __future__ import annotations - -import argparse -import os -import time -from urllib.parse import urlencode - -DEFAULT_CLOUD_HOSTNAME = "cloud.polarsignals.com" -DEFAULT_PROFILE_METRIC = "parca_agent:samples:count:cpu:nanoseconds:delta" -DEFAULT_RELATIVE_WINDOW = "relative:hour|6" - - -def parse_labels(labels: str) -> dict[str, str]: - """Parse the semicolon-delimited label format used by the profiling action.""" - - parsed = {} - for raw_label in labels.split(";"): - key, separator, value = raw_label.strip().partition("=") - if key and separator: - parsed[key.strip()] = value.strip() - return parsed - - -def _quote_label_value(value: str) -> str: - return value.replace("\\", "\\\\").replace('"', '\\"') - - -def label_selector(labels: dict[str, str]) -> str: - """Render labels as a Prometheus-style selector.""" - - if not labels: - return "" - selectors = [f'{key}="{_quote_label_value(value)}"' for key, value in labels.items()] - return "{" + ",".join(selectors) + "}" - - -def _timestamp_ms(value: str | None) -> int | None: - if not value: - return None - try: - return int(value) - except ValueError: - return None - - -def build_query_url( - project_uuid: str, - labels: dict[str, str], - *, - cloud_hostname: str = DEFAULT_CLOUD_HOSTNAME, - profile_metric: str = DEFAULT_PROFILE_METRIC, - start_ms: int | None = None, - end_ms: int | None = None, -) -> str: - """Build a Polar Signals Cloud query URL for one benchmark profile.""" - - expression = f"{profile_metric}{label_selector(labels)}" - params = { - "query_browser_mode": "simple", - "expression_a": expression, - "sum_by_a": "comm", - "selection_a": expression, - } - - if start_ms is not None: - if end_ms is None: - end_ms = int(time.time() * 1000) - duration_seconds = max(0, (end_ms - start_ms) // 1000) - params.update( - { - "step_count": str(min(max(duration_seconds // 10, 50), 500)), - "from_a": str(start_ms), - "to_a": str(end_ms), - "time_selection_a": f"absolute:{start_ms}-{end_ms}", - "merge_from_a": str(start_ms * 1_000_000), - "merge_to_a": str(end_ms * 1_000_000), - } - ) - else: - params.update( - { - "step_count": "50", - "time_selection_a": DEFAULT_RELATIVE_WINDOW, - } - ) - - return f"https://{cloud_hostname}/projects/{project_uuid}?{urlencode(params)}" - - -def build_query_url_from_env(*, now_ms: int | None = None) -> str | None: - """Build a profiling URL from POLARSIGNALS_* environment variables.""" - - project_uuid = os.environ.get("POLARSIGNALS_PROJECT_UUID") - raw_labels = os.environ.get("POLARSIGNALS_LABELS", "") - labels = parse_labels(raw_labels) - if not project_uuid or not labels: - return None - - start_ms = _timestamp_ms(os.environ.get("POLARSIGNALS_START_MS")) - end_ms = _timestamp_ms(os.environ.get("POLARSIGNALS_END_MS")) - if end_ms is None: - end_ms = now_ms - - return build_query_url( - project_uuid, - labels, - cloud_hostname=os.environ.get("POLARSIGNALS_CLOUD_HOSTNAME", DEFAULT_CLOUD_HOSTNAME), - profile_metric=os.environ.get("POLARSIGNALS_PROFILE_METRIC", DEFAULT_PROFILE_METRIC), - start_ms=start_ms, - end_ms=end_ms, - ) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--github-output", - metavar="NAME", - help="also write the generated URL to this GitHub Actions output", - ) - args = parser.parse_args() - - url = build_query_url_from_env() - if url is None: - return - - print(url) - if args.github_output: - output_path = os.environ.get("GITHUB_OUTPUT") - if output_path: - with open(output_path, "a", encoding="utf-8") as output: - print(f"{args.github_output}={url}", file=output) - - -if __name__ == "__main__": - main() diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 8edb062628f..8a74c7dad81 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -6,7 +6,6 @@ import subprocess import sys from pathlib import Path -from urllib.parse import parse_qs, urlparse import pandas as pd @@ -159,49 +158,6 @@ def test_within_engine_analysis_uses_each_engines_own_parquet_control() -> None: assert compare.build_verdict(analyses["duckdb"])["impact"] == "+20.0%" -def test_polarsignals_link_uses_run_labels_and_absolute_time(monkeypatch) -> None: - compare = load_compare_module() - monkeypatch.setenv("POLARSIGNALS_PROJECT_UUID", "project-uuid") - monkeypatch.setenv( - "POLARSIGNALS_LABELS", - "branch=feature/profile-links;gh_run_id=123456;benchmark=tpch-nvme", - ) - monkeypatch.setenv("POLARSIGNALS_START_MS", "1000") - monkeypatch.setenv("POLARSIGNALS_END_MS", "61000") - - field = compare.format_polarsignals_link() - - assert field is not None - assert field.startswith("**Profile**: [Polar Signals](") - url = field.removeprefix("**Profile**: [Polar Signals](").removesuffix(")") - parsed = urlparse(url) - query = parse_qs(parsed.query) - - assert parsed.scheme == "https" - assert parsed.netloc == "cloud.polarsignals.com" - assert parsed.path == "/projects/project-uuid" - assert query["time_selection_a"] == ["absolute:1000-61000"] - assert query["from_a"] == ["1000"] - assert query["to_a"] == ["61000"] - assert query["merge_from_a"] == ["1000000000"] - assert query["merge_to_a"] == ["61000000000"] - expected_expression = ( - 'parca_agent:samples:count:cpu:nanoseconds:delta{branch="feature/profile-links",' - 'gh_run_id="123456",benchmark="tpch-nvme"}' - ) - assert query["expression_a"] == [expected_expression] - assert query["selection_a"] == query["expression_a"] - assert query["sum_by_a"] == ["comm"] - - -def test_polarsignals_link_is_omitted_without_labels(monkeypatch) -> None: - compare = load_compare_module() - monkeypatch.delenv("POLARSIGNALS_PROJECT_UUID", raising=False) - monkeypatch.delenv("POLARSIGNALS_LABELS", raising=False) - - assert compare.format_polarsignals_link() is None - - def file_size_record(commit: str, size: int) -> dict[str, object]: return file_size_record_for(commit, size, "tpch", "10", "vortex-file-compressed", "part-0.vortex") From 0270890b26a54967c22ce47d1ee8fb1d832b77f5 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 13:36:29 +0100 Subject: [PATCH 03/11] Force frame points in benchmarks Signed-off-by: Adam Gutglick --- .github/workflows/bench-pr.yml | 2 +- .github/workflows/bench.yml | 2 +- .github/workflows/sql-benchmarks.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 53273dc89ca..8ecd359abd5 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -59,7 +59,7 @@ jobs: - name: Build binary shell: bash env: - RUSTFLAGS: "-C target-cpu=native" + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 10052ead7f6..d5f732c064d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -73,7 +73,7 @@ jobs: - name: Build binary shell: bash env: - RUSTFLAGS: "-C target-cpu=native" + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index c9ccff7f273..eaefc3217cb 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -90,7 +90,7 @@ jobs: - name: Build binaries shell: bash env: - RUSTFLAGS: "-C target-cpu=native" + RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) if [ "${{ inputs.mode }}" != "pr" ]; then From 3439eb0d9fbc3c5f560c497333dbec6416e5eec2 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 14:22:01 +0100 Subject: [PATCH 04/11] Use newer parca version Signed-off-by: Adam Gutglick --- .github/workflows/bench-pr.yml | 1 + .github/workflows/bench.yml | 1 + .github/workflows/sql-benchmarks.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 8ecd359abd5..2ea8baeea17 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -73,6 +73,7 @@ jobs: job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) if: matrix.benchmark.id == 'random-access-bench' diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index d5f732c064d..32cdc98b701 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -86,6 +86,7 @@ jobs: job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) if: matrix.benchmark.id == 'random-access-bench' diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index eaefc3217cb..f2825cd6d8b 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -136,6 +136,7 @@ jobs: job_name: "${{ matrix.id }}" profiling_frequency: 199 extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + parca_agent_version: "0.49.0" - name: Run ${{ matrix.name }} benchmark if: matrix.remote_key == null || github.event.pull_request.head.repo.fork == true From 2f123431631699118bb5c3caf5ea459a90e9971d Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 14:45:21 +0100 Subject: [PATCH 05/11] Rename binary Signed-off-by: Adam Gutglick --- .github/workflows/sql-benchmarks.yml | 2 +- .../bench_orchestrator/config.py | 11 +++++- .../bench_orchestrator/runner/builder.py | 5 ++- bench-orchestrator/tests/test_builder.py | 39 +++++++++++++++++++ bench-orchestrator/tests/test_cli.py | 4 +- bench-orchestrator/tests/test_config.py | 5 +++ benchmarks/datafusion-bench/Cargo.toml | 3 +- 7 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 bench-orchestrator/tests/test_builder.py diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index f2825cd6d8b..c60267a32d2 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -92,7 +92,7 @@ jobs: env: RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | - packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) + packages=(--bin data-gen --bin df-bench --bin duckdb-bench) if [ "${{ inputs.mode }}" != "pr" ]; then packages+=(--bin lance-bench) fi diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 20a42f902d1..9dbace457cf 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -20,7 +20,16 @@ class Engine(Enum): @property def binary_name(self) -> str: - """Return the cargo binary name for this engine when used to execute benchmarks.""" + """Return the benchmark executable name for this engine.""" + return { + Engine.DUCKDB: "duckdb-bench", + Engine.DATAFUSION: "df-bench", + Engine.LANCE: "lance-bench", + }[self] + + @property + def package_name(self) -> str: + """Return the Cargo package name that provides this engine's benchmark binary.""" return { Engine.DUCKDB: "duckdb-bench", Engine.DATAFUSION: "datafusion-bench", diff --git a/bench-orchestrator/bench_orchestrator/runner/builder.py b/bench-orchestrator/bench_orchestrator/runner/builder.py index 070100f1660..47224f710c9 100644 --- a/bench-orchestrator/bench_orchestrator/runner/builder.py +++ b/bench-orchestrator/bench_orchestrator/runner/builder.py @@ -47,12 +47,15 @@ def build(self, backends: list[Engine]) -> dict[Engine, Path]: for backend in backends: binary_name = backend.binary_name + package_name = backend.package_name console.print(f"[blue]Building {binary_name}...[/blue]") cmd = [ "cargo", "build", - "-p", + "--package", + package_name, + "--bin", binary_name, "--profile", self.config.profile, diff --git a/bench-orchestrator/tests/test_builder.py b/bench-orchestrator/tests/test_builder.py new file mode 100644 index 00000000000..339a14274b4 --- /dev/null +++ b/bench-orchestrator/tests/test_builder.py @@ -0,0 +1,39 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +from pathlib import Path + +from bench_orchestrator.config import Engine +from bench_orchestrator.runner import builder as builder_module +from bench_orchestrator.runner.builder import BenchmarkBuilder + + +def test_datafusion_build_uses_package_and_binary_names(tmp_path: Path, monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_run(cmd, cwd, env, check): + captured["cmd"] = cmd + captured["cwd"] = cwd + captured["env"] = env + captured["check"] = check + + monkeypatch.setattr(builder_module.subprocess, "run", fake_run) + + builder = BenchmarkBuilder(workspace_root=tmp_path) + paths = builder.build([Engine.DATAFUSION]) + + assert captured["cmd"] == [ + "cargo", + "build", + "--package", + "datafusion-bench", + "--bin", + "df-bench", + "--profile", + "release_debug", + "--features", + "unstable_encodings", + ] + assert captured["cwd"] == tmp_path + assert captured["check"] is True + assert paths == {Engine.DATAFUSION: tmp_path / "target" / "release_debug" / "df-bench"} diff --git a/bench-orchestrator/tests/test_cli.py b/bench-orchestrator/tests/test_cli.py index 3d67ffaf177..2d0beb3112b 100644 --- a/bench-orchestrator/tests/test_cli.py +++ b/bench-orchestrator/tests/test_cli.py @@ -50,7 +50,7 @@ def fake_run(cmd: list[str], check: bool) -> None: def test_run_writes_compatibility_results_output(tmp_path, monkeypatch) -> None: run_store = ResultStore(base_dir=tmp_path / "runs") output_path = tmp_path / "artifacts" / "results.json" - binary_path = tmp_path / "datafusion-bench" + binary_path = tmp_path / "df-bench" binary_path.write_text("", encoding="utf-8") sample_line = json.dumps( @@ -111,7 +111,7 @@ def test_run_combines_ingest_output_per_backend(tmp_path, monkeypatch) -> None: run_store = ResultStore(base_dir=tmp_path / "runs") output_path = tmp_path / "artifacts" / "results.ingest.jsonl" binary_paths = { - cli_module.Engine.DATAFUSION: tmp_path / "datafusion-bench", + cli_module.Engine.DATAFUSION: tmp_path / "df-bench", cli_module.Engine.DUCKDB: tmp_path / "duckdb-bench", } for binary_path in binary_paths.values(): diff --git a/bench-orchestrator/tests/test_config.py b/bench-orchestrator/tests/test_config.py index 12c48d21433..89aafa9a669 100644 --- a/bench-orchestrator/tests/test_config.py +++ b/bench-orchestrator/tests/test_config.py @@ -123,3 +123,8 @@ def test_group_targets_by_backend_routes_lance_to_lance_binary() -> None: Engine.DUCKDB, ] assert groups[Engine.LANCE] == [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)] + + +def test_datafusion_package_and_binary_names_are_distinct() -> None: + assert Engine.DATAFUSION.package_name == "datafusion-bench" + assert Engine.DATAFUSION.binary_name == "df-bench" diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index e7e18e6376e..81dd3bb4b99 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -18,7 +18,8 @@ publish = false ignored = ["vortex-cuda"] [[bin]] -name = "datafusion-bench" +name = "df-bench" +path = "src/main.rs" test = false [lib] From 86adaa5db55399808df136262c1a08061686d63b Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 15:07:36 +0100 Subject: [PATCH 06/11] print parca logs Signed-off-by: Adam Gutglick --- .github/workflows/sql-benchmarks.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index c60267a32d2..2a96d479ebc 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -212,6 +212,11 @@ jobs: # this action will *update* the comment instead of posting a new comment. comment-tag: bench-pr-comment-${{ matrix.id }} + - name: Print Parca log + shell: bash + run: | + cat ${{ runner.temp }}/parca-agent.log + - name: Comment PR on failure if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 From b7e1d2774e0b0cf104870add6f580451f04c93d2 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 15:29:10 +0100 Subject: [PATCH 07/11] Upload debug info explicitly Signed-off-by: Adam Gutglick --- .../actions/upload-parca-debuginfo/action.yml | 104 ++++++++++++++++++ .github/workflows/bench-pr.yml | 15 ++- .github/workflows/bench.yml | 12 +- .github/workflows/sql-benchmarks.yml | 13 ++- 4 files changed, 138 insertions(+), 6 deletions(-) create mode 100644 .github/actions/upload-parca-debuginfo/action.yml diff --git a/.github/actions/upload-parca-debuginfo/action.yml b/.github/actions/upload-parca-debuginfo/action.yml new file mode 100644 index 00000000000..fb670638c48 --- /dev/null +++ b/.github/actions/upload-parca-debuginfo/action.yml @@ -0,0 +1,104 @@ +name: "Upload Parca debuginfo" +description: "Upload debuginfo for built binaries to Polar Signals Cloud" + +inputs: + paths: + description: "Newline-delimited paths to binaries whose debuginfo should be uploaded" + required: true + polarsignals-cloud-token: + description: "Polar Signals service account token" + required: true + project-id: + description: "Polar Signals Cloud project ID" + required: true + store-address: + description: "Parca-compatible debuginfo gRPC endpoint" + required: false + default: "grpc.polarsignals.com:443" + version: + description: "parca-debuginfo version to install" + required: false + default: "0.13.0" + force: + description: "Replace existing debuginfo for the same build ID" + required: false + default: "true" + +runs: + using: "composite" + steps: + - name: Install parca-debuginfo + shell: bash + env: + VERSION: ${{ inputs.version }} + run: | + set -Eeu -o pipefail + + base_url="https://github.com/parca-dev/parca-debuginfo/releases/download/v${VERSION}" + asset="parca-debuginfo_${VERSION}_$(uname -s)_$(uname -m)" + checksums="$(mktemp)" + curl -fsSL "${base_url}/checksums.txt" -o "$checksums" + + mkdir -p "$RUNNER_TEMP/parca-debuginfo" + cd "$RUNNER_TEMP/parca-debuginfo" + + if curl -fsSLO "${base_url}/${asset}"; then + shasum --ignore-missing -a 256 --check "$checksums" + chmod +x "$asset" + mv "$asset" parca-debuginfo + else + archive="${asset}.tar.gz" + curl -fsSLO "${base_url}/${archive}" + shasum --ignore-missing -a 256 --check "$checksums" + tar -xzf "$archive" parca-debuginfo + chmod +x parca-debuginfo + fi + + echo "$PWD" >> "$GITHUB_PATH" + + - name: Upload debuginfo + shell: bash + env: + PATHS: ${{ inputs.paths }} + POLARSIGNALS_CLOUD_TOKEN: ${{ inputs.polarsignals-cloud-token }} + PROJECT_ID: ${{ inputs.project-id }} + STORE_ADDRESS: ${{ inputs.store-address }} + FORCE: ${{ inputs.force }} + run: | + set -Eeu -o pipefail + + token_file="$(mktemp)" + trap 'rm -f "$token_file"' EXIT + printf "%s" "$POLARSIGNALS_CLOUD_TOKEN" > "$token_file" + + uploaded=0 + while IFS= read -r binary; do + binary="${binary#"${binary%%[![:space:]]*}"}" + binary="${binary%"${binary##*[![:space:]]}"}" + [[ -z "$binary" ]] && continue + + if [[ ! -f "$binary" ]]; then + echo "::error::Debuginfo upload target does not exist: $binary" + exit 1 + fi + + cmd=( + parca-debuginfo + upload + "--store-address=$STORE_ADDRESS" + "--bearer-token-file=$token_file" + "--grpc-headers=projectID=$PROJECT_ID" + "$binary" + ) + if [[ "$FORCE" == "true" ]]; then + cmd+=(--force) + fi + + "${cmd[@]}" + uploaded=$((uploaded + 1)) + done <<< "$PATHS" + + if [[ "$uploaded" -eq 0 ]]; then + echo "::error::No debuginfo upload targets were provided" + exit 1 + fi diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 2ea8baeea17..f359aff78ed 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -52,7 +52,7 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info @@ -63,6 +63,15 @@ jobs: run: | cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + - name: Upload Parca debuginfo + if: github.event.pull_request.head.repo.fork == false + uses: ./.github/actions/upload-parca-debuginfo + with: + paths: | + target/release_debug/${{ matrix.benchmark.id }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Setup Polar Signals if: github.event.pull_request.head.repo.fork == false uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 @@ -72,7 +81,7 @@ jobs: project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + extra_args: "--off-cpu-threshold=0.03 --debuginfo-upload-disable" # Personally tuned by @brancz parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) @@ -117,7 +126,7 @@ jobs: uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ > comment.md - cat comment.md >> $GITHUB_STEP_SUMMARY + cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR if: github.event.pull_request.head.repo.fork == false diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 32cdc98b701..afc8a067eca 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -66,7 +66,7 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.5/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info @@ -77,6 +77,14 @@ jobs: run: | cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings + - name: Upload Parca debuginfo + uses: ./.github/actions/upload-parca-debuginfo + with: + paths: | + target/release_debug/${{ matrix.benchmark.id }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Setup Polar Signals uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a # v0.8.1 with: @@ -85,7 +93,7 @@ jobs: project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + extra_args: "--off-cpu-threshold=0.03 --debuginfo-upload-disable" # Personally tuned by @brancz parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 2a96d479ebc..da29fceab26 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -98,6 +98,17 @@ jobs: fi cargo build "${packages[@]}" --profile release_debug --features unstable_encodings + - name: Upload Parca debuginfo + if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false + uses: ./.github/actions/upload-parca-debuginfo + with: + paths: | + target/release_debug/df-bench + target/release_debug/duckdb-bench + ${{ inputs.mode != 'pr' && 'target/release_debug/lance-bench' || '' }} + polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} + project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56" + - name: Generate data shell: bash env: @@ -135,7 +146,7 @@ jobs: project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz + extra_args: "--off-cpu-threshold=0.03 --debuginfo-upload-disable" # Personally tuned by @brancz parca_agent_version: "0.49.0" - name: Run ${{ matrix.name }} benchmark From 63bc6be013f1151a70fda9d932cb2de37b9c6179 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 15:41:19 +0100 Subject: [PATCH 08/11] Update parca config Signed-off-by: Adam Gutglick --- .github/actions/upload-parca-debuginfo/action.yml | 8 ++++---- .github/workflows/bench-pr.yml | 4 ++-- .github/workflows/bench.yml | 4 ++-- .github/workflows/sql-benchmarks.yml | 4 ++-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/upload-parca-debuginfo/action.yml b/.github/actions/upload-parca-debuginfo/action.yml index fb670638c48..fc9134d46a8 100644 --- a/.github/actions/upload-parca-debuginfo/action.yml +++ b/.github/actions/upload-parca-debuginfo/action.yml @@ -1,5 +1,5 @@ -name: "Upload Parca debuginfo" -description: "Upload debuginfo for built binaries to Polar Signals Cloud" +name: "Pre-upload benchmark debuginfo to Polar Signals" +description: "Install parca-debuginfo and upload built benchmark binaries to Polar Signals Cloud" inputs: paths: @@ -27,7 +27,7 @@ inputs: runs: using: "composite" steps: - - name: Install parca-debuginfo + - name: Download and verify parca-debuginfo CLI shell: bash env: VERSION: ${{ inputs.version }} @@ -56,7 +56,7 @@ runs: echo "$PWD" >> "$GITHUB_PATH" - - name: Upload debuginfo + - name: Upload benchmark binary debuginfo to Polar Signals shell: bash env: PATHS: ${{ inputs.paths }} diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index f359aff78ed..7782c83420e 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -63,7 +63,7 @@ jobs: run: | cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - name: Upload Parca debuginfo + - name: Pre-upload benchmark debuginfo to Polar Signals if: github.event.pull_request.head.repo.fork == false uses: ./.github/actions/upload-parca-debuginfo with: @@ -81,7 +81,7 @@ jobs: project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03 --debuginfo-upload-disable" # Personally tuned by @brancz + extra_args: "--debuginfo-strip=false" parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index afc8a067eca..dbb8c806789 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -77,7 +77,7 @@ jobs: run: | cargo build --bin ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings - - name: Upload Parca debuginfo + - name: Pre-upload benchmark debuginfo to Polar Signals uses: ./.github/actions/upload-parca-debuginfo with: paths: | @@ -93,7 +93,7 @@ jobs: project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.benchmark.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03 --debuginfo-upload-disable" # Personally tuned by @brancz + extra_args: "--debuginfo-strip=false" parca_agent_version: "0.49.0" - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination) diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index da29fceab26..aa085f573df 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -98,7 +98,7 @@ jobs: fi cargo build "${packages[@]}" --profile release_debug --features unstable_encodings - - name: Upload Parca debuginfo + - name: Pre-upload SQL benchmark debuginfo to Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false uses: ./.github/actions/upload-parca-debuginfo with: @@ -146,7 +146,7 @@ jobs: project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56" job_name: "${{ matrix.id }}" profiling_frequency: 199 - extra_args: "--off-cpu-threshold=0.03 --debuginfo-upload-disable" # Personally tuned by @brancz + extra_args: "--debuginfo-strip=false" parca_agent_version: "0.49.0" - name: Run ${{ matrix.name }} benchmark From cfe38fb1e69146a819998ecddb7b62fdacdc087e Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 15:48:21 +0100 Subject: [PATCH 09/11] fix Signed-off-by: Adam Gutglick --- .../actions/upload-parca-debuginfo/action.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/actions/upload-parca-debuginfo/action.yml b/.github/actions/upload-parca-debuginfo/action.yml index fc9134d46a8..a034c30e828 100644 --- a/.github/actions/upload-parca-debuginfo/action.yml +++ b/.github/actions/upload-parca-debuginfo/action.yml @@ -22,7 +22,7 @@ inputs: force: description: "Replace existing debuginfo for the same build ID" required: false - default: "true" + default: "false" runs: using: "composite" @@ -68,7 +68,8 @@ runs: set -Eeu -o pipefail token_file="$(mktemp)" - trap 'rm -f "$token_file"' EXIT + upload_log="$(mktemp)" + trap 'rm -f "$token_file" "$upload_log"' EXIT printf "%s" "$POLARSIGNALS_CLOUD_TOKEN" > "$token_file" uploaded=0 @@ -94,7 +95,19 @@ runs: cmd+=(--force) fi - "${cmd[@]}" + set +e + "${cmd[@]}" > "$upload_log" 2>&1 + status=$? + set -e + cat "$upload_log" + + if [[ "$status" -ne 0 ]]; then + if grep -Eiq "upload id mismatch|already exists|AlreadyExists" "$upload_log"; then + echo "::notice::Debuginfo already uploaded for $binary; continuing" + else + exit "$status" + fi + fi uploaded=$((uploaded + 1)) done <<< "$PATHS" From 1d6d710ecdffb5be63aefffd5f8304764112cb47 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 16:34:58 +0100 Subject: [PATCH 10/11] rename fix Signed-off-by: Adam Gutglick --- .github/workflows/sql-benchmarks.yml | 9 +---- .../bench_orchestrator/config.py | 2 +- bench-orchestrator/tests/test_builder.py | 39 ------------------- bench-orchestrator/tests/test_cli.py | 4 +- bench-orchestrator/tests/test_config.py | 4 +- benchmarks/datafusion-bench/Cargo.toml | 2 +- scripts/tests/test_benchmark_reporting.py | 1 - 7 files changed, 8 insertions(+), 53 deletions(-) delete mode 100644 bench-orchestrator/tests/test_builder.py diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index aa085f573df..918762cea96 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -92,7 +92,7 @@ jobs: env: RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes" run: | - packages=(--bin data-gen --bin df-bench --bin duckdb-bench) + packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) if [ "${{ inputs.mode }}" != "pr" ]; then packages+=(--bin lance-bench) fi @@ -103,7 +103,7 @@ jobs: uses: ./.github/actions/upload-parca-debuginfo with: paths: | - target/release_debug/df-bench + target/release_debug/datafusion-bench target/release_debug/duckdb-bench ${{ inputs.mode != 'pr' && 'target/release_debug/lance-bench' || '' }} polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }} @@ -223,11 +223,6 @@ jobs: # this action will *update* the comment instead of posting a new comment. comment-tag: bench-pr-comment-${{ matrix.id }} - - name: Print Parca log - shell: bash - run: | - cat ${{ runner.temp }}/parca-agent.log - - name: Comment PR on failure if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b # v3 diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py index 9dbace457cf..dfb1aaa920e 100644 --- a/bench-orchestrator/bench_orchestrator/config.py +++ b/bench-orchestrator/bench_orchestrator/config.py @@ -23,7 +23,7 @@ def binary_name(self) -> str: """Return the benchmark executable name for this engine.""" return { Engine.DUCKDB: "duckdb-bench", - Engine.DATAFUSION: "df-bench", + Engine.DATAFUSION: "datafusion-bench", Engine.LANCE: "lance-bench", }[self] diff --git a/bench-orchestrator/tests/test_builder.py b/bench-orchestrator/tests/test_builder.py deleted file mode 100644 index 339a14274b4..00000000000 --- a/bench-orchestrator/tests/test_builder.py +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright the Vortex contributors - -from pathlib import Path - -from bench_orchestrator.config import Engine -from bench_orchestrator.runner import builder as builder_module -from bench_orchestrator.runner.builder import BenchmarkBuilder - - -def test_datafusion_build_uses_package_and_binary_names(tmp_path: Path, monkeypatch) -> None: - captured: dict[str, object] = {} - - def fake_run(cmd, cwd, env, check): - captured["cmd"] = cmd - captured["cwd"] = cwd - captured["env"] = env - captured["check"] = check - - monkeypatch.setattr(builder_module.subprocess, "run", fake_run) - - builder = BenchmarkBuilder(workspace_root=tmp_path) - paths = builder.build([Engine.DATAFUSION]) - - assert captured["cmd"] == [ - "cargo", - "build", - "--package", - "datafusion-bench", - "--bin", - "df-bench", - "--profile", - "release_debug", - "--features", - "unstable_encodings", - ] - assert captured["cwd"] == tmp_path - assert captured["check"] is True - assert paths == {Engine.DATAFUSION: tmp_path / "target" / "release_debug" / "df-bench"} diff --git a/bench-orchestrator/tests/test_cli.py b/bench-orchestrator/tests/test_cli.py index 2d0beb3112b..3d67ffaf177 100644 --- a/bench-orchestrator/tests/test_cli.py +++ b/bench-orchestrator/tests/test_cli.py @@ -50,7 +50,7 @@ def fake_run(cmd: list[str], check: bool) -> None: def test_run_writes_compatibility_results_output(tmp_path, monkeypatch) -> None: run_store = ResultStore(base_dir=tmp_path / "runs") output_path = tmp_path / "artifacts" / "results.json" - binary_path = tmp_path / "df-bench" + binary_path = tmp_path / "datafusion-bench" binary_path.write_text("", encoding="utf-8") sample_line = json.dumps( @@ -111,7 +111,7 @@ def test_run_combines_ingest_output_per_backend(tmp_path, monkeypatch) -> None: run_store = ResultStore(base_dir=tmp_path / "runs") output_path = tmp_path / "artifacts" / "results.ingest.jsonl" binary_paths = { - cli_module.Engine.DATAFUSION: tmp_path / "df-bench", + cli_module.Engine.DATAFUSION: tmp_path / "datafusion-bench", cli_module.Engine.DUCKDB: tmp_path / "duckdb-bench", } for binary_path in binary_paths.values(): diff --git a/bench-orchestrator/tests/test_config.py b/bench-orchestrator/tests/test_config.py index 89aafa9a669..1652cd82934 100644 --- a/bench-orchestrator/tests/test_config.py +++ b/bench-orchestrator/tests/test_config.py @@ -125,6 +125,6 @@ def test_group_targets_by_backend_routes_lance_to_lance_binary() -> None: assert groups[Engine.LANCE] == [BenchmarkTarget(engine=Engine.DATAFUSION, format=Format.LANCE)] -def test_datafusion_package_and_binary_names_are_distinct() -> None: +def test_datafusion_package_and_binary_names_match() -> None: assert Engine.DATAFUSION.package_name == "datafusion-bench" - assert Engine.DATAFUSION.binary_name == "df-bench" + assert Engine.DATAFUSION.binary_name == "datafusion-bench" diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index 81dd3bb4b99..306669754d8 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -18,7 +18,7 @@ publish = false ignored = ["vortex-cuda"] [[bin]] -name = "df-bench" +name = "datafusion-bench" path = "src/main.rs" test = false diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py index 8a74c7dad81..7925161e91e 100644 --- a/scripts/tests/test_benchmark_reporting.py +++ b/scripts/tests/test_benchmark_reporting.py @@ -15,7 +15,6 @@ def load_compare_module(): - sys.path.insert(0, str(COMPARE_SCRIPT.parent)) spec = importlib.util.spec_from_file_location("compare_benchmark_jsons", COMPARE_SCRIPT) assert spec is not None module = importlib.util.module_from_spec(spec) From 7f3ef0464ee1186ceda320480dd15ee10da9e187 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 28 Jul 2026 16:44:49 +0100 Subject: [PATCH 11/11] more error cases Signed-off-by: Adam Gutglick --- .github/actions/upload-parca-debuginfo/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/upload-parca-debuginfo/action.yml b/.github/actions/upload-parca-debuginfo/action.yml index a034c30e828..40369a8227e 100644 --- a/.github/actions/upload-parca-debuginfo/action.yml +++ b/.github/actions/upload-parca-debuginfo/action.yml @@ -102,8 +102,8 @@ runs: cat "$upload_log" if [[ "$status" -ne 0 ]]; then - if grep -Eiq "upload id mismatch|already exists|AlreadyExists" "$upload_log"; then - echo "::notice::Debuginfo already uploaded for $binary; continuing" + if grep -Eiq "upload id mismatch|already exists|AlreadyExists|previous upload is still in-progress|not stale yet|only stale uploads can be retried" "$upload_log"; then + echo "::notice::Debuginfo upload already exists or is in progress for $binary; continuing" else exit "$status" fi