From 9680f3fb2096189e78da3c2a42ade4ec1d709466 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 31 Jul 2026 12:53:08 +0530 Subject: [PATCH 1/3] adding iterations for perf test --- .github/workflows/test-runner.yml | 39 +++++++++++++++--- .github/workflows/windows-benchmark.yml | 55 ++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index b3aa1e7dbd..2f4cea2af2 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -114,6 +114,14 @@ jobs: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-benchmark-existing cancel-in-progress: true + # Runs *after* windows-benchmark-bq rather than alongside it. Both shards + # benchmark against the same BigQuery project, so running them concurrently + # made each one's numbers depend on the other's API load -- and when that + # tripped rate limits, retry backoff turned the contention into + # multi-second steps. The dependency deliberately does not require the + # BqDriver shard to succeed: '!cancelled()' plus the windows-cmake guard + # preserves the previous run conditions, so the existing-driver baseline is + # still produced when that shard fails or is skipped. if: | !cancelled() && github.event_name == 'workflow_dispatch' && @@ -169,9 +177,19 @@ jobs: import re def parse_gtest_output(filepath): - results = {} + # The suite is run several times (--gtest_repeat), so each test + # appears once per repetition. Take the median of its timings. + # + # These benchmarks talk to a live BigQuery service and fan list + # calls out concurrently, so a single run samples the latency + # tail and moved tens of percent between runs for no code + # reason. The median rejects a single outlier repetition, + # including the cold-cache first one. Files with only one run per + # test (a --gtest_repeat=1 run, or a baseline recorded before + # this change) still work: the median of one sample is itself. + samples = {} if not os.path.exists(filepath): - return results + return {} pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)') try: @@ -179,11 +197,22 @@ jobs: for line in f: match = pattern.search(line) if match: - test_name = match.group(1) - time_taken = match.group(2) - results[test_name] = time_taken + ms = parse_time_to_ms(match.group(2)) + if ms is not None: + samples.setdefault(match.group(1), []).append(ms) except Exception as e: print(f'Error reading {filepath}: {e}') + + results = {} + for test_name, values in samples.items(): + values.sort() + n = len(values) + median = (values[n // 2] if n % 2 == 1 + else (values[n // 2 - 1] + values[n // 2]) / 2.0) + results[test_name] = f'{median}ms' + if n > 1: + print(f'{test_name}: median={median:.0f}ms of {n} runs ' + f'(min={values[0]:.0f}ms max={values[-1]:.0f}ms)') return results def parse_time_to_ms(time_str): diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index a78ecc4300..f3ce9d0336 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -12,9 +12,14 @@ on: description: "The driver shard to test (Core or BqDriver)" type: string default: "BqDriver" - secrets: - BUILD_CACHE_KEY: - required: true + benchmark_iterations: + required: false + description: >- + How many times to run the whole suite (gtest --gtest_repeat). The + results table reports the median per test, which rejects a single + outlier run. Total runtime scales linearly with this. + type: string + default: "3" workflow_dispatch: inputs: build_shard: @@ -25,6 +30,11 @@ on: - BqDriver - Core default: "BqDriver" + benchmark_iterations: + description: "How many times to run the suite; median is reported" + required: false + type: string + default: "3" permissions: contents: read @@ -33,11 +43,16 @@ jobs: run-benchmarks: name: Run ODBC Performance Benchmarks (${{ inputs.build_shard }}) runs-on: windows-2022 - timeout-minutes: 60 + # The suite is now repeated benchmark_iterations times so the results + # table can take a median, so it takes proportionally longer. The timeout + # is sized for that and, more importantly, bounds a hang: without it a + # stuck run holds the runner until GitHub's 6-hour default. + timeout-minutes: 180 env: DRIVER_ARCH: x64 BUILD_SHARD: ${{ inputs.build_shard }} ODBC_GOOGLE_DRIVER_VERSION: 99.99.99 + BENCHMARK_ITERATIONS: ${{ inputs.benchmark_iterations || '3' }} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -143,9 +158,37 @@ jobs: EXE_PATH="c:/b/google/cloud/odbc/integration_tests/Release/performance_test.exe" echo "Running performance benchmark executable against $BUILD_SHARD..." + echo " repeats=${BENCHMARK_ITERATIONS}" set +e - "$EXE_PATH" > "$RESULTS_FILE" 2>&1 - TEST_EXIT_CODE=$? + # Run the suite BENCHMARK_ITERATIONS times and let the results parser + # take the median of each test's timings. A single run is dominated by + # BigQuery/service latency variance, so one sample per test moved tens + # of percent between runs for no code reason. + # + # Deliberately separate processes rather than --gtest_repeat: repeating + # in-process re-allocates and frees SQL_HANDLE_ENV once per test, so + # the Driver Manager loads and unloads the driver DLL on every test. + # Tripling that churn crashed the Simba driver mid-run (abort, exit 3). + # A fresh process per repetition keeps that count identical to a + # single-shot run, and an iteration that dies still leaves the other + # iterations' timings in the results file. + # + # tee -a appends each run; the file is truncated first. tee so progress + # is visible in the live job log -- previously all output went to a + # file only uploaded at the end, making a slow run indistinguishable + # from a hung one. PIPESTATUS keeps the executable's exit code, not + # tee's. + : > "$RESULTS_FILE" + TEST_EXIT_CODE=0 + for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" | tee -a "$RESULTS_FILE" + "$EXE_PATH" 2>&1 | tee -a "$RESULTS_FILE" + RUN_EXIT=${PIPESTATUS[0]} + if [ $RUN_EXIT -ne 0 ]; then + echo "WARNING: iteration ${i} exited with code ${RUN_EXIT}" + TEST_EXIT_CODE=$RUN_EXIT + fi + done set -e echo "Uploading results to GCS..." From 6c07edd708d33bc44f3ab887c49dfc6f79c9ca88 Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 31 Jul 2026 13:08:41 +0530 Subject: [PATCH 2/3] minor changes --- .github/workflows/test-runner.yml | 3 ++- .github/workflows/windows-benchmark.yml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 2f4cea2af2..2e83278521 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -126,7 +126,7 @@ jobs: !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_benchmark_existing == true - needs: [pre-flight] + needs: [pre-flight, windows-benchmark-bq] uses: ./.github/workflows/windows-benchmark.yml with: checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }} @@ -308,3 +308,4 @@ jobs: run: | gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/ echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/benchmark_summary_table.txt" + \ No newline at end of file diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index f3ce9d0336..99dca2e8ee 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -20,6 +20,9 @@ on: outlier run. Total runtime scales linearly with this. type: string default: "3" + secrets: + BUILD_CACHE_KEY: + required: true workflow_dispatch: inputs: build_shard: @@ -198,3 +201,4 @@ jobs: echo "ERROR: Benchmark executable failed with exit code $TEST_EXIT_CODE." exit $TEST_EXIT_CODE fi + \ No newline at end of file From 1e5c7e32ac2663424fea4816f37d4a7840dc64ad Mon Sep 17 00:00:00 2001 From: Anshu6250 Date: Fri, 31 Jul 2026 15:30:21 +0530 Subject: [PATCH 3/3] reverted the console output --- .github/workflows/test-runner.yml | 1 - .github/workflows/windows-benchmark.yml | 13 +++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test-runner.yml b/.github/workflows/test-runner.yml index 2e83278521..3107e75467 100644 --- a/.github/workflows/test-runner.yml +++ b/.github/workflows/test-runner.yml @@ -308,4 +308,3 @@ jobs: run: | gcloud storage cp benchmark_summary_table.txt gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/ echo "Uploaded benchmark table to gs://bq-dev-tools-testing-drivers/odbc-perf/$BRANCH_NAME/results/benchmark_summary_table.txt" - \ No newline at end of file diff --git a/.github/workflows/windows-benchmark.yml b/.github/workflows/windows-benchmark.yml index 99dca2e8ee..d1fe3784fa 100644 --- a/.github/workflows/windows-benchmark.yml +++ b/.github/workflows/windows-benchmark.yml @@ -175,18 +175,12 @@ jobs: # A fresh process per repetition keeps that count identical to a # single-shot run, and an iteration that dies still leaves the other # iterations' timings in the results file. - # - # tee -a appends each run; the file is truncated first. tee so progress - # is visible in the live job log -- previously all output went to a - # file only uploaded at the end, making a slow run indistinguishable - # from a hung one. PIPESTATUS keeps the executable's exit code, not - # tee's. : > "$RESULTS_FILE" TEST_EXIT_CODE=0 for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do - echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" | tee -a "$RESULTS_FILE" - "$EXE_PATH" 2>&1 | tee -a "$RESULTS_FILE" - RUN_EXIT=${PIPESTATUS[0]} + echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" >> "$RESULTS_FILE" + "$EXE_PATH" >> "$RESULTS_FILE" 2>&1 + RUN_EXIT=$? if [ $RUN_EXIT -ne 0 ]; then echo "WARNING: iteration ${i} exited with code ${RUN_EXIT}" TEST_EXIT_CODE=$RUN_EXIT @@ -201,4 +195,3 @@ jobs: echo "ERROR: Benchmark executable failed with exit code $TEST_EXIT_CODE." exit $TEST_EXIT_CODE fi - \ No newline at end of file