test: adding iterations in perf testing

.github/workflows/test-runner.yml

@@ -114,11 +114,19 @@ jobs:
     concurrency:
       group: ${{ github.workflow }}-${{ github.ref }}-benchmark-existing
       cancel-in-progress: true
+    # Runs *after* windows-benchmark-bq rather than alongside it. Both shards
+    # benchmark against the same BigQuery project, so running them concurrently
+    # made each one's numbers depend on the other's API load -- and when that
+    # tripped rate limits, retry backoff turned the contention into
+    # multi-second steps. The dependency deliberately does not require the
+    # BqDriver shard to succeed: '!cancelled()' plus the windows-cmake guard
+    # preserves the previous run conditions, so the existing-driver baseline is
+    # still produced when that shard fails or is skipped.
     if: |
       !cancelled() &&
       github.event_name == 'workflow_dispatch' &&
       inputs.run_benchmark_existing == true
-    needs: [pre-flight]
+    needs: [pre-flight, windows-benchmark-bq]
     uses: ./.github/workflows/windows-benchmark.yml
     with:
       checkout-ref: ${{ needs.pre-flight.outputs.checkout-sha }}
@@ -169,21 +177,42 @@ jobs:
           import re
 
           def parse_gtest_output(filepath):
-              results = {}
+              # The suite is run several times (--gtest_repeat), so each test
+              # appears once per repetition. Take the median of its timings.
+              #
+              # These benchmarks talk to a live BigQuery service and fan list
+              # calls out concurrently, so a single run samples the latency
+              # tail and moved tens of percent between runs for no code
+              # reason. The median rejects a single outlier repetition,
+              # including the cold-cache first one. Files with only one run per
+              # test (a --gtest_repeat=1 run, or a baseline recorded before
+              # this change) still work: the median of one sample is itself.
+              samples = {}
               if not os.path.exists(filepath):
-                  return results
+                  return {}
 
               pattern = re.compile(r'\[\s+OK\s+\]\s+(\S+)\s+\(([^)]+)\)')
               try:
                   with open(filepath, 'r') as f:
                       for line in f:
                           match = pattern.search(line)
                           if match:
-                              test_name = match.group(1)
-                              time_taken = match.group(2)
-                              results[test_name] = time_taken
+                              ms = parse_time_to_ms(match.group(2))
+                              if ms is not None:
+                                  samples.setdefault(match.group(1), []).append(ms)
               except Exception as e:
                   print(f'Error reading {filepath}: {e}')
+
+              results = {}
+              for test_name, values in samples.items():
+                  values.sort()
+                  n = len(values)
+                  median = (values[n // 2] if n % 2 == 1
+                            else (values[n // 2 - 1] + values[n // 2]) / 2.0)
+                  results[test_name] = f'{median}ms'
+                  if n > 1:
+                      print(f'{test_name}: median={median:.0f}ms of {n} runs '
+                            f'(min={values[0]:.0f}ms max={values[-1]:.0f}ms)')
               return results
 
           def parse_time_to_ms(time_str):

.github/workflows/windows-benchmark.yml

@@ -12,6 +12,14 @@ on:
         description: "The driver shard to test (Core or BqDriver)"
         type: string
         default: "BqDriver"
+      benchmark_iterations:
+        required: false
+        description: >-
+          How many times to run the whole suite (gtest --gtest_repeat). The
+          results table reports the median per test, which rejects a single
+          outlier run. Total runtime scales linearly with this.
+        type: string
+        default: "3"
     secrets:
       BUILD_CACHE_KEY:
         required: true
@@ -25,6 +33,11 @@ on:
           - BqDriver
           - Core
         default: "BqDriver"
+      benchmark_iterations:
+        description: "How many times to run the suite; median is reported"
+        required: false
+        type: string
+        default: "3"
 
 permissions:
   contents: read
@@ -33,11 +46,16 @@ jobs:
   run-benchmarks:
     name: Run ODBC Performance Benchmarks (${{ inputs.build_shard }})
     runs-on: windows-2022
-    timeout-minutes: 60
+    # The suite is now repeated benchmark_iterations times so the results
+    # table can take a median, so it takes proportionally longer. The timeout
+    # is sized for that and, more importantly, bounds a hang: without it a
+    # stuck run holds the runner until GitHub's 6-hour default.
+    timeout-minutes: 180
     env:
       DRIVER_ARCH: x64
       BUILD_SHARD: ${{ inputs.build_shard }}
       ODBC_GOOGLE_DRIVER_VERSION: 99.99.99
+      BENCHMARK_ITERATIONS: ${{ inputs.benchmark_iterations || '3' }}
     steps:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
@@ -143,9 +161,31 @@ jobs:
         EXE_PATH="c:/b/google/cloud/odbc/integration_tests/Release/performance_test.exe"
 
         echo "Running performance benchmark executable against $BUILD_SHARD..."     
+        echo "  repeats=${BENCHMARK_ITERATIONS}"
         set +e
-        "$EXE_PATH" > "$RESULTS_FILE" 2>&1
-        TEST_EXIT_CODE=$?
+        # Run the suite BENCHMARK_ITERATIONS times and let the results parser
+        # take the median of each test's timings. A single run is dominated by
+        # BigQuery/service latency variance, so one sample per test moved tens
+        # of percent between runs for no code reason.
+        #
+        # Deliberately separate processes rather than --gtest_repeat: repeating
+        # in-process re-allocates and frees SQL_HANDLE_ENV once per test, so
+        # the Driver Manager loads and unloads the driver DLL on every test.
+        # Tripling that churn crashed the Simba driver mid-run (abort, exit 3).
+        # A fresh process per repetition keeps that count identical to a
+        # single-shot run, and an iteration that dies still leaves the other
+        # iterations' timings in the results file.
+        : > "$RESULTS_FILE"
+        TEST_EXIT_CODE=0
+        for i in $(seq 1 "${BENCHMARK_ITERATIONS}"); do
+          echo "=== benchmark iteration ${i}/${BENCHMARK_ITERATIONS} ===" >> "$RESULTS_FILE"
+          "$EXE_PATH" >> "$RESULTS_FILE" 2>&1
+          RUN_EXIT=$?
+          if [ $RUN_EXIT -ne 0 ]; then
+            echo "WARNING: iteration ${i} exited with code ${RUN_EXIT}"
+            TEST_EXIT_CODE=$RUN_EXIT
+          fi
+        done
         set -e
 
         echo "Uploading results to GCS..."