feat(evals): M2 — Parquet-snapshot fast seed (cache-backed test)

.changeset/eval-parquet-snapshot-seed.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/hdx-eval": minor
+---
+
+Add a Parquet-snapshot fast-seed path for the MCP eval suite (Milestone 2 — kill the seed bottleneck). Full-volume seed generation is a CPU-bound JS loop that takes hours; the snapshot path generates the data once, exports each eval table to Parquet, and on later runs loads the Parquet straight into ClickHouse (I/O-bound, ~an order of magnitude faster). New CLI commands: `export-snapshot`, `load-snapshot`, and `seed-logic-hash`. The load path drops the rollup materialized views, bulk-inserts the raw Parquet (no per-block MV fan-out), then backfills the rollup tables in one shot using the same SELECTs the MVs run, and recreates the MVs — producing byte-identical rollup content much faster than letting the MVs fan out on the bulk insert. In CI the snapshot is stored in an `actions/cache` entry keyed on a hash of the seed-generation source, so it is reused across runs and regenerated only when the seeding logic changes.

.github/workflows/evals.yml

@@ -18,9 +18,11 @@ on:
         type: string
         default: latency-spike
       volume_factor:
-        description: 'Seed volume factor (fraction of full volume)'
+        description:
+          'Snapshot generation volume factor (1 = full; lower = cheaper test
+          seed)'
         type: string
-        default: '0.01'
+        default: '1'
       runs:
         description: 'Runs per (scenario, MCP) cell'
         type: string
@@ -40,13 +42,22 @@ jobs:
       contents: read
       pull-requests: write
     env:
-      # M1 defaults; workflow_dispatch inputs override on manual runs.
+      # M2: single scenario for the fast-seed test, FULL volume via snapshot.
       HDX_EVAL_SCENARIO: ${{ github.event.inputs.scenario || 'latency-spike' }}
-      HDX_EVAL_VOLUME_FACTOR: ${{ github.event.inputs.volume_factor || '0.01' }}
       HDX_EVAL_RUNS: ${{ github.event.inputs.runs || '1' }}
       HDX_EVAL_OUTPUT_DIR: ${{ github.workspace }}/eval-output
       HDX_EVALS_OUTPUT_DIR: ${{ github.workspace }}/eval-output
       HDX_EVALS_RUNS_DIR: ${{ github.workspace }}/eval-output/runs
+      # M2 Parquet snapshot fast-seed path.
+      HDX_EVALS_SNAPSHOT_DIR: ${{ github.workspace }}/eval-snapshot
+      # Path INSIDE the runner container (matches the compose mount target).
+      HDX_EVAL_SNAPSHOT_DIR: /work/eval-snapshot
+      # Generate at FULL volume on a snapshot miss (the point of M2). Override
+      # via workflow_dispatch input for a cheaper test.
+      HDX_EVAL_SNAPSHOT_VOLUME_FACTOR:
+        ${{ github.event.inputs.volume_factor || '1' }}
+      # Stable anchor so cached snapshot timestamps stay valid across runs.
+      HDX_EVAL_ANCHOR: '2026-06-01T00:00:00Z'
     steps:
       - name: Checkout
         uses: actions/checkout@v6
@@ -59,13 +70,45 @@ jobs:
       - name: Expose GitHub Actions cache to BuildKit
         uses: crazy-max/ghaction-github-runtime@v4
 
-      - name: Prepare output dir
+      - name: Prepare output + snapshot dirs
         # World-writable so the runner container's non-root `node` user (uid
-        # 1000) can write runs/ + verdict.md into these bind mounts regardless
-        # of the host runner's uid.
+        # 1000) can write runs/ + verdict.md + the exported snapshot into these
+        # bind mounts regardless of the host runner's uid.
         run: |
           mkdir -p "${HDX_EVAL_OUTPUT_DIR}/runs"
-          chmod -R 777 "${HDX_EVAL_OUTPUT_DIR}"
+          mkdir -p "${HDX_EVALS_SNAPSHOT_DIR}"
+          chmod -R 777 "${HDX_EVAL_OUTPUT_DIR}" "${HDX_EVALS_SNAPSHOT_DIR}"
+
+      # Compute the seed-logic hash — the cache key. It changes ONLY when the
+      # seed-generation source changes, so a cached snapshot is reused across
+      # runs and regenerated exactly when (and only when) seeding logic changes.
+      # hashFiles() here must cover the same files the CLI's seedLogicHash()
+      # hashes (see src/clickhouse/parquetSnapshot.ts SEED_LOGIC_GLOBS).
+      - name: Compute seed-logic hash
+        id: seedhash
+        run: |
+          HASH="${{ hashFiles(
+            'packages/hdx-eval/src/generators/**',
+            'packages/hdx-eval/src/scenarios/**',
+            'packages/hdx-eval/src/rng/**',
+            'packages/hdx-eval/src/clickhouse/insert.ts',
+            'packages/hdx-eval/src/clickhouse/schema.ts',
+            'packages/hdx-eval/src/clickhouse/parquetSnapshot.ts'
+          ) }}"
+          echo "hash=$HASH" >> "$GITHUB_OUTPUT"
+          echo "Seed-logic hash: $HASH"
+
+      # Restore (and, at job end, save) the full-volume Parquet snapshot keyed
+      # on the seed-logic hash + scenario. No restore-keys: a stale snapshot is
+      # never partially reused across a logic change — a miss regenerates.
+      - name: Restore/save eval seed snapshot
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.HDX_EVALS_SNAPSHOT_DIR }}
+          key:
+            eval-seed-${{ env.HDX_EVAL_SCENARIO }}-v${{
+            env.HDX_EVAL_SNAPSHOT_VOLUME_FACTOR }}-${{
+            steps.seedhash.outputs.hash }}
 
       # Build both DISTINCT images with GHA layer caching. The HyperDX
       # all-in-one build is heavy (OTel collector + Next.js); caching keeps

docker-compose.evals.yml

@@ -107,11 +107,22 @@ services:
       HDX_EVAL_OUT_COMMENT: /work/eval-output/verdict.md
       HDX_EVAL_RUN_URL: ${HDX_EVAL_RUN_URL:-}
       HDX_EVAL_COMMIT_SHA: ${HDX_EVAL_COMMIT_SHA:-}
+      HDX_EVAL_ANCHOR: ${HDX_EVAL_ANCHOR:-2026-06-01T00:00:00Z}
+      # M2 Parquet snapshot fast-seed path. When HDX_EVAL_SNAPSHOT_DIR is set,
+      # the runner loads a cached snapshot (fast) or generates+exports one
+      # (miss). The workflow backs this dir with actions/cache keyed on the
+      # seed-logic hash. Empty by default → M1 low-volume live seed.
+      HDX_EVAL_SNAPSHOT_DIR: ${HDX_EVAL_SNAPSHOT_DIR:-}
+      HDX_EVAL_SNAPSHOT_VOLUME_FACTOR: ${HDX_EVAL_SNAPSHOT_VOLUME_FACTOR:-1}
     volumes:
       # Persist runs + rendered verdict to the host so CI can upload artifacts
       # and post the PR comment.
       - ${HDX_EVALS_OUTPUT_DIR:-./eval-output}:/work/eval-output
       - ${HDX_EVALS_RUNS_DIR:-./eval-output/runs}:/work/packages/hdx-eval/runs
+      # Parquet snapshot dir (host-side, cached by the workflow). Mounted even
+      # when unused so the container path is stable; the runner only touches it
+      # when HDX_EVAL_SNAPSHOT_DIR points inside it.
+      - ${HDX_EVALS_SNAPSHOT_DIR:-./eval-snapshot}:/work/eval-snapshot
     networks:
       - evals
 

docker/hdx-eval-runner/run-evals.sh

@@ -23,7 +23,7 @@
 #                           ClickHouse HTTP URL as the HyperDX API sees it
 #                           (e.g. http://localhost:8123 for the all-in-one image)
 #
-# Optional env (with M1 defaults):
+# Optional env (with defaults):
 #   HDX_EVAL_SCENARIO        default: latency-spike
 #   HDX_EVAL_VOLUME_FACTOR   default: 0.01  (1% of full volume — scale up later)
 #   HDX_EVAL_RUNS            default: 1
@@ -33,6 +33,20 @@
 #   HDX_EVAL_OUT_COMMENT     default: /work/eval-output/verdict.md
 #   HDX_EVAL_RUN_URL         workflow run URL (for the comment footer)
 #   HDX_EVAL_COMMIT_SHA      commit SHA (for the comment footer)
+#   HDX_EVAL_ANCHOR          fixed anchor ISO for deterministic seed timestamps
+#                            (default: 2026-06-01T00:00:00Z — must be stable so a
+#                            cached snapshot's timestamps stay valid across runs)
+#
+# ── M2 Parquet snapshot (fast seed) ────────────────────────────────────────
+#   HDX_EVAL_SNAPSHOT_DIR    when set, enables the Parquet snapshot fast path.
+#                            The workflow restores/saves this dir as an
+#                            actions/cache entry keyed on the seed-logic hash.
+#                            - dir has a manifest.json  → LOAD it (fast path)
+#                            - dir empty/no manifest    → GENERATE at full volume,
+#                              then EXPORT so the cache-save persists it
+#   HDX_EVAL_SNAPSHOT_VOLUME_FACTOR
+#                            volume factor to GENERATE at on a snapshot miss.
+#                            default: 1 (FULL volume — the whole point of M2)
 #
 # The verdict is completion-only and ADVISORY: this script exits 0 as long as
 # it produced a summary + verdict, even if the verdict is FAIL. Genuine
@@ -48,6 +62,10 @@ MAX_TURNS="${HDX_EVAL_MAX_TURNS:-15}"
 TIMEOUT_MS="${HDX_EVAL_TIMEOUT_MS:-600000}"
 MCP="${HDX_EVAL_MCP:-hyperdx}"
 OUT_COMMENT="${HDX_EVAL_OUT_COMMENT:-/work/eval-output/verdict.md}"
+# Fixed anchor so a cached snapshot's seeded timestamps stay valid across runs.
+ANCHOR="${HDX_EVAL_ANCHOR:-2026-06-01T00:00:00Z}"
+SNAPSHOT_DIR="${HDX_EVAL_SNAPSHOT_DIR:-}"
+SNAPSHOT_VOLUME_FACTOR="${HDX_EVAL_SNAPSHOT_VOLUME_FACTOR:-1}"
 
 : "${ANTHROPIC_API_KEY:?ANTHROPIC_API_KEY is required}"
 : "${HDX_EVAL_API_URL:?HDX_EVAL_API_URL is required (e.g. http://hyperdx:8000)}"
@@ -78,19 +96,46 @@ echo "::group::[1/5] Setup"
   --connection-ch-url "$CONNECTION_CH_URL"
 echo "::endgroup::"
 
-echo "::group::[2/5] Seed (low volume)"
-"${CLI[@]}" seed "$SCENARIO" --volume-factor "$VOLUME_FACTOR"
+echo "::group::[2/5] Seed"
+if [ -n "$SNAPSHOT_DIR" ]; then
+  # ── M2 Parquet snapshot fast path ──
+  # The workflow restores $SNAPSHOT_DIR from actions/cache (keyed on the
+  # seed-logic hash) BEFORE this container runs. If the cache hit, the dir has
+  # a manifest.json and we LOAD it (fast). If it missed, the dir is empty and
+  # we GENERATE at full volume then EXPORT, so the workflow's cache-save step
+  # persists the snapshot for the next run.
+  SCEN_SNAP_DIR="$SNAPSHOT_DIR/$SCENARIO"
+  if [ -f "$SCEN_SNAP_DIR/manifest.json" ]; then
+    echo "Snapshot cache HIT → loading Parquet from $SCEN_SNAP_DIR"
+    "${CLI[@]}" load-snapshot "$SCENARIO" --dir "$SCEN_SNAP_DIR"
+  else
+    echo "Snapshot cache MISS → generating full-volume seed (factor $SNAPSHOT_VOLUME_FACTOR) then exporting Parquet"
+    "${CLI[@]}" seed "$SCENARIO" \
+      --volume-factor "$SNAPSHOT_VOLUME_FACTOR" \
+      --now "$ANCHOR"
+    "${CLI[@]}" export-snapshot "$SCENARIO" \
+      --dir "$SCEN_SNAP_DIR" \
+      --volume-factor "$SNAPSHOT_VOLUME_FACTOR" \
+      --anchor "$ANCHOR"
+  fi
+else
+  # M1 path: plain low-volume live seed (no snapshot caching).
+  "${CLI[@]}" seed "$SCENARIO" --volume-factor "$VOLUME_FACTOR" --now "$ANCHOR"
+fi
 echo "::endgroup::"
 
 echo "::group::[3/5+4/5] Run + Grade"
 # `run` auto-grades and auto-reports unless told otherwise. We let it do the
-# grade + report inline so a single command drives stages 3–5. --reseed is not
-# passed: the seed above already loaded data, and run reuses it.
+# grade + report inline so a single command drives stages 3–5. Pin the anchor
+# to the SAME value the seed used so the agent's "now" matches the seeded
+# timestamps. Data already exists (seeded or snapshot-loaded above), so `run`
+# skips its auto-seed — no --reseed.
 "${CLI[@]}" run "$SCENARIO" \
   --mcp "$MCP" \
   --runs "$RUNS" \
   --max-turns "$MAX_TURNS" \
-  --timeout "$TIMEOUT_MS"
+  --timeout "$TIMEOUT_MS" \
+  --anchor-time "$ANCHOR"
 echo "::endgroup::"
 
 # Find the batch produced by the run (newest directory under runs/).

packages/hdx-eval/src/__tests__/parquetSnapshot.test.ts

@@ -0,0 +1,86 @@
+import { mkdtempSync, rmSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+
+import {
+  manifestPath,
+  readManifest,
+  seedLogicHash,
+  seedLogicHashShort,
+  snapshotFileName,
+  type SnapshotManifest,
+  snapshotTableFields,
+  writeManifest,
+} from '@/clickhouse/parquetSnapshot';
+
+describe('seedLogicHash', () => {
+  it('is a stable 64-char hex sha256', () => {
+    const h = seedLogicHash();
+    expect(h).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it('is deterministic across calls (content-based, not time-based)', () => {
+    expect(seedLogicHash()).toBe(seedLogicHash());
+  });
+
+  it('short form is the 12-char prefix', () => {
+    expect(seedLogicHashShort()).toBe(seedLogicHash().slice(0, 12));
+    expect(seedLogicHashShort()).toHaveLength(12);
+  });
+});
+
+describe('snapshot table fields', () => {
+  it('covers the raw data tables and excludes rollups', () => {
+    const fields = snapshotTableFields();
+    expect(fields).toContain('traces');
+    expect(fields).toContain('logs');
+    expect(fields).toContain('metricsGauge');
+    // Rollup tables must NOT be snapshotted — they repopulate via MVs on load.
+    expect(fields).not.toContain('tracesKvRollup');
+    expect(fields).not.toContain('tracesKeyRollup');
+    expect(fields).not.toContain('logsKvRollup');
+    expect(fields).not.toContain('logsKeyRollup');
+  });
+});
+
+describe('snapshotFileName', () => {
+  it('maps a table name to <table>.parquet', () => {
+    expect(snapshotFileName('eval_latency_spike_otel_traces')).toBe(
+      'eval_latency_spike_otel_traces.parquet',
+    );
+  });
+});
+
+describe('manifest round-trip', () => {
+  let dir: string;
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'hdx-eval-snap-'));
+  });
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('writes and reads back an identical manifest', () => {
+    const manifest: SnapshotManifest = {
+      scenarioName: 'latency-spike',
+      seedLogicHash: seedLogicHash(),
+      volumeFactor: 1,
+      seed: 42,
+      anchorMs: Date.parse('2026-06-01T00:00:00Z'),
+      anchorIso: '2026-06-01T00:00:00.000Z',
+      createdAt: new Date().toISOString(),
+      tables: [
+        { table: 'eval_latency_spike_otel_traces', rows: 1000, bytes: 5000 },
+      ],
+      totalRows: 1000,
+      totalBytes: 5000,
+    };
+    writeManifest(dir, manifest);
+    expect(readManifest(dir)).toEqual(manifest);
+    expect(manifestPath(dir)).toBe(join(dir, 'manifest.json'));
+  });
+
+  it('returns null when no manifest is present', () => {
+    expect(readManifest(dir)).toBeNull();
+  });
+});