From 3165223de739392c696494553b634f018892a640 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 7 Jul 2026 15:02:32 +0200 Subject: [PATCH 1/3] ci: add incremental update benchmark with PR-vs-main comparison Extracts the incremental update benchmark from #1648 (with the opt-in perf tracing removed, since those hooks only exist on that branch) and adds a workflow that runs it on non-draft PRs touching packages/db, packages/db-ivm, or the benchmark itself. The workflow benchmarks the PR merge commit and its merge-base with the same benchmark code, then posts a comparison as a job summary and sticky PR comment. Co-Authored-By: Claude Fable 5 --- .github/workflows/benchmark.yml | 95 ++ scripts/bench/compare-incremental-update.ts | 344 +++++++ scripts/bench/incremental-update.ts | 1004 +++++++++++++++++++ 3 files changed, 1443 insertions(+) create mode 100644 .github/workflows/benchmark.yml create mode 100644 scripts/bench/compare-incremental-update.ts create mode 100644 scripts/bench/incremental-update.ts diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..26d5ae08ab --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,95 @@ +name: Benchmark + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + paths: + - 'packages/db/**' + - 'packages/db-ivm/**' + - 'scripts/bench/**' + - '.github/workflows/benchmark.yml' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + benchmark: + name: Incremental Update Benchmark + # Skip draft PRs; the benchmark runs once a PR is marked ready for review + # (and on subsequent pushes to it). + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.draft == false + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Setup Tools + uses: tanstack/config/.github/setup@e4b48f16568324f76f467aa4c2aac2f05db632c3 + - name: Run benchmark on PR + run: | + pnpm --filter @tanstack/db-ivm build + pnpm exec tsx scripts/bench/incremental-update.ts \ + --iterations=100 --warmup=20 \ + --outFile="$RUNNER_TEMP/candidate.json" + - name: Run benchmark on base + env: + BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }} + run: | + HEAD_SHA=$(git rev-parse HEAD) + BASE_SHA=$(git merge-base "origin/$BASE_REF" HEAD) + echo "Benchmarking base commit $BASE_SHA" + git checkout --force "$BASE_SHA" + # Run the exact same benchmark code on both sides + git checkout "$HEAD_SHA" -- scripts/bench + pnpm install --frozen-lockfile + pnpm --filter @tanstack/db-ivm build + pnpm exec tsx scripts/bench/incremental-update.ts \ + --iterations=100 --warmup=20 \ + --outFile="$RUNNER_TEMP/base.json" + - name: Compare results + run: | + pnpm exec tsx scripts/bench/compare-incremental-update.ts \ + --base="$RUNNER_TEMP/base.json" \ + --candidate="$RUNNER_TEMP/candidate.json" \ + --outFile="$RUNNER_TEMP/comparison.md" + cat "$RUNNER_TEMP/comparison.md" >> "$GITHUB_STEP_SUMMARY" + - name: Upload reports + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-reports + path: | + ${{ runner.temp }}/base.json + ${{ runner.temp }}/candidate.json + ${{ runner.temp }}/comparison.md + - name: Comment on PR + if: github.event_name == 'pull_request' + # PRs from forks get a read-only token; the job summary and artifacts + # still carry the results. + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: | + MARKER='' + BODY_FILE="$RUNNER_TEMP/comment.md" + { echo "$MARKER"; cat "$RUNNER_TEMP/comparison.md"; } > "$BODY_FILE" + COMMENT_ID=$(gh api "repos/$REPO/issues/$PR_NUMBER/comments" --paginate \ + --jq ".[] | select(.body | startswith(\"$MARKER\")) | .id" | head -n1) + if [ -n "$COMMENT_ID" ]; then + gh api -X PATCH "repos/$REPO/issues/comments/$COMMENT_ID" -F body=@"$BODY_FILE" > /dev/null + else + gh api "repos/$REPO/issues/$PR_NUMBER/comments" -F body=@"$BODY_FILE" > /dev/null + fi diff --git a/scripts/bench/compare-incremental-update.ts b/scripts/bench/compare-incremental-update.ts new file mode 100644 index 0000000000..b349dddeca --- /dev/null +++ b/scripts/bench/compare-incremental-update.ts @@ -0,0 +1,344 @@ +/** + * Compares two JSON reports produced by `scripts/bench/incremental-update.ts` + * and prints a markdown summary of the differences. + * + * Usage: + * tsx scripts/bench/compare-incremental-update.ts \ + * --base=.tmp/perf/base.json \ + * --candidate=.tmp/perf/candidate.json \ + * [--outFile=.tmp/perf/comparison.md] \ + * [--threshold=0.20] \ + * [--failOnRegression=false] + */ +import { readFileSync, writeFileSync } from 'node:fs' + +type Summary = { + iterations: number + medianMs: number + p75Ms: number + p95Ms: number + minMs: number + maxMs: number + stddevMs: number +} + +type FixtureScale = { + label: string + issueCount: number + userCount: number + commentCount: number +} + +type RunResult = { + query: string + scenario: string + scale: FixtureScale + sourceIndexMode: string + mutationMode: string + coldHydrateMs: number + writeSummary: Summary +} + +type Report = { + metadata: { + node: string + platform: string + cpu: string + gitSha: string + iterations: number + warmup: number + } + results: Array +} + +type Comparison = { + key: string + query: string + scenario: string + scale: FixtureScale + sourceIndexMode: string + mutationMode: string + base: RunResult + candidate: RunResult + medianRatio: number + p95Ratio: number +} + +// Relative change below which a result is considered noise, and an absolute +// floor so sub-hundredth-of-a-ms jitter never counts as a change. +const defaultThreshold = 0.2 +const absoluteFloorMs = 0.05 + +const args = parseArgs(process.argv.slice(2)) + +const base = readReport(args.base) +const candidate = readReport(args.candidate) + +const comparisons = compareReports(base, candidate) +const markdown = formatMarkdown(base, candidate, comparisons, args.threshold) + +if (args.outFile) { + writeFileSync(args.outFile, markdown) +} +console.log(markdown) + +const regressions = comparisons.filter((comparison) => + isRegression(comparison, args.threshold), +) +if (args.failOnRegression && regressions.length > 0) { + console.error(`\n${regressions.length} benchmark regression(s) found`) + process.exit(1) +} + +function readReport(path: string): Report { + return JSON.parse(readFileSync(path, `utf8`)) as Report +} + +function resultKey(result: RunResult): string { + return [ + result.query, + result.scenario, + result.scale.label, + result.scale.issueCount, + result.scale.userCount, + result.scale.commentCount, + result.sourceIndexMode, + result.mutationMode, + ].join(`|`) +} + +function compareReports( + baseReport: Report, + candidateReport: Report, +): Array { + const baseByKey = new Map( + baseReport.results.map((result) => [resultKey(result), result]), + ) + + const matched: Array = [] + for (const result of candidateReport.results) { + const key = resultKey(result) + const baseResult = baseByKey.get(key) + if (!baseResult) continue + + matched.push({ + key, + query: result.query, + scenario: result.scenario, + scale: result.scale, + sourceIndexMode: result.sourceIndexMode, + mutationMode: result.mutationMode, + base: baseResult, + candidate: result, + medianRatio: ratio( + baseResult.writeSummary.medianMs, + result.writeSummary.medianMs, + ), + p95Ratio: ratio(baseResult.writeSummary.p95Ms, result.writeSummary.p95Ms), + }) + } + + return matched +} + +function ratio(baseMs: number, candidateMs: number): number { + if (baseMs <= 0) return candidateMs <= 0 ? 1 : Number.POSITIVE_INFINITY + return candidateMs / baseMs +} + +function isSignificant( + baseMs: number, + candidateMs: number, + threshold: number, +): boolean { + const relativeChange = Math.abs(ratio(baseMs, candidateMs) - 1) + const absoluteChange = Math.abs(candidateMs - baseMs) + return relativeChange > threshold && absoluteChange > absoluteFloorMs +} + +function isRegression(comparison: Comparison, threshold: number): boolean { + return ( + comparison.medianRatio > 1 && + isSignificant( + comparison.base.writeSummary.medianMs, + comparison.candidate.writeSummary.medianMs, + threshold, + ) + ) +} + +function isImprovement(comparison: Comparison, threshold: number): boolean { + return ( + comparison.medianRatio < 1 && + isSignificant( + comparison.base.writeSummary.medianMs, + comparison.candidate.writeSummary.medianMs, + threshold, + ) + ) +} + +function marker(comparison: Comparison, threshold: number): string { + if (isRegression(comparison, threshold)) return `🔴` + if (isImprovement(comparison, threshold)) return `🟢` + return `` +} + +function formatMarkdown( + baseReport: Report, + candidateReport: Report, + allComparisons: Array, + threshold: number, +): string { + const lines: Array = [] + const regressed = allComparisons.filter((comparison) => + isRegression(comparison, threshold), + ) + const improved = allComparisons.filter((comparison) => + isImprovement(comparison, threshold), + ) + + lines.push(`## Incremental update benchmark`) + lines.push(``) + lines.push( + `Comparing \`${candidateReport.metadata.gitSha}\` (this PR) against \`${baseReport.metadata.gitSha}\` (base). ` + + `Times are per-write medians over ${candidateReport.metadata.iterations} iterations ` + + `(${candidateReport.metadata.warmup} warmup writes).`, + ) + lines.push(``) + + if (regressed.length === 0 && improved.length === 0) { + lines.push( + `**No significant changes** (threshold: ±${Math.round(threshold * 100)}% and >${absoluteFloorMs}ms).`, + ) + } else { + lines.push( + `**${regressed.length} regression(s), ${improved.length} improvement(s)** ` + + `(threshold: ±${Math.round(threshold * 100)}% and >${absoluteFloorMs}ms).`, + ) + } + lines.push(``) + + const groups = new Map>() + for (const comparison of allComparisons) { + const groupKey = `${formatScale(comparison.scale)} | source indexes: ${ + comparison.sourceIndexMode + } | ${comparison.mutationMode} writes` + const group = groups.get(groupKey) ?? [] + group.push(comparison) + groups.set(groupKey, group) + } + + for (const [groupKey, group] of groups) { + lines.push(`
`) + const changed = group.filter( + (comparison) => marker(comparison, threshold) !== ``, + ).length + const summarySuffix = changed > 0 ? ` — ${changed} change(s)` : `` + lines.push(`${groupKey}${summarySuffix}`) + lines.push(``) + lines.push( + `| Query | Base median | PR median | Δ median | Base p95 | PR p95 | Δ p95 | |`, + ) + lines.push(`| --- | ---: | ---: | ---: | ---: | ---: | ---: | --- |`) + for (const comparison of group) { + lines.push( + `| ${comparison.query} | ${formatMs( + comparison.base.writeSummary.medianMs, + )} | ${formatMs(comparison.candidate.writeSummary.medianMs)} | ${formatDelta( + comparison.medianRatio, + )} | ${formatMs(comparison.base.writeSummary.p95Ms)} | ${formatMs( + comparison.candidate.writeSummary.p95Ms, + )} | ${formatDelta(comparison.p95Ratio)} | ${marker( + comparison, + threshold, + )} |`, + ) + } + lines.push(``) + lines.push(`
`) + lines.push(``) + } + + lines.push( + `Runner: node ${candidate.metadata.node}, ${candidate.metadata.platform}, ${candidate.metadata.cpu}. ` + + `Timings on shared CI runners are noisy; treat small deltas as indicative only.`, + ) + + return lines.join(`\n`) +} + +function formatScale(scale: FixtureScale): string { + if ( + scale.issueCount === scale.userCount && + scale.userCount === scale.commentCount + ) { + return `${scale.issueCount.toLocaleString(`en-US`)} rows/collection` + } + return `issues:${scale.issueCount} users:${scale.userCount} comments:${scale.commentCount}` +} + +function formatMs(value: number): string { + return `${value.toFixed(3)}ms` +} + +function formatDelta(value: number): string { + if (!Number.isFinite(value)) return `n/a` + const percent = (value - 1) * 100 + const sign = percent > 0 ? `+` : `` + return `${sign}${percent.toFixed(1)}%` +} + +function parseArgs(argv: Array): { + base: string + candidate: string + outFile?: string + threshold: number + failOnRegression: boolean +} { + let basePath: string | undefined + let candidatePath: string | undefined + let outFile: string | undefined + let threshold = defaultThreshold + let failOnRegression = false + + for (const arg of argv) { + const [name, value] = arg.replace(/^--/, ``).split(`=`) + if (!name || value === undefined) continue + + switch (name) { + case `base`: + basePath = value + break + case `candidate`: + candidatePath = value + break + case `outFile`: + outFile = value + break + case `threshold`: { + const parsed = Number(value) + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error(`--threshold must be a positive number`) + } + threshold = parsed + break + } + case `failOnRegression`: + failOnRegression = value === `true` + break + } + } + + if (!basePath || !candidatePath) { + throw new Error(`--base and --candidate report paths are required`) + } + + return { + base: basePath, + candidate: candidatePath, + outFile, + threshold, + failOnRegression, + } +} diff --git a/scripts/bench/incremental-update.ts b/scripts/bench/incremental-update.ts new file mode 100644 index 0000000000..70ab0d5132 --- /dev/null +++ b/scripts/bench/incremental-update.ts @@ -0,0 +1,1004 @@ +import { execSync } from 'node:child_process' +import { mkdirSync, writeFileSync } from 'node:fs' +import { cpus, platform, release } from 'node:os' +import { dirname, join } from 'node:path' +import { + BTreeIndex, + count, + createCollection, + createLiveQueryCollection, + createTransaction, + eq, + materialize, +} from '../../packages/db/src/index.js' +import type { + ChangeMessageOrDeleteKeyMessage, + Collection, + SyncConfig, +} from '../../packages/db/src/index.js' + +type Issue = { + id: number + status: `open` | `closed` + authorId: number + createdAt: number + title: string + body: string + updateTick: number +} + +type User = { + id: number + name: string + reputation: number +} + +type Comment = { + id: number + issueId: number + authorId: number + createdAt: number + body: string + updateTick: number +} + +type ManualSyncUtils = { + begin: Parameters[`sync`]>[0][`begin`] + write: (message: ChangeMessageOrDeleteKeyMessage) => void + commit: Parameters[`sync`]>[0][`commit`] +} + +type ManualCollection< + T extends object, + TKey extends string | number, +> = Collection, never, T> + +type BenchmarkCollection = Collection, string | number> + +type Fixture = { + seed: number + issues: ManualCollection + users: ManualCollection + comments: ManualCollection + issueById: Map + userById: Map + commentById: Map + visibleIssueId: number + selectedIssueId: number + nextCommentId: number +} + +type BenchmarkOptions = { + seed: number + levels: Array + sourceIndexModes: Array + mutationModes: Array + issueCount?: number + userCount?: number + commentCount?: number + warmup: number + iterations: number + outDir: string + outFile?: string +} + +type FixtureScale = { + label: string + issueCount: number + userCount: number + commentCount: number +} + +type BenchmarkRunOptions = { + seed: number + scale: FixtureScale + issueCount: number + userCount: number + commentCount: number + sourceIndexMode: SourceIndexMode + warmup: number + iterations: number +} + +type Summary = { + iterations: number + medianMs: number + p75Ms: number + p95Ms: number + minMs: number + maxMs: number + stddevMs: number +} + +type RunResult = { + query: string + scenario: string + scale: FixtureScale + sourceIndexMode: SourceIndexMode + mutationMode: MutationMode + coldHydrateMs: number + writeSummary: Summary +} + +type MutationMode = `synced` | `optimistic` +type SourceIndexMode = `none` | `manual` | `auto` + +type WriteCleanup = () => void | Promise + +type WriteResult = void | { cleanup: WriteCleanup } + +type QueryCase = { + name: string + scenario: string + createQuery: (fixture: Fixture) => BenchmarkCollection + write: ( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, + ) => WriteResult +} + +const defaultOptions: BenchmarkOptions = { + seed: 42, + levels: [100, 1_000, 10_000], + sourceIndexModes: [`none`, `manual`], + mutationModes: [`synced`, `optimistic`], + warmup: 10, + iterations: 50, + outDir: `.tmp/perf`, +} + +const defaultCustomScale: FixtureScale = { + label: `custom`, + issueCount: 10_000, + userCount: 1_000, + commentCount: 40_000, +} + +const options = parseArgs(process.argv.slice(2), defaultOptions) + +const queryCases: Array = [ + { + name: `list: newest 50 open`, + scenario: `visible issue update`, + createQuery: ({ issues }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + createdAt: issue.createdAt, + })), + ) as unknown as BenchmarkCollection, + write: updateVisibleIssue, + }, + { + name: `list + author`, + scenario: `visible author update`, + createQuery: ({ issues, users }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .join({ author: users }, ({ issue, author }) => + eq(issue.authorId, author.id), + ) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue, author }) => ({ + id: issue.id, + title: issue.title, + createdAt: issue.createdAt, + authorName: author.name, + })), + ) as unknown as BenchmarkCollection, + write: updateVisibleAuthor, + }, + { + name: `list + comment count`, + scenario: `visible issue comment insert`, + createQuery: ({ issues, comments }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + commentCount: materialize( + q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .select(({ comment }) => count(comment.id)) + .findOne(), + ), + })), + ) as unknown as BenchmarkCollection, + write: insertVisibleComment, + }, + { + name: `list + 3 recent comments`, + scenario: `visible issue comment insert`, + createQuery: ({ issues, comments }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + recentComments: q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .orderBy(({ comment }) => comment.createdAt, `desc`) + .limit(3) + .select(({ comment }) => ({ + id: comment.id, + body: comment.body, + createdAt: comment.createdAt, + })), + })), + ) as unknown as BenchmarkCollection, + write: insertVisibleComment, + }, + { + name: `issue detail + comments`, + scenario: `selected issue comment insert`, + createQuery: ({ issues, comments, selectedIssueId }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.id, selectedIssueId)) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + comments: q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .orderBy(({ comment }) => comment.createdAt, `desc`) + .select(({ comment }) => ({ + id: comment.id, + body: comment.body, + createdAt: comment.createdAt, + })), + })), + ) as unknown as BenchmarkCollection, + write: insertSelectedIssueComment, + }, +] + +const scales = resolveScales(options) +const results: Array = [] +for (const scale of scales) { + for (const sourceIndexMode of options.sourceIndexModes) { + const runOptions = createRunOptions(options, scale, sourceIndexMode) + for (const mutationMode of options.mutationModes) { + for (const queryCase of queryCases) { + results.push(await runCase(queryCase, mutationMode, runOptions)) + } + } + } +} + +const report = { + metadata: runtimeMetadata(options, scales), + results, +} + +const outputPath = + options.outFile ?? + join(options.outDir, `incremental-update-${Date.now()}.json`) +mkdirSync(dirname(outputPath), { recursive: true }) +writeFileSync(outputPath, JSON.stringify(report, null, 2)) + +console.log(formatTextReport(report, outputPath)) + +async function runCase( + queryCase: QueryCase, + mutationMode: MutationMode, + options: BenchmarkRunOptions, +): Promise { + const fixture = createFixture(options) + const query = queryCase.createQuery(fixture) + + const coldStart = performance.now() + await query.preload() + await flushMicrotasks() + const coldHydrateMs = performance.now() - coldStart + + for (let i = 0; i < options.warmup; i++) { + const writeResult = queryCase.write(fixture, i, mutationMode) + await flushMicrotasks() + await cleanupWrite(writeResult) + } + + const samples: Array = [] + for (let i = 0; i < options.iterations; i++) { + const start = performance.now() + const writeResult = queryCase.write( + fixture, + options.warmup + i, + mutationMode, + ) + await flushMicrotasks() + samples.push(performance.now() - start) + await cleanupWrite(writeResult) + } + + await cleanupFixture(fixture, query) + + return { + query: queryCase.name, + scenario: queryCase.scenario, + scale: options.scale, + sourceIndexMode: options.sourceIndexMode, + mutationMode, + coldHydrateMs, + writeSummary: summarize(samples), + } +} + +function createManualCollection( + id: string, + initialData: Array, + getKey: (item: T) => TKey, + sourceIndexMode: SourceIndexMode, +): ManualCollection { + let begin: Parameters[`sync`]>[0][`begin`] | undefined + let write: Parameters[`sync`]>[0][`write`] | undefined + let commit: Parameters[`sync`]>[0][`commit`] | undefined + + const utils: ManualSyncUtils = { + begin: (syncOptions) => begin!(syncOptions), + write: (message) => write!(message), + commit: () => commit!(), + } + + return createCollection>({ + id, + getKey, + utils, + startSync: true, + autoIndex: sourceIndexMode === `auto` ? `eager` : `off`, + defaultIndexType: BTreeIndex, + sync: { + rowUpdateMode: `full`, + sync: (methods) => { + begin = methods.begin + write = methods.write + commit = methods.commit + + begin() + for (const value of initialData) { + write({ + type: `insert`, + value, + }) + } + commit() + methods.markReady() + + return () => { + begin = undefined + write = undefined + commit = undefined + } + }, + }, + }) +} + +function createFixture(options: BenchmarkRunOptions): Fixture { + const random = seededRandom(options.seed) + const usersData: Array = [] + for (let id = 1; id <= options.userCount; id++) { + usersData.push({ + id, + name: `User ${id}`, + reputation: Math.floor(random() * 10_000), + }) + } + + const issuesData: Array = [] + for (let id = 1; id <= options.issueCount; id++) { + issuesData.push({ + id, + status: id % 3 === 0 ? `closed` : `open`, + authorId: 1 + (id % options.userCount), + createdAt: id, + title: `Issue ${id}`, + body: `Body ${id}`, + updateTick: 0, + }) + } + + const visibleIssueId = findNewestOpenIssueId(issuesData) + const selectedIssueId = visibleIssueId + const commentsData: Array = [] + for (let id = 1; id <= options.commentCount; id++) { + const hotIssueId = + id % 5 === 0 + ? 1 + Math.floor(random() * Math.min(options.issueCount, 50)) + : 1 + Math.floor(random() * options.issueCount) + commentsData.push({ + id, + issueId: hotIssueId, + authorId: 1 + (id % options.userCount), + createdAt: id, + body: `Comment ${id}`, + updateTick: 0, + }) + } + + const fixture = { + seed: options.seed, + issues: createManualCollection( + `bench-issues`, + issuesData, + (issue) => issue.id, + options.sourceIndexMode, + ), + users: createManualCollection( + `bench-users`, + usersData, + (user) => user.id, + options.sourceIndexMode, + ), + comments: createManualCollection( + `bench-comments`, + commentsData, + (comment) => comment.id, + options.sourceIndexMode, + ), + issueById: new Map(issuesData.map((issue) => [issue.id, issue])), + userById: new Map(usersData.map((user) => [user.id, user])), + commentById: new Map(commentsData.map((comment) => [comment.id, comment])), + visibleIssueId, + selectedIssueId, + nextCommentId: options.commentCount + 1, + } + + applySourceIndexes(fixture, options.sourceIndexMode) + + return fixture +} + +function applySourceIndexes( + fixture: Fixture, + sourceIndexMode: SourceIndexMode, +): void { + if (sourceIndexMode !== `manual`) return + + fixture.issues.createIndex((issue) => issue.id) + fixture.issues.createIndex((issue) => issue.status) + fixture.issues.createIndex((issue) => issue.authorId) + fixture.issues.createIndex((issue) => issue.createdAt) + fixture.users.createIndex((user) => user.id) + fixture.comments.createIndex((comment) => comment.issueId) + fixture.comments.createIndex((comment) => comment.createdAt) +} + +function updateVisibleIssue( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + const issue = fixture.issueById.get(fixture.visibleIssueId)! + const next = { + ...issue, + title: `Issue ${issue.id} tick ${iteration}`, + updateTick: iteration, + } + fixture.issueById.set(issue.id, next) + + if (mutationMode === `synced`) { + writeSync(fixture.issues, { + type: `update`, + value: next, + }) + return + } + + const transaction = writeOptimistic(() => { + fixture.issues.update(issue.id, (draft) => { + draft.title = next.title + draft.updateTick = next.updateTick + }) + }) + + return { + cleanup: () => { + transaction.rollback() + fixture.issueById.set(issue.id, issue) + }, + } +} + +function updateVisibleAuthor( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + const issue = fixture.issueById.get(fixture.visibleIssueId)! + const user = fixture.userById.get(issue.authorId)! + const next = { + ...user, + name: `User ${user.id} tick ${iteration}`, + reputation: user.reputation + 1, + } + fixture.userById.set(user.id, next) + + if (mutationMode === `synced`) { + writeSync(fixture.users, { + type: `update`, + value: next, + }) + return + } + + const transaction = writeOptimistic(() => { + fixture.users.update(user.id, (draft) => { + draft.name = next.name + draft.reputation = next.reputation + }) + }) + + return { + cleanup: () => { + transaction.rollback() + fixture.userById.set(user.id, user) + }, + } +} + +function insertVisibleComment( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + return insertCommentForIssue( + fixture, + fixture.visibleIssueId, + iteration, + mutationMode, + ) +} + +function insertSelectedIssueComment( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + return insertCommentForIssue( + fixture, + fixture.selectedIssueId, + iteration, + mutationMode, + ) +} + +function insertCommentForIssue( + fixture: Fixture, + issueId: number, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + const comment: Comment = { + id: fixture.nextCommentId++, + issueId, + authorId: 1 + (iteration % fixture.userById.size), + createdAt: 1_000_000_000 + iteration, + body: `Inserted comment ${iteration}`, + updateTick: iteration, + } + fixture.commentById.set(comment.id, comment) + + if (mutationMode === `synced`) { + writeSync(fixture.comments, { + type: `insert`, + value: comment, + }) + return + } + + const transaction = writeOptimistic(() => { + fixture.comments.insert(comment) + }) + + return { + cleanup: () => { + transaction.rollback() + fixture.commentById.delete(comment.id) + }, + } +} + +function writeSync( + collection: ManualCollection, + message: ChangeMessageOrDeleteKeyMessage, +): void { + collection.utils.begin({ immediate: true }) + collection.utils.write(message) + collection.utils.commit() +} + +function writeOptimistic(write: () => void) { + const transaction = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + transaction.isPersisted.promise.catch(() => undefined) + transaction.mutate(write) + return transaction +} + +async function cleanupWrite(writeResult: WriteResult): Promise { + if (!writeResult) return + await writeResult.cleanup() + await flushMicrotasks() +} + +async function cleanupFixture( + fixture: Fixture, + query: BenchmarkCollection, +): Promise { + await Promise.all([ + query.cleanup(), + fixture.issues.cleanup(), + fixture.users.cleanup(), + fixture.comments.cleanup(), + ]) +} + +function summarize(samples: Array): Summary { + const sorted = [...samples].sort((a, b) => a - b) + const total = sorted.reduce((sum, value) => sum + value, 0) + const mean = total / sorted.length + const variance = + sorted.reduce((sum, value) => sum + (value - mean) ** 2, 0) / sorted.length + return { + iterations: sorted.length, + medianMs: percentile(sorted, 0.5), + p75Ms: percentile(sorted, 0.75), + p95Ms: percentile(sorted, 0.95), + minMs: sorted[0] ?? 0, + maxMs: sorted[sorted.length - 1] ?? 0, + stddevMs: Math.sqrt(variance), + } +} + +function percentile(sorted: Array, p: number): number { + if (sorted.length === 0) return 0 + const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * p) - 1) + return sorted[index]! +} + +function runtimeMetadata( + options: BenchmarkOptions, + scales: Array, +) { + return { + seed: options.seed, + levels: options.levels, + scales, + sourceIndexModes: options.sourceIndexModes, + mutationModes: options.mutationModes, + warmup: options.warmup, + iterations: options.iterations, + node: process.version, + platform: `${platform()} ${release()}`, + cpu: cpus()[0]?.model ?? `unknown`, + gitSha: gitSha(), + gcAvailable: typeof global.gc === `function`, + } +} + +function formatTextReport( + report: { + metadata: ReturnType + results: Array + }, + outputPath: string, +): string { + const lines: Array = [] + lines.push(`Incremental update benchmark`) + lines.push(`seed=${report.metadata.seed}`) + lines.push(`scales=${report.metadata.scales.map(formatScale).join(`; `)}`) + lines.push(`sourceIndexModes=${report.metadata.sourceIndexModes.join(`,`)}`) + lines.push(`mutationModes=${report.metadata.mutationModes.join(`,`)}`) + lines.push( + `runtime=node ${report.metadata.node} ${report.metadata.platform} ${report.metadata.cpu}`, + ) + lines.push(`git=${report.metadata.gitSha}`) + lines.push(`json=${outputPath}`) + lines.push(``) + + for (const scale of uniqueScales(report.results)) { + lines.push(`scale=${formatScale(scale)}`) + for (const sourceIndexMode of report.metadata.sourceIndexModes) { + lines.push(`sourceIndexMode=${sourceIndexMode}`) + for (const mutationMode of report.metadata.mutationModes) { + lines.push(`mutationMode=${mutationMode}`) + for (const result of report.results.filter( + (item) => + scaleKey(item.scale) === scaleKey(scale) && + item.sourceIndexMode === sourceIndexMode && + item.mutationMode === mutationMode, + )) { + lines.push(`${result.query} | ${result.scenario}`) + lines.push( + ` cold=${formatMs(result.coldHydrateMs)} median=${formatMs( + result.writeSummary.medianMs, + )} p95=${formatMs(result.writeSummary.p95Ms)} min=${formatMs( + result.writeSummary.minMs, + )} max=${formatMs(result.writeSummary.maxMs)} stddev=${formatMs( + result.writeSummary.stddevMs, + )}`, + ) + } + lines.push(``) + } + } + } + + return lines.join(`\n`) +} + +function resolveScales(options: BenchmarkOptions): Array { + const hasCustomScale = + options.issueCount !== undefined || + options.userCount !== undefined || + options.commentCount !== undefined + + if (hasCustomScale) { + return [ + { + label: `custom`, + issueCount: options.issueCount ?? defaultCustomScale.issueCount, + userCount: options.userCount ?? defaultCustomScale.userCount, + commentCount: options.commentCount ?? defaultCustomScale.commentCount, + }, + ] + } + + return options.levels.map((level) => ({ + label: String(level), + issueCount: level, + userCount: level, + commentCount: level, + })) +} + +function createRunOptions( + options: BenchmarkOptions, + scale: FixtureScale, + sourceIndexMode: SourceIndexMode, +): BenchmarkRunOptions { + return { + seed: options.seed, + scale, + issueCount: scale.issueCount, + userCount: scale.userCount, + commentCount: scale.commentCount, + sourceIndexMode, + warmup: options.warmup, + iterations: options.iterations, + } +} + +function uniqueScales(results: Array): Array { + const scales = new Map() + for (const result of results) { + scales.set(scaleKey(result.scale), result.scale) + } + return [...scales.values()] +} + +function scaleKey(scale: FixtureScale): string { + return `${scale.label}:${scale.issueCount}:${scale.userCount}:${scale.commentCount}` +} + +function formatScale(scale: FixtureScale): string { + if ( + scale.issueCount === scale.userCount && + scale.userCount === scale.commentCount + ) { + return `${formatInteger(scale.issueCount)} rows/collection` + } + + return `issues:${formatInteger(scale.issueCount)} users:${formatInteger( + scale.userCount, + )} comments:${formatInteger(scale.commentCount)}` +} + +function formatMs(value: number): string { + return `${formatNumber(value)}ms` +} + +function formatNumber(value: number): string { + return value.toFixed(3) +} + +function formatInteger(value: number): string { + return value.toLocaleString(`en-US`) +} + +function parseArgs( + args: Array, + defaults: BenchmarkOptions, +): BenchmarkOptions { + const parsed = { + ...defaults, + levels: [...defaults.levels], + sourceIndexModes: [...defaults.sourceIndexModes], + mutationModes: [...defaults.mutationModes], + } + + for (const arg of args) { + const [name, value] = arg.replace(/^--/, ``).split(`=`) + if (!name || value === undefined) continue + + switch (name) { + case `seed`: + parsed.seed = parseNonNegativeInteger(value, name) + break + case `levels`: + parsed.levels = parseLevels(value) + break + case `sourceIndexes`: + case `sourceIndexModes`: + parsed.sourceIndexModes = parseSourceIndexModes(value) + break + case `mutationModes`: + parsed.mutationModes = parseMutationModes(value) + break + case `issues`: + parsed.issueCount = parsePositiveCount(value, name) + break + case `users`: + parsed.userCount = parsePositiveCount(value, name) + break + case `comments`: + parsed.commentCount = parsePositiveCount(value, name) + break + case `warmup`: + parsed.warmup = parseNonNegativeInteger(value, name) + break + case `iterations`: + parsed.iterations = parsePositiveInteger(value, name) + break + case `outDir`: + parsed.outDir = value + break + case `outFile`: + parsed.outFile = value + break + } + } + + return parsed +} + +function parseLevels(value: string): Array { + const levels = value + .split(`,`) + .map((part) => part.trim()) + .filter((part) => part.length > 0) + .map((part) => parsePositiveCount(part, `levels`)) + + if (levels.length === 0) { + throw new Error(`--levels must contain at least one positive integer`) + } + + return levels +} + +function parseMutationModes(value: string): Array { + const modes = value + .split(`,`) + .map((part) => part.trim()) + .filter((part) => part.length > 0) + + if (modes.length === 0) { + throw new Error(`--mutationModes must contain at least one mode`) + } + + for (const mode of modes) { + if (mode !== `synced` && mode !== `optimistic`) { + throw new Error(`--mutationModes must contain synced and/or optimistic`) + } + } + + return modes +} + +function parseSourceIndexModes(value: string): Array { + const modes = value + .split(`,`) + .map((part) => part.trim()) + .filter((part) => part.length > 0) + + if (modes.length === 0) { + throw new Error(`--sourceIndexes must contain at least one mode`) + } + + for (const mode of modes) { + if (mode !== `none` && mode !== `manual` && mode !== `auto`) { + throw new Error(`--sourceIndexes must contain none, manual, and/or auto`) + } + } + + return modes +} + +function parsePositiveCount(value: string, name: string): number { + const normalized = value.trim().toLowerCase() + const parsed = normalized.endsWith(`k`) + ? Number(normalized.slice(0, -1)) * 1_000 + : Number(normalized) + + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`--${name} must be a positive integer or k-suffixed count`) + } + return parsed +} + +function parsePositiveInteger(value: string, name: string): number { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`--${name} must be a positive integer`) + } + return parsed +} + +function parseNonNegativeInteger(value: string, name: string): number { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`--${name} must be a non-negative integer`) + } + return parsed +} + +function seededRandom(seed: number): () => number { + let state = seed >>> 0 + return () => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return ((state >>> 0) % 1_000_000) / 1_000_000 + } +} + +function findNewestOpenIssueId(issues: Array): number { + for (let index = issues.length - 1; index >= 0; index--) { + const issue = issues[index]! + if (issue.status === `open`) { + return issue.id + } + } + throw new Error(`Fixture must contain at least one open issue`) +} + +function gitSha(): string { + try { + return execSync(`git rev-parse --short HEAD`, { + encoding: `utf8`, + stdio: [`ignore`, `pipe`, `ignore`], + }).trim() + } catch { + return `unknown` + } +} + +async function flushMicrotasks(): Promise { + await Promise.resolve() + await Promise.resolve() +} From 4c1de9506159ebd2bb8df2e20781cbe48bd67e82 Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 7 Jul 2026 15:05:26 +0200 Subject: [PATCH 2/3] ci: add geomean ratio to benchmark comparison summary Per-row significance flags miss consistent sub-floor shifts across many cases; the geometric mean of per-case median ratios surfaces them. Co-Authored-By: Claude Fable 5 --- scripts/bench/compare-incremental-update.ts | 31 +++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/scripts/bench/compare-incremental-update.ts b/scripts/bench/compare-incremental-update.ts index b349dddeca..eeb88a3975 100644 --- a/scripts/bench/compare-incremental-update.ts +++ b/scripts/bench/compare-incremental-update.ts @@ -184,6 +184,23 @@ function marker(comparison: Comparison, threshold: number): string { return `` } +// Individual rows are noisy at sub-ms timings, but a consistent shift across +// many rows is a real change even when no single row clears the per-row +// significance bar — the geometric mean of the ratios captures that. +function geomeanRatio(group: Array): number { + const finite = group + .map((comparison) => comparison.medianRatio) + .filter((value) => Number.isFinite(value) && value > 0) + if (finite.length === 0) return 1 + return Math.exp( + finite.reduce((sum, value) => sum + Math.log(value), 0) / finite.length, + ) +} + +function formatRatio(value: number): string { + return `${value.toFixed(2)}×` +} + function formatMarkdown( baseReport: Report, candidateReport: Report, @@ -218,6 +235,12 @@ function formatMarkdown( ) } lines.push(``) + lines.push( + `Overall median write time vs base: **${formatRatio( + geomeanRatio(allComparisons), + )}** (geometric mean of per-case ratios; lower is faster).`, + ) + lines.push(``) const groups = new Map>() for (const comparison of allComparisons) { @@ -234,8 +257,12 @@ function formatMarkdown( const changed = group.filter( (comparison) => marker(comparison, threshold) !== ``, ).length - const summarySuffix = changed > 0 ? ` — ${changed} change(s)` : `` - lines.push(`${groupKey}${summarySuffix}`) + const changedSuffix = changed > 0 ? `, ${changed} change(s)` : `` + lines.push( + `${groupKey} — geomean ${formatRatio( + geomeanRatio(group), + )}${changedSuffix}`, + ) lines.push(``) lines.push( `| Query | Base median | PR median | Δ median | Base p95 | PR p95 | Δ p95 | |`, From a7e958465ebcc7879288552c9919d2f0b71bbb2a Mon Sep 17 00:00:00 2001 From: Kevin De Porre Date: Tue, 7 Jul 2026 15:26:32 +0200 Subject: [PATCH 3/3] ci: add per-query rollup table to benchmark comment The per-configuration tables are collapsed, so the comment showed no per-query numbers without expanding; a visible rollup (geomean, spread, flags per query) carries the signal. Co-Authored-By: Claude Fable 5 --- scripts/bench/compare-incremental-update.ts | 42 +++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/scripts/bench/compare-incremental-update.ts b/scripts/bench/compare-incremental-update.ts index eeb88a3975..6f2dd880d2 100644 --- a/scripts/bench/compare-incremental-update.ts +++ b/scripts/bench/compare-incremental-update.ts @@ -242,6 +242,48 @@ function formatMarkdown( ) lines.push(``) + const byQuery = new Map>() + for (const comparison of allComparisons) { + const group = byQuery.get(comparison.query) ?? [] + group.push(comparison) + byQuery.set(comparison.query, group) + } + + lines.push(`| Query | Δ median (geomean) | Best case | Worst case | Flags |`) + lines.push(`| --- | ---: | ---: | ---: | :-- |`) + for (const [query, group] of byQuery) { + const ratios = group + .map((comparison) => comparison.medianRatio) + .filter((value) => Number.isFinite(value)) + const best = Math.min(...ratios) + const worst = Math.max(...ratios) + const redCount = group.filter((comparison) => + isRegression(comparison, threshold), + ).length + const greenCount = group.filter((comparison) => + isImprovement(comparison, threshold), + ).length + const flags = + [ + redCount > 0 ? `${redCount} 🔴` : ``, + greenCount > 0 ? `${greenCount} 🟢` : ``, + ] + .filter(Boolean) + .join(` `) || `—` + lines.push( + `| ${query} | ${formatDelta(geomeanRatio(group))} | ${formatDelta( + best, + )} | ${formatDelta(worst)} | ${flags} |`, + ) + } + lines.push(``) + lines.push( + `Each row aggregates the ${allComparisons.length / byQuery.size || 0} ` + + `scale/index/write-mode configurations of that query; per-configuration ` + + `tables below.`, + ) + lines.push(``) + const groups = new Map>() for (const comparison of allComparisons) { const groupKey = `${formatScale(comparison.scale)} | source indexes: ${