diff --git a/INCREMENTAL_UPDATE_FINDINGS.md b/INCREMENTAL_UPDATE_FINDINGS.md new file mode 100644 index 0000000000..da67cb1411 --- /dev/null +++ b/INCREMENTAL_UPDATE_FINDINGS.md @@ -0,0 +1,295 @@ +# Incremental Update Performance Findings + +Date: 2026-07-02 + +Branch: https://github.com/samwillis/db/tree/codex/incremental-update-instrumentation-plan + +Instrumentation commit: https://github.com/samwillis/db/commit/a13fd382a + +Source-index benchmark commit: https://github.com/samwillis/db/commit/1ee5188cf + +## Executive Summary + +We have identified the likely source of the slow incremental update results seen in the shared benchmark table. + +The incremental hot path is doing O(source-collection-size) snapshot work on every write. In particular, both synced and optimistic paths clone `rowOrigins` for the mutated collection: + +- synced path: `collection.commitPendingTransactions.snapshotState` +- optimistic path: `collection.recomputeOptimisticState.snapshotState` + +At 100k source rows this snapshot work is about 95% of traced source-to-result time. At the estimated shared benchmark shape of `100k issues`, `100k users`, and `400k comments`, it is about 98% of traced source-to-result time. + +This makes the reported shared TanStack DB numbers plausible if their benchmark used roughly: + +- `100k` issues +- `100k` users +- `400k` comments + +That shape gives us: + +| Query | Shared TanStack DB | Our estimated-shape best | +|---|---:|---:| +| list: newest 50 open | 5.36ms | 6.25ms | +| list + author | 6.61ms | 6.07ms | +| list + comment count | 25.29ms | 28.79ms | +| list + 3 recent comments | 24.80ms | 29.26ms | +| issue detail + comments | 24.91ms | 28.83ms | + +That is close enough to strongly support the diagnosis. It is not proof because we do not have the shared benchmark source. + +## Important Caveat + +This is not a true benchmark-to-benchmark comparison. + +We do not have the source for the shared benchmark, so we do not know: + +- exact dataset sizes +- data distribution +- whether source collections had indexes +- whether auto-indexing was enabled +- whether incremental writes were synced changes, optimistic changes, or something else +- whether included shapes were fully incremental +- whether timings include rendering, scheduling, framework overhead, or only DB work +- runtime, machine, browser/Node version, warmup, and sampling method + +The comparisons below are best-effort orientation only. They are useful for diagnosis, but they should not be treated as apples-to-apples results. + +## What We Added + +The benchmark covers the externally reported query shapes: + +- `list: newest 50 open` +- `list + author` +- `list + comment count` +- `list + 3 recent comments` +- `issue detail + comments` + +The fixture uses three source collections: + +- `issues` +- `users` +- `comments` + +The benchmark can run: + +- `synced` writes through `begin/write/commit` +- `optimistic` writes through public collection mutations, with rollback cleanup kept outside the measured sample +- source-index modes `none`, `manual`, and `auto` + +Manual source indexes are created on: + +- `issues.id` +- `issues.status` +- `issues.authorId` +- `issues.createdAt` +- `users.id` +- `comments.issueId` +- `comments.createdAt` + +## Benchmark Commands + +Full incremental trace run: + +```sh +pnpm exec tsx scripts/bench/incremental-update.ts +``` + +Cold hydrate source-index comparison: + +```sh +pnpm exec tsx scripts/bench/incremental-update.ts --mutationModes=synced --sourceIndexes=none,manual --outDir=.tmp/perf-index-cold +``` + +100k uniform run: + +```sh +pnpm exec tsx scripts/bench/incremental-update.ts --levels=100k --sourceIndexes=manual --mutationModes=synced,optimistic --outDir=.tmp/perf-100k +``` + +Estimated shared-shape run: + +```sh +pnpm exec tsx scripts/bench/incremental-update.ts --issues=100k --users=100k --comments=400k --sourceIndexes=manual --mutationModes=synced,optimistic --outDir=.tmp/perf-estimated-shape +``` + +Relevant local result files: + +```txt +.tmp/perf/incremental-update-1782978638218.json +.tmp/perf-index-cold/incremental-update-1782979711311.json +.tmp/perf-100k/incremental-update-1782981193934.json +.tmp/perf-estimated-shape/incremental-update-1782981555034.json +``` + +Run metadata: + +- seed: `42` +- warmup: `10` +- iterations: `50` +- runtime: Node `v22.13.0` +- platform: `darwin 24.6.0` +- CPU: `Apple M2` +- `global.gc`: not available + +## Shared External Table + +The shared screenshot reports these incremental update numbers: + +| Query | Shared TanStack DB | Rindle | +|---|---:|---:| +| list: newest 50 open | 5.36ms | 0.113ms | +| list + author | 6.61ms | 0.133ms | +| list + comment count | 25.29ms | 0.086ms | +| list + 3 recent comments | 24.80ms | 0.129ms | +| issue detail + comments | 24.91ms | 0.072ms | + +## Small Sizes Compared With Rindle + +At 100 and 1k rows per source collection, our synthetic TanStack DB benchmark is broadly competitive with or faster than the shared Rindle column. + +### 100 Rows Per Collection + +| Query | Rindle | Our synced | Synced read | Our optimistic | Optimistic read | +|---|---:|---:|---:|---:|---:| +| list: newest 50 open | 0.113 | 0.062 | 1.83x faster | 0.084 | 1.34x faster | +| list + author | 0.133 | 0.100 | 1.33x faster | 0.087 | 1.52x faster | +| list + comment count | 0.086 | 0.074 | 1.17x faster | 0.066 | 1.31x faster | +| list + 3 recent comments | 0.129 | 0.055 | 2.34x faster | 0.052 | 2.50x faster | +| issue detail + comments | 0.072 | 0.042 | 1.70x faster | 0.039 | 1.85x faster | + +### 1,000 Rows Per Collection + +| Query | Rindle | Our synced | Synced read | Our optimistic | Optimistic read | +|---|---:|---:|---:|---:|---:| +| list: newest 50 open | 0.113 | 0.067 | 1.70x faster | 0.083 | 1.36x faster | +| list + author | 0.133 | 0.072 | 1.86x faster | 0.087 | 1.53x faster | +| list + comment count | 0.086 | 0.079 | 1.08x faster | 0.087 | 1.01x slower | +| list + 3 recent comments | 0.129 | 0.070 | 1.84x faster | 0.070 | 1.83x faster | +| issue detail + comments | 0.072 | 0.066 | 1.08x faster | 0.063 | 1.14x faster | + +The issue is not visible at small source sizes. The cliff appears as source collections grow. + +## 100k Uniform Run + +With `100k` rows in each source collection and manual source indexes, incremental writes are around `6ms` regardless of query shape. + +| Query | Shared TanStack DB | Rindle | Our synced | Our optimistic | Best vs shared TanStack DB | Best vs Rindle | +|---|---:|---:|---:|---:|---:|---:| +| list: newest 50 open | 5.36 | 0.113 | 6.088 | 6.170 | 1.14x | 53.9x slower | +| list + author | 6.61 | 0.133 | 6.570 | 6.197 | 0.94x | 46.6x slower | +| list + comment count | 25.29 | 0.086 | 6.326 | 6.501 | 0.25x | 73.6x slower | +| list + 3 recent comments | 24.80 | 0.129 | 5.950 | 6.348 | 0.24x | 46.1x slower | +| issue detail + comments | 24.91 | 0.072 | 6.262 | 6.510 | 0.25x | 87.0x slower | + +This explains the first two shared TanStack DB rows if their issues/users collections are around 100k rows. It does not explain the roughly 25ms comment-driven rows unless the comments collection is larger. + +## Estimated Shared-Shape Run + +We then ran the benchmark with: + +- `100k` issues +- `100k` users +- `400k` comments +- manual source indexes + +This closely reproduces the shared TanStack DB shape. + +| Query | Shared TanStack DB | Rindle | Our synced | Our optimistic | Best | Best vs shared TanStack DB | Best vs Rindle | +|---|---:|---:|---:|---:|---:|---:|---:| +| list: newest 50 open | 5.36 | 0.113 | 6.25 | 6.54 | 6.25 | 1.17x | 55x slower | +| list + author | 6.61 | 0.133 | 6.52 | 6.07 | 6.07 | 0.92x | 46x slower | +| list + comment count | 25.29 | 0.086 | 34.98 | 28.79 | 28.79 | 1.14x | 335x slower | +| list + 3 recent comments | 24.80 | 0.129 | 29.26 | 34.56 | 29.26 | 1.18x | 227x slower | +| issue detail + comments | 24.91 | 0.072 | 31.11 | 28.83 | 28.83 | 1.16x | 400x slower | + +This is strong evidence that the shared TanStack DB numbers can be explained by source collection sizes alone: + +- issue/user updates mutate a roughly 100k-row source collection and cost about 6ms +- comment inserts mutate a roughly 400k-row source collection and cost about 29ms + +Again, this is not proof of their dataset size. It is a strong fit to the observed shape. + +## Why This Happens + +The traced layer breakdown for the estimated shared-shape run shows that snapshot cloning is essentially the whole cost. + +| Mode | Total traced source time | Snapshot clone | Event emit | Query execute | Graph run | +|---|---:|---:|---:|---:|---:| +| synced | 5487ms | 5384ms, 98.1% | 82ms, 1.5% | 73ms, 1.3% | 47ms, 0.8% | +| optimistic | 5268ms | 5169ms, 98.1% | 13ms, 0.2% | 67ms, 1.3% | 42ms, 0.8% | + +The graph is not the problem in this run. Hashing is not the problem in this run. The incremental cost is dominated by cloning collection snapshot state. + +At 10k, each query case copies about 500k `rowOrigins` entries over 50 writes. + +At 100k, each query case copies about 5 million `rowOrigins` entries over 50 writes. + +In the estimated shared-shape run, comment-driven cases mutate the 400k-row comments collection, so they copy about 20 million `rowOrigins` entries over 50 writes. + +## Cold Hydrate Finding + +Cold hydrate is a separate issue. It is extremely sensitive to source indexes. + +### 10,000 Rows Per Collection + +| Query | No indexes | Manual indexes | Speedup | +|---|---:|---:|---:| +| list: newest 50 open | 31.644ms | 0.385ms | 82.2x | +| list + author | 119.735ms | 0.930ms | 128.7x | +| list + comment count | 2308.053ms | 1.322ms | 1745.9x | +| list + 3 recent comments | 2227.533ms | 3.102ms | 718.0x | +| issue detail + comments | 6.590ms | 0.280ms | 23.6x | + +Cold hydrate conclusions: + +- With source indexes, cold hydrate is fast. +- Without source indexes, included/comment shapes can become multi-second at 10k. +- Cold hydrate comparisons are not meaningful unless we know the shared benchmark's index setup. + +## Incremental Writes And Source Indexes + +Manual source indexes barely changed incremental write medians at 10k: + +| Query | No indexes | Manual indexes | +|---|---:|---:| +| list: newest 50 open | 0.574ms | 0.534ms | +| list + author | 0.578ms | 0.552ms | +| list + comment count | 0.576ms | 0.529ms | +| list + 3 recent comments | 0.528ms | 0.529ms | +| issue detail + comments | 0.548ms | 0.543ms | + +This separates the two findings: + +- source indexes are decisive for cold hydrate +- source indexes do not materially change the incremental hot spot + +## Current Diagnosis + +The most likely issue is global snapshot/diff behavior in collection state updates. + +The code should not need to clone or diff full collection-sized maps to emit events for a one-row write. The mutation and sync pipelines already know the changed keys. + +The likely fix is to move both hot paths toward changed-key event construction: + +- synced path: build events from committed operation keys and previous values/origins for those keys +- optimistic path: compute event deltas from active mutation keys rather than diffing cloned global maps + +## Recommended Next Work + +1. Remove the full `rowOrigins` clone from synced and optimistic hot paths. + +2. Replace global snapshot/diff with changed-key event construction. + +3. Keep source-index modes in the benchmark. + - `none` catches accidental full-scan hydrate behavior. + - `manual` gives a fair indexed hydrate baseline. + - `auto` lets us inspect eager auto-index behavior separately. + +4. Re-run this benchmark after the fix. + - The estimated shared-shape run should drop from about `6ms` and `29ms` toward the query graph cost, currently under `1ms` per write in aggregate. + +5. Re-run with the external benchmark source if it becomes available. + - That is the only way to make a real comparison to the shared table. + +6. Re-run with `node --expose-gc` for a cleaner memory/GC profile. + diff --git a/INCREMENTAL_UPDATE_INSTRUMENTATION_PLAN.md b/INCREMENTAL_UPDATE_INSTRUMENTATION_PLAN.md new file mode 100644 index 0000000000..d2f8631581 --- /dev/null +++ b/INCREMENTAL_UPDATE_INSTRUMENTATION_PLAN.md @@ -0,0 +1,540 @@ +# Incremental Update Instrumentation Plan + +## Context + +External benchmarks suggest TanStack DB may have slow steady-state incremental +updates for live queries. The reported query shapes are: + +- `list: newest 50 open` +- `list + author` +- `list + comment count` +- `list + 3 recent comments` +- `issue detail + comments` + +The cold hydrate numbers are slower than the comparison library, but the much +larger gap is steady-state write latency. The instrumentation should therefore +focus on explaining what one source collection write causes downstream: graph +work, operator fanout, hashing, output flushing, collection commits, includes +materialization, and subscriber event delivery. + +## Goals + +- Attribute per-write latency across the full live-query path. +- Separate D2 graph/operator cost from TanStack DB collection commit/event cost. +- Measure hashing and serialization explicitly because this has been a past + bottleneck. +- Capture cardinalities alongside timing so we can identify unnecessary fanout, + not just slow functions. +- Make the benchmark fixture deterministic enough to compare branches. +- Keep instrumentation opt-in and low overhead when disabled. + +## Non-goals + +- Do not expose tracing as a stable public API in the first pass. +- Do not optimize code while adding instrumentation, except for trivial fixes + needed to make measurement reliable. +- Do not add wall-clock performance assertions to normal unit tests. Timing can + be noisy; committed tests should assert behavior and useful cardinality + invariants instead. + +## Benchmark Shapes To Reproduce + +Build a deterministic issue tracker fixture with `issues`, `users`, and +`comments`. Scale should be configurable, but start with enough rows to expose +fanout: + +- Issues: 10k total, mixed `open` and closed. +- Users: 500 to 2k. +- Comments: 0 to many per issue, skewed distribution so some issues are hot. +- Indexes: test both the expected indexed path and a no-index/control path where + useful. + +Queries: + +1. Newest open list + - `issues where status = 'open' orderBy createdAt desc limit 50` + - Measures filter, ordered top-k, and limit placement. + +2. List plus author + - Same base list, joined to `users` on `authorId`. + - Measures whether filtering/limit happens before join and whether join + delta output is proportional to changed rows. + +3. List plus comment count + - Same base list, with `count(comments)` grouped by issue. + - Measures aggregate maintenance and whether a single comment write scans + too many comments for that issue or all issues. + +4. List plus 3 recent comments + - Same base list, with newest 3 comments per issue. + - Measures grouped top-k, nested includes, and child collection updates. + +5. Issue detail plus comments + - Single issue detail with comments. + - Measures the non-list path and whether comment-only writes update only the + issue's child collection. + +For each query, measure: + +- Cold hydrate: preload, first graph run, and first visible result. +- Steady-state write: insert, update, and delete one relevant row. +- Irrelevant write: write a row that should not affect the result. +- Boundary write: write a row that enters or exits the top 50 or top 3 window. + +### Concrete Query Sketches + +Use sketches like these as the starting point, then adjust to exact QueryBuilder +syntax during implementation. + +For included comment shapes, intentionally leave nested queries as plain +subqueries. Do not wrap them with `toArray()` for these benchmarks: plain +subqueries materialize child collections and should stay fully incremental, +whereas inline arrays force parent-row re-materialization. Do not reshape these +benchmarks into lower-level workaround queries just to help the current compiler: +the point is to measure the optimal user-facing DX. If the trace shows child +include work is driven by all filtered parents before the parent `orderBy/limit` +window is applied, treat that as a likely issue exposed by the benchmark. + +```ts +// 1. list: newest 50 open +q.from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + +// 2. list + author +q.from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .join({ author: users }, ({ issue, author }) => + eq(issue.authorId, author.id), + ) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + +// 3. list + comment count +q.from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + issue, + commentCount: materialize( + q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .select(({ comment }) => count(comment.id)) + .findOne(), + ), + })) + +// 4. list + 3 recent comments +q.from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + issue, + recentComments: q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .orderBy(({ comment }) => comment.createdAt, `desc`) + .limit(3), + })) + +// 5. issue detail + comments +q.from({ issue: issues }) + .where(({ issue }) => eq(issue.id, selectedIssueId)) + .select(({ issue }) => ({ + issue, + comments: q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .orderBy(({ comment }) => comment.createdAt, `desc`), + })) +``` + +### Write Scenarios + +Run each scenario against every query where it is relevant: + +- Issue insert outside the open/top-50 window. +- Issue insert that enters the open/top-50 window. +- Issue update that changes non-query fields on a visible issue. +- Issue update that changes `status` so a row enters or exits the list. +- Issue update that changes `createdAt` across the top-50 boundary. +- Issue delete for a visible issue. +- Author update for the author of a visible issue. +- Author update for a user with no visible issues. +- Comment insert on a visible issue. +- Comment insert on a non-visible issue. +- Comment update that does not affect comment count or ordering. +- Comment update that changes `createdAt` across the top-3 boundary. +- Comment delete from a visible issue. +- Comment insert/delete on the selected issue detail row. +- Comment insert/delete on a different issue while the detail query is active. + +### Benchmark Protocol + +The benchmark runner should be a standalone `tsx` script first, with small +Vitest unit tests for trace aggregation behavior. A standalone runner makes it +easier to control warmup, iteration count, output files, and Node flags without +turning normal test runs into noisy performance tests. + +For every benchmark case: + +- Use a fixed fixture seed and print it in the report. +- Print Node version, package manager version, platform, CPU model when + available, and git SHA. +- Run warmup iterations before measured iterations. +- Run enough measured iterations to report median, p75, p95, min, max, and + standard deviation. +- Run with tracing disabled and enabled on the same fixture to estimate trace + overhead. +- Optionally run with `--expose-gc` and call `global.gc()` between measured + cases when available; report whether GC control was active. +- Keep JSON output outside committed source by default, for example under + `.tmp/perf/`, unless the team explicitly wants checked-in snapshots. + +## Trace Architecture + +Add an internal, opt-in trace recorder with two levels: + +1. A shared low-level recorder usable from `@tanstack/db-ivm`. +2. TanStack DB live-query spans that record collection and includes work. + +Suggested files: + +- `packages/db-ivm/src/perf.ts` +- `packages/db/src/query/live/perf.ts` +- `scripts/bench/incremental-update.ts` +- `packages/db/tests/query/perf/trace-aggregation.test.ts` + +The recorder should support: + +- `span(name, tags, fn)` for sync work. +- `spanAsync(name, tags, fn)` for promise-returning work. +- `startSpan(name, tags)` returning an explicit `end(extraTags?)` handle for + work that crosses callback or subscription boundaries. +- `record(name, value, tags)` for counters and gauges. +- `reset()` and `snapshot()` for tests/benchmarks. +- A global symbol-backed sink so both packages can aggregate in the same report + without making a public API commitment. + +Enablement options: + +- Programmatic test helper for benchmarks. +- Optional environment/global flag, for example `TANSTACK_DB_TRACE=1` in Node + and `globalThis.__TANSTACK_DB_TRACE__ = true` in browser-like tests. + +Overhead rules: + +- When disabled, instrumentation should be one cheap branch and no object + allocation in hot loops. +- Use `performance.now()` when available. +- Avoid wrapping tiny inner-loop operations unless the wrapper itself is gated + before allocation. + +## Required Timing Spans + +### D2 Graph + +Files: + +- `packages/db-ivm/src/d2.ts` +- `packages/db-ivm/src/graph.ts` + +Measure: + +- `d2.run`: total time, number of steps. +- `d2.step`: total operators visited. +- `d2.operator.run`: per operator id and class name. +- `d2.pendingWork`: optional count of calls if it shows up in profiles. + +Record per operator: + +- Input message count. +- Input row count. +- Output message count where visible. +- Output row count where visible. + +Output attribution must be explicit. The preferred first implementation is: + +- Each operator that already builds a `MultiSet` or result array records output + row count immediately before `output.sendData(...)`. +- Generic `D2` operator timing records class name and input counts only. +- Do not infer per-operator output rows from `DifferenceStreamWriter` unless the + writer is extended with producer metadata; otherwise downstream readers make + attribution ambiguous. +- Operators that stream through messages, such as `output`, should record + callback input rows and callback duration separately from forwarded rows. + +### Query Operators + +Files: + +- `packages/db-ivm/src/operators/filter.ts` +- `packages/db-ivm/src/operators/filterBy.ts` +- `packages/db-ivm/src/operators/join.ts` +- `packages/db-ivm/src/operators/reduce.ts` +- `packages/db-ivm/src/operators/count.ts` +- `packages/db-ivm/src/operators/groupBy.ts` +- `packages/db-ivm/src/operators/topKWithFractionalIndex.ts` +- `packages/db-ivm/src/operators/groupedTopKWithFractionalIndex.ts` +- `packages/db-ivm/src/operators/orderBy.ts` +- `packages/db-ivm/src/operators/consolidate.ts` +- `packages/db-ivm/src/operators/distinct.ts` +- `packages/db-ivm/src/operators/output.ts` + +Measure operator-specific cardinalities: + +- Filter: rows in, rows passed, predicate time. +- Join: delta A rows, delta B rows, matched bucket sizes, emitted rows. +- Reduce/count: changed keys, values scanned per key, emitted rows. +- Group-by: group keys touched, group key serialization time. +- Top-k/order-by: state size, inserts, deletes, move-ins, move-outs. +- Consolidate: input rows, output rows, keyed fast path vs unkeyed path. +- Distinct: keys touched and hash time. +- Output: messages read, rows accumulated, callback time. + +### Live Query Scheduling And Flush + +Files: + +- `packages/db/src/scheduler.ts` +- `packages/db/src/query/live/collection-config-builder.ts` +- `packages/db/src/query/live/collection-subscriber.ts` + +Measure: + +- `scheduler.schedule`: context vs immediate, dedupe count. +- `scheduler.flush`: jobs run, passes, blocked dependency count. +- `liveQuery.scheduleGraphRun`: schedules per source write. +- `liveQuery.executeGraphRun`: coalesced callbacks per run. +- `liveQuery.maybeRunGraph`: graph run loop count and total time. +- `liveQuery.flushPendingChanges`: total time and parent/child split. +- `collectionSubscriber.sendChangesToPipeline`: source changes received, + duplicate inserts filtered, changes sent to D2. + +### Collection Commit And Events + +Files: + +- `packages/db/src/collection/sync.ts` +- `packages/db/src/collection/state.ts` +- `packages/db/src/collection/changes.ts` +- `packages/db/src/collection/subscription.ts` + +Measure: + +- Sync `begin`, `write`, and `commit` counts and time. +- `commitPendingTransactions`: total time. +- `changedKeys` size. +- committed synced transaction count. +- active optimistic transaction count. +- event count by insert/update/delete. +- index update time. +- virtual prop enrichment time. +- `deepEquals` count and aggregate time. +- subscriber count and subscriber event delivery time. + +### Includes Materialization + +File: + +- `packages/db/src/query/live/collection-config-builder.ts` + +Measure: + +- `flushIncludesState`: total time by level and materialization type. +- Parent insert phase count/time. +- Child change application count/time. +- Nested buffer drain count/time. +- Per-entry recursive flush count/time. +- Inline parent re-emission count/time. +- Parent delete cleanup count/time. +- Child collection create/dispose count. + +This is especially important for: + +- `list + 3 recent comments` +- `issue detail + comments` + +## Hashing And Serialization Instrumentation + +Hashing must be a first-class report section. + +Files: + +- `packages/db-ivm/src/hashing/hash.ts` +- `packages/db-ivm/src/multiset.ts` +- `packages/db-ivm/src/indexes.ts` +- `packages/db-ivm/src/utils.ts` +- `packages/db-ivm/src/operators/distinct.ts` +- `packages/db/src/query/compiler/group-by.ts` +- `packages/db/src/query/live/utils.ts` + +Measure: + +- `hash`: public call count, total time, max single-call time. +- Structural hash calls, total time, and max single-call time. +- Object hash cache hits and misses. These should only count object-like values + that consult the WeakMap, not primitives. +- Primitive hash calls. These are deterministic but not cache-backed. +- Reference identity hash calls for functions, large binary values, and files. +- Hash calls by input kind: + - primitive + - plain object + - array + - date + - map + - set + - Uint8Array or Buffer + - Temporal + - function or reference hash +- `hashPlainObject`: key count, nested value count, and key sort time. +- `hashUint8Array`: byte length buckets. +- `ObjectIdGenerator.getStringId`: calls, time, object WeakMap hits/misses, + and primitive calls. +- `MultiSet.consolidate`: keyed fast path vs unkeyed hash path. +- `Index` fallback hashing: + - `ValueMap.addValue` calls. + - single value to value map transitions. + - prefix map value hashing. +- `serializeValue`: calls, total time, max single-call time. + +The cache counters need clear denominators. A report that says `cacheHits=1000` +must also make clear whether those hits are from `hashObject`, reference identity +hashing, or `ObjectIdGenerator`; primitives should not be counted as misses just +because they do not use a WeakMap. + +Example report section: + +```text +hash: + publicCalls: 12450 + structuralCalls: 3430 + totalMs: 8.42 + maxMs: 0.31 + objectCacheHits: 2010 + objectCacheMisses: 520 + referenceHashCalls: 12 + primitiveCalls: 9020 + byKind: primitive=9020 object=3100 array=180 map=0 set=0 uint8=0 + +objectIdGenerator: + calls: 9000 + totalMs: 1.33 + objectHits: 7200 + objectMisses: 400 + primitiveCalls: 1400 + +multisetConsolidate: + calls: 412 + totalMs: 5.88 + keyedFastPath: 390 + unkeyedHashPath: 22 + +indexHashFallback: + calls: 744 + totalMs: 2.19 + valueMapTransitions: 31 +``` + +## Report Format + +Each benchmark run should emit: + +- Query name. +- Phase: cold hydrate or incremental write. +- Write scenario name. +- Fixture seed and scale. +- Runtime metadata: Node version, platform, git SHA, tracing enabled/disabled. +- Wall-clock duration summary: median, p75, p95, min, max, standard deviation, + and iteration count. +- Trace overhead summary comparing tracing disabled vs enabled for the same + case. +- Top spans by total time. +- Top spans by call count. +- Hashing summary. +- Operator cardinality table. +- Collection commit and event summary. + +Example: + +```text +query: list + comment count +phase: comment insert +scenario: visible issue comment insert +iterations: 100 +medianMs: 25.12 +p95Ms: 29.48 +traceOverheadMs: 0.82 + +top spans: + collection.commitPendingTransactions 11.40ms calls=1 + reduce.count 7.22ms calls=1 + hash 4.80ms calls=9180 + liveQuery.flushPendingChanges 1.60ms calls=1 + +operator cardinality: + filter issues in=1 out=0 + join comments deltaA=1 deltaB=0 emitted=1 + count comments changedKeys=1 valuesScanned=2800 emitted=1 +``` + +## Implementation Steps + +1. Add the disabled-by-default trace recorder. +2. Add unit tests for trace aggregation, disabled-mode behavior, and async span + completion. +3. Add D2 graph and operator-level spans. +4. Add hash, serialize, multiset, and index fallback counters. +5. Add live query scheduling and flush spans. +6. Add collection commit, event, and includes spans. +7. Add the deterministic standalone benchmark fixture and runner. +8. Run benchmark cases with tracing disabled and enabled to estimate overhead. +9. Run baseline on current branch and save JSON output outside committed source + unless the team wants checked-in snapshots. +10. Inspect the highest cost spans and cardinality blowups. +11. Add focused regression tests for any discovered bug or accidental fanout. +12. Implement optimizations in separate commits after instrumentation is trusted. + +## Questions The Data Should Answer + +- Does a single irrelevant write still run expensive downstream work? +- Is filter/order/limit applied before joins and aggregates where possible? +- Does `comment count` scan all comments for an issue, all comments globally, or + only the changed delta? +- Does `3 recent comments` update only affected child collections and avoid + parent row re-emission? +- Does an included child query for a limited parent list run only for the visible + parent window, or for every parent that passes the pre-limit filter? +- Is hashing time material, and if so, is it structural hash, identity key + generation, `serializeValue`, or `Index` fallback hashing? +- Are repeated `deepEquals` calls or virtual prop enrichment dominating after + D2 finishes? +- Are multiple graph runs scheduled for one source write? +- Are child collection commits/events more expensive than the graph update? + +## Acceptance Criteria + +- Running the benchmark produces one readable text report and one machine + readable JSON report. +- For each query shape, the report shows cold hydrate and at least one + steady-state write case. +- Each measured case reports warmup count, measured iteration count, median, + p95, min, max, standard deviation, fixture seed, fixture scale, runtime + metadata, and git SHA. +- Each measured case reports tracing disabled vs enabled overhead. +- Hashing appears as its own aggregate section with call counts and timing. +- Hashing cache metrics separate object WeakMap hits/misses, primitive calls, + and reference identity hashing. +- D2 operator timing can be separated from collection commit/event timing. +- Operator output cardinality is explicitly recorded by operators that emit + data, not inferred ambiguously from downstream readers. +- Async work such as preload, loadSubset, and first visible result can be timed + with async spans or explicit start/end spans. +- The report includes enough cardinality data to explain why a slow span is + slow. +- Disabled tracing adds no hot-loop allocations and only a cheap enabled-check + branch; any enabled-tracing overhead is reported rather than hidden. diff --git a/packages/db-ivm/src/d2.ts b/packages/db-ivm/src/d2.ts index 8451b2affa..5ea1dc508b 100644 --- a/packages/db-ivm/src/d2.ts +++ b/packages/db-ivm/src/d2.ts @@ -1,4 +1,10 @@ import { DifferenceStreamWriter } from './graph.js' +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, + withPerfSpan, +} from './perf.js' import type { BinaryOperator, DifferenceStreamReader, @@ -47,19 +53,85 @@ export class D2 implements ID2 { if (!this.#finalized) { throw new Error(`Graph not finalized`) } - for (const op of this.#operators) { - op.run() + + if (!isPerfEnabled()) { + for (const op of this.#operators) { + op.run() + } + return } + + recordPerfCount(`d2.step.operatorsVisited`, this.#operators.length) + + withPerfSpan( + `d2.step`, + { + operators: this.#operators.length, + }, + () => { + for (const op of this.#operators) { + const inputStats = op.getPendingInputStats() + const tags = { + operatorId: op.id, + operator: op.constructor.name, + } + + recordPerfCount( + `d2.operator.inputMessages`, + inputStats.messageCount, + tags, + ) + recordPerfCount(`d2.operator.inputRows`, inputStats.rowCount, tags) + + withPerfSpan(`d2.operator.run`, tags, () => { + op.run() + }) + } + }, + ) } pendingWork(): boolean { + if (isPerfEnabled()) { + recordPerfCount(`d2.pendingWork.calls`) + } return this.#operators.some((op) => op.hasPendingWork()) } run(): void { + this.#run(false) + } + + /** + * Drains the graph when the caller has already confirmed pending work. + */ + runWithPendingWork(): void { + this.#run(true) + } + + #run(hasPendingWork: boolean): void { + if (!isPerfEnabled()) { + if (hasPendingWork) { + this.step() + } + while (this.pendingWork()) { + this.step() + } + return + } + + const span = startPerfSpan(`d2.run`) + let steps = 0 + if (hasPendingWork) { + this.step() + steps++ + } while (this.pendingWork()) { + steps++ this.step() } + recordPerfCount(`d2.run.steps`, steps) + span.end({ steps }) } } diff --git a/packages/db-ivm/src/graph.ts b/packages/db-ivm/src/graph.ts index 5263bb643a..bde437aeb0 100644 --- a/packages/db-ivm/src/graph.ts +++ b/packages/db-ivm/src/graph.ts @@ -1,4 +1,5 @@ import { MultiSet } from './multiset.js' +import { isPerfEnabled, recordPerfCount } from './perf.js' import type { MultiSetArray } from './multiset.js' import type { IDifferenceStreamReader, @@ -25,6 +26,18 @@ export class DifferenceStreamReader implements IDifferenceStreamReader { isEmpty(): boolean { return this.#queue.length === 0 } + + pendingMessageCount(): number { + return this.#queue.length + } + + pendingRowCount(): number { + let count = 0 + for (const message of this.#queue) { + count += message.getInner().length + } + return count + } } /** @@ -72,6 +85,26 @@ export abstract class Operator implements IOperator { hasPendingWork(): boolean { return this.inputs.some((input) => !input.isEmpty()) } + + getPendingInputStats(): { + inputCount: number + messageCount: number + rowCount: number + } { + let messageCount = 0 + let rowCount = 0 + + for (const input of this.inputs) { + messageCount += input.pendingMessageCount() + rowCount += input.pendingRowCount() + } + + return { + inputCount: this.inputs.length, + messageCount, + rowCount, + } + } } /** @@ -124,8 +157,25 @@ export abstract class LinearUnaryOperator extends UnaryOperator { abstract inner(collection: MultiSet): MultiSet run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + for (const message of this.inputMessages()) { - this.output.sendData(this.inner(message)) + const result = this.inner(message) + if (shouldTrace) { + recordPerfCount( + `d2.operator.outputRows`, + result.getInner().length, + tags, + ) + recordPerfCount(`d2.operator.outputMessages`, 1, tags) + } + this.output.sendData(result) } } } diff --git a/packages/db-ivm/src/hashing/hash.ts b/packages/db-ivm/src/hashing/hash.ts index 813e4ed35c..69fcfc4db0 100644 --- a/packages/db-ivm/src/hashing/hash.ts +++ b/packages/db-ivm/src/hashing/hash.ts @@ -1,3 +1,4 @@ +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import { MurmurHashStream, randomHash } from './murmur.js' import type { Hasher } from './murmur.js' @@ -48,10 +49,23 @@ const UINT8ARRAY_CONTENT_HASH_THRESHOLD = 128 const hashCache = new WeakMap() -export function hash(input: any): number { - const hasher = new MurmurHashStream() - updateHasher(hasher, input) - return hasher.digest() +export function hash(input: unknown): number { + if (!isPerfEnabled()) { + const hasher = new MurmurHashStream() + updateHasher(hasher, input) + return hasher.digest() + } + + const tags = { kind: getHashInputKind(input) } + recordPerfCount(`hash.public.calls`, 1, tags) + const span = startPerfSpan(`hash.public`, tags) + try { + const hasher = new MurmurHashStream() + updateHasher(hasher, input) + return hasher.digest() + } finally { + span.end() + } } function hashObject(input: object): number { @@ -60,6 +74,9 @@ function hashObject(input: object): number { return cachedHash } + const shouldTrace = isPerfEnabled() + const tags = shouldTrace ? { kind: getHashInputKind(input) } : undefined + const span = shouldTrace ? startPerfSpan(`hash.structural`, tags) : undefined let valueHash: number | undefined if (input instanceof Date) { valueHash = hashDate(input) @@ -76,11 +93,11 @@ function hashObject(input: object): number { } else { // Deeply hashing large arrays would be too costly // so we track them by reference and cache them in a weak map - return cachedReferenceHash(input) + valueHash = cachedReferenceHash(input) } - } else if (input instanceof File) { + } else if (typeof File !== `undefined` && input instanceof File) { // Files are always hashed by reference due to their potentially large size - return cachedReferenceHash(input) + valueHash = cachedReferenceHash(input) } else if (isTemporal(input)) { valueHash = hashTemporal(input) } else { @@ -88,23 +105,26 @@ function hashObject(input: object): number { let marker = OBJECT_MARKER if (input instanceof Array) { - marker = ARRAY_MARKER - } - - if (input instanceof Map) { - marker = MAP_MARKER - plainObjectInput = [...input.entries()] - } - - if (input instanceof Set) { + valueHash = canHashDenseArrayByIndex(input) + ? hashDenseArray(input) + : hashPlainObject(input, ARRAY_MARKER) + } else if (input instanceof Set) { marker = SET_MARKER plainObjectInput = [...input.entries()] - } - valueHash = hashPlainObject(plainObjectInput, marker) + valueHash = hashPlainObject(plainObjectInput, marker) + } else { + if (input instanceof Map) { + marker = MAP_MARKER + plainObjectInput = [...input.entries()] + } + + valueHash = hashPlainObject(plainObjectInput, marker) + } } hashCache.set(input, valueHash) + span?.end() return valueHash } @@ -116,6 +136,12 @@ function hashDate(input: Date): number { } function hashUint8Array(input: Uint8Array): number { + if (isPerfEnabled()) { + recordPerfCount(`hash.uint8Array.calls`) + recordPerfCount(`hash.uint8Array.bytes`, input.byteLength, { + bucket: byteLengthBucket(input.byteLength), + }) + } const hasher = new MurmurHashStream() hasher.update(UINT8ARRAY_MARKER) // Hash the byte length first to differentiate arrays of different sizes @@ -127,6 +153,37 @@ function hashUint8Array(input: Uint8Array): number { return hasher.digest() } +function canHashDenseArrayByIndex(input: Array): boolean { + const keys = Object.keys(input) + if (keys.length !== input.length) return false + + for (let i = 0; i < keys.length; i++) { + if (keys[i] !== String(i)) return false + } + + return true +} + +function hashDenseArray(input: Array): number { + const hasher = new MurmurHashStream() + hasher.update(ARRAY_MARKER) + + if (isPerfEnabled()) { + recordPerfCount(`hash.denseArray.keys`, input.length) + } + + for (let i = 0; i < input.length; i++) { + hasher.update(KEY) + hasher.update(String(i)) + if (isPerfEnabled()) { + recordPerfCount(`hash.denseArray.nestedValues`) + } + updateHasher(hasher, input[i]) + } + + return hasher.digest() +} + function hashTemporal(input: TemporalLike): number { const hasher = new MurmurHashStream() hasher.update(TEMPORAL_MARKER) @@ -141,10 +198,22 @@ function hashPlainObject(input: object, marker: number): number { // Mark the type of the input hasher.update(marker) const keys = Object.keys(input) - keys.sort(keySort) + const sortSpan = isPerfEnabled() + ? startPerfSpan(`hash.plainObject.keySort`, { + keyCount: keys.length, + }) + : undefined + keys.sort() + sortSpan?.end() + if (isPerfEnabled()) { + recordPerfCount(`hash.plainObject.keys`, keys.length) + } for (const key of keys) { hasher.update(KEY) hasher.update(key) + if (isPerfEnabled()) { + recordPerfCount(`hash.plainObject.nestedValues`) + } updateHasher(hasher, input[key as keyof typeof input]) } @@ -152,24 +221,40 @@ function hashPlainObject(input: object, marker: number): number { } function updateHasher(hasher: Hasher, input: unknown): void { + const shouldTrace = isPerfEnabled() if (input === null) { + if (shouldTrace) { + recordPerfCount(`hash.primitive.calls`, 1, { kind: `null` }) + } hasher.update(NULL) return } switch (typeof input) { case `undefined`: + if (shouldTrace) { + recordPerfCount(`hash.primitive.calls`, 1, { kind: `undefined` }) + } hasher.update(UNDEFINED) return case `boolean`: + if (shouldTrace) { + recordPerfCount(`hash.primitive.calls`, 1, { kind: `boolean` }) + } hasher.update(input ? TRUE : FALSE) return case `number`: + if (shouldTrace) { + recordPerfCount(`hash.primitive.calls`, 1, { kind: `number` }) + } // Normalize NaNs and -0 hasher.update(isNaN(input) ? NaN : input === 0 ? 0 : input) return case `bigint`: case `string`: case `symbol`: + if (shouldTrace) { + recordPerfCount(`hash.primitive.calls`, 1, { kind: typeof input }) + } hasher.update(input) return case `object`: @@ -190,7 +275,16 @@ function updateHasher(hasher: Hasher, input: unknown): void { function getCachedHash(input: object): number { let valueHash = hashCache.get(input) if (valueHash === undefined) { + if (isPerfEnabled()) { + recordPerfCount(`hash.objectCache.misses`, 1, { + kind: getHashInputKind(input), + }) + } valueHash = hashObject(input) + } else if (isPerfEnabled()) { + recordPerfCount(`hash.objectCache.hits`, 1, { + kind: getHashInputKind(input), + }) } return valueHash } @@ -199,16 +293,48 @@ let nextRefId = 1 function cachedReferenceHash(fn: object): number { let valueHash = hashCache.get(fn) if (valueHash === undefined) { + if (isPerfEnabled()) { + recordPerfCount(`hash.reference.misses`, 1, { + kind: getHashInputKind(fn), + }) + } valueHash = nextRefId ^ FUNCTIONS nextRefId++ hashCache.set(fn, valueHash) + } else if (isPerfEnabled()) { + recordPerfCount(`hash.reference.hits`, 1, { + kind: getHashInputKind(fn), + }) + } + if (isPerfEnabled()) { + recordPerfCount(`hash.reference.calls`, 1, { + kind: getHashInputKind(fn), + }) } return valueHash } -/** - * Strings sorted lexicographically. - */ -function keySort(a: string, b: string): number { - return a.localeCompare(b) +function getHashInputKind(input: unknown): string { + if (input === null) return `null` + if (typeof input !== `object`) return typeof input + if (input instanceof Date) return `date` + if ( + (typeof Buffer !== `undefined` && input instanceof Buffer) || + input instanceof Uint8Array + ) { + return `uint8Array` + } + if (typeof File !== `undefined` && input instanceof File) return `file` + if (isTemporal(input)) return `temporal` + if (Array.isArray(input)) return `array` + if (input instanceof Map) return `map` + if (input instanceof Set) return `set` + return `plainObject` +} + +function byteLengthBucket(byteLength: number): string { + if (byteLength <= 16) return `0-16` + if (byteLength <= 128) return `17-128` + if (byteLength <= 1024) return `129-1024` + return `1025+` } diff --git a/packages/db-ivm/src/indexes.ts b/packages/db-ivm/src/indexes.ts index 07c0fa5d58..9626efd8a4 100644 --- a/packages/db-ivm/src/indexes.ts +++ b/packages/db-ivm/src/indexes.ts @@ -35,6 +35,7 @@ import { MultiSet } from './multiset.js' import { hash } from './hashing/index.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from './perf.js' import type { Hash } from './hashing/index.js' // We use a symbol to represent the absence of a prefix, unprefixed values a stored @@ -62,6 +63,10 @@ class PrefixMap extends Map< addValue(value: TValue, multiplicity: number): boolean { if (multiplicity === 0) return this.size === 0 + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`index.prefixMap.addValue`) + : undefined const prefix = getPrefix(value) const valueMapOrSingleValue = this.get(prefix) @@ -73,7 +78,13 @@ class PrefixMap extends Map< throw new Error(`Mismatching prefixes, this should never happen`) } - if (currentValue === value || hash(currentValue) === hash(value)) { + const sameValue = + currentValue === value || hash(currentValue) === hash(value) + if (shouldTrace && currentValue !== value) { + recordPerfCount(`index.prefixMap.hashComparisons`, 2) + } + + if (sameValue) { // Same value, update multiplicity const newMultiplicity = currentMultiplicity + multiplicity if (newMultiplicity === 0) { @@ -83,6 +94,10 @@ class PrefixMap extends Map< } } else { // Different suffixes, need to create ValueMap + if (shouldTrace) { + recordPerfCount(`index.prefixMap.valueMapTransitions`) + recordPerfCount(`index.prefixMap.hashComparisons`, 2) + } const valueMap = new ValueMap() valueMap.set(hash(currentValue), valueMapOrSingleValue) valueMap.set(hash(value), [value, multiplicity]) @@ -99,6 +114,7 @@ class PrefixMap extends Map< } } + span?.end({ empty: this.size === 0 }) return this.size === 0 } } @@ -114,6 +130,14 @@ class ValueMap extends Map { addValue(value: TValue, multiplicity: number): boolean { if (multiplicity === 0) return this.size === 0 + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`index.valueMap.addValue`) + : undefined + if (shouldTrace) { + recordPerfCount(`index.valueMap.addValue.calls`) + recordPerfCount(`index.valueMap.hashCalls`) + } const key = hash(value) const currentValue = this.get(key) @@ -129,6 +153,7 @@ class ValueMap extends Map { this.set(key, [value, multiplicity]) } + span?.end({ empty: this.size === 0 }) return this.size === 0 } } @@ -162,15 +187,27 @@ export class Index { * @returns A new Index containing all the data from the messages. */ static fromMultiSets(messages: Array>): Index { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`index.fromMultiSets`, { + messageCount: messages.length, + }) + : undefined const index = new Index() + let rowCount = 0 for (const message of messages) { for (const [item, multiplicity] of message.getInner()) { + rowCount++ const [key, value] = item index.addValue(key, [value, multiplicity]) } } + if (shouldTrace) { + recordPerfCount(`index.fromMultiSets.rows`, rowCount) + span?.end({ keys: index.size }) + } return index } @@ -301,6 +338,9 @@ export class Index { const [value, multiplicity] = valueTuple // If the multiplicity is 0, do nothing if (multiplicity === 0) return + if (isPerfEnabled()) { + recordPerfCount(`index.addValue.calls`) + } // Update consolidated multiplicity tracking const newConsolidatedMultiplicity = @@ -364,6 +404,10 @@ export class Index { newValue: TValue, multiplicity: number, ) { + const shouldTrace = isPerfEnabled() + if (shouldTrace) { + recordPerfCount(`index.singleValueTransitions`) + } const [currentValue, currentMultiplicity] = currentSingleValue // Check for exact same value (reference equality) @@ -382,10 +426,18 @@ export class Index { const currentPrefix = getPrefix(currentValue) // Check if they're the same value by prefix/suffix comparison - if ( + const samePrefixedValue = currentPrefix === newPrefix && (currentValue === newValue || hash(currentValue) === hash(newValue)) + if ( + shouldTrace && + currentPrefix === newPrefix && + currentValue !== newValue ) { + recordPerfCount(`index.singleValueTransition.hashComparisons`, 2) + } + + if (samePrefixedValue) { const newMultiplicity = currentMultiplicity + multiplicity if (newMultiplicity === 0) { this.#inner.delete(key) @@ -398,6 +450,10 @@ export class Index { // Different values - choose appropriate map type if (currentPrefix === NO_PREFIX && newPrefix === NO_PREFIX) { // Both have NO_PREFIX, use ValueMap directly + if (shouldTrace) { + recordPerfCount(`index.singleValueTransition.valueMapTransitions`) + recordPerfCount(`index.singleValueTransition.hashComparisons`, 2) + } const valueMap = new ValueMap() valueMap.set(hash(currentValue), currentSingleValue) valueMap.set(hash(newValue), [newValue, multiplicity]) @@ -408,6 +464,12 @@ export class Index { if (currentPrefix === newPrefix) { // Same prefix, different suffixes - need ValueMap within PrefixMap + if (shouldTrace) { + recordPerfCount( + `index.singleValueTransition.prefixValueMapTransitions`, + ) + recordPerfCount(`index.singleValueTransition.hashComparisons`, 2) + } const valueMap = new ValueMap() valueMap.set(hash(currentValue), currentSingleValue) valueMap.set(hash(newValue), [newValue, multiplicity]) @@ -427,9 +489,21 @@ export class Index { * @param other - The index to append to the current index. */ append(other: Index): void { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`index.append`, { + keys: other.size, + }) + : undefined + let rows = 0 for (const [key, value] of other.entries()) { + rows++ this.addValue(key, value) } + if (shouldTrace) { + recordPerfCount(`index.append.rows`, rows) + span?.end() + } } /** @@ -440,6 +514,13 @@ export class Index { join( other: Index, ): MultiSet<[TKey, [TValue, TValue2]]> { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`index.join`, { + leftKeys: this.size, + rightKeys: other.size, + }) + : undefined const result: Array<[[TKey, [TValue, TValue2]], number]> = [] // We want to iterate over the smaller of the two indexes to reduce the // number of operations we need to do. @@ -469,6 +550,10 @@ export class Index { } } + if (shouldTrace) { + recordPerfCount(`index.join.outputRows`, result.length) + span?.end({ outputRows: result.length }) + } return new MultiSet(result) } } diff --git a/packages/db-ivm/src/multiset.ts b/packages/db-ivm/src/multiset.ts index 601e948cf0..8d09a229c4 100644 --- a/packages/db-ivm/src/multiset.ts +++ b/packages/db-ivm/src/multiset.ts @@ -4,6 +4,7 @@ import { globalObjectIdGenerator, } from './utils.js' import { hash } from './hashing/index.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from './perf.js' export type MultiSetArray = Array<[T, number]> export type KeyedData = [key: string, value: T] @@ -71,16 +72,40 @@ export class MultiSet { * (record, multiplicity) pair. */ consolidate(): MultiSet { - // Check if this looks like a keyed multiset (first item is a tuple of length 2) - if (this.#inner.length > 0) { - const firstItem = this.#inner[0]?.[0] - if (Array.isArray(firstItem) && firstItem.length === 2) { - return this.#consolidateKeyed() + if (!isPerfEnabled()) { + // Check if this looks like a keyed multiset (first item is a tuple of length 2) + if (this.#inner.length > 0) { + const firstItem = this.#inner[0]?.[0] + if (Array.isArray(firstItem) && firstItem.length === 2) { + return this.#consolidateKeyed() + } } + + // Fall back to original method for unkeyed data + return this.#consolidateUnkeyed() } - // Fall back to original method for unkeyed data - return this.#consolidateUnkeyed() + const firstItem = this.#inner[0]?.[0] + const keyedCandidate = Array.isArray(firstItem) && firstItem.length === 2 + const path = keyedCandidate ? `keyedCandidate` : `unkeyed` + const span = startPerfSpan(`multiset.consolidate`, { path }) + recordPerfCount(`multiset.consolidate.inputRows`, this.#inner.length, { + path, + }) + + try { + const result = keyedCandidate + ? this.#consolidateKeyed() + : this.#consolidateUnkeyed() + recordPerfCount( + `multiset.consolidate.outputRows`, + result.getInner().length, + { path }, + ) + return result + } finally { + span.end() + } } /** @@ -94,6 +119,10 @@ export class MultiSet { * we unpack them and compare each element individually to maintain proper equality semantics. */ #consolidateKeyed(): MultiSet { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`multiset.consolidate.keyed`) + : undefined const consolidated = new Map() const values = new Map() @@ -117,6 +146,12 @@ export class MultiSet { // Verify this is still a keyed item (should be [key, value] pair) if (!Array.isArray(data) || data.length !== 2) { // Found non-keyed item, fall back to unkeyed consolidation + if (shouldTrace) { + recordPerfCount(`multiset.consolidate.keyedFallbacks`, 1, { + reason: `shape`, + }) + span?.end() + } return this.#consolidateUnkeyed() } @@ -125,6 +160,12 @@ export class MultiSet { // Verify key is string or number as expected for keyed multisets if (typeof key !== `string` && typeof key !== `number`) { // Found non-string/number key, fall back to unkeyed consolidation + if (shouldTrace) { + recordPerfCount(`multiset.consolidate.keyedFallbacks`, 1, { + reason: `keyType`, + }) + span?.end() + } return this.#consolidateUnkeyed() } @@ -159,6 +200,10 @@ export class MultiSet { } } + if (shouldTrace) { + recordPerfCount(`multiset.consolidate.keyedOutputRows`, result.length) + span?.end() + } return new MultiSet(result) } @@ -166,6 +211,10 @@ export class MultiSet { * Private method for consolidating unkeyed multisets using the original approach. */ #consolidateUnkeyed(): MultiSet { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`multiset.consolidate.unkeyed`) + : undefined const consolidated = new DefaultMap(() => 0) const values = new Map() @@ -184,9 +233,17 @@ export class MultiSet { } const requireJson = hasOther || (hasString && hasNumber) + if (shouldTrace) { + recordPerfCount(`multiset.consolidate.unkeyedCalls`, 1, { + hashPath: requireJson, + }) + } for (const [data, multiplicity] of this.#inner) { const key = requireJson ? hash(data) : (data as string | number) + if (shouldTrace && requireJson) { + recordPerfCount(`multiset.consolidate.unkeyedHashRows`) + } if (requireJson && !values.has(key as string)) { values.set(key as string, data) } @@ -201,6 +258,12 @@ export class MultiSet { } } + if (shouldTrace) { + recordPerfCount(`multiset.consolidate.unkeyedOutputRows`, result.length, { + hashPath: requireJson, + }) + span?.end() + } return new MultiSet(result) } diff --git a/packages/db-ivm/src/operators/consolidate.ts b/packages/db-ivm/src/operators/consolidate.ts index b5356b4f4f..5ae8c1beb0 100644 --- a/packages/db-ivm/src/operators/consolidate.ts +++ b/packages/db-ivm/src/operators/consolidate.ts @@ -1,6 +1,7 @@ import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' import { MultiSet } from '../multiset.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { IStreamBuilder, PipedOperator } from '../types.js' /** @@ -8,14 +9,27 @@ import type { IStreamBuilder, PipedOperator } from '../types.js' */ export class ConsolidateOperator extends UnaryOperator { run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.consolidate.run`, tags) + : undefined const messages = this.inputMessages() if (messages.length === 0) { + span?.end() return } // Combine all messages into a single MultiSet const combined = new MultiSet() + let inputRows = 0 for (const message of messages) { + inputRows += message.getInner().length combined.extend(message) } @@ -26,6 +40,16 @@ export class ConsolidateOperator extends UnaryOperator { if (consolidated.getInner().length > 0) { this.output.sendData(consolidated) } + if (shouldTrace) { + recordPerfCount(`operator.consolidate.messages`, messages.length, tags) + recordPerfCount(`operator.consolidate.inputRows`, inputRows, tags) + recordPerfCount( + `operator.consolidate.outputRows`, + consolidated.getInner().length, + tags, + ) + span?.end() + } } } diff --git a/packages/db-ivm/src/operators/count.ts b/packages/db-ivm/src/operators/count.ts index 0fa79b4010..79567dad3c 100644 --- a/packages/db-ivm/src/operators/count.ts +++ b/packages/db-ivm/src/operators/count.ts @@ -1,27 +1,77 @@ -import { DifferenceStreamWriter } from '../graph.js' +import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' -import { ReduceOperator } from './reduce.js' +import { MultiSet } from '../multiset.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { DifferenceStreamReader } from '../graph.js' import type { IStreamBuilder, KeyValue } from '../types.js' /** * Operator that counts elements by key (version-free) */ -export class CountOperator extends ReduceOperator { +export class CountOperator extends UnaryOperator<[K, V], [K, number]> { + #counts = new Map() + #hasOutput = new Set() + constructor( id: number, inputA: DifferenceStreamReader<[K, V]>, output: DifferenceStreamWriter<[K, number]>, ) { - const countInner = (vals: Array<[V, number]>): Array<[number, number]> => { - let totalCount = 0 - for (const [_, diff] of vals) { - totalCount += diff + super(id, inputA, output) + } + + run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.count.run`, tags) + : undefined + + const deltas = new Map() + let inputRows = 0 + for (const message of this.inputMessages()) { + for (const [item, multiplicity] of message.getInner()) { + inputRows++ + const [key] = item + deltas.set(key, (deltas.get(key) ?? 0) + multiplicity) + } + } + + const result: Array<[[K, number], number]> = [] + for (const [key, delta] of deltas) { + const oldCount = this.#counts.get(key) ?? 0 + const newCount = oldCount + delta + const hasOutput = this.#hasOutput.has(key) + + if (!hasOutput) { + result.push([[key, newCount], 1]) + this.#hasOutput.add(key) + } else if (oldCount !== newCount) { + result.push([[key, oldCount], -1]) + result.push([[key, newCount], 1]) + } + + if (newCount === 0) { + this.#counts.delete(key) + } else { + this.#counts.set(key, newCount) } - return [[totalCount, 1]] } - super(id, inputA, output, countInner) + if (result.length > 0) { + this.output.sendData(new MultiSet(result)) + } + if (shouldTrace) { + recordPerfCount(`operator.count.inputRows`, inputRows, tags) + recordPerfCount(`operator.count.changedKeys`, deltas.size, tags) + recordPerfCount(`operator.count.outputRows`, result.length, tags) + span?.end({ outputRows: result.length }) + } } } diff --git a/packages/db-ivm/src/operators/distinct.ts b/packages/db-ivm/src/operators/distinct.ts index 721f380c3b..d9d794ef6d 100644 --- a/packages/db-ivm/src/operators/distinct.ts +++ b/packages/db-ivm/src/operators/distinct.ts @@ -2,6 +2,7 @@ import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' import { hash } from '../hashing/index.js' import { MultiSet } from '../multiset.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { Hash } from '../hashing/index.js' import type { DifferenceStreamReader } from '../graph.js' import type { IStreamBuilder, KeyValue } from '../types.js' @@ -31,11 +32,23 @@ export class DistinctOperator< } run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.distinct.run`, tags) + : undefined const updatedValues = new Map() + let inputRows = 0 // Compute the new multiplicity for each value for (const message of this.inputMessages()) { for (const [value, diff] of message.getInner()) { + inputRows++ const hashedValue = hash(this.#by(value)) const oldMultiplicity = @@ -76,6 +89,12 @@ export class DistinctOperator< if (result.length > 0) { this.output.sendData(new MultiSet(result)) } + if (shouldTrace) { + recordPerfCount(`operator.distinct.inputRows`, inputRows, tags) + recordPerfCount(`operator.distinct.keysTouched`, updatedValues.size, tags) + recordPerfCount(`operator.distinct.outputRows`, result.length, tags) + span?.end() + } } } diff --git a/packages/db-ivm/src/operators/filter.ts b/packages/db-ivm/src/operators/filter.ts index 9e0fccf2ae..9a26997750 100644 --- a/packages/db-ivm/src/operators/filter.ts +++ b/packages/db-ivm/src/operators/filter.ts @@ -1,5 +1,6 @@ import { DifferenceStreamWriter, LinearUnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { IStreamBuilder, PipedOperator } from '../types.js' import type { DifferenceStreamReader } from '../graph.js' import type { MultiSet } from '../multiset.js' @@ -21,7 +22,26 @@ export class FilterOperator extends LinearUnaryOperator { } inner(collection: MultiSet): MultiSet { - return collection.filter(this.#f) + if (!isPerfEnabled()) { + return collection.filter(this.#f) + } + + const tags = { + operatorId: this.id, + operator: this.constructor.name, + } + let rowsPassed = 0 + const rowsIn = collection.getInner().length + const span = startPerfSpan(`operator.filter.predicate`, tags) + const result = collection.filter((data) => { + const passed = this.#f(data) + if (passed) rowsPassed++ + return passed + }) + span.end() + recordPerfCount(`operator.filter.rowsIn`, rowsIn, tags) + recordPerfCount(`operator.filter.rowsPassed`, rowsPassed, tags) + return result } } diff --git a/packages/db-ivm/src/operators/groupBy.ts b/packages/db-ivm/src/operators/groupBy.ts index 9c2fe1e357..5341aa8366 100644 --- a/packages/db-ivm/src/operators/groupBy.ts +++ b/packages/db-ivm/src/operators/groupBy.ts @@ -1,11 +1,20 @@ import { serializeValue } from '../utils.js' +import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' +import { StreamBuilder } from '../d2.js' +import { MultiSet } from '../multiset.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import { map } from './map.js' import { reduce } from './reduce.js' +import type { DifferenceStreamReader } from '../graph.js' import type { IStreamBuilder, KeyValue } from '../types.js' type GroupKey = Record +type GroupValuePrefix = string | number | bigint | undefined +type GroupValue = [GroupValuePrefix, Record] +type IncrementalAggregateKind = `count` | `some` | `every` type BasicAggregateFunction = { + kind?: IncrementalAggregateKind preMap: (data: T) => V reduce: (values: Array<[V, number]>) => V postMap?: (result: V) => R @@ -32,6 +41,232 @@ function isPipedAggregateFunction( return `pipe` in aggregate } +type IncrementalAggregateFunction = { + kind: IncrementalAggregateKind + preMap: (data: T) => unknown +} + +function isIncrementalAggregateFunction( + aggregate: BasicAggregateFunction, +): aggregate is BasicAggregateFunction & + IncrementalAggregateFunction { + return aggregate.kind !== undefined +} + +function getGroupValuePrefix(data: unknown): GroupValuePrefix { + if (!Array.isArray(data)) return undefined + + const key = data[0] + return typeof key === `string` || + typeof key === `number` || + typeof key === `bigint` + ? key + : undefined +} + +type IncrementalGroupState = { + key: K + total: number + values: Array +} + +type IncrementalGroupSnapshot = { + key: K + total: number + values: Array +} + +function aggregateValuesEqual( + left: Array, + right: Array, +): boolean { + if (left.length !== right.length) return false + + for (let i = 0; i < left.length; i++) { + if (left[i] !== right[i]) return false + } + + return true +} + +function incrementalAggregateValue( + aggregate: IncrementalAggregateFunction, + data: T, +): number { + const value = aggregate.preMap(data) + switch (aggregate.kind) { + case `count`: + return value as number + case `some`: + return value ? 1 : 0 + case `every`: + return value ? 0 : 1 + } +} + +function incrementalAggregateOutput( + aggregate: IncrementalAggregateFunction, + value: number, +): unknown { + switch (aggregate.kind) { + case `count`: + return value + case `some`: + return value > 0 + case `every`: + return value <= 0 + } +} + +class IncrementalGroupByOperator< + T, + K extends GroupKey, + ResultType, +> extends UnaryOperator> { + #groups = new Map>() + #keyExtractor: (data: T) => K + #aggregates: Array<[string, IncrementalAggregateFunction]> + + constructor( + id: number, + input: DifferenceStreamReader, + output: DifferenceStreamWriter>, + keyExtractor: (data: T) => K, + aggregates: Array<[string, IncrementalAggregateFunction]>, + ) { + super(id, input, output) + this.#keyExtractor = keyExtractor + this.#aggregates = aggregates + } + + run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.groupBy.incremental.run`, tags) + : undefined + + const changedGroups = new Map< + string, + { + before: IncrementalGroupSnapshot + state: IncrementalGroupState + } + >() + let inputRows = 0 + + for (const message of this.inputMessages()) { + for (const [data, multiplicity] of message.getInner()) { + inputRows++ + const key = this.#keyExtractor(data) + const keyString = serializeValue(key) + let state = this.#groups.get(keyString) + + if (!state) { + state = { + key, + total: 0, + values: new Array(this.#aggregates.length).fill(0), + } + this.#groups.set(keyString, state) + } + + if (!changedGroups.has(keyString)) { + changedGroups.set(keyString, { + before: { + key: state.key, + total: state.total, + values: [...state.values], + }, + state, + }) + } + + if (state.total <= 0 && multiplicity > 0) { + state.key = key + } + state.total += multiplicity + + for (let i = 0; i < this.#aggregates.length; i++) { + state.values[i]! += + incrementalAggregateValue(this.#aggregates[i]![1], data) * + multiplicity + } + } + } + + const result: Array<[KeyValue, number]> = [] + for (const [keyString, { before, state }] of changedGroups) { + const hadOutput = before.total > 0 + const hasOutput = state.total > 0 + + if (!hasOutput) { + this.#groups.delete(keyString) + } + + if (hadOutput && !hasOutput) { + result.push([ + [keyString, this.#buildResult(before.key, before.values)], + -1, + ]) + } else if (!hadOutput && hasOutput) { + result.push([ + [keyString, this.#buildResult(state.key, state.values)], + 1, + ]) + } else if ( + hadOutput && + hasOutput && + !aggregateValuesEqual(before.values, state.values) + ) { + result.push([ + [keyString, this.#buildResult(before.key, before.values)], + -1, + ]) + result.push([ + [keyString, this.#buildResult(state.key, state.values)], + 1, + ]) + } + } + + if (result.length > 0) { + this.output.sendData(new MultiSet(result)) + } + if (shouldTrace) { + recordPerfCount(`operator.groupBy.incremental.inputRows`, inputRows, tags) + recordPerfCount( + `operator.groupBy.incremental.changedGroups`, + changedGroups.size, + tags, + ) + recordPerfCount( + `operator.groupBy.incremental.outputRows`, + result.length, + tags, + ) + span?.end({ outputRows: result.length }) + } + } + + #buildResult(key: K, values: Array): ResultType { + const result: Record = {} + Object.assign(result, key) + + for (let i = 0; i < this.#aggregates.length; i++) { + const [name, aggregate] = this.#aggregates[i]! + result[name] = incrementalAggregateOutput(aggregate, values[i]!) + } + + return result as ResultType + } +} + /** * Groups data by key and applies multiple aggregate operations * @param keyExtractor Function to extract grouping key from data @@ -49,6 +284,7 @@ export function groupBy< ([_, aggregate]) => !isPipedAggregateFunction(aggregate), ), ) as Record> + const basicAggregateEntries = Object.entries(basicAggregates) // @ts-expect-error - TODO: we don't use this yet, but we will // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -61,6 +297,33 @@ export function groupBy< return ( stream: IStreamBuilder, ): IStreamBuilder> => { + const incrementalAggregateEntries = basicAggregateEntries.every( + ([_, aggregate]) => isIncrementalAggregateFunction(aggregate), + ) + ? (basicAggregateEntries as Array< + [string, IncrementalAggregateFunction] + >) + : undefined + + if ( + incrementalAggregateEntries !== undefined && + incrementalAggregateEntries.length > 0 + ) { + const output = new StreamBuilder>( + stream.graph, + new DifferenceStreamWriter>(), + ) + const operator = new IncrementalGroupByOperator( + stream.graph.getNextOperatorId(), + stream.connectReader(), + output.writer, + keyExtractor, + incrementalAggregateEntries, + ) + stream.graph.addOperator(operator) + return output + } + // Special key to store the original key object const KEY_SENTINEL = `__original_key__` @@ -77,11 +340,14 @@ export function groupBy< values[KEY_SENTINEL] = key // Add pre-aggregated values - for (const [name, aggregate] of Object.entries(basicAggregates)) { + for (const [name, aggregate] of basicAggregateEntries) { values[name] = aggregate.preMap(data) } - return [keyString, values] as KeyValue> + return [keyString, [getGroupValuePrefix(data), values]] as KeyValue< + string, + GroupValue + > }), ) @@ -102,13 +368,13 @@ export function groupBy< const result: Record = {} // Get the original key from first value in group - const originalKey = values[0]?.[0]?.[KEY_SENTINEL] + const originalKey = values[0]?.[0]?.[1][KEY_SENTINEL] result[KEY_SENTINEL] = originalKey // Apply each aggregate function - for (const [name, aggregate] of Object.entries(basicAggregates)) { + for (const [name, aggregate] of basicAggregateEntries) { const preValues = values.map( - ([v, m]) => [v[name], m] as [any, number], + ([v, m]) => [v[1][name], m] as [any, number], ) result[name] = aggregate.reduce(preValues) } @@ -130,7 +396,7 @@ export function groupBy< Object.assign(result, key) // Apply postMap if provided - for (const [name, aggregate] of Object.entries(basicAggregates)) { + for (const [name, aggregate] of basicAggregateEntries) { if (aggregate.postMap) { result[name] = aggregate.postMap(values[name]) } else { @@ -167,9 +433,10 @@ export function sum( * Creates a count aggregate function */ export function count( - valueExtractor: (value: T) => any = (v) => v, + valueExtractor: (value: T) => unknown = (v) => v, ): AggregateFunction { return { + kind: `count`, // Count only not-null values (the `== null` comparison gives true for both null and undefined) preMap: (data: T) => (valueExtractor(data) == null ? 0 : 1), reduce: (values: Array<[number, number]>) => { diff --git a/packages/db-ivm/src/operators/groupedTopKWithFractionalIndex.ts b/packages/db-ivm/src/operators/groupedTopKWithFractionalIndex.ts index 47e7a309bd..b9613ec69e 100644 --- a/packages/db-ivm/src/operators/groupedTopKWithFractionalIndex.ts +++ b/packages/db-ivm/src/operators/groupedTopKWithFractionalIndex.ts @@ -1,6 +1,7 @@ import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' import { MultiSet } from '../multiset.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js' import { TopKArray, createKeyedComparator } from './topKArray.js' import type { IndexedValue, TopK } from './topKArray.js' @@ -97,6 +98,16 @@ export class GroupedTopKWithFractionalIndexOperator< * Any changes to the topK are sent to the output. */ #moveTopK({ offset, limit }: { offset?: number; limit?: number }): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.groupedTopK.moveWindow`, tags) + : undefined if (offset !== undefined) { this.#offset = offset } @@ -121,12 +132,38 @@ export class GroupedTopKWithFractionalIndexOperator< if (hasChanges) { this.output.sendData(new MultiSet(result)) } + if (shouldTrace) { + recordPerfCount( + `operator.groupedTopK.groups`, + this.#groupStates.size, + tags, + ) + recordPerfCount(`operator.groupedTopK.outputRows`, result.length, tags) + recordPerfCount( + `operator.groupedTopK.stateSize`, + this.#getTotalSize(), + tags, + ) + span?.end({ outputRows: result.length }) + } } run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.groupedTopK.run`, tags) + : undefined const result: Array<[[K, IndexedValue], number]> = [] + let inputRows = 0 for (const message of this.inputMessages()) { for (const [item, multiplicity] of message.getInner()) { + inputRows++ const [key, value] = item this.#processElement(key, value, multiplicity, result) } @@ -135,6 +172,21 @@ export class GroupedTopKWithFractionalIndexOperator< if (result.length > 0) { this.output.sendData(new MultiSet(result)) } + if (shouldTrace) { + recordPerfCount(`operator.groupedTopK.inputRows`, inputRows, tags) + recordPerfCount(`operator.groupedTopK.outputRows`, result.length, tags) + recordPerfCount( + `operator.groupedTopK.groups`, + this.#groupStates.size, + tags, + ) + recordPerfCount( + `operator.groupedTopK.stateSize`, + this.#getTotalSize(), + tags, + ) + span?.end({ outputRows: result.length }) + } } #processElement( diff --git a/packages/db-ivm/src/operators/join.ts b/packages/db-ivm/src/operators/join.ts index 58b62620f3..51c4d6b71a 100644 --- a/packages/db-ivm/src/operators/join.ts +++ b/packages/db-ivm/src/operators/join.ts @@ -52,6 +52,7 @@ import { BinaryOperator, DifferenceStreamWriter } from '../graph.js' import { StreamBuilder } from '../d2.js' import { MultiSet } from '../multiset.js' import { Index } from '../indexes.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { DifferenceStreamReader } from '../graph.js' import type { IStreamBuilder, KeyValue, PipedOperator } from '../types.js' @@ -82,16 +83,73 @@ export class JoinOperator extends BinaryOperator< } run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + mode: this.#mode, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.join.run`, tags) + : undefined // Build deltas from input messages - const deltaA = Index.fromMultiSets( - this.inputAMessages() as Array>, - ) - const deltaB = Index.fromMultiSets( - this.inputBMessages() as Array>, - ) + const inputAMessages = this.inputAMessages() as Array> + const inputBMessages = this.inputBMessages() as Array> + const deltaASummary = summarizeDelta(inputAMessages) + const deltaBSummary = summarizeDelta(inputBMessages) + const deltaARows = deltaASummary.rowCount + const deltaBRows = deltaBSummary.rowCount + + if (deltaASummary.single && deltaBRows === 0) { + const outputRows = this.runSingleDeltaA(deltaASummary.single) + if (shouldTrace) { + recordPerfCount(`operator.join.deltaRowsA`, deltaARows, tags) + recordPerfCount(`operator.join.deltaRowsB`, deltaBRows, tags) + recordPerfCount( + `operator.join.deltaKeysA`, + deltaASummary.single.multiplicity === 0 ? 0 : 1, + tags, + ) + recordPerfCount(`operator.join.deltaKeysB`, 0, tags) + recordPerfCount(`operator.join.outputRows`, outputRows, tags) + recordPerfCount(`operator.join.singleDeltaA`, 1, tags) + span?.end({ outputRows }) + } + return + } + + if (deltaBSummary.single && deltaARows === 0) { + const outputRows = this.runSingleDeltaB(deltaBSummary.single) + if (shouldTrace) { + recordPerfCount(`operator.join.deltaRowsA`, deltaARows, tags) + recordPerfCount(`operator.join.deltaRowsB`, deltaBRows, tags) + recordPerfCount(`operator.join.deltaKeysA`, 0, tags) + recordPerfCount( + `operator.join.deltaKeysB`, + deltaBSummary.single.multiplicity === 0 ? 0 : 1, + tags, + ) + recordPerfCount(`operator.join.outputRows`, outputRows, tags) + recordPerfCount(`operator.join.singleDeltaB`, 1, tags) + span?.end({ outputRows }) + } + return + } + + const deltaA = Index.fromMultiSets(inputAMessages) + const deltaB = Index.fromMultiSets(inputBMessages) // Early-out if nothing changed - if (deltaA.size === 0 && deltaB.size === 0) return + if (deltaA.size === 0 && deltaB.size === 0) { + if (shouldTrace) { + recordPerfCount(`operator.join.deltaRowsA`, deltaARows, tags) + recordPerfCount(`operator.join.deltaRowsB`, deltaBRows, tags) + span?.end({ outputRows: 0 }) + } + return + } const results = new MultiSet() @@ -125,6 +183,18 @@ export class JoinOperator extends BinaryOperator< if (results.getInner().length > 0) { this.output.sendData(results) } + if (shouldTrace) { + recordPerfCount(`operator.join.deltaRowsA`, deltaARows, tags) + recordPerfCount(`operator.join.deltaRowsB`, deltaBRows, tags) + recordPerfCount(`operator.join.deltaKeysA`, deltaA.size, tags) + recordPerfCount(`operator.join.deltaKeysB`, deltaB.size, tags) + recordPerfCount( + `operator.join.outputRows`, + results.getInner().length, + tags, + ) + span?.end({ outputRows: results.getInner().length }) + } } private emitInnerResults( @@ -138,6 +208,120 @@ export class JoinOperator extends BinaryOperator< if (deltaA.size > 0 && deltaB.size > 0) results.extend(deltaA.join(deltaB)) } + private runSingleDeltaA(change: DeltaChange): number { + const { key, value, multiplicity } = change + if (multiplicity === 0) return 0 + + const results = new MultiSet() + + if (this.#mode !== `anti`) { + for (const [rightValue, rightMultiplicity] of this.#indexB.getIterator( + key, + )) { + if (rightMultiplicity !== 0) { + results.add( + [key, [value, rightValue]], + multiplicity * rightMultiplicity, + ) + } + } + } + + if ( + this.#mode === `left` || + this.#mode === `full` || + this.#mode === `anti` + ) { + const finalMultiplicityB = this.#indexB.getConsolidatedMultiplicity(key) + if (finalMultiplicityB === 0) { + results.add([key, [value, null]], multiplicity) + } + } + + if (this.#mode === `right` || this.#mode === `full`) { + const before = this.#indexA.getConsolidatedMultiplicity(key) + const after = before + multiplicity + if ((before === 0) !== (after === 0)) { + const transitioningToMatched = before === 0 + + for (const [rightValue, rightMultiplicity] of this.#indexB.getIterator( + key, + )) { + if (rightMultiplicity !== 0) { + results.add( + [key, [null, rightValue]], + transitioningToMatched ? -rightMultiplicity : +rightMultiplicity, + ) + } + } + } + } + + this.#indexA.addValue(key, [value, multiplicity]) + const outputRows = results.getInner().length + if (outputRows > 0) { + this.output.sendData(results) + } + return outputRows + } + + private runSingleDeltaB(change: DeltaChange): number { + const { key, value, multiplicity } = change + if (multiplicity === 0) return 0 + + const results = new MultiSet() + + if (this.#mode !== `anti`) { + for (const [leftValue, leftMultiplicity] of this.#indexA.getIterator( + key, + )) { + if (leftMultiplicity !== 0) { + results.add( + [key, [leftValue, value]], + leftMultiplicity * multiplicity, + ) + } + } + } + + if ( + this.#mode === `left` || + this.#mode === `full` || + this.#mode === `anti` + ) { + const before = this.#indexB.getConsolidatedMultiplicity(key) + const after = before + multiplicity + if ((before === 0) !== (after === 0)) { + const transitioningToMatched = before === 0 + + for (const [leftValue, leftMultiplicity] of this.#indexA.getIterator( + key, + )) { + if (leftMultiplicity !== 0) { + results.add( + [key, [leftValue, null]], + transitioningToMatched ? -leftMultiplicity : +leftMultiplicity, + ) + } + } + } + } + + if (this.#mode === `right` || this.#mode === `full`) { + const finalMultiplicityA = this.#indexA.getConsolidatedMultiplicity(key) + if (finalMultiplicityA === 0) { + results.add([key, [null, value]], multiplicity) + } + } + + this.#indexB.addValue(key, [value, multiplicity]) + const outputRows = results.getInner().length + if (outputRows > 0) { + this.output.sendData(results) + } + return outputRows + } + private emitLeftOuterResults( deltaA: Index, deltaB: Index, @@ -247,6 +431,37 @@ export class JoinOperator extends BinaryOperator< } } +type DeltaChange = { + key: K + value: V + multiplicity: number +} + +type DeltaSummary = { + rowCount: number + single?: DeltaChange +} + +function summarizeDelta( + messages: Array>, +): DeltaSummary { + let rowCount = 0 + let single: DeltaChange | undefined + + for (const message of messages) { + for (const [[key, value], multiplicity] of message.getInner()) { + rowCount++ + if (rowCount === 1) { + single = { key, value, multiplicity } + } else { + single = undefined + } + } + } + + return single ? { rowCount, single } : { rowCount } +} + /** * Joins two input streams * @param other - The other stream to join with diff --git a/packages/db-ivm/src/operators/output.ts b/packages/db-ivm/src/operators/output.ts index 2e9795e863..fb8f3a3ca3 100644 --- a/packages/db-ivm/src/operators/output.ts +++ b/packages/db-ivm/src/operators/output.ts @@ -1,5 +1,6 @@ import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { IStreamBuilder, PipedOperator } from '../types.js' import type { DifferenceStreamReader } from '../graph.js' import type { MultiSet } from '../multiset.js' @@ -21,8 +22,27 @@ export class OutputOperator extends UnaryOperator { } run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined for (const message of this.inputMessages()) { + if (shouldTrace) { + recordPerfCount(`operator.output.messages`, 1, tags) + recordPerfCount( + `operator.output.inputRows`, + message.getInner().length, + tags, + ) + } + const span = shouldTrace + ? startPerfSpan(`operator.output.callback`, tags) + : undefined this.#fn(message) + span?.end() this.output.sendData(message) } } diff --git a/packages/db-ivm/src/operators/reduce.ts b/packages/db-ivm/src/operators/reduce.ts index 3a8690e017..89d901f4ed 100644 --- a/packages/db-ivm/src/operators/reduce.ts +++ b/packages/db-ivm/src/operators/reduce.ts @@ -2,6 +2,7 @@ import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' import { MultiSet } from '../multiset.js' import { Index } from '../indexes.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import type { DifferenceStreamReader } from '../graph.js' import type { IStreamBuilder, KeyValue } from '../types.js' @@ -24,10 +25,22 @@ export class ReduceOperator extends UnaryOperator<[K, V1], [K, V2]> { } run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.reduce.run`, tags) + : undefined // Collect all input messages and update the index const keysTodo = new Set() + let inputRows = 0 for (const message of this.inputMessages()) { for (const [item, multiplicity] of message.getInner()) { + inputRows++ const [key, value] = item this.#index.addValue(key, [value, multiplicity]) keysTodo.add(key) @@ -36,9 +49,11 @@ export class ReduceOperator extends UnaryOperator<[K, V1], [K, V2]> { // For each key, compute the reduction and delta const result: Array<[[K, V2], number]> = [] + let valuesScanned = 0 for (const key of keysTodo) { const curr = this.#index.get(key) const currOut = this.#indexOut.get(key) + valuesScanned += curr.length const out = this.#f(curr) // Create maps for current and previous outputs using values directly as keys @@ -94,6 +109,13 @@ export class ReduceOperator extends UnaryOperator<[K, V1], [K, V2]> { if (result.length > 0) { this.output.sendData(new MultiSet(result)) } + if (shouldTrace) { + recordPerfCount(`operator.reduce.inputRows`, inputRows, tags) + recordPerfCount(`operator.reduce.changedKeys`, keysTodo.size, tags) + recordPerfCount(`operator.reduce.valuesScanned`, valuesScanned, tags) + recordPerfCount(`operator.reduce.outputRows`, result.length, tags) + span?.end({ outputRows: result.length }) + } } } diff --git a/packages/db-ivm/src/operators/topKWithFractionalIndex.ts b/packages/db-ivm/src/operators/topKWithFractionalIndex.ts index 740c3b8408..ff6e311be6 100644 --- a/packages/db-ivm/src/operators/topKWithFractionalIndex.ts +++ b/packages/db-ivm/src/operators/topKWithFractionalIndex.ts @@ -1,6 +1,7 @@ import { DifferenceStreamWriter, UnaryOperator } from '../graph.js' import { StreamBuilder } from '../d2.js' import { MultiSet } from '../multiset.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from '../perf.js' import { TopKState, handleMoveIn, handleMoveOut } from './topKState.js' import { TopKArray, createKeyedComparator } from './topKArray.js' import type { IndexedValue, TopK } from './topKArray.js' @@ -60,6 +61,16 @@ export class TopKWithFractionalIndexOperator< * Any changes to the topK are sent to the output. */ moveTopK({ offset, limit }: { offset?: number; limit?: number }) { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.topK.moveWindow`, tags) + : undefined const result: Array<[[K, IndexedValue], number]> = [] const diff = this.#state.move({ offset, limit }) @@ -72,12 +83,31 @@ export class TopKWithFractionalIndexOperator< // because the collection is lazy, so we will run the graph again to load the data this.output.sendData(new MultiSet(result)) } + if (shouldTrace) { + recordPerfCount(`operator.topK.moveIns`, diff.moveIns.length, tags) + recordPerfCount(`operator.topK.moveOuts`, diff.moveOuts.length, tags) + recordPerfCount(`operator.topK.outputRows`, result.length, tags) + recordPerfCount(`operator.topK.stateSize`, this.#state.size, tags) + span?.end({ outputRows: result.length }) + } } run(): void { + const shouldTrace = isPerfEnabled() + const tags = shouldTrace + ? { + operatorId: this.id, + operator: this.constructor.name, + } + : undefined + const span = shouldTrace + ? startPerfSpan(`operator.topK.run`, tags) + : undefined const result: Array<[[K, IndexedValue], number]> = [] + let inputRows = 0 for (const message of this.inputMessages()) { for (const [item, multiplicity] of message.getInner()) { + inputRows++ const [key, value] = item this.processElement(key, value, multiplicity, result) } @@ -86,6 +116,12 @@ export class TopKWithFractionalIndexOperator< if (result.length > 0) { this.output.sendData(new MultiSet(result)) } + if (shouldTrace) { + recordPerfCount(`operator.topK.inputRows`, inputRows, tags) + recordPerfCount(`operator.topK.outputRows`, result.length, tags) + recordPerfCount(`operator.topK.stateSize`, this.#state.size, tags) + span?.end({ outputRows: result.length }) + } } processElement( diff --git a/packages/db-ivm/src/perf.ts b/packages/db-ivm/src/perf.ts new file mode 100644 index 0000000000..f128ab2c8c --- /dev/null +++ b/packages/db-ivm/src/perf.ts @@ -0,0 +1,215 @@ +type PerfTagValue = string | number | boolean | undefined +type PerfTags = Record + +type PerfMetricKind = `span` | `counter` | `gauge` + +interface PerfMetric { + name: string + kind: PerfMetricKind + tags: PerfTags + calls: number + total: number + max: number + min: number + last: number +} + +interface PerfState { + enabled: boolean + metrics: Map +} + +export interface PerfMetricSnapshot { + name: string + kind: PerfMetricKind + tags: PerfTags + calls: number + total: number + max: number + min: number + last: number +} + +export interface PerfTraceSnapshot { + enabled: boolean + metrics: Array +} + +export interface PerfSpanHandle { + end: (extraTags?: PerfTags) => void +} + +const perfStateSymbol = Symbol.for(`@tanstack/db.perfState`) + +type PerfGlobal = typeof globalThis & { + __TANSTACK_DB_TRACE__?: boolean + [perfStateSymbol]?: PerfState +} + +function getPerfState(): PerfState { + const global = globalThis as PerfGlobal + global[perfStateSymbol] ??= { + enabled: readDefaultEnabled(global), + metrics: new Map(), + } + return global[perfStateSymbol] +} + +function readDefaultEnabled(global: PerfGlobal): boolean { + const env = ( + global as { process?: { env?: Record } } + ).process?.env + + return ( + global.__TANSTACK_DB_TRACE__ === true || + env?.TANSTACK_DB_TRACE === `1` || + env?.TANSTACK_DB_TRACE === `true` + ) +} + +function now(): number { + return globalThis.performance.now() +} + +function metricKey(name: string, kind: PerfMetricKind, tags: PerfTags): string { + const tagParts = Object.keys(tags) + .sort() + .map((key) => `${key}:${String(tags[key])}`) + .join(`,`) + return `${kind}:${name}:${tagParts}` +} + +function mergedTags(tags: PerfTags, extraTags?: PerfTags): PerfTags { + return extraTags ? { ...tags, ...extraTags } : tags +} + +function recordMetric( + kind: PerfMetricKind, + name: string, + value: number, + tags: PerfTags = {}, +): void { + const state = getPerfState() + if (!state.enabled) return + + const key = metricKey(name, kind, tags) + let metric = state.metrics.get(key) + if (!metric) { + metric = { + name, + kind, + tags: { ...tags }, + calls: 0, + total: 0, + max: Number.NEGATIVE_INFINITY, + min: Number.POSITIVE_INFINITY, + last: 0, + } + state.metrics.set(key, metric) + } + + metric.calls++ + metric.total += value + metric.max = Math.max(metric.max, value) + metric.min = Math.min(metric.min, value) + metric.last = value +} + +export function isPerfEnabled(): boolean { + return getPerfState().enabled +} + +export function setPerfTracingEnabled(enabled: boolean): void { + getPerfState().enabled = enabled +} + +export function resetPerfTrace(): void { + getPerfState().metrics.clear() +} + +export function getPerfTraceSnapshot(): PerfTraceSnapshot { + const state = getPerfState() + const metrics = Array.from(state.metrics.values()) + .map((metric) => ({ + ...metric, + tags: { ...metric.tags }, + min: metric.min === Number.POSITIVE_INFINITY ? 0 : metric.min, + max: metric.max === Number.NEGATIVE_INFINITY ? 0 : metric.max, + })) + .sort((a, b) => b.total - a.total || b.calls - a.calls) + + return { + enabled: state.enabled, + metrics, + } +} + +export function recordPerfCount( + name: string, + value = 1, + tags?: PerfTags, +): void { + recordMetric(`counter`, name, value, tags) +} + +export function recordPerfGauge( + name: string, + value: number, + tags?: PerfTags, +): void { + recordMetric(`gauge`, name, value, tags) +} + +export function recordPerfSpan( + name: string, + durationMs: number, + tags?: PerfTags, +): void { + recordMetric(`span`, name, durationMs, tags) +} + +export function startPerfSpan( + name: string, + tags: PerfTags = {}, +): PerfSpanHandle { + if (!isPerfEnabled()) { + return { end: () => {} } + } + + const start = now() + return { + end: (extraTags?: PerfTags) => { + recordPerfSpan(name, now() - start, mergedTags(tags, extraTags)) + }, + } +} + +export function withPerfSpan(name: string, tags: PerfTags, fn: () => T): T { + if (!isPerfEnabled()) { + return fn() + } + + const span = startPerfSpan(name, tags) + try { + return fn() + } finally { + span.end() + } +} + +export async function withPerfSpanAsync( + name: string, + tags: PerfTags, + fn: () => Promise, +): Promise { + if (!isPerfEnabled()) { + return fn() + } + + const span = startPerfSpan(name, tags) + try { + return await fn() + } finally { + span.end() + } +} diff --git a/packages/db-ivm/src/utils.ts b/packages/db-ivm/src/utils.ts index 70cfda48c6..f6c2b56302 100644 --- a/packages/db-ivm/src/utils.ts +++ b/packages/db-ivm/src/utils.ts @@ -1,3 +1,5 @@ +import { isPerfEnabled, recordPerfCount, startPerfSpan } from './perf.js' + /** * Simple assertion function for runtime checks. * Throws an error if the condition is false. @@ -91,9 +93,19 @@ export class ObjectIdGenerator { * - Objects: Uses WeakMap for reference-based identity * - Primitives: Uses consistent string-based hashing */ - getId(value: any): number { + getId(value: unknown): number { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`objectIdGenerator.getId`, { + kind: value === null ? `null` : typeof value, + }) + : undefined + // For primitives, use a simple hash of their string representation if (typeof value !== `object` || value === null) { + if (shouldTrace) { + recordPerfCount(`objectIdGenerator.primitiveCalls`) + } const str = String(value) let hashValue = 0 for (let i = 0; i < str.length; i++) { @@ -101,25 +113,46 @@ export class ObjectIdGenerator { hashValue = (hashValue << 5) - hashValue + char hashValue = hashValue & hashValue // Convert to 32-bit integer } + span?.end() return hashValue } // For objects, use WeakMap to assign unique IDs if (!this.objectIds.has(value)) { + if (shouldTrace) { + recordPerfCount(`objectIdGenerator.objectMisses`) + } this.objectIds.set(value, this.nextId++) + } else if (shouldTrace) { + recordPerfCount(`objectIdGenerator.objectHits`) } + span?.end() return this.objectIds.get(value)! } /** * Get a string representation of the ID for use in composite keys. */ - getStringId(value: any): string { - if (value === null) return `null` - if (value === undefined) return `undefined` - if (typeof value !== `object`) return `str_${String(value)}` + getStringId(value: unknown): string { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`objectIdGenerator.getStringId`, { + kind: value === null ? `null` : typeof value, + }) + : undefined - return `obj_${this.getId(value)}` + try { + if (shouldTrace) { + recordPerfCount(`objectIdGenerator.getStringId.calls`) + } + if (value === null) return `null` + if (value === undefined) return `undefined` + if (typeof value !== `object`) return `str_${String(value)}` + + return `obj_${this.getId(value)}` + } finally { + span?.end() + } } } @@ -199,13 +232,31 @@ export function compareKeys(a: string | number, b: string | number): number { * This is used for creating string keys in groupBy operations. */ export function serializeValue(value: unknown): string { - return JSON.stringify(value, (_, val) => { - if (typeof val === 'bigint') { - return val.toString() - } - if (val instanceof Date) { - return val.toISOString() - } - return val - }) + if (!isPerfEnabled()) { + return JSON.stringify(value, (_, val) => { + if (typeof val === 'bigint') { + return val.toString() + } + if (val instanceof Date) { + return val.toISOString() + } + return val + }) + } + + recordPerfCount(`serializeValue.calls`) + const span = startPerfSpan(`serializeValue`) + try { + return JSON.stringify(value, (_, val) => { + if (typeof val === 'bigint') { + return val.toString() + } + if (val instanceof Date) { + return val.toISOString() + } + return val + }) + } finally { + span.end() + } } diff --git a/packages/db-ivm/tests/graph.test.ts b/packages/db-ivm/tests/graph.test.ts index b3e2de1876..bea52896b2 100644 --- a/packages/db-ivm/tests/graph.test.ts +++ b/packages/db-ivm/tests/graph.test.ts @@ -1,6 +1,8 @@ import { beforeEach, describe, expect, test } from 'vitest' +import { D2 } from '../src/d2.js' import { DifferenceStreamWriter } from '../src/graph.js' import { MultiSet } from '../src/multiset.js' +import { output } from '../src/operators/index.js' import type { DifferenceStreamReader } from '../src/graph.js' describe(`DifferenceStreamReader and DifferenceStreamWriter`, () => { @@ -58,3 +60,22 @@ describe(`DifferenceStreamReader and DifferenceStreamWriter`, () => { expect(reader.isEmpty()).toBe(true) }) }) + +describe(`D2`, () => { + test(`runWithPendingWork drains pending input after an explicit pending check`, () => { + const graph = new D2() + const input = graph.newInput() + const messages: Array> = [] + + input.pipe(output((message) => messages.push(message))) + graph.finalize() + + input.sendData(new MultiSet([[1, 1]])) + + expect(graph.pendingWork()).toBe(true) + graph.runWithPendingWork() + + expect(graph.pendingWork()).toBe(false) + expect(messages).toEqual([new MultiSet([[1, 1]])]) + }) +}) diff --git a/packages/db-ivm/tests/operators/groupBy.test.ts b/packages/db-ivm/tests/operators/groupBy.test.ts index fbe50fb33a..9e28b5407f 100644 --- a/packages/db-ivm/tests/operators/groupBy.test.ts +++ b/packages/db-ivm/tests/operators/groupBy.test.ts @@ -374,6 +374,75 @@ describe(`Operators`, () => { expect(latestMessage.getInner()).toEqual(expectedResult) }) + test(`with count-only aggregate skips unchanged null-count rows`, () => { + const graph = new D2() + const input = graph.newInput<{ + category: string + amount: number | null + }>() + const messages: Array> = [] + + input.pipe( + groupBy((data) => ({ category: data.category }), { + countNotNull: count((data) => data.amount), + }), + output((message) => { + messages.push(message) + }), + ) + + graph.finalize() + + input.sendData(new MultiSet([[{ category: `A`, amount: null }, 1]])) + graph.run() + + expect(messages).toHaveLength(1) + expect(messages[0]!.getInner()).toEqual([ + [ + [ + `{"category":"A"}`, + { + category: `A`, + countNotNull: 0, + }, + ], + 1, + ], + ]) + + input.sendData(new MultiSet([[{ category: `A`, amount: null }, 1]])) + graph.run() + + expect(messages).toHaveLength(1) + + input.sendData(new MultiSet([[{ category: `A`, amount: 10 }, 1]])) + graph.run() + + expect(messages).toHaveLength(2) + expect(messages[1]!.getInner()).toEqual([ + [ + [ + `{"category":"A"}`, + { + category: `A`, + countNotNull: 0, + }, + ], + -1, + ], + [ + [ + `{"category":"A"}`, + { + category: `A`, + countNotNull: 1, + }, + ], + 1, + ], + ]) + }) + test(`with avg and count aggregates`, () => { const graph = new D2() const input = graph.newInput<{ diff --git a/packages/db/src/SortedMap.ts b/packages/db/src/SortedMap.ts index 3d59ad687c..2aea7dd94b 100644 --- a/packages/db/src/SortedMap.ts +++ b/packages/db/src/SortedMap.ts @@ -82,6 +82,48 @@ export class SortedMap { return left } + private insertSortedKey(key: TKey, value: TValue): void { + const length = this.sortedKeys.length + if (length === 0) { + this.sortedKeys.push(key) + return + } + + if (!this.comparator) { + const lastKey = this.sortedKeys[length - 1]! + if (compareKeys(key, lastKey) > 0) { + this.sortedKeys.push(key) + return + } + + const firstKey = this.sortedKeys[0]! + if (compareKeys(key, firstKey) < 0) { + this.sortedKeys.unshift(key) + return + } + } + + const index = this.indexOf(key, value) + this.sortedKeys.splice(index, 0, key) + } + + private removeSortedKey(key: TKey, value: TValue): void { + if (!this.comparator) { + const lastIndex = this.sortedKeys.length - 1 + if (this.sortedKeys[lastIndex] === key) { + this.sortedKeys.pop() + return + } + if (this.sortedKeys[0] === key) { + this.sortedKeys.shift() + return + } + } + + const index = this.indexOf(key, value) + this.sortedKeys.splice(index, 1) + } + /** * Sets a key-value pair in the map and maintains sort order * @@ -90,22 +132,58 @@ export class SortedMap { * @returns This SortedMap instance for chaining */ set(key: TKey, value: TValue): this { + if (!this.comparator) { + if (this.map.has(key)) { + this.map.set(key, value) + return this + } + + this.insertSortedKey(key, value) + this.map.set(key, value) + return this + } + if (this.map.has(key)) { // Need to remove the old key from the sorted keys array const oldValue = this.map.get(key)! - const oldIndex = this.indexOf(key, oldValue) - this.sortedKeys.splice(oldIndex, 1) + this.removeSortedKey(key, oldValue) } // Insert the new key at the correct position - const index = this.indexOf(key, value) - this.sortedKeys.splice(index, 0, key) + this.insertSortedKey(key, value) this.map.set(key, value) return this } + /** + * Sets a key-value pair when the caller has already established whether the + * key exists. This avoids a duplicate Map lookup on hot paths. + */ + setWithKnownPresence( + key: TKey, + value: TValue, + keyExists: boolean, + previousValue: TValue | undefined, + ): this { + if (!keyExists) { + this.insertSortedKey(key, value) + this.map.set(key, value) + return this + } + + if (!this.comparator) { + this.map.set(key, value) + return this + } + + this.removeSortedKey(key, previousValue as TValue) + this.insertSortedKey(key, value) + this.map.set(key, value) + return this + } + /** * Gets a value by its key * @@ -125,14 +203,30 @@ export class SortedMap { delete(key: TKey): boolean { if (this.map.has(key)) { const oldValue = this.map.get(key) - const index = this.indexOf(key, oldValue!) - this.sortedKeys.splice(index, 1) + this.removeSortedKey(key, oldValue!) return this.map.delete(key) } return false } + /** + * Deletes a key when the caller has already read the previous value. + * Returns false when the previous read proved the key was absent. + */ + deleteWithKnownPreviousValue( + key: TKey, + keyExists: boolean, + previousValue: TValue | undefined, + ): boolean { + if (!keyExists) { + return false + } + + this.removeSortedKey(key, previousValue as TValue) + return this.map.delete(key) + } + /** * Checks if a key exists in the map * diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index dc07cd3f18..b09c04617e 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -3,6 +3,11 @@ import { createSingleRowRefProxy, toExpression, } from '../query/builder/ref-proxy.js' +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, +} from '../query/live/perf.js' import { CollectionSubscription } from './subscription.js' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessage, SubscribeChangesOptions } from '../types' @@ -77,10 +82,23 @@ export class CollectionChangesManager< changes: Array>, forceEmit = false, ): void { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.changes.emitEvents`, { + collectionId: this.collection.id, + forceEmit, + }) + : undefined // Skip batching for user actions (forceEmit=true) to keep UI responsive if (this.shouldBatchEvents && !forceEmit) { // Add events to the batch this.batchedEvents.push(...changes) + if (shouldTrace) { + recordPerfCount(`collection.changes.batchedEvents`, changes.length, { + collectionId: this.collection.id, + }) + span?.end({ batched: true }) + } return } @@ -99,19 +117,65 @@ export class CollectionChangesManager< } if (rawEvents.length === 0) { + span?.end({ rawEvents: 0 }) + return + } + + if (this.changeSubscriptions.size === 0) { + if (shouldTrace) { + recordPerfCount(`collection.changes.rawEvents`, rawEvents.length, { + collectionId: this.collection.id, + }) + recordPerfCount(`collection.changes.enrichedEvents`, 0, { + collectionId: this.collection.id, + }) + recordPerfCount(`collection.changes.subscriberDeliveries`, 0, { + collectionId: this.collection.id, + }) + span?.end({ rawEvents: rawEvents.length }) + } return } // Enrich all change messages with virtual properties // This uses the "add-if-missing" pattern to preserve pass-through semantics + const enrichSpan = shouldTrace + ? startPerfSpan(`collection.changes.enrichVirtualProps`, { + collectionId: this.collection.id, + }) + : undefined const enrichedEvents: Array< ChangeMessage, TKey> > = rawEvents.map((change) => this.enrichChangeWithVirtualProps(change)) + enrichSpan?.end() // Emit to all listeners + const deliverySpan = shouldTrace + ? startPerfSpan(`collection.changes.subscriberDelivery`, { + collectionId: this.collection.id, + subscriberCount: this.changeSubscriptions.size, + }) + : undefined for (const subscription of this.changeSubscriptions) { subscription.emitEvents(enrichedEvents) } + if (shouldTrace) { + recordPerfCount(`collection.changes.rawEvents`, rawEvents.length, { + collectionId: this.collection.id, + }) + recordPerfCount( + `collection.changes.enrichedEvents`, + enrichedEvents.length, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.changes.subscriberDeliveries`, + this.changeSubscriptions.size, + { collectionId: this.collection.id }, + ) + deliverySpan?.end() + span?.end({ rawEvents: rawEvents.length }) + } } /** diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index abfb6693eb..1e5e5d715b 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -1,6 +1,7 @@ import { withArrayChangeTracking, withChangeTracking } from '../proxy' import { safeRandomUUID } from '../utils/uuid' import { createTransaction, getActiveTransaction } from '../transactions' +import { isPerfEnabled, startPerfSpan } from '../query/live/perf.js' import { DeleteKeyNotFoundError, DuplicateKeyError, @@ -177,6 +178,13 @@ export class CollectionMutationsManager< } const items = Array.isArray(data) ? data : [data] + const span = isPerfEnabled() + ? startPerfSpan(`collection.mutations.insert`, { + collectionId: this.id, + items: items.length, + ambientTransaction: ambientTransaction !== undefined, + }) + : undefined const mutations: Array> = [] const keysInCurrentBatch = new Set() @@ -228,6 +236,7 @@ export class CollectionMutationsManager< state.scheduleTransactionCleanup(ambientTransaction) state.recomputeOptimisticState(true) + span?.end({ mutations: mutations.length }) return ambientTransaction } else { // Create a new transaction with a mutation function that calls the onInsert handler @@ -257,6 +266,7 @@ export class CollectionMutationsManager< state.scheduleTransactionCleanup(directOpTransaction) state.recomputeOptimisticState(true) + span?.end({ mutations: mutations.length }) return directOpTransaction } } @@ -290,6 +300,13 @@ export class CollectionMutationsManager< const isArray = Array.isArray(keys) const keysArray = isArray ? keys : [keys] + const span = isPerfEnabled() + ? startPerfSpan(`collection.mutations.update`, { + collectionId: this.id, + keys: keysArray.length, + ambientTransaction: ambientTransaction !== undefined, + }) + : undefined if (isArray && keysArray.length === 0) { throw new NoKeysPassedToUpdateError() @@ -411,6 +428,7 @@ export class CollectionMutationsManager< emptyTransaction.commit().catch(() => undefined) // Schedule cleanup for empty transaction state.scheduleTransactionCleanup(emptyTransaction) + span?.end({ mutations: 0 }) return emptyTransaction } @@ -422,6 +440,7 @@ export class CollectionMutationsManager< state.scheduleTransactionCleanup(ambientTransaction) state.recomputeOptimisticState(true) + span?.end({ mutations: mutations.length }) return ambientTransaction } @@ -455,6 +474,7 @@ export class CollectionMutationsManager< state.scheduleTransactionCleanup(directOpTransaction) state.recomputeOptimisticState(true) + span?.end({ mutations: mutations.length }) return directOpTransaction } @@ -480,6 +500,13 @@ export class CollectionMutationsManager< } const keysArray = Array.isArray(keys) ? keys : [keys] + const span = isPerfEnabled() + ? startPerfSpan(`collection.mutations.delete`, { + collectionId: this.id, + keys: keysArray.length, + ambientTransaction: ambientTransaction !== undefined, + }) + : undefined const mutations: Array< PendingMutation< TOutput, @@ -527,6 +554,7 @@ export class CollectionMutationsManager< state.scheduleTransactionCleanup(ambientTransaction) state.recomputeOptimisticState(true) + span?.end({ mutations: mutations.length }) return ambientTransaction } @@ -557,6 +585,7 @@ export class CollectionMutationsManager< state.scheduleTransactionCleanup(directOpTransaction) state.recomputeOptimisticState(true) + span?.end({ mutations: mutations.length }) return directOpTransaction } } diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 9cbdebb234..1c6a7ea39a 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -1,6 +1,11 @@ import { deepEquals } from '../utils' import { SortedMap } from '../SortedMap' import { enrichRowWithVirtualProps } from '../virtual-props.js' +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, +} from '../query/live/perf.js' import { DIRECT_TRANSACTION_METADATA_KEY } from './transaction-metadata.js' import type { VirtualOrigin, @@ -54,6 +59,45 @@ type InternalChangeMessage< } } +type SimpleSyncCommitResult = { + changedKeyCount: number + eventCount: number +} + +function hasResolvedVirtualProps( + row: unknown, +): row is WithVirtualProps { + if (typeof row !== 'object' || row === null) { + return false + } + + const virtualRow = row as Record + return ( + virtualRow.$synced !== undefined && + virtualRow.$synced !== null && + virtualRow.$origin !== undefined && + virtualRow.$origin !== null && + virtualRow.$key !== undefined && + virtualRow.$key !== null && + virtualRow.$collectionId !== undefined && + virtualRow.$collectionId !== null + ) +} + +function perfDeepEquals(a: unknown, b: unknown): boolean { + if (!isPerfEnabled()) { + return deepEquals(a, b) + } + + const span = startPerfSpan(`collection.deepEquals`) + try { + return deepEquals(a, b) + } finally { + recordPerfCount(`collection.deepEquals.calls`) + span.end() + } +} + export class CollectionStateManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -226,6 +270,33 @@ export class CollectionStateManager< }) } + private snapshotRowOriginsForKeys( + keys: Iterable, + ): Map { + const rowOrigins = new Map() + + for (const key of keys) { + const origin = this.rowOrigins.get(key) + if (origin !== undefined) { + rowOrigins.set(key, origin) + } + } + + return rowOrigins + } + + private setRowOrigin( + key: TKey, + origin: VirtualOrigin, + hadPreviousVisibleValue: boolean, + ): void { + if (origin === 'local') { + this.rowOrigins.set(key, origin) + } else if (hadPreviousVisibleValue) { + this.rowOrigins.delete(key) + } + } + private enrichWithVirtualPropsSnapshot( row: TOutput, virtualProps: VirtualRowProps, @@ -236,6 +307,15 @@ export class CollectionStateManager< const resolvedKey = existingRow.$key ?? virtualProps.$key const collectionId = existingRow.$collectionId ?? virtualProps.$collectionId + if ( + existingRow.$synced === synced && + existingRow.$origin === origin && + existingRow.$key === resolvedKey && + existingRow.$collectionId === collectionId + ) { + return row as WithVirtualProps + } + const cached = this.virtualPropsCache.get(row as object) if ( cached && @@ -298,6 +378,8 @@ export class CollectionStateManager< const { __virtualProps } = change as InternalChangeMessage const enrichedValue = __virtualProps?.value ? this.enrichWithVirtualPropsSnapshot(change.value, __virtualProps.value) + : hasResolvedVirtualProps(change.value) + ? (change.value as WithVirtualProps) : this.enrichWithVirtualProps(change.value, change.key) const enrichedPreviousValue = change.previousValue ? __virtualProps?.previousValue @@ -305,6 +387,8 @@ export class CollectionStateManager< change.previousValue, __virtualProps.previousValue, ) + : hasResolvedVirtualProps(change.previousValue) + ? (change.previousValue as WithVirtualProps) : this.enrichWithVirtualProps(change.previousValue, change.key) : undefined @@ -465,20 +549,66 @@ export class CollectionStateManager< public recomputeOptimisticState( triggeredByUserAction: boolean = false, ): void { + const shouldTrace = isPerfEnabled() // Skip redundant recalculations when we're in the middle of committing sync transactions // While the sync pipeline is replaying a large batch we still want to honour // fresh optimistic mutations from the UI. Only skip recompute for the // internal sync-driven redraws; user-triggered work (triggeredByUserAction) // must run so live queries stay responsive during long commits. if (this.isCommittingSyncTransactions && !triggeredByUserAction) { + if (shouldTrace) { + recordPerfCount(`collection.recomputeOptimisticState.skipped`, 1, { + collectionId: this.collection.id, + }) + } return } + const span = shouldTrace + ? startPerfSpan(`collection.recomputeOptimisticState`, { + collectionId: this.collection.id, + triggeredByUserAction, + }) + : undefined + let eventCount = 0 + let activeTransactionCount = 0 + + const snapshotSpan = shouldTrace + ? startPerfSpan(`collection.recomputeOptimisticState.snapshotState`, { + collectionId: this.collection.id, + }) + : undefined const previousState = new Map(this.optimisticUpserts) const previousDeletes = new Set(this.optimisticDeletes) - const previousRowOrigins = new Map(this.rowOrigins) + const previousRowOrigins = this.rowOrigins + if (shouldTrace) { + recordPerfCount( + `collection.recomputeOptimisticState.snapshotState.optimisticUpserts`, + previousState.size, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.recomputeOptimisticState.snapshotState.optimisticDeletes`, + previousDeletes.size, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.recomputeOptimisticState.snapshotState.rowOrigins`, + 0, + { collectionId: this.collection.id }, + ) + } + snapshotSpan?.end() // Update pending optimistic state for completed/failed transactions + const pendingStateSpan = shouldTrace + ? startPerfSpan( + `collection.recomputeOptimisticState.updatePendingState`, + { + collectionId: this.collection.id, + }, + ) + : undefined for (const transaction of this.transactions.values()) { const isDirectTransaction = transaction.metadata[DIRECT_TRANSACTION_METADATA_KEY] === true @@ -535,6 +665,7 @@ export class CollectionStateManager< } } } + pendingStateSpan?.end() // Clear current optimistic state this.optimisticUpserts.clear() @@ -581,18 +712,35 @@ export class CollectionStateManager< const activeTransactions: Array> = [] + const activeTransactionSpan = shouldTrace + ? startPerfSpan( + `collection.recomputeOptimisticState.collectActiveTransactions`, + { + collectionId: this.collection.id, + }, + ) + : undefined for (const transaction of this.transactions.values()) { if (![`completed`, `failed`].includes(transaction.state)) { activeTransactions.push(transaction) } } + activeTransactionCount = activeTransactions.length + activeTransactionSpan?.end() // Apply active transactions only (completed transactions are handled by sync operations) + const applyActiveSpan = shouldTrace + ? startPerfSpan(`collection.recomputeOptimisticState.applyActive`, { + collectionId: this.collection.id, + }) + : undefined + let activeMutationCount = 0 for (const transaction of activeTransactions) { for (const mutation of transaction.mutations) { if (!this.isThisCollection(mutation.collection)) { continue } + activeMutationCount++ // Track that this key has pending local changes for $origin tracking this.pendingLocalChanges.add(mutation.key) @@ -615,11 +763,30 @@ export class CollectionStateManager< } } } + if (shouldTrace) { + recordPerfCount( + `collection.recomputeOptimisticState.applyActive.mutations`, + activeMutationCount, + { collectionId: this.collection.id }, + ) + } + applyActiveSpan?.end() // Update cached size + const sizeSpan = shouldTrace + ? startPerfSpan(`collection.recomputeOptimisticState.calculateSize`, { + collectionId: this.collection.id, + }) + : undefined this.size = this.calculateSize() + sizeSpan?.end() // Collect events for changes + const collectEventsSpan = shouldTrace + ? startPerfSpan(`collection.recomputeOptimisticState.collectEvents`, { + collectionId: this.collection.id, + }) + : undefined const events: Array> = [] this.collectOptimisticChanges( previousState, @@ -627,6 +794,7 @@ export class CollectionStateManager< previousRowOrigins, events, ) + collectEventsSpan?.end() // Filter out events for recently synced keys to prevent duplicates // BUT: Only filter out events that are actually from sync operations @@ -645,6 +813,11 @@ export class CollectionStateManager< return false }) + const emitSpan = shouldTrace + ? startPerfSpan(`collection.recomputeOptimisticState.emitEvents`, { + collectionId: this.collection.id, + }) + : undefined // Filter out redundant delete events if there are pending sync transactions // that will immediately restore the same data, but only for completed transactions // IMPORTANT: Skip complex filtering for user-triggered actions to prevent UI blocking @@ -685,6 +858,7 @@ export class CollectionStateManager< if (filteredEvents.length > 0) { this.indexes.updateIndexes(filteredEvents) } + eventCount = filteredEvents.length this.changes.emitEvents(filteredEvents, triggeredByUserAction) } else { // Update indexes for all events @@ -692,8 +866,25 @@ export class CollectionStateManager< this.indexes.updateIndexes(filteredEventsBySyncStatus) } // Emit all events if no pending sync transactions + eventCount = filteredEventsBySyncStatus.length this.changes.emitEvents(filteredEventsBySyncStatus, triggeredByUserAction) } + emitSpan?.end({ events: eventCount }) + if (shouldTrace) { + recordPerfCount( + `collection.recomputeOptimisticState.activeTransactions`, + activeTransactionCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.recomputeOptimisticState.events`, + eventCount, + { + collectionId: this.collection.id, + }, + ) + } + span?.end({ events: eventCount }) } /** @@ -795,529 +986,1220 @@ export class CollectionStateManager< return this.syncedData.get(key) } - /** - * Attempts to commit pending synced transactions if there are no active transactions - * This method processes operations from pending transactions and applies them to the synced data - */ - commitPendingTransactions = () => { - // Check if there are any persisting transaction - let hasPersistingTransaction = false - for (const transaction of this.transactions.values()) { - if (transaction.state === `persisting`) { - hasPersistingTransaction = true - break + private commitSinglePureSyncTransaction( + transaction: PendingSyncedTransaction | undefined, + uncommittedSyncedTransactions: Array< + PendingSyncedTransaction + >, + hasPersistingTransaction: boolean, + hasTruncateSync: boolean, + hasImmediateSync: boolean, + ): SimpleSyncCommitResult | undefined { + if ( + hasTruncateSync || + (hasPersistingTransaction && !hasImmediateSync) || + transaction === undefined || + this.transactions.size !== 0 || + this.optimisticUpserts.size !== 0 || + this.optimisticDeletes.size !== 0 || + this.pendingOptimisticUpserts.size !== 0 || + this.pendingOptimisticDeletes.size !== 0 || + this.pendingOptimisticDirectUpserts.size !== 0 || + this.pendingOptimisticDirectDeletes.size !== 0 || + this.pendingLocalChanges.size !== 0 || + this.pendingLocalOrigins.size !== 0 || + this.preSyncVisibleState.size !== 0 || + this.recentlySyncedKeys.size !== 0 + ) { + return undefined + } + + if ( + transaction.truncate || + transaction.collectionMetadataWrites.size !== 0 + ) { + return undefined + } + + if (transaction.operations.length === 2) { + return this.commitDoublePureSyncTransaction( + transaction, + uncommittedSyncedTransactions, + ) + } + + if (transaction.operations.length !== 1) { + return undefined + } + + const operation = transaction.operations[0]! + const key = operation.key as TKey + for (const [metadataKey] of transaction.rowMetadataWrites) { + if (metadataKey !== key) { + return undefined } } - // pending synced transactions could be either `committed` or still open. - // we only want to process `committed` transactions here - const { - committedSyncedTransactions, - uncommittedSyncedTransactions, - hasTruncateSync, - hasImmediateSync, - } = this.pendingSyncedTransactions.reduce( - (acc, t) => { - if (t.committed) { - acc.committedSyncedTransactions.push(t) - if (t.truncate) { - acc.hasTruncateSync = true + const previousVisibleValue = this.syncedData.get(key) + const hadPreviousVisibleValue = previousVisibleValue !== undefined + const previousVirtualProps = + hadPreviousVisibleValue + ? this.getVirtualPropsSnapshotForState(key) + : undefined + + this.isCommittingSyncTransactions = true + let newVisibleValue: TOutput | undefined + try { + this.syncedKeys.add(key) + const origin: VirtualOrigin = this.isLocalOnly ? 'local' : 'remote' + + switch (operation.type) { + case `insert`: + if (!(`value` in operation)) { + return undefined } - if (t.immediate) { - acc.hasImmediateSync = true + this.syncedData.setWithKnownPresence( + key, + operation.value, + hadPreviousVisibleValue, + previousVisibleValue, + ) + newVisibleValue = operation.value + this.setRowOrigin(key, origin, hadPreviousVisibleValue) + break + case `update`: { + if (!(`value` in operation)) { + return undefined } - } else { - acc.uncommittedSyncedTransactions.push(t) + const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` + if (rowUpdateMode === `partial`) { + const updatedValue = Object.assign( + {}, + previousVisibleValue, + operation.value, + ) + this.syncedData.setWithKnownPresence( + key, + updatedValue, + hadPreviousVisibleValue, + previousVisibleValue, + ) + newVisibleValue = updatedValue + } else { + this.syncedData.setWithKnownPresence( + key, + operation.value, + hadPreviousVisibleValue, + previousVisibleValue, + ) + newVisibleValue = operation.value + } + this.setRowOrigin(key, origin, hadPreviousVisibleValue) + break } - return acc - }, - { - committedSyncedTransactions: [] as Array< - PendingSyncedTransaction - >, - uncommittedSyncedTransactions: [] as Array< - PendingSyncedTransaction - >, - hasTruncateSync: false, - hasImmediateSync: false, - }, - ) + case `delete`: + this.syncedData.deleteWithKnownPreviousValue( + key, + hadPreviousVisibleValue, + previousVisibleValue, + ) + this.syncedMetadata.delete(key) + this.rowOrigins.delete(key) + break + } - // Process committed transactions if: - // 1. No persisting user transaction (normal sync flow), OR - // 2. There's a truncate operation (must be processed immediately), OR - // 3. There's an immediate transaction (manual writes must be processed synchronously) - // - // Note: When hasImmediateSync or hasTruncateSync is true, we process ALL committed - // sync transactions (not just the immediate/truncate ones). This is intentional for - // ordering correctness: if we only processed the immediate transaction, earlier - // non-immediate transactions would be applied later and could overwrite newer state. - // Processing all committed transactions together preserves causal ordering. - if (!hasPersistingTransaction || hasTruncateSync || hasImmediateSync) { - // Set flag to prevent redundant optimistic state recalculations - this.isCommittingSyncTransactions = true - - const previousRowOrigins = new Map(this.rowOrigins) - const previousOptimisticUpserts = new Map(this.optimisticUpserts) - const previousOptimisticDeletes = new Set(this.optimisticDeletes) - - // Get the optimistic snapshot from the truncate transaction (captured when truncate() was called) - const truncateOptimisticSnapshot = hasTruncateSync - ? committedSyncedTransactions.find((t) => t.truncate) - ?.optimisticSnapshot - : null - let truncatePendingLocalChanges: Set | undefined - let truncatePendingLocalOrigins: Set | undefined - - // First collect all keys that will be affected by sync operations - const changedKeys = new Set() - for (const transaction of committedSyncedTransactions) { - for (const operation of transaction.operations) { - changedKeys.add(operation.key as TKey) - } - for (const [key] of transaction.rowMetadataWrites) { - changedKeys.add(key) + for (const [metadataKey, metadataWrite] of transaction.rowMetadataWrites) { + if (metadataWrite.type === `delete`) { + this.syncedMetadata.delete(metadataKey) + } else { + this.syncedMetadata.set(metadataKey, metadataWrite.value) } } + } finally { + this.isCommittingSyncTransactions = false + } - // Use pre-captured state if available (from optimistic scenarios), - // otherwise capture current state (for pure sync scenarios) - let currentVisibleState = this.preSyncVisibleState - if (currentVisibleState.size === 0) { - // No pre-captured state, capture it now for pure sync operations - currentVisibleState = new Map() - for (const key of changedKeys) { - const currentValue = this.get(key) - if (currentValue !== undefined) { - currentVisibleState.set(key, currentValue) - } - } + const events: Array> = [] + + if (previousVisibleValue === undefined && newVisibleValue !== undefined) { + events.push({ + type: `insert`, + key, + value: newVisibleValue, + }) + } else if ( + previousVisibleValue !== undefined && + newVisibleValue === undefined + ) { + events.push({ + type: `delete`, + key, + value: this.enrichWithVirtualPropsSnapshot( + previousVisibleValue, + previousVirtualProps!, + ), + }) + } else if ( + previousVisibleValue !== undefined && + newVisibleValue !== undefined + ) { + const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) + const virtualChanged = + previousVirtualProps!.$synced !== nextVirtualProps.$synced || + previousVirtualProps!.$origin !== nextVirtualProps.$origin + const valuesEqual = perfDeepEquals(previousVisibleValue, newVisibleValue) + + if (!valuesEqual || virtualChanged) { + events.push({ + type: `update`, + key, + value: newVisibleValue, + previousValue: this.enrichWithVirtualPropsSnapshot( + previousVisibleValue, + previousVirtualProps!, + ), + }) } + } - const events: Array> = [] - const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` - const completedOptimisticOps = new Map< - TKey, - { type: string; value: TOutput } - >() + this.size = this.syncedData.size + if (events.length > 0) { + this.indexes.updateIndexes(events) + } + this.changes.emitEvents(events, true) - for (const transaction of this.transactions.values()) { - if (transaction.state === `completed`) { - for (const mutation of transaction.mutations) { - if (this.isThisCollection(mutation.collection)) { - if (mutation.optimistic) { - completedOptimisticOps.set(mutation.key, { - type: mutation.type, - value: mutation.modified as TOutput, - }) - } - } - } - } + this.pendingSyncedTransactions = uncommittedSyncedTransactions + this.preSyncVisibleState.clear() + Promise.resolve().then(() => { + this.recentlySyncedKeys.clear() + }) + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } + + return { + changedKeyCount: 1, + eventCount: events.length, + } + } + + private commitDoublePureSyncTransaction( + transaction: PendingSyncedTransaction, + uncommittedSyncedTransactions: Array< + PendingSyncedTransaction + >, + ): SimpleSyncCommitResult | undefined { + const changedKeys = new Set() + + for (const operation of transaction.operations) { + if (operation.type !== `delete` && !(`value` in operation)) { + return undefined } - for (const transaction of committedSyncedTransactions) { - // Handle truncate operations first - if (transaction.truncate) { - // TRUNCATE PHASE - // 1) Emit a delete for every visible key (synced + optimistic) so downstream listeners/indexes - // observe a clear-before-rebuild. We intentionally skip keys already in - // optimisticDeletes because their delete was previously emitted by the user. - // Use the snapshot to ensure we emit deletes for all items that existed at truncate start. - const visibleKeys = new Set([ - ...this.syncedData.keys(), - ...(truncateOptimisticSnapshot?.upserts.keys() || []), - ]) - for (const key of visibleKeys) { - if (truncateOptimisticSnapshot?.deletes.has(key)) continue - const previousValue = - truncateOptimisticSnapshot?.upserts.get(key) || - this.syncedData.get(key) - if (previousValue !== undefined) { - events.push({ type: `delete`, key, value: previousValue }) - } - } + const key = operation.key as TKey + if (changedKeys.has(key)) { + return undefined + } + changedKeys.add(key) + } - // 2) Clear the authoritative synced base. Subsequent server ops in this - // same commit will rebuild the base atomically. - // Preserve pending local tracking just long enough for operations in this - // truncate batch to retain correct local origin semantics. - truncatePendingLocalChanges = new Set(this.pendingLocalChanges) - truncatePendingLocalOrigins = new Set(this.pendingLocalOrigins) - this.syncedData.clear() - this.syncedMetadata.clear() - this.syncedKeys.clear() - this.clearOriginTrackingState() - - // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations - // are compared against the post-truncate state (undefined) rather than pre-truncate state - // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events - for (const key of changedKeys) { - currentVisibleState.delete(key) - } + for (const [metadataKey] of transaction.rowMetadataWrites) { + if (!changedKeys.has(metadataKey)) { + return undefined + } + } - // 4) Emit truncate event so subscriptions can reset their cursor tracking state - this._events.emit(`truncate`, { - type: `truncate`, - collection: this.collection, - }) - } + const previousValues = new Map() + const previousVirtualProps = new Map>() - for (const operation of transaction.operations) { - const key = operation.key as TKey - this.syncedKeys.add(key) - - // Determine origin: 'local' for local-only collections or pending local changes - const origin: VirtualOrigin = - this.isLocalOnly || - this.pendingLocalChanges.has(key) || - this.pendingLocalOrigins.has(key) || - truncatePendingLocalChanges?.has(key) === true || - truncatePendingLocalOrigins?.has(key) === true - ? 'local' - : 'remote' - - // Update synced data - switch (operation.type) { - case `insert`: - this.syncedData.set(key, operation.value) - this.rowOrigins.set(key, origin) - // Clear pending local changes now that sync has confirmed - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) - break - case `update`: { - if (rowUpdateMode === `partial`) { - const updatedValue = Object.assign( - {}, - this.syncedData.get(key), - operation.value, - ) - this.syncedData.set(key, updatedValue) - } else { - this.syncedData.set(key, operation.value) - } - this.rowOrigins.set(key, origin) - // Clear pending local changes now that sync has confirmed - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) - break - } - case `delete`: - this.syncedData.delete(key) - this.syncedMetadata.delete(key) - // Clean up origin and pending tracking for deleted rows - this.rowOrigins.delete(key) - this.pendingLocalChanges.delete(key) - this.pendingLocalOrigins.delete(key) - this.pendingOptimisticUpserts.delete(key) - this.pendingOptimisticDeletes.delete(key) - this.pendingOptimisticDirectUpserts.delete(key) - this.pendingOptimisticDirectDeletes.delete(key) - break + for (const key of changedKeys) { + const previousValue = this.syncedData.get(key) + if (previousValue !== undefined) { + previousValues.set(key, previousValue) + previousVirtualProps.set(key, this.getVirtualPropsSnapshotForState(key)) + } + } + + this.isCommittingSyncTransactions = true + try { + const origin: VirtualOrigin = this.isLocalOnly ? 'local' : 'remote' + const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` + + for (const operation of transaction.operations) { + const key = operation.key as TKey + const previousValue = previousValues.get(key) + const hadPreviousValue = previousValues.has(key) + this.syncedKeys.add(key) + + switch (operation.type) { + case `insert`: + this.syncedData.setWithKnownPresence( + key, + operation.value, + hadPreviousValue, + previousValue, + ) + this.setRowOrigin(key, origin, hadPreviousValue) + break + case `update`: { + const updatedValue = + rowUpdateMode === `partial` + ? Object.assign({}, previousValue, operation.value) + : operation.value + this.syncedData.setWithKnownPresence( + key, + updatedValue, + hadPreviousValue, + previousValue, + ) + this.setRowOrigin(key, origin, hadPreviousValue) + break } + case `delete`: + this.syncedData.deleteWithKnownPreviousValue( + key, + hadPreviousValue, + previousValue, + ) + this.syncedMetadata.delete(key) + this.rowOrigins.delete(key) + break } + } - for (const [key, metadataWrite] of transaction.rowMetadataWrites) { - if (metadataWrite.type === `delete`) { - this.syncedMetadata.delete(key) - continue - } - this.syncedMetadata.set(key, metadataWrite.value) + for (const [metadataKey, metadataWrite] of transaction.rowMetadataWrites) { + if (metadataWrite.type === `delete`) { + this.syncedMetadata.delete(metadataKey) + } else { + this.syncedMetadata.set(metadataKey, metadataWrite.value) } + } + } finally { + this.isCommittingSyncTransactions = false + } + + const events: Array> = [] - for (const [ + for (const key of changedKeys) { + const hadPreviousValue = previousValues.has(key) + const previousValue = previousValues.get(key) + const newValue = this.syncedData.get(key) + + if (!hadPreviousValue && newValue !== undefined) { + events.push({ + type: `insert`, key, - metadataWrite, - ] of transaction.collectionMetadataWrites) { - if (metadataWrite.type === `delete`) { - this.syncedCollectionMetadata.delete(key) - continue - } - this.syncedCollectionMetadata.set(key, metadataWrite.value) + value: newValue, + }) + } else if (hadPreviousValue && newValue === undefined) { + events.push({ + type: `delete`, + key, + value: this.enrichWithVirtualPropsSnapshot( + previousValue!, + previousVirtualProps.get(key)!, + ), + }) + } else if (hadPreviousValue && newValue !== undefined) { + const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) + const previousProps = previousVirtualProps.get(key)! + const virtualChanged = + previousProps.$synced !== nextVirtualProps.$synced || + previousProps.$origin !== nextVirtualProps.$origin + const valuesEqual = perfDeepEquals(previousValue, newValue) + + if (!valuesEqual || virtualChanged) { + events.push({ + type: `update`, + key, + value: newValue, + previousValue: this.enrichWithVirtualPropsSnapshot( + previousValue!, + previousProps, + ), + }) } } + } - // After applying synced operations, if this commit included a truncate, - // re-apply optimistic mutations on top of the fresh synced base. This ensures - // the UI preserves local intent while respecting server rebuild semantics. - // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. - if (hasTruncateSync) { - // Avoid duplicating keys that were inserted/updated by synced operations in this commit - const syncedInsertedOrUpdatedKeys = new Set() - for (const t of committedSyncedTransactions) { - for (const op of t.operations) { - if (op.type === `insert` || op.type === `update`) { - syncedInsertedOrUpdatedKeys.add(op.key as TKey) - } + this.size = this.syncedData.size + if (events.length > 0) { + this.indexes.updateIndexes(events) + } + this.changes.emitEvents(events, true) + + this.pendingSyncedTransactions = uncommittedSyncedTransactions + this.preSyncVisibleState.clear() + Promise.resolve().then(() => { + this.recentlySyncedKeys.clear() + }) + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } + + return { + changedKeyCount: changedKeys.size, + eventCount: events.length, + } + } + + /** + * Attempts to commit pending synced transactions if there are no active transactions + * This method processes operations from pending transactions and applies them to the synced data + */ + commitPendingTransactions = () => { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.commitPendingTransactions`, { + collectionId: this.collection.id, + }) + : undefined + let committedSyncedTransactionCount = 0 + let activeOptimisticTransactionCount = 0 + let changedKeyCount = 0 + let eventCount = 0 + + try { + const pendingSyncedTransactionCount = + this.pendingSyncedTransactions.length + if ( + pendingSyncedTransactionCount === 1 && + this.transactions.size === 0 + ) { + const transaction = this.pendingSyncedTransactions[0]! + if (transaction.committed) { + const simpleSyncCommit = this.commitSinglePureSyncTransaction( + transaction, + [], + false, + false, + false, + ) + if (simpleSyncCommit) { + committedSyncedTransactionCount = 1 + changedKeyCount = simpleSyncCommit.changedKeyCount + eventCount = simpleSyncCommit.eventCount + return } } + } - // Build re-apply sets from the snapshot taken at the start of this function. - // This prevents losing optimistic state if transactions complete during truncate processing. - const reapplyUpserts = new Map( - truncateOptimisticSnapshot!.upserts, - ) - const reapplyDeletes = new Set( - truncateOptimisticSnapshot!.deletes, - ) + const classifySpan = shouldTrace + ? startPerfSpan(`collection.commitPendingTransactions.classify`, { + collectionId: this.collection.id, + }) + : undefined + // Check if there are any persisting transaction + let hasPersistingTransaction = false + for (const transaction of this.transactions.values()) { + if (transaction.state === `persisting`) { + hasPersistingTransaction = true + break + } + } - // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete. - // If the server also inserted/updated the same key in this batch, override that value - // with the optimistic value to preserve local intent. - for (const [key, value] of reapplyUpserts) { - if (reapplyDeletes.has(key)) continue - if (syncedInsertedOrUpdatedKeys.has(key)) { - let foundInsert = false - for (let i = events.length - 1; i >= 0; i--) { - const evt = events[i]! - if (evt.key === key && evt.type === `insert`) { - evt.value = value - foundInsert = true - break - } + // pending synced transactions could be either `committed` or still open. + // we only want to process `committed` transactions here + const { + committedSyncedTransactions, + uncommittedSyncedTransactions, + hasTruncateSync, + hasImmediateSync, + } = this.pendingSyncedTransactions.reduce( + (acc, t) => { + if (t.committed) { + acc.committedSyncedTransactions.push(t) + if (t.truncate) { + acc.hasTruncateSync = true } - if (!foundInsert) { - events.push({ type: `insert`, key, value }) + if (t.immediate) { + acc.hasImmediateSync = true } } else { - events.push({ type: `insert`, key, value }) + acc.uncommittedSyncedTransactions.push(t) } - } + return acc + }, + { + committedSyncedTransactions: [] as Array< + PendingSyncedTransaction + >, + uncommittedSyncedTransactions: [] as Array< + PendingSyncedTransaction + >, + hasTruncateSync: false, + hasImmediateSync: false, + }, + ) + committedSyncedTransactionCount = committedSyncedTransactions.length + activeOptimisticTransactionCount = this.transactions.size + classifySpan?.end() + + const singleCommittedSyncedTransaction = + committedSyncedTransactions.length === 1 + ? committedSyncedTransactions[0]! + : undefined + const simpleSyncCommit = this.commitSinglePureSyncTransaction( + singleCommittedSyncedTransaction, + uncommittedSyncedTransactions, + hasPersistingTransaction, + hasTruncateSync, + hasImmediateSync, + ) + if (simpleSyncCommit) { + changedKeyCount = simpleSyncCommit.changedKeyCount + eventCount = simpleSyncCommit.eventCount + return + } - // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete. - if (events.length > 0 && reapplyDeletes.size > 0) { - const filtered: Array> = [] - for (const evt of events) { - if (evt.type === `insert` && reapplyDeletes.has(evt.key)) { - continue + // Process committed transactions if: + // 1. No persisting user transaction (normal sync flow), OR + // 2. There's a truncate operation (must be processed immediately), OR + // 3. There's an immediate transaction (manual writes must be processed synchronously) + // + // Note: When hasImmediateSync or hasTruncateSync is true, we process ALL committed + // sync transactions (not just the immediate/truncate ones). This is intentional for + // ordering correctness: if we only processed the immediate transaction, earlier + // non-immediate transactions would be applied later and could overwrite newer state. + // Processing all committed transactions together preserves causal ordering. + if (!hasPersistingTransaction || hasTruncateSync || hasImmediateSync) { + // Set flag to prevent redundant optimistic state recalculations + this.isCommittingSyncTransactions = true + + // Get the optimistic snapshot from the truncate transaction (captured when truncate() was called) + const truncateOptimisticSnapshot = hasTruncateSync + ? committedSyncedTransactions.find((t) => t.truncate) + ?.optimisticSnapshot + : null + let truncatePendingLocalChanges: Set | undefined + let truncatePendingLocalOrigins: Set | undefined + + // First collect all keys that will be affected by sync operations + const changedKeysSpan = shouldTrace + ? startPerfSpan(`collection.commitPendingTransactions.changedKeys`, { + collectionId: this.collection.id, + }) + : undefined + const changedKeys = new Set() + for (const transaction of committedSyncedTransactions) { + for (const operation of transaction.operations) { + changedKeys.add(operation.key as TKey) + } + for (const [key] of transaction.rowMetadataWrites) { + changedKeys.add(key) + } + } + changedKeyCount = changedKeys.size + changedKeysSpan?.end() + + const snapshotSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.snapshotState`, + { + collectionId: this.collection.id, + }, + ) + : undefined + const virtualSnapshotKeys = new Set(changedKeys) + for (const key of this.pendingOptimisticDirectUpserts) { + virtualSnapshotKeys.add(key) + } + for (const key of this.pendingOptimisticDirectDeletes) { + virtualSnapshotKeys.add(key) + } + const previousRowOrigins = + this.snapshotRowOriginsForKeys(virtualSnapshotKeys) + const previousOptimisticUpserts = new Map(this.optimisticUpserts) + const previousOptimisticDeletes = new Set(this.optimisticDeletes) + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.snapshotState.rowOrigins`, + previousRowOrigins.size, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.snapshotState.optimisticUpserts`, + previousOptimisticUpserts.size, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.snapshotState.optimisticDeletes`, + previousOptimisticDeletes.size, + { collectionId: this.collection.id }, + ) + } + snapshotSpan?.end() + + // Use pre-captured state if available (from optimistic scenarios), + // otherwise capture current state (for pure sync scenarios) + const visibleStateSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.captureVisibleState`, + { + collectionId: this.collection.id, + }, + ) + : undefined + let currentVisibleState = this.preSyncVisibleState + if (currentVisibleState.size === 0) { + // No pre-captured state, capture it now for pure sync operations + currentVisibleState = new Map() + for (const key of changedKeys) { + const currentValue = this.get(key) + if (currentValue !== undefined) { + currentVisibleState.set(key, currentValue) } - filtered.push(evt) } - events.length = 0 - events.push(...filtered) } - - // Ensure listeners are active before emitting this critical batch - if (this.lifecycle.status !== `ready`) { - this.lifecycle.markReady() + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.captureVisibleState.keys`, + currentVisibleState.size, + { collectionId: this.collection.id }, + ) } - } - - // Maintain optimistic state appropriately - // Clear optimistic state since sync operations will now provide the authoritative data. - // Any still-active user transactions will be re-applied below in recompute. - this.optimisticUpserts.clear() - this.optimisticDeletes.clear() - - // Reset flag and recompute optimistic state for any remaining active transactions - this.isCommittingSyncTransactions = false - - // If we had a truncate, restore the preserved optimistic state from the snapshot - // This includes items from transactions that may have completed during processing - if (hasTruncateSync && truncateOptimisticSnapshot) { - for (const [key, value] of truncateOptimisticSnapshot.upserts) { - this.optimisticUpserts.set(key, value) + visibleStateSpan?.end() + + const events: Array> = [] + const rowUpdateMode = this.config.sync.rowUpdateMode || `partial` + const completedOptimisticOps = new Map< + TKey, + { type: string; value: TOutput } + >() + + const completedOpsSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.completedOptimisticOps`, + { + collectionId: this.collection.id, + }, + ) + : undefined + for (const transaction of this.transactions.values()) { + if (transaction.state === `completed`) { + for (const mutation of transaction.mutations) { + if (this.isThisCollection(mutation.collection)) { + if (mutation.optimistic) { + completedOptimisticOps.set(mutation.key, { + type: mutation.type, + value: mutation.modified as TOutput, + }) + } + } + } + } } - for (const key of truncateOptimisticSnapshot.deletes) { - this.optimisticDeletes.add(key) + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.completedOptimisticOps.ops`, + completedOptimisticOps.size, + { collectionId: this.collection.id }, + ) } - } + completedOpsSpan?.end() + + const applySyncSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.applySyncedTransactions`, + { + collectionId: this.collection.id, + }, + ) + : undefined + let appliedOperationCount = 0 + let rowMetadataWriteCount = 0 + let collectionMetadataWriteCount = 0 + let truncateCount = 0 + try { + for (const transaction of committedSyncedTransactions) { + // Handle truncate operations first + if (transaction.truncate) { + truncateCount++ + const truncateSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.applyTruncate`, + { + collectionId: this.collection.id, + }, + ) + : undefined + try { + // TRUNCATE PHASE + // 1) Emit a delete for every visible key (synced + optimistic) so downstream listeners/indexes + // observe a clear-before-rebuild. We intentionally skip keys already in + // optimisticDeletes because their delete was previously emitted by the user. + // Use the snapshot to ensure we emit deletes for all items that existed at truncate start. + const visibleKeys = new Set([ + ...this.syncedData.keys(), + ...(truncateOptimisticSnapshot?.upserts.keys() || []), + ]) + for (const key of visibleKeys) { + if (truncateOptimisticSnapshot?.deletes.has(key)) continue + const previousValue = + truncateOptimisticSnapshot?.upserts.get(key) || + this.syncedData.get(key) + if (previousValue !== undefined) { + events.push({ type: `delete`, key, value: previousValue }) + } + } + + // 2) Clear the authoritative synced base. Subsequent server ops in this + // same commit will rebuild the base atomically. + // Preserve pending local tracking just long enough for operations in this + // truncate batch to retain correct local origin semantics. + truncatePendingLocalChanges = new Set(this.pendingLocalChanges) + truncatePendingLocalOrigins = new Set(this.pendingLocalOrigins) + this.syncedData.clear() + this.syncedMetadata.clear() + this.syncedKeys.clear() + this.clearOriginTrackingState() + + // 3) Clear currentVisibleState for truncated keys to ensure subsequent operations + // are compared against the post-truncate state (undefined) rather than pre-truncate state + // This ensures that re-inserted keys are emitted as INSERT events, not UPDATE events + for (const key of changedKeys) { + currentVisibleState.delete(key) + } + + // 4) Emit truncate event so subscriptions can reset their cursor tracking state + this._events.emit(`truncate`, { + type: `truncate`, + collection: this.collection, + }) + } finally { + truncateSpan?.end() + } + } - // Always overlay any still-active optimistic transactions so mutations that started - // after the truncate snapshot are preserved. - for (const transaction of this.transactions.values()) { - if (![`completed`, `failed`].includes(transaction.state)) { - for (const mutation of transaction.mutations) { - if ( - this.isThisCollection(mutation.collection) && - mutation.optimistic - ) { - switch (mutation.type) { + for (const operation of transaction.operations) { + appliedOperationCount++ + const key = operation.key as TKey + this.syncedKeys.add(key) + + // Determine origin: 'local' for local-only collections or pending local changes + const origin: VirtualOrigin = + this.isLocalOnly || + this.pendingLocalChanges.has(key) || + this.pendingLocalOrigins.has(key) || + truncatePendingLocalChanges?.has(key) === true || + truncatePendingLocalOrigins?.has(key) === true + ? 'local' + : 'remote' + + // Update synced data + switch (operation.type) { case `insert`: - case `update`: - this.optimisticUpserts.set( - mutation.key, - mutation.modified as TOutput, - ) - this.optimisticDeletes.delete(mutation.key) + this.syncedData.set(key, operation.value) + this.setRowOrigin(key, origin, currentVisibleState.has(key)) + // Clear pending local changes now that sync has confirmed + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) break + case `update`: { + if (rowUpdateMode === `partial`) { + const updatedValue = Object.assign( + {}, + this.syncedData.get(key), + operation.value, + ) + this.syncedData.set(key, updatedValue) + } else { + this.syncedData.set(key, operation.value) + } + this.setRowOrigin(key, origin, currentVisibleState.has(key)) + // Clear pending local changes now that sync has confirmed + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) + break + } case `delete`: - this.optimisticUpserts.delete(mutation.key) - this.optimisticDeletes.add(mutation.key) + this.syncedData.delete(key) + this.syncedMetadata.delete(key) + // Clean up origin and pending tracking for deleted rows + this.rowOrigins.delete(key) + this.pendingLocalChanges.delete(key) + this.pendingLocalOrigins.delete(key) + this.pendingOptimisticUpserts.delete(key) + this.pendingOptimisticDeletes.delete(key) + this.pendingOptimisticDirectUpserts.delete(key) + this.pendingOptimisticDirectDeletes.delete(key) break } } - } - } - } - // A completed optimistic insert may have used a temporary client key while - // the sync confirmation used a different server-generated key. Once a - // sync commit has been applied, stop retaining completed optimistic keys - // that were not confirmed by this commit so the temporary row is removed. - for (const key of this.pendingOptimisticDirectUpserts) { - if (!changedKeys.has(key)) { - changedKeys.add(key) - if (!currentVisibleState.has(key)) { - const previousValue = previousOptimisticUpserts.get(key) - if (previousValue !== undefined) { - currentVisibleState.set(key, previousValue) + for (const [key, metadataWrite] of transaction.rowMetadataWrites) { + rowMetadataWriteCount++ + if (metadataWrite.type === `delete`) { + this.syncedMetadata.delete(key) + continue + } + this.syncedMetadata.set(key, metadataWrite.value) + } + + for (const [ + key, + metadataWrite, + ] of transaction.collectionMetadataWrites) { + collectionMetadataWriteCount++ + if (metadataWrite.type === `delete`) { + this.syncedCollectionMetadata.delete(key) + continue + } + this.syncedCollectionMetadata.set(key, metadataWrite.value) } } - this.pendingOptimisticUpserts.delete(key) - this.pendingLocalOrigins.delete(key) - } - } - for (const key of this.pendingOptimisticDirectDeletes) { - if (!changedKeys.has(key)) { - changedKeys.add(key) + } finally { + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.applySyncedTransactions.operations`, + appliedOperationCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.applySyncedTransactions.rowMetadataWrites`, + rowMetadataWriteCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.applySyncedTransactions.collectionMetadataWrites`, + collectionMetadataWriteCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.applySyncedTransactions.truncates`, + truncateCount, + { collectionId: this.collection.id }, + ) + } + applySyncSpan?.end() } - this.pendingOptimisticDeletes.delete(key) - this.pendingLocalOrigins.delete(key) - } - this.pendingOptimisticDirectUpserts.clear() - this.pendingOptimisticDirectDeletes.clear() - - // Now check what actually changed in the final visible state - for (const key of changedKeys) { - const previousVisibleValue = currentVisibleState.get(key) - const newVisibleValue = this.get(key) // This returns the new derived state - const previousVirtualProps = this.getVirtualPropsSnapshotForState(key, { - rowOrigins: previousRowOrigins, - optimisticUpserts: previousOptimisticUpserts, - optimisticDeletes: previousOptimisticDeletes, - completedOptimisticKeys: completedOptimisticOps, - }) - const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) - const virtualChanged = - previousVirtualProps.$synced !== nextVirtualProps.$synced || - previousVirtualProps.$origin !== nextVirtualProps.$origin - const previousValueWithVirtual = - previousVisibleValue !== undefined - ? enrichRowWithVirtualProps( - previousVisibleValue, - key, - this.collection.id, - () => previousVirtualProps.$synced, - () => previousVirtualProps.$origin, + + // After applying synced operations, if this commit included a truncate, + // re-apply optimistic mutations on top of the fresh synced base. This ensures + // the UI preserves local intent while respecting server rebuild semantics. + // Ordering: deletes (above) -> server ops (just applied) -> optimistic upserts. + if (hasTruncateSync) { + const truncateReapplySpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.reapplyTruncateOptimisticState`, + { + collectionId: this.collection.id, + }, ) : undefined + try { + // Avoid duplicating keys that were inserted/updated by synced operations in this commit + const syncedInsertedOrUpdatedKeys = new Set() + for (const t of committedSyncedTransactions) { + for (const op of t.operations) { + if (op.type === `insert` || op.type === `update`) { + syncedInsertedOrUpdatedKeys.add(op.key as TKey) + } + } + } + + // Build re-apply sets from the snapshot taken at the start of this function. + // This prevents losing optimistic state if transactions complete during truncate processing. + const reapplyUpserts = new Map( + truncateOptimisticSnapshot!.upserts, + ) + const reapplyDeletes = new Set( + truncateOptimisticSnapshot!.deletes, + ) + + // Emit inserts for re-applied upserts, skipping any keys that have an optimistic delete. + // If the server also inserted/updated the same key in this batch, override that value + // with the optimistic value to preserve local intent. + for (const [key, value] of reapplyUpserts) { + if (reapplyDeletes.has(key)) continue + if (syncedInsertedOrUpdatedKeys.has(key)) { + let foundInsert = false + for (let i = events.length - 1; i >= 0; i--) { + const evt = events[i]! + if (evt.key === key && evt.type === `insert`) { + evt.value = value + foundInsert = true + break + } + } + if (!foundInsert) { + events.push({ type: `insert`, key, value }) + } + } else { + events.push({ type: `insert`, key, value }) + } + } - // Check if this sync operation is redundant with a completed optimistic operation - const completedOp = completedOptimisticOps.get(key) - let isRedundantSync = false - - if (completedOp) { - if ( - completedOp.type === `delete` && - previousVisibleValue !== undefined && - newVisibleValue === undefined && - deepEquals(completedOp.value, previousVisibleValue) - ) { - isRedundantSync = true - } else if ( - newVisibleValue !== undefined && - deepEquals(completedOp.value, newVisibleValue) - ) { - isRedundantSync = true + // Finally, ensure we do NOT insert keys that have an outstanding optimistic delete. + if (events.length > 0 && reapplyDeletes.size > 0) { + const filtered: Array> = [] + for (const evt of events) { + if (evt.type === `insert` && reapplyDeletes.has(evt.key)) { + continue + } + filtered.push(evt) + } + events.length = 0 + events.push(...filtered) + } + + // Ensure listeners are active before emitting this critical batch + if (this.lifecycle.status !== `ready`) { + this.lifecycle.markReady() + } + } finally { + truncateReapplySpan?.end() } } - const shouldEmitVirtualUpdate = - virtualChanged && - previousVisibleValue !== undefined && - newVisibleValue !== undefined && - deepEquals(previousVisibleValue, newVisibleValue) + // Maintain optimistic state appropriately + // Clear optimistic state since sync operations will now provide the authoritative data. + // Any still-active user transactions will be re-applied below in recompute. + const optimisticRebuildSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.rebuildOptimisticState`, + { + collectionId: this.collection.id, + }, + ) + : undefined + let activeOptimisticMutationCount = 0 + try { + this.optimisticUpserts.clear() + this.optimisticDeletes.clear() + + // Reset flag and recompute optimistic state for any remaining active transactions + this.isCommittingSyncTransactions = false + + // If we had a truncate, restore the preserved optimistic state from the snapshot + // This includes items from transactions that may have completed during processing + if (hasTruncateSync && truncateOptimisticSnapshot) { + for (const [key, value] of truncateOptimisticSnapshot.upserts) { + this.optimisticUpserts.set(key, value) + } + for (const key of truncateOptimisticSnapshot.deletes) { + this.optimisticDeletes.add(key) + } + } - if (isRedundantSync && !shouldEmitVirtualUpdate) { - continue + // Always overlay any still-active optimistic transactions so mutations that started + // after the truncate snapshot are preserved. + for (const transaction of this.transactions.values()) { + if (![`completed`, `failed`].includes(transaction.state)) { + for (const mutation of transaction.mutations) { + if ( + this.isThisCollection(mutation.collection) && + mutation.optimistic + ) { + activeOptimisticMutationCount++ + switch (mutation.type) { + case `insert`: + case `update`: + this.optimisticUpserts.set( + mutation.key, + mutation.modified as TOutput, + ) + this.optimisticDeletes.delete(mutation.key) + break + case `delete`: + this.optimisticUpserts.delete(mutation.key) + this.optimisticDeletes.add(mutation.key) + break + } + } + } + } + } + + // A completed optimistic insert may have used a temporary client key while + // the sync confirmation used a different server-generated key. Once a + // sync commit has been applied, stop retaining completed optimistic keys + // that were not confirmed by this commit so the temporary row is removed. + for (const key of this.pendingOptimisticDirectUpserts) { + if (!changedKeys.has(key)) { + changedKeys.add(key) + if (!currentVisibleState.has(key)) { + const previousValue = previousOptimisticUpserts.get(key) + if (previousValue !== undefined) { + currentVisibleState.set(key, previousValue) + } + } + this.pendingOptimisticUpserts.delete(key) + this.pendingLocalOrigins.delete(key) + } + } + for (const key of this.pendingOptimisticDirectDeletes) { + if (!changedKeys.has(key)) { + changedKeys.add(key) + } + this.pendingOptimisticDeletes.delete(key) + this.pendingLocalOrigins.delete(key) + } + this.pendingOptimisticDirectUpserts.clear() + this.pendingOptimisticDirectDeletes.clear() + changedKeyCount = changedKeys.size + } finally { + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.rebuildOptimisticState.activeMutations`, + activeOptimisticMutationCount, + { collectionId: this.collection.id }, + ) + } + optimisticRebuildSpan?.end() } - if ( - previousVisibleValue === undefined && - newVisibleValue !== undefined - ) { - const completedOptimisticOp = completedOptimisticOps.get(key) - if (completedOptimisticOp) { - const previousValueFromCompleted = completedOptimisticOp.value - const previousValueWithVirtualFromCompleted = - enrichRowWithVirtualProps( - previousValueFromCompleted, - key, - this.collection.id, - () => previousVirtualProps.$synced, - () => previousVirtualProps.$origin, - ) - events.push({ - type: `update`, - key, - value: newVisibleValue, - previousValue: previousValueWithVirtualFromCompleted, + // Now check what actually changed in the final visible state + const buildEventsSpan = shouldTrace + ? startPerfSpan(`collection.commitPendingTransactions.buildEvents`, { + collectionId: this.collection.id, }) - } else { - events.push({ - type: `insert`, + : undefined + let insertEventCount = 0 + let updateEventCount = 0 + let deleteEventCount = 0 + let redundantSyncCount = 0 + let virtualUpdateCount = 0 + try { + for (const key of changedKeys) { + const previousVisibleValue = currentVisibleState.get(key) + const newVisibleValue = this.get(key) // This returns the new derived state + const previousVirtualProps = this.getVirtualPropsSnapshotForState( key, - value: newVisibleValue, - }) + { + rowOrigins: previousRowOrigins, + optimisticUpserts: previousOptimisticUpserts, + optimisticDeletes: previousOptimisticDeletes, + completedOptimisticKeys: completedOptimisticOps, + }, + ) + const nextVirtualProps = this.getVirtualPropsSnapshotForState(key) + const virtualChanged = + previousVirtualProps.$synced !== nextVirtualProps.$synced || + previousVirtualProps.$origin !== nextVirtualProps.$origin + const previousValueWithVirtual = + previousVisibleValue !== undefined + ? enrichRowWithVirtualProps( + previousVisibleValue, + key, + this.collection.id, + () => previousVirtualProps.$synced, + () => previousVirtualProps.$origin, + ) + : undefined + + // Check if this sync operation is redundant with a completed optimistic operation + const completedOp = completedOptimisticOps.get(key) + let isRedundantSync = false + + if (completedOp) { + if ( + completedOp.type === `delete` && + previousVisibleValue !== undefined && + newVisibleValue === undefined && + perfDeepEquals(completedOp.value, previousVisibleValue) + ) { + isRedundantSync = true + } else if ( + newVisibleValue !== undefined && + perfDeepEquals(completedOp.value, newVisibleValue) + ) { + isRedundantSync = true + } + } + + const shouldEmitVirtualUpdate = + virtualChanged && + previousVisibleValue !== undefined && + newVisibleValue !== undefined && + perfDeepEquals(previousVisibleValue, newVisibleValue) + + if (shouldEmitVirtualUpdate) { + virtualUpdateCount++ + } + + if (isRedundantSync && !shouldEmitVirtualUpdate) { + redundantSyncCount++ + continue + } + + if ( + previousVisibleValue === undefined && + newVisibleValue !== undefined + ) { + const completedOptimisticOp = completedOptimisticOps.get(key) + if (completedOptimisticOp) { + const previousValueFromCompleted = completedOptimisticOp.value + const previousValueWithVirtualFromCompleted = + enrichRowWithVirtualProps( + previousValueFromCompleted, + key, + this.collection.id, + () => previousVirtualProps.$synced, + () => previousVirtualProps.$origin, + ) + events.push({ + type: `update`, + key, + value: newVisibleValue, + previousValue: previousValueWithVirtualFromCompleted, + }) + updateEventCount++ + } else { + events.push({ + type: `insert`, + key, + value: newVisibleValue, + }) + insertEventCount++ + } + } else if ( + previousVisibleValue !== undefined && + newVisibleValue === undefined + ) { + events.push({ + type: `delete`, + key, + value: previousValueWithVirtual ?? previousVisibleValue, + }) + deleteEventCount++ + } else if ( + previousVisibleValue !== undefined && + newVisibleValue !== undefined && + (!perfDeepEquals(previousVisibleValue, newVisibleValue) || + shouldEmitVirtualUpdate) + ) { + events.push({ + type: `update`, + key, + value: newVisibleValue, + previousValue: previousValueWithVirtual ?? previousVisibleValue, + }) + updateEventCount++ + } } - } else if ( - previousVisibleValue !== undefined && - newVisibleValue === undefined - ) { - events.push({ - type: `delete`, - key, - value: previousValueWithVirtual ?? previousVisibleValue, - }) - } else if ( - previousVisibleValue !== undefined && - newVisibleValue !== undefined && - (!deepEquals(previousVisibleValue, newVisibleValue) || - shouldEmitVirtualUpdate) - ) { - events.push({ - type: `update`, - key, - value: newVisibleValue, - previousValue: previousValueWithVirtual ?? previousVisibleValue, - }) + } finally { + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.buildEvents.insertEvents`, + insertEventCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.buildEvents.updateEvents`, + updateEventCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.buildEvents.deleteEvents`, + deleteEventCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.buildEvents.redundantSyncs`, + redundantSyncCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.buildEvents.virtualUpdates`, + virtualUpdateCount, + { collectionId: this.collection.id }, + ) + } + buildEventsSpan?.end() } - } - // Update cached size after synced data changes - this.size = this.calculateSize() - - // Update indexes for all events before emitting - if (events.length > 0) { - this.indexes.updateIndexes(events) - } + // Update cached size after synced data changes + const sizeSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.calculateSize`, + { + collectionId: this.collection.id, + }, + ) + : undefined + this.size = this.calculateSize() + sizeSpan?.end() + + // Update indexes for all events before emitting + if (events.length > 0) { + const indexSpan = shouldTrace + ? startPerfSpan( + `collection.commitPendingTransactions.indexUpdate`, + { + collectionId: this.collection.id, + }, + ) + : undefined + this.indexes.updateIndexes(events) + indexSpan?.end() + } - // End batching and emit all events (combines any batched events with sync events) - this.changes.emitEvents(events, true) + // End batching and emit all events (combines any batched events with sync events) + eventCount = events.length + const eventSpan = shouldTrace + ? startPerfSpan(`collection.commitPendingTransactions.emitEvents`, { + collectionId: this.collection.id, + }) + : undefined + this.changes.emitEvents(events, true) + eventSpan?.end() - this.pendingSyncedTransactions = uncommittedSyncedTransactions + const cleanupSpan = shouldTrace + ? startPerfSpan(`collection.commitPendingTransactions.cleanup`, { + collectionId: this.collection.id, + }) + : undefined + try { + this.pendingSyncedTransactions = uncommittedSyncedTransactions - // Clear the pre-sync state since sync operations are complete - this.preSyncVisibleState.clear() + // Clear the pre-sync state since sync operations are complete + this.preSyncVisibleState.clear() - // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them - Promise.resolve().then(() => { - this.recentlySyncedKeys.clear() - }) + // Clear recently synced keys after a microtask to allow recomputeOptimisticState to see them + Promise.resolve().then(() => { + this.recentlySyncedKeys.clear() + }) - // Mark that we've received the first commit (for tracking purposes) - if (!this.hasReceivedFirstCommit) { - this.hasReceivedFirstCommit = true + // Mark that we've received the first commit (for tracking purposes) + if (!this.hasReceivedFirstCommit) { + this.hasReceivedFirstCommit = true + } + } finally { + cleanupSpan?.end() + } + } + } finally { + if (shouldTrace) { + recordPerfCount( + `collection.commitPendingTransactions.committedSyncedTransactions`, + committedSyncedTransactionCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.activeOptimisticTransactions`, + activeOptimisticTransactionCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.changedKeys`, + changedKeyCount, + { collectionId: this.collection.id }, + ) + recordPerfCount( + `collection.commitPendingTransactions.events`, + eventCount, + { collectionId: this.collection.id }, + ) + span?.end({ events: eventCount }) } } } diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 2d48add4b6..53d65f46e4 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -4,6 +4,11 @@ import { PropRef, Value } from '../query/ir.js' import { EventEmitter } from '../event-emitter.js' import { compileExpression } from '../query/compiler/evaluators.js' import { buildCursor } from '../utils/cursor.js' +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, +} from '../query/live/perf.js' import { createFilterFunctionFromExpression, createFilteredCallback, @@ -319,7 +324,21 @@ export class CollectionSubscription } emitEvents(changes: Array>) { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.subscription.emitEvents`, { + collectionId: this.collection.id, + }) + : undefined const newChanges = this.filterAndFlipChanges(changes) + if (shouldTrace) { + recordPerfCount(`collection.subscription.eventsIn`, changes.length, { + collectionId: this.collection.id, + }) + recordPerfCount(`collection.subscription.eventsOut`, newChanges.length, { + collectionId: this.collection.id, + }) + } if (this.isBufferingForTruncate) { // Buffer the changes instead of emitting immediately @@ -327,9 +346,12 @@ export class CollectionSubscription if (newChanges.length > 0) { this.truncateBuffer.push(newChanges) } - } else { + } else if (newChanges.length > 0 || changes.length === 0) { + // Empty input is an explicit readiness signal. Non-empty input that + // filtered to an empty delta has no subscriber-visible work to deliver. this.filteredCallback(newChanges) } + span?.end({ outputRows: newChanges.length }) } /** @@ -340,8 +362,16 @@ export class CollectionSubscription * or, the entire state was already loaded. */ requestSnapshot(opts?: RequestSnapshotOptions): boolean { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.subscription.requestSnapshot`, { + collectionId: this.collection.id, + optimizedOnly: opts?.optimizedOnly === true, + }) + : undefined if (this.loadedInitialState) { // Subscription was deoptimized so we already sent the entire initial state + span?.end({ skipped: true }) return false } @@ -394,6 +424,7 @@ export class CollectionSubscription if (snapshot === undefined) { // Couldn't load from indexes + span?.end({ optimized: false }) return false } @@ -411,6 +442,14 @@ export class CollectionSubscription this.snapshotSent = true this.callback(filteredSnapshot) + if (shouldTrace) { + recordPerfCount( + `collection.subscription.snapshotRows`, + filteredSnapshot.length, + { collectionId: this.collection.id }, + ) + span?.end({ rows: filteredSnapshot.length }) + } return true } @@ -442,6 +481,14 @@ export class CollectionSubscription `Ordered snapshot was requested but no index was found. You have to call setOrderByIndex before requesting an ordered snapshot.`, ) } + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.subscription.requestLimitedSnapshot`, { + collectionId: this.collection.id, + limit, + hasMinValues: minValues !== undefined && minValues.length > 0, + }) + : undefined // Check if minValues has a first element (regardless of its value) // This distinguishes between "no min value provided" vs "min value is undefined" @@ -624,6 +671,14 @@ export class CollectionSubscription if (shouldTrackLoadSubsetPromise) { this.trackLoadSubsetPromise(syncResult) } + if (shouldTrace) { + recordPerfCount( + `collection.subscription.limitedSnapshotRows`, + changes.length, + { collectionId: this.collection.id }, + ) + span?.end({ rows: changes.length }) + } } // TODO: also add similar test but that checks that it can also load it from the collection's loadSubset function @@ -650,18 +705,21 @@ export class CollectionSubscription // 3. We're collecting all changes atomically, so filtering doesn't make sense const skipDeleteFilter = this.isBufferingForTruncate - const newChanges = [] - for (const change of changes) { + let newChanges: Array> | undefined + for (let index = 0; index < changes.length; index++) { + const change = changes[index]! let newChange = change const keyInSentKeys = this.sentKeys.has(change.key) if (!keyInSentKeys) { if (change.type === `update`) { + newChanges ??= changes.slice(0, index) newChange = { ...change, type: `insert`, previousValue: undefined } } else if (change.type === `delete`) { // Filter out deletes for keys that have not been sent, // UNLESS we're buffering for truncate (where all deletes should pass through) if (!skipDeleteFilter) { + newChanges ??= changes.slice(0, index) continue } } @@ -673,6 +731,7 @@ export class CollectionSubscription // This prevents D2 multiplicity from going above 1, which would // cause deletes to not properly remove items (multiplicity would // go from 2 to 1 instead of 1 to 0). + newChanges ??= changes.slice(0, index) continue } else if (change.type === `delete`) { // Remove from sentKeys so future inserts for this key are allowed @@ -680,9 +739,9 @@ export class CollectionSubscription this.sentKeys.delete(change.key) } } - newChanges.push(newChange) + newChanges?.push(newChange) } - return newChanges + return newChanges ?? changes } private trackSentKeys(changes: Array>) { diff --git a/packages/db/src/collection/sync.ts b/packages/db/src/collection/sync.ts index af89ed2cf3..1962725c31 100644 --- a/packages/db/src/collection/sync.ts +++ b/packages/db/src/collection/sync.ts @@ -10,6 +10,11 @@ import { } from '../errors' import { deepEquals } from '../utils' import { LIVE_QUERY_INTERNAL } from '../query/live/internal.js' +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, +} from '../query/live/perf.js' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessageOrDeleteKeyMessage, @@ -90,14 +95,30 @@ export class CollectionSyncManager< this.config.sync.sync({ collection: this.collection, begin: (options?: { immediate?: boolean }) => { - this.state.pendingSyncedTransactions.push({ - committed: false, - operations: [], - deletedKeys: new Set(), - rowMetadataWrites: new Map(), - collectionMetadataWrites: new Map(), - immediate: options?.immediate, - }) + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.sync.begin`, { + collectionId: this.id, + immediate: options?.immediate === true, + }) + : undefined + try { + this.state.pendingSyncedTransactions.push({ + committed: false, + operations: [], + deletedKeys: new Set(), + rowMetadataWrites: new Map(), + collectionMetadataWrites: new Map(), + immediate: options?.immediate, + }) + } finally { + if (shouldTrace) { + recordPerfCount(`collection.sync.begin.calls`, 1, { + collectionId: this.id, + }) + span?.end() + } + } }, write: ( messageWithOptionalKey: ChangeMessageOrDeleteKeyMessage< @@ -105,113 +126,161 @@ export class CollectionSyncManager< TKey >, ) => { - const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionWriteError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedWriteError() - } + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.sync.write`, { + collectionId: this.id, + type: messageWithOptionalKey.type, + }) + : undefined + try { + const pendingTransaction = + this.state.pendingSyncedTransactions[ + this.state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionWriteError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedWriteError() + } - let key: TKey | undefined = undefined - if (`key` in messageWithOptionalKey) { - key = messageWithOptionalKey.key - } else { - key = this.config.getKey(messageWithOptionalKey.value) - } + let key: TKey | undefined = undefined + if (`key` in messageWithOptionalKey) { + key = messageWithOptionalKey.key + } else { + key = this.config.getKey(messageWithOptionalKey.value) + } - if (this.state.pendingLocalChanges.has(key)) { - this.state.pendingLocalOrigins.add(key) - } + if (this.state.pendingLocalChanges.has(key)) { + this.state.pendingLocalOrigins.add(key) + } - let messageType = messageWithOptionalKey.type - - // Check if an item with this key already exists when inserting - if (messageWithOptionalKey.type === `insert`) { - const insertingIntoExistingSynced = this.state.syncedData.has(key) - const hasPendingDeleteForKey = - pendingTransaction.deletedKeys.has(key) - const isTruncateTransaction = pendingTransaction.truncate === true - // Allow insert after truncate in the same transaction even if it existed in syncedData - if ( - insertingIntoExistingSynced && - !hasPendingDeleteForKey && - !isTruncateTransaction - ) { - const existingValue = this.state.syncedData.get(key) - const valuesEqual = - existingValue !== undefined && - deepEquals(existingValue, messageWithOptionalKey.value) - if (valuesEqual) { - // The "insert" is an echo of a value we already have locally. - // Treat it as an update so we preserve optimistic intent without - // throwing a duplicate-key error during reconciliation. - messageType = `update` - } else { - const utils = this.config.utils as - | Partial - | undefined - const internal = utils?.[LIVE_QUERY_INTERNAL] - throw new DuplicateKeySyncError(key, this.id, { - hasCustomGetKey: internal?.hasCustomGetKey ?? false, - hasJoins: internal?.hasJoins ?? false, - hasDistinct: internal?.hasDistinct ?? false, - }) + let messageType = messageWithOptionalKey.type + + // Check if an item with this key already exists when inserting + if (messageWithOptionalKey.type === `insert`) { + const insertingIntoExistingSynced = + this.state.syncedData.has(key) + const hasPendingDeleteForKey = + pendingTransaction.deletedKeys.has(key) + const isTruncateTransaction = + pendingTransaction.truncate === true + // Allow insert after truncate in the same transaction even if it existed in syncedData + if ( + insertingIntoExistingSynced && + !hasPendingDeleteForKey && + !isTruncateTransaction + ) { + const existingValue = this.state.syncedData.get(key) + const valuesEqual = + existingValue !== undefined && + deepEquals(existingValue, messageWithOptionalKey.value) + if (valuesEqual) { + // The "insert" is an echo of a value we already have locally. + // Treat it as an update so we preserve optimistic intent without + // throwing a duplicate-key error during reconciliation. + messageType = `update` + } else { + const utils = this.config.utils as + | Partial + | undefined + const internal = utils?.[LIVE_QUERY_INTERNAL] + throw new DuplicateKeySyncError(key, this.id, { + hasCustomGetKey: internal?.hasCustomGetKey ?? false, + hasJoins: internal?.hasJoins ?? false, + hasDistinct: internal?.hasDistinct ?? false, + }) + } } } - } - const message = { - ...messageWithOptionalKey, - type: messageType, - key, - } as OptimisticChangeMessage - pendingTransaction.operations.push(message) - - if (messageType === `delete`) { - pendingTransaction.deletedKeys.add(key) - pendingTransaction.rowMetadataWrites.set(key, { type: `delete` }) - } else if (messageType === `insert`) { - if (message.metadata !== undefined) { + const message = { + ...messageWithOptionalKey, + type: messageType, + key, + } as OptimisticChangeMessage + pendingTransaction.operations.push(message) + + if (messageType === `delete`) { + pendingTransaction.deletedKeys.add(key) + pendingTransaction.rowMetadataWrites.set(key, { + type: `delete`, + }) + } else if (messageType === `insert`) { + if (message.metadata !== undefined) { + pendingTransaction.rowMetadataWrites.set(key, { + type: `set`, + value: message.metadata, + }) + } else { + pendingTransaction.rowMetadataWrites.set(key, { + type: `delete`, + }) + } + } else if (message.metadata !== undefined) { pendingTransaction.rowMetadataWrites.set(key, { type: `set`, value: message.metadata, }) - } else { - pendingTransaction.rowMetadataWrites.set(key, { - type: `delete`, + } + } finally { + if (shouldTrace) { + recordPerfCount(`collection.sync.write.calls`, 1, { + collectionId: this.id, + type: messageWithOptionalKey.type, }) + span?.end() } - } else if (message.metadata !== undefined) { - pendingTransaction.rowMetadataWrites.set(key, { - type: `set`, - value: message.metadata, - }) } }, commit: () => { - const pendingTransaction = - this.state.pendingSyncedTransactions[ - this.state.pendingSyncedTransactions.length - 1 - ] - if (!pendingTransaction) { - throw new NoPendingSyncTransactionCommitError() - } - if (pendingTransaction.committed) { - throw new SyncTransactionAlreadyCommittedError() - } + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collection.sync.commit`, { + collectionId: this.id, + }) + : undefined + try { + const pendingTransaction = + this.state.pendingSyncedTransactions[ + this.state.pendingSyncedTransactions.length - 1 + ] + if (!pendingTransaction) { + throw new NoPendingSyncTransactionCommitError() + } + if (pendingTransaction.committed) { + throw new SyncTransactionAlreadyCommittedError() + } - pendingTransaction.committed = true + pendingTransaction.committed = true + if (shouldTrace) { + recordPerfCount( + `collection.sync.commit.operations`, + pendingTransaction.operations.length, + { collectionId: this.id }, + ) + } - this.state.commitPendingTransactions() + this.state.commitPendingTransactions() + } finally { + if (shouldTrace) { + recordPerfCount(`collection.sync.commit.calls`, 1, { + collectionId: this.id, + }) + span?.end() + } + } }, markReady: () => { this.lifecycle.markReady() }, truncate: () => { + if (isPerfEnabled()) { + recordPerfCount(`collection.sync.truncate.calls`, 1, { + collectionId: this.id, + }) + } const pendingTransaction = this.state.pendingSyncedTransactions[ this.state.pendingSyncedTransactions.length - 1 diff --git a/packages/db/src/indexes/base-index.ts b/packages/db/src/indexes/base-index.ts index 945221e6fa..b6645225d7 100644 --- a/packages/db/src/indexes/base-index.ts +++ b/packages/db/src/indexes/base-index.ts @@ -1,6 +1,10 @@ -import { compileSingleRowExpression } from '../query/compiler/evaluators.js' +import { + + compileSingleRowExpression +} from '../query/compiler/evaluators.js' import { comparisonFunctions } from '../query/builder/functions.js' import { DEFAULT_COMPARE_OPTIONS, deepEquals } from '../utils.js' +import type {CompiledSingleRowExpression} from '../query/compiler/evaluators.js'; import type { RangeQueryOptions } from './btree-index.js' import type { CompareOptions } from '../query/builder/types.js' import type { BasicExpression, OrderByDirection } from '../query/ir.js' @@ -97,8 +101,9 @@ export abstract class BaseIndex< protected lookupCount = 0 protected totalLookupTime = 0 - protected lastUpdated = new Date() + private lastUpdatedTimestamp = Date.now() protected compareOptions: CompareOptions + private readonly expressionEvaluator: CompiledSingleRowExpression /** * Set by subclasses when constructed with a user-supplied comparator, whose * ordering may not match the WHERE evaluator's relational operators. @@ -115,6 +120,7 @@ export abstract class BaseIndex< this.expression = expression this.compareOptions = DEFAULT_COMPARE_OPTIONS this.name = name + this.expressionEvaluator = compileSingleRowExpression(expression) this.initialize(options) } @@ -203,15 +209,14 @@ export abstract class BaseIndex< lookupCount: this.lookupCount, averageLookupTime: this.lookupCount > 0 ? this.totalLookupTime / this.lookupCount : 0, - lastUpdated: this.lastUpdated, + lastUpdated: new Date(this.lastUpdatedTimestamp), } } protected abstract initialize(options?: any): void protected evaluateIndexExpression(item: any): any { - const evaluator = compileSingleRowExpression(this.expression) - return evaluator(item as Record) + return this.expressionEvaluator(item as Record) } protected trackLookup(startTime: number): void { @@ -221,7 +226,7 @@ export abstract class BaseIndex< } protected updateTimestamp(): void { - this.lastUpdated = new Date() + this.lastUpdatedTimestamp = Date.now() } } diff --git a/packages/db/src/indexes/basic-index.ts b/packages/db/src/indexes/basic-index.ts index b9b06d1925..3c6ca8dc14 100644 --- a/packages/db/src/indexes/basic-index.ts +++ b/packages/db/src/indexes/basic-index.ts @@ -89,9 +89,10 @@ export class BasicIndex< const normalizedValue = normalizeValue(indexedValue) - if (this.valueMap.has(normalizedValue)) { + const keySet = this.valueMap.get(normalizedValue) + if (keySet) { // Value already exists, just add the key to the set - this.valueMap.get(normalizedValue)!.add(key) + keySet.add(key) } else { // New value - add to map and insert into sorted array this.valueMap.set(normalizedValue, new Set([key])) @@ -128,8 +129,8 @@ export class BasicIndex< const normalizedValue = normalizeValue(indexedValue) - if (this.valueMap.has(normalizedValue)) { - const keySet = this.valueMap.get(normalizedValue)! + const keySet = this.valueMap.get(normalizedValue) + if (keySet) { keySet.delete(key) if (keySet.size === 0) { @@ -175,8 +176,9 @@ export class BasicIndex< // Group by value for (const { key, value } of entriesArray) { - if (this.valueMap.has(value)) { - this.valueMap.get(value)!.add(key) + const keySet = this.valueMap.get(value) + if (keySet) { + keySet.add(key) } else { this.valueMap.set(value, new Set([key])) } diff --git a/packages/db/src/indexes/btree-index.ts b/packages/db/src/indexes/btree-index.ts index 8b92095f01..ecd5a3bad8 100644 --- a/packages/db/src/indexes/btree-index.ts +++ b/packages/db/src/indexes/btree-index.ts @@ -94,14 +94,14 @@ export class BTreeIndex< // Normalize the value for Map key usage const normalizedValue = normalizeForBTree(indexedValue) - // Check if this value already exists - if (this.valueMap.has(normalizedValue)) { + const keySet = this.valueMap.get(normalizedValue) + if (keySet) { // Add to existing set - this.valueMap.get(normalizedValue)!.add(key) + keySet.add(key) } else { // Create new set for this value - const keySet = new Set([key]) - this.valueMap.set(normalizedValue, keySet) + const newKeySet = new Set([key]) + this.valueMap.set(normalizedValue, newKeySet) this.orderedEntries.set(normalizedValue, undefined) } @@ -127,8 +127,8 @@ export class BTreeIndex< // Normalize the value for Map key usage const normalizedValue = normalizeForBTree(indexedValue) - if (this.valueMap.has(normalizedValue)) { - const keySet = this.valueMap.get(normalizedValue)! + const keySet = this.valueMap.get(normalizedValue) + if (keySet) { keySet.delete(key) // If set is now empty, remove the entry entirely diff --git a/packages/db/src/query/compiler/evaluators.ts b/packages/db/src/query/compiler/evaluators.ts index fa2e90725d..82f8b503ab 100644 --- a/packages/db/src/query/compiler/evaluators.ts +++ b/packages/db/src/query/compiler/evaluators.ts @@ -216,7 +216,26 @@ function compileRef(ref: PropRef): CompiledExpression { function compileSingleRowRef(ref: PropRef): CompiledSingleRowExpression { const propertyPath = ref.path - // This function works for all path lengths including empty path + if (propertyPath.length === 0) { + return (item) => item + } + + if (propertyPath.length === 1) { + const prop = propertyPath[0]! + return (item) => item[prop] + } + + if (propertyPath.length === 2) { + const firstProp = propertyPath[0]! + const secondProp = propertyPath[1]! + return (item) => { + const firstValue = item[firstProp] + return firstValue == null + ? firstValue + : (firstValue as Record)[secondProp] + } + } + return (item) => { let value: any = item for (const prop of propertyPath) { diff --git a/packages/db/src/query/compiler/group-by.ts b/packages/db/src/query/compiler/group-by.ts index c670de9649..0112cc2b3e 100644 --- a/packages/db/src/query/compiler/group-by.ts +++ b/packages/db/src/query/compiler/group-by.ts @@ -135,6 +135,7 @@ export function processGroupBy( ): NamespacedAndKeyedStream { const virtualAggregates: Record = { [VIRTUAL_SYNCED_KEY]: { + kind: `every`, preMap: ([, row]: [string, NamespacedRow]) => getRowVirtualMetadata(row).synced, reduce: (values: Array<[boolean, number]>) => { @@ -147,6 +148,7 @@ export function processGroupBy( }, }, [VIRTUAL_HAS_LOCAL_KEY]: { + kind: `some`, preMap: ([, row]: [string, NamespacedRow]) => getRowVirtualMetadata(row).hasLocal, reduce: (values: Array<[boolean, number]>) => { diff --git a/packages/db/src/query/effect.ts b/packages/db/src/query/effect.ts index 0237519212..bc64b2f819 100644 --- a/packages/db/src/query/effect.ts +++ b/packages/db/src/query/effect.ts @@ -731,7 +731,7 @@ class EffectPipelineRunner { this.isGraphRunning = true try { while (this.graph.pendingWork()) { - this.graph.run() + this.graph.runWithPendingWork() // A handler (via onBatchProcessed) or source error callback may have // called dispose() during graph.run(). Stop early to avoid operating // on stale state. TS narrows disposed to false from the guard above diff --git a/packages/db/src/query/live/collection-config-builder.ts b/packages/db/src/query/live/collection-config-builder.ts index a6a51b4788..e702b0f336 100644 --- a/packages/db/src/query/live/collection-config-builder.ts +++ b/packages/db/src/query/live/collection-config-builder.ts @@ -14,6 +14,12 @@ import { getActiveTransaction } from '../../transactions.js' import { CollectionSubscriber } from './collection-subscriber.js' import { getCollectionBuilder } from './collection-registry.js' import { LIVE_QUERY_INTERNAL } from './internal.js' +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, + withPerfSpan, +} from './perf.js' import { buildQueryFromConfig, extractCollectionAliases, @@ -349,6 +355,11 @@ export class CollectionConfigBuilder< // no nested runs of the graph // which is possible if the `callback` // would call `maybeRunGraph` e.g. after it has loaded some more data + if (isPerfEnabled()) { + recordPerfCount(`liveQuery.maybeRunGraph.nestedSkips`, 1, { + builderId: this.id, + }) + } return } @@ -360,6 +371,15 @@ export class CollectionConfigBuilder< } this.isGraphRunning = true + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`liveQuery.maybeRunGraph`, { + builderId: this.id, + }) + : undefined + let graphRuns = 0 + let flushes = 0 + let callbacks = 0 try { const { begin, commit } = this.currentSyncConfig @@ -374,12 +394,23 @@ export class CollectionConfigBuilder< if (syncState.subscribedToAllCollections) { let callbackCalled = false while (syncState.graph.pendingWork()) { - syncState.graph.run() + graphRuns++ + withPerfSpan( + `liveQuery.graph.run`, + { + builderId: this.id, + }, + () => { + syncState.graph.runWithPendingWork() + }, + ) // Flush accumulated changes after each graph step to commit them as one transaction. // This ensures intermediate join states (like null on one side) don't cause // duplicate key errors when the full join result arrives in the same step. syncState.flushPendingChanges?.() + flushes++ callback?.() + callbacks++ callbackCalled = true } @@ -388,6 +419,7 @@ export class CollectionConfigBuilder< // an async loadSubset completes and we need to re-check if more data is needed. if (!callbackCalled) { callback?.() + callbacks++ } // On the initial run, we may need to do an empty commit to ensure that @@ -405,6 +437,18 @@ export class CollectionConfigBuilder< this.updateLiveQueryStatus(this.currentSyncConfig) } } finally { + if (shouldTrace) { + recordPerfCount(`liveQuery.maybeRunGraph.graphRuns`, graphRuns, { + builderId: this.id, + }) + recordPerfCount(`liveQuery.maybeRunGraph.flushes`, flushes, { + builderId: this.id, + }) + recordPerfCount(`liveQuery.maybeRunGraph.callbacks`, callbacks, { + builderId: this.id, + }) + span?.end({ graphRuns }) + } this.isGraphRunning = false } } @@ -437,6 +481,20 @@ export class CollectionConfigBuilder< }, ) { const contextId = options?.contextId ?? getActiveTransaction()?.id + if (isPerfEnabled()) { + recordPerfCount(`liveQuery.scheduleGraphRun`, 1, { + builderId: this.id, + alias: options?.alias, + context: contextId !== undefined, + hasCallback: callback !== undefined, + }) + } + + if (typeof contextId === `undefined`) { + this.executeImmediateGraphRun(callback) + return + } + // Use the builder instance as the job ID for deduplication. This is memory-safe // because the scheduler's context Map is deleted after flushing (no long-term retention). const jobId = options?.jobId ?? this @@ -462,11 +520,9 @@ export class CollectionConfigBuilder< // Ensure dependent builders are actually scheduled in this context so that // dependency edges always point to a real job (or a deduped no-op if already scheduled). - if (contextId) { - for (const dep of dependentBuilders) { - if (typeof dep.scheduleGraphRun === `function`) { - dep.scheduleGraphRun(undefined, { contextId }) - } + for (const dep of dependentBuilders) { + if (typeof dep.scheduleGraphRun === `function`) { + dep.scheduleGraphRun(undefined, { contextId }) } } @@ -486,27 +542,58 @@ export class CollectionConfigBuilder< pending = { loadCallbacks: new Set(), } - if (contextId) { - this.pendingGraphRuns.set(contextId, pending) - } + this.pendingGraphRuns.set(contextId, pending) } // Add callback if provided (this is what accumulates between schedules) if (callback) { pending.loadCallbacks.add(callback) + if (isPerfEnabled()) { + recordPerfCount(`liveQuery.scheduleGraphRun.callbacks`, 1, { + builderId: this.id, + }) + } } // Schedule execution (scheduler just orchestrates order, we manage state) - // For immediate execution (no contextId), pass pending directly since it won't be in the map - const pendingToPass = contextId ? undefined : pending transactionScopedScheduler.schedule({ contextId, jobId, dependencies: dependentBuilders, - run: () => this.executeGraphRun(contextId, pendingToPass), + run: () => this.executeGraphRun(contextId), }) } + private executeImmediateGraphRun(callback?: () => boolean): void { + if (!this.currentSyncConfig || !this.currentSyncState) { + throw new Error( + `scheduleGraphRun called without active sync session. This should not happen.`, + ) + } + + const shouldTrace = isPerfEnabled() + const callbackCount = callback ? 1 : 0 + const span = shouldTrace + ? startPerfSpan(`liveQuery.executeGraphRun`, { + builderId: this.id, + callbackCount, + context: false, + }) + : undefined + if (shouldTrace) { + recordPerfCount(`liveQuery.executeGraphRun.callbacks`, callbackCount, { + builderId: this.id, + }) + } + + this.incrementRunCount() + try { + this.maybeRunGraph(callback) + } finally { + span?.end() + } + } + /** * Clears pending graph run state for a specific context. * Called when the scheduler clears a context (e.g., transaction rollback/abort). @@ -528,46 +615,73 @@ export class CollectionConfigBuilder< * create fresh state and don't interfere with the current execution. * Uses instance sync state - if sync has ended, gracefully returns without executing. * - * @param contextId - Optional context ID to look up pending state - * @param pendingParam - For immediate execution (no context), pending state is passed directly + * @param contextId - Context ID used to look up pending state */ - private executeGraphRun( - contextId?: SchedulerContextId, - pendingParam?: PendingGraphRun, - ): void { - // Get pending state: either from parameter (no context) or from map (with context) + private executeGraphRun(contextId: SchedulerContextId): void { // Remove from map BEFORE checking sync state to prevent leaking entries when sync ends // before the transaction flushes (e.g., unsubscribe during in-flight transaction) - const pending = - pendingParam ?? - (contextId ? this.pendingGraphRuns.get(contextId) : undefined) - if (contextId) { - this.pendingGraphRuns.delete(contextId) - } + const pending = this.pendingGraphRuns.get(contextId) + this.pendingGraphRuns.delete(contextId) // If no pending state, nothing to execute (context was cleared) if (!pending) { + if (isPerfEnabled()) { + recordPerfCount(`liveQuery.executeGraphRun.skips`, 1, { + builderId: this.id, + reason: `missingPending`, + }) + } return } // If sync session has ended, don't execute (graph is finalized, subscriptions cleared) if (!this.currentSyncConfig || !this.currentSyncState) { + if (isPerfEnabled()) { + recordPerfCount(`liveQuery.executeGraphRun.skips`, 1, { + builderId: this.id, + reason: `inactiveSync`, + }) + } return } + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`liveQuery.executeGraphRun`, { + builderId: this.id, + callbackCount: pending.loadCallbacks.size, + context: true, + }) + : undefined + if (shouldTrace) { + recordPerfCount( + `liveQuery.executeGraphRun.callbacks`, + pending.loadCallbacks.size, + { builderId: this.id }, + ) + } this.incrementRunCount() const combinedLoader = () => { let allDone = true let firstError: unknown - pending.loadCallbacks.forEach((loader) => { - try { - allDone = loader() && allDone - } catch (error) { - allDone = false - firstError ??= error - } - }) + withPerfSpan( + `liveQuery.executeGraphRun.loadCallbacks`, + { + builderId: this.id, + callbackCount: pending.loadCallbacks.size, + }, + () => { + pending.loadCallbacks.forEach((loader) => { + try { + allDone = loader() && allDone + } catch (error) { + allDone = false + firstError ??= error + } + }) + }, + ) if (firstError) { throw firstError } @@ -575,7 +689,11 @@ export class CollectionConfigBuilder< return allDone } - this.maybeRunGraph(combinedLoader) + try { + this.maybeRunGraph(combinedLoader) + } finally { + span?.end() + } } private getSyncConfig(): SyncConfig { @@ -753,12 +871,24 @@ export class CollectionConfigBuilder< pipeline.pipe( output((data) => { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`liveQuery.output.parent`, { + builderId: this.id, + }) + : undefined const messages = data.getInner() syncState.messagesCount += messages.length + if (shouldTrace) { + recordPerfCount(`liveQuery.output.parentRows`, messages.length, { + builderId: this.id, + }) + } // Accumulate changes from this output callback into the pending changes map. // Changes for the same key are merged (inserts/deletes are added together). messages.reduce(accumulateChanges, pendingChanges) + span?.end() }), ) @@ -771,57 +901,85 @@ export class CollectionConfigBuilder< // Flush pending changes and reset the accumulator. // Called at the end of each graph run to commit all accumulated changes. syncState.flushPendingChanges = () => { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`liveQuery.flushPendingChanges`, { + builderId: this.id, + }) + : undefined const hasParentChanges = pendingChanges.size > 0 const hasChildChanges = hasPendingIncludesChanges(includesState) + const pendingParentChangeCount = pendingChanges.size - if (!hasParentChanges && !hasChildChanges) { - return - } + try { + if (!hasParentChanges && !hasChildChanges) { + return + } - let changesToApply = pendingChanges - - // When a custom getKey is provided, multiple D2 internal keys may map - // to the same user-visible key. Re-accumulate by custom key so that a - // retract + insert for the same logical row merges into an UPDATE - // instead of a separate DELETE and INSERT that can race. - if (this.config.getKey) { - const merged = new Map>() - for (const [, changes] of pendingChanges) { - const customKey = this.config.getKey(changes.value) - const existing = merged.get(customKey) - if (existing) { - existing.inserts += changes.inserts - existing.deletes += changes.deletes - // Keep the value from the insert side (the new value) - if (changes.inserts > 0) { - existing.value = changes.value - if (changes.orderByIndex !== undefined) { - existing.orderByIndex = changes.orderByIndex + let changesToApply = pendingChanges + + // When a custom getKey is provided, multiple D2 internal keys may map + // to the same user-visible key. Re-accumulate by custom key so that a + // retract + insert for the same logical row merges into an UPDATE + // instead of a separate DELETE and INSERT that can race. + if (this.config.getKey) { + const merged = new Map>() + for (const [, changes] of pendingChanges) { + const customKey = this.config.getKey(changes.value) + const existing = merged.get(customKey) + if (existing) { + existing.inserts += changes.inserts + existing.deletes += changes.deletes + // Keep the value from the insert side (the new value) + if (changes.inserts > 0) { + existing.value = changes.value + if (changes.orderByIndex !== undefined) { + existing.orderByIndex = changes.orderByIndex + } } + } else { + merged.set(customKey, { ...changes }) } - } else { - merged.set(customKey, { ...changes }) } + changesToApply = merged } - changesToApply = merged - } - // 1. Flush parent changes - if (hasParentChanges) { - begin() - changesToApply.forEach(this.applyChanges.bind(this, config)) - commit() + // 1. Flush parent changes + if (hasParentChanges) { + begin() + for (const [key, changes] of changesToApply) { + this.applyChanges(config, changes, key) + } + commit() + } + pendingChanges = new Map() + + // 2. Process includes: create/dispose child Collections, route child changes + flushIncludesState( + includesState, + config.collection, + this.id, + hasParentChanges ? changesToApply : null, + config, + ) + } finally { + if (shouldTrace) { + recordPerfCount( + `liveQuery.flushPendingChanges.parentChanges`, + pendingParentChangeCount, + { builderId: this.id }, + ) + recordPerfCount( + `liveQuery.flushPendingChanges.includesStates`, + includesState.length, + { builderId: this.id }, + ) + span?.end({ + hasParentChanges, + hasChildChanges, + }) + } } - pendingChanges = new Map() - - // 2. Process includes: create/dispose child Collections, route child changes - flushIncludesState( - includesState, - config.collection, - this.id, - hasParentChanges ? changesToApply : null, - config, - ) } graph.finalize() @@ -857,14 +1015,30 @@ export class CollectionConfigBuilder< scalarField: entry.scalarField, childRegistry: new Map(), pendingChildChanges: new Map(), + dirtyChildEntries: new Set(), correlationToParentKeys: new Map(), } // Attach output callback on the child pipeline entry.pipeline.pipe( output((data) => { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`liveQuery.output.includes`, { + builderId: this.id, + fieldName: state.fieldName, + materialization: state.materialization, + }) + : undefined const messages = data.getInner() syncState.messagesCount += messages.length + if (shouldTrace) { + recordPerfCount(`liveQuery.output.includesRows`, messages.length, { + builderId: this.id, + fieldName: state.fieldName, + materialization: state.materialization, + }) + } for (const [[childKey, tupleData], multiplicity] of messages) { const [childResult, _orderByIndex, correlationKey, parentContext] = @@ -900,6 +1074,7 @@ export class CollectionConfigBuilder< byChild.set(childKey, existing) } + span?.end() }), ) @@ -1219,10 +1394,14 @@ type IncludesOutputState = { childRegistry: Map /** Pending child changes: correlationKey → Map */ pendingChildChanges: Map>> + /** Child entries whose nested include state has pending work */ + dirtyChildEntries: Set /** Reverse index: correlation key → Set of parent collection keys */ correlationToParentKeys: Map> /** Shared nested pipeline setups (one per nested includes level) */ nestedSetups?: Array + parentState?: IncludesOutputState + parentCorrelationKey?: unknown } type ChildCollectionEntry = { @@ -1238,6 +1417,33 @@ function materializesInline(state: IncludesOutputState): boolean { return state.materialization !== `collection` } +function markIncludesStateDirty(state: IncludesOutputState): void { + let current = state + while (current.parentState) { + current.parentState.dirtyChildEntries.add(current.parentCorrelationKey) + current = current.parentState + } +} + +function markNestedBufferRoutesDirty( + setup: NestedIncludesSetup, + nestedRoutingKey: unknown, +): void { + const stateRoutes = setup.routingIndex.get(nestedRoutingKey) + if (!stateRoutes) return + + for (const [targetState, parentRoutes] of stateRoutes) { + const targetSetupIndex = targetState.nestedSetups?.indexOf(setup) ?? -1 + if (targetSetupIndex < 0) continue + + for (const parentCorrelationKey of parentRoutes.keys()) { + const entry = targetState.childRegistry.get(parentCorrelationKey) + const entryState = entry?.includesStates?.[targetSetupIndex] + if (entryState) markIncludesStateDirty(entryState) + } + } +} + function materializeIncludedValue( state: IncludesOutputState, entry: ChildCollectionEntry | undefined, @@ -1293,6 +1499,7 @@ function setupNestedPipelines( output((data) => { const messages = data.getInner() syncState.messagesCount += messages.length + const touchedRoutingKeys = new Set() for (const [[childKey, tupleData], multiplicity] of messages) { const [childResult, _orderByIndex, correlationKey, parentContext] = @@ -1326,6 +1533,11 @@ function setupNestedPipelines( } byChild.set(childKey, existing) + touchedRoutingKeys.add(routingKey) + } + + for (const routingKey of touchedRoutingKeys) { + markNestedBufferRoutesDirty(setup, routingKey) } }), ) @@ -1357,6 +1569,8 @@ function setupNestedPipelines( */ function createPerEntryIncludesStates( setups: Array, + parentState?: IncludesOutputState, + parentCorrelationKey?: unknown, ): Array { return setups.map((setup) => { const state: IncludesOutputState = { @@ -1368,9 +1582,15 @@ function createPerEntryIncludesStates( scalarField: setup.compilationResult.scalarField, childRegistry: new Map(), pendingChildChanges: new Map(), + dirtyChildEntries: new Set(), correlationToParentKeys: new Map(), } + if (parentState) { + state.parentState = parentState + state.parentCorrelationKey = parentCorrelationKey + } + if (setup.nestedSetups) { state.nestedSetups = setup.nestedSetups } @@ -1464,6 +1684,7 @@ function seedParentFromSnapshot( orderByIndex: row.orderByIndex, }) } + markIncludesStateDirty(entryState) } /** @@ -1523,6 +1744,7 @@ function drainNestedBuffers(state: IncludesOutputState): Set { byChild.set(childKey, { ...changes }) } } + markIncludesStateDirty(entryState) if (targetState === state) { dirtyCorrelationKeys.add(parentCorrelationKey) @@ -1814,6 +2036,7 @@ function createChildCollectionEntry( correlationKey: unknown, hasOrderBy: boolean, nestedSetups?: Array, + parentState?: IncludesOutputState, ): ChildCollectionEntry { const resultKeys = new WeakMap() const orderByIndices = hasOrderBy ? new WeakMap() : null @@ -1850,7 +2073,11 @@ function createChildCollectionEntry( } if (nestedSetups) { - entry.includesStates = createPerEntryIncludesStates(nestedSetups) + entry.includesStates = createPerEntryIncludesStates( + nestedSetups, + parentState, + correlationKey, + ) } return entry @@ -1872,172 +2099,213 @@ function flushIncludesState( parentChanges: Map> | null, parentSyncMethods: SyncMethods | null, ): void { - for (const state of includesState) { - // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent - if (parentChanges) { - for (const [parentKey, changes] of parentChanges) { - if (changes.inserts > 0) { - const parentResult = changes.value - // Extract routing info from INCLUDES_ROUTING symbol (set by compiler) - const routing = parentResult[INCLUDES_ROUTING]?.[state.fieldName] - const correlationKey = routing?.correlationKey - const parentContext = routing?.parentContext ?? null - const routingKey = computeRoutingKey(correlationKey, parentContext) + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`includes.flushState`, { + parentId, + stateCount: includesState.length, + }) + : undefined + let parentInsertCount = 0 + let parentDeleteCount = 0 + let childChangeGroups = 0 + let childChangeRows = 0 + let childCollectionsCreated = 0 + let nestedBufferDirtyCount = 0 + let recursiveFlushes = 0 + let inlineReEmitEvents = 0 + let childCollectionsDeleted = 0 + + try { + for (const state of includesState) { + // Phase 1: Parent INSERTs — ensure a child Collection exists for every parent + if (parentChanges) { + for (const [parentKey, changes] of parentChanges) { + if (changes.inserts > 0) { + parentInsertCount++ + const parentResult = changes.value + // Extract routing info from INCLUDES_ROUTING symbol (set by compiler) + const routing = parentResult[INCLUDES_ROUTING]?.[state.fieldName] + const correlationKey = routing?.correlationKey + const parentContext = routing?.parentContext ?? null + const routingKey = computeRoutingKey(correlationKey, parentContext) - if (correlationKey != null) { - // Ensure child Collection exists for this routing key - if (!state.childRegistry.has(routingKey)) { - const entry = createChildCollectionEntry( - parentId, - state.fieldName, - routingKey, - state.hasOrderBy, - state.nestedSetups, - ) - state.childRegistry.set(routingKey, entry) - } - // Update reverse index: routing key → parent keys - let parentKeys = state.correlationToParentKeys.get(routingKey) - if (!parentKeys) { - parentKeys = new Set() - state.correlationToParentKeys.set(routingKey, parentKeys) - } - parentKeys.add(parentKey) + if (correlationKey != null) { + // Ensure child Collection exists for this routing key + if (!state.childRegistry.has(routingKey)) { + const entry = createChildCollectionEntry( + parentId, + state.fieldName, + routingKey, + state.hasOrderBy, + state.nestedSetups, + state, + ) + state.childRegistry.set(routingKey, entry) + childCollectionsCreated++ + } + // Update reverse index: routing key → parent keys + let parentKeys = state.correlationToParentKeys.get(routingKey) + if (!parentKeys) { + parentKeys = new Set() + state.correlationToParentKeys.set(routingKey, parentKeys) + } + parentKeys.add(parentKey) - const childValue = materializeIncludedValue( - state, - state.childRegistry.get(routingKey), - ) - setIncludedValue(parentResult, state.resultPath, childValue) + const childValue = materializeIncludedValue( + state, + state.childRegistry.get(routingKey), + ) + setIncludedValue(parentResult, state.resultPath, childValue) - // Parent rows may already be materialized in the live collection by the - // time includes state is flushed, so update the stored row as well. - const storedParent = parentCollection.get(parentKey as any) - if (storedParent && storedParent !== parentResult) { - setIncludedValue(storedParent, state.resultPath, childValue) + // Parent rows may already be materialized in the live collection by the + // time includes state is flushed, so update the stored row as well. + const storedParent = parentCollection.get(parentKey as any) + if (storedParent && storedParent !== parentResult) { + setIncludedValue(storedParent, state.resultPath, childValue) + } } } } } - } - // Track affected correlation keys for inline materializations before clearing child changes. - const affectedCorrelationKeys = materializesInline(state) - ? new Set(state.pendingChildChanges.keys()) - : null - - // Phase 2: Child changes — apply to child Collections - // Track which entries had child changes and capture their childChanges maps - const entriesWithChildChanges = new Map< - unknown, - { entry: ChildCollectionEntry; childChanges: Map> } - >() - if (state.pendingChildChanges.size > 0) { - for (const [correlationKey, childChanges] of state.pendingChildChanges) { - // Ensure child Collection exists for this correlation key - let entry = state.childRegistry.get(correlationKey) - if (!entry) { - entry = createChildCollectionEntry( - parentId, - state.fieldName, - correlationKey, - state.hasOrderBy, - state.nestedSetups, - ) - state.childRegistry.set(correlationKey, entry) - } - - if (state.materialization === `collection`) { - attachChildCollectionToParent( - parentCollection, - state.resultPath, - correlationKey, - state.correlationToParentKeys, - entry.collection, - ) + // Track affected correlation keys for inline materializations before clearing child changes. + const affectedCorrelationKeys = materializesInline(state) + ? new Set(state.pendingChildChanges.keys()) + : null + + // Phase 2: Child changes — apply to child Collections + // Track which entries had child changes and capture their childChanges maps + const entriesWithChildChanges = new Map< + unknown, + { + entry: ChildCollectionEntry + childChanges: Map> } + >() + if (state.pendingChildChanges.size > 0) { + childChangeGroups += state.pendingChildChanges.size + for (const [ + correlationKey, + childChanges, + ] of state.pendingChildChanges) { + childChangeRows += childChanges.size + // Ensure child Collection exists for this correlation key + let entry = state.childRegistry.get(correlationKey) + if (!entry) { + entry = createChildCollectionEntry( + parentId, + state.fieldName, + correlationKey, + state.hasOrderBy, + state.nestedSetups, + state, + ) + state.childRegistry.set(correlationKey, entry) + childCollectionsCreated++ + } - // Apply child changes to the child Collection - if (entry.syncMethods) { - entry.syncMethods.begin() - for (const [childKey, change] of childChanges) { - entry.resultKeys.set(change.value, childKey) - if (entry.orderByIndices && change.orderByIndex !== undefined) { - entry.orderByIndices.set(change.value, change.orderByIndex) - } - const key = entry.syncMethods.collection.getKeyFromItem( - change.value, + if (state.materialization === `collection`) { + attachChildCollectionToParent( + parentCollection, + state.resultPath, + correlationKey, + state.correlationToParentKeys, + entry.collection, ) - const childAlreadyExists = entry.syncMethods.collection.has(key) + } - if (change.inserts > 0 && change.deletes === 0) { - entry.syncMethods.write({ - value: change.value, - type: childAlreadyExists ? `update` : `insert`, - }) - } else if ( - change.inserts > change.deletes || - (change.inserts === change.deletes && childAlreadyExists) - ) { - entry.syncMethods.write({ value: change.value, type: `update` }) - } else if (change.deletes > 0) { - entry.syncMethods.write({ value: change.value, type: `delete` }) + // Apply child changes to the child Collection + if (entry.syncMethods) { + entry.syncMethods.begin() + for (const [childKey, change] of childChanges) { + entry.resultKeys.set(change.value, childKey) + if (entry.orderByIndices && change.orderByIndex !== undefined) { + entry.orderByIndices.set(change.value, change.orderByIndex) + } + const key = entry.syncMethods.collection.getKeyFromItem( + change.value, + ) + const childAlreadyExists = entry.syncMethods.collection.has(key) + + if (change.inserts > 0 && change.deletes === 0) { + entry.syncMethods.write({ + value: change.value, + type: childAlreadyExists ? `update` : `insert`, + }) + } else if ( + change.inserts > change.deletes || + (change.inserts === change.deletes && childAlreadyExists) + ) { + entry.syncMethods.write({ value: change.value, type: `update` }) + } else if (change.deletes > 0) { + entry.syncMethods.write({ value: change.value, type: `delete` }) + } } + entry.syncMethods.commit() } - entry.syncMethods.commit() - } - // Update routing index for nested includes - updateRoutingIndex(state, correlationKey, childChanges) + // Update routing index for nested includes + updateRoutingIndex(state, correlationKey, childChanges) - entriesWithChildChanges.set(correlationKey, { entry, childChanges }) + entriesWithChildChanges.set(correlationKey, { entry, childChanges }) + } + state.pendingChildChanges.clear() } - state.pendingChildChanges.clear() - } - // Phase 3: Drain nested buffers — route buffered grandchild changes to per-entry states - const dirtyFromBuffers = drainNestedBuffers(state) + // Phase 3: Drain nested buffers — route buffered grandchild changes to per-entry states + const dirtyFromBuffers = drainNestedBuffers(state) + nestedBufferDirtyCount += dirtyFromBuffers.size - // Phase 4: Flush per-entry states - // First: entries that had child changes in Phase 2 - for (const [, { entry, childChanges }] of entriesWithChildChanges) { - if (entry.includesStates) { - flushIncludesState( - entry.includesStates, - entry.collection, - entry.collection.id, - childChanges, - entry.syncMethods, - ) - } - } - // Then: entries that only had buffer-routed changes (no child changes at this level) - for (const correlationKey of dirtyFromBuffers) { - if (entriesWithChildChanges.has(correlationKey)) continue - const entry = state.childRegistry.get(correlationKey) - if (entry?.includesStates) { - flushIncludesState( - entry.includesStates, - entry.collection, - entry.collection.id, - null, - entry.syncMethods, - ) + // Phase 4: Flush per-entry states + // First: entries that had child changes in Phase 2 + for (const [, { entry, childChanges }] of entriesWithChildChanges) { + if (entry.includesStates) { + recursiveFlushes++ + flushIncludesState( + entry.includesStates, + entry.collection, + entry.collection.id, + childChanges, + entry.syncMethods, + ) + } } - } - // Finally: entries with deep nested buffer changes (grandchild-or-deeper buffers - // have pending data, but neither this level nor the immediate child level changed). - // Without this pass, changes at depth 3+ are stranded because drainNestedBuffers - // only drains one level and Phase 4 only flushes entries dirty from Phase 2/3. - const deepBufferDirty = new Set() - if (state.nestedSetups) { - for (const [correlationKey, entry] of state.childRegistry) { + // Then: entries that only had buffer-routed changes (no child changes at this level) + for (const correlationKey of dirtyFromBuffers) { if (entriesWithChildChanges.has(correlationKey)) continue - if (dirtyFromBuffers.has(correlationKey)) continue + const entry = state.childRegistry.get(correlationKey) + if (entry?.includesStates) { + recursiveFlushes++ + flushIncludesState( + entry.includesStates, + entry.collection, + entry.collection.id, + null, + entry.syncMethods, + ) + } + } + // Finally: entries with deep nested buffer changes (grandchild-or-deeper buffers + // have pending data, but neither this level nor the immediate child level changed). + // Dirty entries are marked when nested buffers route into per-entry state, so this + // remains proportional to changed routes instead of scanning every child entry. + const deepBufferDirty = new Set() + for (const correlationKey of state.dirtyChildEntries) { + if (entriesWithChildChanges.has(correlationKey)) { + state.dirtyChildEntries.delete(correlationKey) + continue + } + if (dirtyFromBuffers.has(correlationKey)) { + state.dirtyChildEntries.delete(correlationKey) + continue + } + const entry = state.childRegistry.get(correlationKey) if ( - entry.includesStates && + entry?.includesStates && hasPendingIncludesChanges(entry.includesStates) ) { + recursiveFlushes++ flushIncludesState( entry.includesStates, entry.collection, @@ -2047,85 +2315,131 @@ function flushIncludesState( ) deepBufferDirty.add(correlationKey) } + state.dirtyChildEntries.delete(correlationKey) } - } - // For inline materializations: re-emit affected parents with updated snapshots. - // We mutate items in-place (so collection.get() reflects changes immediately) - // and emit UPDATE events directly. We bypass the sync methods because - // commitPendingTransactions compares previous vs new visible state using - // deepEquals, but in-place mutation means both sides reference the same - // object, so the comparison always returns true and suppresses the event. - const inlineReEmitKeys = materializesInline(state) - ? new Set([ - ...(affectedCorrelationKeys || []), - ...dirtyFromBuffers, - ...deepBufferDirty, - ]) - : null - if (parentSyncMethods && inlineReEmitKeys && inlineReEmitKeys.size > 0) { - const events: Array> = [] - for (const correlationKey of inlineReEmitKeys) { - const parentKeys = state.correlationToParentKeys.get(correlationKey) - if (!parentKeys) continue - const entry = state.childRegistry.get(correlationKey) - for (const parentKey of parentKeys) { - const item = parentCollection.get(parentKey as any) - if (item) { - // Capture previous value before in-place mutation - const previousValue = cloneForIncludesUpdate(item, state.resultPath) - setIncludedValue( - item, - state.resultPath, - materializeIncludedValue(state, entry), - ) - const nextValue = cloneForIncludesUpdate(item, state.resultPath) - events.push({ - type: `update`, - key: parentKey as any, - value: nextValue, - previousValue, - }) + // For inline materializations: re-emit affected parents with updated snapshots. + // We mutate items in-place (so collection.get() reflects changes immediately) + // and emit UPDATE events directly. We bypass the sync methods because + // commitPendingTransactions compares previous vs new visible state using + // deepEquals, but in-place mutation means both sides reference the same + // object, so the comparison always returns true and suppresses the event. + const inlineReEmitKeys = materializesInline(state) + ? new Set([ + ...(affectedCorrelationKeys || []), + ...dirtyFromBuffers, + ...deepBufferDirty, + ]) + : null + if (parentSyncMethods && inlineReEmitKeys && inlineReEmitKeys.size > 0) { + const events: Array> = [] + for (const correlationKey of inlineReEmitKeys) { + const parentKeys = state.correlationToParentKeys.get(correlationKey) + if (!parentKeys) continue + const entry = state.childRegistry.get(correlationKey) + for (const parentKey of parentKeys) { + const item = parentCollection.get(parentKey as any) + if (item) { + // Capture previous value before in-place mutation + const previousValue = cloneForIncludesUpdate( + item, + state.resultPath, + ) + setIncludedValue( + item, + state.resultPath, + materializeIncludedValue(state, entry), + ) + const nextValue = cloneForIncludesUpdate(item, state.resultPath) + events.push({ + type: `update`, + key: parentKey as any, + value: nextValue, + previousValue, + }) + inlineReEmitEvents++ + } } } - } - if (events.length > 0) { - // Emit directly — the in-place mutation already updated the data in - // syncedData, so we only need to notify subscribers. - const changesManager = (parentCollection as any)._changes as { - emitEvents: ( - changes: Array>, - forceEmit?: boolean, - ) => void + if (events.length > 0) { + // Emit directly — the in-place mutation already updated the data in + // syncedData, so we only need to notify subscribers. + const changesManager = (parentCollection as any)._changes as { + emitEvents: ( + changes: Array>, + forceEmit?: boolean, + ) => void + } + changesManager.emitEvents(events, true) } - changesManager.emitEvents(events, true) } - } - // Phase 5: Parent DELETEs — dispose child Collections and clean up - if (parentChanges) { - for (const [parentKey, changes] of parentChanges) { - if (changes.deletes > 0 && changes.inserts === 0) { - const routing = changes.value[INCLUDES_ROUTING]?.[state.fieldName] - const correlationKey = routing?.correlationKey - const parentContext = routing?.parentContext ?? null - const routingKey = computeRoutingKey(correlationKey, parentContext) - if (correlationKey != null) { - // Clean up reverse index first, only delete child collection - // when the last parent referencing it is removed - const parentKeys = state.correlationToParentKeys.get(routingKey) - if (parentKeys) { - parentKeys.delete(parentKey) - if (parentKeys.size === 0) { - cleanRoutingIndexOnDelete(state, routingKey) - state.childRegistry.delete(routingKey) - state.correlationToParentKeys.delete(routingKey) + // Phase 5: Parent DELETEs — dispose child Collections and clean up + if (parentChanges) { + for (const [parentKey, changes] of parentChanges) { + if (changes.deletes > 0 && changes.inserts === 0) { + parentDeleteCount++ + const routing = changes.value[INCLUDES_ROUTING]?.[state.fieldName] + const correlationKey = routing?.correlationKey + const parentContext = routing?.parentContext ?? null + const routingKey = computeRoutingKey(correlationKey, parentContext) + if (correlationKey != null) { + // Clean up reverse index first, only delete child collection + // when the last parent referencing it is removed + const parentKeys = state.correlationToParentKeys.get(routingKey) + if (parentKeys) { + parentKeys.delete(parentKey) + if (parentKeys.size === 0) { + cleanRoutingIndexOnDelete(state, routingKey) + state.childRegistry.delete(routingKey) + state.correlationToParentKeys.delete(routingKey) + childCollectionsDeleted++ + } } } } } } } + } finally { + if (shouldTrace) { + recordPerfCount(`includes.flush.parentInserts`, parentInsertCount, { + parentId, + }) + recordPerfCount(`includes.flush.parentDeletes`, parentDeleteCount, { + parentId, + }) + recordPerfCount(`includes.flush.childChangeGroups`, childChangeGroups, { + parentId, + }) + recordPerfCount(`includes.flush.childChangeRows`, childChangeRows, { + parentId, + }) + recordPerfCount( + `includes.flush.childCollectionsCreated`, + childCollectionsCreated, + { parentId }, + ) + recordPerfCount( + `includes.flush.childCollectionsDeleted`, + childCollectionsDeleted, + { parentId }, + ) + recordPerfCount( + `includes.flush.nestedBufferDirty`, + nestedBufferDirtyCount, + { + parentId, + }, + ) + recordPerfCount(`includes.flush.recursiveFlushes`, recursiveFlushes, { + parentId, + }) + recordPerfCount(`includes.flush.inlineReEmitEvents`, inlineReEmitEvents, { + parentId, + }) + span?.end() + } } // Clean up the internal routing stamp from parent/child results @@ -2147,13 +2461,7 @@ function hasPendingIncludesChanges( if (state.pendingChildChanges.size > 0) return true if (state.nestedSetups && hasNestedBufferChanges(state.nestedSetups)) return true - for (const entry of state.childRegistry.values()) { - if ( - entry.includesStates && - hasPendingIncludesChanges(entry.includesStates) - ) - return true - } + if (state.dirtyChildEntries.size > 0) return true } return false } diff --git a/packages/db/src/query/live/collection-subscriber.ts b/packages/db/src/query/live/collection-subscriber.ts index 69c8220ad3..d5c6eae0a8 100644 --- a/packages/db/src/query/live/collection-subscriber.ts +++ b/packages/db/src/query/live/collection-subscriber.ts @@ -2,6 +2,7 @@ import { normalizeExpressionPaths, normalizeOrderByPaths, } from '../compiler/expressions.js' +import { isPerfEnabled, recordPerfCount, startPerfSpan } from './perf.js' import { computeOrderedLoadCursor, computeSubscriptionOrderByHints, @@ -153,6 +154,13 @@ export class CollectionSubscriber< changes: Iterable>, callback?: () => boolean, ) { + const shouldTrace = isPerfEnabled() + const span = shouldTrace + ? startPerfSpan(`collectionSubscriber.sendChangesToPipeline`, { + alias: this.alias, + collectionId: this.collectionId, + }) + : undefined const changesArray = Array.isArray(changes) ? changes : [...changes] const filteredChanges = filterDuplicateInserts( changesArray, @@ -164,6 +172,23 @@ export class CollectionSubscriber< const input = this.collectionConfigBuilder.currentSyncState!.inputs[this.alias]! const sentChanges = sendChangesToInput(input, filteredChanges) + if (shouldTrace) { + const tags = { + alias: this.alias, + collectionId: this.collectionId, + } + recordPerfCount( + `collectionSubscriber.sourceChanges`, + changesArray.length, + tags, + ) + recordPerfCount( + `collectionSubscriber.filteredChanges`, + filteredChanges.length, + tags, + ) + recordPerfCount(`collectionSubscriber.sentChanges`, sentChanges, tags) + } // Do not provide the callback that loads more data // if there's no more data to load @@ -176,6 +201,7 @@ export class CollectionSubscriber< this.collectionConfigBuilder.scheduleGraphRun(dataLoader, { alias: this.alias, }) + span?.end({ sentChanges }) } private subscribeToMatchingChanges( @@ -255,8 +281,12 @@ export class CollectionSubscriber< this.trackSentValues(changesArray, orderByInfo.comparator) - // Split live updates into a delete of the old value and an insert of the new value - const splittedChanges = splitUpdates(changesArray) + // Split live updates into a delete of the old value and an insert of the + // new value. Insert/delete-only batches are already in the shape the + // pipeline needs, so keep the original array and avoid generator + // materialization in the hot ordered-change path. + const hasUpdates = changesArray.some((change) => change.type === `update`) + const splittedChanges = hasUpdates ? splitUpdates(changesArray) : changesArray this.sendChangesToPipelineWithTracking( splittedChanges, subscriptionHolder.current!, diff --git a/packages/db/src/query/live/perf.ts b/packages/db/src/query/live/perf.ts new file mode 100644 index 0000000000..f128ab2c8c --- /dev/null +++ b/packages/db/src/query/live/perf.ts @@ -0,0 +1,215 @@ +type PerfTagValue = string | number | boolean | undefined +type PerfTags = Record + +type PerfMetricKind = `span` | `counter` | `gauge` + +interface PerfMetric { + name: string + kind: PerfMetricKind + tags: PerfTags + calls: number + total: number + max: number + min: number + last: number +} + +interface PerfState { + enabled: boolean + metrics: Map +} + +export interface PerfMetricSnapshot { + name: string + kind: PerfMetricKind + tags: PerfTags + calls: number + total: number + max: number + min: number + last: number +} + +export interface PerfTraceSnapshot { + enabled: boolean + metrics: Array +} + +export interface PerfSpanHandle { + end: (extraTags?: PerfTags) => void +} + +const perfStateSymbol = Symbol.for(`@tanstack/db.perfState`) + +type PerfGlobal = typeof globalThis & { + __TANSTACK_DB_TRACE__?: boolean + [perfStateSymbol]?: PerfState +} + +function getPerfState(): PerfState { + const global = globalThis as PerfGlobal + global[perfStateSymbol] ??= { + enabled: readDefaultEnabled(global), + metrics: new Map(), + } + return global[perfStateSymbol] +} + +function readDefaultEnabled(global: PerfGlobal): boolean { + const env = ( + global as { process?: { env?: Record } } + ).process?.env + + return ( + global.__TANSTACK_DB_TRACE__ === true || + env?.TANSTACK_DB_TRACE === `1` || + env?.TANSTACK_DB_TRACE === `true` + ) +} + +function now(): number { + return globalThis.performance.now() +} + +function metricKey(name: string, kind: PerfMetricKind, tags: PerfTags): string { + const tagParts = Object.keys(tags) + .sort() + .map((key) => `${key}:${String(tags[key])}`) + .join(`,`) + return `${kind}:${name}:${tagParts}` +} + +function mergedTags(tags: PerfTags, extraTags?: PerfTags): PerfTags { + return extraTags ? { ...tags, ...extraTags } : tags +} + +function recordMetric( + kind: PerfMetricKind, + name: string, + value: number, + tags: PerfTags = {}, +): void { + const state = getPerfState() + if (!state.enabled) return + + const key = metricKey(name, kind, tags) + let metric = state.metrics.get(key) + if (!metric) { + metric = { + name, + kind, + tags: { ...tags }, + calls: 0, + total: 0, + max: Number.NEGATIVE_INFINITY, + min: Number.POSITIVE_INFINITY, + last: 0, + } + state.metrics.set(key, metric) + } + + metric.calls++ + metric.total += value + metric.max = Math.max(metric.max, value) + metric.min = Math.min(metric.min, value) + metric.last = value +} + +export function isPerfEnabled(): boolean { + return getPerfState().enabled +} + +export function setPerfTracingEnabled(enabled: boolean): void { + getPerfState().enabled = enabled +} + +export function resetPerfTrace(): void { + getPerfState().metrics.clear() +} + +export function getPerfTraceSnapshot(): PerfTraceSnapshot { + const state = getPerfState() + const metrics = Array.from(state.metrics.values()) + .map((metric) => ({ + ...metric, + tags: { ...metric.tags }, + min: metric.min === Number.POSITIVE_INFINITY ? 0 : metric.min, + max: metric.max === Number.NEGATIVE_INFINITY ? 0 : metric.max, + })) + .sort((a, b) => b.total - a.total || b.calls - a.calls) + + return { + enabled: state.enabled, + metrics, + } +} + +export function recordPerfCount( + name: string, + value = 1, + tags?: PerfTags, +): void { + recordMetric(`counter`, name, value, tags) +} + +export function recordPerfGauge( + name: string, + value: number, + tags?: PerfTags, +): void { + recordMetric(`gauge`, name, value, tags) +} + +export function recordPerfSpan( + name: string, + durationMs: number, + tags?: PerfTags, +): void { + recordMetric(`span`, name, durationMs, tags) +} + +export function startPerfSpan( + name: string, + tags: PerfTags = {}, +): PerfSpanHandle { + if (!isPerfEnabled()) { + return { end: () => {} } + } + + const start = now() + return { + end: (extraTags?: PerfTags) => { + recordPerfSpan(name, now() - start, mergedTags(tags, extraTags)) + }, + } +} + +export function withPerfSpan(name: string, tags: PerfTags, fn: () => T): T { + if (!isPerfEnabled()) { + return fn() + } + + const span = startPerfSpan(name, tags) + try { + return fn() + } finally { + span.end() + } +} + +export async function withPerfSpanAsync( + name: string, + tags: PerfTags, + fn: () => Promise, +): Promise { + if (!isPerfEnabled()) { + return fn() + } + + const span = startPerfSpan(name, tags) + try { + return await fn() + } finally { + span.end() + } +} diff --git a/packages/db/src/query/live/utils.ts b/packages/db/src/query/live/utils.ts index c7f701124f..fe8927817e 100644 --- a/packages/db/src/query/live/utils.ts +++ b/packages/db/src/query/live/utils.ts @@ -331,19 +331,23 @@ export function filterDuplicateInserts( changes: Array>, sentKeys: Set, ): Array> { - const filtered: Array> = [] - for (const change of changes) { + let filtered: Array> | undefined + + for (let i = 0; i < changes.length; i++) { + const change = changes[i]! if (change.type === `insert`) { if (sentKeys.has(change.key)) { + filtered ??= changes.slice(0, i) continue // Skip duplicate } sentKeys.add(change.key) } else if (change.type === `delete`) { sentKeys.delete(change.key) } - filtered.push(change) + filtered?.push(change) } - return filtered + + return filtered ?? changes } /** diff --git a/packages/db/src/scheduler.ts b/packages/db/src/scheduler.ts index 8ffe8b768c..0357b0a91a 100644 --- a/packages/db/src/scheduler.ts +++ b/packages/db/src/scheduler.ts @@ -1,3 +1,9 @@ +import { + isPerfEnabled, + recordPerfCount, + startPerfSpan, +} from './query/live/perf.js' + /** * Identifier used to scope scheduled work. Maps to a transaction id for live queries. */ @@ -76,14 +82,35 @@ export class Scheduler { */ schedule({ contextId, jobId, dependencies, run }: ScheduleOptions): void { if (typeof contextId === `undefined`) { - run() + if (!isPerfEnabled()) { + run() + return + } + + recordPerfCount(`scheduler.schedule`, 1, { + context: false, + deduped: false, + }) + const span = startPerfSpan(`scheduler.schedule.immediate`) + try { + run() + } finally { + span.end() + } return } const context = this.getOrCreateContext(contextId) + const isNewJob = !context.jobs.has(jobId) + if (isPerfEnabled()) { + recordPerfCount(`scheduler.schedule`, 1, { + context: true, + deduped: !isNewJob, + }) + } // If this is a new job, add it to the queue - if (!context.jobs.has(jobId)) { + if (isNewJob) { context.queue.push(jobId) } @@ -112,67 +139,87 @@ export class Scheduler { if (!context) return const { queue, jobs, dependencies, completed } = context + const shouldTrace = isPerfEnabled() + const span = shouldTrace ? startPerfSpan(`scheduler.flush`) : undefined + let passes = 0 + let jobsRun = 0 + let blockedDependencies = 0 - while (queue.length > 0) { - let ranThisPass = false - const jobsThisPass = queue.length - - for (let i = 0; i < jobsThisPass; i++) { - const jobId = queue.shift()! - const run = jobs.get(jobId) - if (!run) { - dependencies.delete(jobId) - completed.delete(jobId) - continue - } + try { + while (queue.length > 0) { + passes++ + let ranThisPass = false + const jobsThisPass = queue.length + + for (let i = 0; i < jobsThisPass; i++) { + const jobId = queue.shift()! + const run = jobs.get(jobId) + if (!run) { + dependencies.delete(jobId) + completed.delete(jobId) + continue + } - const deps = dependencies.get(jobId) - let ready = !deps - if (deps) { - ready = true - for (const dep of deps) { - if (dep === jobId) continue - - const depHasPending = - isPendingAwareJob(dep) && dep.hasPendingGraphRun(contextId) - - // Treat dependencies as blocking if the dep has a pending run in this - // context or if it's enqueued and not yet complete. If the dep is - // neither pending nor enqueued, consider it satisfied to avoid deadlocks - // on lazy sources that never schedule work. - if ( - (jobs.has(dep) && !completed.has(dep)) || - (!jobs.has(dep) && depHasPending) - ) { - ready = false - break + const deps = dependencies.get(jobId) + let ready = !deps + if (deps) { + ready = true + for (const dep of deps) { + if (dep === jobId) continue + + const depHasPending = + isPendingAwareJob(dep) && dep.hasPendingGraphRun(contextId) + + // Treat dependencies as blocking if the dep has a pending run in this + // context or if it's enqueued and not yet complete. If the dep is + // neither pending nor enqueued, consider it satisfied to avoid deadlocks + // on lazy sources that never schedule work. + if ( + (jobs.has(dep) && !completed.has(dep)) || + (!jobs.has(dep) && depHasPending) + ) { + ready = false + blockedDependencies++ + break + } } } + + if (ready) { + jobs.delete(jobId) + dependencies.delete(jobId) + // Run the job. If it throws, we don't mark it complete, allowing the + // error to propagate while maintaining scheduler state consistency. + run() + completed.add(jobId) + ranThisPass = true + jobsRun++ + } else { + queue.push(jobId) + } } - if (ready) { - jobs.delete(jobId) - dependencies.delete(jobId) - // Run the job. If it throws, we don't mark it complete, allowing the - // error to propagate while maintaining scheduler state consistency. - run() - completed.add(jobId) - ranThisPass = true - } else { - queue.push(jobId) + if (!ranThisPass) { + throw new Error( + `Scheduler detected unresolved dependencies for context ${String( + contextId, + )}.`, + ) } } - if (!ranThisPass) { - throw new Error( - `Scheduler detected unresolved dependencies for context ${String( - contextId, - )}.`, + this.contexts.delete(contextId) + } finally { + if (shouldTrace) { + recordPerfCount(`scheduler.flush.passes`, passes) + recordPerfCount(`scheduler.flush.jobsRun`, jobsRun) + recordPerfCount( + `scheduler.flush.blockedDependencies`, + blockedDependencies, ) + span?.end({ jobsRun }) } } - - this.contexts.delete(contextId) } /** diff --git a/packages/db/src/transactions.ts b/packages/db/src/transactions.ts index fe2f61c0fd..84cbbcfe58 100644 --- a/packages/db/src/transactions.ts +++ b/packages/db/src/transactions.ts @@ -334,27 +334,39 @@ class Transaction> { * @param mutations - Array of new mutations to apply */ applyMutations(mutations: Array>): void { + // Merge via a globalKey-keyed map rather than a findIndex scan per + // mutation, which is O(n²) for bulk operations (e.g. inserting many rows + // in one call). Map preserves insertion order, matching the previous + // replace-in-place / remove / append semantics. + const merged = new Map>() + for (const mutation of this.mutations) { + merged.set(mutation.globalKey, mutation) + } + for (const newMutation of mutations) { - const existingIndex = this.mutations.findIndex( - (m) => m.globalKey === newMutation.globalKey, - ) + const existingMutation = merged.get(newMutation.globalKey) - if (existingIndex >= 0) { - const existingMutation = this.mutations[existingIndex]! + if (existingMutation) { const mergeResult = mergePendingMutations(existingMutation, newMutation) if (mergeResult === null) { // Remove the mutation (e.g., delete after insert cancels both) - this.mutations.splice(existingIndex, 1) + merged.delete(newMutation.globalKey) } else { // Replace with merged mutation - this.mutations[existingIndex] = mergeResult + merged.set(newMutation.globalKey, mergeResult) } } else { // Insert new mutation - this.mutations.push(newMutation) + merged.set(newMutation.globalKey, newMutation) } } + + // Rebuild in place to preserve the array's identity for external holders + this.mutations.length = 0 + for (const mutation of merged.values()) { + this.mutations.push(mutation) + } } /** diff --git a/packages/db/tests/SortedMap.test.ts b/packages/db/tests/SortedMap.test.ts index e2a46da61b..b85dd07b46 100644 --- a/packages/db/tests/SortedMap.test.ts +++ b/packages/db/tests/SortedMap.test.ts @@ -35,6 +35,62 @@ describe(`SortedMap`, () => { expect(values).toEqual([2, 3]) }) + it(`keeps key order when updating values without a comparator`, () => { + const map = new SortedMap() + map.set(`a`, 1) + map.set(`b`, 2) + map.set(`c`, 3) + + map.set(`b`, 20) + + expect(Array.from(map.entries())).toEqual([ + [`a`, 1], + [`b`, 20], + [`c`, 3], + ]) + }) + + it(`sets entries with known presence`, () => { + const map = new SortedMap() + map.set(`a`, 1) + map.set(`c`, 3) + + map.setWithKnownPresence(`b`, 2, false, undefined) + map.setWithKnownPresence(`c`, 30, true, 3) + + expect(Array.from(map.entries())).toEqual([ + [`a`, 1], + [`b`, 2], + [`c`, 30], + ]) + }) + + it(`sets entries with known presence and custom comparator`, () => { + const map = new SortedMap((a, b) => b - a) + map.set(`a`, 1) + map.set(`b`, 2) + + map.setWithKnownPresence(`a`, 3, true, 1) + + expect(Array.from(map.entries())).toEqual([ + [`a`, 3], + [`b`, 2], + ]) + }) + + it(`sets entries with known presence when the previous value is undefined`, () => { + const map = new SortedMap() + map.set(`a`, undefined) + map.set(`c`, 3) + + map.setWithKnownPresence(`a`, 1, true, undefined) + + expect(Array.from(map.entries())).toEqual([ + [`a`, 1], + [`c`, 3], + ]) + }) + it(`correctly handles deletions`, () => { const map = new SortedMap() map.set(`a`, 1) @@ -47,6 +103,36 @@ describe(`SortedMap`, () => { expect(values).toEqual([1, 3]) }) + it(`handles edge inserts and deletes without a comparator`, () => { + const map = new SortedMap() + map.set(20, `twenty`) + map.set(30, `thirty`) + map.set(10, `ten`) + map.set(40, `forty`) + + expect(Array.from(map.keys())).toEqual([10, 20, 30, 40]) + expect(map.delete(40)).toBe(true) + expect(map.delete(10)).toBe(true) + expect(Array.from(map.entries())).toEqual([ + [20, `twenty`], + [30, `thirty`], + ]) + }) + + it(`deletes entries with a known previous value`, () => { + const map = new SortedMap() + map.set(10, `ten`) + map.set(20, `twenty`) + map.set(30, `thirty`) + + expect(map.deleteWithKnownPreviousValue(30, true, `thirty`)).toBe(true) + expect(map.deleteWithKnownPreviousValue(40, false, undefined)).toBe(false) + expect(Array.from(map.entries())).toEqual([ + [10, `ten`], + [20, `twenty`], + ]) + }) + it(`implements iteration methods correctly`, () => { const map = new SortedMap() map.set(`b`, 2) diff --git a/packages/db/tests/collection-subscription.test.ts b/packages/db/tests/collection-subscription.test.ts index 65465a6a8b..0514572dd4 100644 --- a/packages/db/tests/collection-subscription.test.ts +++ b/packages/db/tests/collection-subscription.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../src/collection/index.js' -import { flushPromises } from './utils' +import { flushPromises, mockSyncCollectionOptions } from './utils' describe(`CollectionSubscription status tracking`, () => { it(`subscription starts with status 'ready'`, () => { @@ -274,3 +274,28 @@ describe(`CollectionSubscription status tracking`, () => { expect(eventCount).toBe(0) }) }) + +describe(`CollectionSubscription event filtering`, () => { + it(`does not emit when a non-empty batch filters to an empty delta`, () => { + type Row = { id: string; value: string } + const initialRow: Row = { id: `row-1`, value: `one` } + const options = mockSyncCollectionOptions({ + id: `filtered-empty-delta-test`, + initialData: [initialRow], + getKey: (item) => item.id, + }) + const collection = createCollection(options) + const batches: Array> = [] + + const subscription = collection.subscribeChanges((changes) => { + batches.push(changes) + }) + + options.utils.begin() + options.utils.write({ type: `delete`, value: initialRow }) + options.utils.commit() + + expect(batches).toHaveLength(0) + subscription.unsubscribe() + }) +}) diff --git a/packages/db/tests/query/perf/trace-aggregation.test.ts b/packages/db/tests/query/perf/trace-aggregation.test.ts new file mode 100644 index 0000000000..9d0c8f4a7a --- /dev/null +++ b/packages/db/tests/query/perf/trace-aggregation.test.ts @@ -0,0 +1,84 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { + getPerfTraceSnapshot, + recordPerfCount, + resetPerfTrace, + setPerfTracingEnabled, + withPerfSpanAsync, +} from '../../../src/query/live/perf.js' +import { recordPerfCount as recordIvmPerfCount } from '../../../../db-ivm/src/perf.js' + +describe(`perf trace aggregation`, () => { + beforeEach(() => { + resetPerfTrace() + setPerfTracingEnabled(false) + }) + + afterEach(() => { + resetPerfTrace() + setPerfTracingEnabled(false) + }) + + it(`does not aggregate metrics while disabled`, () => { + recordPerfCount(`disabled.counter`) + + expect(getPerfTraceSnapshot().metrics).toEqual([]) + }) + + it(`aggregates repeated metrics by name, kind, and tags`, () => { + setPerfTracingEnabled(true) + + recordPerfCount(`example.counter`, 2, { path: `a` }) + recordPerfCount(`example.counter`, 3, { path: `a` }) + + const metric = getPerfTraceSnapshot().metrics.find( + (entry) => entry.name === `example.counter`, + ) + + expect(metric).toMatchObject({ + kind: `counter`, + calls: 2, + total: 5, + min: 2, + max: 3, + last: 3, + tags: { path: `a` }, + }) + }) + + it(`completes async spans`, async () => { + setPerfTracingEnabled(true) + + const result = await withPerfSpanAsync(`example.async`, {}, async () => { + return 42 + }) + + const metric = getPerfTraceSnapshot().metrics.find( + (entry) => entry.name === `example.async`, + ) + + expect(result).toBe(42) + expect(metric).toMatchObject({ + kind: `span`, + calls: 1, + }) + expect(metric?.total).toBeGreaterThanOrEqual(0) + }) + + it(`shares one sink with db-ivm instrumentation`, () => { + setPerfTracingEnabled(true) + + recordIvmPerfCount(`ivm.counter`, 4, { package: `db-ivm` }) + + const metric = getPerfTraceSnapshot().metrics.find( + (entry) => entry.name === `ivm.counter`, + ) + + expect(metric).toMatchObject({ + kind: `counter`, + calls: 1, + total: 4, + tags: { package: `db-ivm` }, + }) + }) +}) diff --git a/scripts/bench/incremental-update.ts b/scripts/bench/incremental-update.ts new file mode 100644 index 0000000000..0827636e04 --- /dev/null +++ b/scripts/bench/incremental-update.ts @@ -0,0 +1,1121 @@ +import { execSync } from 'node:child_process' +import { mkdirSync, writeFileSync } from 'node:fs' +import { cpus, platform, release } from 'node:os' +import { join } from 'node:path' +import { + BTreeIndex, + count, + createCollection, + createLiveQueryCollection, + createTransaction, + eq, + materialize, +} from '../../packages/db/src/index.js' +import { + getPerfTraceSnapshot, + resetPerfTrace, + setPerfTracingEnabled, +} from '../../packages/db/src/query/live/perf.js' +import type { + ChangeMessageOrDeleteKeyMessage, + Collection, + SyncConfig, +} from '../../packages/db/src/index.js' +import type { PerfTraceSnapshot } from '../../packages/db/src/query/live/perf.js' + +type Issue = { + id: number + status: `open` | `closed` + authorId: number + createdAt: number + title: string + body: string + updateTick: number +} + +type User = { + id: number + name: string + reputation: number +} + +type Comment = { + id: number + issueId: number + authorId: number + createdAt: number + body: string + updateTick: number +} + +type ManualSyncUtils = { + begin: Parameters[`sync`]>[0][`begin`] + write: (message: ChangeMessageOrDeleteKeyMessage) => void + commit: Parameters[`sync`]>[0][`commit`] +} + +type ManualCollection< + T extends object, + TKey extends string | number, +> = Collection, never, T> + +type BenchmarkCollection = Collection, string | number> + +type Fixture = { + seed: number + issues: ManualCollection + users: ManualCollection + comments: ManualCollection + issueById: Map + userById: Map + commentById: Map + visibleIssueId: number + selectedIssueId: number + nextCommentId: number +} + +type BenchmarkOptions = { + seed: number + levels: Array + sourceIndexModes: Array + mutationModes: Array + issueCount?: number + userCount?: number + commentCount?: number + warmup: number + iterations: number + outDir: string +} + +type FixtureScale = { + label: string + issueCount: number + userCount: number + commentCount: number +} + +type BenchmarkRunOptions = { + seed: number + scale: FixtureScale + issueCount: number + userCount: number + commentCount: number + sourceIndexMode: SourceIndexMode + warmup: number + iterations: number +} + +type Summary = { + iterations: number + medianMs: number + p75Ms: number + p95Ms: number + minMs: number + maxMs: number + stddevMs: number +} + +type RunResult = { + query: string + scenario: string + scale: FixtureScale + sourceIndexMode: SourceIndexMode + mutationMode: MutationMode + tracing: boolean + coldHydrateMs: number + writeSummary: Summary + trace: PerfTraceSnapshot +} + +type MutationMode = `synced` | `optimistic` +type SourceIndexMode = `none` | `manual` | `auto` + +type WriteCleanup = () => void | Promise + +type WriteResult = void | { cleanup: WriteCleanup } + +type QueryCase = { + name: string + scenario: string + createQuery: (fixture: Fixture) => BenchmarkCollection + write: ( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, + ) => WriteResult +} + +const defaultOptions: BenchmarkOptions = { + seed: 42, + levels: [100, 1_000, 10_000], + sourceIndexModes: [`none`, `manual`], + mutationModes: [`synced`, `optimistic`], + warmup: 10, + iterations: 50, + outDir: `.tmp/perf`, +} + +const defaultCustomScale: FixtureScale = { + label: `custom`, + issueCount: 10_000, + userCount: 1_000, + commentCount: 40_000, +} + +const options = parseArgs(process.argv.slice(2), defaultOptions) + +const queryCases: Array = [ + { + name: `list: newest 50 open`, + scenario: `visible issue update`, + createQuery: ({ issues }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + createdAt: issue.createdAt, + })), + ) as unknown as BenchmarkCollection, + write: updateVisibleIssue, + }, + { + name: `list + author`, + scenario: `visible author update`, + createQuery: ({ issues, users }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .join({ author: users }, ({ issue, author }) => + eq(issue.authorId, author.id), + ) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue, author }) => ({ + id: issue.id, + title: issue.title, + createdAt: issue.createdAt, + authorName: author.name, + })), + ) as unknown as BenchmarkCollection, + write: updateVisibleAuthor, + }, + { + name: `list + comment count`, + scenario: `visible issue comment insert`, + createQuery: ({ issues, comments }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + commentCount: materialize( + q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .select(({ comment }) => count(comment.id)) + .findOne(), + ), + })), + ) as unknown as BenchmarkCollection, + write: insertVisibleComment, + }, + { + name: `list + 3 recent comments`, + scenario: `visible issue comment insert`, + createQuery: ({ issues, comments }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.status, `open`)) + .orderBy(({ issue }) => issue.createdAt, `desc`) + .limit(50) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + recentComments: q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .orderBy(({ comment }) => comment.createdAt, `desc`) + .limit(3) + .select(({ comment }) => ({ + id: comment.id, + body: comment.body, + createdAt: comment.createdAt, + })), + })), + ) as unknown as BenchmarkCollection, + write: insertVisibleComment, + }, + { + name: `issue detail + comments`, + scenario: `selected issue comment insert`, + createQuery: ({ issues, comments, selectedIssueId }) => + createLiveQueryCollection((q) => + q + .from({ issue: issues }) + .where(({ issue }) => eq(issue.id, selectedIssueId)) + .select(({ issue }) => ({ + id: issue.id, + title: issue.title, + comments: q + .from({ comment: comments }) + .where(({ comment }) => eq(comment.issueId, issue.id)) + .orderBy(({ comment }) => comment.createdAt, `desc`) + .select(({ comment }) => ({ + id: comment.id, + body: comment.body, + createdAt: comment.createdAt, + })), + })), + ) as unknown as BenchmarkCollection, + write: insertSelectedIssueComment, + }, +] + +const scales = resolveScales(options) +const results: Array = [] +for (const scale of scales) { + for (const sourceIndexMode of options.sourceIndexModes) { + const runOptions = createRunOptions(options, scale, sourceIndexMode) + for (const mutationMode of options.mutationModes) { + for (const queryCase of queryCases) { + const disabled = await runCase( + queryCase, + mutationMode, + false, + runOptions, + ) + const enabled = await runCase(queryCase, mutationMode, true, runOptions) + results.push(disabled, enabled) + } + } + } +} + +const report = { + metadata: runtimeMetadata(options, scales), + results, + overhead: computeOverhead(results), +} + +mkdirSync(options.outDir, { recursive: true }) +const outputPath = join(options.outDir, `incremental-update-${Date.now()}.json`) +writeFileSync(outputPath, JSON.stringify(report, null, 2)) + +console.log(formatTextReport(report, outputPath)) + +async function runCase( + queryCase: QueryCase, + mutationMode: MutationMode, + tracing: boolean, + options: BenchmarkRunOptions, +): Promise { + setPerfTracingEnabled(tracing) + resetPerfTrace() + + const fixture = createFixture(options) + const query = queryCase.createQuery(fixture) + + const coldStart = performance.now() + await query.preload() + await flushMicrotasks() + const coldHydrateMs = performance.now() - coldStart + + for (let i = 0; i < options.warmup; i++) { + const writeResult = queryCase.write(fixture, i, mutationMode) + await flushMicrotasks() + await cleanupWrite(writeResult, tracing) + } + + resetPerfTrace() + const samples: Array = [] + for (let i = 0; i < options.iterations; i++) { + const start = performance.now() + const writeResult = queryCase.write( + fixture, + options.warmup + i, + mutationMode, + ) + await flushMicrotasks() + samples.push(performance.now() - start) + await cleanupWrite(writeResult, tracing) + } + + const trace = getPerfTraceSnapshot() + await cleanupFixture(fixture, query) + + return { + query: queryCase.name, + scenario: queryCase.scenario, + scale: options.scale, + sourceIndexMode: options.sourceIndexMode, + mutationMode, + tracing, + coldHydrateMs, + writeSummary: summarize(samples), + trace, + } +} + +function createManualCollection( + id: string, + initialData: Array, + getKey: (item: T) => TKey, + sourceIndexMode: SourceIndexMode, +): ManualCollection { + let begin: Parameters[`sync`]>[0][`begin`] | undefined + let write: Parameters[`sync`]>[0][`write`] | undefined + let commit: Parameters[`sync`]>[0][`commit`] | undefined + + const utils: ManualSyncUtils = { + begin: (syncOptions) => begin!(syncOptions), + write: (message) => write!(message), + commit: () => commit!(), + } + + return createCollection>({ + id, + getKey, + utils, + startSync: true, + autoIndex: sourceIndexMode === `auto` ? `eager` : `off`, + defaultIndexType: BTreeIndex, + sync: { + rowUpdateMode: `full`, + sync: (methods) => { + begin = methods.begin + write = methods.write + commit = methods.commit + + begin() + for (const value of initialData) { + write({ + type: `insert`, + value, + }) + } + commit() + methods.markReady() + + return () => { + begin = undefined + write = undefined + commit = undefined + } + }, + }, + }) +} + +function createFixture(options: BenchmarkRunOptions): Fixture { + const random = seededRandom(options.seed) + const usersData: Array = [] + for (let id = 1; id <= options.userCount; id++) { + usersData.push({ + id, + name: `User ${id}`, + reputation: Math.floor(random() * 10_000), + }) + } + + const issuesData: Array = [] + for (let id = 1; id <= options.issueCount; id++) { + issuesData.push({ + id, + status: id % 3 === 0 ? `closed` : `open`, + authorId: 1 + (id % options.userCount), + createdAt: id, + title: `Issue ${id}`, + body: `Body ${id}`, + updateTick: 0, + }) + } + + const visibleIssueId = findNewestOpenIssueId(issuesData) + const selectedIssueId = visibleIssueId + const commentsData: Array = [] + for (let id = 1; id <= options.commentCount; id++) { + const hotIssueId = + id % 5 === 0 + ? 1 + Math.floor(random() * Math.min(options.issueCount, 50)) + : 1 + Math.floor(random() * options.issueCount) + commentsData.push({ + id, + issueId: hotIssueId, + authorId: 1 + (id % options.userCount), + createdAt: id, + body: `Comment ${id}`, + updateTick: 0, + }) + } + + const fixture = { + seed: options.seed, + issues: createManualCollection( + `bench-issues`, + issuesData, + (issue) => issue.id, + options.sourceIndexMode, + ), + users: createManualCollection( + `bench-users`, + usersData, + (user) => user.id, + options.sourceIndexMode, + ), + comments: createManualCollection( + `bench-comments`, + commentsData, + (comment) => comment.id, + options.sourceIndexMode, + ), + issueById: new Map(issuesData.map((issue) => [issue.id, issue])), + userById: new Map(usersData.map((user) => [user.id, user])), + commentById: new Map(commentsData.map((comment) => [comment.id, comment])), + visibleIssueId, + selectedIssueId, + nextCommentId: options.commentCount + 1, + } + + applySourceIndexes(fixture, options.sourceIndexMode) + + return fixture +} + +function applySourceIndexes( + fixture: Fixture, + sourceIndexMode: SourceIndexMode, +): void { + if (sourceIndexMode !== `manual`) return + + fixture.issues.createIndex((issue) => issue.id) + fixture.issues.createIndex((issue) => issue.status) + fixture.issues.createIndex((issue) => issue.authorId) + fixture.issues.createIndex((issue) => issue.createdAt) + fixture.users.createIndex((user) => user.id) + fixture.comments.createIndex((comment) => comment.issueId) + fixture.comments.createIndex((comment) => comment.createdAt) +} + +function updateVisibleIssue( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + const issue = fixture.issueById.get(fixture.visibleIssueId)! + const next = { + ...issue, + title: `Issue ${issue.id} tick ${iteration}`, + updateTick: iteration, + } + fixture.issueById.set(issue.id, next) + + if (mutationMode === `synced`) { + writeSync(fixture.issues, { + type: `update`, + value: next, + }) + return + } + + const transaction = writeOptimistic(() => { + fixture.issues.update(issue.id, (draft) => { + draft.title = next.title + draft.updateTick = next.updateTick + }) + }) + + return { + cleanup: () => { + transaction.rollback() + fixture.issueById.set(issue.id, issue) + }, + } +} + +function updateVisibleAuthor( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + const issue = fixture.issueById.get(fixture.visibleIssueId)! + const user = fixture.userById.get(issue.authorId)! + const next = { + ...user, + name: `User ${user.id} tick ${iteration}`, + reputation: user.reputation + 1, + } + fixture.userById.set(user.id, next) + + if (mutationMode === `synced`) { + writeSync(fixture.users, { + type: `update`, + value: next, + }) + return + } + + const transaction = writeOptimistic(() => { + fixture.users.update(user.id, (draft) => { + draft.name = next.name + draft.reputation = next.reputation + }) + }) + + return { + cleanup: () => { + transaction.rollback() + fixture.userById.set(user.id, user) + }, + } +} + +function insertVisibleComment( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + return insertCommentForIssue( + fixture, + fixture.visibleIssueId, + iteration, + mutationMode, + ) +} + +function insertSelectedIssueComment( + fixture: Fixture, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + return insertCommentForIssue( + fixture, + fixture.selectedIssueId, + iteration, + mutationMode, + ) +} + +function insertCommentForIssue( + fixture: Fixture, + issueId: number, + iteration: number, + mutationMode: MutationMode, +): WriteResult { + const comment: Comment = { + id: fixture.nextCommentId++, + issueId, + authorId: 1 + (iteration % fixture.userById.size), + createdAt: 1_000_000_000 + iteration, + body: `Inserted comment ${iteration}`, + updateTick: iteration, + } + fixture.commentById.set(comment.id, comment) + + if (mutationMode === `synced`) { + writeSync(fixture.comments, { + type: `insert`, + value: comment, + }) + return + } + + const transaction = writeOptimistic(() => { + fixture.comments.insert(comment) + }) + + return { + cleanup: () => { + transaction.rollback() + fixture.commentById.delete(comment.id) + }, + } +} + +function writeSync( + collection: ManualCollection, + message: ChangeMessageOrDeleteKeyMessage, +): void { + collection.utils.begin({ immediate: true }) + collection.utils.write(message) + collection.utils.commit() +} + +function writeOptimistic(write: () => void) { + const transaction = createTransaction({ + autoCommit: false, + mutationFn: async () => {}, + }) + transaction.isPersisted.promise.catch(() => undefined) + transaction.mutate(write) + return transaction +} + +async function cleanupWrite( + writeResult: WriteResult, + suppressTracing: boolean, +): Promise { + if (!writeResult) return + if (suppressTracing) { + setPerfTracingEnabled(false) + } + try { + await writeResult.cleanup() + await flushMicrotasks() + } finally { + if (suppressTracing) { + setPerfTracingEnabled(true) + } + } +} + +async function cleanupFixture( + fixture: Fixture, + query: BenchmarkCollection, +): Promise { + await Promise.all([ + query.cleanup(), + fixture.issues.cleanup(), + fixture.users.cleanup(), + fixture.comments.cleanup(), + ]) +} + +function summarize(samples: Array): Summary { + const sorted = [...samples].sort((a, b) => a - b) + const total = sorted.reduce((sum, value) => sum + value, 0) + const mean = total / sorted.length + const variance = + sorted.reduce((sum, value) => sum + (value - mean) ** 2, 0) / sorted.length + return { + iterations: sorted.length, + medianMs: percentile(sorted, 0.5), + p75Ms: percentile(sorted, 0.75), + p95Ms: percentile(sorted, 0.95), + minMs: sorted[0] ?? 0, + maxMs: sorted[sorted.length - 1] ?? 0, + stddevMs: Math.sqrt(variance), + } +} + +function percentile(sorted: Array, p: number): number { + if (sorted.length === 0) return 0 + const index = Math.min(sorted.length - 1, Math.ceil(sorted.length * p) - 1) + return sorted[index]! +} + +function computeOverhead(results: Array) { + const overhead: Array<{ + query: string + scenario: string + scale: FixtureScale + sourceIndexMode: SourceIndexMode + mutationMode: MutationMode + medianOverheadMs: number + p95OverheadMs: number + }> = [] + + for (const disabled of results.filter((result) => !result.tracing)) { + const enabled = results.find( + (result) => + result.tracing && + result.query === disabled.query && + result.scenario === disabled.scenario && + result.sourceIndexMode === disabled.sourceIndexMode && + result.mutationMode === disabled.mutationMode && + scaleKey(result.scale) === scaleKey(disabled.scale), + ) + if (!enabled) continue + overhead.push({ + query: disabled.query, + scenario: disabled.scenario, + scale: disabled.scale, + sourceIndexMode: disabled.sourceIndexMode, + mutationMode: disabled.mutationMode, + medianOverheadMs: + enabled.writeSummary.medianMs - disabled.writeSummary.medianMs, + p95OverheadMs: enabled.writeSummary.p95Ms - disabled.writeSummary.p95Ms, + }) + } + + return overhead +} + +function runtimeMetadata( + options: BenchmarkOptions, + scales: Array, +) { + return { + seed: options.seed, + levels: options.levels, + scales, + sourceIndexModes: options.sourceIndexModes, + mutationModes: options.mutationModes, + warmup: options.warmup, + iterations: options.iterations, + node: process.version, + platform: `${platform()} ${release()}`, + cpu: cpus()[0]?.model ?? `unknown`, + gitSha: gitSha(), + gcAvailable: typeof global.gc === `function`, + } +} + +function formatTextReport( + report: { + metadata: ReturnType + results: Array + overhead: ReturnType + }, + outputPath: string, +): string { + const lines: Array = [] + lines.push(`Incremental update benchmark`) + lines.push(`seed=${report.metadata.seed}`) + lines.push(`scales=${report.metadata.scales.map(formatScale).join(`; `)}`) + lines.push(`sourceIndexModes=${report.metadata.sourceIndexModes.join(`,`)}`) + lines.push(`mutationModes=${report.metadata.mutationModes.join(`,`)}`) + lines.push( + `runtime=node ${report.metadata.node} ${report.metadata.platform} ${report.metadata.cpu}`, + ) + lines.push(`git=${report.metadata.gitSha}`) + lines.push(`json=${outputPath}`) + lines.push(``) + + for (const scale of uniqueScales(report.results)) { + lines.push(`scale=${formatScale(scale)}`) + for (const sourceIndexMode of report.metadata.sourceIndexModes) { + lines.push(`sourceIndexMode=${sourceIndexMode}`) + for (const mutationMode of report.metadata.mutationModes) { + lines.push(`mutationMode=${mutationMode}`) + for (const result of report.results.filter( + (item) => + scaleKey(item.scale) === scaleKey(scale) && + item.sourceIndexMode === sourceIndexMode && + item.mutationMode === mutationMode, + )) { + lines.push( + `${result.query} | ${result.scenario} | tracing=${result.tracing}`, + ) + lines.push( + ` cold=${formatMs(result.coldHydrateMs)} median=${formatMs( + result.writeSummary.medianMs, + )} p95=${formatMs(result.writeSummary.p95Ms)} min=${formatMs( + result.writeSummary.minMs, + )} max=${formatMs(result.writeSummary.maxMs)} stddev=${formatMs( + result.writeSummary.stddevMs, + )}`, + ) + if (result.tracing) { + lines.push(` top spans:`) + for (const metric of topMetrics(result.trace, `span`, 5)) { + lines.push( + ` ${metric.name} total=${formatMs(metric.total)} calls=${ + metric.calls + } max=${formatMs(metric.max)}`, + ) + } + lines.push(` hash:`) + for (const metric of hashMetrics(result.trace, 6)) { + lines.push( + ` ${metric.name} total=${formatNumber(metric.total)} calls=${ + metric.calls + } tags=${JSON.stringify(metric.tags)}`, + ) + } + } + } + lines.push(``) + } + } + } + + lines.push(`trace overhead:`) + for (const overhead of report.overhead) { + lines.push( + ` ${formatScale(overhead.scale)} | sourceIndexMode=${ + overhead.sourceIndexMode + } | ${overhead.mutationMode} | ${overhead.query}: median=${formatMs( + overhead.medianOverheadMs, + )} p95=${formatMs(overhead.p95OverheadMs)}`, + ) + } + + return lines.join(`\n`) +} + +function resolveScales(options: BenchmarkOptions): Array { + const hasCustomScale = + options.issueCount !== undefined || + options.userCount !== undefined || + options.commentCount !== undefined + + if (hasCustomScale) { + return [ + { + label: `custom`, + issueCount: options.issueCount ?? defaultCustomScale.issueCount, + userCount: options.userCount ?? defaultCustomScale.userCount, + commentCount: options.commentCount ?? defaultCustomScale.commentCount, + }, + ] + } + + return options.levels.map((level) => ({ + label: String(level), + issueCount: level, + userCount: level, + commentCount: level, + })) +} + +function createRunOptions( + options: BenchmarkOptions, + scale: FixtureScale, + sourceIndexMode: SourceIndexMode, +): BenchmarkRunOptions { + return { + seed: options.seed, + scale, + issueCount: scale.issueCount, + userCount: scale.userCount, + commentCount: scale.commentCount, + sourceIndexMode, + warmup: options.warmup, + iterations: options.iterations, + } +} + +function uniqueScales(results: Array): Array { + const scales = new Map() + for (const result of results) { + scales.set(scaleKey(result.scale), result.scale) + } + return [...scales.values()] +} + +function scaleKey(scale: FixtureScale): string { + return `${scale.label}:${scale.issueCount}:${scale.userCount}:${scale.commentCount}` +} + +function formatScale(scale: FixtureScale): string { + if ( + scale.issueCount === scale.userCount && + scale.userCount === scale.commentCount + ) { + return `${formatInteger(scale.issueCount)} rows/collection` + } + + return `issues:${formatInteger(scale.issueCount)} users:${formatInteger( + scale.userCount, + )} comments:${formatInteger(scale.commentCount)}` +} + +function topMetrics( + trace: PerfTraceSnapshot, + kind: `span` | `counter` | `gauge`, + limit: number, +) { + return trace.metrics + .filter((metric) => metric.kind === kind) + .sort((a, b) => b.total - a.total || b.calls - a.calls) + .slice(0, limit) +} + +function hashMetrics(trace: PerfTraceSnapshot, limit: number) { + return trace.metrics + .filter((metric) => metric.name.startsWith(`hash.`)) + .sort((a, b) => b.total - a.total || b.calls - a.calls) + .slice(0, limit) +} + +function formatMs(value: number): string { + return `${formatNumber(value)}ms` +} + +function formatNumber(value: number): string { + return value.toFixed(3) +} + +function formatInteger(value: number): string { + return value.toLocaleString(`en-US`) +} + +function parseArgs( + args: Array, + defaults: BenchmarkOptions, +): BenchmarkOptions { + const parsed = { + ...defaults, + levels: [...defaults.levels], + sourceIndexModes: [...defaults.sourceIndexModes], + mutationModes: [...defaults.mutationModes], + } + + for (const arg of args) { + const [name, value] = arg.replace(/^--/, ``).split(`=`) + if (!name || value === undefined) continue + + switch (name) { + case `seed`: + parsed.seed = parseNonNegativeInteger(value, name) + break + case `levels`: + parsed.levels = parseLevels(value) + break + case `sourceIndexes`: + case `sourceIndexModes`: + parsed.sourceIndexModes = parseSourceIndexModes(value) + break + case `mutationModes`: + parsed.mutationModes = parseMutationModes(value) + break + case `issues`: + parsed.issueCount = parsePositiveCount(value, name) + break + case `users`: + parsed.userCount = parsePositiveCount(value, name) + break + case `comments`: + parsed.commentCount = parsePositiveCount(value, name) + break + case `warmup`: + parsed.warmup = parseNonNegativeInteger(value, name) + break + case `iterations`: + parsed.iterations = parsePositiveInteger(value, name) + break + case `outDir`: + parsed.outDir = value + break + } + } + + return parsed +} + +function parseLevels(value: string): Array { + const levels = value + .split(`,`) + .map((part) => part.trim()) + .filter((part) => part.length > 0) + .map((part) => parsePositiveCount(part, `levels`)) + + if (levels.length === 0) { + throw new Error(`--levels must contain at least one positive integer`) + } + + return levels +} + +function parseMutationModes(value: string): Array { + const modes = value + .split(`,`) + .map((part) => part.trim()) + .filter((part) => part.length > 0) + + if (modes.length === 0) { + throw new Error(`--mutationModes must contain at least one mode`) + } + + for (const mode of modes) { + if (mode !== `synced` && mode !== `optimistic`) { + throw new Error(`--mutationModes must contain synced and/or optimistic`) + } + } + + return modes +} + +function parseSourceIndexModes(value: string): Array { + const modes = value + .split(`,`) + .map((part) => part.trim()) + .filter((part) => part.length > 0) + + if (modes.length === 0) { + throw new Error(`--sourceIndexes must contain at least one mode`) + } + + for (const mode of modes) { + if (mode !== `none` && mode !== `manual` && mode !== `auto`) { + throw new Error(`--sourceIndexes must contain none, manual, and/or auto`) + } + } + + return modes +} + +function parsePositiveCount(value: string, name: string): number { + const normalized = value.trim().toLowerCase() + const parsed = normalized.endsWith(`k`) + ? Number(normalized.slice(0, -1)) * 1_000 + : Number(normalized) + + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`--${name} must be a positive integer or k-suffixed count`) + } + return parsed +} + +function parsePositiveInteger(value: string, name: string): number { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`--${name} must be a positive integer`) + } + return parsed +} + +function parseNonNegativeInteger(value: string, name: string): number { + const parsed = Number(value) + if (!Number.isInteger(parsed) || parsed < 0) { + throw new Error(`--${name} must be a non-negative integer`) + } + return parsed +} + +function seededRandom(seed: number): () => number { + let state = seed >>> 0 + return () => { + state ^= state << 13 + state ^= state >>> 17 + state ^= state << 5 + return ((state >>> 0) % 1_000_000) / 1_000_000 + } +} + +function findNewestOpenIssueId(issues: Array): number { + for (let index = issues.length - 1; index >= 0; index--) { + const issue = issues[index]! + if (issue.status === `open`) { + return issue.id + } + } + throw new Error(`Fixture must contain at least one open issue`) +} + +function gitSha(): string { + try { + return execSync(`git rev-parse --short HEAD`, { + encoding: `utf8`, + stdio: [`ignore`, `pipe`, `ignore`], + }).trim() + } catch { + return `unknown` + } +} + +async function flushMicrotasks(): Promise { + await Promise.resolve() + await Promise.resolve() +}