feat: distributed cursor loadMore() + real MCP pagination + docs sweep — Query Phase 4 (SPEC-306a/b/c)

apps/docs-astro/public/llms-full.txt

@@ -1542,20 +1542,38 @@ const handle = client.query<Todo>('todos', {
 const unsubscribe = handle.subscribe((results) => {
   console.log('Results:', results);
 });
+```
 
-// Cursor pagination
-const { nextCursor, hasMore } = handle.getPaginationInfo();
-if (hasMore && nextCursor) {
-  const nextPage = client.query<Todo>('todos', {
-    where: { completed: false },
-    sort: { createdAt: 'desc' },
-    limit: 10,
-    cursor: nextCursor,
-  });
+#### Cursor pagination — `loadMore()`
+
+Call `handle.loadMore()` to fetch the next page and append its rows to the existing result set. This is the recommended path: it guards against duplicate in-flight requests for the same cursor and merges the new page using an append-only strategy so earlier rows are never lost.
+
+```typescript
+const handle = client.query<Todo>('todos', {
+  where: { completed: false },
+  sort: { createdAt: 'desc' },
+  limit: 10,
+});
+
+// Render the first page
+const unsubscribe = handle.subscribe((results) => render(results));
+
+// Load subsequent pages on demand (e.g. a "Load more" button)
+await handle.loadMore();
+// The handle's subscriber fires again with the merged result set
+// (first page rows + new page rows combined).
+
+// Check whether more pages exist before calling again
+const { hasMore } = handle.getPaginationInfo();
+if (hasMore) {
+  await handle.loadMore();
 }
 ```
 
-`QueryHandle<T>` methods: `subscribe(cb)`, `onDelta(listener)`, `consumeChanges()`, `getLastChange()`, `getPendingChanges()`, `clearChanges()`, `resetChangeTracker()`, `getFilter()`, `getMapName()`, `getPaginationInfo()`, `onPaginationChange(listener)`, `updatePaginationInfo(info)`, `syncState` (getter), `onSyncStateChange(listener)`.
+`loadMore()` resolves immediately when there are no more pages (`hasMore` is false) or when a request for the same cursor is already in flight, so it is safe to call on every scroll event without extra debouncing.
+
+
+`QueryHandle<T>` methods: `subscribe(cb)`, `loadMore()`, `onDelta(listener)`, `consumeChanges()`, `getLastChange()`, `getPendingChanges()`, `clearChanges()`, `resetChangeTracker()`, `getFilter()`, `getMapName()`, `getPaginationInfo()`, `onPaginationChange(listener)`, `updatePaginationInfo(info)`, `syncState` (getter), `onSyncStateChange(listener)`.
 
 #### Knowing when a result is authoritative — `subscribe` `{ settled }`
 
@@ -1763,20 +1781,31 @@ const unsubscribe = handle.subscribe((results) => setResults(results));
 handle.dispose();
 ```
 
-### `sql(query)`
+### `sql(query)` — opt-in, server-feature-gated
 
 ```typescript
 public async sql(query: string): Promise<SqlQueryResult>
 ```
 
-Execute a SQL query server-side via DataFusion. Map names are table names. Requires the server's DataFusion feature and registered schemas.
+Execute a one-shot SQL query server-side via the DataFusion engine. Map names are used as table names; the server must have schemas registered for them.
+
+**Off by default.** The server's DataFusion feature is a compile-time opt-in (`--features datafusion`). The default server build (`default = ["redb"]`) does NOT include DataFusion. If you send a SQL query to a server built without this feature, the server returns an error.
+
+**Niche — use only for what the structured query API cannot express.** `client.sql()` exists for queries that genuinely require SQL: multi-map **joins**, **window functions**, **HAVING** clauses, and **DISTINCT** aggregation. For everything else — filtering, sorting, full-text search, group-by, aggregate functions — use the structured query API (`client.query()`, `client.queryOnce()`, `client.queryOncePaged()`) backed by the canonical DAG engine. Reaching for SQL when the structured API suffices adds a server-side dependency and bypasses the local-first offline path.
 
 ```typescript
-const result = await client.sql('SELECT name, age FROM users WHERE age > 21 ORDER BY age');
-// result.columns: ['name', 'age']
-// result.rows: [[...], [...]]
+// Only appropriate for cross-map joins or window/HAVING/DISTINCT queries
+const result = await client.sql(
+  'SELECT u.name, COUNT(t.id) AS task_count ' +
+  'FROM users u JOIN tasks t ON t.userId = u.id ' +
+  'GROUP BY u.name HAVING COUNT(t.id) > 3'
+);
+// result.columns: ['name', 'task_count']
+// result.rows: [['Alice', 5], ['Bob', 4]]
 ```
 
+`SqlQueryResult` shape: `{ columns: string[]; rows: unknown[][] }`. There is no cursor or pagination — SQL queries return the full result set. For large result sets, add a `LIMIT` clause.
+
 ### `vectorSearch(mapName, queryVector, options?)`
 
 ```typescript

apps/docs-astro/src/content/docs/reference/client.mdx

@@ -188,20 +188,39 @@ const handle = client.query<Todo>('todos', {
 const unsubscribe = handle.subscribe((results) => {
   console.log('Results:', results);
 });
+```
+
+#### Cursor pagination — `loadMore()`
+
+Call `handle.loadMore()` to fetch the next page and append its rows to the existing result set. This is the recommended path: it guards against duplicate in-flight requests for the same cursor and merges the new page using an append-only strategy so earlier rows are never lost.
+
+```typescript
+const handle = client.query<Todo>('todos', {
+  where: { completed: false },
+  sort: { createdAt: 'desc' },
+  limit: 10,
+});
+
+// Render the first page
+const unsubscribe = handle.subscribe((results) => render(results));
+
+// Load subsequent pages on demand (e.g. a "Load more" button)
+await handle.loadMore();
+// The handle's subscriber fires again with the merged result set
+// (first page rows + new page rows combined).
 
-// Cursor pagination
-const { nextCursor, hasMore } = handle.getPaginationInfo();
-if (hasMore && nextCursor) {
-  const nextPage = client.query<Todo>('todos', {
-    where: { completed: false },
-    sort: { createdAt: 'desc' },
-    limit: 10,
-    cursor: nextCursor,
-  });
+// Check whether more pages exist before calling again
+const { hasMore } = handle.getPaginationInfo();
+if (hasMore) {
+  await handle.loadMore();
 }
 ```
 
-`QueryHandle<T>` methods: `subscribe(cb)`, `onDelta(listener)`, `consumeChanges()`, `getLastChange()`, `getPendingChanges()`, `clearChanges()`, `resetChangeTracker()`, `getFilter()`, `getMapName()`, `getPaginationInfo()`, `onPaginationChange(listener)`, `updatePaginationInfo(info)`, `syncState` (getter), `onSyncStateChange(listener)`.
+`loadMore()` resolves immediately when there are no more pages (`hasMore` is false) or when a request for the same cursor is already in flight, so it is safe to call on every scroll event without extra debouncing.
+
+<AlertBox variant="info" title="Advanced: manual cursor re-query" text="For cases where you need explicit control over the cursor — building your own pagination state, issuing the next-page query from a different component, or integrating with a URL-driven page parameter — use getPaginationInfo() to read { nextCursor, hasMore, cursorStatus }, then pass cursor: nextCursor to a new client.query() call. loadMore() covers the common append-on-scroll use case; the manual form is for scenarios where append-only merge is not what you want." />
+
+`QueryHandle<T>` methods: `subscribe(cb)`, `loadMore()`, `onDelta(listener)`, `consumeChanges()`, `getLastChange()`, `getPendingChanges()`, `clearChanges()`, `resetChangeTracker()`, `getFilter()`, `getMapName()`, `getPaginationInfo()`, `onPaginationChange(listener)`, `updatePaginationInfo(info)`, `syncState` (getter), `onSyncStateChange(listener)`.
 
 #### Knowing when a result is authoritative — `subscribe` `{ settled }`
 
@@ -409,20 +428,31 @@ const unsubscribe = handle.subscribe((results) => setResults(results));
 handle.dispose();
 ```
 
-### `sql(query)`
+### `sql(query)` — opt-in, server-feature-gated
 
 ```typescript
 public async sql(query: string): Promise<SqlQueryResult>
 ```
 
-Execute a SQL query server-side via DataFusion. Map names are table names. Requires the server's DataFusion feature and registered schemas.
+Execute a one-shot SQL query server-side via the DataFusion engine. Map names are used as table names; the server must have schemas registered for them.
+
+**Off by default.** The server's DataFusion feature is a compile-time opt-in (`--features datafusion`). The default server build (`default = ["redb"]`) does NOT include DataFusion. If you send a SQL query to a server built without this feature, the server returns an error.
+
+**Niche — use only for what the structured query API cannot express.** `client.sql()` exists for queries that genuinely require SQL: multi-map **joins**, **window functions**, **HAVING** clauses, and **DISTINCT** aggregation. For everything else — filtering, sorting, full-text search, group-by, aggregate functions — use the structured query API (`client.query()`, `client.queryOnce()`, `client.queryOncePaged()`) backed by the canonical DAG engine. Reaching for SQL when the structured API suffices adds a server-side dependency and bypasses the local-first offline path.
 
 ```typescript
-const result = await client.sql('SELECT name, age FROM users WHERE age > 21 ORDER BY age');
-// result.columns: ['name', 'age']
-// result.rows: [[...], [...]]
+// Only appropriate for cross-map joins or window/HAVING/DISTINCT queries
+const result = await client.sql(
+  'SELECT u.name, COUNT(t.id) AS task_count ' +
+  'FROM users u JOIN tasks t ON t.userId = u.id ' +
+  'GROUP BY u.name HAVING COUNT(t.id) > 3'
+);
+// result.columns: ['name', 'task_count']
+// result.rows: [['Alice', 5], ['Bob', 4]]
 ```
 
+`SqlQueryResult` shape: `{ columns: string[]; rows: unknown[][] }`. There is no cursor or pagination — SQL queries return the full result set. For large result sets, add a `LIMIT` clause.
+
 ### `vectorSearch(mapName, queryVector, options?)`
 
 ```typescript

apps/docs-astro/src/content/docs/reference/mcp.mdx

@@ -134,8 +134,8 @@ List every map the server knows about.
 
 Read records from a map with filters and sorting. Returns **authoritative server data** — the assistant gets the live, server-confirmed answer, not a stale local guess.
 
-- **Parameters:** `{ map, filter?, sort?, limit?, fields? }`
-- **Response:** `[{ _key, ...fields }]` (a plain settled array)
+- **Parameters:** `{ map, filter?, sort?, limit?, fields?, cursor? }`
+- **Response:** `[{ _key, ...fields }]` (a settled array) with an optional continuation note when `hasMore` is true
 - **Security:** rejects maps outside `allowedMaps`; clamps `limit` to `maxLimit`
 
 ```text
@@ -147,7 +147,11 @@ A settled result with no matching records renders as `No results found in map '<
 
 **Offline / not-settled behavior.** `topgun_query` never silently returns stale local data and never conflates "offline" with "empty". If the server cannot be reached, or the query does not settle within the default 5000 ms window, the tool returns an error (`isError: true`) explaining that no authoritative data was returned — so the assistant knows to retry rather than reporting an empty list. The message distinguishes the two causes (server unreachable / client offline, vs. timed out waiting for the server).
 
-**No cursor pagination, but truncation is never silent.** `topgun_query` returns a plain settled array; the previous `nextCursor` / `hasMore` continuation hint and `cursor` parameter have been removed (continuation cursors are an anti-pattern for an LLM caller). It does **not** drop the one signal that matters: when more rows match than the returned `limit`, the response appends a `More rows match than were returned…` note so the assistant knows the view was capped and can narrow `filter` / `sort` (or raise `limit` up to `maxLimit`) — rather than reporting a truncated list as the whole answer.
+**Cursor pagination.** When the server has more results than the returned `limit`, the response includes a `cursor` token and appends a continuation note: "To fetch the next page, call this tool again with `cursor: <value>`." Pass that token as the optional `cursor` parameter on the next call to receive the following page. Repeat until `hasMore` is false (no continuation note appears). This lets an agent walk through a large result set page by page without re-fetching rows it has already seen.
+
+If you do not need further pages, the continuation note also tells you to narrow with `filter` / `sort` or raise `limit` (up to `maxLimit`) — useful when you want a different slice rather than sequential pages.
+
+A settled result with no matching records renders as `No results found in map '<map>'...` — that means the server genuinely has no such rows, not that the query failed.
 
 ### topgun_mutate
 

packages/client/src/QueryHandle.ts

@@ -413,6 +413,85 @@ export class QueryHandle<T> {
 
   // ============== Pagination Methods ==============
 
+  /**
+   * In-flight cursor token for loadMore. Stores the cursor being fetched so
+   * concurrent calls with the same cursor are deduplicated to a single request.
+   */
+  private _loadMoreInFlight: string | null = null;
+
+  /**
+   * Append-only merge: adds/updates keys from the page batch without removing
+   * any existing keys. This preserves results from prior pages that are absent
+   * from the new page batch — a full-set reconciliation via onResult would
+   * prune those prior-page rows.
+   */
+  private mergePageResults(items: { key: string; value: T }[]): void {
+    for (const item of items) {
+      this.currentResults.set(item.key, item.value);
+    }
+    this.computeAndNotifyChanges(Date.now());
+    this.notify();
+  }
+
+  /**
+   * Load the next page of results and append them to the current result set.
+   *
+   * Uses the cursor from the most recent server response. If no further pages
+   * are available (`hasMore` is false) or a request for the same cursor is
+   * already in flight, resolves immediately without issuing a duplicate request.
+   *
+   * Results from the new page are merged with the existing result set using an
+   * append-only strategy: prior-page rows are never removed.
+   */
+  public async loadMore(): Promise<void> {
+    const { nextCursor, hasMore } = this._paginationInfo;
+
+    // No more pages available — nothing to fetch.
+    if (!hasMore || !nextCursor) return;
+
+    // Deduplicate concurrent calls for the same cursor.
+    if (this._loadMoreInFlight === nextCursor) return;
+
+    this._loadMoreInFlight = nextCursor;
+
+    try {
+      // Create a temporary one-shot handle with the next cursor, exactly
+      // mirroring the queryOnce pattern: subscribe → settle → unsubscribe.
+      const tempHandle = new QueryHandle<T>(this.syncEngine, this.mapName, {
+        ...this.filter,
+        cursor: nextCursor,
+      });
+
+      const pageItems: { key: string; value: T }[] = [];
+
+      const unsub = tempHandle.subscribe((results) => {
+        pageItems.length = 0;
+        for (const item of results) {
+          // Destructure the synthetic _key field added by getSortedResults.
+          const { _key, ...rest } = item as T & { _key: string };
+          pageItems.push({ key: _key, value: rest as T });
+        }
+      });
+
+      await tempHandle.whenSettled();
+      unsub();
+
+      // Append-only merge: preserves rows from all prior pages.
+      this.mergePageResults(pageItems);
+
+      // Advance pagination state to reflect the new page's cursor/hasMore.
+      const newPaginationInfo = tempHandle.getPaginationInfo();
+      this._paginationInfo = {
+        nextCursor: newPaginationInfo.nextCursor,
+        hasMore: newPaginationInfo.hasMore,
+        cursorStatus: newPaginationInfo.cursorStatus,
+      };
+      this.notifyPaginationListeners();
+    } finally {
+      this._loadMoreInFlight = null;
+    }
+  }
+
   /**
    * Get current pagination info.
    * Returns nextCursor, hasMore, and cursorStatus.