From a8bb927dffb621c84fcaba3d5f93e45c1a6d97a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 04:34:04 +0000 Subject: [PATCH 1/6] fix(objectql): floor a plugin driver on an external datasource: the engine applies the query, up to 1000 rows A datasource whose schemaMode is not managed and whose driver id is not built in is served through a floored driver, stored once in the engine's drivers map. The driver is sent where, orderBy and limit 1001; the engine applies where, orderBy, offset, limit and fields itself (applyDriverQueryInMemory, over the having walker with $like/$ilike lifted into its where position). findOne and count read the same way, aggregate is hidden, writes pass through. More than 1000 rows is EXTERNAL_ROW_CAP_EXCEEDED (422); a remote 4xx on a write is EXTERNAL_WRITE_REFUSED (422); any other remote failure is ERR_DATASOURCE_UNAVAILABLE (503, reason remote). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2dgsoHnkCiSyXF8isz --- .changeset/27277-plugin-driver-floor.md | 27 ++ packages/objectql/src/core.ts | 1 + .../objectql/src/driver-connect-errors.ts | 24 +- ...driver-query-in-memory.conformance.test.ts | 148 ++++++++++ .../objectql/src/driver-query-in-memory.ts | 206 ++++++++++++++ .../objectql/src/driver-where-like.test.ts | 72 +++++ packages/objectql/src/engine.ts | 50 +++- .../objectql/src/external-driver-errors.ts | 99 +++++++ .../src/external-driver-floor.test.ts | 269 ++++++++++++++++++ .../objectql/src/external-driver-floor.ts | 224 +++++++++++++++ packages/objectql/src/having-filter.ts | 103 ++++++- packages/objectql/src/index.ts | 15 + ...lugin-driver-external-object.spike.test.ts | 49 +++- 13 files changed, 1263 insertions(+), 24 deletions(-) create mode 100644 .changeset/27277-plugin-driver-floor.md create mode 100644 packages/objectql/src/driver-query-in-memory.conformance.test.ts create mode 100644 packages/objectql/src/driver-query-in-memory.ts create mode 100644 packages/objectql/src/driver-where-like.test.ts create mode 100644 packages/objectql/src/external-driver-errors.ts create mode 100644 packages/objectql/src/external-driver-floor.test.ts create mode 100644 packages/objectql/src/external-driver-floor.ts diff --git a/.changeset/27277-plugin-driver-floor.md b/.changeset/27277-plugin-driver-floor.md new file mode 100644 index 00000000000..e833abd35a2 --- /dev/null +++ b/.changeset/27277-plugin-driver-floor.md @@ -0,0 +1,27 @@ +--- +'@objectstack/objectql': patch +'@objectstack/spec': patch +--- + +fix(objectql)!: a plugin driver on an external datasource answers every read correctly or refuses: the engine applies the whole query to the rows the driver returns, up to 1000 of them (#27277) + +Clause-②: yes (two codes join ERROR_CODE_LEDGER under `@objectstack/objectql`: `EXTERNAL_WRITE_REFUSED` and `EXTERNAL_ROW_CAP_EXCEEDED`, both 422) +Metadata-protocol: none + + + +**BREAKING** accept-set narrowing for reads on a plugin-driver external datasource (`@objectstack/objectql`). Only a datasource with both of these is affected: `schemaMode` is not `managed`, and `driver` names a driver a server plugin published (not a built-in id: memory, sqlite, sqlite-wasm, postgres, mysql, mongodb, turso). A managed datasource and an ADR-0015 federated database on a built-in driver are unchanged. + +- **Before.** The engine passed the whole query to the plugin driver and served whatever came back. A driver that ignored part of the filter, the sort or the paging answered the wrong rows, and nothing said so. +- **Now.** The engine holds such a driver behind a correctness floor. The driver is sent `where` and `orderBy`, which it may push down, and `limit: 1001`. It is never sent `offset` or `fields`. The engine then applies the whole query to the rows it gets back (`applyDriverQueryInMemory`): filter, sort (nulls last, and an `id` tie-break on a paged read), offset and limit, and projection. A driver that returned every row still answers the right subset. + - `findOne` and `count` are answered from the same read. + - `aggregate` is hidden, so the engine aggregates in memory over that read. + - The filter is evaluated by the engine's own walker (the one `having` uses). `$like` and `$ilike` now work in its `where` position. It is held to the spec tables `FILTER_LOGIC_CASES`, `FILTER_TEXT_CASES`, `PAGINATION_*_CASES` and `TEMPORAL_CASES`. +- **Row cap.** A read that matches more than 1000 rows is refused with `EXTERNAL_ROW_CAP_EXCEEDED` (422), whose message says to narrow the filter or mirror the data into a real object with DataSync. +- **Writes pass through.** A remote refusal with a 4xx status is `EXTERNAL_WRITE_REFUSED` (422). Its message is the remote's own, and `details.remoteStatus` carries the status. A remote that fails, does not answer, or answers 5xx is `ERR_DATASOURCE_UNAVAILABLE` (503, `reason: 'remote'`); its cause goes to the server log at `warn` and is never sent to the caller. +- **One instance.** The engine stores the floored driver in place of the driver, so `getDriverByName`, `datasource()` and the routing for an object all return the same instance. `unwrapExternalDriverFloor` answers the driver behind it. +- **Not covered.** `updateMany` and `deleteMany` pass the predicate through to the driver unchanged. A `where` with a `ReadableIds` leaf is refused (ADR-0139 D3), because the engine has no semi-join over a plugin driver's rows. + +**The remedy.** If a read now meets the cap, narrow its filter, or mirror the data with a `mapping` pulled by a `job` and read the real object. + +New exports from `@objectstack/objectql`: `applyDriverQueryInMemory`, `ExternalWriteRefusedError`, `ExternalRowCapExceededError`, their `*_CODE` constants, `EXTERNAL_ROW_CAP`, `needsExternalDriverFloor`, `unwrapExternalDriverFloor`, and the types `DriverQueryInMemory`, `DriverQueryInMemoryOptions`, `ExternalWriteOperation` and `DatasourceUnavailableReason`. `DatasourceDef` gains an optional `driver`. diff --git a/packages/objectql/src/core.ts b/packages/objectql/src/core.ts index 5ed73f4d103..222394f85da 100644 --- a/packages/objectql/src/core.ts +++ b/packages/objectql/src/core.ts @@ -72,6 +72,7 @@ export type { DriverHealth, DatasourceUnavailableInfo, DatasourceUnavailableKind, + DatasourceUnavailableReason, } from './driver-connect-errors.js'; // In-memory aggregation fallback diff --git a/packages/objectql/src/driver-connect-errors.ts b/packages/objectql/src/driver-connect-errors.ts index 3642639839f..8fd71892528 100644 --- a/packages/objectql/src/driver-connect-errors.ts +++ b/packages/objectql/src/driver-connect-errors.ts @@ -178,6 +178,15 @@ export class DriverConnectError extends Error { /** Why a declared datasource has no live driver (framework#3828). */ export type DatasourceUnavailableKind = 'blocked' | 'failed'; +/** + * [#27277] The reasons {@link DatasourceUnavailableError} reports: the two a + * datasource with no live driver has, plus `remote` — the driver IS live, and + * the remote system behind a plugin driver failed this one request (it threw, + * was unreachable, or answered 5xx). Per request, so it is never a value the + * connection layer records ({@link DatasourceUnavailableInfo} keeps the two). + */ +export type DatasourceUnavailableReason = DatasourceUnavailableKind | 'remote'; + /** What the connection layer recorded about a datasource it could not connect. */ export interface DatasourceUnavailableInfo { kind: DatasourceUnavailableKind; @@ -245,19 +254,24 @@ export class DatasourceUnavailableError extends Error { constructor( public readonly datasource: string, public readonly objectName: string, - public readonly kind: DatasourceUnavailableKind, + public readonly kind: DatasourceUnavailableReason, publicDetail?: string, ) { - const head = - `[ObjectQL] Datasource '${datasource}' configured for object '${objectName}' is declared but not connected`; + const head = kind === 'remote' + ? `[ObjectQL] Datasource '${datasource}' could not answer this request for object '${objectName}'` + : `[ObjectQL] Datasource '${datasource}' configured for object '${objectName}' is declared but not connected`; const why = - kind === 'blocked' + kind === 'remote' + ? `: the remote system behind its plugin driver failed, did not respond, or answered with a server ` + + `error. It may clear; retry later.` + : kind === 'blocked' ? `: the host's datasource connect policy refused it.` : `: it failed to connect at startup and the server was started with ` + `OS_ALLOW_DRIVER_CONNECT_FAILURE. Nothing re-runs the connect — the server must be ` + `restarted once the datasource is reachable.`; const detail = publicDetail ? ` ${publicDetail}` : ''; - super(`${head}${why}${detail} See the startup logs or Setup → Datasources for the cause.`); + const where = kind === 'remote' ? 'the server log' : 'the startup logs or Setup → Datasources'; + super(`${head}${why}${detail} See ${where} for the cause.`); this.name = 'DatasourceUnavailableError'; } } diff --git a/packages/objectql/src/driver-query-in-memory.conformance.test.ts b/packages/objectql/src/driver-query-in-memory.conformance.test.ts new file mode 100644 index 00000000000..5b0d974c5d2 --- /dev/null +++ b/packages/objectql/src/driver-query-in-memory.conformance.test.ts @@ -0,0 +1,148 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27277] `applyDriverQueryInMemory` answers the spec's driver conformance + * tables — the standard every built-in driver is held to — because it is what + * answers a query for a plugin driver on an external datasource + * (`external-driver-floor.ts`). The tables are the oracle, not any one driver. + * + * - `FILTER_LOGIC_CASES`: keys, operators, combinators, the null family. + * - `FILTER_TEXT_CASES`: the text family, its comparand literalness, and every + * rejection row in the ADR-0112 envelope. + * - `PAGINATION_CASES` / `PAGINATION_UNORDERED_CASES` / + * `PAGINATION_ZERO_LIMIT_CASES`: a paged read is a partition; `limit` by + * presence. + * - `TEMPORAL_CASES`: the `where` lowered as the engine hands it to a driver + * (`lowerFilterCondition`, tokens resolved against `TEMPORAL_NOW`), over rows + * in both writer forms, with the declared field classes. + */ + +import { describe, it, expect } from 'vitest'; +import { + FILTER_LOGIC_CASES, + FILTER_LOGIC_ROWS, + FILTER_TEXT_CASES, + FILTER_TEXT_ROWS, + PAGINATION_ALL_IDS, + PAGINATION_CASES, + PAGINATION_ROWS, + PAGINATION_UNORDERED_CASES, + PAGINATION_ZERO_LIMIT_CASES, + TEMPORAL_CASES, + TEMPORAL_NOW, + TEMPORAL_ROWS, + lowerFilterCondition, + type FilterTextCase, + type FilterTextRejectionCase, +} from '@objectstack/spec/data'; +import { resolveFilterTokens } from '@objectstack/core'; +import { applyDriverQueryInMemory } from './driver-query-in-memory.js'; +import { declaredFieldClasses } from './having-filter.js'; + +const idsOf = (rows: Record[]) => rows.map((row) => row.id as string); +const sortedIds = (rows: Record[]) => idsOf(rows).sort(); + +describe('[#27277] applyDriverQueryInMemory — FILTER_LOGIC_CASES', () => { + const rows = FILTER_LOGIC_ROWS.map((row) => ({ ...row })); + for (const c of FILTER_LOGIC_CASES) { + it(c.name, () => { + expect(sortedIds(applyDriverQueryInMemory(rows, { where: c.filter })), c.note).toEqual([...c.expected]); + }); + } +}); + +describe('[#27277] applyDriverQueryInMemory — FILTER_TEXT_CASES', () => { + const rows = FILTER_TEXT_ROWS.map((row) => ({ ...row })); + const isRejection = (c: FilterTextCase): c is FilterTextRejectionCase => c.expectRejection === true; + + for (const c of FILTER_TEXT_CASES) { + if (isRejection(c)) { + it(`${c.name} — refused in the ADR-0112 envelope`, () => { + let err: (Error & { code?: string; status?: number }) | undefined; + try { + applyDriverQueryInMemory(rows, { where: c.filter }); + } catch (e) { + err = e as Error & { code?: string; status?: number }; + } + expect(err, c.note ?? 'expected a refusal').toBeInstanceOf(Error); + expect(err!.code).toBe(c.code); + expect(err!.status).toBe(400); + for (const mention of c.mustMention) expect(err!.message).toContain(mention); + }); + } else { + it(c.name, () => { + expect(sortedIds(applyDriverQueryInMemory(rows, { where: c.filter })), c.note).toEqual([...c.expected]); + }); + } + } +}); + +describe('[#27277] applyDriverQueryInMemory — PAGINATION cases', () => { + // Seeded in the table's order; the helper sorts, so arrival order is not a + // sort the walk could lean on. + const rows = PAGINATION_ROWS.map((row) => ({ ...row })); + + const walk = (query: { orderBy?: ReadonlyArray<{ field: string; order: 'asc' | 'desc' }> }, pageSize: number) => { + const visited: string[] = []; + for (let offset = 0; offset < rows.length; offset += pageSize) { + visited.push(...idsOf(applyDriverQueryInMemory(rows, { ...query, limit: pageSize, offset }))); + } + return visited; + }; + + for (const c of PAGINATION_CASES) { + it(`${c.name}: every row exactly once, in the requested order`, () => { + const visited = walk({ orderBy: c.orderBy }, c.pageSize); + expect([...visited].sort()).toEqual([...PAGINATION_ALL_IDS].sort()); + // The walk agrees with one unpaged read's order on every requested key. + const whole = applyDriverQueryInMemory(rows, { orderBy: c.orderBy }); + const byId = new Map(rows.map((row) => [row.id, row])); + const keys = (id: string) => c.orderBy.map(({ field }) => (byId.get(id) as Record)[field]); + expect(visited.map(keys)).toEqual(idsOf(whole).map(keys)); + }); + } + + for (const c of PAGINATION_UNORDERED_CASES) { + it(`unordered — ${c.name}: every row exactly once`, () => { + const visited = walk({}, c.pageSize); + expect(visited).toHaveLength(PAGINATION_ALL_IDS.length); + expect([...visited].sort()).toEqual([...PAGINATION_ALL_IDS].sort()); + }); + } + + for (const c of PAGINATION_ZERO_LIMIT_CASES) { + it(`zero limit — ${c.name}`, () => { + expect(applyDriverQueryInMemory(rows, c.query)).toHaveLength(c.expectedRowCount); + }); + } +}); + +describe('[#27277] applyDriverQueryInMemory — TEMPORAL_CASES', () => { + const FIELDS = { at: { type: 'datetime' }, on: { type: 'date' }, why: { type: 'text' } }; + const fieldClasses = declaredFieldClasses(FIELDS); + const lowered = (filter: T): T => + lowerFilterCondition(filter, { isDatetimeColumn: (column) => column === 'at' }); + const resolved = (filter: T): T => resolveFilterTokens(filter, { now: new Date(TEMPORAL_NOW) }); + + // A remote answers in whatever form it stores: the wire form on some rows, + // a native `Date` on others (the table's writer axis). + const rows = TEMPORAL_ROWS.map((r) => ({ + id: r.id, + at: r.writerForm === 'native' ? new Date(r.at) : r.at, + on: r.on, + why: r.why, + })); + + for (const c of TEMPORAL_CASES) { + it(c.name, () => { + const got = sortedIds(applyDriverQueryInMemory(rows, { where: lowered(c.filter) }, { fieldClasses })); + expect(got, c.note).toEqual([...c.expected].sort()); + }); + if (c.tokenFilter) { + it(`${c.name} — via relative tokens`, () => { + const got = sortedIds(applyDriverQueryInMemory(rows, { where: lowered(resolved(c.tokenFilter)) }, { fieldClasses })); + expect(got, c.note).toEqual([...c.expected].sort()); + }); + } + } +}); diff --git a/packages/objectql/src/driver-query-in-memory.ts b/packages/objectql/src/driver-query-in-memory.ts new file mode 100644 index 00000000000..717ad4b6b90 --- /dev/null +++ b/packages/objectql/src/driver-query-in-memory.ts @@ -0,0 +1,206 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `applyDriverQueryInMemory` — the engine's own reading of a driver query over + * rows already in memory: `where`, `orderBy`, `offset` / `limit` and `fields`, + * in that order (ADR-0015 Tier A, epic #27261 P1). + * + * ## Why the engine owns this + * + * A plugin driver (a `driver.` service a server plugin publishes) is + * written against a remote system that may filter, sort and page some of a + * query or none of it. `DriverCapabilities` carries no claim for any of the + * three (the bits were retired in 17.0.0 with the prescription that `find()` + * executes the whole AST), so the engine cannot know which parts such a + * driver honoured. The floor in `external-driver-floor.ts` therefore applies + * the whole query here, over whatever rows the driver returned: a driver that + * already filtered loses nothing (filtering a filtered set is the identity), + * and one that returned everything still answers the right subset. A driver + * cannot opt out of correctness, only of performance (ruling 6471382887). + * + * ## One filter semantics, not a second one + * + * `where` is evaluated by objectql's own walker (`matchesDriverWhere`, + * `having-filter.ts`), the one that already serves `having` and the + * per-aggregation `filter`: one vocabulary, one ADR-0112 envelope, the #5298 + * null-safe negations, and a temporal comparand read by the column's storage + * rule when `fieldClasses` names the column's class. The spec's conformance + * tables (`FILTER_LOGIC_CASES`, `FILTER_TEXT_CASES`, `PAGINATION_*_CASES`, + * `TEMPORAL_CASES`) are its oracle, not any one driver + * (`driver-query-in-memory.conformance.test.ts`). + * + * The `where` is expected as the engine hands it to a driver: tokens resolved + * and the shared lowering applied (`lowerFilterCondition`, ADR-0053 D-D1). The + * helper applies no whole-day rule of its own, as no driver does. + * + * ## Sort and paging + * + * - Keys compare as `driver-memory`'s reference reading does: a value with no + * value (`null` / absent) sorts LAST in either direction, `Date`s as + * instants, everything else by `<` / `>`. + * - A PAGED read (`limit` or `offset` present) is a partition of the result + * set (`IDataDriver.find`'s contract, `PAGINATION_CASES`): `id` is appended + * as the tie-breaker in the last key's direction (`asc` when there is none), + * as `driver-sql` orders a paged read. An unpaged read keeps the rows' + * arrival order when it carries no `orderBy`. + * - `limit` and `offset` are read by PRESENCE: `limit: 0` returns no rows + * (#6485). + * + * `fields` projects each row to the named columns plus `id`, which every row + * keeps (the projection `driver-memory` applies). Every row returned is a + * fresh object. + */ + +import type { FilterCondition } from '@objectstack/spec/data'; +import { findMisplacedReadableIds, readableIdsDriverRefusal } from '@objectstack/core'; +import { matchesDriverWhere, type AggregatedColumnClass } from './having-filter.js'; + +/** The query keys this helper applies. */ +export interface DriverQueryInMemory { + where?: FilterCondition; + orderBy?: ReadonlyArray<{ field: string; order?: 'asc' | 'desc' }>; + limit?: number; + offset?: number; + fields?: readonly string[]; +} + +/** What the helper is told about the object the rows belong to. */ +export interface DriverQueryInMemoryOptions { + /** + * Each declared field's comparison class (`declaredFieldClasses(fields)`): a + * temporal comparand is compared by its column's storage rule. Absent ⇒ + * every comparand as written. + */ + fieldClasses?: ReadonlyMap; + /** + * The declared JSON-stored fields (`declaredJsonStoredFields(fields)`), on + * which `$contains` / `$notContains` ask membership and the scalar + * comparisons are refused, as `where` refuses them on every SQL dialect. + */ + jsonStored?: ReadonlySet; +} + +/** + * Keys a driver query may carry that this helper does not apply. Each is + * refused by name rather than ignored: ignoring `groupBy` would answer raw + * rows for a grouped read, ignoring `search` every row for a search. + * `object` and `expand` are not here — the first is the call's routing, the + * second the engine's own post-read step. + */ +const UNAPPLIED_KEYS = ['groupBy', 'aggregations', 'having', 'search', 'searchFields', 'top'] as const; + +/** The refusal name a composed `ReadableIds` leaf is refused under (ADR-0139 D3). */ +const READABLE_IDS_REFUSER = 'objectql in-memory query'; + +/** + * Refuse a `where` this helper cannot evaluate before any row is read — or a + * remote is asked: a `ReadableIds` leaf, composed or not (ADR-0139 D3). The + * walker would read a composed leaf in a `$in` slot as an empty list and + * answer no rows, a short list nobody was told about. + */ +export function assertDriverWhereEvaluableInMemory(where: unknown): void { + const site = findMisplacedReadableIds(where, false); + if (site) { + throw readableIdsDriverRefusal( + READABLE_IDS_REFUSER, + site, + 'the engine evaluates a plugin driver\'s rows in memory and has no semi-join to read the leaf with', + ); + } +} + +/** + * Apply `query` to `rows` in memory: `where`, then `orderBy`, then `offset` / + * `limit`, then `fields`. See the module comment for each step's reading. + * + * @throws `INVALID_FILTER` / 400 for a `where` the walker refuses, and a plain + * error for a key this helper does not apply or a malformed `orderBy`, + * `limit` or `offset`. + */ +export function applyDriverQueryInMemory( + rows: readonly Record[], + query: DriverQueryInMemory, + options: DriverQueryInMemoryOptions = {}, +): Record[] { + for (const key of UNAPPLIED_KEYS) { + if ((query as Record)[key] !== undefined) { + throw new Error( + `applyDriverQueryInMemory does not apply \`${key}\`. It reads where, orderBy, limit, offset and fields; ` + + `a query carrying \`${key}\` is refused rather than answered without it.`, + ); + } + } + assertDriverWhereEvaluableInMemory(query.where); + const orderBy = readOrderBy(query.orderBy); + const limit = readCount('limit', query.limit); + const offset = readCount('offset', query.offset); + + let result = query.where === undefined + ? [...rows] + : rows.filter((row) => matchesDriverWhere(row, query.where, options.fieldClasses, options.jsonStored)); + + const paged = limit !== undefined || offset !== undefined; + const keys = paged && !orderBy.some((key) => key.field === 'id') + ? [...orderBy, { field: 'id', order: orderBy.length > 0 ? orderBy[orderBy.length - 1]!.order : 'asc' as const }] + : orderBy; + if (keys.length > 0) result = [...result].sort((a, b) => compareRows(a, b, keys)); + + if (offset !== undefined) result = result.slice(offset); + if (limit !== undefined) result = result.slice(0, limit); + + const fields = query.fields; + if (Array.isArray(fields) && fields.length > 0) return result.map((row) => project(row, fields)); + return result.map((row) => ({ ...row })); +} + +type SortKey = { field: string; order: 'asc' | 'desc' }; + +function readOrderBy(orderBy: unknown): SortKey[] { + if (orderBy === undefined) return []; + if (!Array.isArray(orderBy)) { + throw new Error(`applyDriverQueryInMemory: \`orderBy\` must be an array of { field, order } nodes, received ${JSON.stringify(orderBy)}.`); + } + return orderBy.map((node: unknown) => { + const { field, order } = (node ?? {}) as { field?: unknown; order?: unknown }; + if (typeof field !== 'string' || field === '' || (order !== undefined && order !== 'asc' && order !== 'desc')) { + throw new Error(`applyDriverQueryInMemory: an \`orderBy\` node must be { field: string, order?: 'asc' | 'desc' }, received ${JSON.stringify(node)}.`); + } + return { field, order: order ?? 'asc' }; + }); +} + +function readCount(key: 'limit' | 'offset', value: unknown): number | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) { + throw new Error(`applyDriverQueryInMemory: \`${key}\` must be a non-negative integer, received ${JSON.stringify(value)}.`); + } + return value; +} + +function compareRows(a: Record, b: Record, keys: readonly SortKey[]): number { + for (const { field, order } of keys) { + const x = a[field]; + const y = b[field]; + const xMissing = x === undefined || x === null; + const yMissing = y === undefined || y === null; + if (xMissing || yMissing) { + if (xMissing && yMissing) continue; + // No value sorts last in either direction. + return xMissing ? 1 : -1; + } + const left = x instanceof Date ? x.getTime() : x; + const right = y instanceof Date ? y.getTime() : y; + if ((left as any) < (right as any)) return order === 'desc' ? 1 : -1; + if ((left as any) > (right as any)) return order === 'desc' ? -1 : 1; + } + return 0; +} + +function project(row: Record, fields: readonly string[]): Record { + const out: Record = {}; + for (const field of fields) { + if (row[field] !== undefined) out[field] = row[field]; + } + if (out.id === undefined && row.id !== undefined) out.id = row.id; + return out; +} diff --git a/packages/objectql/src/driver-where-like.test.ts b/packages/objectql/src/driver-where-like.test.ts new file mode 100644 index 00000000000..c22130116a0 --- /dev/null +++ b/packages/objectql/src/driver-where-like.test.ts @@ -0,0 +1,72 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27277] `$like` / `$ilike` in the walker's `where` position — the pattern + * operators a driver `where` carries and `having` never did. The walker reads + * a pattern with the spec's one compiler (`likePatternToRegExp`) and refuses + * the shapes the spec refuses (a dangling escape, a NUL, a non-string), in the + * ADR-0112 envelope; `having` and the per-aggregation `filter` still refuse + * both operators as outside their vocabulary. + */ + +import { describe, it, expect } from 'vitest'; +import { applyHaving, matchesDriverWhere } from './having-filter.js'; + +const ROWS = [ + { id: '1', name: 'ACME Corp' }, + { id: '2', name: 'acme corp' }, + { id: '3', name: '100% match' }, + { id: '4', name: '100X match' }, + { id: '5', name: 'a_b' }, + { id: '6', name: 'axb' }, + { id: '7', name: null }, +]; + +const select = (where: Record) => + ROWS.filter((row) => matchesDriverWhere(row, where as any)).map((row) => row.id); + +function refusal(run: () => unknown): Error & { code?: string; status?: number } { + try { + run(); + } catch (e) { + return e as Error & { code?: string; status?: number }; + } + throw new Error('expected a refusal'); +} + +describe('[#27277] $like / $ilike in a driver where', () => { + it('$like is case-sensitive; % and _ are wildcards', () => { + expect(select({ name: { $like: 'ACME%' } })).toEqual(['1']); + expect(select({ name: { $like: 'a_b' } })).toEqual(['5', '6']); + }); + + it('$ilike folds ASCII case', () => { + expect(select({ name: { $ilike: 'acme%' } })).toEqual(['1', '2']); + }); + + it('an escaped wildcard is literal', () => { + expect(select({ name: { $like: '100\\%%' } })).toEqual(['3']); + expect(select({ name: { $like: 'a\\_b' } })).toEqual(['5']); + }); + + it('a row with no value never matches', () => { + expect(select({ name: { $like: '%' } })).toEqual(['1', '2', '3', '4', '5', '6']); + }); + + it('a dangling escape, a NUL and a non-string pattern are refused INVALID_FILTER / 400', () => { + for (const pattern of ['abc\\', 'a\u0000b', 42]) { + const err = refusal(() => select({ name: { $like: pattern } })); + expect(err.code, String(pattern)).toBe('INVALID_FILTER'); + expect(err.status).toBe(400); + expect(err.message).toContain('$like'); + } + }); + + it('`having` still refuses both as outside its vocabulary', () => { + for (const op of ['$like', '$ilike']) { + const err = refusal(() => applyHaving([{ n: 'x' }], { n: { [op]: 'x%' } } as any)); + expect(err.code, op).toBe('INVALID_FILTER'); + expect(err.message).toContain(op); + } + }); +}); diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 50a504332d7..f4b46501001 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -366,6 +366,13 @@ import { // [#21777] "Is this schema the remote's?" One predicate, shared with the boot sync. // [#21910, #21918] And its injected-column refinement, which both cascade walks ask. import { isFederatedObject, isFederatedUnprovisionedInjectedColumn } from './federated-object.js'; +// [#27277] The correctness floor over a plugin driver on an external datasource. +import { + createExternalDriverFloor, + isExternalDriverFloor, + needsExternalDriverFloor, + unwrapExternalDriverFloor, +} from './external-driver-floor.js'; // [#25522 cut 14] The master-detail landing, the preview's parents and the // related reads a validation rule makes; the methods of the same names below // hand each body the members it reads (`MasterDetailRelatedHost`). Imported after @@ -4519,6 +4526,12 @@ interface TransactionScope { */ export interface DatasourceDef { name: string; + /** + * [#27277] The driver id the datasource selects. Read to decide whether the + * engine floors the datasource's driver (`needsExternalDriverFloor`: a + * non-managed datasource on a driver id that is not built in). + */ + driver?: string; schemaMode?: string; external?: { /** Datasource-wide write gate — ADR-0015 §5.3 Gate 3. */ @@ -9056,7 +9069,8 @@ export class ObjectQL implements IObjectQLEngine { registerDriver(driver: IDataDriver, isDefault: boolean = false) { const existing = this.drivers.get(driver.name); if (existing) { - if (existing !== driver) { + // [#27277] The entry may be the floored instance of this very driver. + if (unwrapExternalDriverFloor(existing) !== unwrapExternalDriverFloor(driver)) { this.logger.warn( 'Driver name collision — KEEPING the already-registered driver and DISCARDING the one just supplied. ' + 'Two different driver instances claim one name, so whatever configuration the discarded instance carried ' + @@ -9087,6 +9101,7 @@ export class ObjectQL implements IObjectQLEngine { this.drivers.set(driver.name, driver); this.supplyFileColumnsMovedResolver(driver); this.supplyTenantKeyCertifiedResolver(driver); + this.applyExternalDriverFloor(driver.name); this.logger.info('Registered driver', { driverName: driver.name, version: driver.version @@ -9229,7 +9244,38 @@ export class ObjectQL implements IObjectQLEngine { */ registerDatasourceDef(def: DatasourceDef): void { if (!def?.name) return; - this.datasourceDefs.set(def.name, { schemaMode: def.schemaMode, external: def.external }); + this.datasourceDefs.set(def.name, { driver: def.driver, schemaMode: def.schemaMode, external: def.external }); + this.applyExternalDriverFloor(def.name); + } + + /** + * [#27277] Put the registered driver of datasource `name` behind the engine's + * correctness floor when its definition calls for one, or take it back out + * when a later definition no longer does (`external-driver-floor.ts`). + * + * Called from both doors — {@link registerDriver} and + * {@link registerDatasourceDef} — because either can come first. The floored + * driver REPLACES the entry in `drivers`, once, so every reader of the map + * (`getDriver`, `getDriverByName`, the default-driver and transaction-scope + * identity checks) holds one instance. + */ + private applyExternalDriverFloor(name: string): void { + const current = this.drivers.get(name); + if (!current) return; + const floored = isExternalDriverFloor(current); + if (needsExternalDriverFloor(this.datasourceDefs.get(name))) { + if (floored) return; + this.drivers.set(name, createExternalDriverFloor(current, { + datasource: name, + fieldsOf: (object) => this._registry.getObject(object)?.fields, + logger: this.logger, + })); + this.logger.info('Datasource served by a plugin driver: the engine applies each query in memory', { + datasource: name, + }); + } else if (floored) { + this.drivers.set(name, unwrapExternalDriverFloor(current)); + } } /** diff --git a/packages/objectql/src/external-driver-errors.ts b/packages/objectql/src/external-driver-errors.ts new file mode 100644 index 00000000000..333f3461912 --- /dev/null +++ b/packages/objectql/src/external-driver-errors.ts @@ -0,0 +1,99 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27277] The two refusals the engine's floor over a plugin driver raises + * (`external-driver-floor.ts`, ADR-0015 Tier A, epic #27261 P1). + * + * Both are 4xx: the caller can change the request and try again, and the + * message is theirs to read. That is what keeps them apart from the third + * outcome of the same floor, a remote that failed or did not answer, which is + * `ERR_DATASOURCE_UNAVAILABLE` / 503 (`DatasourceUnavailableError`, reason + * `remote`) and withholds the remote's text by the 5xx rule. + * + * Each declares its `status` on the thrown error (the convergent HTTP-exit + * convention `driver-connect-errors.ts` and `concurrent-update-error.ts` + * follow), so every door answers the same status and keeps the `code`. + * ⛔ Neither is an ADR-0015 gate refusal: `EXTERNAL_WRITE_FORBIDDEN` (403) is + * the platform's own write switch being off, fixed by turning it on; these are + * fixed by changing the payload or the filter. They are registered under this + * package in `ERROR_CODE_LEDGER`, beside the gate codes' own owner. + */ + +/** The `code` {@link ExternalWriteRefusedError} carries. */ +export const EXTERNAL_WRITE_REFUSED_CODE = 'EXTERNAL_WRITE_REFUSED' as const; + +/** The `code` {@link ExternalRowCapExceededError} carries. */ +export const EXTERNAL_ROW_CAP_EXCEEDED_CODE = 'EXTERNAL_ROW_CAP_EXCEEDED' as const; + +/** + * The most rows the engine reads from a plugin driver for one request, and so + * the most matching rows such a request can be answered over. Fixed in P1 — + * not an authorable key (maintainer ruling 2 on #27261, comment 6472620372). + */ +export const EXTERNAL_ROW_CAP = 1000; + +/** The write a refusal names. */ +export type ExternalWriteOperation = + | 'create' | 'update' | 'upsert' | 'delete' + | 'bulkCreate' | 'bulkUpdate' | 'bulkDelete' + | 'updateMany' | 'deleteMany'; + +/** + * The remote system behind a plugin driver refused a write with a 4xx answer — + * its validation failed, the record is locked, it does not exist there. 422: + * the request reached the remote and was judged, and a changed payload may + * pass. The message is the remote's own, so the person who typed the value + * reads why it was refused. + */ +export class ExternalWriteRefusedError extends Error { + readonly code = EXTERNAL_WRITE_REFUSED_CODE; + readonly status = 422; + readonly httpStatus = 422; + readonly details: { + datasource: string; + object: string; + operation: ExternalWriteOperation; + /** The status the remote answered with, as the driver reported it. */ + remoteStatus: number; + }; + + constructor( + datasource: string, + object: string, + operation: ExternalWriteOperation, + remoteStatus: number, + remoteMessage: string, + options?: { cause?: unknown }, + ) { + const said = remoteMessage.trim() || `it answered ${remoteStatus} with no message`; + super(`The system behind datasource '${datasource}' refused this ${operation} on '${object}': ${said}`); + this.name = 'ExternalWriteRefusedError'; + if (options && 'cause' in options) (this as Error & { cause?: unknown }).cause = options.cause; + this.details = { datasource, object, operation, remoteStatus }; + } +} + +/** + * A read on an object served by a plugin driver matched more rows than the + * engine reads from such a driver ({@link EXTERNAL_ROW_CAP}). The engine + * applies the query itself over the rows the driver returns, so it cannot + * answer correctly over a set it only partly holds, and refuses instead of + * returning a short or wrongly ordered list. 422 with the prescription: narrow + * the filter, or mirror the data into a real object with DataSync. + */ +export class ExternalRowCapExceededError extends Error { + readonly code = EXTERNAL_ROW_CAP_EXCEEDED_CODE; + readonly status = 422; + readonly httpStatus = 422; + readonly details: { datasource: string; object: string; cap: number }; + + constructor(datasource: string, object: string, cap: number = EXTERNAL_ROW_CAP) { + super( + `This read on '${object}' matched more than ${cap} rows from datasource '${datasource}', the most the ` + + `engine reads from a plugin driver for one request. Narrow the filter so fewer than ${cap + 1} rows ` + + `match, or mirror the data into a real object with DataSync and read that instead.`, + ); + this.name = 'ExternalRowCapExceededError'; + this.details = { datasource, object, cap }; + } +} diff --git a/packages/objectql/src/external-driver-floor.test.ts b/packages/objectql/src/external-driver-floor.test.ts new file mode 100644 index 00000000000..c975471137f --- /dev/null +++ b/packages/objectql/src/external-driver-floor.test.ts @@ -0,0 +1,269 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27277] The engine's floor over a plugin driver on an external datasource + * (`external-driver-floor.ts`), driven through the real engine. + * + * The driver under test is the worst honest plugin driver: it pushes NOTHING + * down — no filter, no sort — and only stops reading at the `limit` it is + * sent, the way a remote API's page size does. Every read must still answer + * exactly the rows the query asks for, or refuse. + */ + +import { describe, it, expect } from 'vitest'; +import type { IDataDriver } from '@objectstack/spec/contracts'; +import { ObjectQL } from './engine'; +import { EXTERNAL_ROW_CAP, ExternalRowCapExceededError, ExternalWriteRefusedError } from './external-driver-errors.js'; +import { DatasourceUnavailableError } from './driver-connect-errors.js'; +import { isExternalDriverFloor, needsExternalDriverFloor, unwrapExternalDriverFloor } from './external-driver-floor.js'; + +type Row = Record; + +interface RemoteDouble { + driver: IDataDriver; + /** Every query `find` received, as received. */ + sent: Row[]; +} + +/** A plugin driver over `rows` that applies no part of a query but its `limit`. */ +function remoteDriver(name: string, rows: Row[], extra: Record = {}): RemoteDouble { + const sent: Row[] = []; + const driver = { + name, + version: '0.0.1', + supports: { transactionsUnsupported: true }, + async connect() {}, + async disconnect() {}, + async find(_object: string, query: Row = {}) { + sent.push({ ...query }); + const page = rows.map((r) => ({ ...r })); + return typeof query.limit === 'number' ? page.slice(0, query.limit) : page; + }, + async findOne() { throw new Error('the floor answers findOne itself'); }, + async count() { throw new Error('the floor answers count itself'); }, + async aggregate() { throw new Error('the floor hides aggregate'); }, + async syncSchema() {}, + ...extra, + } as unknown as IDataDriver; + return { driver, sent }; +} + +const TREE_ROWS: Row[] = [ + { id: 'n1', kind: 'leaf', label: 'Echo', weight: 3 }, + { id: 'n2', kind: 'branch', label: 'Alpha', weight: 1 }, + { id: 'n3', kind: 'leaf', label: 'Charlie', weight: 2 }, + { id: 'n4', kind: 'leaf', label: 'Bravo', weight: null }, + { id: 'n5', kind: 'branch', label: 'Delta', weight: 5 }, + { id: 'n6', kind: 'leaf', label: 'Foxtrot', weight: 2 }, +]; + +function makeEngine(opts: { + rows?: Row[]; + driverId?: string; + schemaMode?: 'managed' | 'external' | 'validate-only'; + extra?: Record; + order?: 'driver-first' | 'def-first'; +} = {}) { + const engine = new ObjectQL(); + const remote = remoteDriver('tree', opts.rows ?? TREE_ROWS, opts.extra); + const def = { + name: 'tree', + driver: opts.driverId ?? 'spike_tree', + schemaMode: opts.schemaMode ?? 'external', + external: { allowWrites: true }, + }; + if (opts.order === 'def-first') { + engine.registerDatasourceDef(def); + engine.registerDriver(remote.driver); + } else { + engine.registerDriver(remote.driver); + engine.registerDatasourceDef(def); + } + engine.registerApp({ + id: 'tree_pkg', + name: 'Tree', + objects: [{ + name: 'tree_node', + datasource: 'tree', + external: { remoteName: 'nodes', writable: true }, + fields: { kind: { type: 'text' }, label: { type: 'text' }, weight: { type: 'number' } }, + }], + } as any); + return { engine, remote }; +} + +const ids = (rows: Row[]) => rows.map((r) => r.id); + +describe('[#27277] which datasources get the floor', () => { + it('a non-managed datasource on a driver id that is not built in', () => { + expect(needsExternalDriverFloor({ driver: 'spike_tree', schemaMode: 'external' })).toBe(true); + expect(needsExternalDriverFloor({ driver: 'spike_tree', schemaMode: 'validate-only' })).toBe(true); + }); + + it('never a managed datasource, a built-in driver (federated SQL), or a def that names no driver', () => { + expect(needsExternalDriverFloor({ driver: 'spike_tree', schemaMode: 'managed' })).toBe(false); + expect(needsExternalDriverFloor({ driver: 'spike_tree' })).toBe(false); + for (const builtin of ['memory', 'sqlite', 'postgres', 'mysql', 'mongodb', 'turso']) { + expect(needsExternalDriverFloor({ driver: builtin, schemaMode: 'external' }), builtin).toBe(false); + } + expect(needsExternalDriverFloor({ schemaMode: 'external' })).toBe(false); + expect(needsExternalDriverFloor(undefined)).toBe(false); + }); + + it('a built-in driver on an external datasource is the registered instance itself — zero change', () => { + const { engine, remote } = makeEngine({ driverId: 'sqlite' }); + expect(engine.getDriverByName('tree')).toBe(remote.driver); + }); + + it('a managed datasource is the registered instance itself', () => { + const { engine, remote } = makeEngine({ schemaMode: 'managed' }); + expect(engine.getDriverByName('tree')).toBe(remote.driver); + }); + + it('either registration order floors the driver, and a managed redefinition takes it back out', () => { + for (const order of ['driver-first', 'def-first'] as const) { + const { engine, remote } = makeEngine({ order }); + const held = engine.getDriverByName('tree')!; + expect(isExternalDriverFloor(held), order).toBe(true); + expect(unwrapExternalDriverFloor(held)).toBe(remote.driver); + engine.registerDatasourceDef({ name: 'tree', driver: 'spike_tree', schemaMode: 'managed' }); + expect(engine.getDriverByName('tree'), order).toBe(remote.driver); + } + }); +}); + +describe('[#27277] one instance', () => { + it('every reader of the drivers map holds the same floored instance', () => { + const { engine } = makeEngine(); + const held = engine.getDriverByName('tree')!; + expect(isExternalDriverFloor(held)).toBe(true); + expect(engine.datasource('tree')).toBe(held); + expect((engine as any).getDriver('tree_node')).toBe(held); + // Re-applying the def does not wrap twice. + engine.registerDatasourceDef({ name: 'tree', driver: 'spike_tree', schemaMode: 'external' }); + expect(engine.getDriverByName('tree')).toBe(held); + }); + + it('re-registering the same driver is not a name collision with its floored instance', () => { + const { engine, remote } = makeEngine(); + const held = engine.getDriverByName('tree'); + expect(() => engine.registerDriver(remote.driver)).not.toThrow(); + expect(engine.getDriverByName('tree')).toBe(held); + }); +}); + +describe('[#27277] reads: the engine applies the query', () => { + it('a driver that pushes nothing down still answers the filtered, sorted, paged, projected rows', async () => { + const { engine, remote } = makeEngine(); + const rows = await engine.find('tree_node', { + where: { kind: 'leaf' }, + orderBy: [{ field: 'weight', order: 'asc' }], + limit: 2, + offset: 1, + fields: ['label'], + }); + // leaves by weight asc, nulls last: n3(2), n6(2), n1(3), n4(null) — page 2 of size 2 from offset 1 + expect(ids(rows)).toEqual(['n6', 'n1']); + expect(rows.map((r) => r.label)).toEqual(['Foxtrot', 'Echo']); + // The driver was sent the filter and sort it may push down, the cap, and nothing else. + const query = remote.sent.at(-1)!; + expect(query.where).toEqual({ kind: 'leaf' }); + expect(query.orderBy).toEqual([{ field: 'weight', order: 'asc' }]); + expect(query.limit).toBe(EXTERNAL_ROW_CAP + 1); + expect(query).not.toHaveProperty('offset'); + expect(query).not.toHaveProperty('fields'); + }); + + it('findOne answers the first matching row under the requested order', async () => { + const { engine } = makeEngine(); + const row = await engine.findOne('tree_node', { where: { kind: 'branch' }, orderBy: [{ field: 'label', order: 'desc' }] }); + expect(row?.id).toBe('n5'); + expect(await engine.findOne('tree_node', { where: { kind: 'none' } })).toBeNull(); + }); + + it('count answers the number of matching rows', async () => { + const { engine } = makeEngine(); + expect(await engine.count('tree_node', { where: { kind: 'leaf' } })).toBe(4); + expect(await engine.count('tree_node')).toBe(TREE_ROWS.length); + }); + + it('aggregate is hidden, so the engine aggregates the floored rows in memory', async () => { + const { engine } = makeEngine(); + expect('aggregate' in engine.getDriverByName('tree')!).toBe(false); + const groups = await engine.aggregate('tree_node', { + where: { kind: 'leaf' }, + groupBy: ['kind'], + aggregations: [{ function: 'count', alias: 'n' }], + }); + expect(groups).toEqual([{ kind: 'leaf', n: 4 }]); + }); + + it(`exactly ${EXTERNAL_ROW_CAP} rows is answered`, async () => { + const rows = Array.from({ length: EXTERNAL_ROW_CAP }, (_, i) => ({ id: `r${i}`, kind: 'leaf' })); + const { engine } = makeEngine({ rows }); + expect(await engine.count('tree_node')).toBe(EXTERNAL_ROW_CAP); + }); + + it(`more than ${EXTERNAL_ROW_CAP} rows is refused 422 with the prescription, on every read`, async () => { + const rows = Array.from({ length: EXTERNAL_ROW_CAP + 5 }, (_, i) => ({ id: `r${i}`, kind: 'leaf' })); + const { engine } = makeEngine({ rows }); + for (const read of [ + () => engine.find('tree_node', { limit: 5 }), + () => engine.findOne('tree_node', { where: { kind: 'leaf' } }), + () => engine.count('tree_node'), + ]) { + const err = await read().then(() => undefined, (e) => e); + expect(err).toBeInstanceOf(ExternalRowCapExceededError); + expect(err.code).toBe('EXTERNAL_ROW_CAP_EXCEEDED'); + expect(err.status).toBe(422); + expect(err.message).toContain('Narrow the filter'); + expect(err.message).toContain('DataSync'); + } + }); + + it('a read the remote fails is ERR_DATASOURCE_UNAVAILABLE / 503, the cause logged and never in the message', async () => { + const { engine } = makeEngine({ + extra: { async find() { throw new Error('ECONNRESET talking to tree.internal:8443'); } }, + }); + const err = await engine.find('tree_node', {}).then(() => undefined, (e) => e); + expect(err).toBeInstanceOf(DatasourceUnavailableError); + expect(err.code).toBe('ERR_DATASOURCE_UNAVAILABLE'); + expect(err.kind).toBe('remote'); + expect(err.message).not.toContain('tree.internal'); + }); +}); + +describe('[#27277] writes pass through; a remote refusal is 422', () => { + it('a write the remote accepts is the remote\'s answer', async () => { + const { engine } = makeEngine({ + extra: { async create(_o: string, data: Row) { return { id: 'n9', ...data }; } }, + }); + const row = await engine.insert('tree_node', { kind: 'leaf', label: 'Golf' }); + expect(row).toMatchObject({ id: 'n9', label: 'Golf' }); + }); + + it('a 4xx from the remote is EXTERNAL_WRITE_REFUSED / 422 carrying the remote\'s message and status', async () => { + const { engine } = makeEngine({ + extra: { + async create() { + throw Object.assign(new Error('label must be unique within a parent'), { status: 409 }); + }, + }, + }); + const err = await engine.insert('tree_node', { kind: 'leaf', label: 'Echo' }).then(() => undefined, (e) => e); + expect(err).toBeInstanceOf(ExternalWriteRefusedError); + expect(err.code).toBe('EXTERNAL_WRITE_REFUSED'); + expect(err.status).toBe(422); + expect(err.message).toContain('label must be unique within a parent'); + expect(err.details).toMatchObject({ datasource: 'tree', object: 'tree_node', operation: 'create', remoteStatus: 409 }); + }); + + it('a write failure with no 4xx status is ERR_DATASOURCE_UNAVAILABLE / 503', async () => { + const { engine } = makeEngine({ + extra: { async create() { throw Object.assign(new Error('upstream exploded'), { statusCode: 502 }); } }, + }); + const err = await engine.insert('tree_node', { kind: 'leaf' }).then(() => undefined, (e) => e); + expect(err).toBeInstanceOf(DatasourceUnavailableError); + expect(err.message).not.toContain('upstream exploded'); + }); +}); diff --git a/packages/objectql/src/external-driver-floor.ts b/packages/objectql/src/external-driver-floor.ts new file mode 100644 index 00000000000..1bce5913ab5 --- /dev/null +++ b/packages/objectql/src/external-driver-floor.ts @@ -0,0 +1,224 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27277] The engine's correctness floor over a plugin driver on an external + * datasource (ADR-0015 Tier A, epic #27261 P1). + * + * ## Which drivers + * + * A datasource gets the floor when BOTH hold ({@link needsExternalDriverFloor}): + * + * - its `schemaMode` is not `managed` — the platform does not own the storage; + * - its `driver` id is not a built-in one (`resolveDriverId` answers nothing, + * so it is not in `BUILTIN_DRIVER_IDS`) — a server plugin published the + * driver as `driver.`. + * + * Built-in drivers keep executing the whole query, as `find()`'s contract + * requires and their conformance suites pin; a managed datasource and an + * ADR-0015 federated database (a built-in SQL driver on `schemaMode: + * 'external'`) are untouched. A plugin driver declares nothing and cannot opt + * out: `DriverCapabilities` carries no pushdown claim (the query bits were + * retired in 17.0.0), so the engine treats every part of the query as + * unclaimed (ruling 6471382887, maintainer decision Q1 = B on #27277). A + * verified-pushdown credential that lets a conformance-tested driver skip the + * floor is the epic's P3 and does not exist yet. + * + * ## What it does + * + * - `find`, `findOne`, `count`: the driver is sent `where` and `orderBy` (it + * may push either down) and `limit` {@link EXTERNAL_ROW_CAP} + 1 — never + * `offset` or `fields`, which the engine applies after it filters. More than + * {@link EXTERNAL_ROW_CAP} rows back ⇒ {@link ExternalRowCapExceededError}; + * otherwise the engine applies the whole query to the rows + * (`applyDriverQueryInMemory`), so a driver that returned every row still + * answers the right subset. `findOne` is that read with `limit: 1`, `count` + * its length. + * - `aggregate` is hidden, so the engine's existing find + in-memory + * aggregation path runs over the floored `find`. + * - Writes pass through. A write the remote refuses with a 4xx is + * {@link ExternalWriteRefusedError} (422, the remote's message); any other + * failure of a read or a write is `ERR_DATASOURCE_UNAVAILABLE` (503, reason + * `remote`), with the cause in the server log and never on the wire. + * - Everything else (`connect`, `supports`, `syncSchema`, `introspectSchema`, + * `beginTransaction`, …) is the driver's own, unchanged. + * + * ## One instance + * + * The engine stores the floored driver in its `drivers` map in place of the + * driver, once ({@link createExternalDriverFloor} is idempotent per driver), so + * every identity comparison the engine makes (`getDriver(object) === + * defaultDriver`, a transaction's scope driver) compares one instance. + * {@link unwrapExternalDriverFloor} answers the driver behind it. + */ + +import type { DriverQuery, IDataDriver } from '@objectstack/spec/contracts'; +import { resolveDriverId } from '@objectstack/spec/data'; +import { applyDriverQueryInMemory, assertDriverWhereEvaluableInMemory } from './driver-query-in-memory.js'; +import { declaredFieldClasses, declaredJsonStoredFields } from './having-filter.js'; +import { DatasourceUnavailableError } from './driver-connect-errors.js'; +import { + EXTERNAL_ROW_CAP, + ExternalRowCapExceededError, + ExternalWriteRefusedError, + type ExternalWriteOperation, +} from './external-driver-errors.js'; + +/** + * Does a datasource with this definition get the floor? `schemaMode` absent + * reads as `managed`, the schema's default. + */ +export function needsExternalDriverFloor(def: { driver?: unknown; schemaMode?: unknown } | undefined): boolean { + if (!def) return false; + if (def.schemaMode === undefined || def.schemaMode === 'managed') return false; + return typeof def.driver === 'string' && resolveDriverId(def.driver) === undefined; +} + +/** What the floor needs from the engine. */ +export interface ExternalDriverFloorHost { + /** The datasource the driver serves — named in every refusal. */ + datasource: string; + /** The object's declared `fields` record, for its comparand classes; `undefined` when unknown. */ + fieldsOf(object: string): unknown; + /** Where a remote failure's cause is written: the wire answer withholds it. */ + logger: { warn(message: string, meta?: Record): void }; +} + +const WRITE_METHODS: ReadonlySet = new Set([ + 'create', 'update', 'upsert', 'delete', + 'bulkCreate', 'bulkUpdate', 'bulkDelete', + 'updateMany', 'deleteMany', +]); + +/** floored driver → the driver behind it */ +const innerOf = new WeakMap(); +/** driver → its floored driver, so wrapping one driver twice answers one instance */ +const floorOf = new WeakMap(); + +/** The driver behind a floored one, or `driver` itself when it is not floored. */ +export function unwrapExternalDriverFloor(driver: IDataDriver): IDataDriver { + return innerOf.get(driver) ?? driver; +} + +/** Is `driver` a floored driver? */ +export function isExternalDriverFloor(driver: unknown): boolean { + return typeof driver === 'object' && driver !== null && innerOf.has(driver); +} + +/** + * The floored driver for `driver` — the same instance on every call for one + * driver. See the module comment. + */ +export function createExternalDriverFloor(driver: IDataDriver, host: ExternalDriverFloorHost): IDataDriver { + const inner = unwrapExternalDriverFloor(driver); + const existing = floorOf.get(inner); + if (existing) return existing; + + const classesOf = (object: string) => { + const fields = host.fieldsOf(object); + return { fieldClasses: declaredFieldClasses(fields), jsonStored: declaredJsonStoredFields(fields) }; + }; + + const remoteFailure = (object: string, operation: string, error: unknown): Error => { + const message = error instanceof Error ? error.message : String(error); + host.logger.warn( + `[ObjectQL] the plugin driver of datasource '${host.datasource}' failed a ${operation} on '${object}'; ` + + 'the caller was answered ERR_DATASOURCE_UNAVAILABLE (503) without this cause', + { datasource: host.datasource, object, operation, error: message }, + ); + const refusal = new DatasourceUnavailableError(host.datasource, object, 'remote'); + (refusal as Error & { cause?: unknown }).cause = error; + return refusal; + }; + + const readCapped = async (object: string, query: DriverQuery | undefined, options: unknown) => { + assertDriverWhereEvaluableInMemory(query?.where); + const routed = (query as { object?: unknown } | undefined)?.object; + const sent = { + ...(routed !== undefined ? { object: routed } : {}), + ...(query?.where !== undefined ? { where: query.where } : {}), + ...(query?.orderBy !== undefined ? { orderBy: query.orderBy } : {}), + limit: EXTERNAL_ROW_CAP + 1, + } as DriverQuery; + let rows: unknown; + try { + rows = await inner.find(object, sent, options as Parameters[2]); + } catch (error) { + throw remoteFailure(object, 'find', error); + } + if (!Array.isArray(rows)) { + throw remoteFailure(object, 'find', new Error(`find answered ${typeof rows}, not an array of rows`)); + } + if (rows.length > EXTERNAL_ROW_CAP) throw new ExternalRowCapExceededError(host.datasource, object); + return rows as Record[]; + }; + + const find = async (object: string, query: DriverQuery = {}, options?: unknown) => + applyDriverQueryInMemory(await readCapped(object, query, options), withoutRouting(query), classesOf(object)); + + const findOne = async (object: string, query: DriverQuery = {}, options?: unknown) => + applyDriverQueryInMemory( + await readCapped(object, query, options), { ...withoutRouting(query), limit: 1 }, classesOf(object), + )[0] ?? null; + + const count = async (object: string, query: DriverQuery = {}, options?: unknown) => + applyDriverQueryInMemory( + await readCapped(object, query, options), + query?.where !== undefined ? { where: query.where } : {}, + classesOf(object), + ).length; + + const write = (operation: string, method: (...args: unknown[]) => Promise) => + async (object: string, ...rest: unknown[]) => { + try { + return await method.call(inner, object, ...rest); + } catch (error) { + const status = declaredStatus(error); + if (status !== undefined && status >= 400 && status < 500) { + throw new ExternalWriteRefusedError( + host.datasource, object, operation as ExternalWriteOperation, status, + error instanceof Error ? error.message : String(error), { cause: error }, + ); + } + throw remoteFailure(object, operation, error); + } + }; + + const bound = new Map(); + const floored = new Proxy(inner, { + get(target, prop) { + if (prop === 'find') return find; + if (prop === 'findOne') return findOne; + if (prop === 'count') return count; + if (prop === 'aggregate') return undefined; + const value = Reflect.get(target, prop, target); + if (typeof value !== 'function') return value; + if (!bound.has(prop)) { + bound.set(prop, typeof prop === 'string' && WRITE_METHODS.has(prop) + ? write(prop, value as (...args: unknown[]) => Promise) + : (value as (...args: unknown[]) => unknown).bind(target)); + } + return bound.get(prop); + }, + has(target, prop) { + if (prop === 'aggregate') return false; + return Reflect.has(target, prop); + }, + }) as IDataDriver; + + innerOf.set(floored, inner); + floorOf.set(inner, floored); + return floored; +} + +/** The query as the in-memory helper reads it: without the call's routing (`object`) and the engine's own `expand`. */ +function withoutRouting(query: DriverQuery): DriverQuery { + const { object: _object, expand: _expand, ...rest } = query as DriverQuery & { object?: unknown; expand?: unknown }; + return rest as DriverQuery; +} + +/** The HTTP status a thrown error declares, read as the repo's HTTP exits read it: `status`, then `statusCode`. */ +function declaredStatus(error: unknown): number | undefined { + const { status, statusCode } = (error ?? {}) as { status?: unknown; statusCode?: unknown }; + const value = typeof status === 'number' ? status : typeof statusCode === 'number' ? statusCode : undefined; + return value !== undefined && Number.isInteger(value) && value >= 100 && value < 600 ? value : undefined; +} diff --git a/packages/objectql/src/having-filter.ts b/packages/objectql/src/having-filter.ts index f46fee53b29..8b748f087c8 100644 --- a/packages/objectql/src/having-filter.ts +++ b/packages/objectql/src/having-filter.ts @@ -210,6 +210,10 @@ import { utcInstantMs } from '@objectstack/spec/data'; // [#20444] `$empty`'s value-level half — the spec's one definition of what a // stored value counts as empty for a face that judges by value. import { isEmptyFilterValue } from '@objectstack/spec/data'; +// [#27277] `$like` / `$ilike`'s pattern language — the spec's one compilation, +// the one `driver-memory` and `formula` evaluate and `driver-sql` hands to +// `LIKE` / `GLOB` — read by the driver-where position only. +import { hasDanglingLikeEscape, hasNulInLikePattern, likePatternToRegExp } from '@objectstack/spec/data'; // [#20176] The storage rule a temporal column puts a value in — ONE function, // shared with `driver-sql`'s and `driver-memory`'s `where`. import { temporalStorageForm, type TemporalComparandKind } from '@objectstack/core'; @@ -248,11 +252,36 @@ interface FilterClause { root: string; /** One-sentence semantics printed before the supported-operator list. */ semantics: string; + /** [#27277] What ignoring an unknown operator would silently return, in the refusal's last sentence. */ + ignoring: string; + /** + * [#27277] Whether the pattern pair `$like` / `$ilike` is evaluated here. Only + * the driver-where position takes it: a `where` carries it to every driver + * (#7536), while `having` and the per-aggregation `filter` never declared it. + */ + patterns?: true; } const HAVING_CLAUSE: FilterClause = { root: 'having', semantics: 'HAVING filters the aggregated rows (aggregation aliases + groupBy projections)', + ignoring: 'return unfiltered aggregates', +}; + +/** + * [#27277] The driver-where position: a `where` the engine evaluates over the + * rows a plugin driver returned (`applyDriverQueryInMemory`, the ADR-0015 Tier + * A correctness floor). The rows are the object's own flat columns, as for the + * per-aggregation `filter`, and the vocabulary is the full `where` one — the + * pattern pair included. + */ +const DRIVER_WHERE_CLAUSE: FilterClause = { + root: 'where', + semantics: + 'The engine evaluates this `where` over the rows a plugin driver returned ' + + '(the object\'s own columns, the `where` operator vocabulary)', + ignoring: 'return rows the filter excludes', + patterns: true, }; /** @@ -266,6 +295,7 @@ export function aggregationFilterClause(index: number): FilterClause { semantics: 'A per-aggregation `filter` narrows the SOURCE rows that one aggregation reads ' + '(raw column namespace, the `where` operator vocabulary)', + ignoring: 'return unfiltered aggregates', }; } @@ -294,6 +324,9 @@ const CONDITION_OPERATORS = [ '$empty', ] as const; +/** [#27277] The pattern pair, evaluated at the driver-where position only ({@link FilterClause.patterns}). */ +const PATTERN_OPERATORS = ['$like', '$ilike'] as const; + /** * [#20099] The operators whose WHOLE comparand may be a `{ $field }` reference — * the six scalar comparisons `FieldReferenceSchema` names, and the only @@ -407,11 +440,11 @@ function unknownOperator( } const supported = where === 'logical' ? `${LOGICAL_OPERATORS.join(', ')} (or a column condition)` - : CONDITION_OPERATORS.join(', '); + : [...CONDITION_OPERATORS, ...(clause.patterns ? PATTERN_OPERATORS : [])].join(', '); return invalidFilterError( `Unsupported operator '${op}' in \`${clause.root}\`. ${clause.semantics} and supports: ${supported}. ` + `An unknown operator is refused rather than ignored — ignoring it would silently ` - + `return unfiltered aggregates (ADR-0078).`, + + `${clause.ignoring} (ADR-0078).`, ); } @@ -581,6 +614,30 @@ function icontainsComparandError(field: string, value: unknown, path: string): E * and `parseFilterAST` lowers it there — so an author carrying it over from a * `where` is told which slot takes it rather than that it is malformed. */ +/** + * [#27277] What is wrong with a `$like` / `$ilike` comparand, or `undefined`. + * The three shapes `driver-memory` and `driver-sql` refuse: a non-string, a + * trailing unpaired backslash, a U+0000. + */ +function likePatternViolation(pattern: unknown): string | undefined { + if (typeof pattern !== 'string') { + return `requires a string comparand, received ${preview(pattern)}. A pattern's "%" matches any ` + + `sequence, "_" one character, and a backslash escapes either, so a non-string comparand cannot ` + + `be coerced without inventing wildcards the caller never wrote. For a literal substring search ` + + `write "$contains", whose comparand IS text.`; + } + if (hasDanglingLikeEscape(pattern)) { + return `has a pattern ending in a lone unpaired backslash (${JSON.stringify(pattern)}). A backslash ` + + `escapes the character after it, so a trailing one escapes nothing. Write two backslashes to ` + + `match a literal backslash, or drop the trailing one.`; + } + if (hasNulInLikePattern(pattern)) { + return `has a pattern holding the NUL character U+0000, which no backend reads portably. Remove it ` + + `from the pattern.`; + } + return undefined; +} + function havingNotAConditionError(having: unknown): Error { const shape = 'a filter condition OBJECT over the aggregated rows — { "ALIAS": { "$gt": 100 } }'; if (Array.isArray(having)) { @@ -2018,6 +2075,32 @@ export function matchesHaving( return true; } +/** + * [#27277] Evaluate one row a plugin driver returned against the `where` the + * engine sent it — the driver-where position of this walker, read by + * `applyDriverQueryInMemory`. + * + * The SAME walker as `having` and the per-aggregation `filter`, for their + * reason: one vocabulary, one ADR-0112 envelope, the #5298 null-safe + * negations, and the column's storage rule for a temporal comparand. Only the + * wording and the pattern pair differ ({@link DRIVER_WHERE_CLAUSE}). The `where` + * arrives already judged and lowered by the engine's doors; the per-row + * refusals here are the floor for a caller that evaluates rows directly. + * + * `classes` and `jsonStored` are the object's declared field classes and + * JSON-stored fields ({@link declaredFieldClasses}, + * {@link declaredJsonStoredFields}); absent ⇒ every comparand as written and the + * substring reading of `$contains`. + */ +export function matchesDriverWhere( + row: Record, + where: FilterCondition | null | undefined, + classes?: ReadonlyMap, + jsonStored?: ReadonlySet, +): boolean { + return matchesHaving(row, where, DRIVER_WHERE_CLAUSE.root, DRIVER_WHERE_CLAUSE, classes, jsonStored); +} + /** * [#10576] Evaluate one raw source row against one `aggregations[i].filter` * predicate — the per-aggregation filter of `AggregationNodeSchema` (the @@ -2245,6 +2328,12 @@ function checkCondition( if ((op === '$exists' || op === '$null') && typeof target !== 'boolean') { throw nonBooleanFlagComparandError(op, field, target, `${path}.${op}`); } + // [#27277] …and the pattern pair's, where the clause evaluates it. Off that + // position the pair is an unknown operator and `default:` refuses it. + if ((op === '$like' || op === '$ilike') && clause.patterns) { + const violation = likePatternViolation(target); + if (violation) throw invalidFilterError(`Operator "${op}" on field "${field}" at ${path}.${op} ${violation}`); + } // [#21007] A scalar comparison on a declared JSON-stored column — refused, // as `where` refuses it. The backstop under the one-time judgment // (assertAggregationFilterSparesJsonStoredFields), for a row that gets here. @@ -2384,6 +2473,16 @@ function checkCondition( if (typeof value !== 'string' || !asciiCaseInsensitiveContains(value, target)) return false; break; + // [#27277] `$like` / `$ilike` — the caller's OWN pattern, anchored to the + // whole value, compiled by the spec (`likePatternToRegExp`: `u` flag, the + // `$ilike` fold ASCII-only and in the source). Not `escapeRegex`d — a `%` + // here is the caller's wildcard (#7536). A value that is not text matches + // no pattern, as `$contains` answers it. + case '$like': + case '$ilike': + if (!clause.patterns) throw unknownOperator(op, 'condition', keys, clause); + if (typeof value !== 'string' || !likePatternToRegExp(target, op === '$ilike').test(value)) return false; + break; // [#5702] The `$regex` arm is GONE. It built `new RegExp(target, $options)` // and, on an illegal pattern, `return false` — an unrunnable filter // answered as "this row does not match", i.e. a silent empty result rather diff --git a/packages/objectql/src/index.ts b/packages/objectql/src/index.ts index f27fbd8524d..52b5036b8d1 100644 --- a/packages/objectql/src/index.ts +++ b/packages/objectql/src/index.ts @@ -295,9 +295,24 @@ export type { DriverHealth, DatasourceUnavailableInfo, DatasourceUnavailableKind, + DatasourceUnavailableReason, PrimaryDatasourceVerdict, PrimaryDatasourceUnresolvedReason, } from './driver-connect-errors.js'; +// [#27277] The engine's floor over a plugin driver on an external datasource +// (ADR-0015 Tier A): the in-memory reading of a driver query it applies, and +// the two 422 refusals it raises (registered in ERROR_CODE_LEDGER). +export { applyDriverQueryInMemory } from './driver-query-in-memory.js'; +export type { DriverQueryInMemory, DriverQueryInMemoryOptions } from './driver-query-in-memory.js'; +export { + ExternalWriteRefusedError, + ExternalRowCapExceededError, + EXTERNAL_WRITE_REFUSED_CODE, + EXTERNAL_ROW_CAP_EXCEEDED_CODE, + EXTERNAL_ROW_CAP, +} from './external-driver-errors.js'; +export type { ExternalWriteOperation } from './external-driver-errors.js'; +export { needsExternalDriverFloor, unwrapExternalDriverFloor } from './external-driver-floor.js'; export type { InsertManyRowOutcome } from './engine.js'; // [#5696] Thrown by `transaction(cb, base, { require: true })` when the // datasource cannot give a real transaction. Exported so a caller that fails diff --git a/packages/runtime/src/plugin-driver-external-object.spike.test.ts b/packages/runtime/src/plugin-driver-external-object.spike.test.ts index c3145d9f146..d185f786965 100644 --- a/packages/runtime/src/plugin-driver-external-object.spike.test.ts +++ b/packages/runtime/src/plugin-driver-external-object.spike.test.ts @@ -25,13 +25,20 @@ * datasources: `DatasourceAdminServicePlugin` (the `datasource-connection` * service AppPlugin auto-connects through), `ExternalDatasourceServicePlugin` * and the external-validation plugin. The boot is paid in `beforeAll`. + * + * [#27277] Since P1 the engine holds this driver behind its correctness floor + * (`external-driver-floor.ts` in objectql): a plugin driver on a non-managed + * datasource is sent `where` and `orderBy` with `limit` EXTERNAL_ROW_CAP + 1, + * and the engine applies the whole query to the rows it returns. `findOne` and + * `count` are answered from that same read. The steps below pin what the + * remote is asked under the floor. */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import { ObjectKernel, type Plugin, type PluginContext } from '@objectstack/core'; import { defineStack, defineView } from '@objectstack/spec'; import type { DriverQuery, IDataDriver, IExternalDatasourceService } from '@objectstack/spec/contracts'; -import { ObjectQLPlugin } from '@objectstack/objectql'; +import { ObjectQLPlugin, EXTERNAL_ROW_CAP, unwrapExternalDriverFloor } from '@objectstack/objectql'; import { HonoServerPlugin } from '@objectstack/plugin-hono-server'; import { createRestApiPlugin } from '@objectstack/rest'; import { AuthPlugin } from '@objectstack/plugin-auth'; @@ -464,8 +471,11 @@ afterAll(async () => { describe('(a) defineStack accepts a datasource whose driver is a plugin-published `driver.` service', () => { it('the datasource names the driver by the service suffix, and the engine routes the object to that instance', () => { expect(STACK.datasources?.[0]).toMatchObject({ name: DATASOURCE, driver: DATASOURCE, config: {} }); - expect(engine.getDriverByName(DATASOURCE)).toBe(driver); - expect(engine.getDriverForObject(OBJECT)).toBe(driver); + // [#27277] Behind the floor: one floored instance, wrapping this driver. + const held = engine.getDriverByName(DATASOURCE)!; + expect(held).not.toBe(driver); + expect(unwrapExternalDriverFloor(held)).toBe(driver); + expect(engine.getDriverForObject(OBJECT)).toBe(held); }); it('auto-connect finds the driver already registered and leaves it alone; the boot connects it once', () => { @@ -491,7 +501,7 @@ describe('(b) the stack boots with `checkOnBoot: false` and no `introspectSchema }); describe('(c) GET /data/spike_node with the view\'s filter and sort reaches `driver.find` with a DriverQuery', () => { - it('the view\'s filter arrives as `where`, its sort as `orderBy`, `top` as `limit`; no projection, no offset', async () => { + it('the view\'s filter arrives as `where`, its sort as `orderBy`; `top` is the engine\'s, the remote is asked for the cap', async () => { // The plan view's filter as the console lowers it (`toFilterNode`), its // sort as `serializeOrderBy` writes it, and the console's page size. const plan = (STACK.views?.[0] as any)?.listViews?.plan; @@ -503,18 +513,23 @@ describe('(c) GET /data/spike_node with the view\'s filter and sort reaches `dri const body = await readJson(res); expect(res.status, JSON.stringify(body)).toBe(200); - // Exactly what the remote system is asked — the whole DriverQuery, - // `object` included (the engine spreads it in although `DriverQuery` - // omits it), then a second round trip for the total. + // Exactly what the remote system is asked — `where` and `orderBy`, which + // it may push down, `object` (the engine spreads it in although + // `DriverQuery` omits it), and the floor's cap in place of `top`; then a + // second round trip for the total, which the floor answers from a read + // of the same shape (#27277). expect(calls.slice(mark)).toEqual([ { method: 'find', object: OBJECT, - query: { where: { archived: false }, orderBy: [{ field: 'start_date', order: 'asc' }], limit: 50, object: OBJECT }, + query: { where: { archived: false }, orderBy: [{ field: 'start_date', order: 'asc' }], limit: EXTERNAL_ROW_CAP + 1, object: OBJECT }, }, - { method: 'count', object: OBJECT, query: { object: OBJECT, where: { archived: false } } }, + { method: 'find', object: OBJECT, query: { object: OBJECT, where: { archived: false }, limit: EXTERNAL_ROW_CAP + 1 } }, ]); - expect((body.records as Row[]).map((r) => r.id)).toEqual(['n_root', 'n_cut', 'n_weld']); + // `n_root` and `n_cut` tie on `start_date`. The engine now orders the + // page itself, and a paged read breaks a tie by `id` (the partition rule + // `driver-sql` follows), so `n_cut` comes first. + expect((body.records as Row[]).map((r) => r.id)).toEqual(['n_cut', 'n_root', 'n_weld']); }); }); @@ -546,7 +561,8 @@ describe('(e) PATCH /data/spike_node/:id runs `beforeUpdate` first, then `driver }); // The hook read the pre-image through the driver, and nothing else. expect(calls.slice(mark).map((c) => c.method)).not.toContain('update'); - expect(callsSince(mark, 'findOne').length).toBeGreaterThanOrEqual(1); + // [#27277] The floor answers `findOne` from a capped `find`. + expect(callsSince(mark, 'find').length).toBeGreaterThanOrEqual(1); expect(rows.find((r) => r.id === 'n_weld')).toMatchObject({ start_date: '2026-11-09', end_date: '2026-11-13' }); }); @@ -579,11 +595,13 @@ describe('(e) PATCH /data/spike_node/:id runs `beforeUpdate` first, then `driver expect(seen.map((c) => c.method)).not.toContain('update'); // What the remote is asked for: `owner_id` (the sharing check) and // `created_by == current_user.id` (the platform ownership floor). - expect(seen).toContainEqual({ method: 'find', object: OWNED, query: { where: { id: 'o_1' }, fields: ['id', 'owner_id'], limit: 1, object: OWNED } }); + // [#27277] Under the floor the remote sees each as a capped `find`; the + // engine applies the projection and `limit: 1` itself. + expect(seen).toContainEqual({ method: 'find', object: OWNED, query: { where: { id: 'o_1' }, limit: EXTERNAL_ROW_CAP + 1, object: OWNED } }); expect(seen).toContainEqual({ - method: 'findOne', + method: 'find', object: OWNED, - query: { where: { $and: [{ id: 'o_1' }, { created_by: plannerId }] }, object: OWNED, limit: 1 }, + query: { where: { $and: [{ id: 'o_1' }, { created_by: plannerId }] }, object: OWNED, limit: EXTERNAL_ROW_CAP + 1 }, }); expect(owned[0]).toEqual({ id: 'o_1', name: 'Remote row' }); }); @@ -607,8 +625,9 @@ describe('(g) object permission sets apply to the external object', () => { expect(body?.code).toBe('PERMISSION_DENIED'); // The write door loads the record before it refuses — on a plugin driver // that is a round trip to the remote on behalf of a caller who may not read. + // [#27277] The floor answers that `findOne` from one capped `find`. expect(calls.slice(mark)).toEqual([ - { method: 'findOne', object: OBJECT, query: { where: { id: 'n_cut' }, object: OBJECT, limit: 1 } }, + { method: 'find', object: OBJECT, query: { where: { id: 'n_cut' }, object: OBJECT, limit: EXTERNAL_ROW_CAP + 1 } }, ]); expect(rows.find((r) => r.id === 'n_cut')?.name).toBe('Cut'); }); From 4d04d12e29354170282ea06eae4058824a018c72 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 04:35:44 +0000 Subject: [PATCH 2/6] feat(spec): register EXTERNAL_WRITE_REFUSED and EXTERNAL_ROW_CAP_EXCEEDED under @objectstack/objectql The two 422 refusals of the engine's floor over a plugin driver on an external datasource (ADR-0015 Tier A). Neither is an ADR-0015 gate refusal: EXTERNAL_WRITE_FORBIDDEN (403) stays the platform's own write switch. The closed ApiErrorSchema.code vocabulary gains both members; the generated references are re-recorded (gen:schema + gen:docs). The api surface is unchanged (check:api-surface). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2dgsoHnkCiSyXF8isz --- .../docs/references/api/automation-api.mdx | 4 +++- content/docs/references/api/contract.mdx | 4 +++- .../docs/references/api/error-code-ledger.mdx | 2 ++ .../spec/src/api/error-code-ledger.zod.ts | 19 +++++++++++++++++++ 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/content/docs/references/api/automation-api.mdx b/content/docs/references/api/automation-api.mdx index 46ead571b7f..c3acf40ca8a 100644 --- a/content/docs/references/api/automation-api.mdx +++ b/content/docs/references/api/automation-api.mdx @@ -347,7 +347,7 @@ const result = AutomationApiErrorCode.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +347 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +349 more>` | ✅ | The inner refusal's code in its WIRE spelling - the ADR-0112 member a record save of the same write answers in `error.code` (e.g. `UNIQUE_VIOLATION`); the closed set `ApiErrorSchema.code` parses against | | **message** | `string` | ✅ | The inner refusal's own diagnostic text, verbatim, without the node and operation prefixes the run's failure text adds | ### Allowed Values: `FlowInnerRefusal.code` @@ -496,9 +496,11 @@ const result = AutomationApiErrorCode.parse(data); * `EXPORT_NOT_PERMITTED` * `EXTERNAL_DATASOURCE_ERROR` * `EXTERNAL_IMPORT_ERROR` +* `EXTERNAL_ROW_CAP_EXCEEDED` * `EXTERNAL_SCHEMA_MISMATCH` * `EXTERNAL_SCHEMA_MODE_VIOLATION` * `EXTERNAL_WRITE_FORBIDDEN` +* `EXTERNAL_WRITE_REFUSED` * `FEEDS_DISABLED` * `FIELD_VISIBILITY_UNRESOLVED` * `FILES_DISABLED` diff --git a/content/docs/references/api/contract.mdx b/content/docs/references/api/contract.mdx index a8b16daab90..b522fa9201e 100644 --- a/content/docs/references/api/contract.mdx +++ b/content/docs/references/api/contract.mdx @@ -28,7 +28,7 @@ const result = ApiErrorSchema.parse(data); | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +347 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | +| **code** | `Enum<'VALIDATION_ERROR' \| 'INVALID_FIELD' \| 'MISSING_REQUIRED_FIELD' \| 'INVALID_FORMAT' \| 'VALUE_TOO_LONG' \| 'VALUE_TOO_SHORT' \| 'VALUE_OUT_OF_RANGE' \| … +349 more>` | ✅ | Error code (e.g. VALIDATION_ERROR; StandardErrorCode ∪ the ledger the serving side registers — ERROR_CODE_LEDGER for framework packages) | | **declaredCode** | `string` | optional | The producer-declared code, verbatim, when it is not a member of the closed `code` vocabulary — the open, author-authored channel (app-specific spellings; ADR-0112) | | **message** | `string` | ✅ | Readable error message | | **userMessage** | `string` | optional | Producer-marked user-facing refusal text, verbatim. Present exactly when the producer opted in at throw time; consumers render it to end users and keep their generic substitution for anything unmarked. Status-agnostic; never replaces `message`. | @@ -184,9 +184,11 @@ const result = ApiErrorSchema.parse(data); * `EXPORT_NOT_PERMITTED` * `EXTERNAL_DATASOURCE_ERROR` * `EXTERNAL_IMPORT_ERROR` +* `EXTERNAL_ROW_CAP_EXCEEDED` * `EXTERNAL_SCHEMA_MISMATCH` * `EXTERNAL_SCHEMA_MODE_VIOLATION` * `EXTERNAL_WRITE_FORBIDDEN` +* `EXTERNAL_WRITE_REFUSED` * `FEEDS_DISABLED` * `FIELD_VISIBILITY_UNRESOLVED` * `FILES_DISABLED` diff --git a/content/docs/references/api/error-code-ledger.mdx b/content/docs/references/api/error-code-ledger.mdx index 45028637f64..f35d01d46e7 100644 --- a/content/docs/references/api/error-code-ledger.mdx +++ b/content/docs/references/api/error-code-ledger.mdx @@ -351,9 +351,11 @@ const result = ErrorCode.parse(data); * `EXPORT_NOT_PERMITTED` * `EXTERNAL_DATASOURCE_ERROR` * `EXTERNAL_IMPORT_ERROR` +* `EXTERNAL_ROW_CAP_EXCEEDED` * `EXTERNAL_SCHEMA_MISMATCH` * `EXTERNAL_SCHEMA_MODE_VIOLATION` * `EXTERNAL_WRITE_FORBIDDEN` +* `EXTERNAL_WRITE_REFUSED` * `FEEDS_DISABLED` * `FIELD_VISIBILITY_UNRESOLVED` * `FILES_DISABLED` diff --git a/packages/spec/src/api/error-code-ledger.zod.ts b/packages/spec/src/api/error-code-ledger.zod.ts index 91f7a2bf848..87fc732a334 100644 --- a/packages/spec/src/api/error-code-ledger.zod.ts +++ b/packages/spec/src/api/error-code-ledger.zod.ts @@ -814,6 +814,25 @@ export const ERROR_CODE_LEDGER = { // been written (`TransactionUnsupportedError`, `transaction-errors.ts`; // ADR-0119 D1/D4 fail-closed posture). Same #8087-gate family. 'ERR_TRANSACTION_UNSUPPORTED', + // [#27277] The engine's floor over a plugin driver on an external + // datasource (ADR-0015 Tier A, `external-driver-floor.ts`). Both are 422 + // with their status declared on the thrown error (the + // `driver-connect-errors.ts` pattern), and neither is an ADR-0015 gate + // refusal: `EXTERNAL_WRITE_FORBIDDEN` (403, `@objectstack/spec/shared`) + // is the platform's own write switch being off; these are fixed by + // changing the payload or the filter. + // + // The remote system refused a write with a 4xx (its validation, a lock, + // a missing record). The message is the remote's own; `details` carries + // `remoteStatus`. A remote that failed or did not answer is + // ERR_DATASOURCE_UNAVAILABLE / 503 above, not this. + 'EXTERNAL_WRITE_REFUSED', + // A read matched more rows than the engine reads from a plugin driver + // for one request (1000). The engine applies the query to the rows the + // driver returns, so it refuses rather than answer over a set it only + // partly holds. The message prescribes narrowing the filter or mirroring + // the data with DataSync. + 'EXTERNAL_ROW_CAP_EXCEEDED', // [#15823] an `afterFind` handler REPLACED `ctx.result` with something that // is not an array, breaking `find()`'s declared `Promise`. It was the // FIRST of the four `return hookContext.result` sites in `engine.ts` to be From 55356b9fe1318d51f56c0485d30c6bbc0aa46b59 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 05:20:50 +0000 Subject: [PATCH 3/6] docs(data-modeling): virtual objects run behind the engine's floor: three-step driver path, the 1000-row cap, fail-closed owner columns The two "planned" sentences become the shipped behaviour (the floor; 503 ERR_DATASOURCE_UNAVAILABLE / 422 EXTERNAL_WRITE_REFUSED for driver failures). New passages: the engine re-applies every query (what the driver is sent, what the engine does), the driver's three steps, what the 1000-row cap means and what the floor does not cover, and that a missing owner column fails closed. The "what your driver receives" table is re-measured under the floor. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2dgsoHnkCiSyXF8isz --- .../docs/data-modeling/virtual-objects.mdx | 104 ++++++++++++++---- 1 file changed, 84 insertions(+), 20 deletions(-) diff --git a/content/docs/data-modeling/virtual-objects.mdx b/content/docs/data-modeling/virtual-objects.mdx index 6af687b6fb1..1780a85ecfa 100644 --- a/content/docs/data-modeling/virtual-objects.mdx +++ b/content/docs/data-modeling/virtual-objects.mdx @@ -35,8 +35,9 @@ If the remote is a SQL database you can reach, you need no driver of your own built-in one through [External datasources](/docs/data-modeling/external-datasources). A virtual object pays for its freshness on every request: each list read is a round -trip to the remote, and the remote's query power is the object's query power. That is -the trade the table above decides. +trip to the remote, and a read can match at most 1000 rows +([the cap](#the-engine-re-applies-every-query)). That is the trade the table above +decides. ## The recipe @@ -178,20 +179,25 @@ export function createPortScheduleDriver(transport: PortScheduleTransport): IDat } ``` -**A driver cannot opt out of correctness.** The rules it keeps: - -- **Apply every condition, sort and page it receives.** The engine hands the driver - the view's filter, the user's search, the sort and the page size, and serves whatever - comes back as the answer. A condition the driver drops widens the result silently — a - user sees rows the filter excluded, and nothing on the page says so. An engine-side - floor for drivers that cannot push a condition down is planned - ([#27277](https://github.com/objectstack-ai/objectstack/issues/27277)); until it lands - the whole obligation is the driver's. -- **Refuse by name what the remote cannot run** — an operator, a combinator, a column it - does not carry — and fail when the remote fails. Throw; ⛔ never return an empty - array, which reads exactly like "no rows match". A thrown driver error answers the - caller `500 INTERNAL_ERROR` today; a dedicated refusal for driver failures is planned - with the floor above. +**A driver cannot opt out of correctness, only of performance.** The engine does not +trust a plugin driver to have applied the query: it re-applies it +([below](#the-engine-re-applies-every-query)). The rules the driver keeps: + +- **Never return fewer rows than match.** The engine filters, sorts and pages the rows + your driver returns, so extra rows cost only transfer, but a row your driver leaves out + is lost. When you cannot translate part of a `where`, send the remote a WIDER filter, + never a narrower one: drop an untranslatable member of a top-level `$and` (or the whole + `where`), and drop a whole `$or` or `$not` that contains one. ⛔ Dropping one branch + of an `$or` narrows the result. The skeleton above takes the stricter road and refuses + instead, which is also safe. +- **Fail when the remote fails.** Throw; ⛔ never return an empty array, which reads + exactly like "no rows match". A read that throws (a refusal by name included) answers + the caller `503 ERR_DATASOURCE_UNAVAILABLE` with `reason: 'remote'`. Its message says + the remote could not answer and to retry; your error's own text goes to the server log + at `warn` and never to the caller. A write the remote refuses with a 4xx is answered + `422 EXTERNAL_WRITE_REFUSED` carrying your error's message, so set `status` (or + `statusCode`) on the error you throw. A write that fails any other way is `503`, as a + read is. - **Send the remote only the columns it owns.** `update` receives the patch plus any field a hook stamped, plus `id`, `updated_at` and `updated_by`, which the engine adds on its own and no remote column declares. @@ -372,8 +378,62 @@ remote row does not carry — so every row reads as someone else's `sharingModel: 'public_read_write'` and decide who may edit in `beforeUpdate`, as above. Serving an owner column the checks can read is the other way out; the reference test does not cover it, so pin it in your own test if you take it. + +Missing owner columns fail closed: the edit is refused, never allowed by mistake. So pick +one of the two ways out on purpose. Either set `sharingModel` explicitly, or have your +driver return the owner under the field name the checks read. On a plugin driver that +mapping is the driver's own job: `external.columnMap` is applied by the built-in SQL +drivers, not by the engine. A validation warning for an external object on an +owner-keyed sharing model is tracked in +[#27332](https://github.com/objectstack-ai/objectstack/issues/27332). +### The engine re-applies every query + +For a datasource whose `schemaMode` is not `managed` and whose `driver` is not a built-in +id (memory, sqlite, sqlite-wasm, postgres, mysql, mongodb, turso), the engine holds the +driver behind a correctness floor. It does not change a federated database on a built-in +driver. + +- **What the driver is sent.** `find` gets `where` and `orderBy`, which it may push + down, and `limit: 1001`. It never gets `offset` or `fields`. `findOne` and `count` + reach your driver as that same `find`. `aggregate` is never called; the engine + aggregates in memory. +- **What the engine does with the rows.** It applies the whole query itself: `where` + (the engine's own operator vocabulary, `$like` and `$ilike` included), the sort (no + value sorts last), `offset` and `limit` (a paged read breaks ties by `id`), and the + projection. +- **Writes** go to your driver unchanged. + +So a driver grows in three steps, each correct: + +1. **Return every row.** Ignore the query and answer the resource's rows. The answer is + correct for any resource with at most 1000 rows. +2. **Push `where` and `orderBy` down.** The remote returns only the matching rows, so the + 1000-row cap now counts matching rows, not the whole resource, and less travels. The + engine still re-checks every row, so a translation slip shows up as a missing row + (step 1's rule), never as an extra one. +3. **Skip the floor with a conformance-tested driver.** This is planned for a later + phase ([#27261](https://github.com/objectstack-ai/objectstack/issues/27261)): a driver + that passes the platform's conformance kit would be trusted with paging and the cap. + It does not exist yet. + +**The 1000-row cap.** A read that matches more than 1000 rows is refused with +`422 EXTERNAL_ROW_CAP_EXCEEDED` rather than answered from a partial set. The fix is to +narrow the filter so fewer rows match, or, for data that size, to mirror it into a real +object with a [scheduled pull](/docs/automation/jobs#pulling-a-mapping) and read that. +The cap applies to plugin drivers only. It also means a list cannot page past its first +1000 matching rows: the page at offset 1000 is refused, not empty. The cap is fixed and +not configurable. + +**Not covered by the floor.** + +- `updateMany` and `deleteMany` pass their `where` to your driver unchanged, so a driver + that offers them must apply the whole `where`. +- A read whose scope composes a `ReadableIds` leaf, such as an object + `controlled_by_parent`, is refused (ADR-0139): the engine has no semi-join over a + plugin driver's rows. + ### Credentials ⛔ Never put a credential in the stack — not in `config`, not in a hook body, not in an @@ -389,16 +449,20 @@ repository. ## What your driver receives -Measured on the reference test, request by request: +Measured on the reference test, request by request, with the engine's floor +([above](#the-engine-re-applies-every-query)) in place: | The platform does | Your driver is called with | | :--- | :--- | | Boots | `connect()` once. No `syncSchema`, no `introspectSchema` (`checkOnBoot: false`). | -| Serves a list (`GET /data/berth_slot` with the view's filter, sort and page size) | `find(object, { where, orderBy, limit })`, then `count(object, { where })` for the total | -| Serves a record (`GET /data/berth_slot/:id`) | `findOne(object, { where: { id }, limit: 1 })` | -| Writes (`PATCH /data/berth_slot/:id`) | several by-id reads (the pre-image for hooks and checks), then `update(object, id, patch)` with the extra keys above | +| Serves a list (`GET /data/berth_slot` with the view's filter, sort and page size) | `find(object, { where, orderBy, limit: 1001 })`, then `find(object, { where, limit: 1001 })` for the total. The page size is applied by the engine. | +| Serves a record (`GET /data/berth_slot/:id`) | `find(object, { where: { id }, limit: 1001 })` | +| Writes (`PATCH /data/berth_slot/:id`) | several by-id reads as `find` (the pre-image for hooks and checks), then `update(object, id, patch)` with the extra keys above | | Refuses a caller with no grant on a list read | nothing | +Your driver should still implement `findOne` and `count`: the engine calls them on a +driver that is not behind the floor, and the shape test below exercises them. + Two details catch every first driver: - **`object` is the object name** (`berth_slot`), never `external.remoteName` (`slots`). From 3d06ce76b86c24083a3b4b173bac42c91e011d72 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 05:42:22 +0000 Subject: [PATCH 4/6] fix(objectql): keep the floor on a driver-less re-registration; hide predicate writes; only request-shaped 4xx is a write refusal Rework round 1 on the plugin-driver floor: - registerDatasourceDef without `driver` keeps the stored driver id, so the datasource-connection service's driver-less re-registration (after a connect, restoreOldPool) no longer unwraps the floor. - updateMany / deleteMany are hidden on the floored instance, as aggregate is: a precondition write goes through update by id, and a multi-row predicate write is refused. - Only 400/404/409/412/422 from the remote is EXTERNAL_WRITE_REFUSED; a 401/403/407/429 is ERR_DATASOURCE_UNAVAILABLE (503) with its text withheld. - The in-memory sort compares a Date and an ISO string in one column as instants. - Docs and changeset follow. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2dgsoHnkCiSyXF8isz --- .changeset/27277-plugin-driver-floor.md | 9 +- .../docs/data-modeling/virtual-objects.mdx | 28 +++-- ...ver-query-in-memory.mixed-temporal.test.ts | 36 ++++++ .../objectql/src/driver-query-in-memory.ts | 18 ++- packages/objectql/src/engine.ts | 7 +- .../objectql/src/external-driver-errors.ts | 23 +++- .../src/external-driver-floor.test.ts | 107 ++++++++++++++++++ .../objectql/src/external-driver-floor.ts | 28 +++-- packages/objectql/src/index.ts | 1 + 9 files changed, 229 insertions(+), 28 deletions(-) create mode 100644 packages/objectql/src/driver-query-in-memory.mixed-temporal.test.ts diff --git a/.changeset/27277-plugin-driver-floor.md b/.changeset/27277-plugin-driver-floor.md index e833abd35a2..b658820fef0 100644 --- a/.changeset/27277-plugin-driver-floor.md +++ b/.changeset/27277-plugin-driver-floor.md @@ -18,10 +18,13 @@ Metadata-protocol: none - `aggregate` is hidden, so the engine aggregates in memory over that read. - The filter is evaluated by the engine's own walker (the one `having` uses). `$like` and `$ilike` now work in its `where` position. It is held to the spec tables `FILTER_LOGIC_CASES`, `FILTER_TEXT_CASES`, `PAGINATION_*_CASES` and `TEMPORAL_CASES`. - **Row cap.** A read that matches more than 1000 rows is refused with `EXTERNAL_ROW_CAP_EXCEEDED` (422), whose message says to narrow the filter or mirror the data into a real object with DataSync. -- **Writes pass through.** A remote refusal with a 4xx status is `EXTERNAL_WRITE_REFUSED` (422). Its message is the remote's own, and `details.remoteStatus` carries the status. A remote that fails, does not answer, or answers 5xx is `ERR_DATASOURCE_UNAVAILABLE` (503, `reason: 'remote'`); its cause goes to the server log at `warn` and is never sent to the caller. +- **Writes pass through, by id.** The floored driver does not offer `updateMany` or `deleteMany`, so a by-id write under a precondition goes through `update`, and a multi-row predicate update or delete is refused. + - A remote refusal for the request's own sake (400, 404, 409, 412 or 422) is `EXTERNAL_WRITE_REFUSED` (422). Its message is the remote's own, and `details.remoteStatus` carries the status. + - Any other failure is `ERR_DATASOURCE_UNAVAILABLE` (503, `reason: 'remote'`): no answer, a 5xx, or a 401/403/407/429 credential or rate-limit condition. The cause goes to the server log at `warn` and is never sent to the caller. - **One instance.** The engine stores the floored driver in place of the driver, so `getDriverByName`, `datasource()` and the routing for an object all return the same instance. `unwrapExternalDriverFloor` answers the driver behind it. -- **Not covered.** `updateMany` and `deleteMany` pass the predicate through to the driver unchanged. A `where` with a `ReadableIds` leaf is refused (ADR-0139 D3), because the engine has no semi-join over a plugin driver's rows. +- **Re-registration.** A datasource definition registered again without `driver` keeps the driver id stored before it, so the floor stays on. +- **Not covered.** A `where` with a `ReadableIds` leaf is refused (ADR-0139 D3), because the engine has no semi-join over a plugin driver's rows. **The remedy.** If a read now meets the cap, narrow its filter, or mirror the data with a `mapping` pulled by a `job` and read the real object. -New exports from `@objectstack/objectql`: `applyDriverQueryInMemory`, `ExternalWriteRefusedError`, `ExternalRowCapExceededError`, their `*_CODE` constants, `EXTERNAL_ROW_CAP`, `needsExternalDriverFloor`, `unwrapExternalDriverFloor`, and the types `DriverQueryInMemory`, `DriverQueryInMemoryOptions`, `ExternalWriteOperation` and `DatasourceUnavailableReason`. `DatasourceDef` gains an optional `driver`. +New exports from `@objectstack/objectql`: `applyDriverQueryInMemory`, `ExternalWriteRefusedError`, `ExternalRowCapExceededError`, their `*_CODE` constants, `EXTERNAL_ROW_CAP`, `EXTERNAL_WRITE_REFUSED_STATUSES`, `needsExternalDriverFloor`, `unwrapExternalDriverFloor`, and the types `DriverQueryInMemory`, `DriverQueryInMemoryOptions`, `ExternalWriteOperation` and `DatasourceUnavailableReason`. `DatasourceDef` gains an optional `driver`. diff --git a/content/docs/data-modeling/virtual-objects.mdx b/content/docs/data-modeling/virtual-objects.mdx index 1780a85ecfa..f8abf358cdc 100644 --- a/content/docs/data-modeling/virtual-objects.mdx +++ b/content/docs/data-modeling/virtual-objects.mdx @@ -194,10 +194,12 @@ trust a plugin driver to have applied the query: it re-applies it exactly like "no rows match". A read that throws (a refusal by name included) answers the caller `503 ERR_DATASOURCE_UNAVAILABLE` with `reason: 'remote'`. Its message says the remote could not answer and to retry; your error's own text goes to the server log - at `warn` and never to the caller. A write the remote refuses with a 4xx is answered - `422 EXTERNAL_WRITE_REFUSED` carrying your error's message, so set `status` (or - `statusCode`) on the error you throw. A write that fails any other way is `503`, as a - read is. + at `warn` and never to the caller. A write the remote refuses for the request's own + sake (400, 404, 409, 412 or 422) is answered `422 EXTERNAL_WRITE_REFUSED` carrying your + error's message, so set `status` (or `statusCode`) on the error you throw. A write + that fails any other way is `503`, as a read is. That includes 401, 403, 407 and 429: + an expired credential or a rate limit is the operator's to fix, and its text may name + the credential. - **Send the remote only the columns it owns.** `update` receives the patch plus any field a hook stamped, plus `id`, `updated_at` and `updated_by`, which the engine adds on its own and no remote column declares. @@ -403,12 +405,17 @@ driver. (the engine's own operator vocabulary, `$like` and `$ilike` included), the sort (no value sorts last), `offset` and `limit` (a paged read breaks ties by `id`), and the projection. -- **Writes** go to your driver unchanged. +- **Writes** go to your driver unchanged, one record at a time. The floored driver does + not offer `updateMany` or `deleteMany`, so no `where` the engine cannot re-check + reaches a write. A by-id write under a precondition goes through `update` by id, and + a multi-row predicate update or delete is refused. So a driver grows in three steps, each correct: 1. **Return every row.** Ignore the query and answer the resource's rows. The answer is - correct for any resource with at most 1000 rows. + correct for any resource with at most 1000 rows. Honour `limit` by paging the remote: + a remote whose page is smaller than 1001 rows, and that you do not page through, + answers fewer rows than match, and the engine cannot detect it. 2. **Push `where` and `orderBy` down.** The remote returns only the matching rows, so the 1000-row cap now counts matching rows, not the whole resource, and less travels. The engine still re-checks every row, so a translation slip shows up as a missing row @@ -423,13 +430,14 @@ So a driver grows in three steps, each correct: narrow the filter so fewer rows match, or, for data that size, to mirror it into a real object with a [scheduled pull](/docs/automation/jobs#pulling-a-mapping) and read that. The cap applies to plugin drivers only. It also means a list cannot page past its first -1000 matching rows: the page at offset 1000 is refused, not empty. The cap is fixed and -not configurable. +1000 matching rows: when more than 1000 rows match, every page of that read is refused, +the one at offset 1000 included. The cap is fixed and not configurable. **Not covered by the floor.** -- `updateMany` and `deleteMany` pass their `where` to your driver unchanged, so a driver - that offers them must apply the whole `where`. +- Predicate writes. `updateMany` and `deleteMany` are not offered on a floored datasource, + so a write that selects its rows by a filter is refused rather than handed to your + driver. Read the ids, then write each one. - A read whose scope composes a `ReadableIds` leaf, such as an object `controlled_by_parent`, is refused (ADR-0139): the engine has no semi-join over a plugin driver's rows. diff --git a/packages/objectql/src/driver-query-in-memory.mixed-temporal.test.ts b/packages/objectql/src/driver-query-in-memory.mixed-temporal.test.ts new file mode 100644 index 00000000000..7728446d50b --- /dev/null +++ b/packages/objectql/src/driver-query-in-memory.mixed-temporal.test.ts @@ -0,0 +1,36 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#27277] A remote may answer one column in both forms — a `Date` on some rows, + * an ISO string on others. `applyDriverQueryInMemory` sorts such a column by + * instant: before, a `Date` (as epoch ms) against a string compared as + * `number < string`, false both ways, so every mixed pair read as equal. + */ + +import { describe, it, expect } from 'vitest'; +import { applyDriverQueryInMemory } from './driver-query-in-memory.js'; + +const ROWS = [ + { id: 'a', at: '2026-07-28T09:00:00.000Z' }, + { id: 'b', at: new Date('2026-07-27T09:00:00.000Z') }, + { id: 'c', at: '2026-07-29T09:00:00.000Z' }, + { id: 'd', at: new Date('2026-07-28T21:00:00.000Z') }, + { id: 'e', at: null }, +]; + +describe('[#27277] a column mixing Date and ISO-string values sorts by instant', () => { + it('ascending, no value last', () => { + const got = applyDriverQueryInMemory(ROWS, { orderBy: [{ field: 'at', order: 'asc' }] }).map((r) => r.id); + expect(got).toEqual(['b', 'a', 'd', 'c', 'e']); + }); + + it('descending, no value still last', () => { + const got = applyDriverQueryInMemory(ROWS, { orderBy: [{ field: 'at', order: 'desc' }] }).map((r) => r.id); + expect(got).toEqual(['c', 'd', 'a', 'b', 'e']); + }); + + it('a page cuts the instant order, not the arrival order', () => { + const got = applyDriverQueryInMemory(ROWS, { orderBy: [{ field: 'at', order: 'asc' }], limit: 2, offset: 1 }).map((r) => r.id); + expect(got).toEqual(['a', 'd']); + }); +}); diff --git a/packages/objectql/src/driver-query-in-memory.ts b/packages/objectql/src/driver-query-in-memory.ts index 717ad4b6b90..03cf61d237a 100644 --- a/packages/objectql/src/driver-query-in-memory.ts +++ b/packages/objectql/src/driver-query-in-memory.ts @@ -188,14 +188,28 @@ function compareRows(a: Record, b: Record, key // No value sorts last in either direction. return xMissing ? 1 : -1; } - const left = x instanceof Date ? x.getTime() : x; - const right = y instanceof Date ? y.getTime() : y; + // A remote may answer one column in both forms (a `Date` on some rows, an + // ISO string on others): when either side is a `Date`, both compare as + // instants, or `number < string` would read every pair as equal. + const temporal = x instanceof Date || y instanceof Date; + const left = temporal ? instantOf(x) : x; + const right = temporal ? instantOf(y) : y; if ((left as any) < (right as any)) return order === 'desc' ? 1 : -1; if ((left as any) > (right as any)) return order === 'desc' ? -1 : 1; } return 0; } +/** A value as epoch ms for a temporal comparison; a string that is not a date stays as written. */ +function instantOf(value: unknown): unknown { + if (value instanceof Date) return value.getTime(); + if (typeof value === 'string') { + const ms = Date.parse(value); + return Number.isNaN(ms) ? value : ms; + } + return value; +} + function project(row: Record, fields: readonly string[]): Record { const out: Record = {}; for (const field of fields) { diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index f4b46501001..9344467f0e6 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -9244,7 +9244,12 @@ export class ObjectQL implements IObjectQLEngine { */ registerDatasourceDef(def: DatasourceDef): void { if (!def?.name) return; - this.datasourceDefs.set(def.name, { driver: def.driver, schemaMode: def.schemaMode, external: def.external }); + // [#27277] A def that names no driver keeps the id stored before it: the + // datasource-connection service re-registers a def without `driver` after + // a connect (`EngineDatasourceDef` has no such key), and reading that + // absence as "not a plugin driver" would take the floor back off. + const driver = def.driver ?? this.datasourceDefs.get(def.name)?.driver; + this.datasourceDefs.set(def.name, { driver, schemaMode: def.schemaMode, external: def.external }); this.applyExternalDriverFloor(def.name); } diff --git a/packages/objectql/src/external-driver-errors.ts b/packages/objectql/src/external-driver-errors.ts index 333f3461912..f82f8ed439b 100644 --- a/packages/objectql/src/external-driver-errors.ts +++ b/packages/objectql/src/external-driver-errors.ts @@ -32,15 +32,28 @@ export const EXTERNAL_ROW_CAP_EXCEEDED_CODE = 'EXTERNAL_ROW_CAP_EXCEEDED' as con */ export const EXTERNAL_ROW_CAP = 1000; -/** The write a refusal names. */ +/** + * The write a refusal names. `updateMany` / `deleteMany` are not among them: + * the floor does not offer predicate writes (`external-driver-floor.ts`). + */ export type ExternalWriteOperation = | 'create' | 'update' | 'upsert' | 'delete' - | 'bulkCreate' | 'bulkUpdate' | 'bulkDelete' - | 'updateMany' | 'deleteMany'; + | 'bulkCreate' | 'bulkUpdate' | 'bulkDelete'; + +/** + * The remote statuses read as a refusal of THIS request — its payload, its + * target, its precondition — and so answered {@link ExternalWriteRefusedError} + * with the remote's message. Every other 4xx is a condition of the connection, + * not of the request (401 / 403 / 407 an expired or refused credential, 429 a + * rate limit), whose text may name the credential: it is answered as a remote + * failure (`ERR_DATASOURCE_UNAVAILABLE`, 503), the cause in the server log only. + */ +export const EXTERNAL_WRITE_REFUSED_STATUSES: ReadonlySet = new Set([400, 404, 409, 412, 422]); /** - * The remote system behind a plugin driver refused a write with a 4xx answer — - * its validation failed, the record is locked, it does not exist there. 422: + * The remote system behind a plugin driver refused a write with a request-shaped + * 4xx ({@link EXTERNAL_WRITE_REFUSED_STATUSES}) — its validation failed, the + * record is locked, it does not exist there. 422: * the request reached the remote and was judged, and a changed payload may * pass. The message is the remote's own, so the person who typed the value * reads why it was refused. diff --git a/packages/objectql/src/external-driver-floor.test.ts b/packages/objectql/src/external-driver-floor.test.ts index c975471137f..8f96b3bceb4 100644 --- a/packages/objectql/src/external-driver-floor.test.ts +++ b/packages/objectql/src/external-driver-floor.test.ts @@ -267,3 +267,110 @@ describe('[#27277] writes pass through; a remote refusal is 422', () => { expect(err.message).not.toContain('upstream exploded'); }); }); + +describe('[#27277] only a request-shaped 4xx is a refusal of the write', () => { + const refusing = (status: number, message: string) => makeEngine({ + extra: { async create() { throw Object.assign(new Error(message), { status }); } }, + }); + + for (const status of [400, 404, 409, 412, 422]) { + it(`${status} is EXTERNAL_WRITE_REFUSED / 422`, async () => { + const { engine } = refusing(status, `remote said ${status}`); + const err = await engine.insert('tree_node', { kind: 'leaf' }).then(() => undefined, (e) => e); + expect(err).toBeInstanceOf(ExternalWriteRefusedError); + expect(err.details.remoteStatus).toBe(status); + }); + } + + for (const status of [401, 403, 407, 429]) { + it(`${status} (a credential or rate-limit condition) is ERR_DATASOURCE_UNAVAILABLE / 503, its text withheld`, async () => { + const { engine } = refusing(status, 'token sk_live_123 expired'); + const err = await engine.insert('tree_node', { kind: 'leaf' }).then(() => undefined, (e) => e); + expect(err).toBeInstanceOf(DatasourceUnavailableError); + expect(err.code).toBe('ERR_DATASOURCE_UNAVAILABLE'); + expect(err.kind).toBe('remote'); + expect(err.message).not.toContain('sk_live_123'); + }); + } +}); + +describe('[#27277] predicate writes are not offered: every write reaches the driver by id', () => { + const predicateDouble = () => { + const writes: Row[] = []; + const { engine, remote } = makeEngine({ + extra: { + async update(_o: string, id: string, data: Row) { writes.push({ via: 'update', id }); return { id, ...data }; }, + async updateMany() { writes.push({ via: 'updateMany' }); return 99; }, + async deleteMany() { writes.push({ via: 'deleteMany' }); return 99; }, + }, + }); + return { engine, remote, writes }; + }; + + it('updateMany and deleteMany are hidden on the floored instance', () => { + const { engine, remote } = predicateDouble(); + const held = engine.getDriverByName('tree')!; + expect('updateMany' in held).toBe(false); + expect('deleteMany' in held).toBe(false); + expect(held.updateMany).toBeUndefined(); + expect(held.deleteMany).toBeUndefined(); + expect(typeof (remote.driver as any).updateMany).toBe('function'); + }); + + it('a by-id write under a precondition (ADR-0142 D4) takes the by-id path, not updateMany', async () => { + const { engine, writes } = predicateDouble(); + await (engine as any).updateByIdExpecting( + engine.getDriverByName('tree'), 'tree_node', 'n1', { label: 'Echo 2' }, { label: 'Echo' }, {}, + ); + expect(writes).toEqual([{ via: 'update', id: 'n1' }]); + }); + + it('a multi-row predicate update is refused, and the driver writes nothing', async () => { + const { engine, writes } = predicateDouble(); + const err = await engine.update('tree_node', { label: 'x' }, { where: { kind: 'leaf' }, multi: true }) + .then(() => undefined, (e) => e); + // The engine's generic refusal for a `multi` verdict on a driver with no + // predicate write — the floored instance offers none. + expect(err).toBeInstanceOf(Error); + expect(err.message).toContain('options.multi=true'); + expect(writes).toEqual([]); + }); + + it('a multi-row predicate delete is refused, and the driver writes nothing', async () => { + const { engine, writes } = predicateDouble(); + const err = await engine.delete('tree_node', { where: { kind: 'leaf' }, multi: true }) + .then(() => undefined, (e) => e); + // The engine's generic refusal for a `multi` verdict on a driver with no + // predicate write — the floored instance offers none. + expect(err).toBeInstanceOf(Error); + expect(err.message).toContain('options.multi=true'); + expect(writes).toEqual([]); + }); +}); + +describe('[#27277] a re-registration that names no driver keeps the floor', () => { + it('the datasource-connection sequence: def with driver, driver, then a def without driver', async () => { + const engine = new ObjectQL(); + const remote = remoteDriver('tree', TREE_ROWS); + engine.registerDatasourceDef({ name: 'tree', driver: 'spike_tree', schemaMode: 'external', external: { allowWrites: true } }); + engine.registerDriver(remote.driver); + // What `datasource-connection-service.ts` registers after a connect and in + // `restoreOldPool`: no `driver` key (`EngineDatasourceDef` has none). + engine.registerDatasourceDef({ name: 'tree', schemaMode: 'external', external: { allowWrites: true } }); + engine.registerApp({ + id: 'tree_pkg', + name: 'Tree', + objects: [{ + name: 'tree_node', + datasource: 'tree', + external: { remoteName: 'nodes', writable: true }, + fields: { kind: { type: 'text' }, label: { type: 'text' }, weight: { type: 'number' } }, + }], + } as any); + const held = engine.getDriverByName('tree')!; + expect(isExternalDriverFloor(held)).toBe(true); + expect(unwrapExternalDriverFloor(held)).toBe(remote.driver); + // The where-ignoring driver still answers only the matching rows. + expect(ids(await engine.find('tree_node', { where: { kind: 'branch' } }))).toEqual(['n2', 'n5']); + }); +}); diff --git a/packages/objectql/src/external-driver-floor.ts b/packages/objectql/src/external-driver-floor.ts index 1bce5913ab5..2b09f08cedc 100644 --- a/packages/objectql/src/external-driver-floor.ts +++ b/packages/objectql/src/external-driver-floor.ts @@ -35,10 +35,13 @@ * its length. * - `aggregate` is hidden, so the engine's existing find + in-memory * aggregation path runs over the floored `find`. - * - Writes pass through. A write the remote refuses with a 4xx is + * - Writes pass through, by id: `updateMany` / `deleteMany` are hidden too, so + * no `where` the engine cannot re-check reaches a write. A write the remote + * refuses with a request-shaped 4xx (`EXTERNAL_WRITE_REFUSED_STATUSES`) is * {@link ExternalWriteRefusedError} (422, the remote's message); any other - * failure of a read or a write is `ERR_DATASOURCE_UNAVAILABLE` (503, reason - * `remote`), with the cause in the server log and never on the wire. + * failure of a read or a write — an auth or rate-limit 4xx included — is + * `ERR_DATASOURCE_UNAVAILABLE` (503, reason `remote`), with the cause in the + * server log and never on the wire. * - Everything else (`connect`, `supports`, `syncSchema`, `introspectSchema`, * `beginTransaction`, …) is the driver's own, unchanged. * @@ -58,6 +61,7 @@ import { declaredFieldClasses, declaredJsonStoredFields } from './having-filter. import { DatasourceUnavailableError } from './driver-connect-errors.js'; import { EXTERNAL_ROW_CAP, + EXTERNAL_WRITE_REFUSED_STATUSES, ExternalRowCapExceededError, ExternalWriteRefusedError, type ExternalWriteOperation, @@ -86,9 +90,19 @@ export interface ExternalDriverFloorHost { const WRITE_METHODS: ReadonlySet = new Set([ 'create', 'update', 'upsert', 'delete', 'bulkCreate', 'bulkUpdate', 'bulkDelete', - 'updateMany', 'deleteMany', ]); +/** + * Members the floored driver does not offer, whatever the driver has. + * `aggregate`: the engine aggregates in memory over the floored `find`. + * `updateMany` / `deleteMany`: a predicate write would hand the driver a + * `where` the engine cannot re-check, so a driver that ignored it would write + * rows the caller never selected. Without them the engine writes by id (the + * ADR-0142 D4 precondition takes its documented no-`updateMany` path) and + * refuses a multi-row predicate write. + */ +const HIDDEN: ReadonlySet = new Set(['aggregate', 'updateMany', 'deleteMany']); + /** floored driver → the driver behind it */ const innerOf = new WeakMap(); /** driver → its floored driver, so wrapping one driver twice answers one instance */ @@ -173,7 +187,7 @@ export function createExternalDriverFloor(driver: IDataDriver, host: ExternalDri return await method.call(inner, object, ...rest); } catch (error) { const status = declaredStatus(error); - if (status !== undefined && status >= 400 && status < 500) { + if (status !== undefined && EXTERNAL_WRITE_REFUSED_STATUSES.has(status)) { throw new ExternalWriteRefusedError( host.datasource, object, operation as ExternalWriteOperation, status, error instanceof Error ? error.message : String(error), { cause: error }, @@ -189,7 +203,7 @@ export function createExternalDriverFloor(driver: IDataDriver, host: ExternalDri if (prop === 'find') return find; if (prop === 'findOne') return findOne; if (prop === 'count') return count; - if (prop === 'aggregate') return undefined; + if (HIDDEN.has(prop)) return undefined; const value = Reflect.get(target, prop, target); if (typeof value !== 'function') return value; if (!bound.has(prop)) { @@ -200,7 +214,7 @@ export function createExternalDriverFloor(driver: IDataDriver, host: ExternalDri return bound.get(prop); }, has(target, prop) { - if (prop === 'aggregate') return false; + if (HIDDEN.has(prop)) return false; return Reflect.has(target, prop); }, }) as IDataDriver; diff --git a/packages/objectql/src/index.ts b/packages/objectql/src/index.ts index 52b5036b8d1..1bdeb934f67 100644 --- a/packages/objectql/src/index.ts +++ b/packages/objectql/src/index.ts @@ -310,6 +310,7 @@ export { EXTERNAL_WRITE_REFUSED_CODE, EXTERNAL_ROW_CAP_EXCEEDED_CODE, EXTERNAL_ROW_CAP, + EXTERNAL_WRITE_REFUSED_STATUSES, } from './external-driver-errors.js'; export type { ExternalWriteOperation } from './external-driver-errors.js'; export { needsExternalDriverFloor, unwrapExternalDriverFloor } from './external-driver-floor.js'; From 7561b1487577198b5a49ea394ba2d00db86ec074 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 06:06:07 +0000 Subject: [PATCH 5/6] docs(data-modeling): the sharing callout says what actually lets a member edit and delete a virtual object's rows An explicit sharingModel is not a way out on its own ('private' is refused the same way): it is public_read_write with a beforeUpdate hook, or a declared owner field the driver returns under its name (columnMap only on a built-in SQL driver). Member deletes stay refused on public_read_write unless the driver returns created_by or a policy or share grants delete. Matches the security-external-owner-unmapped warning (#27332). Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2dgsoHnkCiSyXF8isz --- .../docs/data-modeling/virtual-objects.mdx | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/content/docs/data-modeling/virtual-objects.mdx b/content/docs/data-modeling/virtual-objects.mdx index f8abf358cdc..bf38ecbad43 100644 --- a/content/docs/data-modeling/virtual-objects.mdx +++ b/content/docs/data-modeling/virtual-objects.mdx @@ -381,12 +381,19 @@ remote row does not carry — so every row reads as someone else's Serving an owner column the checks can read is the other way out; the reference test does not cover it, so pin it in your own test if you take it. -Missing owner columns fail closed: the edit is refused, never allowed by mistake. So pick -one of the two ways out on purpose. Either set `sharingModel` explicitly, or have your -driver return the owner under the field name the checks read. On a plugin driver that -mapping is the driver's own job: `external.columnMap` is applied by the built-in SQL -drivers, not by the engine. A validation warning for an external object on an -owner-keyed sharing model is tracked in +Missing owner columns fail closed: the edit is refused, never allowed by mistake. Setting +`sharingModel` is not enough by itself, because an explicit `private` is refused the +same way. There are two ways out: + +- `sharingModel: 'public_read_write'` with a `beforeUpdate` hook that decides, as above; +- a declared owner field that your driver returns under that field name. A built-in SQL + driver maps it through `external.columnMap`; on a plugin driver the mapping is your + driver's own job, because the engine does not apply `columnMap`. + +On `public_read_write`, a member's delete is still refused: the ownership floor keeps +deletes by design. A delete passes only when your driver returns `created_by`, or when a +policy or a share grants it. The validation warning for an external object whose owner +is unmapped is tracked in [#27332](https://github.com/objectstack-ai/objectstack/issues/27332). From 490c3c331a2132a8b60506d5b82127807822265d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 29 Oct 2026 06:44:50 +0000 Subject: [PATCH 6/6] fix(objectql): name the refusal of a multi-row predicate write on a plugin-driver datasource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A floored plugin driver offers no updateMany/deleteMany, so a multi:true predicate update or delete fell to the engine's generic "requires an ID or options.multi=true" — false for a caller who passed multi, and with no prescription. Both fallback arms now say the datasource is served by a plugin driver, why the engine does not run the write there, and to read the matching ids and write each by id. Every other driver keeps the generic text. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_017ihL2dgsoHnkCiSyXF8isz --- packages/objectql/src/engine.ts | 9 ++++++++- .../objectql/src/external-driver-floor.test.ts | 18 ++++++++++++------ packages/objectql/src/external-driver-floor.ts | 15 +++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index 9344467f0e6..8acc6add4da 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -369,6 +369,7 @@ import { isFederatedObject, isFederatedUnprovisionedInjectedColumn } from './fed // [#27277] The correctness floor over a plugin driver on an external datasource. import { createExternalDriverFloor, + externalPredicateWriteRefusal, isExternalDriverFloor, needsExternalDriverFloor, unwrapExternalDriverFloor, @@ -16402,6 +16403,9 @@ export class ObjectQL implements IObjectQLEngine { // boundary's status passthrough — decorated; the #5748/#11009 // family stays a plain Error, byte-identical to before. if (dispatch.kind === 'reject') throw engineUpdateDispatchRejectError(dispatch); + // [#27277] A floored plugin driver offers no `updateMany` on purpose: + // say so, rather than ask for the `multi` this caller already passed. + if (isExternalDriverFloor(driver)) throw externalPredicateWriteRefusal(driver.name, object, 'update'); throw new Error(ENGINE_UPDATE_REJECT_MESSAGE); } @@ -19914,7 +19918,10 @@ export class ObjectQL implements IObjectQLEngine { // refusal names the dropped `where` keys; the classic shape keeps // `ENGINE_DELETE_REJECT_MESSAGE`; the fallback arm is a `multi` // verdict against a driver with no `deleteMany`. - throw new Error(dispatch.kind === 'reject' ? dispatch.message : ENGINE_DELETE_REJECT_MESSAGE); + if (dispatch.kind === 'reject') throw new Error(dispatch.message); + // [#27277] See update()'s twin: a floored plugin driver offers no `deleteMany`. + if (isExternalDriverFloor(driver)) throw externalPredicateWriteRefusal(driver.name, object, 'delete'); + throw new Error(ENGINE_DELETE_REJECT_MESSAGE); } // [#6966] See update()'s twin — same rule, same single binding point. diff --git a/packages/objectql/src/external-driver-floor.test.ts b/packages/objectql/src/external-driver-floor.test.ts index 8f96b3bceb4..5c2019eb3ad 100644 --- a/packages/objectql/src/external-driver-floor.test.ts +++ b/packages/objectql/src/external-driver-floor.test.ts @@ -329,10 +329,13 @@ describe('[#27277] predicate writes are not offered: every write reaches the dri const { engine, writes } = predicateDouble(); const err = await engine.update('tree_node', { label: 'x' }, { where: { kind: 'leaf' }, multi: true }) .then(() => undefined, (e) => e); - // The engine's generic refusal for a `multi` verdict on a driver with no - // predicate write — the floored instance offers none. + // A named refusal with its prescription, not the generic "requires an ID + // or options.multi=true" this caller already satisfied. expect(err).toBeInstanceOf(Error); - expect(err.message).toContain('options.multi=true'); + expect(err.message).toContain("Datasource 'tree' is served by a plugin driver"); + expect(err.message).toContain('multi-row predicate update'); + expect(err.message).toContain('update each record by id'); + expect(err.message).not.toContain('options.multi=true'); expect(writes).toEqual([]); }); @@ -340,10 +343,13 @@ describe('[#27277] predicate writes are not offered: every write reaches the dri const { engine, writes } = predicateDouble(); const err = await engine.delete('tree_node', { where: { kind: 'leaf' }, multi: true }) .then(() => undefined, (e) => e); - // The engine's generic refusal for a `multi` verdict on a driver with no - // predicate write — the floored instance offers none. + // A named refusal with its prescription, not the generic "requires an ID + // or options.multi=true" this caller already satisfied. expect(err).toBeInstanceOf(Error); - expect(err.message).toContain('options.multi=true'); + expect(err.message).toContain("Datasource 'tree' is served by a plugin driver"); + expect(err.message).toContain('multi-row predicate delete'); + expect(err.message).toContain('delete each record by id'); + expect(err.message).not.toContain('options.multi=true'); expect(writes).toEqual([]); }); }); diff --git a/packages/objectql/src/external-driver-floor.ts b/packages/objectql/src/external-driver-floor.ts index 2b09f08cedc..4c831fc3954 100644 --- a/packages/objectql/src/external-driver-floor.ts +++ b/packages/objectql/src/external-driver-floor.ts @@ -103,6 +103,21 @@ const WRITE_METHODS: ReadonlySet = new Set([ */ const HIDDEN: ReadonlySet = new Set(['aggregate', 'updateMany', 'deleteMany']); +/** + * The refusal of a multi-row predicate update / delete on a floored datasource, + * where the floored driver offers no `updateMany` / `deleteMany` ({@link HIDDEN}). + * A plain `Error`, as the engine's generic no-predicate-write refusal it stands + * in for (that one tells the caller to pass `options.multi=true`, which this + * caller already did). + */ +export function externalPredicateWriteRefusal(datasource: string, object: string, verb: 'update' | 'delete'): Error { + return new Error( + `Datasource '${datasource}' is served by a plugin driver; the engine does not run a multi-row predicate ` + + `${verb} on '${object}' there, because it cannot re-check which rows the driver's where selects. ` + + `Read the matching ids, then ${verb} each record by id.`, + ); +} + /** floored driver → the driver behind it */ const innerOf = new WeakMap(); /** driver → its floored driver, so wrapping one driver twice answers one instance */