From f71b41c3ed426f57b68c6da38abf0cd621d4b1fa Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 10 Jul 2026 17:55:46 +0000 Subject: [PATCH] refactor(sql): extract shared SQLite layer; close deleteAttachment race; enforce FKs First phase of #46 (SQLite split), plus the two follow-up work items from its comments: - New internal package @haverstack/sqlite-shared: schema DDL, WHERE/ ORDER building, cursor codec, row mappers, and the FTS4 sanitizer, extracted verbatim from record-adapter-sqljs so a future native SQLite adapter shares this SQL instead of drifting from a second copy. Cursor encoding moves from Node's Buffer to btoa/atob so the codec is portable to a browser adapter without change. - record-adapter-sqljs enables PRAGMA foreign_keys = ON and maps FK constraint violations to StackNotFoundError in associate(), so associating a nonexistent record fails loudly instead of silently creating an orphan row. Also fixes an sql.js quirk where db.export() (called by persist() after every write) silently resets connection pragmas, which was quietly turning FK enforcement back off after the first write. - StackRecordAdapter gains an optional deleteUnreferencedAttachmentRecords(fileId, metadataTypeId), and record-adapter-sqljs implements it as a single atomic SQL transaction. Stack.deleteAttachment() uses it when available, closing the check-then-act race where a concurrent associate() could add a reference between the reference check and the metadata delete; adapters without it fall back to the previous non-atomic sequence. Byte deletion still runs after the metadata step commits, so a crash in between leaves orphaned bytes (GC-reclaimable) rather than a dangling reference. - MemoryAdapter gains attachmentFileId filtering, needed to actually exercise deleteAttachment()'s reference check in core's test suite (previously untested). Native adapter creation, sqljs's browser-only cleanup, the adapter-local swap, and the token-store-out-of-file default are left for a follow-up pass on #46. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01RRLiGUZC344E3pgNEGAe5W --- docs/spec.md | 6 + packages/adapter-local/vitest.config.ts | 1 + packages/core/src/stack.ts | 37 +- packages/core/src/testing.ts | 7 + packages/core/src/types.ts | 14 + packages/core/tests/stack.test.ts | 75 +++ packages/record-adapter-sqljs/package.json | 1 + packages/record-adapter-sqljs/src/index.ts | 511 ++++-------------- .../record-adapter-sqljs/tests/record.test.ts | 118 +++- .../record-adapter-sqljs/vitest.config.ts | 1 + packages/sqlite-shared/package.json | 43 ++ packages/sqlite-shared/src/cursor.ts | 61 +++ packages/sqlite-shared/src/fts.ts | 62 +++ packages/sqlite-shared/src/index.ts | 18 + packages/sqlite-shared/src/mappers.ts | 77 +++ packages/sqlite-shared/src/query.ts | 147 +++++ packages/sqlite-shared/src/schema.ts | 95 ++++ packages/sqlite-shared/tests/cursor.test.ts | 55 ++ packages/sqlite-shared/tests/fts.test.ts | 41 ++ packages/sqlite-shared/tsconfig.build.json | 8 + packages/sqlite-shared/tsconfig.json | 8 + packages/sqlite-shared/vitest.config.ts | 13 + pnpm-lock.yaml | 19 + 23 files changed, 1010 insertions(+), 408 deletions(-) create mode 100644 packages/sqlite-shared/package.json create mode 100644 packages/sqlite-shared/src/cursor.ts create mode 100644 packages/sqlite-shared/src/fts.ts create mode 100644 packages/sqlite-shared/src/index.ts create mode 100644 packages/sqlite-shared/src/mappers.ts create mode 100644 packages/sqlite-shared/src/query.ts create mode 100644 packages/sqlite-shared/src/schema.ts create mode 100644 packages/sqlite-shared/tests/cursor.test.ts create mode 100644 packages/sqlite-shared/tests/fts.test.ts create mode 100644 packages/sqlite-shared/tsconfig.build.json create mode 100644 packages/sqlite-shared/tsconfig.json create mode 100644 packages/sqlite-shared/vitest.config.ts diff --git a/docs/spec.md b/docs/spec.md index b452354..b0761f5 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -481,6 +481,10 @@ const stack = await Stack.create(adapter); All adapters support the full Record API. Performance guarantees differ; correctness does not. +**`@haverstack/sqlite-shared`** is an internal, non-public package holding the SQL that's identical across every SQLite-backed record adapter — schema DDL, `WHERE`/`ORDER` building, the cursor codec, row mappers, and the FTS4 sanitizer. `record-adapter-sqljs` consumes it today; a future native SQLite adapter would too, sharing everything but the FTS dialect (FTS4 vs FTS5) and the engine binding itself. Keeping this logic in one place is what keeps a cursor minted by one SQLite-backed adapter decodable by another. + +`record-adapter-sqljs` enables SQLite foreign-key enforcement (`PRAGMA foreign_keys = ON`) so that operations like `associate()` against a nonexistent record fail loudly (`StackNotFoundError`) instead of silently creating an orphan row. + --- ## Concurrency & storage ownership @@ -538,6 +542,8 @@ const meta = results.records[0]?.content as AttachmentContent | undefined; - `Stack.deleteAttachment(fileId)` — deletes bytes and all `_attachment@1` metadata records for the file. Throws `StackConflictError` if any record still references the file. Throws `StackNotFoundError` if the file doesn't exist. - `ScopedStack.deleteAttachment(fileId)` — owner only. Throws `StackPermissionError` for non-owners. Delegates to `Stack.deleteAttachment()`. +**Atomicity of the reference check.** The reference check and the metadata-record deletes must happen as one unit — otherwise a concurrent `associate()` can add a new reference in the gap between them, leaving a dangling association after the delete completes. Adapters MAY implement `StackRecordAdapter.deleteUnreferencedAttachmentRecords(fileId, metadataTypeId)` to close this race (`Stack.deleteAttachment()` uses it when present, falling back to a non-atomic check-then-act sequence otherwise). Byte deletion always happens after the metadata step commits: a crash in between leaves orphaned bytes, which is harmless and later reclaimed by garbage collection, rather than a dangling reference, which is not. + **Deduplication:** Bytes are deduplicated — uploading the same content twice stores the binary only once. However, each call to `putAttachment()` creates a new `_attachment@1` record with its own `mimeType` and `filename`. The same `fileId` may have multiple `_attachment@1` records from separate uploads. --- diff --git a/packages/adapter-local/vitest.config.ts b/packages/adapter-local/vitest.config.ts index 97fad83..9eb9cf6 100644 --- a/packages/adapter-local/vitest.config.ts +++ b/packages/adapter-local/vitest.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ __dirname, '../record-adapter-sqljs/src/index.ts', ), + '@haverstack/sqlite-shared': resolve(__dirname, '../sqlite-shared/src/index.ts'), '@haverstack/blob-adapter-disk': resolve(__dirname, '../blob-adapter-disk/src/index.ts'), }, }, diff --git a/packages/core/src/stack.ts b/packages/core/src/stack.ts index d7de5d9..e6eb494 100644 --- a/packages/core/src/stack.ts +++ b/packages/core/src/stack.ts @@ -765,6 +765,31 @@ export class Stack implements StackClient { * Throws StackNotFoundError if neither metadata records nor bytes exist. */ async deleteAttachment(fileId: string): Promise { + const metadataTypeId = `${SYSTEM_TYPES.ATTACHMENT}@1`; + const deletedRecordIds = this.adapter.deleteUnreferencedAttachmentRecords + ? await this.adapter.deleteUnreferencedAttachmentRecords(fileId, metadataTypeId) + : await this.deleteUnreferencedAttachmentRecordsFallback(fileId, metadataTypeId); + + if (!deletedRecordIds.length) { + try { + await this.adapter.getAttachment(fileId); + } catch { + throw new StackNotFoundError(`Attachment not found: "${fileId}"`); + } + } + + await this.adapter.deleteAttachment(fileId); + } + + /** + * Non-atomic fallback for adapters that don't implement + * deleteUnreferencedAttachmentRecords(): a concurrent associate() can + * race between the reference check below and the deletes it guards. + */ + private async deleteUnreferencedAttachmentRecordsFallback( + fileId: string, + metadataTypeId: string, + ): Promise { const refResult = await this.query({ filter: { attachmentFileId: fileId }, limit: 1 }); if (refResult.records.length > 0) { throw new StackConflictError('Attachment is still referenced by one or more records'); @@ -772,7 +797,7 @@ export class Stack implements StackClient { const metaResult = await this.query({ filter: { - typeId: `${SYSTEM_TYPES.ATTACHMENT}@1`, + typeId: metadataTypeId, ...(this.features.contentFieldQuery && { content: { fileId } }), }, }); @@ -780,19 +805,11 @@ export class Stack implements StackClient { ? metaResult.records : metaResult.records.filter((r) => (r.content as AttachmentContent).fileId === fileId); - if (!metaRecords.length) { - try { - await this.adapter.getAttachment(fileId); - } catch { - throw new StackNotFoundError(`Attachment not found: "${fileId}"`); - } - } - for (const record of metaRecords) { await this.delete(record.id, { hard: true }); } - await this.adapter.deleteAttachment(fileId); + return metaRecords.map((r) => r.id); } // ------------------------------------------------------- diff --git a/packages/core/src/testing.ts b/packages/core/src/testing.ts index 9090a3c..8386aad 100644 --- a/packages/core/src/testing.ts +++ b/packages/core/src/testing.ts @@ -99,6 +99,13 @@ export class MemoryAdapter implements StackAdapter { const ids = Array.isArray(f.entityId) ? f.entityId : [f.entityId]; results = results.filter((r) => r.entityId !== undefined && ids.includes(r.entityId)); } + if (f.attachmentFileId) { + results = results.filter((r) => + (r.associations ?? []).some( + (a) => a.kind === 'attachment' && a.fileId === f.attachmentFileId, + ), + ); + } const limit = query.limit ?? 50; const start = query.cursor ? Number(query.cursor) : 0; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 33a3608..e764c3e 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -388,6 +388,20 @@ export interface StackRecordAdapter { getType(id: TypeId): Promise; listTypes(): Promise; + /** + * Atomically verify fileId is unreferenced by any record's attachment + * association, then hard-delete every record of `metadataTypeId` whose + * content.fileId matches it — all within a single adapter call, so + * nothing can add a new reference between the check and the delete. + * Returns the ids of the deleted metadata records (empty if none exist, + * e.g. bare bytes left by an interrupted upload). Throws + * StackConflictError if fileId is still referenced. + * + * Optional: Stack.deleteAttachment() falls back to a non-atomic + * check-then-act sequence for adapters that don't implement this. + */ + deleteUnreferencedAttachmentRecords?(fileId: FileId, metadataTypeId: TypeId): Promise; + // Lifecycle flush?(): Promise; close?(): Promise; diff --git a/packages/core/tests/stack.test.ts b/packages/core/tests/stack.test.ts index 010c93d..e63ab72 100644 --- a/packages/core/tests/stack.test.ts +++ b/packages/core/tests/stack.test.ts @@ -853,3 +853,78 @@ describe('putAttachment', () => { expect(result.records[0].entityId).toBeUndefined(); }); }); + +// ------------------------------------------------------- +// deleteAttachment +// ------------------------------------------------------- + +describe('deleteAttachment', () => { + test('throws StackConflictError when a record still references the file (fallback path)', async () => { + const data = new Uint8Array([1, 2, 3]); + const fileId = await stack.putAttachment(data, 'image/png'); + const note = await stack.create(NOTE_V1, { text: 'hi' }); + await stack.associate(note.id, { + kind: 'attachment', + label: 'cover', + fileId, + mimeType: 'image/png', + }); + + await expect(stack.deleteAttachment(fileId)).rejects.toThrow(StackConflictError); + }); + + test('deletes the _attachment@1 metadata record when unreferenced (fallback path)', async () => { + const data = new Uint8Array([1, 2, 3]); + const fileId = await stack.putAttachment(data, 'image/png'); + + await stack.deleteAttachment(fileId); + + const result = await stack.query({ filter: { typeId: '_attachment@1' } }); + expect(result.records).toHaveLength(0); + }); + + test('throws StackNotFoundError when neither metadata nor bytes exist', async () => { + class NoBytesAdapter extends MemoryAdapter { + async getAttachment(_fileId: string): Promise { + throw new Error('not found'); + } + } + const noBytesStack = await Stack.create( + new NoBytesAdapter({ ownerEntityId: 'owner-123', timezone: 'UTC' }), + ); + + await expect(noBytesStack.deleteAttachment('nonexistent-file')).rejects.toThrow( + StackNotFoundError, + ); + }); + + test('prefers the adapter atomic path over the fallback when the adapter implements it', async () => { + const calls: string[] = []; + class AtomicAdapter extends MemoryAdapter { + async deleteUnreferencedAttachmentRecords( + fileId: string, + metadataTypeId: string, + ): Promise { + calls.push('atomic'); + const toDelete = [...this.records.values()].filter( + (r) => + r.typeId === metadataTypeId && (r.content as Record).fileId === fileId, + ); + for (const r of toDelete) { + this.records.delete(r.id); + this.order.splice(this.order.indexOf(r.id), 1); + } + return toDelete.map((r) => r.id); + } + } + const atomicStack = await Stack.create( + new AtomicAdapter({ ownerEntityId: 'owner-123', timezone: 'UTC' }), + ); + await atomicStack.defineType(NOTE_V1, 'Note', { text: { kind: 'text', required: true } }); + + const fileId = await atomicStack.putAttachment(new Uint8Array([9]), 'image/png'); + await atomicStack.deleteAttachment(fileId); + + expect(calls).toEqual(['atomic']); + }); +}); diff --git a/packages/record-adapter-sqljs/package.json b/packages/record-adapter-sqljs/package.json index da72057..64209a7 100644 --- a/packages/record-adapter-sqljs/package.json +++ b/packages/record-adapter-sqljs/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@haverstack/core": "workspace:*", + "@haverstack/sqlite-shared": "workspace:*", "sql.js": "^1.14.0" }, "devDependencies": { diff --git a/packages/record-adapter-sqljs/src/index.ts b/packages/record-adapter-sqljs/src/index.ts index ee9bf82..bdc3765 100644 --- a/packages/record-adapter-sqljs/src/index.ts +++ b/packages/record-adapter-sqljs/src/index.ts @@ -19,6 +19,11 @@ * Also exposes token management methods (createToken, * lookupToken, listTokens, revokeToken) used by server * implementations to issue and validate bearer tokens. + * + * Schema DDL, WHERE/ORDER building, cursor codec, row mappers, + * and the FTS4 sanitizer live in @haverstack/sqlite-shared — + * shared with the native record-adapter-sqlite so the two + * engines can't drift on engine-independent SQL. */ import initSqlJs from 'sql.js'; @@ -31,6 +36,8 @@ import type { StackRecord, StackType, TypeId, + RecordId, + FileId, RecordVersion, StackQuery, QueryResult, @@ -38,7 +45,23 @@ import type { Permission, AdapterCapabilities, } from '@haverstack/core'; -import { applyMergePatch, StackQueryError } from '@haverstack/core'; +import { applyMergePatch, StackConflictError, StackNotFoundError } from '@haverstack/core'; +import { + RECORD_SCHEMA_SQL, + FTS4_SCHEMA_SQL, + PRAGMA_FOREIGN_KEYS_ON, + buildWhereClause, + buildOrderClause, + getSortField, + makeCursor, + rowToRecord, + rowToAssociation, + rowToType, + rowToVersion, + toMs, + fromMs, + sanitizeFts4Query, +} from '@haverstack/sqlite-shared'; // ------------------------------------------------------- // Types @@ -75,157 +98,7 @@ export type TokenInfo = { expiresAt?: Date; }; -// ------------------------------------------------------- -// SQL — schema -// ------------------------------------------------------- - -const SCHEMA_SQL = ` - CREATE TABLE IF NOT EXISTS records ( - id TEXT PRIMARY KEY, - type_id TEXT NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - content TEXT NOT NULL CHECK (json_valid(content)), - version INTEGER NOT NULL DEFAULT 1, - parent_id TEXT, - entity_id TEXT, - app_id TEXT, - deleted_at INTEGER, - permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)) - ) STRICT; - - CREATE TABLE IF NOT EXISTS associations ( - record_id TEXT NOT NULL REFERENCES records(id), - kind TEXT NOT NULL CHECK (kind IN ('tag', 'attachment', 'relationship')), - label TEXT NOT NULL, - file_id TEXT NOT NULL DEFAULT '', - mime_type TEXT, - related_id TEXT NOT NULL DEFAULT '', - PRIMARY KEY (record_id, kind, label, file_id, related_id) - ) STRICT; - - CREATE TABLE IF NOT EXISTS versions ( - record_id TEXT NOT NULL REFERENCES records(id), - version INTEGER NOT NULL, - content TEXT NOT NULL CHECK (json_valid(content)), - updated_at INTEGER NOT NULL, - entity_id TEXT, - associations TEXT CHECK (associations IS NULL OR json_valid(associations)), - permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)), - PRIMARY KEY (record_id, version) - ) STRICT; - - CREATE TABLE IF NOT EXISTS types ( - id TEXT PRIMARY KEY, - base_id TEXT NOT NULL, - version INTEGER NOT NULL, - name TEXT NOT NULL, - schema TEXT NOT NULL CHECK (json_valid(schema)), - schema_hash TEXT NOT NULL, - migrates_from TEXT, - created_at INTEGER NOT NULL - ) STRICT; - - CREATE TABLE IF NOT EXISTS tokens ( - id TEXT PRIMARY KEY, - token_hash TEXT NOT NULL UNIQUE, - entity_id TEXT NOT NULL, - label TEXT, - created_at INTEGER NOT NULL, - expires_at INTEGER - ) STRICT; - - -- Indexes - CREATE INDEX IF NOT EXISTS idx_records_type_id ON records(type_id); - CREATE INDEX IF NOT EXISTS idx_records_parent_id ON records(parent_id); - CREATE INDEX IF NOT EXISTS idx_records_entity_id ON records(entity_id); - CREATE INDEX IF NOT EXISTS idx_records_app_id ON records(app_id); - CREATE INDEX IF NOT EXISTS idx_records_deleted_at ON records(deleted_at); - CREATE INDEX IF NOT EXISTS idx_records_created_at ON records(created_at); - CREATE INDEX IF NOT EXISTS idx_records_updated_at ON records(updated_at); - CREATE INDEX IF NOT EXISTS idx_assoc_record_id ON associations(record_id); - CREATE INDEX IF NOT EXISTS idx_assoc_kind_label ON associations(kind, label); - CREATE INDEX IF NOT EXISTS idx_assoc_kind_file_id ON associations(kind, file_id); - CREATE INDEX IF NOT EXISTS idx_types_base_id ON types(base_id); - CREATE INDEX IF NOT EXISTS idx_tokens_hash ON tokens(token_hash); - - -- Full-text search (FTS4 — compatible with sql.js) - CREATE VIRTUAL TABLE IF NOT EXISTS records_fts USING fts4( - content, - content='records' - ); -`; - -// ------------------------------------------------------- -// Helpers -// ------------------------------------------------------- - -const toMs = (d: Date): number => d.getTime(); -const fromMs = (ms: number): Date => new Date(ms); - -const rowToRecord = (row: Record, associations: Association[]): StackRecord => { - const record: StackRecord = { - id: row.id as string, - typeId: row.type_id as string, - createdAt: fromMs(row.created_at as number), - updatedAt: fromMs(row.updated_at as number), - content: JSON.parse(row.content as string), - version: row.version as number, - }; - if (row.parent_id) record.parentId = row.parent_id as string; - if (row.entity_id) record.entityId = row.entity_id as string; - if (row.app_id) record.appId = row.app_id as string; - if (row.deleted_at) record.deletedAt = fromMs(row.deleted_at as number); - if (row.permissions) record.permissions = JSON.parse(row.permissions as string); - if (associations.length) record.associations = associations; - return record; -}; - -const rowToAssociation = (row: Record): Association => { - if (row.kind === 'tag') { - return { kind: 'tag', label: row.label as string }; - } - if (row.kind === 'attachment') { - return { - kind: 'attachment', - label: row.label as string, - fileId: row.file_id as string, - mimeType: row.mime_type as string, - }; - } - // relationship - return { - kind: 'relationship', - label: row.label as string, - recordId: row.related_id as string, - }; -}; - -const rowToType = (row: Record): StackType => { - const t: StackType = { - id: row.id as string, - baseId: row.base_id as string, - version: row.version as number, - name: row.name as string, - schema: JSON.parse(row.schema as string), - schemaHash: row.schema_hash as string, - createdAt: fromMs(row.created_at as number), - }; - if (row.migrates_from) t.migratesFrom = row.migrates_from as string; - return t; -}; - -const rowToVersion = (row: Record): RecordVersion => { - const v: RecordVersion = { - version: row.version as number, - content: JSON.parse(row.content as string), - updatedAt: fromMs(row.updated_at as number), - }; - if (row.entity_id) v.entityId = row.entity_id as string; - if (row.associations) v.associations = JSON.parse(row.associations as string); - if (row.permissions) v.permissions = JSON.parse(row.permissions as string); - return v; -}; +const SCHEMA_SQL = `${RECORD_SCHEMA_SQL}\n${FTS4_SCHEMA_SQL}`; // ------------------------------------------------------- // Storage ownership lock @@ -278,232 +151,12 @@ const releaseLock = (dbPath: string): void => { }; // ------------------------------------------------------- -// FTS4 query sanitization +// Foreign key enforcement // ------------------------------------------------------- -/** - * Sanitizes user input for safe use in an FTS4 MATCH clause. - * - * FTS4's query language supports operators (AND/OR/NOT, phrases, NEAR, wildcards) - * that can be expensive or cause parse errors with untrusted input. - * This function removes the dangerous operators while preserving useful ones: - * kept: AND/OR/NOT (with a left operand), phrase queries ("…"), implicit AND - * removed: wildcards (*), NEAR/N, bare NOT (no left operand) - * capped: parenthesis nesting depth (default: 2) - * - * A query timeout is not implementable here: sql.js runs SQLite as synchronous - * WASM that blocks the JS event loop, and db.interrupt() is not exposed. Moving - * sql.js into a Worker thread would enable a timeout via interrupt() but is out - * of scope for this change. - */ -const sanitizeFts4Query = (query: string, maxDepth = 2): string => { - if (!query) return ''; - - // Remove wildcards - let clean = query.replace(/\*/g, ''); - - // Remove NEAR operator (handles NEAR and NEAR/N variants) - clean = clean.replace(/\bNEAR(?:\/\d+)?\b/gi, ''); - - // Strip bare NOT with no left operand — FTS4 requires "term NOT term", not "NOT term" - clean = clean.replace(/(?:^|\(\s*)NOT\s+/gi, (m) => m.replace(/NOT\s+/i, '')); - - // Enforce max nesting depth; replace excess ( with a space and discard unmatched ) - let currentDepth = 0; - let result = ''; - for (const char of clean) { - if (char === '(') { - if (currentDepth < maxDepth) { - currentDepth++; - result += char; - } else result += ' '; - } else if (char === ')') { - if (currentDepth > 0) { - currentDepth--; - result += char; - } else result += ' '; - } else { - result += char; - } - } - - // Auto-close any unclosed parens - if (currentDepth > 0) result += ')'.repeat(currentDepth); - - // Iteratively remove empty paren pairs left behind by NEAR/NOT stripping - let prev: string; - do { - prev = result; - result = result.replace(/\(\s*\)/g, ' '); - } while (result !== prev); - - return result.replace(/\s+/g, ' ').replace(/\(\s+/g, '(').replace(/\s+\)/g, ')').trim(); -}; - -// ------------------------------------------------------- -// Query building -// ------------------------------------------------------- - -type SortField = 'createdAt' | 'updatedAt' | 'version'; - -const getSortField = (query: StackQuery): SortField => query.sort?.field ?? 'createdAt'; - -const getSortColumn = (field: SortField): string => - field === 'createdAt' ? 'created_at' : field === 'updatedAt' ? 'updated_at' : 'version'; - -const buildWhereClause = (query: StackQuery): { sql: string; params: unknown[] } => { - const conditions: string[] = ["r.id != '_config'"]; - const params: unknown[] = []; - const f = query.filter ?? {}; - - if (!f.includeDeleted) { - conditions.push('r.deleted_at IS NULL'); - } - - if (f.typeId !== undefined) { - const ids = Array.isArray(f.typeId) ? f.typeId : [f.typeId]; - conditions.push(`r.type_id IN (${ids.map(() => '?').join(',')})`); - params.push(...ids); - } - - if (f.parentId !== undefined) { - if (f.parentId === null) { - conditions.push('r.parent_id IS NULL'); - } else { - conditions.push('r.parent_id = ?'); - params.push(f.parentId); - } - } - - if (f.appId !== undefined) { - const ids = Array.isArray(f.appId) ? f.appId : [f.appId]; - conditions.push(`r.app_id IN (${ids.map(() => '?').join(',')})`); - params.push(...ids); - } - - if (f.entityId !== undefined) { - const ids = Array.isArray(f.entityId) ? f.entityId : [f.entityId]; - conditions.push(`r.entity_id IN (${ids.map(() => '?').join(',')})`); - params.push(...ids); - } - - if (f.createdAt?.after) { - conditions.push('r.created_at > ?'); - params.push(toMs(f.createdAt.after)); - } - if (f.createdAt?.before) { - conditions.push('r.created_at < ?'); - params.push(toMs(f.createdAt.before)); - } - if (f.updatedAt?.after) { - conditions.push('r.updated_at > ?'); - params.push(toMs(f.updatedAt.after)); - } - if (f.updatedAt?.before) { - conditions.push('r.updated_at < ?'); - params.push(toMs(f.updatedAt.before)); - } - - // Tag filter — record must have ALL specified tags - if (f.tags?.length) { - for (const tag of f.tags) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'tag' AND a.label = ?)`, - ); - params.push(tag); - } - } - - // Attachment label filter - if (f.hasAttachment) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.label = ?)`, - ); - params.push(f.hasAttachment); - } - - // Attachment file ID filter — find records that reference a specific file - if (f.attachmentFileId) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.file_id = ?)`, - ); - params.push(f.attachmentFileId); - } - - // Relationship filter - if (f.relatedTo) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'relationship' AND a.related_id = ?` + - (f.relatedTo.label ? ` AND a.label = ?` : '') + - `)`, - ); - params.push(f.relatedTo.recordId); - if (f.relatedTo.label) params.push(f.relatedTo.label); - } - - // Content field filters (top-level scalar exact match) - if (f.content) { - for (const [key, value] of Object.entries(f.content)) { - conditions.push(`json_extract(r.content, ?) = ?`); - params.push(`$.${key}`, value); - } - } - - if (f.search) { - const sanitized = sanitizeFts4Query(f.search); - if (sanitized) { - conditions.push(`r.rowid IN (SELECT rowid FROM records_fts WHERE records_fts MATCH ?)`); - params.push(sanitized); - } - } - - // Cursor (sort-field value + id for stable pagination) - if (query.cursor) { - const parts = Buffer.from(query.cursor, 'base64').toString().split('|'); - // New format: field|value|id (3 parts). Legacy format: value|id (2 parts, implies createdAt). - const [cursorField, cursorValue, cursorId] = - parts.length === 3 - ? (parts as [string, string, string]) - : (['createdAt', parts[0], parts[1]] as [string, string, string]); - if (cursorField === undefined || cursorValue === undefined || cursorId === undefined) { - throw new StackQueryError(`Invalid cursor: malformed "${query.cursor}"`); - } - const validSortFields: SortField[] = ['createdAt', 'updatedAt', 'version']; - if (!validSortFields.includes(cursorField as SortField)) { - throw new StackQueryError(`Invalid cursor: unknown sort field "${cursorField}"`); - } - const numericValue = Number(cursorValue); - if (!isFinite(numericValue)) { - throw new StackQueryError(`Invalid cursor: non-numeric sort value`); - } - const col = getSortColumn(cursorField as SortField); - const sortDir = query.sort?.direction ?? 'desc'; - const op = sortDir === 'asc' ? '>' : '<'; - conditions.push(`(r.${col} ${op} ? OR (r.${col} = ? AND r.id ${op} ?))`); - params.push(numericValue, numericValue, cursorId); - } - - return { - sql: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '', - params, - }; -}; - -const buildOrderClause = (query: StackQuery): string => { - const field = getSortField(query); - const dir = (query.sort?.direction ?? 'desc').toUpperCase(); - return `ORDER BY r.${getSortColumn(field)} ${dir}, r.id ${dir}`; -}; - -const makeCursor = (record: StackRecord, field: SortField): string => { - const value = - field === 'updatedAt' - ? toMs(record.updatedAt) - : field === 'version' - ? record.version - : toMs(record.createdAt); - return Buffer.from(`${field}|${value}|${record.id}`).toString('base64'); -}; +/** sql.js's SQLite build reports FK violations as a plain Error with this message. */ +const isForeignKeyViolation = (err: unknown): boolean => + err instanceof Error && err.message.includes('FOREIGN KEY constraint failed'); // ------------------------------------------------------- // SQLiteRecordAdapter @@ -541,6 +194,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { const SQL = await initSqlJs(); const adapter = new SQLiteRecordAdapter(SQL, opts.path); adapter.db = new SQL.Database(); + adapter.db.run(PRAGMA_FOREIGN_KEYS_ON); adapter.db.run(SCHEMA_SQL); const now = Date.now(); adapter.db.run( @@ -570,6 +224,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { const adapter = new SQLiteRecordAdapter(SQL, opts.path); const fileBuffer = readFileSync(opts.path); adapter.db = new SQL.Database(fileBuffer); + adapter.db.run(PRAGMA_FOREIGN_KEYS_ON); adapter.db.run(SCHEMA_SQL); adapter.readConfig(); return adapter; @@ -587,6 +242,10 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { */ private persist(): void { const data = this.db.export(); + // sql.js quirk: export() resets connection-level pragmas (including + // foreign_keys) on the live Database object — reapply immediately so + // FK enforcement doesn't silently go stale after the first write. + this.db.run(PRAGMA_FOREIGN_KEYS_ON); const tmpPath = join( dirname(this.path), `.${basename(this.path)}.tmp-${randomBytes(6).toString('hex')}`, @@ -671,13 +330,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { async deleteRecord(id: string, opts: { hard?: boolean } = {}): Promise { if (opts.hard) { - this.db.run('DELETE FROM associations WHERE record_id = ?', [id]); - this.db.run('DELETE FROM versions WHERE record_id = ?', [id]); - this.db.run( - `DELETE FROM records_fts WHERE docid = (SELECT rowid FROM records WHERE id = ?)`, - [id], - ); - this.db.run('DELETE FROM records WHERE id = ?', [id]); + this.hardDeleteRecord(id); } else { this.db.run( 'UPDATE records SET deleted_at = ?, version = version + 1, updated_at = ? WHERE id = ?', @@ -687,6 +340,16 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { this.persist(); } + /** Deletes a record's associations, versions, FTS entry, and row — no persist(). */ + private hardDeleteRecord(id: string): void { + this.db.run('DELETE FROM associations WHERE record_id = ?', [id]); + this.db.run('DELETE FROM versions WHERE record_id = ?', [id]); + this.db.run(`DELETE FROM records_fts WHERE docid = (SELECT rowid FROM records WHERE id = ?)`, [ + id, + ]); + this.db.run('DELETE FROM records WHERE id = ?', [id]); + } + async undeleteRecord(id: string): Promise { this.db.run( 'UPDATE records SET deleted_at = NULL, version = version + 1, updated_at = ? WHERE id = ?', @@ -745,7 +408,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { } async queryRecords(query: StackQuery): Promise { - const { sql: where, params } = buildWhereClause(query); + const { sql: where, params } = buildWhereClause(query, sanitizeFts4Query); const order = buildOrderClause(query); const limit = query.limit ?? 50; @@ -776,6 +439,46 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { return { records, cursor, total }; } + /** + * Atomically verify fileId is unreferenced, then hard-delete every + * metadataTypeId record whose content.fileId matches it. See + * StackRecordAdapter.deleteUnreferencedAttachmentRecords(). Runs as a + * real SQL transaction and, being entirely synchronous SQLite calls + * with no `await` in between, nothing else can interleave between the + * reference check and the deletes. + */ + async deleteUnreferencedAttachmentRecords( + fileId: FileId, + metadataTypeId: TypeId, + ): Promise { + this.db.run('BEGIN'); + try { + const referenced = this.execQuery<{ found: number }>( + `SELECT 1 as found FROM associations WHERE kind = 'attachment' AND file_id = ? LIMIT 1`, + [fileId], + ); + if (referenced.length) { + throw new StackConflictError('Attachment is still referenced by one or more records'); + } + + const metaRows = this.execQuery<{ id: string }>( + `SELECT id FROM records WHERE type_id = ? AND json_extract(content, '$.fileId') = ?`, + [metadataTypeId, fileId], + ); + const deletedIds = metaRows.map((row) => row.id); + for (const id of deletedIds) { + this.hardDeleteRecord(id); + } + + this.db.run('COMMIT'); + if (deletedIds.length) this.persist(); + return deletedIds; + } catch (err) { + this.db.run('ROLLBACK'); + throw err; + } + } + // ------------------------------------------------------- // Versions // ------------------------------------------------------- @@ -946,21 +649,35 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { ]); } + /** + * FK enforcement (PRAGMA_FOREIGN_KEYS_ON) means inserting an + * association against a record that doesn't exist throws — mapped + * here to StackNotFoundError so associate() on a nonexistent record + * fails loudly with a proper error instead of silently creating an + * orphan row. + */ private insertAssociations(recordId: string, associations: Association[]): void { for (const assoc of associations) { - this.db.run( - `INSERT OR IGNORE INTO associations - (record_id, kind, label, file_id, mime_type, related_id) - VALUES (?, ?, ?, ?, ?, ?)`, - [ - recordId, - assoc.kind, - assoc.label, - assoc.kind === 'attachment' ? assoc.fileId : '', - assoc.kind === 'attachment' ? assoc.mimeType : null, - assoc.kind === 'relationship' ? assoc.recordId : '', - ], - ); + try { + this.db.run( + `INSERT OR IGNORE INTO associations + (record_id, kind, label, file_id, mime_type, related_id) + VALUES (?, ?, ?, ?, ?, ?)`, + [ + recordId, + assoc.kind, + assoc.label, + assoc.kind === 'attachment' ? assoc.fileId : '', + assoc.kind === 'attachment' ? assoc.mimeType : null, + assoc.kind === 'relationship' ? assoc.recordId : '', + ], + ); + } catch (err) { + if (isForeignKeyViolation(err)) { + throw new StackNotFoundError(`Record not found: "${recordId}"`); + } + throw err; + } } } diff --git a/packages/record-adapter-sqljs/tests/record.test.ts b/packages/record-adapter-sqljs/tests/record.test.ts index 1c925b4..d7a6e3b 100644 --- a/packages/record-adapter-sqljs/tests/record.test.ts +++ b/packages/record-adapter-sqljs/tests/record.test.ts @@ -3,7 +3,7 @@ import { mkdirSync, rmSync, existsSync, readdirSync, writeFileSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { SQLiteRecordAdapter } from '../src/index.js'; -import { StackQueryError } from '@haverstack/core'; +import { StackConflictError, StackNotFoundError, StackQueryError } from '@haverstack/core'; import type { StackRecord } from '@haverstack/core'; // ------------------------------------------------------- @@ -840,6 +840,27 @@ describe('associations', () => { const retrieved = await adapter.getRecord(record.id); expect(retrieved?.version).toBe(3); }); + + test('associate on a nonexistent record throws StackNotFoundError instead of creating an orphan row', async () => { + const adapter = await initAdapter(); + await expect( + adapter.associate('does-not-exist', { kind: 'tag', label: 'starred' }), + ).rejects.toThrow(StackNotFoundError); + }); + + test('FK enforcement survives across persist() calls (sql.js resets pragmas on export())', async () => { + const adapter = await initAdapter(); + const record = makeRecord(); + await adapter.createRecord(record); + // Several persist()-triggering writes first, to make sure the pragma + // reset sql.js applies on export() doesn't leak past the first one. + await adapter.associate(record.id, { kind: 'tag', label: 'starred' }); + await adapter.patchContent(record.id, { text: 'updated' }); + + await expect( + adapter.associate('still-does-not-exist', { kind: 'tag', label: 'x' }), + ).rejects.toThrow(StackNotFoundError); + }); }); // ------------------------------------------------------- @@ -1015,6 +1036,101 @@ describe('commitMigration', () => { }); }); +// ------------------------------------------------------- +// deleteUnreferencedAttachmentRecords +// ------------------------------------------------------- + +describe('deleteUnreferencedAttachmentRecords', () => { + const ATTACHMENT_TYPE = 'com.example.test/_attachment@1'; + + test('throws StackConflictError when a record still references the file', async () => { + const adapter = await initAdapter(); + const record = makeRecord(); + await adapter.createRecord(record); + await adapter.associate(record.id, { + kind: 'attachment', + label: 'cover', + fileId: 'file-1', + mimeType: 'image/png', + }); + + await expect( + adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE), + ).rejects.toThrow(StackConflictError); + }); + + test('deletes an unreferenced metadata record and returns its id', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'file-1' } }), + ); + + const deleted = await adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE); + expect(deleted).toEqual(['meta1']); + expect(await adapter.getRecord('meta1')).toBeNull(); + }); + + test('deletes every metadata record sharing the same fileId (dedup case)', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'shared-file' } }), + ); + await adapter.createRecord( + makeRecord({ id: 'meta2', typeId: ATTACHMENT_TYPE, content: { fileId: 'shared-file' } }), + ); + + const deleted = await adapter.deleteUnreferencedAttachmentRecords( + 'shared-file', + ATTACHMENT_TYPE, + ); + expect(deleted.sort()).toEqual(['meta1', 'meta2']); + }); + + test('returns an empty array when no metadata records exist for the file', async () => { + const adapter = await initAdapter(); + const deleted = await adapter.deleteUnreferencedAttachmentRecords( + 'nonexistent-file', + ATTACHMENT_TYPE, + ); + expect(deleted).toEqual([]); + }); + + test('rolls back and leaves the metadata record intact when the reference check fails', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'file-1' } }), + ); + const referencing = makeRecord({ id: 'referencing' }); + await adapter.createRecord(referencing); + await adapter.associate(referencing.id, { + kind: 'attachment', + label: 'cover', + fileId: 'file-1', + mimeType: 'image/png', + }); + + await expect( + adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE), + ).rejects.toThrow(StackConflictError); + expect(await adapter.getRecord('meta1')).not.toBeNull(); + }); + + test('deleting a metadata record removes its version history too', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'file-1' } }), + ); + await adapter.saveVersion('meta1', { + version: 1, + content: { fileId: 'file-1' }, + updatedAt: new Date(), + }); + + await adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE); + expect(await adapter.getVersions('meta1')).toEqual([]); + }); +}); + // ------------------------------------------------------- // Tokens // ------------------------------------------------------- diff --git a/packages/record-adapter-sqljs/vitest.config.ts b/packages/record-adapter-sqljs/vitest.config.ts index e0ec2fb..cd0e1e5 100644 --- a/packages/record-adapter-sqljs/vitest.config.ts +++ b/packages/record-adapter-sqljs/vitest.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ resolve: { alias: { '@haverstack/core': resolve(__dirname, '../core/src/index.ts'), + '@haverstack/sqlite-shared': resolve(__dirname, '../sqlite-shared/src/index.ts'), }, }, test: { diff --git a/packages/sqlite-shared/package.json b/packages/sqlite-shared/package.json new file mode 100644 index 0000000..1054104 --- /dev/null +++ b/packages/sqlite-shared/package.json @@ -0,0 +1,43 @@ +{ + "name": "@haverstack/sqlite-shared", + "version": "0.1.0", + "description": "Engine-independent SQL building blocks shared between Haverstack's SQLite-backed record adapters. Internal package, not a StackAdapter implementation.", + "type": "module", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "https://github.com/haverstack/core", + "directory": "packages/sqlite-shared" + }, + "license": "CC0-1.0", + "keywords": [ + "haverstack", + "sqlite", + "internal" + ], + "scripts": { + "prepublishOnly": "pnpm run build", + "build": "tsc -p tsconfig.build.json", + "test": "vitest run", + "typecheck": "tsc --noEmit", + "lint": "eslint src tests" + }, + "dependencies": { + "@haverstack/core": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.5.0", + "vitest": "^2.0.0" + } +} diff --git a/packages/sqlite-shared/src/cursor.ts b/packages/sqlite-shared/src/cursor.ts new file mode 100644 index 0000000..13704d4 --- /dev/null +++ b/packages/sqlite-shared/src/cursor.ts @@ -0,0 +1,61 @@ +/** + * Cursor encode/decode for keyset pagination. Uses btoa/atob rather than + * Node's Buffer so the same codec works unmodified in a browser adapter — + * cursor payloads here are plain ASCII ("field|value|id"), so no UTF-8 + * encoding step is needed. + */ + +import { StackQueryError } from '@haverstack/core'; +import type { StackRecord } from '@haverstack/core'; + +export type SortField = 'createdAt' | 'updatedAt' | 'version'; + +export const SORT_FIELDS: readonly SortField[] = ['createdAt', 'updatedAt', 'version']; + +export const encodeCursor = (field: SortField, value: number, id: string): string => + btoa(`${field}|${value}|${id}`); + +export type DecodedCursor = { field: SortField; value: number; id: string }; + +/** Throws StackQueryError for any malformed, corrupt, or unrecognized cursor. */ +export const decodeCursor = (cursor: string): DecodedCursor => { + let decoded: string; + try { + decoded = atob(cursor); + } catch { + throw new StackQueryError(`Invalid cursor: malformed "${cursor}"`); + } + const parts = decoded.split('|'); + // New format: field|value|id (3 parts). Legacy format: value|id (2 parts, implies createdAt). + const [field, value, id] = + parts.length === 3 + ? (parts as [string, string, string]) + : (['createdAt', ...parts] as [string, string, string]); + if (field === undefined || value === undefined || id === undefined) { + throw new StackQueryError(`Invalid cursor: malformed "${cursor}"`); + } + if (!SORT_FIELDS.includes(field as SortField)) { + throw new StackQueryError(`Invalid cursor: unknown sort field "${field}"`); + } + const numericValue = Number(value); + if (!isFinite(numericValue)) { + throw new StackQueryError(`Invalid cursor: non-numeric sort value`); + } + return { field: field as SortField, value: numericValue, id }; +}; + +export const getSortField = (query: { sort?: { field?: SortField } }): SortField => + query.sort?.field ?? 'createdAt'; + +export const getSortColumn = (field: SortField): string => + field === 'createdAt' ? 'created_at' : field === 'updatedAt' ? 'updated_at' : 'version'; + +export const makeCursor = (record: StackRecord, field: SortField): string => { + const value = + field === 'updatedAt' + ? record.updatedAt.getTime() + : field === 'version' + ? record.version + : record.createdAt.getTime(); + return encodeCursor(field, value, record.id); +}; diff --git a/packages/sqlite-shared/src/fts.ts b/packages/sqlite-shared/src/fts.ts new file mode 100644 index 0000000..d5c3db5 --- /dev/null +++ b/packages/sqlite-shared/src/fts.ts @@ -0,0 +1,62 @@ +/** + * FTS4 query sanitization. + * + * FTS4's query language supports operators (AND/OR/NOT, phrases, NEAR, wildcards) + * that can be expensive or cause parse errors with untrusted input. + * This function removes the dangerous operators while preserving useful ones: + * kept: AND/OR/NOT (with a left operand), phrase queries ("…"), implicit AND + * removed: wildcards (*), NEAR/N, bare NOT (no left operand) + * capped: parenthesis nesting depth (default: 2) + * + * A query timeout is not implementable for a synchronous, in-process SQLite + * engine (sql.js runs as WASM that blocks the JS event loop; native SQLite's + * interrupt() would need a worker thread to be usable as a timeout) — out of + * scope here. + * + * FTS5 (used by the native adapter) has different grammar and needs its own + * sanitizer reviewed against FTS5's rules rather than assuming this one + * transfers — tracked with the native adapter's work, not duplicated here. + */ +export const sanitizeFts4Query = (query: string, maxDepth = 2): string => { + if (!query) return ''; + + // Remove wildcards + let clean = query.replace(/\*/g, ''); + + // Remove NEAR operator (handles NEAR and NEAR/N variants) + clean = clean.replace(/\bNEAR(?:\/\d+)?\b/gi, ''); + + // Strip bare NOT with no left operand — FTS4 requires "term NOT term", not "NOT term" + clean = clean.replace(/(?:^|\(\s*)NOT\s+/gi, (m) => m.replace(/NOT\s+/i, '')); + + // Enforce max nesting depth; replace excess ( with a space and discard unmatched ) + let currentDepth = 0; + let result = ''; + for (const char of clean) { + if (char === '(') { + if (currentDepth < maxDepth) { + currentDepth++; + result += char; + } else result += ' '; + } else if (char === ')') { + if (currentDepth > 0) { + currentDepth--; + result += char; + } else result += ' '; + } else { + result += char; + } + } + + // Auto-close any unclosed parens + if (currentDepth > 0) result += ')'.repeat(currentDepth); + + // Iteratively remove empty paren pairs left behind by NEAR/NOT stripping + let prev: string; + do { + prev = result; + result = result.replace(/\(\s*\)/g, ' '); + } while (result !== prev); + + return result.replace(/\s+/g, ' ').replace(/\(\s+/g, '(').replace(/\s+\)/g, ')').trim(); +}; diff --git a/packages/sqlite-shared/src/index.ts b/packages/sqlite-shared/src/index.ts new file mode 100644 index 0000000..374ea3a --- /dev/null +++ b/packages/sqlite-shared/src/index.ts @@ -0,0 +1,18 @@ +export { RECORD_SCHEMA_SQL, FTS4_SCHEMA_SQL, PRAGMA_FOREIGN_KEYS_ON } from './schema.js'; +export { + buildWhereClause, + buildOrderClause, + getSortField, + getSortColumn, + type SanitizeSearch, +} from './query.js'; +export { + encodeCursor, + decodeCursor, + makeCursor, + SORT_FIELDS, + type SortField, + type DecodedCursor, +} from './cursor.js'; +export { rowToRecord, rowToAssociation, rowToType, rowToVersion, toMs, fromMs } from './mappers.js'; +export { sanitizeFts4Query } from './fts.js'; diff --git a/packages/sqlite-shared/src/mappers.ts b/packages/sqlite-shared/src/mappers.ts new file mode 100644 index 0000000..5841ffb --- /dev/null +++ b/packages/sqlite-shared/src/mappers.ts @@ -0,0 +1,77 @@ +/** + * Row <-> domain object mappers shared by every SQLite-backed record + * adapter. Column names and JSON-encoding choices are the storage + * contract; keeping one copy means the adapters can't drift on them. + */ + +import type { StackRecord, StackType, RecordVersion, Association } from '@haverstack/core'; + +export const toMs = (d: Date): number => d.getTime(); +export const fromMs = (ms: number): Date => new Date(ms); + +export const rowToRecord = ( + row: Record, + associations: Association[], +): StackRecord => { + const record: StackRecord = { + id: row.id as string, + typeId: row.type_id as string, + createdAt: fromMs(row.created_at as number), + updatedAt: fromMs(row.updated_at as number), + content: JSON.parse(row.content as string), + version: row.version as number, + }; + if (row.parent_id) record.parentId = row.parent_id as string; + if (row.entity_id) record.entityId = row.entity_id as string; + if (row.app_id) record.appId = row.app_id as string; + if (row.deleted_at) record.deletedAt = fromMs(row.deleted_at as number); + if (row.permissions) record.permissions = JSON.parse(row.permissions as string); + if (associations.length) record.associations = associations; + return record; +}; + +export const rowToAssociation = (row: Record): Association => { + if (row.kind === 'tag') { + return { kind: 'tag', label: row.label as string }; + } + if (row.kind === 'attachment') { + return { + kind: 'attachment', + label: row.label as string, + fileId: row.file_id as string, + mimeType: row.mime_type as string, + }; + } + // relationship + return { + kind: 'relationship', + label: row.label as string, + recordId: row.related_id as string, + }; +}; + +export const rowToType = (row: Record): StackType => { + const t: StackType = { + id: row.id as string, + baseId: row.base_id as string, + version: row.version as number, + name: row.name as string, + schema: JSON.parse(row.schema as string), + schemaHash: row.schema_hash as string, + createdAt: fromMs(row.created_at as number), + }; + if (row.migrates_from) t.migratesFrom = row.migrates_from as string; + return t; +}; + +export const rowToVersion = (row: Record): RecordVersion => { + const v: RecordVersion = { + version: row.version as number, + content: JSON.parse(row.content as string), + updatedAt: fromMs(row.updated_at as number), + }; + if (row.entity_id) v.entityId = row.entity_id as string; + if (row.associations) v.associations = JSON.parse(row.associations as string); + if (row.permissions) v.permissions = JSON.parse(row.permissions as string); + return v; +}; diff --git a/packages/sqlite-shared/src/query.ts b/packages/sqlite-shared/src/query.ts new file mode 100644 index 0000000..f60bc93 --- /dev/null +++ b/packages/sqlite-shared/src/query.ts @@ -0,0 +1,147 @@ +/** + * WHERE/ORDER clause building for StackQuery. Engine-independent SQL — + * shared verbatim between SQLite-backed record adapters. The one caveat is + * full-text search: `f.search` here assumes an `records_fts` virtual table + * with a MATCH-able `content` column and a sanitizer that accepts the raw + * query string, which both FTS4 and FTS5 setups can satisfy even though + * their grammars differ (see fts.ts). + */ + +import type { StackQuery } from '@haverstack/core'; +import { decodeCursor, getSortColumn, getSortField, type SortField } from './cursor.js'; + +export { getSortField, getSortColumn }; +export type { SortField }; + +export type SanitizeSearch = (query: string) => string; + +export const buildWhereClause = ( + query: StackQuery, + sanitizeSearch: SanitizeSearch, +): { sql: string; params: unknown[] } => { + const conditions: string[] = ["r.id != '_config'"]; + const params: unknown[] = []; + const f = query.filter ?? {}; + + if (!f.includeDeleted) { + conditions.push('r.deleted_at IS NULL'); + } + + if (f.typeId !== undefined) { + const ids = Array.isArray(f.typeId) ? f.typeId : [f.typeId]; + conditions.push(`r.type_id IN (${ids.map(() => '?').join(',')})`); + params.push(...ids); + } + + if (f.parentId !== undefined) { + if (f.parentId === null) { + conditions.push('r.parent_id IS NULL'); + } else { + conditions.push('r.parent_id = ?'); + params.push(f.parentId); + } + } + + if (f.appId !== undefined) { + const ids = Array.isArray(f.appId) ? f.appId : [f.appId]; + conditions.push(`r.app_id IN (${ids.map(() => '?').join(',')})`); + params.push(...ids); + } + + if (f.entityId !== undefined) { + const ids = Array.isArray(f.entityId) ? f.entityId : [f.entityId]; + conditions.push(`r.entity_id IN (${ids.map(() => '?').join(',')})`); + params.push(...ids); + } + + if (f.createdAt?.after) { + conditions.push('r.created_at > ?'); + params.push(f.createdAt.after.getTime()); + } + if (f.createdAt?.before) { + conditions.push('r.created_at < ?'); + params.push(f.createdAt.before.getTime()); + } + if (f.updatedAt?.after) { + conditions.push('r.updated_at > ?'); + params.push(f.updatedAt.after.getTime()); + } + if (f.updatedAt?.before) { + conditions.push('r.updated_at < ?'); + params.push(f.updatedAt.before.getTime()); + } + + // Tag filter — record must have ALL specified tags + if (f.tags?.length) { + for (const tag of f.tags) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'tag' AND a.label = ?)`, + ); + params.push(tag); + } + } + + // Attachment label filter + if (f.hasAttachment) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.label = ?)`, + ); + params.push(f.hasAttachment); + } + + // Attachment file ID filter — find records that reference a specific file + if (f.attachmentFileId) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.file_id = ?)`, + ); + params.push(f.attachmentFileId); + } + + // Relationship filter + if (f.relatedTo) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'relationship' AND a.related_id = ?` + + (f.relatedTo.label ? ` AND a.label = ?` : '') + + `)`, + ); + params.push(f.relatedTo.recordId); + if (f.relatedTo.label) params.push(f.relatedTo.label); + } + + // Content field filters (top-level scalar exact match) + if (f.content) { + for (const [key, value] of Object.entries(f.content)) { + conditions.push(`json_extract(r.content, ?) = ?`); + params.push(`$.${key}`, value); + } + } + + if (f.search) { + const sanitized = sanitizeSearch(f.search); + if (sanitized) { + conditions.push(`r.rowid IN (SELECT rowid FROM records_fts WHERE records_fts MATCH ?)`); + params.push(sanitized); + } + } + + // Cursor (sort-field value + id for stable pagination) + if (query.cursor) { + const { field: cursorField, value: numericValue, id: cursorId } = decodeCursor(query.cursor); + const col = getSortColumn(cursorField); + const sortDir = query.sort?.direction ?? 'desc'; + const op = sortDir === 'asc' ? '>' : '<'; + conditions.push(`(r.${col} ${op} ? OR (r.${col} = ? AND r.id ${op} ?))`); + params.push(numericValue, numericValue, cursorId); + } + + return { + sql: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '', + params, + }; +}; + +export const buildOrderClause = (query: StackQuery): string => { + const field = getSortField(query); + const dir = (query.sort?.direction ?? 'desc').toUpperCase(); + return `ORDER BY r.${getSortColumn(field)} ${dir}, r.id ${dir}`; +}; diff --git a/packages/sqlite-shared/src/schema.ts b/packages/sqlite-shared/src/schema.ts new file mode 100644 index 0000000..2588595 --- /dev/null +++ b/packages/sqlite-shared/src/schema.ts @@ -0,0 +1,95 @@ +/** + * Schema DDL shared by SQLite-backed record adapters. FTS4 is what sql.js's + * WASM build supports; the native adapter uses FTS5 and defines its own + * virtual table DDL rather than sharing this one — the rest of the schema + * (tables, indexes) is identical across engines. + */ + +export const RECORD_SCHEMA_SQL = ` + CREATE TABLE IF NOT EXISTS records ( + id TEXT PRIMARY KEY, + type_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + content TEXT NOT NULL CHECK (json_valid(content)), + version INTEGER NOT NULL DEFAULT 1, + parent_id TEXT, + entity_id TEXT, + app_id TEXT, + deleted_at INTEGER, + permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)) + ) STRICT; + + CREATE TABLE IF NOT EXISTS associations ( + record_id TEXT NOT NULL REFERENCES records(id), + kind TEXT NOT NULL CHECK (kind IN ('tag', 'attachment', 'relationship')), + label TEXT NOT NULL, + file_id TEXT NOT NULL DEFAULT '', + mime_type TEXT, + related_id TEXT NOT NULL DEFAULT '', + PRIMARY KEY (record_id, kind, label, file_id, related_id) + ) STRICT; + + CREATE TABLE IF NOT EXISTS versions ( + record_id TEXT NOT NULL REFERENCES records(id), + version INTEGER NOT NULL, + content TEXT NOT NULL CHECK (json_valid(content)), + updated_at INTEGER NOT NULL, + entity_id TEXT, + associations TEXT CHECK (associations IS NULL OR json_valid(associations)), + permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)), + PRIMARY KEY (record_id, version) + ) STRICT; + + CREATE TABLE IF NOT EXISTS types ( + id TEXT PRIMARY KEY, + base_id TEXT NOT NULL, + version INTEGER NOT NULL, + name TEXT NOT NULL, + schema TEXT NOT NULL CHECK (json_valid(schema)), + schema_hash TEXT NOT NULL, + migrates_from TEXT, + created_at INTEGER NOT NULL + ) STRICT; + + CREATE TABLE IF NOT EXISTS tokens ( + id TEXT PRIMARY KEY, + token_hash TEXT NOT NULL UNIQUE, + entity_id TEXT NOT NULL, + label TEXT, + created_at INTEGER NOT NULL, + expires_at INTEGER + ) STRICT; + + -- Indexes + CREATE INDEX IF NOT EXISTS idx_records_type_id ON records(type_id); + CREATE INDEX IF NOT EXISTS idx_records_parent_id ON records(parent_id); + CREATE INDEX IF NOT EXISTS idx_records_entity_id ON records(entity_id); + CREATE INDEX IF NOT EXISTS idx_records_app_id ON records(app_id); + CREATE INDEX IF NOT EXISTS idx_records_deleted_at ON records(deleted_at); + CREATE INDEX IF NOT EXISTS idx_records_created_at ON records(created_at); + CREATE INDEX IF NOT EXISTS idx_records_updated_at ON records(updated_at); + CREATE INDEX IF NOT EXISTS idx_assoc_record_id ON associations(record_id); + CREATE INDEX IF NOT EXISTS idx_assoc_kind_label ON associations(kind, label); + CREATE INDEX IF NOT EXISTS idx_assoc_kind_file_id ON associations(kind, file_id); + CREATE INDEX IF NOT EXISTS idx_types_base_id ON types(base_id); + CREATE INDEX IF NOT EXISTS idx_tokens_hash ON tokens(token_hash); +`; + +/** FTS4 — compatible with sql.js's WASM SQLite build. */ +export const FTS4_SCHEMA_SQL = ` + CREATE VIRTUAL TABLE IF NOT EXISTS records_fts USING fts4( + content, + content='records' + ); +`; + +/** + * Enforces the REFERENCES constraints declared above (off by default per + * SQLite connection). Without it, associating/versioning a nonexistent + * record silently creates an orphan row instead of failing loudly. + * Adapters that enable this must map the resulting constraint-violation + * error to StackNotFoundError at the call sites that can trigger it + * (chiefly `associate()` on a record that doesn't exist). + */ +export const PRAGMA_FOREIGN_KEYS_ON = `PRAGMA foreign_keys = ON;`; diff --git a/packages/sqlite-shared/tests/cursor.test.ts b/packages/sqlite-shared/tests/cursor.test.ts new file mode 100644 index 0000000..b9d9aa7 --- /dev/null +++ b/packages/sqlite-shared/tests/cursor.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect } from 'vitest'; +import { StackQueryError } from '@haverstack/core'; +import { encodeCursor, decodeCursor, makeCursor } from '../src/cursor.js'; +import type { StackRecord } from '@haverstack/core'; + +const makeRecord = (overrides: Partial = {}): StackRecord => ({ + id: 'rec01', + typeId: 'com.example/note@1', + createdAt: new Date(1000), + updatedAt: new Date(2000), + content: {}, + version: 3, + ...overrides, +}); + +describe('encodeCursor / decodeCursor', () => { + test('roundtrips field, value, and id', () => { + const cursor = encodeCursor('createdAt', 12345, 'rec01'); + expect(decodeCursor(cursor)).toEqual({ field: 'createdAt', value: 12345, id: 'rec01' }); + }); + + test('accepts the legacy 2-part (value|id) format as createdAt', () => { + const legacy = btoa('12345|rec01'); + expect(decodeCursor(legacy)).toEqual({ field: 'createdAt', value: 12345, id: 'rec01' }); + }); + + test('throws StackQueryError for non-base64 garbage', () => { + expect(() => decodeCursor('!!!not-a-cursor!!!')).toThrow(StackQueryError); + }); + + test('throws StackQueryError for an unknown sort field', () => { + expect(() => decodeCursor(btoa('bogusField|123|r1'))).toThrow(StackQueryError); + }); + + test('throws StackQueryError for a non-numeric sort value', () => { + expect(() => decodeCursor(btoa('createdAt|not-a-number|r1'))).toThrow(StackQueryError); + }); +}); + +describe('makeCursor', () => { + test('encodes createdAt by default field selection', () => { + const cursor = makeCursor(makeRecord(), 'createdAt'); + expect(decodeCursor(cursor)).toEqual({ field: 'createdAt', value: 1000, id: 'rec01' }); + }); + + test('encodes updatedAt', () => { + const cursor = makeCursor(makeRecord(), 'updatedAt'); + expect(decodeCursor(cursor)).toEqual({ field: 'updatedAt', value: 2000, id: 'rec01' }); + }); + + test('encodes version', () => { + const cursor = makeCursor(makeRecord(), 'version'); + expect(decodeCursor(cursor)).toEqual({ field: 'version', value: 3, id: 'rec01' }); + }); +}); diff --git a/packages/sqlite-shared/tests/fts.test.ts b/packages/sqlite-shared/tests/fts.test.ts new file mode 100644 index 0000000..6c1e2cd --- /dev/null +++ b/packages/sqlite-shared/tests/fts.test.ts @@ -0,0 +1,41 @@ +import { describe, test, expect } from 'vitest'; +import { sanitizeFts4Query } from '../src/fts.js'; + +describe('sanitizeFts4Query', () => { + test('returns empty string unchanged', () => { + expect(sanitizeFts4Query('')).toBe(''); + }); + + test('strips wildcards', () => { + expect(sanitizeFts4Query('hel*lo')).toBe('hello'); + }); + + test('strips NEAR and NEAR/N operators', () => { + expect(sanitizeFts4Query('foo NEAR bar')).toBe('foo bar'); + expect(sanitizeFts4Query('foo NEAR/5 bar')).toBe('foo bar'); + }); + + test('strips bare NOT with no left operand', () => { + expect(sanitizeFts4Query('NOT foo')).toBe('foo'); + }); + + test('keeps NOT with a left operand', () => { + expect(sanitizeFts4Query('foo NOT bar')).toBe('foo NOT bar'); + }); + + test('caps parenthesis nesting depth', () => { + expect(sanitizeFts4Query('(((deep)))', 2)).toBe('((deep))'); + }); + + test('auto-closes unclosed parens', () => { + expect(sanitizeFts4Query('(unclosed')).toBe('(unclosed)'); + }); + + test('discards unmatched closing parens', () => { + expect(sanitizeFts4Query('unmatched)')).toBe('unmatched'); + }); + + test('keeps phrase queries and implicit AND', () => { + expect(sanitizeFts4Query('"exact phrase" other')).toBe('"exact phrase" other'); + }); +}); diff --git a/packages/sqlite-shared/tsconfig.build.json b/packages/sqlite-shared/tsconfig.build.json new file mode 100644 index 0000000..57d0596 --- /dev/null +++ b/packages/sqlite-shared/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "noEmit": false + }, + "exclude": ["tests/**/*.ts"] +} diff --git a/packages/sqlite-shared/tsconfig.json b/packages/sqlite-shared/tsconfig.json new file mode 100644 index 0000000..dbd41f4 --- /dev/null +++ b/packages/sqlite-shared/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/packages/sqlite-shared/vitest.config.ts b/packages/sqlite-shared/vitest.config.ts new file mode 100644 index 0000000..e0ec2fb --- /dev/null +++ b/packages/sqlite-shared/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; +import { resolve } from 'path'; + +export default defineConfig({ + resolve: { + alias: { + '@haverstack/core': resolve(__dirname, '../core/src/index.ts'), + }, + }, + test: { + environment: 'node', + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd24460..b6fb6a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,6 +123,9 @@ importers: '@haverstack/core': specifier: workspace:* version: link:../core + '@haverstack/sqlite-shared': + specifier: workspace:* + version: link:../sqlite-shared sql.js: specifier: ^1.14.0 version: 1.14.1 @@ -140,6 +143,22 @@ importers: specifier: ^2.0.0 version: 2.1.9(@types/node@22.19.17) + packages/sqlite-shared: + dependencies: + '@haverstack/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.19.17 + typescript: + specifier: ^5.5.0 + version: 5.9.3 + vitest: + specifier: ^2.0.0 + version: 2.1.9(@types/node@22.19.17) + packages/wire-types: dependencies: '@haverstack/core':