diff --git a/docs/spec.md b/docs/spec.md index b452354..b0761f5 100644 --- a/docs/spec.md +++ b/docs/spec.md @@ -481,6 +481,10 @@ const stack = await Stack.create(adapter); All adapters support the full Record API. Performance guarantees differ; correctness does not. +**`@haverstack/sqlite-shared`** is an internal, non-public package holding the SQL that's identical across every SQLite-backed record adapter — schema DDL, `WHERE`/`ORDER` building, the cursor codec, row mappers, and the FTS4 sanitizer. `record-adapter-sqljs` consumes it today; a future native SQLite adapter would too, sharing everything but the FTS dialect (FTS4 vs FTS5) and the engine binding itself. Keeping this logic in one place is what keeps a cursor minted by one SQLite-backed adapter decodable by another. + +`record-adapter-sqljs` enables SQLite foreign-key enforcement (`PRAGMA foreign_keys = ON`) so that operations like `associate()` against a nonexistent record fail loudly (`StackNotFoundError`) instead of silently creating an orphan row. + --- ## Concurrency & storage ownership @@ -538,6 +542,8 @@ const meta = results.records[0]?.content as AttachmentContent | undefined; - `Stack.deleteAttachment(fileId)` — deletes bytes and all `_attachment@1` metadata records for the file. Throws `StackConflictError` if any record still references the file. Throws `StackNotFoundError` if the file doesn't exist. - `ScopedStack.deleteAttachment(fileId)` — owner only. Throws `StackPermissionError` for non-owners. Delegates to `Stack.deleteAttachment()`. +**Atomicity of the reference check.** The reference check and the metadata-record deletes must happen as one unit — otherwise a concurrent `associate()` can add a new reference in the gap between them, leaving a dangling association after the delete completes. Adapters MAY implement `StackRecordAdapter.deleteUnreferencedAttachmentRecords(fileId, metadataTypeId)` to close this race (`Stack.deleteAttachment()` uses it when present, falling back to a non-atomic check-then-act sequence otherwise). Byte deletion always happens after the metadata step commits: a crash in between leaves orphaned bytes, which is harmless and later reclaimed by garbage collection, rather than a dangling reference, which is not. + **Deduplication:** Bytes are deduplicated — uploading the same content twice stores the binary only once. However, each call to `putAttachment()` creates a new `_attachment@1` record with its own `mimeType` and `filename`. The same `fileId` may have multiple `_attachment@1` records from separate uploads. --- diff --git a/packages/adapter-local/vitest.config.ts b/packages/adapter-local/vitest.config.ts index 97fad83..9eb9cf6 100644 --- a/packages/adapter-local/vitest.config.ts +++ b/packages/adapter-local/vitest.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ __dirname, '../record-adapter-sqljs/src/index.ts', ), + '@haverstack/sqlite-shared': resolve(__dirname, '../sqlite-shared/src/index.ts'), '@haverstack/blob-adapter-disk': resolve(__dirname, '../blob-adapter-disk/src/index.ts'), }, }, diff --git a/packages/core/src/stack.ts b/packages/core/src/stack.ts index d7de5d9..e6eb494 100644 --- a/packages/core/src/stack.ts +++ b/packages/core/src/stack.ts @@ -765,6 +765,31 @@ export class Stack implements StackClient { * Throws StackNotFoundError if neither metadata records nor bytes exist. */ async deleteAttachment(fileId: string): Promise { + const metadataTypeId = `${SYSTEM_TYPES.ATTACHMENT}@1`; + const deletedRecordIds = this.adapter.deleteUnreferencedAttachmentRecords + ? await this.adapter.deleteUnreferencedAttachmentRecords(fileId, metadataTypeId) + : await this.deleteUnreferencedAttachmentRecordsFallback(fileId, metadataTypeId); + + if (!deletedRecordIds.length) { + try { + await this.adapter.getAttachment(fileId); + } catch { + throw new StackNotFoundError(`Attachment not found: "${fileId}"`); + } + } + + await this.adapter.deleteAttachment(fileId); + } + + /** + * Non-atomic fallback for adapters that don't implement + * deleteUnreferencedAttachmentRecords(): a concurrent associate() can + * race between the reference check below and the deletes it guards. + */ + private async deleteUnreferencedAttachmentRecordsFallback( + fileId: string, + metadataTypeId: string, + ): Promise { const refResult = await this.query({ filter: { attachmentFileId: fileId }, limit: 1 }); if (refResult.records.length > 0) { throw new StackConflictError('Attachment is still referenced by one or more records'); @@ -772,7 +797,7 @@ export class Stack implements StackClient { const metaResult = await this.query({ filter: { - typeId: `${SYSTEM_TYPES.ATTACHMENT}@1`, + typeId: metadataTypeId, ...(this.features.contentFieldQuery && { content: { fileId } }), }, }); @@ -780,19 +805,11 @@ export class Stack implements StackClient { ? metaResult.records : metaResult.records.filter((r) => (r.content as AttachmentContent).fileId === fileId); - if (!metaRecords.length) { - try { - await this.adapter.getAttachment(fileId); - } catch { - throw new StackNotFoundError(`Attachment not found: "${fileId}"`); - } - } - for (const record of metaRecords) { await this.delete(record.id, { hard: true }); } - await this.adapter.deleteAttachment(fileId); + return metaRecords.map((r) => r.id); } // ------------------------------------------------------- diff --git a/packages/core/src/testing.ts b/packages/core/src/testing.ts index 9090a3c..8386aad 100644 --- a/packages/core/src/testing.ts +++ b/packages/core/src/testing.ts @@ -99,6 +99,13 @@ export class MemoryAdapter implements StackAdapter { const ids = Array.isArray(f.entityId) ? f.entityId : [f.entityId]; results = results.filter((r) => r.entityId !== undefined && ids.includes(r.entityId)); } + if (f.attachmentFileId) { + results = results.filter((r) => + (r.associations ?? []).some( + (a) => a.kind === 'attachment' && a.fileId === f.attachmentFileId, + ), + ); + } const limit = query.limit ?? 50; const start = query.cursor ? Number(query.cursor) : 0; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 33a3608..e764c3e 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -388,6 +388,20 @@ export interface StackRecordAdapter { getType(id: TypeId): Promise; listTypes(): Promise; + /** + * Atomically verify fileId is unreferenced by any record's attachment + * association, then hard-delete every record of `metadataTypeId` whose + * content.fileId matches it — all within a single adapter call, so + * nothing can add a new reference between the check and the delete. + * Returns the ids of the deleted metadata records (empty if none exist, + * e.g. bare bytes left by an interrupted upload). Throws + * StackConflictError if fileId is still referenced. + * + * Optional: Stack.deleteAttachment() falls back to a non-atomic + * check-then-act sequence for adapters that don't implement this. + */ + deleteUnreferencedAttachmentRecords?(fileId: FileId, metadataTypeId: TypeId): Promise; + // Lifecycle flush?(): Promise; close?(): Promise; diff --git a/packages/core/tests/stack.test.ts b/packages/core/tests/stack.test.ts index 010c93d..e63ab72 100644 --- a/packages/core/tests/stack.test.ts +++ b/packages/core/tests/stack.test.ts @@ -853,3 +853,78 @@ describe('putAttachment', () => { expect(result.records[0].entityId).toBeUndefined(); }); }); + +// ------------------------------------------------------- +// deleteAttachment +// ------------------------------------------------------- + +describe('deleteAttachment', () => { + test('throws StackConflictError when a record still references the file (fallback path)', async () => { + const data = new Uint8Array([1, 2, 3]); + const fileId = await stack.putAttachment(data, 'image/png'); + const note = await stack.create(NOTE_V1, { text: 'hi' }); + await stack.associate(note.id, { + kind: 'attachment', + label: 'cover', + fileId, + mimeType: 'image/png', + }); + + await expect(stack.deleteAttachment(fileId)).rejects.toThrow(StackConflictError); + }); + + test('deletes the _attachment@1 metadata record when unreferenced (fallback path)', async () => { + const data = new Uint8Array([1, 2, 3]); + const fileId = await stack.putAttachment(data, 'image/png'); + + await stack.deleteAttachment(fileId); + + const result = await stack.query({ filter: { typeId: '_attachment@1' } }); + expect(result.records).toHaveLength(0); + }); + + test('throws StackNotFoundError when neither metadata nor bytes exist', async () => { + class NoBytesAdapter extends MemoryAdapter { + async getAttachment(_fileId: string): Promise { + throw new Error('not found'); + } + } + const noBytesStack = await Stack.create( + new NoBytesAdapter({ ownerEntityId: 'owner-123', timezone: 'UTC' }), + ); + + await expect(noBytesStack.deleteAttachment('nonexistent-file')).rejects.toThrow( + StackNotFoundError, + ); + }); + + test('prefers the adapter atomic path over the fallback when the adapter implements it', async () => { + const calls: string[] = []; + class AtomicAdapter extends MemoryAdapter { + async deleteUnreferencedAttachmentRecords( + fileId: string, + metadataTypeId: string, + ): Promise { + calls.push('atomic'); + const toDelete = [...this.records.values()].filter( + (r) => + r.typeId === metadataTypeId && (r.content as Record).fileId === fileId, + ); + for (const r of toDelete) { + this.records.delete(r.id); + this.order.splice(this.order.indexOf(r.id), 1); + } + return toDelete.map((r) => r.id); + } + } + const atomicStack = await Stack.create( + new AtomicAdapter({ ownerEntityId: 'owner-123', timezone: 'UTC' }), + ); + await atomicStack.defineType(NOTE_V1, 'Note', { text: { kind: 'text', required: true } }); + + const fileId = await atomicStack.putAttachment(new Uint8Array([9]), 'image/png'); + await atomicStack.deleteAttachment(fileId); + + expect(calls).toEqual(['atomic']); + }); +}); diff --git a/packages/record-adapter-sqljs/package.json b/packages/record-adapter-sqljs/package.json index da72057..64209a7 100644 --- a/packages/record-adapter-sqljs/package.json +++ b/packages/record-adapter-sqljs/package.json @@ -39,6 +39,7 @@ }, "dependencies": { "@haverstack/core": "workspace:*", + "@haverstack/sqlite-shared": "workspace:*", "sql.js": "^1.14.0" }, "devDependencies": { diff --git a/packages/record-adapter-sqljs/src/index.ts b/packages/record-adapter-sqljs/src/index.ts index ee9bf82..bdc3765 100644 --- a/packages/record-adapter-sqljs/src/index.ts +++ b/packages/record-adapter-sqljs/src/index.ts @@ -19,6 +19,11 @@ * Also exposes token management methods (createToken, * lookupToken, listTokens, revokeToken) used by server * implementations to issue and validate bearer tokens. + * + * Schema DDL, WHERE/ORDER building, cursor codec, row mappers, + * and the FTS4 sanitizer live in @haverstack/sqlite-shared — + * shared with the native record-adapter-sqlite so the two + * engines can't drift on engine-independent SQL. */ import initSqlJs from 'sql.js'; @@ -31,6 +36,8 @@ import type { StackRecord, StackType, TypeId, + RecordId, + FileId, RecordVersion, StackQuery, QueryResult, @@ -38,7 +45,23 @@ import type { Permission, AdapterCapabilities, } from '@haverstack/core'; -import { applyMergePatch, StackQueryError } from '@haverstack/core'; +import { applyMergePatch, StackConflictError, StackNotFoundError } from '@haverstack/core'; +import { + RECORD_SCHEMA_SQL, + FTS4_SCHEMA_SQL, + PRAGMA_FOREIGN_KEYS_ON, + buildWhereClause, + buildOrderClause, + getSortField, + makeCursor, + rowToRecord, + rowToAssociation, + rowToType, + rowToVersion, + toMs, + fromMs, + sanitizeFts4Query, +} from '@haverstack/sqlite-shared'; // ------------------------------------------------------- // Types @@ -75,157 +98,7 @@ export type TokenInfo = { expiresAt?: Date; }; -// ------------------------------------------------------- -// SQL — schema -// ------------------------------------------------------- - -const SCHEMA_SQL = ` - CREATE TABLE IF NOT EXISTS records ( - id TEXT PRIMARY KEY, - type_id TEXT NOT NULL, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - content TEXT NOT NULL CHECK (json_valid(content)), - version INTEGER NOT NULL DEFAULT 1, - parent_id TEXT, - entity_id TEXT, - app_id TEXT, - deleted_at INTEGER, - permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)) - ) STRICT; - - CREATE TABLE IF NOT EXISTS associations ( - record_id TEXT NOT NULL REFERENCES records(id), - kind TEXT NOT NULL CHECK (kind IN ('tag', 'attachment', 'relationship')), - label TEXT NOT NULL, - file_id TEXT NOT NULL DEFAULT '', - mime_type TEXT, - related_id TEXT NOT NULL DEFAULT '', - PRIMARY KEY (record_id, kind, label, file_id, related_id) - ) STRICT; - - CREATE TABLE IF NOT EXISTS versions ( - record_id TEXT NOT NULL REFERENCES records(id), - version INTEGER NOT NULL, - content TEXT NOT NULL CHECK (json_valid(content)), - updated_at INTEGER NOT NULL, - entity_id TEXT, - associations TEXT CHECK (associations IS NULL OR json_valid(associations)), - permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)), - PRIMARY KEY (record_id, version) - ) STRICT; - - CREATE TABLE IF NOT EXISTS types ( - id TEXT PRIMARY KEY, - base_id TEXT NOT NULL, - version INTEGER NOT NULL, - name TEXT NOT NULL, - schema TEXT NOT NULL CHECK (json_valid(schema)), - schema_hash TEXT NOT NULL, - migrates_from TEXT, - created_at INTEGER NOT NULL - ) STRICT; - - CREATE TABLE IF NOT EXISTS tokens ( - id TEXT PRIMARY KEY, - token_hash TEXT NOT NULL UNIQUE, - entity_id TEXT NOT NULL, - label TEXT, - created_at INTEGER NOT NULL, - expires_at INTEGER - ) STRICT; - - -- Indexes - CREATE INDEX IF NOT EXISTS idx_records_type_id ON records(type_id); - CREATE INDEX IF NOT EXISTS idx_records_parent_id ON records(parent_id); - CREATE INDEX IF NOT EXISTS idx_records_entity_id ON records(entity_id); - CREATE INDEX IF NOT EXISTS idx_records_app_id ON records(app_id); - CREATE INDEX IF NOT EXISTS idx_records_deleted_at ON records(deleted_at); - CREATE INDEX IF NOT EXISTS idx_records_created_at ON records(created_at); - CREATE INDEX IF NOT EXISTS idx_records_updated_at ON records(updated_at); - CREATE INDEX IF NOT EXISTS idx_assoc_record_id ON associations(record_id); - CREATE INDEX IF NOT EXISTS idx_assoc_kind_label ON associations(kind, label); - CREATE INDEX IF NOT EXISTS idx_assoc_kind_file_id ON associations(kind, file_id); - CREATE INDEX IF NOT EXISTS idx_types_base_id ON types(base_id); - CREATE INDEX IF NOT EXISTS idx_tokens_hash ON tokens(token_hash); - - -- Full-text search (FTS4 — compatible with sql.js) - CREATE VIRTUAL TABLE IF NOT EXISTS records_fts USING fts4( - content, - content='records' - ); -`; - -// ------------------------------------------------------- -// Helpers -// ------------------------------------------------------- - -const toMs = (d: Date): number => d.getTime(); -const fromMs = (ms: number): Date => new Date(ms); - -const rowToRecord = (row: Record, associations: Association[]): StackRecord => { - const record: StackRecord = { - id: row.id as string, - typeId: row.type_id as string, - createdAt: fromMs(row.created_at as number), - updatedAt: fromMs(row.updated_at as number), - content: JSON.parse(row.content as string), - version: row.version as number, - }; - if (row.parent_id) record.parentId = row.parent_id as string; - if (row.entity_id) record.entityId = row.entity_id as string; - if (row.app_id) record.appId = row.app_id as string; - if (row.deleted_at) record.deletedAt = fromMs(row.deleted_at as number); - if (row.permissions) record.permissions = JSON.parse(row.permissions as string); - if (associations.length) record.associations = associations; - return record; -}; - -const rowToAssociation = (row: Record): Association => { - if (row.kind === 'tag') { - return { kind: 'tag', label: row.label as string }; - } - if (row.kind === 'attachment') { - return { - kind: 'attachment', - label: row.label as string, - fileId: row.file_id as string, - mimeType: row.mime_type as string, - }; - } - // relationship - return { - kind: 'relationship', - label: row.label as string, - recordId: row.related_id as string, - }; -}; - -const rowToType = (row: Record): StackType => { - const t: StackType = { - id: row.id as string, - baseId: row.base_id as string, - version: row.version as number, - name: row.name as string, - schema: JSON.parse(row.schema as string), - schemaHash: row.schema_hash as string, - createdAt: fromMs(row.created_at as number), - }; - if (row.migrates_from) t.migratesFrom = row.migrates_from as string; - return t; -}; - -const rowToVersion = (row: Record): RecordVersion => { - const v: RecordVersion = { - version: row.version as number, - content: JSON.parse(row.content as string), - updatedAt: fromMs(row.updated_at as number), - }; - if (row.entity_id) v.entityId = row.entity_id as string; - if (row.associations) v.associations = JSON.parse(row.associations as string); - if (row.permissions) v.permissions = JSON.parse(row.permissions as string); - return v; -}; +const SCHEMA_SQL = `${RECORD_SCHEMA_SQL}\n${FTS4_SCHEMA_SQL}`; // ------------------------------------------------------- // Storage ownership lock @@ -278,232 +151,12 @@ const releaseLock = (dbPath: string): void => { }; // ------------------------------------------------------- -// FTS4 query sanitization +// Foreign key enforcement // ------------------------------------------------------- -/** - * Sanitizes user input for safe use in an FTS4 MATCH clause. - * - * FTS4's query language supports operators (AND/OR/NOT, phrases, NEAR, wildcards) - * that can be expensive or cause parse errors with untrusted input. - * This function removes the dangerous operators while preserving useful ones: - * kept: AND/OR/NOT (with a left operand), phrase queries ("…"), implicit AND - * removed: wildcards (*), NEAR/N, bare NOT (no left operand) - * capped: parenthesis nesting depth (default: 2) - * - * A query timeout is not implementable here: sql.js runs SQLite as synchronous - * WASM that blocks the JS event loop, and db.interrupt() is not exposed. Moving - * sql.js into a Worker thread would enable a timeout via interrupt() but is out - * of scope for this change. - */ -const sanitizeFts4Query = (query: string, maxDepth = 2): string => { - if (!query) return ''; - - // Remove wildcards - let clean = query.replace(/\*/g, ''); - - // Remove NEAR operator (handles NEAR and NEAR/N variants) - clean = clean.replace(/\bNEAR(?:\/\d+)?\b/gi, ''); - - // Strip bare NOT with no left operand — FTS4 requires "term NOT term", not "NOT term" - clean = clean.replace(/(?:^|\(\s*)NOT\s+/gi, (m) => m.replace(/NOT\s+/i, '')); - - // Enforce max nesting depth; replace excess ( with a space and discard unmatched ) - let currentDepth = 0; - let result = ''; - for (const char of clean) { - if (char === '(') { - if (currentDepth < maxDepth) { - currentDepth++; - result += char; - } else result += ' '; - } else if (char === ')') { - if (currentDepth > 0) { - currentDepth--; - result += char; - } else result += ' '; - } else { - result += char; - } - } - - // Auto-close any unclosed parens - if (currentDepth > 0) result += ')'.repeat(currentDepth); - - // Iteratively remove empty paren pairs left behind by NEAR/NOT stripping - let prev: string; - do { - prev = result; - result = result.replace(/\(\s*\)/g, ' '); - } while (result !== prev); - - return result.replace(/\s+/g, ' ').replace(/\(\s+/g, '(').replace(/\s+\)/g, ')').trim(); -}; - -// ------------------------------------------------------- -// Query building -// ------------------------------------------------------- - -type SortField = 'createdAt' | 'updatedAt' | 'version'; - -const getSortField = (query: StackQuery): SortField => query.sort?.field ?? 'createdAt'; - -const getSortColumn = (field: SortField): string => - field === 'createdAt' ? 'created_at' : field === 'updatedAt' ? 'updated_at' : 'version'; - -const buildWhereClause = (query: StackQuery): { sql: string; params: unknown[] } => { - const conditions: string[] = ["r.id != '_config'"]; - const params: unknown[] = []; - const f = query.filter ?? {}; - - if (!f.includeDeleted) { - conditions.push('r.deleted_at IS NULL'); - } - - if (f.typeId !== undefined) { - const ids = Array.isArray(f.typeId) ? f.typeId : [f.typeId]; - conditions.push(`r.type_id IN (${ids.map(() => '?').join(',')})`); - params.push(...ids); - } - - if (f.parentId !== undefined) { - if (f.parentId === null) { - conditions.push('r.parent_id IS NULL'); - } else { - conditions.push('r.parent_id = ?'); - params.push(f.parentId); - } - } - - if (f.appId !== undefined) { - const ids = Array.isArray(f.appId) ? f.appId : [f.appId]; - conditions.push(`r.app_id IN (${ids.map(() => '?').join(',')})`); - params.push(...ids); - } - - if (f.entityId !== undefined) { - const ids = Array.isArray(f.entityId) ? f.entityId : [f.entityId]; - conditions.push(`r.entity_id IN (${ids.map(() => '?').join(',')})`); - params.push(...ids); - } - - if (f.createdAt?.after) { - conditions.push('r.created_at > ?'); - params.push(toMs(f.createdAt.after)); - } - if (f.createdAt?.before) { - conditions.push('r.created_at < ?'); - params.push(toMs(f.createdAt.before)); - } - if (f.updatedAt?.after) { - conditions.push('r.updated_at > ?'); - params.push(toMs(f.updatedAt.after)); - } - if (f.updatedAt?.before) { - conditions.push('r.updated_at < ?'); - params.push(toMs(f.updatedAt.before)); - } - - // Tag filter — record must have ALL specified tags - if (f.tags?.length) { - for (const tag of f.tags) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'tag' AND a.label = ?)`, - ); - params.push(tag); - } - } - - // Attachment label filter - if (f.hasAttachment) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.label = ?)`, - ); - params.push(f.hasAttachment); - } - - // Attachment file ID filter — find records that reference a specific file - if (f.attachmentFileId) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.file_id = ?)`, - ); - params.push(f.attachmentFileId); - } - - // Relationship filter - if (f.relatedTo) { - conditions.push( - `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'relationship' AND a.related_id = ?` + - (f.relatedTo.label ? ` AND a.label = ?` : '') + - `)`, - ); - params.push(f.relatedTo.recordId); - if (f.relatedTo.label) params.push(f.relatedTo.label); - } - - // Content field filters (top-level scalar exact match) - if (f.content) { - for (const [key, value] of Object.entries(f.content)) { - conditions.push(`json_extract(r.content, ?) = ?`); - params.push(`$.${key}`, value); - } - } - - if (f.search) { - const sanitized = sanitizeFts4Query(f.search); - if (sanitized) { - conditions.push(`r.rowid IN (SELECT rowid FROM records_fts WHERE records_fts MATCH ?)`); - params.push(sanitized); - } - } - - // Cursor (sort-field value + id for stable pagination) - if (query.cursor) { - const parts = Buffer.from(query.cursor, 'base64').toString().split('|'); - // New format: field|value|id (3 parts). Legacy format: value|id (2 parts, implies createdAt). - const [cursorField, cursorValue, cursorId] = - parts.length === 3 - ? (parts as [string, string, string]) - : (['createdAt', parts[0], parts[1]] as [string, string, string]); - if (cursorField === undefined || cursorValue === undefined || cursorId === undefined) { - throw new StackQueryError(`Invalid cursor: malformed "${query.cursor}"`); - } - const validSortFields: SortField[] = ['createdAt', 'updatedAt', 'version']; - if (!validSortFields.includes(cursorField as SortField)) { - throw new StackQueryError(`Invalid cursor: unknown sort field "${cursorField}"`); - } - const numericValue = Number(cursorValue); - if (!isFinite(numericValue)) { - throw new StackQueryError(`Invalid cursor: non-numeric sort value`); - } - const col = getSortColumn(cursorField as SortField); - const sortDir = query.sort?.direction ?? 'desc'; - const op = sortDir === 'asc' ? '>' : '<'; - conditions.push(`(r.${col} ${op} ? OR (r.${col} = ? AND r.id ${op} ?))`); - params.push(numericValue, numericValue, cursorId); - } - - return { - sql: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '', - params, - }; -}; - -const buildOrderClause = (query: StackQuery): string => { - const field = getSortField(query); - const dir = (query.sort?.direction ?? 'desc').toUpperCase(); - return `ORDER BY r.${getSortColumn(field)} ${dir}, r.id ${dir}`; -}; - -const makeCursor = (record: StackRecord, field: SortField): string => { - const value = - field === 'updatedAt' - ? toMs(record.updatedAt) - : field === 'version' - ? record.version - : toMs(record.createdAt); - return Buffer.from(`${field}|${value}|${record.id}`).toString('base64'); -}; +/** sql.js's SQLite build reports FK violations as a plain Error with this message. */ +const isForeignKeyViolation = (err: unknown): boolean => + err instanceof Error && err.message.includes('FOREIGN KEY constraint failed'); // ------------------------------------------------------- // SQLiteRecordAdapter @@ -541,6 +194,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { const SQL = await initSqlJs(); const adapter = new SQLiteRecordAdapter(SQL, opts.path); adapter.db = new SQL.Database(); + adapter.db.run(PRAGMA_FOREIGN_KEYS_ON); adapter.db.run(SCHEMA_SQL); const now = Date.now(); adapter.db.run( @@ -570,6 +224,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { const adapter = new SQLiteRecordAdapter(SQL, opts.path); const fileBuffer = readFileSync(opts.path); adapter.db = new SQL.Database(fileBuffer); + adapter.db.run(PRAGMA_FOREIGN_KEYS_ON); adapter.db.run(SCHEMA_SQL); adapter.readConfig(); return adapter; @@ -587,6 +242,10 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { */ private persist(): void { const data = this.db.export(); + // sql.js quirk: export() resets connection-level pragmas (including + // foreign_keys) on the live Database object — reapply immediately so + // FK enforcement doesn't silently go stale after the first write. + this.db.run(PRAGMA_FOREIGN_KEYS_ON); const tmpPath = join( dirname(this.path), `.${basename(this.path)}.tmp-${randomBytes(6).toString('hex')}`, @@ -671,13 +330,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { async deleteRecord(id: string, opts: { hard?: boolean } = {}): Promise { if (opts.hard) { - this.db.run('DELETE FROM associations WHERE record_id = ?', [id]); - this.db.run('DELETE FROM versions WHERE record_id = ?', [id]); - this.db.run( - `DELETE FROM records_fts WHERE docid = (SELECT rowid FROM records WHERE id = ?)`, - [id], - ); - this.db.run('DELETE FROM records WHERE id = ?', [id]); + this.hardDeleteRecord(id); } else { this.db.run( 'UPDATE records SET deleted_at = ?, version = version + 1, updated_at = ? WHERE id = ?', @@ -687,6 +340,16 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { this.persist(); } + /** Deletes a record's associations, versions, FTS entry, and row — no persist(). */ + private hardDeleteRecord(id: string): void { + this.db.run('DELETE FROM associations WHERE record_id = ?', [id]); + this.db.run('DELETE FROM versions WHERE record_id = ?', [id]); + this.db.run(`DELETE FROM records_fts WHERE docid = (SELECT rowid FROM records WHERE id = ?)`, [ + id, + ]); + this.db.run('DELETE FROM records WHERE id = ?', [id]); + } + async undeleteRecord(id: string): Promise { this.db.run( 'UPDATE records SET deleted_at = NULL, version = version + 1, updated_at = ? WHERE id = ?', @@ -745,7 +408,7 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { } async queryRecords(query: StackQuery): Promise { - const { sql: where, params } = buildWhereClause(query); + const { sql: where, params } = buildWhereClause(query, sanitizeFts4Query); const order = buildOrderClause(query); const limit = query.limit ?? 50; @@ -776,6 +439,46 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { return { records, cursor, total }; } + /** + * Atomically verify fileId is unreferenced, then hard-delete every + * metadataTypeId record whose content.fileId matches it. See + * StackRecordAdapter.deleteUnreferencedAttachmentRecords(). Runs as a + * real SQL transaction and, being entirely synchronous SQLite calls + * with no `await` in between, nothing else can interleave between the + * reference check and the deletes. + */ + async deleteUnreferencedAttachmentRecords( + fileId: FileId, + metadataTypeId: TypeId, + ): Promise { + this.db.run('BEGIN'); + try { + const referenced = this.execQuery<{ found: number }>( + `SELECT 1 as found FROM associations WHERE kind = 'attachment' AND file_id = ? LIMIT 1`, + [fileId], + ); + if (referenced.length) { + throw new StackConflictError('Attachment is still referenced by one or more records'); + } + + const metaRows = this.execQuery<{ id: string }>( + `SELECT id FROM records WHERE type_id = ? AND json_extract(content, '$.fileId') = ?`, + [metadataTypeId, fileId], + ); + const deletedIds = metaRows.map((row) => row.id); + for (const id of deletedIds) { + this.hardDeleteRecord(id); + } + + this.db.run('COMMIT'); + if (deletedIds.length) this.persist(); + return deletedIds; + } catch (err) { + this.db.run('ROLLBACK'); + throw err; + } + } + // ------------------------------------------------------- // Versions // ------------------------------------------------------- @@ -946,21 +649,35 @@ export class SQLiteRecordAdapter implements StackRecordAdapter { ]); } + /** + * FK enforcement (PRAGMA_FOREIGN_KEYS_ON) means inserting an + * association against a record that doesn't exist throws — mapped + * here to StackNotFoundError so associate() on a nonexistent record + * fails loudly with a proper error instead of silently creating an + * orphan row. + */ private insertAssociations(recordId: string, associations: Association[]): void { for (const assoc of associations) { - this.db.run( - `INSERT OR IGNORE INTO associations - (record_id, kind, label, file_id, mime_type, related_id) - VALUES (?, ?, ?, ?, ?, ?)`, - [ - recordId, - assoc.kind, - assoc.label, - assoc.kind === 'attachment' ? assoc.fileId : '', - assoc.kind === 'attachment' ? assoc.mimeType : null, - assoc.kind === 'relationship' ? assoc.recordId : '', - ], - ); + try { + this.db.run( + `INSERT OR IGNORE INTO associations + (record_id, kind, label, file_id, mime_type, related_id) + VALUES (?, ?, ?, ?, ?, ?)`, + [ + recordId, + assoc.kind, + assoc.label, + assoc.kind === 'attachment' ? assoc.fileId : '', + assoc.kind === 'attachment' ? assoc.mimeType : null, + assoc.kind === 'relationship' ? assoc.recordId : '', + ], + ); + } catch (err) { + if (isForeignKeyViolation(err)) { + throw new StackNotFoundError(`Record not found: "${recordId}"`); + } + throw err; + } } } diff --git a/packages/record-adapter-sqljs/tests/record.test.ts b/packages/record-adapter-sqljs/tests/record.test.ts index 1c925b4..d7a6e3b 100644 --- a/packages/record-adapter-sqljs/tests/record.test.ts +++ b/packages/record-adapter-sqljs/tests/record.test.ts @@ -3,7 +3,7 @@ import { mkdirSync, rmSync, existsSync, readdirSync, writeFileSync } from 'fs'; import { join } from 'path'; import { tmpdir } from 'os'; import { SQLiteRecordAdapter } from '../src/index.js'; -import { StackQueryError } from '@haverstack/core'; +import { StackConflictError, StackNotFoundError, StackQueryError } from '@haverstack/core'; import type { StackRecord } from '@haverstack/core'; // ------------------------------------------------------- @@ -840,6 +840,27 @@ describe('associations', () => { const retrieved = await adapter.getRecord(record.id); expect(retrieved?.version).toBe(3); }); + + test('associate on a nonexistent record throws StackNotFoundError instead of creating an orphan row', async () => { + const adapter = await initAdapter(); + await expect( + adapter.associate('does-not-exist', { kind: 'tag', label: 'starred' }), + ).rejects.toThrow(StackNotFoundError); + }); + + test('FK enforcement survives across persist() calls (sql.js resets pragmas on export())', async () => { + const adapter = await initAdapter(); + const record = makeRecord(); + await adapter.createRecord(record); + // Several persist()-triggering writes first, to make sure the pragma + // reset sql.js applies on export() doesn't leak past the first one. + await adapter.associate(record.id, { kind: 'tag', label: 'starred' }); + await adapter.patchContent(record.id, { text: 'updated' }); + + await expect( + adapter.associate('still-does-not-exist', { kind: 'tag', label: 'x' }), + ).rejects.toThrow(StackNotFoundError); + }); }); // ------------------------------------------------------- @@ -1015,6 +1036,101 @@ describe('commitMigration', () => { }); }); +// ------------------------------------------------------- +// deleteUnreferencedAttachmentRecords +// ------------------------------------------------------- + +describe('deleteUnreferencedAttachmentRecords', () => { + const ATTACHMENT_TYPE = 'com.example.test/_attachment@1'; + + test('throws StackConflictError when a record still references the file', async () => { + const adapter = await initAdapter(); + const record = makeRecord(); + await adapter.createRecord(record); + await adapter.associate(record.id, { + kind: 'attachment', + label: 'cover', + fileId: 'file-1', + mimeType: 'image/png', + }); + + await expect( + adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE), + ).rejects.toThrow(StackConflictError); + }); + + test('deletes an unreferenced metadata record and returns its id', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'file-1' } }), + ); + + const deleted = await adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE); + expect(deleted).toEqual(['meta1']); + expect(await adapter.getRecord('meta1')).toBeNull(); + }); + + test('deletes every metadata record sharing the same fileId (dedup case)', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'shared-file' } }), + ); + await adapter.createRecord( + makeRecord({ id: 'meta2', typeId: ATTACHMENT_TYPE, content: { fileId: 'shared-file' } }), + ); + + const deleted = await adapter.deleteUnreferencedAttachmentRecords( + 'shared-file', + ATTACHMENT_TYPE, + ); + expect(deleted.sort()).toEqual(['meta1', 'meta2']); + }); + + test('returns an empty array when no metadata records exist for the file', async () => { + const adapter = await initAdapter(); + const deleted = await adapter.deleteUnreferencedAttachmentRecords( + 'nonexistent-file', + ATTACHMENT_TYPE, + ); + expect(deleted).toEqual([]); + }); + + test('rolls back and leaves the metadata record intact when the reference check fails', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'file-1' } }), + ); + const referencing = makeRecord({ id: 'referencing' }); + await adapter.createRecord(referencing); + await adapter.associate(referencing.id, { + kind: 'attachment', + label: 'cover', + fileId: 'file-1', + mimeType: 'image/png', + }); + + await expect( + adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE), + ).rejects.toThrow(StackConflictError); + expect(await adapter.getRecord('meta1')).not.toBeNull(); + }); + + test('deleting a metadata record removes its version history too', async () => { + const adapter = await initAdapter(); + await adapter.createRecord( + makeRecord({ id: 'meta1', typeId: ATTACHMENT_TYPE, content: { fileId: 'file-1' } }), + ); + await adapter.saveVersion('meta1', { + version: 1, + content: { fileId: 'file-1' }, + updatedAt: new Date(), + }); + + await adapter.deleteUnreferencedAttachmentRecords('file-1', ATTACHMENT_TYPE); + expect(await adapter.getVersions('meta1')).toEqual([]); + }); +}); + // ------------------------------------------------------- // Tokens // ------------------------------------------------------- diff --git a/packages/record-adapter-sqljs/vitest.config.ts b/packages/record-adapter-sqljs/vitest.config.ts index e0ec2fb..cd0e1e5 100644 --- a/packages/record-adapter-sqljs/vitest.config.ts +++ b/packages/record-adapter-sqljs/vitest.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ resolve: { alias: { '@haverstack/core': resolve(__dirname, '../core/src/index.ts'), + '@haverstack/sqlite-shared': resolve(__dirname, '../sqlite-shared/src/index.ts'), }, }, test: { diff --git a/packages/sqlite-shared/package.json b/packages/sqlite-shared/package.json new file mode 100644 index 0000000..1054104 --- /dev/null +++ b/packages/sqlite-shared/package.json @@ -0,0 +1,43 @@ +{ + "name": "@haverstack/sqlite-shared", + "version": "0.1.0", + "description": "Engine-independent SQL building blocks shared between Haverstack's SQLite-backed record adapters. Internal package, not a StackAdapter implementation.", + "type": "module", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "files": [ + "dist" + ], + "repository": { + "type": "git", + "url": "https://github.com/haverstack/core", + "directory": "packages/sqlite-shared" + }, + "license": "CC0-1.0", + "keywords": [ + "haverstack", + "sqlite", + "internal" + ], + "scripts": { + "prepublishOnly": "pnpm run build", + "build": "tsc -p tsconfig.build.json", + "test": "vitest run", + "typecheck": "tsc --noEmit", + "lint": "eslint src tests" + }, + "dependencies": { + "@haverstack/core": "workspace:*" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "typescript": "^5.5.0", + "vitest": "^2.0.0" + } +} diff --git a/packages/sqlite-shared/src/cursor.ts b/packages/sqlite-shared/src/cursor.ts new file mode 100644 index 0000000..13704d4 --- /dev/null +++ b/packages/sqlite-shared/src/cursor.ts @@ -0,0 +1,61 @@ +/** + * Cursor encode/decode for keyset pagination. Uses btoa/atob rather than + * Node's Buffer so the same codec works unmodified in a browser adapter — + * cursor payloads here are plain ASCII ("field|value|id"), so no UTF-8 + * encoding step is needed. + */ + +import { StackQueryError } from '@haverstack/core'; +import type { StackRecord } from '@haverstack/core'; + +export type SortField = 'createdAt' | 'updatedAt' | 'version'; + +export const SORT_FIELDS: readonly SortField[] = ['createdAt', 'updatedAt', 'version']; + +export const encodeCursor = (field: SortField, value: number, id: string): string => + btoa(`${field}|${value}|${id}`); + +export type DecodedCursor = { field: SortField; value: number; id: string }; + +/** Throws StackQueryError for any malformed, corrupt, or unrecognized cursor. */ +export const decodeCursor = (cursor: string): DecodedCursor => { + let decoded: string; + try { + decoded = atob(cursor); + } catch { + throw new StackQueryError(`Invalid cursor: malformed "${cursor}"`); + } + const parts = decoded.split('|'); + // New format: field|value|id (3 parts). Legacy format: value|id (2 parts, implies createdAt). + const [field, value, id] = + parts.length === 3 + ? (parts as [string, string, string]) + : (['createdAt', ...parts] as [string, string, string]); + if (field === undefined || value === undefined || id === undefined) { + throw new StackQueryError(`Invalid cursor: malformed "${cursor}"`); + } + if (!SORT_FIELDS.includes(field as SortField)) { + throw new StackQueryError(`Invalid cursor: unknown sort field "${field}"`); + } + const numericValue = Number(value); + if (!isFinite(numericValue)) { + throw new StackQueryError(`Invalid cursor: non-numeric sort value`); + } + return { field: field as SortField, value: numericValue, id }; +}; + +export const getSortField = (query: { sort?: { field?: SortField } }): SortField => + query.sort?.field ?? 'createdAt'; + +export const getSortColumn = (field: SortField): string => + field === 'createdAt' ? 'created_at' : field === 'updatedAt' ? 'updated_at' : 'version'; + +export const makeCursor = (record: StackRecord, field: SortField): string => { + const value = + field === 'updatedAt' + ? record.updatedAt.getTime() + : field === 'version' + ? record.version + : record.createdAt.getTime(); + return encodeCursor(field, value, record.id); +}; diff --git a/packages/sqlite-shared/src/fts.ts b/packages/sqlite-shared/src/fts.ts new file mode 100644 index 0000000..d5c3db5 --- /dev/null +++ b/packages/sqlite-shared/src/fts.ts @@ -0,0 +1,62 @@ +/** + * FTS4 query sanitization. + * + * FTS4's query language supports operators (AND/OR/NOT, phrases, NEAR, wildcards) + * that can be expensive or cause parse errors with untrusted input. + * This function removes the dangerous operators while preserving useful ones: + * kept: AND/OR/NOT (with a left operand), phrase queries ("…"), implicit AND + * removed: wildcards (*), NEAR/N, bare NOT (no left operand) + * capped: parenthesis nesting depth (default: 2) + * + * A query timeout is not implementable for a synchronous, in-process SQLite + * engine (sql.js runs as WASM that blocks the JS event loop; native SQLite's + * interrupt() would need a worker thread to be usable as a timeout) — out of + * scope here. + * + * FTS5 (used by the native adapter) has different grammar and needs its own + * sanitizer reviewed against FTS5's rules rather than assuming this one + * transfers — tracked with the native adapter's work, not duplicated here. + */ +export const sanitizeFts4Query = (query: string, maxDepth = 2): string => { + if (!query) return ''; + + // Remove wildcards + let clean = query.replace(/\*/g, ''); + + // Remove NEAR operator (handles NEAR and NEAR/N variants) + clean = clean.replace(/\bNEAR(?:\/\d+)?\b/gi, ''); + + // Strip bare NOT with no left operand — FTS4 requires "term NOT term", not "NOT term" + clean = clean.replace(/(?:^|\(\s*)NOT\s+/gi, (m) => m.replace(/NOT\s+/i, '')); + + // Enforce max nesting depth; replace excess ( with a space and discard unmatched ) + let currentDepth = 0; + let result = ''; + for (const char of clean) { + if (char === '(') { + if (currentDepth < maxDepth) { + currentDepth++; + result += char; + } else result += ' '; + } else if (char === ')') { + if (currentDepth > 0) { + currentDepth--; + result += char; + } else result += ' '; + } else { + result += char; + } + } + + // Auto-close any unclosed parens + if (currentDepth > 0) result += ')'.repeat(currentDepth); + + // Iteratively remove empty paren pairs left behind by NEAR/NOT stripping + let prev: string; + do { + prev = result; + result = result.replace(/\(\s*\)/g, ' '); + } while (result !== prev); + + return result.replace(/\s+/g, ' ').replace(/\(\s+/g, '(').replace(/\s+\)/g, ')').trim(); +}; diff --git a/packages/sqlite-shared/src/index.ts b/packages/sqlite-shared/src/index.ts new file mode 100644 index 0000000..374ea3a --- /dev/null +++ b/packages/sqlite-shared/src/index.ts @@ -0,0 +1,18 @@ +export { RECORD_SCHEMA_SQL, FTS4_SCHEMA_SQL, PRAGMA_FOREIGN_KEYS_ON } from './schema.js'; +export { + buildWhereClause, + buildOrderClause, + getSortField, + getSortColumn, + type SanitizeSearch, +} from './query.js'; +export { + encodeCursor, + decodeCursor, + makeCursor, + SORT_FIELDS, + type SortField, + type DecodedCursor, +} from './cursor.js'; +export { rowToRecord, rowToAssociation, rowToType, rowToVersion, toMs, fromMs } from './mappers.js'; +export { sanitizeFts4Query } from './fts.js'; diff --git a/packages/sqlite-shared/src/mappers.ts b/packages/sqlite-shared/src/mappers.ts new file mode 100644 index 0000000..5841ffb --- /dev/null +++ b/packages/sqlite-shared/src/mappers.ts @@ -0,0 +1,77 @@ +/** + * Row <-> domain object mappers shared by every SQLite-backed record + * adapter. Column names and JSON-encoding choices are the storage + * contract; keeping one copy means the adapters can't drift on them. + */ + +import type { StackRecord, StackType, RecordVersion, Association } from '@haverstack/core'; + +export const toMs = (d: Date): number => d.getTime(); +export const fromMs = (ms: number): Date => new Date(ms); + +export const rowToRecord = ( + row: Record, + associations: Association[], +): StackRecord => { + const record: StackRecord = { + id: row.id as string, + typeId: row.type_id as string, + createdAt: fromMs(row.created_at as number), + updatedAt: fromMs(row.updated_at as number), + content: JSON.parse(row.content as string), + version: row.version as number, + }; + if (row.parent_id) record.parentId = row.parent_id as string; + if (row.entity_id) record.entityId = row.entity_id as string; + if (row.app_id) record.appId = row.app_id as string; + if (row.deleted_at) record.deletedAt = fromMs(row.deleted_at as number); + if (row.permissions) record.permissions = JSON.parse(row.permissions as string); + if (associations.length) record.associations = associations; + return record; +}; + +export const rowToAssociation = (row: Record): Association => { + if (row.kind === 'tag') { + return { kind: 'tag', label: row.label as string }; + } + if (row.kind === 'attachment') { + return { + kind: 'attachment', + label: row.label as string, + fileId: row.file_id as string, + mimeType: row.mime_type as string, + }; + } + // relationship + return { + kind: 'relationship', + label: row.label as string, + recordId: row.related_id as string, + }; +}; + +export const rowToType = (row: Record): StackType => { + const t: StackType = { + id: row.id as string, + baseId: row.base_id as string, + version: row.version as number, + name: row.name as string, + schema: JSON.parse(row.schema as string), + schemaHash: row.schema_hash as string, + createdAt: fromMs(row.created_at as number), + }; + if (row.migrates_from) t.migratesFrom = row.migrates_from as string; + return t; +}; + +export const rowToVersion = (row: Record): RecordVersion => { + const v: RecordVersion = { + version: row.version as number, + content: JSON.parse(row.content as string), + updatedAt: fromMs(row.updated_at as number), + }; + if (row.entity_id) v.entityId = row.entity_id as string; + if (row.associations) v.associations = JSON.parse(row.associations as string); + if (row.permissions) v.permissions = JSON.parse(row.permissions as string); + return v; +}; diff --git a/packages/sqlite-shared/src/query.ts b/packages/sqlite-shared/src/query.ts new file mode 100644 index 0000000..f60bc93 --- /dev/null +++ b/packages/sqlite-shared/src/query.ts @@ -0,0 +1,147 @@ +/** + * WHERE/ORDER clause building for StackQuery. Engine-independent SQL — + * shared verbatim between SQLite-backed record adapters. The one caveat is + * full-text search: `f.search` here assumes an `records_fts` virtual table + * with a MATCH-able `content` column and a sanitizer that accepts the raw + * query string, which both FTS4 and FTS5 setups can satisfy even though + * their grammars differ (see fts.ts). + */ + +import type { StackQuery } from '@haverstack/core'; +import { decodeCursor, getSortColumn, getSortField, type SortField } from './cursor.js'; + +export { getSortField, getSortColumn }; +export type { SortField }; + +export type SanitizeSearch = (query: string) => string; + +export const buildWhereClause = ( + query: StackQuery, + sanitizeSearch: SanitizeSearch, +): { sql: string; params: unknown[] } => { + const conditions: string[] = ["r.id != '_config'"]; + const params: unknown[] = []; + const f = query.filter ?? {}; + + if (!f.includeDeleted) { + conditions.push('r.deleted_at IS NULL'); + } + + if (f.typeId !== undefined) { + const ids = Array.isArray(f.typeId) ? f.typeId : [f.typeId]; + conditions.push(`r.type_id IN (${ids.map(() => '?').join(',')})`); + params.push(...ids); + } + + if (f.parentId !== undefined) { + if (f.parentId === null) { + conditions.push('r.parent_id IS NULL'); + } else { + conditions.push('r.parent_id = ?'); + params.push(f.parentId); + } + } + + if (f.appId !== undefined) { + const ids = Array.isArray(f.appId) ? f.appId : [f.appId]; + conditions.push(`r.app_id IN (${ids.map(() => '?').join(',')})`); + params.push(...ids); + } + + if (f.entityId !== undefined) { + const ids = Array.isArray(f.entityId) ? f.entityId : [f.entityId]; + conditions.push(`r.entity_id IN (${ids.map(() => '?').join(',')})`); + params.push(...ids); + } + + if (f.createdAt?.after) { + conditions.push('r.created_at > ?'); + params.push(f.createdAt.after.getTime()); + } + if (f.createdAt?.before) { + conditions.push('r.created_at < ?'); + params.push(f.createdAt.before.getTime()); + } + if (f.updatedAt?.after) { + conditions.push('r.updated_at > ?'); + params.push(f.updatedAt.after.getTime()); + } + if (f.updatedAt?.before) { + conditions.push('r.updated_at < ?'); + params.push(f.updatedAt.before.getTime()); + } + + // Tag filter — record must have ALL specified tags + if (f.tags?.length) { + for (const tag of f.tags) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'tag' AND a.label = ?)`, + ); + params.push(tag); + } + } + + // Attachment label filter + if (f.hasAttachment) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.label = ?)`, + ); + params.push(f.hasAttachment); + } + + // Attachment file ID filter — find records that reference a specific file + if (f.attachmentFileId) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'attachment' AND a.file_id = ?)`, + ); + params.push(f.attachmentFileId); + } + + // Relationship filter + if (f.relatedTo) { + conditions.push( + `EXISTS (SELECT 1 FROM associations a WHERE a.record_id = r.id AND a.kind = 'relationship' AND a.related_id = ?` + + (f.relatedTo.label ? ` AND a.label = ?` : '') + + `)`, + ); + params.push(f.relatedTo.recordId); + if (f.relatedTo.label) params.push(f.relatedTo.label); + } + + // Content field filters (top-level scalar exact match) + if (f.content) { + for (const [key, value] of Object.entries(f.content)) { + conditions.push(`json_extract(r.content, ?) = ?`); + params.push(`$.${key}`, value); + } + } + + if (f.search) { + const sanitized = sanitizeSearch(f.search); + if (sanitized) { + conditions.push(`r.rowid IN (SELECT rowid FROM records_fts WHERE records_fts MATCH ?)`); + params.push(sanitized); + } + } + + // Cursor (sort-field value + id for stable pagination) + if (query.cursor) { + const { field: cursorField, value: numericValue, id: cursorId } = decodeCursor(query.cursor); + const col = getSortColumn(cursorField); + const sortDir = query.sort?.direction ?? 'desc'; + const op = sortDir === 'asc' ? '>' : '<'; + conditions.push(`(r.${col} ${op} ? OR (r.${col} = ? AND r.id ${op} ?))`); + params.push(numericValue, numericValue, cursorId); + } + + return { + sql: conditions.length ? `WHERE ${conditions.join(' AND ')}` : '', + params, + }; +}; + +export const buildOrderClause = (query: StackQuery): string => { + const field = getSortField(query); + const dir = (query.sort?.direction ?? 'desc').toUpperCase(); + return `ORDER BY r.${getSortColumn(field)} ${dir}, r.id ${dir}`; +}; diff --git a/packages/sqlite-shared/src/schema.ts b/packages/sqlite-shared/src/schema.ts new file mode 100644 index 0000000..2588595 --- /dev/null +++ b/packages/sqlite-shared/src/schema.ts @@ -0,0 +1,95 @@ +/** + * Schema DDL shared by SQLite-backed record adapters. FTS4 is what sql.js's + * WASM build supports; the native adapter uses FTS5 and defines its own + * virtual table DDL rather than sharing this one — the rest of the schema + * (tables, indexes) is identical across engines. + */ + +export const RECORD_SCHEMA_SQL = ` + CREATE TABLE IF NOT EXISTS records ( + id TEXT PRIMARY KEY, + type_id TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + content TEXT NOT NULL CHECK (json_valid(content)), + version INTEGER NOT NULL DEFAULT 1, + parent_id TEXT, + entity_id TEXT, + app_id TEXT, + deleted_at INTEGER, + permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)) + ) STRICT; + + CREATE TABLE IF NOT EXISTS associations ( + record_id TEXT NOT NULL REFERENCES records(id), + kind TEXT NOT NULL CHECK (kind IN ('tag', 'attachment', 'relationship')), + label TEXT NOT NULL, + file_id TEXT NOT NULL DEFAULT '', + mime_type TEXT, + related_id TEXT NOT NULL DEFAULT '', + PRIMARY KEY (record_id, kind, label, file_id, related_id) + ) STRICT; + + CREATE TABLE IF NOT EXISTS versions ( + record_id TEXT NOT NULL REFERENCES records(id), + version INTEGER NOT NULL, + content TEXT NOT NULL CHECK (json_valid(content)), + updated_at INTEGER NOT NULL, + entity_id TEXT, + associations TEXT CHECK (associations IS NULL OR json_valid(associations)), + permissions TEXT CHECK (permissions IS NULL OR json_valid(permissions)), + PRIMARY KEY (record_id, version) + ) STRICT; + + CREATE TABLE IF NOT EXISTS types ( + id TEXT PRIMARY KEY, + base_id TEXT NOT NULL, + version INTEGER NOT NULL, + name TEXT NOT NULL, + schema TEXT NOT NULL CHECK (json_valid(schema)), + schema_hash TEXT NOT NULL, + migrates_from TEXT, + created_at INTEGER NOT NULL + ) STRICT; + + CREATE TABLE IF NOT EXISTS tokens ( + id TEXT PRIMARY KEY, + token_hash TEXT NOT NULL UNIQUE, + entity_id TEXT NOT NULL, + label TEXT, + created_at INTEGER NOT NULL, + expires_at INTEGER + ) STRICT; + + -- Indexes + CREATE INDEX IF NOT EXISTS idx_records_type_id ON records(type_id); + CREATE INDEX IF NOT EXISTS idx_records_parent_id ON records(parent_id); + CREATE INDEX IF NOT EXISTS idx_records_entity_id ON records(entity_id); + CREATE INDEX IF NOT EXISTS idx_records_app_id ON records(app_id); + CREATE INDEX IF NOT EXISTS idx_records_deleted_at ON records(deleted_at); + CREATE INDEX IF NOT EXISTS idx_records_created_at ON records(created_at); + CREATE INDEX IF NOT EXISTS idx_records_updated_at ON records(updated_at); + CREATE INDEX IF NOT EXISTS idx_assoc_record_id ON associations(record_id); + CREATE INDEX IF NOT EXISTS idx_assoc_kind_label ON associations(kind, label); + CREATE INDEX IF NOT EXISTS idx_assoc_kind_file_id ON associations(kind, file_id); + CREATE INDEX IF NOT EXISTS idx_types_base_id ON types(base_id); + CREATE INDEX IF NOT EXISTS idx_tokens_hash ON tokens(token_hash); +`; + +/** FTS4 — compatible with sql.js's WASM SQLite build. */ +export const FTS4_SCHEMA_SQL = ` + CREATE VIRTUAL TABLE IF NOT EXISTS records_fts USING fts4( + content, + content='records' + ); +`; + +/** + * Enforces the REFERENCES constraints declared above (off by default per + * SQLite connection). Without it, associating/versioning a nonexistent + * record silently creates an orphan row instead of failing loudly. + * Adapters that enable this must map the resulting constraint-violation + * error to StackNotFoundError at the call sites that can trigger it + * (chiefly `associate()` on a record that doesn't exist). + */ +export const PRAGMA_FOREIGN_KEYS_ON = `PRAGMA foreign_keys = ON;`; diff --git a/packages/sqlite-shared/tests/cursor.test.ts b/packages/sqlite-shared/tests/cursor.test.ts new file mode 100644 index 0000000..b9d9aa7 --- /dev/null +++ b/packages/sqlite-shared/tests/cursor.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect } from 'vitest'; +import { StackQueryError } from '@haverstack/core'; +import { encodeCursor, decodeCursor, makeCursor } from '../src/cursor.js'; +import type { StackRecord } from '@haverstack/core'; + +const makeRecord = (overrides: Partial = {}): StackRecord => ({ + id: 'rec01', + typeId: 'com.example/note@1', + createdAt: new Date(1000), + updatedAt: new Date(2000), + content: {}, + version: 3, + ...overrides, +}); + +describe('encodeCursor / decodeCursor', () => { + test('roundtrips field, value, and id', () => { + const cursor = encodeCursor('createdAt', 12345, 'rec01'); + expect(decodeCursor(cursor)).toEqual({ field: 'createdAt', value: 12345, id: 'rec01' }); + }); + + test('accepts the legacy 2-part (value|id) format as createdAt', () => { + const legacy = btoa('12345|rec01'); + expect(decodeCursor(legacy)).toEqual({ field: 'createdAt', value: 12345, id: 'rec01' }); + }); + + test('throws StackQueryError for non-base64 garbage', () => { + expect(() => decodeCursor('!!!not-a-cursor!!!')).toThrow(StackQueryError); + }); + + test('throws StackQueryError for an unknown sort field', () => { + expect(() => decodeCursor(btoa('bogusField|123|r1'))).toThrow(StackQueryError); + }); + + test('throws StackQueryError for a non-numeric sort value', () => { + expect(() => decodeCursor(btoa('createdAt|not-a-number|r1'))).toThrow(StackQueryError); + }); +}); + +describe('makeCursor', () => { + test('encodes createdAt by default field selection', () => { + const cursor = makeCursor(makeRecord(), 'createdAt'); + expect(decodeCursor(cursor)).toEqual({ field: 'createdAt', value: 1000, id: 'rec01' }); + }); + + test('encodes updatedAt', () => { + const cursor = makeCursor(makeRecord(), 'updatedAt'); + expect(decodeCursor(cursor)).toEqual({ field: 'updatedAt', value: 2000, id: 'rec01' }); + }); + + test('encodes version', () => { + const cursor = makeCursor(makeRecord(), 'version'); + expect(decodeCursor(cursor)).toEqual({ field: 'version', value: 3, id: 'rec01' }); + }); +}); diff --git a/packages/sqlite-shared/tests/fts.test.ts b/packages/sqlite-shared/tests/fts.test.ts new file mode 100644 index 0000000..6c1e2cd --- /dev/null +++ b/packages/sqlite-shared/tests/fts.test.ts @@ -0,0 +1,41 @@ +import { describe, test, expect } from 'vitest'; +import { sanitizeFts4Query } from '../src/fts.js'; + +describe('sanitizeFts4Query', () => { + test('returns empty string unchanged', () => { + expect(sanitizeFts4Query('')).toBe(''); + }); + + test('strips wildcards', () => { + expect(sanitizeFts4Query('hel*lo')).toBe('hello'); + }); + + test('strips NEAR and NEAR/N operators', () => { + expect(sanitizeFts4Query('foo NEAR bar')).toBe('foo bar'); + expect(sanitizeFts4Query('foo NEAR/5 bar')).toBe('foo bar'); + }); + + test('strips bare NOT with no left operand', () => { + expect(sanitizeFts4Query('NOT foo')).toBe('foo'); + }); + + test('keeps NOT with a left operand', () => { + expect(sanitizeFts4Query('foo NOT bar')).toBe('foo NOT bar'); + }); + + test('caps parenthesis nesting depth', () => { + expect(sanitizeFts4Query('(((deep)))', 2)).toBe('((deep))'); + }); + + test('auto-closes unclosed parens', () => { + expect(sanitizeFts4Query('(unclosed')).toBe('(unclosed)'); + }); + + test('discards unmatched closing parens', () => { + expect(sanitizeFts4Query('unmatched)')).toBe('unmatched'); + }); + + test('keeps phrase queries and implicit AND', () => { + expect(sanitizeFts4Query('"exact phrase" other')).toBe('"exact phrase" other'); + }); +}); diff --git a/packages/sqlite-shared/tsconfig.build.json b/packages/sqlite-shared/tsconfig.build.json new file mode 100644 index 0000000..57d0596 --- /dev/null +++ b/packages/sqlite-shared/tsconfig.build.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "rootDir": "src", + "noEmit": false + }, + "exclude": ["tests/**/*.ts"] +} diff --git a/packages/sqlite-shared/tsconfig.json b/packages/sqlite-shared/tsconfig.json new file mode 100644 index 0000000..dbd41f4 --- /dev/null +++ b/packages/sqlite-shared/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist", + "noEmit": true + }, + "include": ["src/**/*.ts", "tests/**/*.ts"] +} diff --git a/packages/sqlite-shared/vitest.config.ts b/packages/sqlite-shared/vitest.config.ts new file mode 100644 index 0000000..e0ec2fb --- /dev/null +++ b/packages/sqlite-shared/vitest.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from 'vitest/config'; +import { resolve } from 'path'; + +export default defineConfig({ + resolve: { + alias: { + '@haverstack/core': resolve(__dirname, '../core/src/index.ts'), + }, + }, + test: { + environment: 'node', + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fd24460..b6fb6a2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -123,6 +123,9 @@ importers: '@haverstack/core': specifier: workspace:* version: link:../core + '@haverstack/sqlite-shared': + specifier: workspace:* + version: link:../sqlite-shared sql.js: specifier: ^1.14.0 version: 1.14.1 @@ -140,6 +143,22 @@ importers: specifier: ^2.0.0 version: 2.1.9(@types/node@22.19.17) + packages/sqlite-shared: + dependencies: + '@haverstack/core': + specifier: workspace:* + version: link:../core + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.19.17 + typescript: + specifier: ^5.5.0 + version: 5.9.3 + vitest: + specifier: ^2.0.0 + version: 2.1.9(@types/node@22.19.17) + packages/wire-types: dependencies: '@haverstack/core':