diff --git a/templates/content/actions/_builder-cms-read-client.test.ts b/templates/content/actions/_builder-cms-read-client.test.ts index 07dc474514..e847b1a7f1 100644 --- a/templates/content/actions/_builder-cms-read-client.test.ts +++ b/templates/content/actions/_builder-cms-read-client.test.ts @@ -2,6 +2,7 @@ import { resolveBuilderCredential } from "@agent-native/core/server"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { + builderCmsListEntryFields, listBuilderCmsModels, readBuilderCmsContentEntry, readBuilderCmsContentEntries, @@ -24,6 +25,46 @@ describe("Builder CMS read client", () => { delete process.env.BUILDER_CMS_READ_LIMIT; }); + it("builds additive list projections without reintroducing heavy body fields", () => { + const fields = builderCmsListEntryFields([ + "topics", + "data.tags", + "data.customModelField", + "data.published", + "data.Status", + "data.status", + "data.tags", + "data.blocks", + "DATA.BLOCKS", + "data.blocks.children", + "data.blocksString", + "data.BlocksString", + "sys.sync_state", + "bad,field", + ]).split(","); + + expect(fields).toEqual( + expect.arrayContaining([ + "data.title", + "data.topics", + "data.tags", + "data.customModelField", + "data.published", + "data.Status", + "data.status", + ]), + ); + expect(fields).not.toContain("data.blocks"); + expect(fields).not.toContain("data.blocks.children"); + expect(fields).not.toContain("data.blocksString"); + expect(fields).not.toContain("DATA.BLOCKS"); + expect(fields).not.toContain("data.BlocksString"); + expect(fields).not.toContain("sys.sync_state"); + expect(fields.filter((field) => field === "data.tags")).toHaveLength(1); + expect(fields).toContain("published"); + expect(fields).toContain("data.published"); + }); + it("does not call Builder when the public key is not configured", async () => { resolveBuilderCredentialMock.mockResolvedValue(null); const fetchImpl = vi.fn(); @@ -55,6 +96,38 @@ describe("Builder CMS read client", () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it("keeps unconfigured model-field discovery as an empty-field fallback", async () => { + resolveBuilderCredentialMock.mockResolvedValue(null); + const fetchImpl = vi.fn(); + + await expect( + readBuilderCmsModelFields({ + model: "blog-article", + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).resolves.toEqual([]); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("throws when production model discovery returns an error state", async () => { + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PRIVATE_KEY" ? "private-key" : null, + ); + const fetchImpl = vi.fn().mockResolvedValue( + new Response("Builder unavailable", { + status: 503, + }), + ); + + await expect( + readBuilderCmsModelFields({ + model: "blog-article", + fetchImpl: fetchImpl as unknown as typeof fetch, + }), + ).rejects.toThrow("Builder MCP request failed with HTTP 503."); + expect(fetchImpl).toHaveBeenCalledTimes(1); + }); + it("lists Builder models through the MCP read endpoint", async () => { resolveBuilderCredentialMock.mockImplementation(async (key) => key === "BUILDER_PRIVATE_KEY" ? "private-key" : null, @@ -236,6 +309,11 @@ describe("Builder CMS read client", () => { expect(input.searchParams.get("limit")).toBe("100"); expect(input.searchParams.get("offset")).toBe("0"); expect(input.searchParams.get("fields")).toContain("data.title"); + expect(input.searchParams.get("fields")).toContain("data.topics"); + expect(input.searchParams.get("fields")).toContain("data.tags"); + expect(input.searchParams.get("fields")).toContain( + "data.customModelField", + ); expect(input.searchParams.get("fields")).not.toContain("data.blocks"); expect(init?.headers).toMatchObject({ accept: "application/json", @@ -250,6 +328,11 @@ describe("Builder CMS read client", () => { data: { title: "Builder title", url: "/blog/builder-title", + topics: ["AI", "CMS"], + tags: ["Agents"], + customModelField: "Preserved", + Status: "Editorial", + status: "published", }, }, ], @@ -261,6 +344,14 @@ describe("Builder CMS read client", () => { await expect( readBuilderCmsContentEntries({ model: "blog_article", + fieldPaths: [ + "data.topics", + "data.tags", + "data.customModelField", + "data.Status", + "data.status", + "data.blocks", + ], fetchImpl: fetchImpl as unknown as typeof fetch, }), ).resolves.toMatchObject({ @@ -272,11 +363,90 @@ describe("Builder CMS read client", () => { title: "Builder title", urlPath: "/blog/builder-title", updatedAt: "2026-06-08T12:00:00.000Z", + sourceValues: { + "data.topics": ["AI", "CMS"], + "data.tags": ["Agents"], + "data.customModelField": "Preserved", + "data.Status": "Editorial", + "data.status": "published", + }, }, ], }); }); + it("requests mapped model fields through Builder MCP list reads", async () => { + resolveBuilderCredentialMock.mockImplementation(async (key) => + key === "BUILDER_PRIVATE_KEY" ? "private-key" : null, + ); + const fetchImpl = vi + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify({ jsonrpc: "2.0", result: {} }), { + status: 200, + headers: { "mcp-session-id": "session-1" }, + }), + ) + .mockResolvedValueOnce( + new Response(JSON.stringify({ jsonrpc: "2.0", result: {} }), { + status: 200, + }), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + jsonrpc: "2.0", + result: { + content: [ + { + type: "text", + text: JSON.stringify({ + content: [ + { + id: "builder-entry-mcp", + lastUpdated: "2026-06-08T12:00:00.000Z", + data: { + title: "MCP title", + topics: ["AI"], + tags: ["CMS"], + customModelField: "MCP preserved", + }, + }, + ], + }), + }, + ], + }, + }), + { status: 200 }, + ), + ); + + const result = await readBuilderCmsContentEntries({ + model: "blog_article", + fieldPaths: [ + "topics", + "data.tags", + "data.customModelField", + "data.blocksString", + ], + fetchImpl: fetchImpl as unknown as typeof fetch, + }); + + expect(result.entries[0]?.sourceValues).toMatchObject({ + "data.topics": ["AI"], + "data.tags": ["CMS"], + "data.customModelField": "MCP preserved", + }); + const [, request] = fetchImpl.mock.calls[2] as [string, RequestInit]; + const fields = JSON.parse(String(request.body)).params.arguments.fields; + expect(fields).toContain("data.topics"); + expect(fields).toContain("data.tags"); + expect(fields).toContain("data.customModelField"); + expect(fields).not.toContain("data.blocks"); + expect(fields).not.toContain("data.blocksString"); + }); + it("paginates Builder content through the Content API up to the read limit", async () => { process.env.BUILDER_CONTENT_API_HOST = "https://cdn.test.builder.io"; resolveBuilderCredentialMock.mockImplementation(async (key) => diff --git a/templates/content/actions/_builder-cms-read-client.ts b/templates/content/actions/_builder-cms-read-client.ts index b63f9ed629..d82be98ea6 100644 --- a/templates/content/actions/_builder-cms-read-client.ts +++ b/templates/content/actions/_builder-cms-read-client.ts @@ -59,9 +59,75 @@ const BUILDER_CMS_DEFAULT_READ_LIMIT = 500; const BUILDER_CMS_MAX_READ_LIMIT = 1000; const BUILDER_CMS_PAGE_SIZE = 100; const BUILDER_CMS_READ_RETRIES = 2; -const BUILDER_CMS_METADATA_ENTRY_FIELDS = - "id,name,published,lastUpdated,createdDate,data.title,data.handle,data.url,data.slug,data.date,data.description,data.status,data.author,data.image"; -const BUILDER_CMS_BODY_ENTRY_FIELDS = `${BUILDER_CMS_METADATA_ENTRY_FIELDS},data.blocks,data.blocksString`; +const BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS = [ + "id", + "name", + "published", + "lastUpdated", + "createdDate", + "data.title", + "data.handle", + "data.url", + "data.slug", + "data.date", + "data.description", + "data.status", + "data.author", + "data.image", +] as const; +const BUILDER_CMS_TOP_LEVEL_METADATA_FIELDS = new Set( + BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS.filter( + (fieldPath) => !fieldPath.startsWith("data."), + ).map((fieldPath) => fieldPath.toLowerCase()), +); +const BUILDER_CMS_HEAVY_BODY_FIELD_PATHS = [ + "data.blocks", + "data.blocksString", +] as const; +const BUILDER_CMS_FIELD_PATH_PATTERN = + /^[A-Za-z0-9_$-]+(?:\.[A-Za-z0-9_$-]+)*$/; + +function normalizeBuilderCmsListFieldPath(fieldPath: string) { + const trimmed = fieldPath.trim(); + if (!trimmed || !BUILDER_CMS_FIELD_PATH_PATTERN.test(trimmed)) return null; + const normalized = trimmed.includes(".") + ? trimmed + : BUILDER_CMS_TOP_LEVEL_METADATA_FIELDS.has(trimmed.toLowerCase()) + ? trimmed + : `data.${trimmed}`; + const lower = normalized.toLowerCase(); + if ( + BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.some((heavyFieldPath) => { + const heavyLower = heavyFieldPath.toLowerCase(); + return lower === heavyLower || lower.startsWith(`${heavyLower}.`); + }) + ) { + return null; + } + if ( + normalized.includes(".") && + !normalized.toLowerCase().startsWith("data.") + ) { + return null; + } + return normalized; +} + +export function builderCmsListEntryFields(fieldPaths: readonly string[] = []) { + const fields = new Map(); + for (const fieldPath of [ + ...BUILDER_CMS_METADATA_ENTRY_FIELD_PATHS, + ...fieldPaths, + ]) { + const normalized = normalizeBuilderCmsListFieldPath(fieldPath); + if (!normalized) continue; + if (!fields.has(normalized)) fields.set(normalized, normalized); + } + return Array.from(fields.values()).join(","); +} + +const BUILDER_CMS_METADATA_ENTRY_FIELDS = builderCmsListEntryFields(); +const BUILDER_CMS_BODY_ENTRY_FIELDS = `${BUILDER_CMS_METADATA_ENTRY_FIELDS},${BUILDER_CMS_HEAVY_BODY_FIELD_PATHS.join(",")}`; function builderContentApiHost() { return ( @@ -444,6 +510,7 @@ async function initializeBuilderMcp(args: { async function readBuilderCmsContentEntriesViaMcp(args: { model: string; + fieldPaths?: readonly string[]; limit?: number; maxPages?: number; offset?: number; @@ -464,6 +531,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: { ? Math.max(0, Math.floor(args.offset)) : 0; const contentEntries: BuilderCmsSourceEntry[] = []; + const fields = builderCmsListEntryFields(args.fieldPaths); const seenContentIds = new Set(); let pagesRead = 0; let hasMore = false; @@ -490,7 +558,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: { modelName: args.model, limit: pageLimit, offset, - fields: BUILDER_CMS_METADATA_ENTRY_FIELDS, + fields, enrich: true, }, }, @@ -600,7 +668,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: { modelName: args.model, limit: 1, query: { id: entry.id }, - fields: BUILDER_CMS_METADATA_ENTRY_FIELDS, + fields, enrich: true, }, }, @@ -633,6 +701,7 @@ async function readBuilderCmsContentEntriesViaMcp(args: { async function readBuilderCmsContentEntriesViaContentApi(args: { model: string; + fieldPaths?: readonly string[]; limit?: number; maxPages?: number; offset?: number; @@ -650,7 +719,7 @@ async function readBuilderCmsContentEntriesViaContentApi(args: { // bare reference id. url.searchParams.set("enrich", "true"); url.searchParams.set("noCache", "true"); - url.searchParams.set("fields", BUILDER_CMS_METADATA_ENTRY_FIELDS); + url.searchParams.set("fields", builderCmsListEntryFields(args.fieldPaths)); const limit = readLimit(args.limit); const startOffset = @@ -902,7 +971,10 @@ export async function readBuilderCmsModelFields(args: { fetchImpl?: FetchLike; }): Promise { const models = await listBuilderCmsModels({ fetchImpl: args.fetchImpl }); - if (models.state !== "live") return []; + if (models.state === "unconfigured") return []; + if (models.state === "error") { + throw new Error(models.message ?? "Builder CMS model discovery failed."); + } const modelName = args.model.trim().toLowerCase(); return ( models.models.find((model) => { @@ -917,6 +989,7 @@ export async function readBuilderCmsModelFields(args: { export async function readBuilderCmsContentEntries(args: { model: string; + fieldPaths?: readonly string[]; limit?: number; maxPages?: number; offset?: number; @@ -929,6 +1002,7 @@ export async function readBuilderCmsContentEntries(args: { if (publicKey) { const contentApiRead = await readBuilderCmsContentEntriesViaContentApi({ model: args.model, + fieldPaths: args.fieldPaths, limit: args.limit, maxPages: args.maxPages, offset: args.offset, @@ -945,6 +1019,7 @@ export async function readBuilderCmsContentEntries(args: { try { return await readBuilderCmsContentEntriesViaMcp({ model: args.model, + fieldPaths: args.fieldPaths, limit: args.limit, maxPages: args.maxPages, offset: args.offset, diff --git a/templates/content/actions/_database-source-utils.test.ts b/templates/content/actions/_database-source-utils.test.ts index 37fb2b4460..c7b08659d1 100644 --- a/templates/content/actions/_database-source-utils.test.ts +++ b/templates/content/actions/_database-source-utils.test.ts @@ -26,6 +26,7 @@ import { normalizeSourceFreshness, serializeBuilderCmsSourceReadMetadataRecord, serializeSourceMetadataRecord, + sourceSnapshotValuesJsonProjectionSql, sourceValuesForSnapshot, sourceValuesForSeededSourceRow, sourceChangeSetKey, @@ -127,6 +128,24 @@ describe("database source helpers", () => { ).toBe(values); }); + it("strips heavy Builder bodies in the database snapshot projection", () => { + const sqliteProjection = sourceSnapshotValuesJsonProjectionSql("sqlite"); + const postgresProjection = + sourceSnapshotValuesJsonProjectionSql("postgres"); + + expect(sqliteProjection).toContain("json_remove"); + expect(postgresProjection).toContain("::jsonb"); + for (const key of [ + BUILDER_CMS_BODY_CONTENT_KEY, + BUILDER_CMS_BODY_LOSSLESS_CONTENT_KEY, + BUILDER_CMS_BODY_READABLE_MAP_KEY, + BUILDER_CMS_BODY_SIDECARS_KEY, + ]) { + expect(sqliteProjection).toContain(key); + expect(postgresProjection).toContain(key); + } + }); + it("drops stored federation metadata with unsafe regex formulas", () => { expect( normalizeSourceFederation({ @@ -179,6 +198,27 @@ describe("database source helpers", () => { }); }); + it("records suspicious empty Builder reads without calling them healthy", () => { + expect( + JSON.parse( + serializeBuilderCmsSourceReadMetadataRecord({ + sourceTable: "blog-article", + readState: "live", + entryCount: 0, + matchedRowCount: 0, + suspiciousEmpty: true, + sourceFetchState: "error", + }), + ), + ).toMatchObject({ + liveReadConfigured: true, + lastReadEntryCount: 0, + lastReadSuspiciousEmpty: true, + sourceFetchState: "error", + activeReadSourceRowIds: [], + }); + }); + it("preserves existing Builder model fields during metadata rewrites", () => { const existingMetadataJson = JSON.stringify({ builderModelFields: [ diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index c144f92c84..8942a07bcc 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -128,6 +128,7 @@ type SourceMetadataRecord = { lastReadPartial?: boolean; lastReadHasMore?: boolean; lastReadNextOffset?: number; + lastReadSuspiciousEmpty?: boolean; activeReadSourceRowIds?: string[]; sourceFetchState?: "idle" | "fetching" | "error" | string; allowDraftWrites?: boolean; @@ -336,6 +337,48 @@ const HEAVY_BUILDER_BODY_SOURCE_VALUE_KEYS = new Set([ BUILDER_CMS_BODY_SIDECARS_KEY, ]); +const SOURCE_VALUES_JSON_COLUMN = + '"content_database_source_rows"."source_values_json"'; + +export function sourceSnapshotValuesJsonProjectionSql(dialect: Dialect) { + const keys = Array.from(HEAVY_BUILDER_BODY_SOURCE_VALUE_KEYS); + if (dialect === "postgres") { + return `COALESCE((${SOURCE_VALUES_JSON_COLUMN}::jsonb${keys + .map((key) => ` - '${key}'`) + .join("")})::text, '{}')`; + } + const paths = keys.map((key) => `'$."${key}"'`); + return `COALESCE(json_remove(${SOURCE_VALUES_JSON_COLUMN}, ${paths.join(", ")}), '{}')`; +} + +function sourceSnapshotRowSelection(args: { + stripHeavyBuilderBodyValues: boolean; +}) { + const row = schema.contentDatabaseSourceRows; + return { + id: row.id, + ownerEmail: row.ownerEmail, + sourceId: row.sourceId, + databaseItemId: row.databaseItemId, + documentId: row.documentId, + sourceRowId: row.sourceRowId, + sourceQualifiedId: row.sourceQualifiedId, + sourceDisplayKey: row.sourceDisplayKey, + sourceValuesJson: args.stripHeavyBuilderBodyValues + ? sql`${sql.raw( + sourceSnapshotValuesJsonProjectionSql(getDialect()), + )}` + : row.sourceValuesJson, + provenance: row.provenance, + syncState: row.syncState, + freshness: row.freshness, + lastSyncedAt: row.lastSyncedAt, + lastSourceUpdatedAt: row.lastSourceUpdatedAt, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + }; +} + export function sourceValuesForSnapshot( sourceValues: Record, options: { includeHeavyBuilderBodyValues?: boolean } = {}, @@ -2389,10 +2432,16 @@ async function readSourceSnapshotRowsOnce(args: { source: ContentDatabaseSourceRowDb; database: ContentDatabaseRow | ContentDatabase; isBuilderSource: boolean; + includeHeavyBuilderBodyValues: boolean; }) { const db = getDb(); const rowRows = await db - .select() + .select( + sourceSnapshotRowSelection({ + stripHeavyBuilderBodyValues: + args.isBuilderSource && !args.includeHeavyBuilderBodyValues, + }), + ) .from(schema.contentDatabaseSourceRows) .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)) .orderBy(asc(schema.contentDatabaseSourceRows.createdAt)); @@ -2467,6 +2516,7 @@ async function sourceSnapshotConsistencyMarker(args: { source: ContentDatabaseSourceRowDb; database: ContentDatabaseRow | ContentDatabase; isBuilderSource: boolean; + includeHeavyBuilderBodyValues: boolean; }) { const db = getDb(); const [rows] = await db @@ -2518,6 +2568,7 @@ async function loadSourceSnapshotRowsOptimistically(args: { source: ContentDatabaseSourceRowDb; database: ContentDatabaseRow | ContentDatabase; isBuilderSource: boolean; + includeHeavyBuilderBodyValues: boolean; }) { let latest: Awaited> | null = null; @@ -2616,6 +2667,7 @@ async function loadSourceSnapshot( source, database, isBuilderSource, + includeHeavyBuilderBodyValues: options.includeHeavyBuilderBodyValues, }); const rows = rowRows.map((row) => serializeSourceRowRecord(row, { @@ -2722,7 +2774,7 @@ async function loadSourceSnapshot( string, ContentDatabaseSourceBodyChange >(); - if (isBuilderSource) { + if (isBuilderSource && options.includeHeavyBuilderBodyValues) { const sourceRowByDocumentId = new Map( rowRows.map((row) => [row.documentId, row]), ); @@ -2860,6 +2912,10 @@ async function loadSourceSnapshot( typeof metadata.lastReadNextOffset === "number" ? metadata.lastReadNextOffset : undefined, + lastReadSuspiciousEmpty: + typeof metadata.lastReadSuspiciousEmpty === "boolean" + ? metadata.lastReadSuspiciousEmpty + : undefined, sourceFetchState: metadata.sourceFetchState === "idle" || metadata.sourceFetchState === "fetching" || @@ -3019,6 +3075,7 @@ export function serializeBuilderCmsSourceReadMetadataRecord(args: { progress?: BuilderCmsReadProgress; sourceFetchState?: "idle" | "fetching" | "error"; activeReadSourceRowIds?: string[]; + suspiciousEmpty?: boolean; builderModelFields?: BuilderCmsModelFieldSummary[]; existingMetadataJson?: string | null; }) { @@ -3038,7 +3095,10 @@ export function serializeBuilderCmsSourceReadMetadataRecord(args: { lastReadPartial: args.progress?.partial, lastReadHasMore: args.progress?.hasMore, lastReadNextOffset: args.progress?.nextOffset, - activeReadSourceRowIds: args.activeReadSourceRowIds, + lastReadSuspiciousEmpty: args.suspiciousEmpty === true, + activeReadSourceRowIds: args.suspiciousEmpty + ? [] + : args.activeReadSourceRowIds, sourceFetchState: args.sourceFetchState ?? (args.progress?.partial @@ -3122,6 +3182,11 @@ export async function seedMockSourceFields(args: { }) { const db = getDb(); const isBuilder = args.sourceType === "builder-cms"; + const existingBuilderFieldByPropertyId = new Map( + (args.existingFields ?? []) + .filter((field) => isBuilder && field.propertyId) + .map((field) => [field.propertyId!, field]), + ); const rows = [ { id: crypto.randomUUID(), @@ -3224,10 +3289,12 @@ export async function seedMockSourceFields(args: { propertyId: property.definition.id, localFieldKey: property.definition.id, sourceFieldKey: isBuilder - ? builderCmsSourceFieldKey( + ? (existingBuilderFieldByPropertyId.get(property.definition.id) + ?.sourceFieldKey ?? + builderCmsSourceFieldKey( property.definition.id, property.definition.name, - ) + )) : `fields.${slugifySourceField(property.definition.name)}`, sourceFieldLabel: property.definition.name, sourceFieldType: property.definition.type, @@ -3259,15 +3326,19 @@ export async function seedMockSourceFields(args: { updatedAt: args.now, })), ]; + const sourceFieldIdentityKey = (sourceFieldKey: string) => { + const trimmed = sourceFieldKey.trim(); + return isBuilder ? trimmed : trimmed.toLowerCase(); + }; if (isBuilder) { const existingSourceFieldKeys = new Set( - rows.map((row) => row.sourceFieldKey.trim().toLowerCase()), + rows.map((row) => sourceFieldIdentityKey(row.sourceFieldKey)), ); for (const field of args.builderModelFields ?? []) { const fieldName = field.name.trim(); if (!fieldName) continue; const sourceFieldKey = `data.${fieldName}`; - const normalizedKey = sourceFieldKey.toLowerCase(); + const normalizedKey = sourceFieldIdentityKey(sourceFieldKey); if (existingSourceFieldKeys.has(normalizedKey)) continue; existingSourceFieldKeys.add(normalizedKey); rows.push({ @@ -3292,7 +3363,7 @@ export async function seedMockSourceFields(args: { for (const entry of args.builderSampleEntries ?? []) { for (const sourceFieldKey of Object.keys(entry.sourceValues)) { if (!sourceFieldKey.startsWith("data.")) continue; - const normalizedKey = sourceFieldKey.toLowerCase(); + const normalizedKey = sourceFieldIdentityKey(sourceFieldKey); if (existingSourceFieldKeys.has(normalizedKey)) continue; existingSourceFieldKeys.add(normalizedKey); const value = entry.sourceValues[sourceFieldKey]; @@ -3329,13 +3400,13 @@ export async function seedMockSourceFields(args: { const existingFieldBySourceKey = new Map( (args.existingFields ?? []).map((field) => [ - field.sourceFieldKey.trim().toLowerCase(), + sourceFieldIdentityKey(field.sourceFieldKey), field, ]), ); const mergedRows = rows.map((row) => { const existing = existingFieldBySourceKey.get( - row.sourceFieldKey.trim().toLowerCase(), + sourceFieldIdentityKey(row.sourceFieldKey), ); if (!existing) return row; return { @@ -4152,8 +4223,33 @@ export async function resyncBuilderCmsSourceSnapshot(args: { sourceMetadata.lastReadNextOffset > 0 ? sourceMetadata.lastReadNextOffset : 0; + const existingFields = await db + .select() + .from(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id)); + let builderModelFields: BuilderCmsModelFieldSummary[] | undefined; + let builderModelFieldsReadFailed = false; + try { + builderModelFields = await readBuilderCmsModelFields({ + model: args.source.sourceTable, + }); + } catch (error) { + builderModelFieldsReadFailed = true; + const message = error instanceof Error ? error.message : String(error); + console.warn( + `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`, + ); + } + const projectionModelFields = + builderModelFields && builderModelFields.length > 0 + ? builderModelFields + : (sourceMetadata.builderModelFields ?? []); const builderRead = await readBuilderCmsContentEntries({ model: args.source.sourceTable, + fieldPaths: [ + ...existingFields.map((field) => field.sourceFieldKey), + ...projectionModelFields.map((field) => `data.${field.name}`), + ], maxPages: args.runFullRefresh ? undefined : BUILDER_CMS_REFRESH_INITIAL_PAGES, @@ -4170,15 +4266,41 @@ export async function resyncBuilderCmsSourceSnapshot(args: { .select() .from(schema.contentDatabaseSourceRows) .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); + const readStartOffset = builderRead.progress?.startOffset ?? 0; + const activeReadSourceRowIdSet = new Set(activeReadSourceRowIds); + const suspiciousEmptyRead = + builderRead.state === "live" && + builderRead.entries.length === 0 && + existingRows.length > 0 && + (readStartOffset === 0 || + existingRows.some( + (row) => !activeReadSourceRowIdSet.has(row.sourceRowId), + )); + if (suspiciousEmptyRead) { + const message = + "Builder CMS returned no entries for a source with existing rows. The previous snapshot was preserved; retry the refresh before treating the source as empty."; + await updateBuilderCmsSourceReadMetadata({ + sourceId: args.source.id, + sourceTable: args.source.sourceTable, + readState: builderRead.state, + entryCount: builderRead.entries.length, + matchedRowCount: 0, + fetchedAt: builderRead.fetchedAt, + now: args.now, + message, + builderModelFields, + progress: builderRead.progress, + sourceFetchState: "error", + syncState: "error", + suspiciousEmpty: true, + }); + return; + } await enqueueEmptyHydratedBuilderBodiesFromStoredRows({ source: args.source, now: args.now, }); let importedEntriesByDocumentId = new Map(); - const existingFields = await db - .select() - .from(schema.contentDatabaseSourceFields) - .where(eq(schema.contentDatabaseSourceFields.sourceId, args.source.id)); if (builderRead.state === "live") { const importResult = await importBuilderCmsEntriesAsDatabaseItems({ database: args.database, @@ -4196,19 +4318,6 @@ export async function resyncBuilderCmsSourceSnapshot(args: { .where(eq(schema.contentDatabaseSourceRows.sourceId, args.source.id)); } } - let builderModelFields: BuilderCmsModelFieldSummary[] | undefined; - let builderModelFieldsReadFailed = false; - try { - builderModelFields = await readBuilderCmsModelFields({ - model: args.source.sourceTable, - }); - } catch (error) { - builderModelFieldsReadFailed = true; - const message = error instanceof Error ? error.message : String(error); - console.warn( - `[content] Builder model field read failed for ${args.source.sourceTable}; continuing source row sync without model field metadata. ${message}`, - ); - } const builderEntriesByDocumentId = builderRead.state === "live" ? mapBuilderCmsEntriesToLocalItems({ @@ -4797,12 +4906,14 @@ export async function updateBuilderCmsSourceReadMetadata(args: { activeReadSourceRowIds?: string[]; syncState?: ContentDatabaseSourceSyncState; builderModelFields?: BuilderCmsModelFieldSummary[]; + suspiciousEmpty?: boolean; }) { const db = getDb(); const [currentSource] = await db .select({ capabilitiesJson: schema.contentDatabaseSources.capabilitiesJson, metadataJson: schema.contentDatabaseSources.metadataJson, + lastSourceUpdatedAt: schema.contentDatabaseSources.lastSourceUpdatedAt, }) .from(schema.contentDatabaseSources) .where(eq(schema.contentDatabaseSources.id, args.sourceId)) @@ -4820,6 +4931,7 @@ export async function updateBuilderCmsSourceReadMetadata(args: { progress: args.progress, sourceFetchState: args.sourceFetchState, activeReadSourceRowIds: args.activeReadSourceRowIds, + suspiciousEmpty: args.suspiciousEmpty, builderModelFields: args.builderModelFields, existingMetadataJson: currentSource?.metadataJson, }), @@ -4829,14 +4941,21 @@ export async function updateBuilderCmsSourceReadMetadata(args: { .set({ syncState: args.syncState ?? "linked", freshness: - args.readState === "error" || args.progress?.partial + args.readState === "error" || + args.progress?.partial || + args.suspiciousEmpty ? "stale" : "fresh", capabilitiesJson: nextJson.capabilitiesJson, metadataJson: nextJson.metadataJson, lastRefreshedAt: args.now, - lastSourceUpdatedAt: args.fetchedAt, - lastError: args.readState === "error" ? args.message : null, + lastSourceUpdatedAt: args.suspiciousEmpty + ? (currentSource?.lastSourceUpdatedAt ?? null) + : args.fetchedAt, + lastError: + args.readState === "error" || args.suspiciousEmpty + ? args.message + : null, updatedAt: args.now, }) .where(eq(schema.contentDatabaseSources.id, args.sourceId)); diff --git a/templates/content/actions/add-content-database-source-field-property.test.ts b/templates/content/actions/add-content-database-source-field-property.test.ts index fc79a66e61..b2e20ecd1b 100644 --- a/templates/content/actions/add-content-database-source-field-property.test.ts +++ b/templates/content/actions/add-content-database-source-field-property.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { + builderMetadataForSourceField, propertyTypeForSourceField, sourceFieldPropertyOptions, } from "./add-content-database-source-field-property.js"; @@ -29,6 +30,45 @@ describe("propertyTypeForSourceField", () => { ).toBe("text"); }); + it("matches case-distinct Builder field metadata exactly", () => { + const sourceMetadataJson = JSON.stringify({ + builderModelFields: [ + { + name: "Status", + type: "list", + inputType: "tags", + required: false, + options: ["Editorial", "Legal"], + }, + { + name: "status", + type: "boolean", + required: false, + }, + ], + }); + + expect( + builderMetadataForSourceField({ + sourceFieldKey: "data.Status", + sourceMetadataJson, + }), + ).toMatchObject({ + name: "Status", + type: "list", + options: ["Editorial", "Legal"], + }); + expect( + builderMetadataForSourceField({ + sourceFieldKey: "data.status", + sourceMetadataJson, + }), + ).toMatchObject({ + name: "status", + type: "boolean", + }); + }); + it("generates unique option ids for distinct Builder choices with matching slugs", () => { expect( sourceFieldPropertyOptions({ diff --git a/templates/content/actions/add-content-database-source-field-property.ts b/templates/content/actions/add-content-database-source-field-property.ts index 1639892c51..355e651487 100644 --- a/templates/content/actions/add-content-database-source-field-property.ts +++ b/templates/content/actions/add-content-database-source-field-property.ts @@ -201,10 +201,7 @@ export function sourceFieldPropertyOptions(args: { } function builderFieldNameForSourceKey(sourceFieldKey: string) { - return sourceFieldKey - .replace(/^data\./, "") - .trim() - .toLowerCase(); + return sourceFieldKey.replace(/^data\./, "").trim(); } function builderModelFieldsFromMetadata( @@ -231,15 +228,15 @@ function builderModelFieldsFromMetadata( } } -function builderMetadataForSourceField(args: { +export function builderMetadataForSourceField(args: { sourceFieldKey: string; sourceMetadataJson: string | null | undefined; }) { const fieldName = builderFieldNameForSourceKey(args.sourceFieldKey); - const sourceFieldKey = args.sourceFieldKey.trim().toLowerCase(); + const sourceFieldKey = args.sourceFieldKey.trim(); return ( builderModelFieldsFromMetadata(args.sourceMetadataJson).find((field) => { - const name = field.name.trim().toLowerCase(); + const name = field.name.trim(); return name === fieldName || `data.${name}` === sourceFieldKey; }) ?? null ); diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index 4fb7a3e1f2..bf6cb1bd7a 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -48,13 +48,41 @@ const BUILDER_CMS_ATTACH_INITIAL_PAGES = 1; export async function readInitialBuilderCmsAttachEntries( sourceTable: string, readEntries: typeof readBuilderCmsContentEntries = readBuilderCmsContentEntries, + fieldPaths: readonly string[] = [], ) { return readEntries({ model: sourceTable, + fieldPaths, maxPages: BUILDER_CMS_ATTACH_INITIAL_PAGES, }); } +export async function readInitialBuilderCmsAttachSource( + sourceTable: string, + dependencies: { + readModelFields?: typeof readBuilderCmsModelFields; + readEntries?: typeof readBuilderCmsContentEntries; + } = {}, +) { + const readModelFields = + dependencies.readModelFields ?? readBuilderCmsModelFields; + const readEntries = dependencies.readEntries ?? readBuilderCmsContentEntries; + let modelFields: BuilderCmsModelFieldSummary[] = []; + let modelFieldsError: unknown = null; + try { + modelFields = await readModelFields({ model: sourceTable }); + } catch (error) { + modelFieldsError = error; + } + const read = await readInitialBuilderCmsAttachEntries( + sourceTable, + readEntries, + modelFields.map((field) => `data.${field.name}`), + ); + if (modelFieldsError) throw modelFieldsError; + return { read, modelFields }; +} + function builderReadHasMore(read: BuilderCmsReadResult | null | undefined) { return read?.state === "live" && read.progress?.hasMore === true; } @@ -238,9 +266,10 @@ export default defineAction({ let modelFields: BuilderCmsModelFieldSummary[]; let builderRead: BuilderCmsReadResult | null = null; if (sourceType === "builder-cms") { - builderRead = await readInitialBuilderCmsAttachEntries(sourceTable); + const initial = await readInitialBuilderCmsAttachSource(sourceTable); + modelFields = initial.modelFields; + builderRead = initial.read; entries = builderRead.state === "live" ? builderRead.entries : []; - modelFields = await readBuilderCmsModelFields({ model: sourceTable }); } else if (sourceType === "local-table") { // sourceTable carries the target database id for a local-table source. ({ entries, modelFields } = await readLocalTableEntries(sourceTable, { @@ -327,13 +356,12 @@ export default defineAction({ if (await databaseSourceExistsForTable(database.id, sourceTable)) { throw new Error(`"${sourceTable}" is already attached as a source.`); } - const additionalRead = - await readInitialBuilderCmsAttachEntries(sourceTable); + const additionalInitial = + await readInitialBuilderCmsAttachSource(sourceTable); + const additionalModelFields = additionalInitial.modelFields; + const additionalRead = additionalInitial.read; const additionalEntries = additionalRead.state === "live" ? additionalRead.entries : []; - const additionalModelFields = await readBuilderCmsModelFields({ - model: sourceTable, - }); const additionalSourceId = await insertSecondarySource({ database, sourceType, @@ -436,6 +464,10 @@ export default defineAction({ const existingSourceRows = existingSource ? await getSourceRows(existingSource.id) : []; + const builderInitial = + sourceType === "builder-cms" + ? await readInitialBuilderCmsAttachSource(sourceTable) + : null; const sourceId = await replaceSourceMetadata({ database, source: existingSource, @@ -444,18 +476,10 @@ export default defineAction({ sourceTable, now, }); - const builderRead = - sourceType === "builder-cms" - ? await readInitialBuilderCmsAttachEntries(sourceTable) - : null; + const builderModelFields = builderInitial?.modelFields ?? []; + const builderRead = builderInitial?.read ?? null; const builderEntries = builderRead?.state === "live" ? builderRead.entries : []; - const builderModelFields = - sourceType === "builder-cms" - ? await readBuilderCmsModelFields({ - model: sourceTable, - }) - : []; let importedEntriesByDocumentId = new Map(); if (builderRead?.state === "live") { const importResult = await importBuilderCmsEntriesAsDatabaseItems({ diff --git a/templates/content/actions/change-content-database-source-role.ts b/templates/content/actions/change-content-database-source-role.ts index ecff21f913..78f3d6109d 100644 --- a/templates/content/actions/change-content-database-source-role.ts +++ b/templates/content/actions/change-content-database-source-role.ts @@ -187,6 +187,37 @@ async function removeRowsOwnedOnlyBySource(args: { ); } +export async function readBuilderCmsEntriesForRoleChange( + args: { + model: string; + existingFieldPaths?: readonly string[]; + }, + dependencies: { + readModelFields?: typeof readBuilderCmsModelFields; + readEntries?: typeof readBuilderCmsContentEntries; + } = {}, +) { + const readModelFields = + dependencies.readModelFields ?? readBuilderCmsModelFields; + const readEntries = dependencies.readEntries ?? readBuilderCmsContentEntries; + let modelFields: BuilderCmsModelFieldSummary[] = []; + let modelFieldsError: unknown = null; + try { + modelFields = await readModelFields({ model: args.model }); + } catch (error) { + modelFieldsError = error; + } + const read = await readEntries({ + model: args.model, + fieldPaths: [ + ...(args.existingFieldPaths ?? []), + ...modelFields.map((field) => `data.${field.name}`), + ], + }); + if (modelFieldsError) throw modelFieldsError; + return { read, modelFields }; +} + async function readSourceEntries(args: { sourceType: ContentDatabaseSourceType; sourceTable: string; @@ -200,12 +231,12 @@ async function readSourceEntries(args: { message: string | null; }> { if (args.sourceType === "builder-cms") { - const read = await readBuilderCmsContentEntries({ + const { read, modelFields } = await readBuilderCmsEntriesForRoleChange({ model: args.sourceTable, }); return { entries: read.state === "live" ? read.entries : [], - modelFields: await readBuilderCmsModelFields({ model: args.sourceTable }), + modelFields, readState: read.state, fetchedAt: read.fetchedAt, message: read.message, @@ -328,13 +359,20 @@ export default defineAction({ throw new Error("Only Builder sources can add more items right now."); } - const read = await readBuilderCmsContentEntries({ - model: source.sourceTable, - }); + const existingSourceFields = await db + .select({ + sourceFieldKey: schema.contentDatabaseSourceFields.sourceFieldKey, + }) + .from(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.sourceId, source.id)); + const { read, modelFields: builderModelFields } = + await readBuilderCmsEntriesForRoleChange({ + model: source.sourceTable, + existingFieldPaths: existingSourceFields.map( + (field) => field.sourceFieldKey, + ), + }); const entries = read.state === "live" ? read.entries : []; - const builderModelFields = await readBuilderCmsModelFields({ - model: source.sourceTable, - }); await db .delete(schema.contentDatabaseSourceFields) .where(eq(schema.contentDatabaseSourceFields.sourceId, source.id)); diff --git a/templates/content/actions/content-database-source-actions.test.ts b/templates/content/actions/content-database-source-actions.test.ts index 9210ad2144..c0842e526d 100644 --- a/templates/content/actions/content-database-source-actions.test.ts +++ b/templates/content/actions/content-database-source-actions.test.ts @@ -11,8 +11,11 @@ import addSourceFieldProperty, { import attachSource, { builderCmsAttachReadMetadata, readInitialBuilderCmsAttachEntries, + readInitialBuilderCmsAttachSource, } from "./attach-content-database-source"; -import changeSourceRole from "./change-content-database-source-role"; +import changeSourceRole, { + readBuilderCmsEntriesForRoleChange, +} from "./change-content-database-source-role"; import disconnectSource from "./disconnect-content-database-source"; import executeBatch from "./execute-builder-source-batch"; import executeExecution from "./execute-builder-source-execution"; @@ -133,11 +136,19 @@ describe("content database source actions", () => { }); it("bounds initial Builder source attachment to a single continuation page", async () => { - const calls: Array<{ model: string; maxPages?: number }> = []; + const calls: Array<{ + model: string; + maxPages?: number; + fieldPaths?: readonly string[]; + }> = []; const result = await readInitialBuilderCmsAttachEntries( "blog-article", async (args) => { - calls.push({ model: args.model, maxPages: args.maxPages }); + calls.push({ + model: args.model, + maxPages: args.maxPages, + fieldPaths: args.fieldPaths, + }); return { state: "live", entries: [], @@ -155,10 +166,91 @@ describe("content database source actions", () => { }, }; }, + ["topics", "tags"], ); expect(result.state).toBe("live"); - expect(calls).toEqual([{ model: "blog-article", maxPages: 1 }]); + expect(calls).toEqual([ + { + model: "blog-article", + maxPages: 1, + fieldPaths: ["topics", "tags"], + }, + ]); + }); + + it("fails Builder attachment preparation before durable source mutation when model discovery fails", async () => { + const calls: string[] = []; + await expect( + readInitialBuilderCmsAttachSource("blog-article", { + readModelFields: async () => { + calls.push("model-fields"); + throw new Error("model discovery unavailable"); + }, + readEntries: async () => { + calls.push("entries"); + return { + state: "live", + entries: [], + fetchedAt: "2026-01-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset: 0, + nextOffset: 0, + fetchedEntryCount: 0, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + }, + }), + ).rejects.toThrow("model discovery unavailable"); + expect(calls).toEqual(["model-fields", "entries"]); + }); + + it("fails role-change preparation before mappings can be rewritten when model discovery fails", async () => { + const calls: string[] = []; + const existingMappings = ["data.topics", "data.tags"]; + + await expect( + readBuilderCmsEntriesForRoleChange( + { + model: "blog-article", + existingFieldPaths: existingMappings, + }, + { + readModelFields: async () => { + calls.push("model-fields"); + throw new Error("model discovery unavailable"); + }, + readEntries: async (args) => { + calls.push("entries"); + expect(args.fieldPaths).toEqual(existingMappings); + return { + state: "live", + entries: [], + fetchedAt: "2026-01-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset: 0, + nextOffset: 0, + fetchedEntryCount: 0, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + }, + }, + ), + ).rejects.toThrow("model discovery unavailable"); + expect(calls).toEqual(["model-fields", "entries"]); + expect(existingMappings).toEqual(["data.topics", "data.tags"]); }); it("marks partial Builder source attachment reads as continuing work", () => { diff --git a/templates/content/actions/resync-content-database-source.db.test.ts b/templates/content/actions/resync-content-database-source.db.test.ts index 60f8cc94ab..2d11501bbb 100644 --- a/templates/content/actions/resync-content-database-source.db.test.ts +++ b/templates/content/actions/resync-content-database-source.db.test.ts @@ -26,7 +26,12 @@ import { const builderReadMock = vi.hoisted(() => ({ mode: "full" as "full" | "paged", - calls: [] as Array<{ model: string; maxPages?: number; offset?: number }>, + calls: [] as Array<{ + model: string; + fieldPaths?: readonly string[]; + maxPages?: number; + offset?: number; + }>, modelFieldsErrorFor: null as string | null, singleEntryCalls: [] as Array<{ model: string; entryId: string }>, singleEntryErrorFor: null as string | null, @@ -47,6 +52,16 @@ vi.mock("./_builder-cms-read-client.js", async () => { if (builderReadMock.modelFieldsErrorFor === model) { throw new Error("read ECONNRESET"); } + if (model === "collection-mapped-fields") { + return [ + { name: "topics", type: "list", required: false }, + { name: "tags", type: "list", required: false }, + { name: "customModelField", type: "string", required: false }, + { name: "published", type: "boolean", required: false }, + { name: "Status", type: "string", required: false }, + { name: "status", type: "string", required: false }, + ]; + } return []; }), readBuilderCmsContentEntry: vi.fn( @@ -114,14 +129,89 @@ vi.mock("./_builder-cms-read-client.js", async () => { readBuilderCmsContentEntries: vi.fn( async ({ model, + fieldPaths, maxPages, offset, }: { model: string; + fieldPaths?: readonly string[]; maxPages?: number; offset?: number; }) => { - builderReadMock.calls.push({ model, maxPages, offset }); + builderReadMock.calls.push({ model, fieldPaths, maxPages, offset }); + if (model === "collection-suspicious-empty") { + return { + state: "live", + entries: [], + fetchedAt: "2026-02-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset: 0, + nextOffset: 0, + fetchedEntryCount: 0, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + } + if (model === "collection-suspicious-empty-continuation") { + const startOffset = offset ?? 1; + return { + state: "live", + entries: [], + fetchedAt: "2026-02-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset, + nextOffset: startOffset, + fetchedEntryCount: startOffset, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + } + if (model === "collection-mapped-fields") { + const entries = [ + { + id: "entry-mapped-fields", + model, + title: "Mapped fields", + urlPath: "/blog/mapped-fields", + updatedAt: "2026-02-01T00:00:00.000Z", + sourceValues: { + "data.title": "Mapped fields", + "data.topics": ["AI", "CMS"], + "data.tags": ["Agents", "Content"], + "data.customModelField": "Arbitrary value", + "data.published": true, + "data.Status": "Editorial", + "data.status": "published", + }, + }, + ]; + return { + state: "live", + entries, + fetchedAt: "2026-02-01T00:00:00.000Z", + message: null, + progress: { + requestedLimit: 500, + pageSize: 100, + startOffset: 0, + nextOffset: entries.length, + fetchedEntryCount: entries.length, + hasMore: false, + partial: false, + readMode: "builder-api", + }, + }; + } if (model === "collection-duplicates") { return { state: "live", @@ -328,6 +418,7 @@ let resync: typeof import("./_database-source-utils.js").resyncBuilderCmsSourceS let importBuilderEntries: typeof import("./_database-source-utils.js").importBuilderCmsEntriesAsDatabaseItems; let materializeSourceFields: typeof import("./_database-source-utils.js").materializeSourceFieldPropertyValues; let getSnapshot: typeof import("./_database-source-utils.js").getContentDatabaseSourceSnapshotById; +let getWriteSnapshot: typeof import("./_database-source-utils.js").getContentDatabaseSourceSnapshotForWrite; let hydrateQueuedBodies: typeof import("./_database-source-utils.js").processBuilderBodyHydrationQueue; const OWNER = "owner@example.com"; @@ -347,6 +438,8 @@ beforeAll(async () => { .materializeSourceFieldPropertyValues; getSnapshot = (await import("./_database-source-utils.js")) .getContentDatabaseSourceSnapshotById; + getWriteSnapshot = (await import("./_database-source-utils.js")) + .getContentDatabaseSourceSnapshotForWrite; hydrateQueuedBodies = (await import("./_database-source-utils.js")) .processBuilderBodyHydrationQueue; }, 60000); @@ -363,6 +456,613 @@ afterAll(() => { } }); +it("preserves an established source snapshot when Builder unexpectedly returns zero entries", async () => { + builderReadMock.mode = "full"; + builderReadMock.calls = []; + const db = getDb(); + const createdAt = "2026-01-01T00:00:00.000Z"; + const refreshAt = "2026-02-01T00:05:00.000Z"; + const sourceValues = { + "data.title": "Existing Builder row", + "data.tags": ["preserve-me"], + [BUILDER_CMS_BODY_CONTENT_KEY]: "A large hydrated body stays stored.", + [BUILDER_CMS_BODY_BLOCKS_HASH_KEY]: "body-hash", + }; + + await db.insert(schema.documents).values([ + { + id: "doc-suspicious-db", + ownerEmail: OWNER, + title: "Suspicious empty DB", + createdAt, + updatedAt: createdAt, + }, + { + id: "doc-suspicious-row", + ownerEmail: OWNER, + parentId: "doc-suspicious-db", + title: "Existing Builder row", + content: "A large hydrated body stays stored.", + createdAt, + updatedAt: createdAt, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: "db-suspicious-empty", + ownerEmail: OWNER, + documentId: "doc-suspicious-db", + title: "Suspicious empty DB", + createdAt, + updatedAt: createdAt, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: "item-suspicious-row", + ownerEmail: OWNER, + databaseId: "db-suspicious-empty", + documentId: "doc-suspicious-row", + bodyHydrationStatus: "hydrated", + createdAt, + updatedAt: createdAt, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: "source-suspicious-empty", + ownerEmail: OWNER, + databaseId: "db-suspicious-empty", + sourceType: "builder-cms", + sourceName: "Suspicious Builder source", + sourceTable: "collection-suspicious-empty", + syncState: "idle", + freshness: "fresh", + lastSourceUpdatedAt: createdAt, + createdAt, + updatedAt: createdAt, + }); + await db.insert(schema.contentDatabaseSourceRows).values({ + id: "source-row-suspicious-empty", + ownerEmail: OWNER, + sourceId: "source-suspicious-empty", + databaseItemId: "item-suspicious-row", + documentId: "doc-suspicious-row", + sourceRowId: "builder-entry-existing", + sourceQualifiedId: + "builder-cms://collection-suspicious-empty/builder-entry-existing", + sourceDisplayKey: "Existing Builder row", + sourceValuesJson: JSON.stringify(sourceValues), + provenance: "Builder CMS read adapter", + freshness: "fresh", + createdAt, + updatedAt: createdAt, + }); + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, "db-suspicious-empty")); + const [source] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, "source-suspicious-empty")); + + await resync({ database, source, now: refreshAt }); + + const rows = await db + .select() + .from(schema.contentDatabaseSourceRows) + .where( + eq(schema.contentDatabaseSourceRows.sourceId, "source-suspicious-empty"), + ); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + id: "source-row-suspicious-empty", + sourceValuesJson: JSON.stringify(sourceValues), + updatedAt: createdAt, + }); + const [updatedSource] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, "source-suspicious-empty")); + expect(updatedSource).toMatchObject({ + freshness: "stale", + syncState: "error", + lastSourceUpdatedAt: createdAt, + }); + expect(updatedSource.lastError).toContain("previous snapshot was preserved"); + expect(JSON.parse(updatedSource.metadataJson)).toMatchObject({ + lastReadEntryCount: 0, + lastReadSuspiciousEmpty: true, + sourceFetchState: "error", + activeReadSourceRowIds: [], + }); + + const snapshot = await getSnapshot(database, source.id); + expect(snapshot?.rows[0]?.sourceValues).toMatchObject({ + "data.title": "Existing Builder row", + "data.tags": ["preserve-me"], + [BUILDER_CMS_BODY_BLOCKS_HASH_KEY]: "body-hash", + }); + expect(snapshot?.rows[0]?.sourceValues).not.toHaveProperty( + BUILDER_CMS_BODY_CONTENT_KEY, + ); + const writeSnapshot = await getWriteSnapshot(database, source.id); + expect(writeSnapshot?.rows[0]?.sourceValues).toMatchObject({ + [BUILDER_CMS_BODY_CONTENT_KEY]: "A large hydrated body stays stored.", + [BUILDER_CMS_BODY_BLOCKS_HASH_KEY]: "body-hash", + }); +}); + +it("preserves unvisited rows when an empty continuation page would prune them", async () => { + builderReadMock.mode = "full"; + builderReadMock.calls = []; + const db = getDb(); + const createdAt = "2026-01-01T00:00:00.000Z"; + const refreshAt = "2026-02-01T00:10:00.000Z"; + await db.insert(schema.documents).values([ + { + id: "doc-continuation-empty-db", + ownerEmail: OWNER, + title: "Continuation empty DB", + createdAt, + updatedAt: createdAt, + }, + { + id: "doc-continuation-seen", + ownerEmail: OWNER, + parentId: "doc-continuation-empty-db", + title: "Seen on the first page", + createdAt, + updatedAt: createdAt, + }, + { + id: "doc-continuation-unvisited", + ownerEmail: OWNER, + parentId: "doc-continuation-empty-db", + title: "Not yet revisited", + createdAt, + updatedAt: createdAt, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: "db-continuation-empty", + ownerEmail: OWNER, + documentId: "doc-continuation-empty-db", + title: "Continuation empty DB", + createdAt, + updatedAt: createdAt, + }); + await db.insert(schema.contentDatabaseItems).values([ + { + id: "item-continuation-seen", + ownerEmail: OWNER, + databaseId: "db-continuation-empty", + documentId: "doc-continuation-seen", + createdAt, + updatedAt: createdAt, + }, + { + id: "item-continuation-unvisited", + ownerEmail: OWNER, + databaseId: "db-continuation-empty", + documentId: "doc-continuation-unvisited", + createdAt, + updatedAt: createdAt, + }, + ]); + await db.insert(schema.contentDatabaseSources).values({ + id: "source-continuation-empty", + ownerEmail: OWNER, + databaseId: "db-continuation-empty", + sourceType: "builder-cms", + sourceName: "Continuation Builder source", + sourceTable: "collection-suspicious-empty-continuation", + syncState: "refreshing", + freshness: "stale", + lastSourceUpdatedAt: createdAt, + metadataJson: JSON.stringify({ + sourceFetchState: "fetching", + lastReadHasMore: true, + lastReadNextOffset: 1, + activeReadSourceRowIds: ["entry-seen"], + }), + createdAt, + updatedAt: createdAt, + }); + await db.insert(schema.contentDatabaseSourceRows).values([ + { + id: "source-row-continuation-seen", + ownerEmail: OWNER, + sourceId: "source-continuation-empty", + databaseItemId: "item-continuation-seen", + documentId: "doc-continuation-seen", + sourceRowId: "entry-seen", + sourceQualifiedId: + "builder-cms://collection-suspicious-empty-continuation/entry-seen", + sourceDisplayKey: "Seen on the first page", + sourceValuesJson: JSON.stringify({ + "data.title": "Seen on the first page", + }), + provenance: "Builder CMS read adapter", + freshness: "fresh", + createdAt, + updatedAt: createdAt, + }, + { + id: "source-row-continuation-unvisited", + ownerEmail: OWNER, + sourceId: "source-continuation-empty", + databaseItemId: "item-continuation-unvisited", + documentId: "doc-continuation-unvisited", + sourceRowId: "entry-unvisited", + sourceQualifiedId: + "builder-cms://collection-suspicious-empty-continuation/entry-unvisited", + sourceDisplayKey: "Not yet revisited", + sourceValuesJson: JSON.stringify({ + "data.title": "Not yet revisited", + }), + provenance: "Builder CMS read adapter", + freshness: "fresh", + createdAt, + updatedAt: createdAt, + }, + ]); + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, "db-continuation-empty")); + const [source] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, "source-continuation-empty")); + await resync({ database, source, now: refreshAt }); + + const rows = await db + .select({ + id: schema.contentDatabaseSourceRows.id, + sourceRowId: schema.contentDatabaseSourceRows.sourceRowId, + updatedAt: schema.contentDatabaseSourceRows.updatedAt, + }) + .from(schema.contentDatabaseSourceRows) + .where( + eq( + schema.contentDatabaseSourceRows.sourceId, + "source-continuation-empty", + ), + ); + expect(rows).toEqual( + expect.arrayContaining([ + { + id: "source-row-continuation-seen", + sourceRowId: "entry-seen", + updatedAt: createdAt, + }, + { + id: "source-row-continuation-unvisited", + sourceRowId: "entry-unvisited", + updatedAt: createdAt, + }, + ]), + ); + expect(rows).toHaveLength(2); + const [updatedSource] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, "source-continuation-empty")); + expect(updatedSource).toMatchObject({ + freshness: "stale", + syncState: "error", + lastSourceUpdatedAt: createdAt, + }); + expect(updatedSource.lastError).toContain("previous snapshot was preserved"); + expect(JSON.parse(updatedSource.metadataJson)).toMatchObject({ + lastReadSuspiciousEmpty: true, + sourceFetchState: "error", + activeReadSourceRowIds: [], + }); + expect( + builderReadMock.calls.find( + (call) => call.model === "collection-suspicious-empty-continuation", + )?.offset, + ).toBe(1); +}); + +it("accepts a zero-entry read for a source that has never had rows", async () => { + builderReadMock.mode = "full"; + builderReadMock.calls = []; + const db = getDb(); + const now = "2026-02-01T00:30:00.000Z"; + await db.insert(schema.documents).values({ + id: "doc-new-empty-db", + ownerEmail: OWNER, + title: "New empty DB", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabases).values({ + id: "db-new-empty", + ownerEmail: OWNER, + documentId: "doc-new-empty-db", + title: "New empty DB", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSources).values({ + id: "source-new-empty", + ownerEmail: OWNER, + databaseId: "db-new-empty", + sourceType: "builder-cms", + sourceName: "New empty Builder source", + sourceTable: "collection-suspicious-empty", + syncState: "linked", + freshness: "unknown", + createdAt: now, + updatedAt: now, + }); + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, "db-new-empty")); + const [source] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, "source-new-empty")); + await resync({ database, source, now }); + + const [updatedSource] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, source.id)); + expect(updatedSource).toMatchObject({ + freshness: "fresh", + syncState: "idle", + lastError: null, + lastSourceUpdatedAt: "2026-02-01T00:00:00.000Z", + }); + expect(JSON.parse(updatedSource.metadataJson)).toMatchObject({ + lastReadEntryCount: 0, + lastReadSuspiciousEmpty: false, + sourceFetchState: "idle", + }); +}); + +it("materializes topics, tags, and arbitrary Builder model fields", async () => { + builderReadMock.mode = "full"; + builderReadMock.calls = []; + const db = getDb(); + const now = "2026-02-01T01:00:00.000Z"; + await db.insert(schema.documents).values({ + id: "doc-mapped-fields-db", + ownerEmail: OWNER, + title: "Mapped fields DB", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabases).values({ + id: "db-mapped-fields", + ownerEmail: OWNER, + documentId: "doc-mapped-fields-db", + title: "Mapped fields DB", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.documentPropertyDefinitions).values([ + { + id: "prop-topics", + ownerEmail: OWNER, + databaseId: "db-mapped-fields", + name: "Topics", + type: "multi_select", + createdAt: now, + updatedAt: now, + }, + { + id: "prop-tags", + ownerEmail: OWNER, + databaseId: "db-mapped-fields", + name: "Tags", + type: "multi_select", + createdAt: now, + updatedAt: now, + }, + { + id: "prop-custom", + ownerEmail: OWNER, + databaseId: "db-mapped-fields", + name: "Custom model field", + type: "text", + createdAt: now, + updatedAt: now, + }, + { + id: "prop-status-upper", + ownerEmail: OWNER, + databaseId: "db-mapped-fields", + name: "Status", + type: "text", + createdAt: now, + updatedAt: now, + }, + { + id: "prop-status-lower", + ownerEmail: OWNER, + databaseId: "db-mapped-fields", + name: "Status", + type: "text", + createdAt: now, + updatedAt: now, + }, + ]); + await db.insert(schema.contentDatabaseSources).values({ + id: "source-mapped-fields", + ownerEmail: OWNER, + databaseId: "db-mapped-fields", + sourceType: "builder-cms", + sourceName: "Mapped Builder source", + sourceTable: "collection-mapped-fields", + createdAt: now, + updatedAt: now, + }); + await db.insert(schema.contentDatabaseSourceFields).values([ + { + id: "field-topics", + ownerEmail: OWNER, + sourceId: "source-mapped-fields", + propertyId: "prop-topics", + localFieldKey: "prop-topics", + sourceFieldKey: "data.topics", + sourceFieldLabel: "Topics", + sourceFieldType: "list", + mappingType: "property", + writeOwner: "source", + createdAt: now, + updatedAt: now, + }, + { + id: "field-tags", + ownerEmail: OWNER, + sourceId: "source-mapped-fields", + propertyId: "prop-tags", + localFieldKey: "prop-tags", + sourceFieldKey: "data.tags", + sourceFieldLabel: "Tags", + sourceFieldType: "list", + mappingType: "property", + writeOwner: "source", + createdAt: now, + updatedAt: now, + }, + { + id: "field-custom", + ownerEmail: OWNER, + sourceId: "source-mapped-fields", + propertyId: "prop-custom", + localFieldKey: "prop-custom", + sourceFieldKey: "data.customModelField", + sourceFieldLabel: "Custom model field", + sourceFieldType: "text", + mappingType: "property", + writeOwner: "source", + createdAt: now, + updatedAt: now, + }, + { + id: "field-status-upper", + ownerEmail: OWNER, + sourceId: "source-mapped-fields", + propertyId: "prop-status-upper", + localFieldKey: "prop-status-upper", + sourceFieldKey: "data.Status", + sourceFieldLabel: "Status upper", + sourceFieldType: "text", + mappingType: "property", + writeOwner: "source", + createdAt: now, + updatedAt: now, + }, + { + id: "field-status-lower", + ownerEmail: OWNER, + sourceId: "source-mapped-fields", + propertyId: "prop-status-lower", + localFieldKey: "prop-status-lower", + sourceFieldKey: "data.status", + sourceFieldLabel: "Status lower", + sourceFieldType: "text", + mappingType: "property", + writeOwner: "source", + createdAt: now, + updatedAt: now, + }, + ]); + + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, "db-mapped-fields")); + const [source] = await db + .select() + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, "source-mapped-fields")); + await resync({ database, source, now }); + + const [sourceRow] = await db + .select() + .from(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, source.id)); + expect(JSON.parse(sourceRow.sourceValuesJson)).toMatchObject({ + "data.topics": ["AI", "CMS"], + "data.tags": ["Agents", "Content"], + "data.customModelField": "Arbitrary value", + "data.published": true, + "data.Status": "Editorial", + "data.status": "published", + }); + const values = await db + .select({ + propertyId: schema.documentPropertyValues.propertyId, + valueJson: schema.documentPropertyValues.valueJson, + }) + .from(schema.documentPropertyValues) + .where(eq(schema.documentPropertyValues.documentId, sourceRow.documentId)); + expect( + Object.fromEntries( + values.map((value: { propertyId: string; valueJson: string }) => [ + value.propertyId, + JSON.parse(value.valueJson), + ]), + ), + ).toMatchObject({ + "prop-topics": ["AI", "CMS"], + "prop-tags": ["Agents", "Content"], + "prop-custom": "Arbitrary value", + "prop-status-upper": "Editorial", + "prop-status-lower": "published", + }); + const statusFieldMappings = await db + .select({ + id: schema.contentDatabaseSourceFields.id, + propertyId: schema.contentDatabaseSourceFields.propertyId, + sourceFieldKey: schema.contentDatabaseSourceFields.sourceFieldKey, + }) + .from(schema.contentDatabaseSourceFields) + .where( + eq(schema.contentDatabaseSourceFields.sourceId, "source-mapped-fields"), + ); + expect( + statusFieldMappings + .filter((field: { propertyId: string | null }) => + ["prop-status-upper", "prop-status-lower"].includes( + field.propertyId ?? "", + ), + ) + .sort((a, b) => String(a.propertyId).localeCompare(String(b.propertyId))), + ).toEqual([ + { + id: "field-status-lower", + propertyId: "prop-status-lower", + sourceFieldKey: "data.status", + }, + { + id: "field-status-upper", + propertyId: "prop-status-upper", + sourceFieldKey: "data.Status", + }, + ]); + const readCall = builderReadMock.calls.find( + (call) => call.model === "collection-mapped-fields", + ); + expect(readCall?.fieldPaths).toEqual( + expect.arrayContaining([ + "data.topics", + "data.tags", + "data.customModelField", + "data.published", + "data.Status", + "data.status", + ]), + ); +}); + it("resync re-links only the source's own rows, never another collection's (self-heal)", async () => { builderReadMock.mode = "full"; builderReadMock.calls = []; @@ -937,9 +1637,13 @@ it("full Builder refresh reads every page in one resync call", async () => { expect( rows.map((row: { sourceRowId: string }) => row.sourceRowId).sort(), ).toEqual(["entry-a1", "entry-a2"]); - expect(builderReadMock.calls).toEqual([ - { model: "collection-a", maxPages: undefined, offset: 0 }, - ]); + expect( + builderReadMock.calls.map(({ model, maxPages, offset }) => ({ + model, + maxPages, + offset, + })), + ).toEqual([{ model: "collection-a", maxPages: undefined, offset: 0 }]); }); it("finishes a Builder continuation when optional model field metadata fails", async () => { @@ -1027,9 +1731,13 @@ it("finishes a Builder continuation when optional model field metadata fails", a expect(metadata.lastReadPartial).toBe(false); expect(metadata.sourceFetchState).toBe("idle"); expect(metadata.activeReadSourceRowIds).toBeUndefined(); - expect(builderReadMock.calls).toEqual([ - { model: "collection-a", maxPages: 1, offset: 1 }, - ]); + expect( + builderReadMock.calls.map(({ model, maxPages, offset }) => ({ + model, + maxPages, + offset, + })), + ).toEqual([{ model: "collection-a", maxPages: 1, offset: 1 }]); const preservedFields = await db .select({ id: schema.contentDatabaseSourceFields.id, diff --git a/templates/content/actions/suggest-source-join-key.ts b/templates/content/actions/suggest-source-join-key.ts index b7eb1beb92..c5f53b25ee 100644 --- a/templates/content/actions/suggest-source-join-key.ts +++ b/templates/content/actions/suggest-source-join-key.ts @@ -6,7 +6,10 @@ import type { DocumentPropertyValue, SuggestSourceJoinKeyResponse, } from "../shared/api.js"; -import { readBuilderCmsContentEntries } from "./_builder-cms-read-client.js"; +import { + readBuilderCmsContentEntries, + readBuilderCmsModelFields, +} from "./_builder-cms-read-client.js"; import { getContentDatabaseSourceSnapshot, resolveDatabaseForSourceMutation, @@ -52,8 +55,12 @@ export default defineAction({ let secondaryValues: Record[]; if (args.candidateSourceType === "builder-cms") { + const modelFields = await readBuilderCmsModelFields({ + model: args.candidateSourceTable, + }).catch(() => []); const read = await readBuilderCmsContentEntries({ model: args.candidateSourceTable, + fieldPaths: modelFields.map((field) => `data.${field.name}`), }); secondaryValues = (read.state === "live" ? read.entries : []) .map((entry) => entry.sourceValues) diff --git a/templates/content/changelog/2026-07-09-builder-source-refreshes-now-keep-mapped-fields-such-as-topi.md b/templates/content/changelog/2026-07-09-builder-source-refreshes-now-keep-mapped-fields-such-as-topi.md new file mode 100644 index 0000000000..42b99168ce --- /dev/null +++ b/templates/content/changelog/2026-07-09-builder-source-refreshes-now-keep-mapped-fields-such-as-topi.md @@ -0,0 +1,6 @@ +--- +type: fixed +date: 2026-07-09 +--- + +Builder source refreshes now keep mapped fields such as topics and tags, preserve existing rows when a read unexpectedly returns empty, and load large databases more reliably. diff --git a/templates/content/shared/api.ts b/templates/content/shared/api.ts index 080b5db55e..a98d70bea6 100644 --- a/templates/content/shared/api.ts +++ b/templates/content/shared/api.ts @@ -618,6 +618,7 @@ export interface ContentDatabaseSource { lastReadPartial?: boolean; lastReadHasMore?: boolean; lastReadNextOffset?: number; + lastReadSuspiciousEmpty?: boolean; sourceFetchState?: "idle" | "fetching" | "error"; allowDraftWrites?: boolean; allowPublishWrites?: boolean;