From 6168cff79abc8c889c5aee8a2a19b8529e7f8387 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 13:57:18 -0700 Subject: [PATCH 01/13] fix(icons): fix Meta icon clipping in model dropdown (#5542) MetaIcon's viewBox (265x165) was smaller than the true bounding box of its three path elements (287.56x191, computed from the actual path geometry), so the bottom-right of the mark was silently clipped by the SVG viewport. It was also non-square, so the dropdown row's forced 14x14 icon slot letterboxed it asymmetrically instead of filling it like every other provider icon. Fixed the viewBox to the exact computed bounding box, padded to a square and vertically centered (0 -48.28 287.56 287.56) - matches the sibling icon convention (GeminiIcon, etc.) of a square viewBox filling the icon slot edge to edge. Verified by rasterizing both the old and new viewBox at the actual deployed 14px size. --- apps/sim/components/icons.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/sim/components/icons.tsx b/apps/sim/components/icons.tsx index c2b6ed56e2e..09d00b0e7c7 100644 --- a/apps/sim/components/icons.tsx +++ b/apps/sim/components/icons.tsx @@ -3696,7 +3696,7 @@ export function MetaIcon(props: SVGProps) { From ff5353cd8376306d86e3df944c462e178f1b4f21 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 14:54:13 -0700 Subject: [PATCH 02/13] fix(rich-markdown-editor): highlight text-shift + hover block drag handle (#5543) * fix(rich-markdown-editor): keep highlight from shifting text The highlight added horizontal padding, which pushed the highlighted text (and the text after it) to the right when the mark was applied. Cancel the padding with an equal negative margin so the amber tint still bleeds slightly past the text but the text never moves as a highlight is applied or removed. * feat(rich-markdown-editor): hover block drag handle (+ / grip) Adds a left-margin block handle revealed on hover (only when the editor is editable): a + that inserts a paragraph below the hovered block and opens the slash menu, and a grip that drags to reorder blocks (via @tiptap/extension-drag-handle) or, on a plain click, selects the block. The keyboard equivalent of the reorder is Mod-Shift-Arrow (block-mover). The control buttons reset their own chrome (no default border/background/padding) so they render consistently regardless of the surrounding reset. --- .../menus/drag-handle.test.ts | 63 ++++++++++++++ .../menus/drag-handle.tsx | 85 +++++++++++++++++++ .../rich-markdown-editor.css | 50 ++++++++++- .../rich-markdown-editor.tsx | 2 + apps/sim/package.json | 1 + bun.lock | 19 +++++ 6 files changed, 219 insertions(+), 1 deletion(-) create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts new file mode 100644 index 00000000000..d836ef588fd --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts @@ -0,0 +1,63 @@ +/** + * @vitest-environment jsdom + */ +import { Editor } from '@tiptap/core' +import { NodeSelection } from '@tiptap/pm/state' +import { afterEach, describe, expect, it } from 'vitest' +import { createMarkdownContentExtensions } from '../extensions' +import { insertBlockBelow, selectBlockAt } from './drag-handle' + +let editor: Editor | null = null + +afterEach(() => { + editor?.destroy() + editor = null +}) + +function mount(markdown: string): Editor { + const created = new Editor({ extensions: createMarkdownContentExtensions() }) + created.commands.setContent(markdown, { contentType: 'markdown' }) + return created +} + +/** Position before the first top-level block whose text contains `word`, or -1. */ +function blockPos(target: Editor, word: string): number { + let pos = -1 + target.state.doc.forEach((node, offset) => { + if (pos < 0 && node.textContent.includes(word)) pos = offset + }) + return pos +} + +describe('drag-handle block operations', () => { + it('inserts a paragraph after the hovered block and opens the slash menu', () => { + editor = mount('# One\n\nTwo para') + insertBlockBelow(editor, blockPos(editor, 'Two para')) + const md = editor.getMarkdown().trim() + expect(md).toContain('Two para') + expect(md.split('Two para')[1]).toContain('/') + }) + + it('inserts a sibling after a whole list, not a nested list item', () => { + editor = mount('- a\n- b') + insertBlockBelow(editor, blockPos(editor, 'a')) + expect(editor.getJSON().content?.[0]?.type).toBe('bulletList') + expect(editor.getJSON().content?.some((node) => node.type === 'paragraph')).toBe(true) + }) + + it('selects the block as a NodeSelection', () => { + editor = mount('# One\n\nTwo para') + selectBlockAt(editor, blockPos(editor, 'Two para')) + const { selection } = editor.state + expect(selection instanceof NodeSelection).toBe(true) + expect((selection as NodeSelection).node.textContent).toBe('Two para') + }) + + it('is a no-op at an unresolved position', () => { + editor = mount('# One') + const before = editor.getMarkdown() + insertBlockBelow(editor, -1) + selectBlockAt(editor, -1) + expect(editor.getMarkdown()).toBe(before) + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx new file mode 100644 index 00000000000..3bceaca653a --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx @@ -0,0 +1,85 @@ +'use client' + +import { useCallback, useRef } from 'react' +import DragHandle from '@tiptap/extension-drag-handle-react' +import type { Editor } from '@tiptap/react' +import { GripVertical, Plus } from 'lucide-react' + +interface BlockDragHandleProps { + editor: Editor +} + +interface NodeChangeData { + pos: number +} + +/** + * Inserts an empty paragraph immediately after the top-level block at `pos` and opens the slash menu + * there, so the `+` control adds a new block below the hovered one. A no-op if `pos` doesn't resolve to + * a node (e.g. the handle hasn't hovered a block yet). + */ +export function insertBlockBelow(editor: Editor, pos: number): void { + const node = pos >= 0 ? editor.state.doc.nodeAt(pos) : null + if (!node) return + const insertAt = pos + node.nodeSize + editor + .chain() + .focus() + .insertContentAt(insertAt, { type: 'paragraph' }) + .setTextSelection(insertAt + 1) + .insertContent('/') + .run() +} + +/** Selects the top-level block at `pos` as a NodeSelection (the grip's click affordance). */ +export function selectBlockAt(editor: Editor, pos: number): void { + if (pos < 0) return + editor.chain().setNodeSelection(pos).run() + editor.view.focus() +} + +/** + * Left-margin block controls revealed on block hover: a `+` that inserts a paragraph below the hovered + * block and opens the slash menu ({@link insertBlockBelow}), and a `⠿` grip that drags to reorder (via + * `@tiptap/extension-drag-handle`) or, on a plain click, selects the block ({@link selectBlockAt}). The + * keyboard equivalent of the reorder is `Mod-Shift-Arrow` (see the block-mover extension). + */ +export function BlockDragHandle({ editor }: BlockDragHandleProps) { + const hoveredPosRef = useRef(-1) + + const handleNodeChange = useCallback((data: NodeChangeData) => { + hoveredPosRef.current = data.pos + }, []) + + const insertBelow = useCallback(() => { + insertBlockBelow(editor, hoveredPosRef.current) + }, [editor]) + + const selectBlock = useCallback(() => { + selectBlockAt(editor, hoveredPosRef.current) + }, [editor]) + + return ( + +
+ + +
+ + ) +} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css index a5d5a82eee4..ec8315ff105 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css @@ -400,17 +400,65 @@ /* * Highlight mark (`==text==`). An opacity-based amber tint so it reads on both light and dark * surfaces without a theme override; `color: inherit` keeps the text at the surrounding body color - * and `box-decoration-break: clone` keeps the tint clean where a highlight wraps across lines. + * and `box-decoration-break: clone` keeps the tint clean where a highlight wraps across lines. The + * horizontal padding is cancelled by an equal negative margin so the tint bleeds slightly past the + * text without ever shifting the text (or any following text) as the highlight is applied/removed. */ .rich-markdown-prose mark { background-color: rgba(255, 212, 0, 0.4); color: inherit; border-radius: 2px; padding: 0 0.1em; + margin: 0 -0.1em; box-decoration-break: clone; -webkit-box-decoration-break: clone; } +/* + * Left-margin block controls (the `+` / `⠿` handle revealed on block hover). `@tiptap/extension-drag- + * handle` positions the wrapper; these rules style the two buttons — a subtle icon pair that darkens on + * hover, with the grip showing a grab cursor and the add button a pointer. + */ +.rich-md-block-controls { + display: flex; + align-items: center; + gap: 1px; + padding-right: 4px; +} + +.rich-md-block-btn { + display: flex; + align-items: center; + justify-content: center; + width: 18px; + height: 22px; + padding: 0; + border: none; + border-radius: 4px; + background: transparent; + appearance: none; + -webkit-appearance: none; + color: var(--text-subtle); + cursor: pointer; + transition: + background-color 0.12s ease, + color 0.12s ease; +} + +.rich-md-block-btn:hover, +.rich-md-block-btn:focus-visible { + background-color: var(--surface-active); + color: var(--text-icon); +} + +.rich-md-block-grip { + cursor: grab; +} + +.rich-md-block-grip:active { + cursor: grabbing; +} + /* * Field variant (modal embed): match the surrounding chip fields' typography exactly — * body at the chip `text-sm` (14px) scale and the placeholder at `--text-muted` (not the diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index d6fc15224a1..00e58de9889 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -23,6 +23,7 @@ import { import { parseMarkdownToDoc } from './markdown-parse' import { useEditorMentions } from './mention' import { EditorBubbleMenu } from './menus/bubble-menu' +import { BlockDragHandle } from './menus/drag-handle' import { LinkHoverCard } from './menus/link-hover-card' import { TableBubbleMenu } from './menus/table-menu' import { normalizeMarkdownContent } from './normalize-content' @@ -446,6 +447,7 @@ export function LoadedRichMarkdownEditor({ {editor && } {editor && } {editor && } + {editor && isEditable && } Date: Thu, 9 Jul 2026 14:54:46 -0700 Subject: [PATCH 03/13] fix(knowledge): fix chunk_index race and storage-quota check/increment race (#5544) * fix(knowledge): fix chunk_index race and storage-quota check/increment race - createChunk now serializes concurrent writes to the same document via a transactional advisory lock before computing the next chunk_index, fixing the unique-constraint collision behind the ITSM 'Failed to create chunk' incident (two overlapping workflow runs appending to the same shared KB document). - Document uploads now check and increment storage quota atomically in a single conditional UPDATE inside the insert transaction, instead of check-then-insert-then-increment-after-commit, closing the window where two concurrent uploads could both pass the quota check and over-commit storage. * fix(knowledge): bound the chunk advisory-lock wait with lock_timeout Address Greptile P1: the advisory lock in createChunk could wait indefinitely on a stalled same-document transaction while holding a pooled connection. Set a 5s lock_timeout before acquiring it, matching the set_config + pg_advisory_xact_lock pattern already used by every other advisory lock in this codebase (org membership, table schema/row-ordering, BYOK keys, workspace env, usage-log flush, execution-log reconciliation). --- apps/sim/lib/billing/storage/index.ts | 2 + apps/sim/lib/billing/storage/tracking.ts | 98 ++++++++++++++++++++- apps/sim/lib/knowledge/chunks/service.ts | 29 +++++- apps/sim/lib/knowledge/documents/service.ts | 90 +++++++++++++++---- 4 files changed, 195 insertions(+), 24 deletions(-) diff --git a/apps/sim/lib/billing/storage/index.ts b/apps/sim/lib/billing/storage/index.ts index 9203c24b455..8820af428ba 100644 --- a/apps/sim/lib/billing/storage/index.ts +++ b/apps/sim/lib/billing/storage/index.ts @@ -1,6 +1,8 @@ export { checkStorageQuota, getUserStorageLimit, getUserStorageUsage } from './limits' export { + checkAndIncrementStorageUsageInTx, decrementStorageUsage, decrementStorageUsageInTx, incrementStorageUsage, + maybeNotifyStorageLimit, } from './tracking' diff --git a/apps/sim/lib/billing/storage/tracking.ts b/apps/sim/lib/billing/storage/tracking.ts index 6311a881846..3c31bb8eb3b 100644 --- a/apps/sim/lib/billing/storage/tracking.ts +++ b/apps/sim/lib/billing/storage/tracking.ts @@ -7,11 +7,12 @@ import { db } from '@sim/db' import { organization, userStats } from '@sim/db/schema' import { createLogger } from '@sim/logger' -import { eq, sql } from 'drizzle-orm' +import { generateId } from '@sim/utils/id' +import { and, eq, sql } from 'drizzle-orm' import { maybeNotifyLimit } from '@/lib/billing/core/limit-notifications' import type { HighestPrioritySubscription } from '@/lib/billing/core/plan' import { getUserStorageLimit, getUserStorageUsage } from '@/lib/billing/storage/limits' -import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils' +import { getFreeTierLimit, isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils' import { isBillingEnabled } from '@/lib/core/config/env-flags' const logger = createLogger('StorageTracking') @@ -33,7 +34,7 @@ function formatGb(bytes: number, decimals: number): string { * @param rearmOnly - True on decrements, so a shrink that leaves usage above a * threshold re-arms but never sends (a drop is not a fresh crossing). */ -async function maybeNotifyStorageLimit( +export async function maybeNotifyStorageLimit( userId: string, workspaceId: string, sub: HighestPrioritySubscription | null, @@ -195,3 +196,94 @@ export async function decrementStorageUsageInTx( .where(eq(userStats.userId, userId)) } } + +/** + * Atomically check quota and increment a user's (or their org's) storage + * counter inside an existing transaction, using a pre-resolved subscription. + * The check and the increment are a single conditional `UPDATE`, so two + * concurrent callers can no longer both read the same pre-increment usage, + * both pass the check, and both commit past the limit — the second caller's + * `UPDATE` re-evaluates the WHERE clause against the first caller's already + * -committed-within-the-same-DB-round-trip row and correctly fails. Replaces + * the old read-then-decide-then-increment-after-commit split (`checkStorageQuota` + * + a fire-and-forget `incrementStorageUsage` after the transaction), which left + * a window between the read and the increment. + * + * On success, callers should best-effort call {@link maybeNotifyStorageLimit} + * after the transaction commits (mirrors the existing post-increment threshold + * check) — this helper doesn't do it itself since it runs mid-transaction. + * + * For a personal (non-org-scoped) `userId`, this first upserts the + * `userStats` row on `tx` — a documented possibility for OAuth account + * linking (see `ensureUserStatsExists` in `lib/billing/core/usage.ts`, whose + * insert values this mirrors) — because the conditional `UPDATE` below + * matches 0 rows, and therefore reads as "quota exceeded", if that row + * doesn't exist yet. `ensureUserStatsExists` itself isn't reused here since + * it writes through the standalone `db` client, which would open a second + * pooled connection while this transaction's is held. + */ +export async function checkAndIncrementStorageUsageInTx( + tx: StorageTransaction, + sub: HighestPrioritySubscription | null, + userId: string, + bytes: number +): Promise<{ allowed: boolean; currentUsage: number; limit: number; error?: string }> { + if (!isBillingEnabled) { + return { allowed: true, currentUsage: 0, limit: Number.MAX_SAFE_INTEGER } + } + + const limit = await getUserStorageLimit(userId, sub) + + if (bytes <= 0) { + return { allowed: true, currentUsage: await getUserStorageUsage(userId, sub), limit } + } + + const orgScoped = isOrgScopedSubscription(sub, userId) && sub + + if (!orgScoped) { + await tx + .insert(userStats) + .values({ + id: generateId(), + userId, + currentUsageLimit: getFreeTierLimit().toString(), + usageLimitUpdatedAt: new Date(), + }) + .onConflictDoNothing({ target: userStats.userId }) + } + + const [updated] = orgScoped + ? await tx + .update(organization) + .set({ storageUsedBytes: sql`${organization.storageUsedBytes} + ${bytes}` }) + .where( + and( + eq(organization.id, sub.referenceId), + sql`${organization.storageUsedBytes} + ${bytes} <= ${limit}` + ) + ) + .returning({ storageUsedBytes: organization.storageUsedBytes }) + : await tx + .update(userStats) + .set({ storageUsedBytes: sql`${userStats.storageUsedBytes} + ${bytes}` }) + .where( + and( + eq(userStats.userId, userId), + sql`${userStats.storageUsedBytes} + ${bytes} <= ${limit}` + ) + ) + .returning({ storageUsedBytes: userStats.storageUsedBytes }) + + if (updated) { + return { allowed: true, currentUsage: updated.storageUsedBytes - bytes, limit } + } + + const currentUsage = await getUserStorageUsage(userId, sub) + const newUsage = currentUsage + bytes + return { + allowed: false, + currentUsage, + limit, + error: `Storage limit exceeded. Used: ${(newUsage / (1024 * 1024 * 1024)).toFixed(2)}GB, Limit: ${(limit / (1024 * 1024 * 1024)).toFixed(0)}GB`, + } +} diff --git a/apps/sim/lib/knowledge/chunks/service.ts b/apps/sim/lib/knowledge/chunks/service.ts index fe34a40b543..6660027d470 100644 --- a/apps/sim/lib/knowledge/chunks/service.ts +++ b/apps/sim/lib/knowledge/chunks/service.ts @@ -17,6 +17,8 @@ import { estimateTokenCount } from '@/lib/tokenization/estimators' const logger = createLogger('ChunksService') +const KB_CHUNK_LOCK_TIMEOUT_MS = 5_000 + /** * Query chunks for a document with filtering and pagination */ @@ -101,7 +103,23 @@ export async function queryChunks( } /** - * Create a new chunk for a document + * Create a new chunk for a document. + * + * Assigns `chunkIndex` as `max(chunkIndex) + 1` under a transactional + * `pg_advisory_xact_lock` keyed on the document, so concurrent calls for the + * same document serialize instead of computing the same index and colliding + * on the `(document_id, chunk_index)` unique constraint. A `SELECT ... FOR + * UPDATE` on the current max row doesn't prevent that collision, since the + * row it would lock is unrelated to the not-yet-inserted next row. A row + * lock on `document` instead of an advisory lock would also work, but would + * invert the embedding-before-document lock order every other chunk + * mutation path uses (see lock-order.test.ts) — the advisory lock is a + * separate namespace, so it can't deadlock against that convention. + * + * `pg_advisory_xact_lock` auto-releases at transaction end, so there's no + * session lock to leak onto a pooled connection, and `lock_timeout` bounds + * the wait (it raises SQLSTATE 55P03 instead of hanging a pooled connection) + * if a same-document holder is stuck. */ export async function createChunk( knowledgeBaseId: string, @@ -135,8 +153,14 @@ export async function createChunk( const chunkId = generateId() const now = new Date() - // Use transaction to atomically get next index and insert chunk const newChunk = await db.transaction(async (tx) => { + await tx.execute( + sql`select set_config('lock_timeout', ${`${KB_CHUNK_LOCK_TIMEOUT_MS}ms`}, true)` + ) + await tx.execute( + sql`select pg_advisory_xact_lock(hashtextextended(${`kb_chunk_seq:${documentId}`}, 0))` + ) + const activeDocument = await tx .select({ id: document.id }) .from(document) @@ -156,7 +180,6 @@ export async function createChunk( throw new Error('Document not found') } - // Get the next chunk index atomically within the transaction const lastChunk = await tx .select({ chunkIndex: embedding.chunkIndex }) .from(embedding) diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 56300b1d837..88b33ff7a62 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -18,9 +18,9 @@ import type { HighestPrioritySubscription } from '@/lib/billing/core/plan' import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' import { recordUsage } from '@/lib/billing/core/usage-log' import { - checkStorageQuota, + checkAndIncrementStorageUsageInTx, decrementStorageUsageInTx, - incrementStorageUsage, + maybeNotifyStorageLimit, } from '@/lib/billing/storage' import { checkAndBillOverageThreshold } from '@/lib/billing/threshold-billing' import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' @@ -864,6 +864,33 @@ export function isTriggerAvailable(): boolean { return Boolean(env.TRIGGER_SECRET_KEY) && isTriggerDevEnabled } +/** + * Resolves the subscription to bill for a storage-metered upload, ahead of + * the FOR UPDATE transaction that will insert the document row. + * + * Must run before that transaction opens: `knowledgeBase.userId` is + * immutable once a KB is created, so resolving it here can't go stale + * relative to the transaction's own read of it, and resolving it inside the + * transaction would open a second pooled-database-connection checkout while + * the first is held. + */ +async function resolveQuotaSubscription( + knowledgeBaseId: string, + uploadedBy: string | null +): Promise { + const billedUserId = + uploadedBy ?? + ( + await db + .select({ userId: knowledgeBase.userId }) + .from(knowledgeBase) + .where(eq(knowledgeBase.id, knowledgeBaseId)) + .limit(1) + )[0]?.userId + + return billedUserId ? getHighestPrioritySubscription(billedUserId) : null +} + export async function createDocumentRecords( documents: Array<{ filename: string @@ -883,7 +910,15 @@ export async function createDocumentRecords( requestId: string, uploadedBy: string | null = null ): Promise { - let storageBilling: { userId: string; workspaceId: string | null; bytes: number } | null = null + let storageBilling: { + userId: string + workspaceId: string | null + bytes: number + sub: HighestPrioritySubscription | null + } | null = null + + const totalBytes = documents.reduce((sum, docData) => sum + (docData.fileSize || 0), 0) + const sub = totalBytes > 0 ? await resolveQuotaSubscription(knowledgeBaseId, uploadedBy) : null const returnData = await db.transaction(async (tx) => { await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`) @@ -912,13 +947,12 @@ export async function createDocumentRecords( ) const billedUserId = uploadedBy ?? kb[0].userId - const totalBytes = documents.reduce((sum, docData) => sum + (docData.fileSize || 0), 0) if (totalBytes > 0) { - const quotaCheck = await checkStorageQuota(billedUserId, totalBytes) + const quotaCheck = await checkAndIncrementStorageUsageInTx(tx, sub, billedUserId, totalBytes) if (!quotaCheck.allowed) { throw new Error(quotaCheck.error || 'Storage limit exceeded') } - storageBilling = { userId: billedUserId, workspaceId: kbWorkspaceId, bytes: totalBytes } + storageBilling = { userId: billedUserId, workspaceId: kbWorkspaceId, bytes: totalBytes, sub } } // One load per batch (was N+1); skip entirely if no doc carries tags. @@ -1011,11 +1045,14 @@ export async function createDocumentRecords( }) if (storageBilling) { - const billing: { userId: string; workspaceId: string | null; bytes: number } = storageBilling - try { - await incrementStorageUsage(billing.userId, billing.bytes, billing.workspaceId ?? undefined) - } catch (storageError) { - logger.error(`[${requestId}] Failed to update storage tracking:`, storageError) + const billing: { + userId: string + workspaceId: string | null + bytes: number + sub: HighestPrioritySubscription | null + } = storageBilling + if (billing.workspaceId) { + void maybeNotifyStorageLimit(billing.userId, billing.workspaceId, billing.sub) } } @@ -1338,7 +1375,15 @@ export async function createSingleDocument( ...processedTags, } - let storageBilling: { userId: string; workspaceId: string | null; bytes: number } | null = null + let storageBilling: { + userId: string + workspaceId: string | null + bytes: number + sub: HighestPrioritySubscription | null + } | null = null + + const sub = + documentData.fileSize > 0 ? await resolveQuotaSubscription(knowledgeBaseId, uploadedBy) : null await db.transaction(async (tx) => { await tx.execute(sql`SELECT 1 FROM knowledge_base WHERE id = ${knowledgeBaseId} FOR UPDATE`) @@ -1367,7 +1412,12 @@ export async function createSingleDocument( const billedUserId = uploadedBy ?? kb[0].userId if (documentData.fileSize > 0) { - const quotaCheck = await checkStorageQuota(billedUserId, documentData.fileSize) + const quotaCheck = await checkAndIncrementStorageUsageInTx( + tx, + sub, + billedUserId, + documentData.fileSize + ) if (!quotaCheck.allowed) { throw new Error(quotaCheck.error || 'Storage limit exceeded') } @@ -1375,6 +1425,7 @@ export async function createSingleDocument( userId: billedUserId, workspaceId: kb[0].workspaceId, bytes: documentData.fileSize, + sub, } } @@ -1387,11 +1438,14 @@ export async function createSingleDocument( }) if (storageBilling) { - const billing: { userId: string; workspaceId: string | null; bytes: number } = storageBilling - try { - await incrementStorageUsage(billing.userId, billing.bytes, billing.workspaceId ?? undefined) - } catch (storageError) { - logger.error(`[${requestId}] Failed to update storage tracking:`, storageError) + const billing: { + userId: string + workspaceId: string | null + bytes: number + sub: HighestPrioritySubscription | null + } = storageBilling + if (billing.workspaceId) { + void maybeNotifyStorageLimit(billing.userId, billing.workspaceId, billing.sub) } } From aa5b1d5569ddab6ad49e32ea4c80573f896ac91b Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 15:55:58 -0700 Subject: [PATCH 04/13] fix(suggested-actions): swap unaudited filled icons for EMCN outline set (#5548) * fix(suggested-actions): swap unaudited filled icons for EMCN outline set Suggested-action template icons on the home page mixed filled/solid icons in with the app's outline icon convention. Swapped them for consistent outline icons and removed the unused filled Card icon. - gmail.ts: Card -> ClipboardList - clickhouse.ts, sftp.ts: Trash -> TrashOutline - ssh.ts: TerminalWindow (emcn) -> SshTerminalIcon (moved to components/icons.tsx alongside the other block/brand icons) - deleted unused packages/emcn/src/icons/card.tsx - regenerated docs * fix(docs): revert jira.mdx regen regression generate-docs.ts is dropping the Configuration and generic-webhook Output tables for Jira triggers even though the trigger schemas still define those fields (caught by Greptile review). Unrelated to the icon changes in this PR, so reverting jira.mdx to its prior content rather than debugging the generator here. * fix(build): remove last Card icon consumer in playground gallery apps/sim/app/playground/page.tsx imported Card from the top-level @sim/emcn barrel for the icon showcase grid, which I missed when auditing @sim/emcn/icons consumers. Broke the production build after card.tsx was deleted. Removed the import and its gallery entry. Verified with a local `bun run build`. --- apps/docs/components/icons.tsx | 77 +++++++++++++++++++ .../content/docs/en/integrations/ashby.mdx | 9 ++- .../content/docs/en/integrations/gitlab.mdx | 4 +- .../content/docs/en/integrations/gmail.mdx | 2 +- apps/sim/app/playground/page.tsx | 2 - apps/sim/blocks/blocks/clickhouse.ts | 13 +++- apps/sim/blocks/blocks/gmail.ts | 4 +- apps/sim/blocks/blocks/sftp.ts | 12 ++- apps/sim/blocks/blocks/ssh.ts | 14 +--- apps/sim/components/icons.tsx | 21 +++++ apps/sim/lib/integrations/integrations.json | 7 +- packages/emcn/src/icons/card.tsx | 26 ------- packages/emcn/src/icons/index.ts | 1 - 13 files changed, 140 insertions(+), 52 deletions(-) delete mode 100644 packages/emcn/src/icons/card.tsx diff --git a/apps/docs/components/icons.tsx b/apps/docs/components/icons.tsx index c213134b0e1..fe869a5b77e 100644 --- a/apps/docs/components/icons.tsx +++ b/apps/docs/components/icons.tsx @@ -3687,6 +3687,62 @@ export const SakanaIcon = (props: SVGProps) => ( ) +export function MetaIcon(props: SVGProps) { + const id = useId() + const gradient1Id = `meta_gradient_1_${id}` + const gradient2Id = `meta_gradient_2_${id}` + + return ( + + Meta + + + + + + + + + + + + + + + + + ) +} + export function GeminiIcon(props: SVGProps) { const id = useId() const gradientId = `gemini_gradient_${id}` @@ -5572,6 +5628,27 @@ export function SshIcon(props: SVGProps) { ) } +export function SshTerminalIcon(props: SVGProps) { + return ( + + + + + + + ) +} + export function SftpIcon(props: SVGProps) { return ( ) { ) } +export function SshTerminalIcon(props: SVGProps) { + return ( + + + + + + + ) +} + export function SftpIcon(props: SVGProps) { return ( ) { - return ( - - ) -} diff --git a/packages/emcn/src/icons/index.ts b/packages/emcn/src/icons/index.ts index 1af0459200b..6cf6d1bbf20 100644 --- a/packages/emcn/src/icons/index.ts +++ b/packages/emcn/src/icons/index.ts @@ -12,7 +12,6 @@ export { BubbleChatDelay } from './bubble-chat-delay' export { BubbleChatPreview } from './bubble-chat-preview' export { Bug } from './bug' export { Calendar } from './calendar' -export { Card } from './card' export { Check } from './check' export { ChevronDown } from './chevron-down' export { CircleAlert } from './circle-alert' From bbde749dcb6e2752b67d556bed9d2b90b2e71005 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 16:09:04 -0700 Subject: [PATCH 05/13] fix(rich-markdown-editor): remove the hover block drag handle and + button (#5550) The hover handle (drag-to-reorder grip + insert button) added surface area and edge cases for marginal value in a file editor. Block reordering is covered by the keyboard shortcut (Mod-Shift-Arrow) and block insertion by the slash menu, so the handle and + are redundant. Removes the component, its styles, the editor wiring, and the now-unused @tiptap/extension-drag-handle-react dependency. The highlight text-shift fix from the same feature branch is unaffected. --- .../menus/drag-handle.test.ts | 63 -------------- .../menus/drag-handle.tsx | 85 ------------------- .../rich-markdown-editor.css | 45 ---------- .../rich-markdown-editor.tsx | 2 - apps/sim/package.json | 1 - bun.lock | 19 ----- 6 files changed, 215 deletions(-) delete mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts delete mode 100644 apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts deleted file mode 100644 index d836ef588fd..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { Editor } from '@tiptap/core' -import { NodeSelection } from '@tiptap/pm/state' -import { afterEach, describe, expect, it } from 'vitest' -import { createMarkdownContentExtensions } from '../extensions' -import { insertBlockBelow, selectBlockAt } from './drag-handle' - -let editor: Editor | null = null - -afterEach(() => { - editor?.destroy() - editor = null -}) - -function mount(markdown: string): Editor { - const created = new Editor({ extensions: createMarkdownContentExtensions() }) - created.commands.setContent(markdown, { contentType: 'markdown' }) - return created -} - -/** Position before the first top-level block whose text contains `word`, or -1. */ -function blockPos(target: Editor, word: string): number { - let pos = -1 - target.state.doc.forEach((node, offset) => { - if (pos < 0 && node.textContent.includes(word)) pos = offset - }) - return pos -} - -describe('drag-handle block operations', () => { - it('inserts a paragraph after the hovered block and opens the slash menu', () => { - editor = mount('# One\n\nTwo para') - insertBlockBelow(editor, blockPos(editor, 'Two para')) - const md = editor.getMarkdown().trim() - expect(md).toContain('Two para') - expect(md.split('Two para')[1]).toContain('/') - }) - - it('inserts a sibling after a whole list, not a nested list item', () => { - editor = mount('- a\n- b') - insertBlockBelow(editor, blockPos(editor, 'a')) - expect(editor.getJSON().content?.[0]?.type).toBe('bulletList') - expect(editor.getJSON().content?.some((node) => node.type === 'paragraph')).toBe(true) - }) - - it('selects the block as a NodeSelection', () => { - editor = mount('# One\n\nTwo para') - selectBlockAt(editor, blockPos(editor, 'Two para')) - const { selection } = editor.state - expect(selection instanceof NodeSelection).toBe(true) - expect((selection as NodeSelection).node.textContent).toBe('Two para') - }) - - it('is a no-op at an unresolved position', () => { - editor = mount('# One') - const before = editor.getMarkdown() - insertBlockBelow(editor, -1) - selectBlockAt(editor, -1) - expect(editor.getMarkdown()).toBe(before) - }) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx deleted file mode 100644 index 3bceaca653a..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/menus/drag-handle.tsx +++ /dev/null @@ -1,85 +0,0 @@ -'use client' - -import { useCallback, useRef } from 'react' -import DragHandle from '@tiptap/extension-drag-handle-react' -import type { Editor } from '@tiptap/react' -import { GripVertical, Plus } from 'lucide-react' - -interface BlockDragHandleProps { - editor: Editor -} - -interface NodeChangeData { - pos: number -} - -/** - * Inserts an empty paragraph immediately after the top-level block at `pos` and opens the slash menu - * there, so the `+` control adds a new block below the hovered one. A no-op if `pos` doesn't resolve to - * a node (e.g. the handle hasn't hovered a block yet). - */ -export function insertBlockBelow(editor: Editor, pos: number): void { - const node = pos >= 0 ? editor.state.doc.nodeAt(pos) : null - if (!node) return - const insertAt = pos + node.nodeSize - editor - .chain() - .focus() - .insertContentAt(insertAt, { type: 'paragraph' }) - .setTextSelection(insertAt + 1) - .insertContent('/') - .run() -} - -/** Selects the top-level block at `pos` as a NodeSelection (the grip's click affordance). */ -export function selectBlockAt(editor: Editor, pos: number): void { - if (pos < 0) return - editor.chain().setNodeSelection(pos).run() - editor.view.focus() -} - -/** - * Left-margin block controls revealed on block hover: a `+` that inserts a paragraph below the hovered - * block and opens the slash menu ({@link insertBlockBelow}), and a `⠿` grip that drags to reorder (via - * `@tiptap/extension-drag-handle`) or, on a plain click, selects the block ({@link selectBlockAt}). The - * keyboard equivalent of the reorder is `Mod-Shift-Arrow` (see the block-mover extension). - */ -export function BlockDragHandle({ editor }: BlockDragHandleProps) { - const hoveredPosRef = useRef(-1) - - const handleNodeChange = useCallback((data: NodeChangeData) => { - hoveredPosRef.current = data.pos - }, []) - - const insertBelow = useCallback(() => { - insertBlockBelow(editor, hoveredPosRef.current) - }, [editor]) - - const selectBlock = useCallback(() => { - selectBlockAt(editor, hoveredPosRef.current) - }, [editor]) - - return ( - -
- - -
- - ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css index ec8315ff105..c8b56385b56 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.css @@ -414,51 +414,6 @@ -webkit-box-decoration-break: clone; } -/* - * Left-margin block controls (the `+` / `⠿` handle revealed on block hover). `@tiptap/extension-drag- - * handle` positions the wrapper; these rules style the two buttons — a subtle icon pair that darkens on - * hover, with the grip showing a grab cursor and the add button a pointer. - */ -.rich-md-block-controls { - display: flex; - align-items: center; - gap: 1px; - padding-right: 4px; -} - -.rich-md-block-btn { - display: flex; - align-items: center; - justify-content: center; - width: 18px; - height: 22px; - padding: 0; - border: none; - border-radius: 4px; - background: transparent; - appearance: none; - -webkit-appearance: none; - color: var(--text-subtle); - cursor: pointer; - transition: - background-color 0.12s ease, - color 0.12s ease; -} - -.rich-md-block-btn:hover, -.rich-md-block-btn:focus-visible { - background-color: var(--surface-active); - color: var(--text-icon); -} - -.rich-md-block-grip { - cursor: grab; -} - -.rich-md-block-grip:active { - cursor: grabbing; -} - /* * Field variant (modal embed): match the surrounding chip fields' typography exactly — * body at the chip `text-sm` (14px) scale and the placeholder at `--text-muted` (not the diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index 00e58de9889..d6fc15224a1 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -23,7 +23,6 @@ import { import { parseMarkdownToDoc } from './markdown-parse' import { useEditorMentions } from './mention' import { EditorBubbleMenu } from './menus/bubble-menu' -import { BlockDragHandle } from './menus/drag-handle' import { LinkHoverCard } from './menus/link-hover-card' import { TableBubbleMenu } from './menus/table-menu' import { normalizeMarkdownContent } from './normalize-content' @@ -447,7 +446,6 @@ export function LoadedRichMarkdownEditor({ {editor && } {editor && } {editor && } - {editor && isEditable && } Date: Thu, 9 Jul 2026 16:47:21 -0700 Subject: [PATCH 06/13] fix(custom-blocks): stop draft-load sanitization deleting consumer-typed input values (#5551) --- .../workflows/sanitization/subblocks.test.ts | 137 ++++++++++++++++++ .../lib/workflows/sanitization/subblocks.ts | 17 ++- 2 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 apps/sim/lib/workflows/sanitization/subblocks.test.ts diff --git a/apps/sim/lib/workflows/sanitization/subblocks.test.ts b/apps/sim/lib/workflows/sanitization/subblocks.test.ts new file mode 100644 index 00000000000..a9a714a400a --- /dev/null +++ b/apps/sim/lib/workflows/sanitization/subblocks.test.ts @@ -0,0 +1,137 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it, vi } from 'vitest' + +vi.unmock('@/blocks/registry') + +import { migrateSubblockIds } from '@/lib/workflows/migrations/subblock-migrations' +import { sanitizeMalformedSubBlocks } from '@/lib/workflows/sanitization/subblocks' +import type { BlockState } from '@/stores/workflows/workflow/types' + +const FIELD_ID = 'cd7e4a16-c608-4087-8f2d-61f9672baeda' + +/** + * A placed custom block whose stored structure only has the wiring sub-blocks. + * `getBlock` cannot resolve `custom_block_*` types outside the org overlay — + * exactly the draft-load context where sanitization runs. + */ +function makeCustomBlock(subBlocks: Record) { + return { id: 'block-1', type: 'custom_block_abc123', subBlocks } +} + +describe('sanitizeMalformedSubBlocks', () => { + describe('custom blocks (schema-agnostic)', () => { + it('keeps and repairs a consumer-typed field value stored with the realtime "unknown" fallback', () => { + const { subBlocks, changed } = sanitizeMalformedSubBlocks( + makeCustomBlock({ + workflowId: { id: 'workflowId', type: 'short-input', value: 'wf-1' }, + inputMapping: { id: 'inputMapping', type: 'code', value: '{}' }, + [FIELD_ID]: { id: FIELD_ID, type: 'unknown', value: 'theo' }, + }) + ) + + expect(changed).toBe(true) + expect(subBlocks[FIELD_ID]).toEqual({ id: FIELD_ID, type: 'short-input', value: 'theo' }) + expect(subBlocks.workflowId.value).toBe('wf-1') + expect(subBlocks.inputMapping.value).toBe('{}') + }) + + it('keeps a raw non-record field value by wrapping it in a repaired entry', () => { + const { subBlocks, changed } = sanitizeMalformedSubBlocks( + makeCustomBlock({ [FIELD_ID]: 'theo' }) + ) + + expect(changed).toBe(true) + expect(subBlocks[FIELD_ID]).toEqual({ id: FIELD_ID, type: 'short-input', value: 'theo' }) + }) + + it('keeps an entry with missing metadata, keying it by the map key', () => { + const { subBlocks, changed } = sanitizeMalformedSubBlocks( + makeCustomBlock({ [FIELD_ID]: { value: 'theo' } }) + ) + + expect(changed).toBe(true) + expect(subBlocks[FIELD_ID]).toEqual({ id: FIELD_ID, type: 'short-input', value: 'theo' }) + }) + + it('leaves well-formed sub-blocks untouched and reports no change', () => { + const input = { + workflowId: { id: 'workflowId', type: 'short-input', value: 'wf-1' }, + [FIELD_ID]: { id: FIELD_ID, type: 'short-input', value: 'theo' }, + } + const { subBlocks, changed } = sanitizeMalformedSubBlocks(makeCustomBlock(input)) + + expect(changed).toBe(false) + expect(subBlocks).toBe(input) + }) + + it('still drops the literal "undefined" key', () => { + const { subBlocks, changed } = sanitizeMalformedSubBlocks( + makeCustomBlock({ undefined: { id: 'undefined', type: 'unknown', value: 'x' } }) + ) + + expect(changed).toBe(true) + expect(subBlocks).toEqual({}) + }) + }) + + describe('through migrateSubblockIds (the draft-load migration pipeline)', () => { + it('a typed custom-block field value survives the load-time migration pass', () => { + const blocks: Record = { + b1: { + id: 'b1', + name: 'Update Internal Allowlist 1', + position: { x: 0, y: 0 }, + type: 'custom_block_abc123', + subBlocks: { + workflowId: { id: 'workflowId', type: 'short-input', value: 'wf-child' }, + inputMapping: { id: 'inputMapping', type: 'code', value: '{}' }, + [FIELD_ID]: { id: FIELD_ID, type: 'unknown', value: 'test' }, + }, + outputs: {}, + enabled: true, + } as BlockState, + } + + const { blocks: migrated, migrated: changed } = migrateSubblockIds(blocks) + + expect(changed).toBe(true) + expect(migrated.b1.subBlocks[FIELD_ID]).toEqual({ + id: FIELD_ID, + type: 'short-input', + value: 'test', + }) + }) + }) + + describe('regular blocks (config is the schema)', () => { + it('still drops an "unknown"-typed entry that matches no configured sub-block', () => { + const { subBlocks, changed } = sanitizeMalformedSubBlocks({ + id: 'block-1', + type: 'function', + subBlocks: { + code: { id: 'code', type: 'code', value: 'return 1' }, + stale: { id: 'stale', type: 'unknown', value: 'x' }, + }, + }) + + expect(changed).toBe(true) + expect(subBlocks.stale).toBeUndefined() + expect(subBlocks.code.value).toBe('return 1') + }) + + it('repairs an "unknown"-typed entry that matches a configured sub-block', () => { + const { subBlocks, changed } = sanitizeMalformedSubBlocks({ + id: 'block-1', + type: 'function', + subBlocks: { + code: { id: 'code', type: 'unknown', value: 'return 1' }, + }, + }) + + expect(changed).toBe(true) + expect(subBlocks.code).toEqual({ id: 'code', type: 'code', value: 'return 1' }) + }) + }) +}) diff --git a/apps/sim/lib/workflows/sanitization/subblocks.ts b/apps/sim/lib/workflows/sanitization/subblocks.ts index a97bb7b479b..1f0d222ce2a 100644 --- a/apps/sim/lib/workflows/sanitization/subblocks.ts +++ b/apps/sim/lib/workflows/sanitization/subblocks.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { isPlainRecord } from '@sim/utils/object' import { DEFAULT_SUBBLOCK_TYPE } from '@sim/workflow-persistence/subblocks' import { getBlock } from '@/blocks' +import { isCustomBlockType } from '@/blocks/custom/build-config' import type { BlockState } from '@/stores/workflows/workflow/types' const logger = createLogger('WorkflowSubblockSanitization') @@ -19,6 +20,15 @@ interface SanitizableBlock { /** * Repairs legacy subBlock metadata when the map key identifies a real field, * and drops entries that cannot be associated with a stable subBlock. + * + * Custom blocks are schema-agnostic here: their server-side config never + * declares the per-field input sub-blocks (the execution overlay passes bare + * wiring rows, and this may run with no overlay at all), so "not in config" + * carries no signal for them. A consumer-typed field value stored via the + * realtime `type: 'unknown'` fallback must be repaired to a concrete type and + * kept — dropping it would delete user input from the draft. Values for fields + * the source workflow no longer has are filtered at serialization/execution + * (`customBlockHasDeclaredInputs`, `remapCustomBlockInputKeys`), never at rest. */ export function sanitizeMalformedSubBlocks( block: SanitizableBlock, @@ -26,6 +36,7 @@ export function sanitizeMalformedSubBlocks( ): { subBlocks: Record; changed: boolean } { let changed = false const blockConfig = getBlock(block.type) + const schemaAgnostic = isCustomBlockType(block.type) const result: Record = {} for (const [subBlockId, subBlock] of Object.entries(block.subBlocks || {})) { @@ -38,7 +49,7 @@ export function sanitizeMalformedSubBlocks( const configuredType = blockConfig?.subBlocks?.find((config) => config.id === subBlockId)?.type if (!isPlainRecord(subBlock)) { - if (!configuredType) { + if (!configuredType && !schemaAgnostic) { logger.warn('Skipping malformed subBlock: unrecognized value entry', { blockId: block.id, subBlockId, @@ -57,7 +68,7 @@ export function sanitizeMalformedSubBlocks( continue } - if (subBlock.type === 'unknown' && !configuredType) { + if (subBlock.type === 'unknown' && !configuredType && !schemaAgnostic) { logger.warn('Skipping malformed subBlock: type is "unknown"', { blockId: block.id, subBlockId, @@ -75,7 +86,7 @@ export function sanitizeMalformedSubBlocks( typeof subBlock.type !== 'string' || subBlock.type.length === 0 - if (missingMetadata && !typeFromConfig) { + if (missingMetadata && !typeFromConfig && !schemaAgnostic) { logger.warn('Skipping malformed subBlock: unrecognized metadata entry', { blockId: block.id, subBlockId, From cbe5e0f3f45bdda5ded168970e3ebb51ac2c9d0a Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 18:09:27 -0700 Subject: [PATCH 07/13] fix(uploads): fix Azure Blob connection-string-only auth and document self-host parity (#5553) * fix(uploads): fix Azure Blob connection-string-only auth and document self-host parity Every Azure Blob operation (upload/download/delete/head/presigned URLs) threw when only AZURE_CONNECTION_STRING was set, despite that being the documented alternative to AZURE_ACCOUNT_NAME/KEY across .env.example, env.ts, and the Helm chart. createBlobConfig required accountName unconditionally, and the upload-SAS path had no fallback to derive credentials from the connection string. Fixed both, verified end-to-end against a live Azurite emulator (upload, download, head, delete, multipart/block-blob upload, and real HTTP PUT/GET through generated SAS URLs), and added regression tests. Also closes the remaining self-host Azure documentation gaps: AZURE_ACS_CONNECTION_STRING, OCR_AZURE_*, KB_OPENAI_MODEL_NAME, and WAND_OPENAI_MODEL_NAME are now documented in the Helm chart (values.yaml, values.schema.json, values-azure.yaml) and .env.example alongside their AWS/S3 counterparts. * fix(uploads): map new Azure keys in the ESO remoteRefs example Greptile flagged that the values-azure.yaml External Secrets example didn't map KB_OPENAI_MODEL_NAME, WAND_OPENAI_MODEL_NAME, OCR_AZURE_ENDPOINT, and OCR_AZURE_MODEL_NAME, so a user who fills those in and switches to ESO would hit a Helm template render failure. Added the remoteRefs entries and verified with an isolated helm template render. --- apps/sim/.env.example | 13 +++ ...age-service.blob-connection-string.test.ts | 82 +++++++++++++++++++ apps/sim/lib/uploads/core/storage-service.ts | 49 ++++++----- .../lib/uploads/providers/blob/client.test.ts | 22 +++++ apps/sim/lib/uploads/providers/blob/client.ts | 2 +- apps/sim/lib/uploads/providers/blob/types.ts | 2 +- helm/sim/examples/values-azure.yaml | 18 ++++ helm/sim/values.schema.json | 24 ++++++ helm/sim/values.yaml | 8 ++ 9 files changed, 196 insertions(+), 24 deletions(-) create mode 100644 apps/sim/lib/uploads/core/storage-service.blob-connection-string.test.ts diff --git a/apps/sim/.env.example b/apps/sim/.env.example index 4083be6cdb0..4536f8a8638 100644 --- a/apps/sim/.env.example +++ b/apps/sim/.env.example @@ -93,6 +93,19 @@ API_ENCRYPTION_KEY=your_api_encryption_key # Use `openssl rand -hex 32` to gener # S3_ENDPOINT= # Custom endpoint for S3-compatible storage (Cloudflare R2, MinIO, Backblaze B2). Leave unset for AWS S3 # S3_FORCE_PATH_STYLE=true # Required for MinIO/Ceph RGW. Leave unset for AWS S3 and R2 +# Azure Blob Storage takes precedence over S3 if both are configured +# AZURE_ACCOUNT_NAME= # Azure storage account name +# AZURE_ACCOUNT_KEY= # Azure storage account key +# AZURE_CONNECTION_STRING= # Alternative to account name/key +# AZURE_STORAGE_CONTAINER_NAME= # General workspace files container +# AZURE_STORAGE_KB_CONTAINER_NAME= # Knowledge base documents +# AZURE_STORAGE_EXECUTION_FILES_CONTAINER_NAME= # Workflow execution files +# AZURE_STORAGE_CHAT_CONTAINER_NAME= # Deployed chat assets +# AZURE_STORAGE_COPILOT_CONTAINER_NAME= # Copilot attachments +# AZURE_STORAGE_PROFILE_PICTURES_CONTAINER_NAME= # User profile pictures +# AZURE_STORAGE_OG_IMAGES_CONTAINER_NAME= # OpenGraph preview images (falls back to AZURE_STORAGE_CONTAINER_NAME) +# AZURE_STORAGE_WORKSPACE_LOGOS_CONTAINER_NAME= # Workspace logos (falls back to AZURE_STORAGE_CONTAINER_NAME) + # Admin API (Optional - for self-hosted GitOps) # ADMIN_API_KEY= # Use `openssl rand -hex 32` to generate. Enables admin API for workflow export/import. # Usage: curl -H "x-admin-key: your_key" https://your-instance/api/v1/admin/workspaces diff --git a/apps/sim/lib/uploads/core/storage-service.blob-connection-string.test.ts b/apps/sim/lib/uploads/core/storage-service.blob-connection-string.test.ts new file mode 100644 index 00000000000..dffb79607eb --- /dev/null +++ b/apps/sim/lib/uploads/core/storage-service.blob-connection-string.test.ts @@ -0,0 +1,82 @@ +/** + * Regression tests: Azure Blob storage must be fully usable with ONLY + * AZURE_CONNECTION_STRING set (no AZURE_ACCOUNT_NAME/AZURE_ACCOUNT_KEY) — this + * is the connection-string auth mode documented as a standalone alternative + * across .env.example, helm/sim/values.yaml, and env.ts. + * + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const CONNECTION_STRING = + 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;' + +const { mockHeadBlobObject, mockGetBlobServiceClient, mockGenerateBlobSASQueryParameters } = + vi.hoisted(() => ({ + mockHeadBlobObject: vi.fn(), + mockGetBlobServiceClient: vi.fn(), + mockGenerateBlobSASQueryParameters: vi.fn(() => ({ toString: () => 'sig=fake' })), + })) + +vi.mock('@/lib/uploads/config', () => ({ + USE_S3_STORAGE: false, + USE_BLOB_STORAGE: true, + // Connection-string-only: accountName/accountKey intentionally absent. + getStorageConfig: () => ({ + containerName: 'workspace-files', + accountName: undefined, + accountKey: undefined, + connectionString: CONNECTION_STRING, + }), +})) + +vi.mock('@/lib/uploads/providers/blob/client', () => ({ + headBlobObject: mockHeadBlobObject, + getBlobServiceClient: mockGetBlobServiceClient, + parseConnectionString: (connectionString: string) => { + const accountName = connectionString.match(/AccountName=([^;]+)/)?.[1] + const accountKey = connectionString.match(/AccountKey=([^;]+)/)?.[1] + if (!accountName || !accountKey) throw new Error('cannot parse') + return { accountName, accountKey } + }, +})) + +vi.mock('@azure/storage-blob', () => ({ + StorageSharedKeyCredential: vi.fn(), + BlobSASPermissions: { parse: vi.fn(() => 'w') }, + generateBlobSASQueryParameters: mockGenerateBlobSASQueryParameters, +})) + +import { generatePresignedUploadUrl, headObject } from '@/lib/uploads/core/storage-service' + +describe('Azure Blob storage — connection-string-only auth', () => { + beforeEach(() => { + vi.clearAllMocks() + mockHeadBlobObject.mockResolvedValue({ size: 42, contentType: 'text/plain' }) + mockGetBlobServiceClient.mockResolvedValue({ + getContainerClient: () => ({ + getBlockBlobClient: () => ({ url: 'https://devstoreaccount1.blob.core.windows.net/c/k' }), + }), + }) + }) + + it('headObject does not throw when only connectionString is configured', async () => { + await expect(headObject('some-key', 'workspace')).resolves.toEqual({ + size: 42, + contentType: 'text/plain', + }) + expect(mockHeadBlobObject).toHaveBeenCalled() + }) + + it('generatePresignedUploadUrl derives SAS credentials from connectionString when accountName/accountKey are absent', async () => { + const result = await generatePresignedUploadUrl({ + fileName: 'report.csv', + contentType: 'text/csv', + context: 'workspace', + fileSize: 100, + }) + + expect(mockGenerateBlobSASQueryParameters).toHaveBeenCalled() + expect(result.url).toContain('sig=fake') + }) +}) diff --git a/apps/sim/lib/uploads/core/storage-service.ts b/apps/sim/lib/uploads/core/storage-service.ts index 9584bb89968..d88f931db67 100644 --- a/apps/sim/lib/uploads/core/storage-service.ts +++ b/apps/sim/lib/uploads/core/storage-service.ts @@ -29,13 +29,13 @@ const logger = createLogger('StorageService') * @throws Error if required properties are missing */ function createBlobConfig(config: StorageConfig): BlobConfig { - if (!config.containerName || !config.accountName) { - throw new Error('Blob configuration missing required properties: containerName and accountName') + if (!config.containerName) { + throw new Error('Blob configuration missing required property: containerName') } - if (!config.connectionString && !config.accountKey) { + if (!config.connectionString && !(config.accountName && config.accountKey)) { throw new Error( - 'Blob configuration missing authentication: either connectionString or accountKey must be provided' + 'Blob configuration missing authentication: either connectionString or both accountName and accountKey must be provided' ) } @@ -635,7 +635,9 @@ async function generateBlobPresignedUrl( }, expirationSeconds: number ): Promise { - const { getBlobServiceClient } = await import('@/lib/uploads/providers/blob/client') + const { getBlobServiceClient, parseConnectionString } = await import( + '@/lib/uploads/providers/blob/client' + ) const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = await import('@azure/storage-blob') @@ -650,27 +652,30 @@ async function generateBlobPresignedUrl( const startsOn = new Date() const expiresOn = new Date(startsOn.getTime() + expirationSeconds * 1000) - let sasToken: string + let accountName = config.accountName + let accountKey = config.accountKey + if ((!accountName || !accountKey) && config.connectionString) { + ;({ accountName, accountKey } = parseConnectionString(config.connectionString)) + } - if (config.accountName && config.accountKey) { - const sharedKeyCredential = new StorageSharedKeyCredential( - config.accountName, - config.accountKey + if (!accountName || !accountKey) { + throw new Error( + 'Azure Blob SAS generation requires accountName/accountKey or a connectionString' ) - sasToken = generateBlobSASQueryParameters( - { - containerName: config.containerName, - blobName: key, - permissions: BlobSASPermissions.parse('w'), // write permission for upload - startsOn, - expiresOn, - }, - sharedKeyCredential - ).toString() - } else { - throw new Error('Azure Blob SAS generation requires accountName and accountKey') } + const sharedKeyCredential = new StorageSharedKeyCredential(accountName, accountKey) + const sasToken = generateBlobSASQueryParameters( + { + containerName: config.containerName, + blobName: key, + permissions: BlobSASPermissions.parse('w'), // write permission for upload + startsOn, + expiresOn, + }, + sharedKeyCredential + ).toString() + return { url: `${blobClient.url}?${sasToken}`, key, diff --git a/apps/sim/lib/uploads/providers/blob/client.test.ts b/apps/sim/lib/uploads/providers/blob/client.test.ts index 024fa2d794f..19b06c93a91 100644 --- a/apps/sim/lib/uploads/providers/blob/client.test.ts +++ b/apps/sim/lib/uploads/providers/blob/client.test.ts @@ -54,6 +54,7 @@ import { deleteFromBlob, downloadFromBlob, getPresignedUrl, + parseConnectionString, uploadToBlob, } from '@/lib/uploads/providers/blob/client' import { sanitizeFilenameForMetadata } from '@/lib/uploads/utils/file-utils' @@ -207,6 +208,27 @@ describe('Azure Blob Storage Client', () => { }) }) + describe('parseConnectionString', () => { + it('extracts accountName and accountKey from a well-formed connection string', () => { + const result = parseConnectionString( + 'DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey123;EndpointSuffix=core.windows.net' + ) + expect(result).toEqual({ accountName: 'myaccount', accountKey: 'mykey123' }) + }) + + it('throws when AccountName is missing', () => { + expect(() => + parseConnectionString('DefaultEndpointsProtocol=https;AccountKey=mykey123') + ).toThrow('Cannot extract account name from connection string') + }) + + it('throws when AccountKey is missing', () => { + expect(() => + parseConnectionString('DefaultEndpointsProtocol=https;AccountName=myaccount') + ).toThrow('Cannot extract account key from connection string') + }) + }) + describe('sanitizeFilenameForMetadata', () => { const testCases = [ { input: 'test file.txt', expected: 'test file.txt' }, diff --git a/apps/sim/lib/uploads/providers/blob/client.ts b/apps/sim/lib/uploads/providers/blob/client.ts index f2fa5925220..eda490a8b82 100644 --- a/apps/sim/lib/uploads/providers/blob/client.ts +++ b/apps/sim/lib/uploads/providers/blob/client.ts @@ -30,7 +30,7 @@ interface ParsedCredentials { * Extract account name and key from an Azure connection string. * Connection strings have the format: DefaultEndpointsProtocol=https;AccountName=...;AccountKey=...;EndpointSuffix=... */ -function parseConnectionString(connectionString: string): ParsedCredentials { +export function parseConnectionString(connectionString: string): ParsedCredentials { const accountNameMatch = connectionString.match(/AccountName=([^;]+)/) if (!accountNameMatch) { throw new Error('Cannot extract account name from connection string') diff --git a/apps/sim/lib/uploads/providers/blob/types.ts b/apps/sim/lib/uploads/providers/blob/types.ts index b024e2a6363..a24a87e384e 100644 --- a/apps/sim/lib/uploads/providers/blob/types.ts +++ b/apps/sim/lib/uploads/providers/blob/types.ts @@ -1,6 +1,6 @@ export interface BlobConfig { containerName: string - accountName: string + accountName?: string accountKey?: string connectionString?: string } diff --git a/helm/sim/examples/values-azure.yaml b/helm/sim/examples/values-azure.yaml index 9de564a8923..2404088b221 100644 --- a/helm/sim/examples/values-azure.yaml +++ b/helm/sim/examples/values-azure.yaml @@ -98,6 +98,16 @@ app: AZURE_ANTHROPIC_API_KEY: "" # Azure Anthropic API key AZURE_ANTHROPIC_API_VERSION: "" # Azure Anthropic API version (e.g., 2023-06-01) NEXT_PUBLIC_AZURE_CONFIGURED: "true" # Set to "true" once credentials are configured above + KB_OPENAI_MODEL_NAME: "" # Azure deployment name serving the KB embedding model (used only when AZURE_OPENAI_* is set) + WAND_OPENAI_MODEL_NAME: "" # Wand generation model deployment name + + # Azure Mistral OCR (optional — Azure-hosted document OCR for knowledge base processing) + OCR_AZURE_ENDPOINT: "" # Azure Mistral OCR service endpoint + OCR_AZURE_MODEL_NAME: "" # Azure Mistral OCR model name + OCR_AZURE_API_KEY: "" # Azure Mistral OCR API key + + # Email — Azure Communication Services (recommended on Azure; alternative to Resend/SES/SMTP) + AZURE_ACS_CONNECTION_STRING: "" # Azure Communication Services connection string # Azure Blob Storage Configuration (RECOMMENDED for production) # Create a storage account and containers in your Azure subscription @@ -308,5 +318,13 @@ ingressInternal: # AZURE_ACCOUNT_KEY: "sim-azure-storage-account-key" # # Use AZURE_CONNECTION_STRING instead if you prefer a single connection string: # # AZURE_CONNECTION_STRING: "sim-azure-storage-connection-string" +# OCR_AZURE_API_KEY: "sim-azure-ocr-api-key" +# AZURE_ACS_CONNECTION_STRING: "sim-azure-acs-connection-string" +# # Non-secret config (deployment names, endpoints) — still required here: ESO mode requires +# # every non-empty app.env key to be mapped, even ones that aren't sensitive. +# KB_OPENAI_MODEL_NAME: "sim-kb-openai-model-name" +# WAND_OPENAI_MODEL_NAME: "sim-wand-openai-model-name" +# OCR_AZURE_ENDPOINT: "sim-azure-ocr-endpoint" +# OCR_AZURE_MODEL_NAME: "sim-azure-ocr-model-name" # postgresql: # password: "sim-postgresql-password" diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index 37f518542b2..f6641894638 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -186,6 +186,10 @@ "type": "string", "description": "AWS region for SES (e.g., 'us-east-1'). Credentials are resolved via the standard AWS provider chain (env vars, IRSA, EC2/ECS task role, SSO)." }, + "AZURE_ACS_CONNECTION_STRING": { + "type": "string", + "description": "Azure Communication Services connection string (email provider — used when Resend/SES/SMTP are not configured)." + }, "SMTP_HOST": { "type": "string", "description": "SMTP server hostname. When set together with SMTP_PORT, enables the SMTP mail provider." @@ -270,6 +274,26 @@ "type": "string", "description": "ElevenLabs API key for text-to-speech in deployed chat" }, + "KB_OPENAI_MODEL_NAME": { + "type": "string", + "description": "Azure deployment name serving the configured KB embedding model (used only when AZURE_OPENAI_* credentials are set)." + }, + "WAND_OPENAI_MODEL_NAME": { + "type": "string", + "description": "Wand generation model deployment name (works with both regular OpenAI and Azure OpenAI)." + }, + "OCR_AZURE_ENDPOINT": { + "type": "string", + "description": "Azure Mistral OCR service endpoint." + }, + "OCR_AZURE_MODEL_NAME": { + "type": "string", + "description": "Azure Mistral OCR model name." + }, + "OCR_AZURE_API_KEY": { + "type": "string", + "description": "Azure Mistral OCR API key." + }, "RATE_LIMIT_WINDOW_MS": { "type": "string", "description": "Rate limit window duration in milliseconds" diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index a12713a5bec..592d2d4f4f7 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -117,6 +117,7 @@ app: FROM_EMAIL_ADDRESS: "" # Complete from address (e.g., "Sim " or "DoNotReply@domain.com") EMAIL_DOMAIN: "" # Domain for sending emails (fallback when FROM_EMAIL_ADDRESS not set) AWS_SES_REGION: "" # AWS region for SES (e.g., "us-east-1"); credentials resolved via the standard AWS provider chain (env, IRSA, instance profile) + AZURE_ACS_CONNECTION_STRING: "" # Azure Communication Services connection string (email provider — used when Resend/SES/SMTP are not configured) SMTP_HOST: "" # SMTP server hostname (alternative to Resend/SES) SMTP_PORT: "" # SMTP server port (465 for TLS, 587 for STARTTLS, 25 for plain) SMTP_USER: "" # SMTP username (optional — omit for unauthenticated relays like MailHog) @@ -145,6 +146,13 @@ app: AZURE_ANTHROPIC_ENDPOINT: "" # Azure AI Foundry endpoint for Anthropic models AZURE_ANTHROPIC_API_KEY: "" # Azure Anthropic API key AZURE_ANTHROPIC_API_VERSION: "" # Azure Anthropic API version (e.g., 2023-06-01) + KB_OPENAI_MODEL_NAME: "" # Azure deployment name serving the configured KB embedding model (used only when AZURE_OPENAI_* credentials are set) + WAND_OPENAI_MODEL_NAME: "" # Wand generation model deployment name (works with both regular OpenAI and Azure OpenAI) + + # Azure Mistral OCR Configuration (leave empty if not using Azure-hosted OCR for document processing) + OCR_AZURE_ENDPOINT: "" # Azure Mistral OCR service endpoint + OCR_AZURE_MODEL_NAME: "" # Azure Mistral OCR model name + OCR_AZURE_API_KEY: "" # Azure Mistral OCR API key # AI Provider API Keys (leave empty if not using) OPENAI_API_KEY: "" # Primary OpenAI API key From a3487da8a1734f92737a76d591b673a0e9fa86f4 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 18:09:41 -0700 Subject: [PATCH 08/13] improvement(files): remove Save UI in favor of silent autosave with local-first draft recovery (#5549) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * improvement(files): remove Save UI in favor of silent autosave with local-first draft recovery Files editor no longer shows a Save/Saving/Save failed button - autosave already ran in the background, the button was vestigial. Cmd+S still works. Adds local-first draft persistence to the shared useAutosave hook (opt-in via draftKey): edits mirror into IndexedDB on a 400ms debounce, independent of the 1.5s network save, and flush best-effort on visibilitychange/pagehide. On reopen, a newer local draft is silently recovered and resynced. This replaces the beforeunload "leave site?" warning, which only blocked navigation - it never actually saved anything. A toast (with a Retry action) surfaces a save failure, since there's no more persistent status indicator to show it. * improvement(files): simplify autosave draft persistence and dedupe editor resync Structural cleanup after the local-draft feature: draftKey is now ANDed with enabled inside useAutosave itself (rather than trusting every caller to replicate that gating), the combined dirty-transition/debounce effect is split into two single-purpose effects, redundant back-to-back IndexedDB writes on visibilitychange+pagehide are deduped, a dead identity wrapper around setDraftContent is removed, and the three near-identical "resync editor body if changed" blocks in rich-markdown-editor.tsx collapse into one local helper. * fix(files): flush pending local draft on unmount, fix stale Retry target Two real bugs from Greptile's first review pass: - Unmounting before the 400ms local-draft debounce fired cancelled the pending timer without ever writing the draft, so if the network flush also failed on the way out, the edit had no backup anywhere. The unmount cleanup now calls persistLocalDraft() synchronously before attempting the network flush. - The save-failure toast's Retry action read saveRef.current lazily at click time, so navigating to a different file before clicking Retry would retry-save the wrong file. It now captures the failing file's save function at the moment the toast is created. * fix(files): Discard Changes now actually resets editor content Discard previously only cleared the parent's mirrored isDirty/saveStatus state; the editor's own content was never reset to match the server baseline. On unmount, useAutosave's flush logic saw content still diverged and (a) re-saved the "discarded" edit to the server, and (b) after this PR's local-draft addition, also persisted it to IndexedDB — so even a future fix to (a) would still have the draft resurrect the discarded text on next open. Adds a discardRef bridge (mirrors the existing saveRef pattern) so Discard resets the editor's draft content back to savedContent before navigating away, closing both paths at the root. * fix(files): make Discard deterministic, independent of state-update timing The previous discard fix (setDraftContent(savedContent) before navigating) relied on that dispatch landing before the FileViewer unmounts. If unmount raced ahead of it, the autosave cleanup would still see stale dirty content and could resurrect the discarded edit via the local draft. useAutosave now exposes discard(): it flags the instance as discarded, cancels any pending timers, and clears the local draft immediately. Every write path (persistLocalDraft, save, the unmount flush) checks that flag first, so nothing written after discard() can bring the edit back, regardless of whether the content-reset render has committed yet. * fix(files): correct in-flight save after discard, fix IndexedDB write/delete ordering Two more real races from round 4 of review: - discard() couldn't stop a save that had already started (discardedRef only blocks saves not yet begun). Once that in-flight save lands, it now schedules a corrective save to push the reverted content, rather than leaving the discarded edit on the server permanently. Only fires when a save was genuinely in flight at discard time. - persistLocalDraft's set() and clearLocalDraft's del() were independent promises with no ordering guarantee. A slow write starting before discard could resolve after discard's delete and resurrect the draft. Both now go through a single serialized queue per hook instance, so a delete queued after a write always runs after it completes. * fix(files): make discard's corrective save use an explicit baseline The corrective save (from the previous fix) relied on the caller's setDraftContent(savedContent) having landed by the time it ran — a real race, not a guarantee: React commits that render on its own schedule, and if the correction's continuation runs first, onSave still reads the ambient (still-dirty) content ref and re-persists the discarded edit. onSave now accepts an optional override content; discard() captures savedContentRef.current as an explicit target at the moment it's called and passes it through, so the correction always pushes the true reverted baseline regardless of render timing. Widened the shared onSave type is backward compatible — the other useAutosave caller (chunk-editor) ignores the extra optional param. * fix(files): retry no longer depends on a remount-able shared ref, purge stale drafts Two more from round 6: - The failure toast's Retry action captured saveRef.current inside the effect reacting to saveStatus='error' — but if the user switched files between the failure occurring and that effect committing, the keyed remount could have already repointed saveRef at the new file's save function first. onSaveStatusChange now passes the failing instance's own saveImmediately alongside the 'error' status directly from the hook that owns it, so retry can never be sourced from the wrong file regardless of remount timing. - A local draft with a stale (mismatched) baseline was left in IndexedDB after being correctly skipped for recovery, so it could resurrect later if the server baseline ever coincidentally matched it again. It's now purged as soon as it's identified as stale. * fix(files): clear inFlightRef once a save settles inFlightRef.current was never reset after a save resolved or rejected — it stayed pointing at the (now-fulfilled) promise indefinitely. discard() reads it to decide whether a save is genuinely in flight; since a resolved promise is still truthy, discard() treated any prior completed save as still pending, captured savedContentRef.current as the "corrective" target, and could push a stale baseline if that capture happened before the save's own dispatch had updated it. Now cleared to null as soon as the save settles, so discard()'s in-flight check reflects reality regardless of how long ago the last save finished. * fix(files): surface a failed discard correction, resume autosave after discard if editing continues Two more from round 8: - If discard()'s corrective save failed, it was only logged — the server could permanently keep the discarded edit with zero user-facing signal. Now surfaced via a dedicated onDiscardCorrectionFailed callback, which closes over the specific file's own name rather than routing through the shared onSaveStatusChange path (that path reads whichever file is currently selected, which by the time this fires is already the file the user navigated to, not the discarded one). - discardedRef never cleared once set, so if the editor stayed mounted briefly after discard (before navigation completes) and the user typed again, every save path silently no-op'd forever for that new edit too. It now clears itself as soon as a genuinely new edit (content diverging from savedContent again) is observed. * fix(files): serialize local drafts by key across mounts, not just within one idbQueueRef was a per-instance ref, so it only ordered IndexedDB ops issued by the same hook instance. A slow write queued by an unmount's flush lived on as a bare promise after that instance was gone, with nothing sequencing it against a freshly-mounted instance for the same file — its del() or recovery read could run first, and the late write would land afterward and resurrect a draft that was supposed to be gone. Replaced the per-instance ref with a module-level queue keyed by draft key, shared by every useAutosave instance (past or present) touching that key, so ordering holds across a fast unmount+remount of the same file. * improvement(files): consolidate autosave hook after 9 rounds of incremental fixes Cosmetic-only pass, no behavior change: - Hoisted MIN_SAVING_DISPLAY_MS to module scope alongside LOCAL_DRAFT_DELAY_MS (was declared inside the hook body, re-allocated every render, inconsistent with its sibling constant). - Grouped the ~15 refs by concern (save/network, content mirrors, draft-key + callbacks, local-draft persistence, discard) instead of the chronological order they were added across nine review rounds. - Removed a provably-dead re-check in the unmount cleanup: content/savedContent can't change between the outer guard and the inner one (no renders happen post-unmount), so only the discardedRef half of the inner check was live. - Removed an unnecessary useCallback around onDiscardCorrectionFailed — its reference is never observed by anything (useAutosave copies it into a ref every render regardless of identity), unlike handleSaveStatusChange in files.tsx, which is correctly memoized because it flows through React.memo-wrapped TextEditor/RichMarkdownEditor. Validated against external research: the two-tier debounce (network + local IndexedDB draft) matches how Tiptap/Notion describe their own local-first persistence; the discardedRef+corrective-save approach over AbortController is a deliberate, justified choice (onSave has no signal parameter, and an abort can't undo a write that's already landed server-side); the module-level per-key promise queue is a recognized idiomatic pattern. Splitting this hook into three smaller ones (useDebouncedSave/useLocalDraft/useDiscard) is a legitimate future refactor, deliberately deferred given the risk of touching this heavily-interdependent, already-hardened state this late in review. * fix(files): serialize discard correction against newer saves, recover local drafts only once per mount Two more real races, both interactions between earlier fixes: - discard()'s corrective save and a genuinely new edit made right after could race independently: if the user typed again before the correction fired, and that correction landed after the new edit's own save, the server would end up with the discarded baseline instead of the user's latest content. The correction now shares the same inFlightRef/savingRef mutual exclusion normal saves use, and skips entirely once content has moved on to something that's neither the discarded baseline nor what it was at the moment discard() was called. - The local-draft recovery effect re-ran every time draftKey toggled through enabled (e.g. autosave turning off during agent streaming and back on once it settles), re-scanning IndexedDB as if freshly mounted. If the settled content coincidentally matched the stale draft's stored baseline, a pre-stream local edit could silently overwrite the agent's work. Recovery now attempts exactly once per mount. --- .../components/file-viewer/file-viewer.tsx | 9 +- .../rich-markdown-editor.tsx | 23 +- .../components/file-viewer/text-editor.tsx | 8 +- .../file-viewer/use-editable-file-content.ts | 54 +- .../workspace/[workspaceId]/files/files.tsx | 53 +- apps/sim/hooks/use-autosave.test.tsx | 587 +++++++++++++++++- apps/sim/hooks/use-autosave.ts | 201 +++++- 7 files changed, 865 insertions(+), 70 deletions(-) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx index 3d4f4b5d3e9..9984e16b0df 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/file-viewer.tsx @@ -99,8 +99,12 @@ interface FileViewerProps { previewMode?: PreviewMode autoFocus?: boolean onDirtyChange?: (isDirty: boolean) => void - onSaveStatusChange?: (status: 'idle' | 'saving' | 'saved' | 'error') => void + onSaveStatusChange?: ( + status: 'idle' | 'saving' | 'saved' | 'error', + retry?: () => Promise + ) => void saveRef?: React.MutableRefObject<(() => Promise) | null> + discardRef?: React.MutableRefObject<(() => void) | null> streamingContent?: string isAgentEditing?: boolean streamIsIncremental?: boolean @@ -131,6 +135,7 @@ function FileViewerContent({ onDirtyChange, onSaveStatusChange, saveRef, + discardRef, streamingContent, isAgentEditing, streamIsIncremental, @@ -174,6 +179,7 @@ function FileViewerContent({ onDirtyChange={onDirtyChange} onSaveStatusChange={onSaveStatusChange} saveRef={saveRef} + discardRef={discardRef} streamingContent={streamingContent} isAgentEditing={isAgentEditing} streamIsIncremental={streamIsIncremental} @@ -193,6 +199,7 @@ function FileViewerContent({ onDirtyChange={onDirtyChange} onSaveStatusChange={onSaveStatusChange} saveRef={saveRef} + discardRef={discardRef} streamingContent={streamingContent} isAgentEditing={isAgentEditing} disableStreamingAutoScroll={disableStreamingAutoScroll} diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx index d6fc15224a1..7114210f8ca 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/rich-markdown-editor.tsx @@ -45,8 +45,9 @@ interface RichMarkdownEditorProps { canEdit: boolean autoFocus?: boolean onDirtyChange?: (isDirty: boolean) => void - onSaveStatusChange?: (status: SaveStatus) => void + onSaveStatusChange?: (status: SaveStatus, retry?: () => Promise) => void saveRef?: React.MutableRefObject<(() => Promise) | null> + discardRef?: React.MutableRefObject<(() => void) | null> streamingContent?: string isAgentEditing?: boolean /** @@ -70,6 +71,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ onDirtyChange, onSaveStatusChange, saveRef, + discardRef, streamingContent, isAgentEditing, streamIsIncremental, @@ -93,6 +95,7 @@ export const RichMarkdownEditor = memo(function RichMarkdownEditor({ onDirtyChange, onSaveStatusChange, saveRef, + discardRef, normalizeBaseline: normalizeMarkdownContent, }) @@ -356,6 +359,14 @@ export function LoadedRichMarkdownEditor({ const lastStreamParseAtRef = useRef(0) useEffect(() => { if (!editor) return + const syncEditorBody = (body: string) => { + if (body === lastSyncedBodyRef.current) return + lastSyncedBodyRef.current = body + editor.commands.setContent(parseMarkdownToDoc(body), { + contentType: 'json', + emitUpdate: false, + }) + } if (isStreaming) { wasStreamingRef.current = true if (editor.isEditable) editor.setEditable(false) @@ -407,14 +418,7 @@ export function LoadedRichMarkdownEditor({ if (isInitialSettle || wasStreamingRef.current) { wasStreamingRef.current = false settledRef.current = lockSettled(content) - const body = splitFrontmatter(content).body - if (body !== lastSyncedBodyRef.current) { - lastSyncedBodyRef.current = body - editor.commands.setContent(parseMarkdownToDoc(body), { - contentType: 'json', - emitUpdate: false, - }) - } + syncEditorBody(splitFrontmatter(content).body) // `setContent` maps any pre-existing selection onto the new doc rather than clearing it — a // select-all survives as "select everything," permanently painting every divider/image with the // `rich-leaf-in-selection` decoration (keymap.ts) until the user clicks elsewhere. This must run @@ -428,6 +432,7 @@ export function LoadedRichMarkdownEditor({ if (isInitialSettle && autoFocus) editor.commands.focus('end') return } + syncEditorBody(splitFrontmatter(content).body) if (settledRef.current) editor.setEditable(canEdit && settledRef.current.verdict) }, [editor, content, isStreaming, canEdit, autoFocus, disableStreamingAutoScroll]) diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/text-editor.tsx b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/text-editor.tsx index d0fa0369a1a..6316f141f7d 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/text-editor.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/text-editor.tsx @@ -328,8 +328,12 @@ interface TextEditorProps { previewMode: PreviewMode autoFocus?: boolean onDirtyChange?: (isDirty: boolean) => void - onSaveStatusChange?: (status: 'idle' | 'saving' | 'saved' | 'error') => void + onSaveStatusChange?: ( + status: 'idle' | 'saving' | 'saved' | 'error', + retry?: () => Promise + ) => void saveRef?: React.MutableRefObject<(() => Promise) | null> + discardRef?: React.MutableRefObject<(() => void) | null> streamingContent?: string isAgentEditing?: boolean disableStreamingAutoScroll: boolean @@ -345,6 +349,7 @@ export const TextEditor = memo(function TextEditor({ onDirtyChange, onSaveStatusChange, saveRef, + discardRef, streamingContent, isAgentEditing, disableStreamingAutoScroll, @@ -385,6 +390,7 @@ export const TextEditor = memo(function TextEditor({ onDirtyChange, onSaveStatusChange, saveRef, + discardRef, }) contentRef.current = content diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts index 245476d2d92..f466249e74f 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/use-editable-file-content.ts @@ -1,6 +1,7 @@ 'use client' import { useCallback, useEffect, useMemo, useReducer, useRef } from 'react' +import { toast } from '@sim/emcn' import type { WorkspaceFileRecord } from '@/lib/uploads/contexts/workspace' import { useUpdateWorkspaceFileContent, @@ -34,8 +35,11 @@ interface UseEditableFileContentOptions { streamingContent?: string isAgentEditing?: boolean onDirtyChange?: (isDirty: boolean) => void - onSaveStatusChange?: (status: SaveStatus) => void + /** `retry` is this instance's own `saveImmediately`, passed alongside an `'error'` status so a caller-side retry never depends on a shared, remount-able ref. */ + onSaveStatusChange?: (status: SaveStatus, retry?: () => Promise) => void saveRef?: React.MutableRefObject<(() => Promise) | null> + /** Bridges an imperative "discard the current draft" command up to the caller, mirroring `saveRef`. */ + discardRef?: React.MutableRefObject<(() => void) | null> /** * Optional transform applied to the fetched content before it becomes the editor's baseline. A * surface whose editor re-serializes its content to a canonical form (the rich markdown editor) @@ -115,6 +119,7 @@ export function useEditableFileContent({ onDirtyChange, onSaveStatusChange, saveRef, + discardRef, normalizeBaseline, }: UseEditableFileContentOptions): EditableFileContent { const onDirtyChangeRef = useRef(onDirtyChange) @@ -182,17 +187,28 @@ export function useEditableFileContent({ const contentRef = useRef(content) contentRef.current = content - const onSave = useCallback(async () => { - const next = contentRef.current - await updateContentRef.current.mutateAsync({ workspaceId, fileId: file.id, content: next }) - markSavedContent(next) - }, [workspaceId, file.id, markSavedContent]) + const onSave = useCallback( + async (overrideContent?: string) => { + const next = overrideContent ?? contentRef.current + await updateContentRef.current.mutateAsync({ workspaceId, fileId: file.id, content: next }) + markSavedContent(next) + }, + [workspaceId, file.id, markSavedContent] + ) + + const autosaveEnabled = canEdit && isInitialized && !isStreamInteractionLocked - const { saveStatus, saveImmediately, isDirty } = useAutosave({ + const { saveStatus, saveImmediately, isDirty, discard } = useAutosave({ content, savedContent, onSave, - enabled: canEdit && isInitialized && !isStreamInteractionLocked, + enabled: autosaveEnabled, + draftKey: autosaveEnabled ? `${workspaceId}:${file.id}` : undefined, + onRestoreDraft: setDraftContent, + onDiscardCorrectionFailed: () => + toast.error( + `Failed to discard "${file.name}" — the server may still have the discarded edit` + ), }) useEffect(() => { @@ -200,8 +216,11 @@ export function useEditableFileContent({ }, [isDirty]) useEffect(() => { - onSaveStatusChangeRef.current?.(saveStatus) - }, [saveStatus]) + onSaveStatusChangeRef.current?.( + saveStatus, + saveStatus === 'error' ? saveImmediately : undefined + ) + }, [saveStatus, saveImmediately]) useEffect(() => { if (!saveRef) return @@ -213,6 +232,21 @@ export function useEditableFileContent({ } }, [saveImmediately, saveRef]) + const discardChanges = useCallback(() => { + discard() + setDraftContent(savedContent) + }, [discard, setDraftContent, savedContent]) + + useEffect(() => { + if (!discardRef) return + discardRef.current = discardChanges + return () => { + if (discardRef.current === discardChanges) { + discardRef.current = null + } + } + }, [discardChanges, discardRef]) + return { content: displayContent, setDraftContent, diff --git a/apps/sim/app/workspace/[workspaceId]/files/files.tsx b/apps/sim/app/workspace/[workspaceId]/files/files.tsx index e8be45d8fa6..1d55ca37628 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/files.tsx +++ b/apps/sim/app/workspace/[workspaceId]/files/files.tsx @@ -171,6 +171,7 @@ function formatFileType(mimeType: string | null, filename: string): string { export function Files() { const fileInputRef = useRef(null) const saveRef = useRef<(() => Promise) | null>(null) + const discardRef = useRef<(() => void) | null>(null) const params = useParams() const router = useRouter() @@ -970,6 +971,15 @@ export function Files() { await saveRef.current() }, []) + const handleSaveStatusChange = useCallback((status: SaveStatus, retry?: () => Promise) => { + setSaveStatus(status) + if (status === 'error') { + toast.error(`Failed to save "${selectedFileRef.current?.name ?? 'file'}"`, { + action: { label: 'Retry', onClick: () => void retry?.() }, + }) + } + }, []) + const handleNavigateFromFileDetail = useCallback( (url: string) => { if (isDirtyRef.current) { @@ -1106,6 +1116,7 @@ export function Files() { ]) const handleDiscardChanges = () => { + discardRef.current?.() setShowUnsavedChangesAlert(false) setIsDirty(false) setSaveStatus('idle') @@ -1358,16 +1369,8 @@ export function Files() { handleSave() } } - const handleBeforeUnload = (e: BeforeUnloadEvent) => { - if (!isDirtyRef.current) return - e.preventDefault() - } window.addEventListener('keydown', handleKeyDown) - window.addEventListener('beforeunload', handleBeforeUnload) - return () => { - window.removeEventListener('keydown', handleKeyDown) - window.removeEventListener('beforeunload', handleBeforeUnload) - } + return () => window.removeEventListener('keydown', handleKeyDown) }, [handleSave]) const selectedRowIdsRef = useRef(selectedRowIds) @@ -1428,43 +1431,21 @@ export function Files() { const fileActions = useMemo(() => { if (!selectedFile) return [] // A large CSV renders as a read-only streamed preview (no editor), so it gets neither the - // Save action nor the edit/split/preview toggle — just like a non-editable file. + // edit/split/preview toggle nor autosave — just like a non-editable file. const streamOnly = isCsvStreamOnly(selectedFile) const canEditText = isTextEditable(selectedFile) && !streamOnly const canPreview = isPreviewable(selectedFile) && !streamOnly - // Markdown renders in the single-surface inline editor, which has no raw/split/preview - // modes — so it keeps Save but drops the mode toggle. + // Markdown renders in the single-surface inline editor, which has no raw/split/preview modes. const isInlineMarkdown = isMarkdownFile(selectedFile) const hasSplitView = canEditText && canPreview && !isInlineMarkdown const showPreviewToggle = canPreview && !isInlineMarkdown - const saveLabel = - saveStatus === 'saving' - ? 'Saving...' - : saveStatus === 'saved' - ? 'Saved' - : saveStatus === 'error' - ? 'Save failed' - : 'Save' - const nextModeLabel = previewMode === 'editor' ? 'Split' : previewMode === 'split' ? 'Preview' : 'Edit' const nextModeIcon = previewMode === 'editor' ? Columns2 : previewMode === 'split' ? Eye : Pencil return [ - ...(canEditText - ? [ - { - text: saveLabel, - onSelect: handleSave, - disabled: - (!isDirty && saveStatus === 'idle') || - saveStatus === 'saving' || - saveStatus === 'saved', - }, - ] - : []), ...(hasSplitView ? [ { @@ -1505,12 +1486,9 @@ export function Files() { }, [ selectedFile, canEdit, - saveStatus, previewMode, - isDirty, handleCyclePreviewMode, handleTogglePreview, - handleSave, handleDownloadSelected, handleShareSelected, handleDeleteSelected, @@ -1897,8 +1875,9 @@ export function Files() { previewMode={previewMode} autoFocus={isNewFile || justCreatedFileIdRef.current === selectedFile.id} onDirtyChange={setIsDirty} - onSaveStatusChange={setSaveStatus} + onSaveStatusChange={handleSaveStatusChange} saveRef={saveRef} + discardRef={discardRef} /> ({ fakeDraftStore: new Map() })) +vi.mock('idb-keyval', () => ({ + get: vi.fn((key: string) => Promise.resolve(fakeDraftStore.get(key))), + set: vi.fn((key: string, value: unknown) => { + fakeDraftStore.set(key, value) + return Promise.resolve() + }), + del: vi.fn((key: string) => { + fakeDraftStore.delete(key) + return Promise.resolve() + }), +})) + import { type SaveStatus, useAutosave } from '@/hooks/use-autosave' interface ProbeProps { content: string savedContent: string - onSave: () => Promise + onSave: (overrideContent?: string) => Promise delay?: number enabled?: boolean + draftKey?: string + onRestoreDraft?: (content: string) => void + onDiscardCorrectionFailed?: () => void } interface HookHandle { status: () => SaveStatus isDirty: () => boolean saveImmediately: () => Promise + discard: () => void rerender: (next: Partial) => void unmount: () => void } @@ -31,7 +51,12 @@ function renderAutosave(initial: ProbeProps): { handle: HookHandle; props: Probe const container = document.createElement('div') const root: Root = createRoot(container) const props = { ...initial } - let latest = { saveStatus: 'idle' as SaveStatus, isDirty: false, saveImmediately: async () => {} } + let latest = { + saveStatus: 'idle' as SaveStatus, + isDirty: false, + saveImmediately: async () => {}, + discard: () => {}, + } function Probe(p: ProbeProps) { latest = useAutosave(p) @@ -49,6 +74,7 @@ function renderAutosave(initial: ProbeProps): { handle: HookHandle; props: Probe status: () => latest.saveStatus, isDirty: () => latest.isDirty, saveImmediately: () => latest.saveImmediately(), + discard: () => latest.discard(), rerender: (next) => { Object.assign(props, next) render() @@ -69,6 +95,7 @@ async function flush() { describe('useAutosave', () => { beforeEach(() => { vi.useFakeTimers() + fakeDraftStore.clear() }) afterEach(() => { vi.runOnlyPendingTimers() @@ -257,6 +284,28 @@ describe('useAutosave', () => { expect(onSave).toHaveBeenCalledTimes(1) }) + it('writes a local backup on unmount even if the network flush fails', async () => { + const onSave = vi.fn(async () => { + throw new Error('offline') + }) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-unmount', + }) + + handle.rerender({ content: 'a1' }) + // Unmount before the 400ms local-draft debounce fires — the pending timer is cancelled, so + // the cleanup itself must persist the draft, not just attempt (and here, fail) the network flush. + handle.unmount() + await flush() + expect(fakeDraftStore.get('autosave-draft:file-unmount')).toEqual({ + content: 'a1', + savedContent: 'a', + }) + }) + it('does not flush on unmount when the document is clean', async () => { const onSave = vi.fn(async () => {}) const { handle } = renderAutosave({ content: 'a', savedContent: 'a', onSave }) @@ -264,4 +313,538 @@ describe('useAutosave', () => { await flush() expect(onSave).not.toHaveBeenCalled() }) + + describe('local draft persistence (draftKey)', () => { + it('mirrors dirty edits into IndexedDB on a short debounce, independent of the network save', async () => { + let resolveSave: (() => void) | undefined + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolveSave = resolve + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-1', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(400) + }) + await flush() + // The local draft lands well before the (longer) network debounce fires. + expect(fakeDraftStore.get('autosave-draft:file-1')).toEqual({ + content: 'a1', + savedContent: 'a', + }) + expect(onSave).not.toHaveBeenCalled() + resolveSave?.() + }) + + it('clears the local draft once the network save succeeds and the caller advances savedContent', async () => { + let resolveSave: (() => void) | undefined + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolveSave = resolve + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-2', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + // The network save is in flight; the local draft is still the only record of the edit. + expect(fakeDraftStore.has('autosave-draft:file-2')).toBe(true) + + handle.rerender({ savedContent: 'a1' }) + await act(async () => { + resolveSave?.() + }) + await flush() + expect(fakeDraftStore.has('autosave-draft:file-2')).toBe(false) + }) + + it('does not clear the local draft when a newer edit lands while the save is in flight', async () => { + let resolveSave: (() => void) | undefined + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolveSave = resolve + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-2b', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(fakeDraftStore.has('autosave-draft:file-2b')).toBe(true) + + // A further edit lands while the first save is still in flight. + handle.rerender({ content: 'a12' }) + await act(async () => { + vi.advanceTimersByTime(400) + }) + await flush() + + // The first save resolves; the caller advances savedContent only to the snapshot it saved. + handle.rerender({ savedContent: 'a1' }) + await act(async () => { + resolveSave?.() + }) + await flush() + // Still dirty ('a12' !== 'a1') — the local backup for the untransmitted edit must survive, + // even though `save()`'s success no longer explicitly clears it. + expect(fakeDraftStore.has('autosave-draft:file-2b')).toBe(true) + expect(fakeDraftStore.get('autosave-draft:file-2b')).toMatchObject({ content: 'a12' }) + }) + + it('flushes the draft to IndexedDB when the page becomes hidden', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-3', + }) + + handle.rerender({ content: 'a1' }) + // No timers advanced — simulates a tab close mid-keystroke, before either debounce fires. + Object.defineProperty(document, 'visibilityState', { value: 'hidden', configurable: true }) + document.dispatchEvent(new Event('visibilitychange')) + await flush() + expect(fakeDraftStore.get('autosave-draft:file-3')).toEqual({ + content: 'a1', + savedContent: 'a', + }) + }) + + it('restores a draft left behind by a prior session, once, on mount', async () => { + fakeDraftStore.set('autosave-draft:file-4', { content: 'recovered', savedContent: 'a' }) + const onSave = vi.fn(async () => {}) + const onRestoreDraft = vi.fn() + renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-4', + onRestoreDraft, + }) + + await flush() + expect(onRestoreDraft).toHaveBeenCalledTimes(1) + expect(onRestoreDraft).toHaveBeenCalledWith('recovered') + }) + + it('does not clobber a fresh edit made while the recovery read was still in flight', async () => { + fakeDraftStore.set('autosave-draft:file-6', { content: 'recovered', savedContent: 'a' }) + const onSave = vi.fn(async () => {}) + const onRestoreDraft = vi.fn() + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-6', + onRestoreDraft, + }) + + // The user types before the async IndexedDB read resolves. + handle.rerender({ content: 'user-typed' }) + await flush() + expect(onRestoreDraft).not.toHaveBeenCalled() + }) + + it('discards and purges a stale draft when the server baseline has moved on', async () => { + fakeDraftStore.set('autosave-draft:file-5', { + content: 'stale-edit', + savedContent: 'old-baseline', + }) + const onSave = vi.fn(async () => {}) + const onRestoreDraft = vi.fn() + renderAutosave({ + content: 'new-baseline', + savedContent: 'new-baseline', + onSave, + draftKey: 'file-5', + onRestoreDraft, + }) + + await flush() + expect(onRestoreDraft).not.toHaveBeenCalled() + // Purged, not merely skipped — otherwise a later open where the baseline coincidentally + // matches this stale snapshot again would silently resurrect the outdated edit. + expect(fakeDraftStore.has('autosave-draft:file-5')).toBe(false) + }) + + it('attempts recovery only once per mount, not every time draftKey toggles across a streaming lock', async () => { + fakeDraftStore.set('autosave-draft:file-once', { content: 'recovered', savedContent: 'a' }) + const onSave = vi.fn(async () => {}) + const onRestoreDraft = vi.fn() + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-once', + enabled: true, + onRestoreDraft, + }) + + await flush() + expect(onRestoreDraft).toHaveBeenCalledTimes(1) + + // Mirrors autosave being disabled during agent streaming (effectiveDraftKey -> undefined) + // and re-enabled once the stream settles (effectiveDraftKey -> defined again). A stale + // pre-stream draft left behind in the meantime must not be re-offered on the second pass. + fakeDraftStore.set('autosave-draft:file-once', { + content: 'pre-agent-edit', + savedContent: 'a', + }) + handle.rerender({ enabled: false }) + await flush() + handle.rerender({ enabled: true }) + await flush() + expect(onRestoreDraft).toHaveBeenCalledTimes(1) + }) + + it('discard clears the local draft immediately and blocks any further write, even mid-race with unmount', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(400) + }) + await flush() + expect(fakeDraftStore.has('autosave-draft:file-discard')).toBe(true) + + // Discard fires before the caller's content===savedContent reset has landed — the hook's + // own flag must block persistence regardless of that race, not just the IndexedDB delete. + act(() => handle.discard()) + await flush() + expect(fakeDraftStore.has('autosave-draft:file-discard')).toBe(false) + + // Simulate the unmount flush racing in right after discard, while still (from the hook's + // perspective) dirty: neither the local draft nor the network save must fire. + handle.unmount() + await flush() + expect(fakeDraftStore.has('autosave-draft:file-discard')).toBe(false) + expect(onSave).not.toHaveBeenCalled() + }) + + it('discard prevents a pending debounced network save from firing', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-2', + }) + + handle.rerender({ content: 'a1' }) + act(() => handle.discard()) + + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).not.toHaveBeenCalled() + }) + + it('corrects the server with the captured baseline, even if the caller never resets content in time', async () => { + let resolveSave: (() => void) | undefined + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolveSave = resolve + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-3', + }) + + // A save is genuinely in flight (discardedRef can't stop it — it already started). + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + + // The user discards while that save is still pending. Deliberately do NOT rerender content + // back to the baseline here — the correction must not depend on that caller-side reset + // landing before this continuation runs; it must push the baseline it captured at discard(). + act(() => handle.discard()) + + await act(async () => { + resolveSave?.() + }) + await flush() + // The stale in-flight write landed; a corrective save fires with the captured baseline ('a'), + // not whatever the still-dirty ambient content ('a1') happened to be. + expect(onSave).toHaveBeenCalledTimes(2) + expect(onSave).toHaveBeenNthCalledWith(2, 'a') + }) + + it('surfaces a corrective save failure instead of only logging it', async () => { + let resolveSave: (() => void) | undefined + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolveSave = resolve + }) + ) + const onDiscardCorrectionFailed = vi.fn() + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-6', + onDiscardCorrectionFailed, + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + act(() => handle.discard()) + + // The corrective save (the second call) fails. + onSave.mockImplementationOnce(() => Promise.reject(new Error('offline'))) + await act(async () => { + resolveSave?.() + }) + await flush() + expect(onDiscardCorrectionFailed).toHaveBeenCalledTimes(1) + }) + + it('resumes normal autosave for a genuinely new edit made after discard', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-7', + }) + + handle.rerender({ content: 'a1' }) + act(() => handle.discard()) + // The caller resets content back to the baseline, mirroring discardChanges. + handle.rerender({ content: 'a' }) + + // The editor stays mounted a moment longer and the user starts a fresh edit. + handle.rerender({ content: 'b' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + // A brand-new edit made after discard must not be silently swallowed forever. + expect(onSave).toHaveBeenCalledTimes(1) + expect(onSave).toHaveBeenCalledWith() + }) + + it('does not issue a corrective save when discard finds nothing in flight', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-4', + }) + + handle.rerender({ content: 'a1' }) + act(() => handle.discard()) + await flush() + expect(onSave).not.toHaveBeenCalled() + }) + + it('does not treat a long-settled save as in flight when discard runs much later', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-5', + }) + + // A save fully completes well before discard is ever called. + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + handle.rerender({ savedContent: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(600) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + + // A later, unrelated edit is discarded. inFlightRef must have been cleared when the first + // save settled — otherwise this reads it as still "in flight" and fires a stale corrective + // save with whatever savedContent happened to be at that (now long-past) moment. + handle.rerender({ content: 'a12' }) + act(() => handle.discard()) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + }) + + it('does not let a stale discard correction clobber a genuinely new edit made afterward', async () => { + const resolvers: Array<() => void> = [] + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-8', + }) + + // A save is in flight when the user discards. + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + act(() => handle.discard()) + + // The caller resets content to the baseline, then the user types something genuinely new + // before the in-flight save (and its scheduled correction) has settled. + handle.rerender({ content: 'a' }) + handle.rerender({ content: 'a2' }) + + // The in-flight save resolves; the correction runs but must defer, since content has + // moved on to something that's neither the discarded baseline nor what it was at discard. + await act(async () => { + resolvers[0]?.() + }) + await flush() + await act(async () => { + vi.advanceTimersByTime(600) + }) + await flush() + + // A fresh save for the new edit fires instead of a correction pushing the stale baseline. + const targets = onSave.mock.calls.map((call) => call[0]) + expect(targets).not.toContain('a') + expect(onSave.mock.calls.length).toBeGreaterThan(1) + + resolvers[resolvers.length - 1]?.() + await flush() + }) + + it('serializes IndexedDB writes and deletes so a slow write cannot resurrect a discarded draft', async () => { + let resolveWrite: (() => void) | undefined + const idbKeyval = await import('idb-keyval') + vi.mocked(idbKeyval.set).mockImplementationOnce( + (key: string, value: unknown) => + new Promise((resolve) => { + resolveWrite = () => { + fakeDraftStore.set(key, value) + resolve() + } + }) + ) + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-race', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(400) + }) + await flush() + // The write is now in flight (not yet resolved) when discard's delete is queued behind it. + act(() => handle.discard()) + await flush() + expect(fakeDraftStore.has('autosave-draft:file-race')).toBe(false) + + // The slow write finally resolves — it must not resurrect the entry the queued delete removed. + resolveWrite?.() + await flush() + expect(fakeDraftStore.has('autosave-draft:file-race')).toBe(false) + }) + + it('serializes IndexedDB ops across an unmount and a fresh remount of the same draft key', async () => { + let resolveWrite: (() => void) | undefined + const idbKeyval = await import('idb-keyval') + vi.mocked(idbKeyval.set).mockImplementationOnce( + (key: string, value: unknown) => + new Promise((resolve) => { + resolveWrite = () => { + fakeDraftStore.set(key, value) + resolve() + } + }) + ) + const onSaveA = vi.fn(async () => {}) + const { handle: handleA } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave: onSaveA, + draftKey: 'file-cross-mount', + }) + + handleA.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(400) + }) + await flush() + // The write is in flight when this instance unmounts — its own idbQueueRef dies with it, + // but the operation must still be ordered relative to whatever comes next for this key. + handleA.unmount() + await flush() + + // A fresh instance mounts for the same file and immediately discards. + const onSaveB = vi.fn(async () => {}) + const { handle: handleB } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave: onSaveB, + draftKey: 'file-cross-mount', + }) + act(() => handleB.discard()) + await flush() + + // The old instance's slow write finally resolves — it must not resurrect the entry the + // new instance's delete already removed, even though they belong to different mounts. + resolveWrite?.() + await flush() + expect(fakeDraftStore.has('autosave-draft:file-cross-mount')).toBe(false) + }) + }) }) diff --git a/apps/sim/hooks/use-autosave.ts b/apps/sim/hooks/use-autosave.ts index 824e2b415d0..05e1f10d6d0 100644 --- a/apps/sim/hooks/use-autosave.ts +++ b/apps/sim/hooks/use-autosave.ts @@ -1,21 +1,63 @@ 'use client' import { useCallback, useEffect, useRef, useState } from 'react' +import { createLogger } from '@sim/logger' +import { del, get, set } from 'idb-keyval' + +const logger = createLogger('Autosave') export type SaveStatus = 'idle' | 'saving' | 'saved' | 'error' +interface LocalDraft { + content: string + savedContent: string +} + +const LOCAL_DRAFT_DELAY_MS = 400 +const MIN_SAVING_DISPLAY_MS = 600 + +function localDraftDbKey(draftKey: string) { + return `autosave-draft:${draftKey}` +} + +const draftOpQueues = new Map>() + +/** Serializes IndexedDB reads/writes for a given draft key across every `useAutosave` instance (including one that already unmounted), so a late write from a just-unmounted instance can't land after a newly-mounted instance's delete or recovery read. */ +function enqueueDraftOp(key: string, op: () => Promise): Promise { + const prev = draftOpQueues.get(key) ?? Promise.resolve() + const result = prev.then(op, op) + const settled = result.catch(() => {}) + draftOpQueues.set(key, settled) + void settled.then(() => { + if (draftOpQueues.get(key) === settled) draftOpQueues.delete(key) + }) + return result +} + interface UseAutosaveOptions { content: string savedContent: string - onSave: () => Promise + /** `overrideContent`, when passed, is what `discard()`'s corrective save pushes — the reverted baseline captured at discard time, not whatever the ambient content ref reads by the time the in-flight save it's correcting for has settled. */ + onSave: (overrideContent?: string) => Promise delay?: number enabled?: boolean + /** + * Uniquely identifies the document being edited (e.g. a file id). When set, the draft is + * mirrored into IndexedDB on a short debounce, independent of the network save, and recovered + * via `onRestoreDraft` on mount if newer than `savedContent`. + */ + draftKey?: string + onRestoreDraft?: (content: string) => void + /** Called if `discard()`'s corrective save fails — the only way that failure can surface, since it happens after the component may already have unmounted. */ + onDiscardCorrectionFailed?: () => void } interface UseAutosaveReturn { saveStatus: SaveStatus saveImmediately: () => Promise isDirty: boolean + /** Abandons the current draft: blocks any save/local-draft write not yet started, clears the local draft immediately, and corrects the server if a save already in flight lands afterward. */ + discard: () => void } /** @@ -29,29 +71,77 @@ export function useAutosave({ onSave, delay = 1500, enabled = true, + draftKey, + onRestoreDraft, + onDiscardCorrectionFailed, }: UseAutosaveOptions): UseAutosaveReturn { const [saveStatus, setSaveStatus] = useState('idle') + const timerRef = useRef>(undefined) const idleTimerRef = useRef>(undefined) const displayTimerRef = useRef>(undefined) const savingRef = useRef(false) + const savingStartRef = useRef(0) + const inFlightRef = useRef | null>(null) + const unmountedRef = useRef(false) const onSaveRef = useRef(onSave) onSaveRef.current = onSave const enabledRef = useRef(enabled) enabledRef.current = enabled + const savedContentRef = useRef(savedContent) savedContentRef.current = savedContent const contentRef = useRef(content) contentRef.current = content + const effectiveDraftKey = enabled ? draftKey : undefined + const draftKeyRef = useRef(effectiveDraftKey) + draftKeyRef.current = effectiveDraftKey + const onRestoreDraftRef = useRef(onRestoreDraft) + onRestoreDraftRef.current = onRestoreDraft + const onDiscardCorrectionFailedRef = useRef(onDiscardCorrectionFailed) + onDiscardCorrectionFailedRef.current = onDiscardCorrectionFailed + + const localDraftTimerRef = useRef>(undefined) + const lastPersistedContentRef = useRef(null) + + const discardedRef = useRef(false) + const isDirty = content !== savedContent - const savingStartRef = useRef(0) - const inFlightRef = useRef | null>(null) - const unmountedRef = useRef(false) - const MIN_SAVING_DISPLAY_MS = 600 + if (discardedRef.current && isDirty) discardedRef.current = false + + const persistLocalDraft = useCallback(() => { + const key = draftKeyRef.current + if (discardedRef.current || !key || contentRef.current === savedContentRef.current) return + if (contentRef.current === lastPersistedContentRef.current) return + const content = contentRef.current + const savedContentSnapshot = savedContentRef.current + void enqueueDraftOp(key, () => + set(localDraftDbKey(key), { + content, + savedContent: savedContentSnapshot, + } satisfies LocalDraft) + ) + .then(() => { + lastPersistedContentRef.current = content + }) + .catch((error) => { + logger.warn('IndexedDB draft write failed', { key, error }) + }) + }, []) + + const clearLocalDraft = useCallback(() => { + const key = draftKeyRef.current + lastPersistedContentRef.current = null + if (!key) return + void enqueueDraftOp(key, () => del(localDraftDbKey(key))).catch((error) => { + logger.warn('IndexedDB draft delete failed', { key, error }) + }) + }, []) const save = useCallback(async () => { if ( + discardedRef.current || !enabledRef.current || savingRef.current || contentRef.current === savedContentRef.current @@ -68,6 +158,7 @@ export function useAutosave({ } catch { nextStatus = 'error' } finally { + inFlightRef.current = null if (unmountedRef.current) { savingRef.current = false } else { @@ -106,23 +197,113 @@ export function useAutosave({ clearTimeout(timerRef.current) clearTimeout(idleTimerRef.current) clearTimeout(displayTimerRef.current) - if (!enabledRef.current || contentRef.current === savedContentRef.current) return + clearTimeout(localDraftTimerRef.current) + persistLocalDraft() + if ( + discardedRef.current || + !enabledRef.current || + contentRef.current === savedContentRef.current + ) { + return + } // Flush the latest content on unmount, but chain it AFTER any in-flight save rather than // firing a concurrent PUT: the in-flight save captured an older snapshot, so writing the // latest sequentially (last) prevents an out-of-order completion from clobbering it. void (async () => { await inFlightRef.current - if (contentRef.current !== savedContentRef.current) { - await onSaveRef.current().catch(() => {}) + if (!discardedRef.current) { + await onSaveRef.current().then(clearLocalDraft, () => {}) } })() } - }, []) + }, [clearLocalDraft, persistLocalDraft]) + + const wasDirtyRef = useRef(isDirty) + + useEffect(() => { + if (effectiveDraftKey && !isDirty && wasDirtyRef.current) clearLocalDraft() + wasDirtyRef.current = isDirty + }, [effectiveDraftKey, isDirty, clearLocalDraft]) + + useEffect(() => { + if (!effectiveDraftKey || !isDirty) return + clearTimeout(localDraftTimerRef.current) + localDraftTimerRef.current = setTimeout(persistLocalDraft, LOCAL_DRAFT_DELAY_MS) + return () => clearTimeout(localDraftTimerRef.current) + }, [content, effectiveDraftKey, isDirty, persistLocalDraft]) + + useEffect(() => { + if (!effectiveDraftKey) return + const handleVisibility = () => { + if (document.visibilityState === 'hidden') persistLocalDraft() + } + window.addEventListener('pagehide', persistLocalDraft) + document.addEventListener('visibilitychange', handleVisibility) + return () => { + window.removeEventListener('pagehide', persistLocalDraft) + document.removeEventListener('visibilitychange', handleVisibility) + } + }, [effectiveDraftKey, persistLocalDraft]) + + const recoveryAttemptedRef = useRef(false) + + useEffect(() => { + if (!effectiveDraftKey || recoveryAttemptedRef.current) return + recoveryAttemptedRef.current = true + let cancelled = false + void enqueueDraftOp(effectiveDraftKey, () => + get(localDraftDbKey(effectiveDraftKey)) + ) + .then((draft) => { + if (cancelled || !draft) return + if (draft.savedContent !== savedContentRef.current) { + clearLocalDraft() + return + } + if (draft.content === draft.savedContent) return + if (contentRef.current !== savedContentRef.current) return + onRestoreDraftRef.current?.(draft.content) + }) + .catch((error) => { + logger.warn('IndexedDB draft read failed', { draftKey: effectiveDraftKey, error }) + }) + return () => { + cancelled = true + } + }, [effectiveDraftKey, clearLocalDraft]) const saveImmediately = useCallback(async () => { clearTimeout(timerRef.current) await save() }, [save]) - return { saveStatus, saveImmediately, isDirty } + const discard = useCallback(() => { + discardedRef.current = true + clearTimeout(timerRef.current) + clearTimeout(localDraftTimerRef.current) + clearLocalDraft() + const pendingSave = inFlightRef.current + if (!pendingSave) return + const target = savedContentRef.current + const contentAtDiscard = contentRef.current + void pendingSave.then(() => { + const current = contentRef.current + if (inFlightRef.current || (current !== target && current !== contentAtDiscard)) return + savingRef.current = true + const correctionRun = onSaveRef + .current(target) + .catch((error) => { + logger.warn('Corrective save after discard failed', { error }) + onDiscardCorrectionFailedRef.current?.() + }) + .finally(() => { + savingRef.current = false + inFlightRef.current = null + }) + inFlightRef.current = correctionRun + return correctionRun + }) + }, [clearLocalDraft]) + + return { saveStatus, saveImmediately, isDirty, discard } } From 5b7513f15d4460ae5a231af7408ee87a69fe01b4 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Thu, 9 Jul 2026 19:38:01 -0700 Subject: [PATCH 09/13] feat(blocks): add block visibility gating (preview blocks + AppConfig reveals) (#5526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(deps): install xlsx from @e965/xlsx npm mirror The dependency was pinned to a direct tarball on cdn.sheetjs.com, which now returns 403 (Cloudflare bot-challenge) to automated clients, breaking bun install in CI. npm's own xlsx is frozen at 0.18.5, so switch to the @e965/xlsx mirror which republishes the identical 0.20.3 CDN build to the npm registry. No code changes needed — all imports use bare 'xlsx'. * feat(blocks): add block visibility gating (preview blocks + AppConfig reveals) * fix(blocks): reset visibility to fail-closed empty state on workspace switch * fix(blocks): carry kill-switch entries across workspace-switch visibility resets * chore(deps): revert stray local xlsx-mirror commit (keep staging's pinned source) * chore(skills): rename gate-block skill to add-block-preview --- .claude/commands/add-block-preview.md | 60 +++++++ .cursor/commands/add-block-preview.md | 60 +++++++ .../app/api/blocks/visibility/route.test.ts | 84 ++++++++++ apps/sim/app/api/blocks/visibility/route.ts | 46 +++++ .../app/workspace/[workspaceId]/layout.tsx | 2 + .../providers/block-visibility-loader.tsx | 41 +++++ .../user-input/hooks/use-mention-data.ts | 6 +- .../panel/components/toolbar/toolbar.tsx | 42 ++++- apps/sim/blocks/custom/client-overlay.ts | 15 +- apps/sim/blocks/integration-matcher.ts | 13 ++ apps/sim/blocks/registry.ts | 68 +++++++- apps/sim/blocks/types.ts | 10 ++ apps/sim/blocks/visibility/client.test.ts | 89 ++++++++++ apps/sim/blocks/visibility/client.ts | 70 ++++++++ apps/sim/blocks/visibility/context.ts | 73 ++++++++ .../blocks/visibility/registry-inert.test.ts | 60 +++++++ apps/sim/blocks/visibility/server-context.ts | 25 +++ apps/sim/blocks/visibility/visibility.test.ts | 149 +++++++++++++++++ .../components/group-detail.tsx | 27 ++- .../utils/permission-check.test.ts | 13 ++ apps/sim/hooks/queries/block-visibility.ts | 31 ++++ .../sim/lib/api/contracts/block-visibility.ts | 37 +++++ apps/sim/lib/copilot/block-visibility.ts | 58 +++++++ apps/sim/lib/copilot/chat/payload.test.ts | 10 ++ apps/sim/lib/copilot/chat/payload.ts | 31 +++- apps/sim/lib/copilot/integration-tools.ts | 61 +++++-- .../tools/handlers/integration-tools.ts | 13 +- apps/sim/lib/copilot/tools/handlers/vfs.ts | 19 ++- .../server/blocks/get-blocks-metadata-tool.ts | 10 ++ apps/sim/lib/copilot/tools/server/router.ts | 37 ++++- apps/sim/lib/copilot/vfs/workspace-vfs.ts | 73 ++++++-- .../lib/core/config/appconfig-rules.test.ts | 69 ++++++++ apps/sim/lib/core/config/appconfig-rules.ts | 80 +++++++++ .../lib/core/config/block-visibility.test.ts | 157 ++++++++++++++++++ apps/sim/lib/core/config/block-visibility.ts | 110 ++++++++++++ apps/sim/lib/core/config/env-flags.ts | 13 ++ apps/sim/lib/core/config/env.ts | 1 + apps/sim/lib/core/config/feature-flags.ts | 60 ++----- apps/sim/lib/search/tool-operations.ts | 9 +- apps/sim/scripts/check-block-registry.ts | 7 +- scripts/check-api-validation-contracts.ts | 4 +- scripts/generate-docs.ts | 32 +++- 42 files changed, 1750 insertions(+), 125 deletions(-) create mode 100644 .claude/commands/add-block-preview.md create mode 100644 .cursor/commands/add-block-preview.md create mode 100644 apps/sim/app/api/blocks/visibility/route.test.ts create mode 100644 apps/sim/app/api/blocks/visibility/route.ts create mode 100644 apps/sim/app/workspace/[workspaceId]/providers/block-visibility-loader.tsx create mode 100644 apps/sim/blocks/visibility/client.test.ts create mode 100644 apps/sim/blocks/visibility/client.ts create mode 100644 apps/sim/blocks/visibility/context.ts create mode 100644 apps/sim/blocks/visibility/registry-inert.test.ts create mode 100644 apps/sim/blocks/visibility/server-context.ts create mode 100644 apps/sim/blocks/visibility/visibility.test.ts create mode 100644 apps/sim/hooks/queries/block-visibility.ts create mode 100644 apps/sim/lib/api/contracts/block-visibility.ts create mode 100644 apps/sim/lib/copilot/block-visibility.ts create mode 100644 apps/sim/lib/core/config/appconfig-rules.test.ts create mode 100644 apps/sim/lib/core/config/appconfig-rules.ts create mode 100644 apps/sim/lib/core/config/block-visibility.test.ts create mode 100644 apps/sim/lib/core/config/block-visibility.ts diff --git a/.claude/commands/add-block-preview.md b/.claude/commands/add-block-preview.md new file mode 100644 index 00000000000..8345170767f --- /dev/null +++ b/.claude/commands/add-block-preview.md @@ -0,0 +1,60 @@ +--- +description: Gate a block's visibility — ship an unreleased block as a preview (hidden until revealed via AppConfig/env), reveal it to admins/orgs, GA it, or kill-switch a shipped block +argument-hint: +--- + +# Add Block Preview Skill + +You manage **block visibility gating** in Sim — hiding blocks from every discovery surface (toolbar, cmd+K search, copilot @-mentions, agent tool picker, mothership VFS/metadata/tools, Access Control list, public docs/catalog) while **never** gating execution of already-placed instances. + +## The model + +Three levers, evaluated in `apps/sim/lib/core/config/block-visibility.ts` and folded into the registry accessors (`apps/sim/blocks/registry.ts`): + +1. **`preview: true`** on the `BlockConfig` (static, in code) — the block is default-hidden EVERYWHERE (hosted, self-hosted, dev, SSR) until revealed. Fail-closed. +2. **The hosted `block-visibility` AppConfig document** — per-block rule keyed by the existing block type: + + ```jsonc + { + "": { + "enabled": false, // required. true = GA (visible to everyone) + "orgIds": ["org_..."], // optional allowlist clauses (any match reveals) + "userIds": ["user_..."], + "adminEnabled": true // platform admins (user.role === 'admin') + } + } + ``` + +3. **`PREVIEW_BLOCKS` env** (comma-separated block types) — the off-AppConfig reveal path for self-hosters and local dev. + +A revealed block that is not globally GA (`enabled !== true`, or env-revealed) renders with a **" (Preview)"** name suffix on discovery surfaces. `getBlock()` stays pure, so placed instances keep their canonical name and always execute. + +## Lifecycle of a preview block + +1. **Author** the block normally (`/add-block` etc.) and set `preview: true` on its `BlockConfig`. **Ship no `BlockMeta` and no docs until GA** — `check-block-registry` deliberately skips preview blocks in meta coverage, and `generate-docs` skips them at every gate. +2. **Local dev:** set `PREVIEW_BLOCKS=` in your env to see it (with the suffix). +3. **Merge/deploy.** The block's code is live everywhere but visible nowhere — no AppConfig rule exists and self-hosters have no env entry. +4. **Hosted preview:** add a rule to the `block-visibility` AppConfig document and start a deployment (no code deploy): + - Admins only: `{ "enabled": false, "adminEnabled": true }` + - Design-partner org: `{ "enabled": false, "orgIds": ["org_123"] }` + - GA via config (code cleanup pending): `{ "enabled": true }` — suffix disappears everywhere within ~30s (AppConfig TTL) + client refetch. + + Same runbook as `feature-flags`: edit the hosted document, `aws appconfig start-deployment` with the `sim--fast` strategy (see the infra README). +5. **GA cleanup:** delete `preview: true` from the block (now visible to self-hosters on their next upgrade), add its `BlockMeta` + regen docs, and drop the AppConfig entry. For a v2 upgrade, this is also when v1 gets `hideFromToolbar: true` (the superseded-version paradigm). + +## Kill switch (shipped blocks) + +To pull an already-GA block from discovery surfaces on hosted (incident, deprecation): add `{ "": { "enabled": false } }` to the document. Allowlist clauses can carve out exceptions. **Execution is NOT stopped** — workflows already using the block keep running; the kill switch only prevents new placement/discovery. + +## Invariants (do not violate) + +- **Execution is never gated.** The executor, serializer, drop-naming, and `isBlockTypeAccessControlExempt` resolve via pure `getBlock`. Do not add visibility checks to execution paths. +- **Clone-not-remove:** gated blocks stay in `getAllBlocks()` output as clones with `hideFromToolbar: true` — `.find`-by-type consumers rely on this. Never filter them out. +- **Keys are registry block types.** Never `custom_block_*` (parse drops them — custom blocks have their own enabled/disabled lifecycle). +- **The shared hidden-predicate is `isHiddenUnder`** (`apps/sim/blocks/visibility/context.ts`). Never restate the preview/disabled rule inline at a new consumer. +- **Process-global caches stay ungated.** `getStaticComponentFiles` (VFS) and `getExposedIntegrationTools` build the ungated universe; per-viewer filtering happens at stamp/consumer time. Never move gating into a shared builder. +- Gating is **surface hiding, not secrecy** — the full config ships in the client JS bundle. Anything truly secret cannot be a registered block. + +## Tests + +Evaluation semantics: `apps/sim/lib/core/config/block-visibility.test.ts`. Registry projection: `apps/sim/blocks/visibility/visibility.test.ts`. When gating behavior changes, extend those — mock `isPlatformAdmin` for the admin clause; use the local `withAppConfig` harness. diff --git a/.cursor/commands/add-block-preview.md b/.cursor/commands/add-block-preview.md new file mode 100644 index 00000000000..8345170767f --- /dev/null +++ b/.cursor/commands/add-block-preview.md @@ -0,0 +1,60 @@ +--- +description: Gate a block's visibility — ship an unreleased block as a preview (hidden until revealed via AppConfig/env), reveal it to admins/orgs, GA it, or kill-switch a shipped block +argument-hint: +--- + +# Add Block Preview Skill + +You manage **block visibility gating** in Sim — hiding blocks from every discovery surface (toolbar, cmd+K search, copilot @-mentions, agent tool picker, mothership VFS/metadata/tools, Access Control list, public docs/catalog) while **never** gating execution of already-placed instances. + +## The model + +Three levers, evaluated in `apps/sim/lib/core/config/block-visibility.ts` and folded into the registry accessors (`apps/sim/blocks/registry.ts`): + +1. **`preview: true`** on the `BlockConfig` (static, in code) — the block is default-hidden EVERYWHERE (hosted, self-hosted, dev, SSR) until revealed. Fail-closed. +2. **The hosted `block-visibility` AppConfig document** — per-block rule keyed by the existing block type: + + ```jsonc + { + "": { + "enabled": false, // required. true = GA (visible to everyone) + "orgIds": ["org_..."], // optional allowlist clauses (any match reveals) + "userIds": ["user_..."], + "adminEnabled": true // platform admins (user.role === 'admin') + } + } + ``` + +3. **`PREVIEW_BLOCKS` env** (comma-separated block types) — the off-AppConfig reveal path for self-hosters and local dev. + +A revealed block that is not globally GA (`enabled !== true`, or env-revealed) renders with a **" (Preview)"** name suffix on discovery surfaces. `getBlock()` stays pure, so placed instances keep their canonical name and always execute. + +## Lifecycle of a preview block + +1. **Author** the block normally (`/add-block` etc.) and set `preview: true` on its `BlockConfig`. **Ship no `BlockMeta` and no docs until GA** — `check-block-registry` deliberately skips preview blocks in meta coverage, and `generate-docs` skips them at every gate. +2. **Local dev:** set `PREVIEW_BLOCKS=` in your env to see it (with the suffix). +3. **Merge/deploy.** The block's code is live everywhere but visible nowhere — no AppConfig rule exists and self-hosters have no env entry. +4. **Hosted preview:** add a rule to the `block-visibility` AppConfig document and start a deployment (no code deploy): + - Admins only: `{ "enabled": false, "adminEnabled": true }` + - Design-partner org: `{ "enabled": false, "orgIds": ["org_123"] }` + - GA via config (code cleanup pending): `{ "enabled": true }` — suffix disappears everywhere within ~30s (AppConfig TTL) + client refetch. + + Same runbook as `feature-flags`: edit the hosted document, `aws appconfig start-deployment` with the `sim--fast` strategy (see the infra README). +5. **GA cleanup:** delete `preview: true` from the block (now visible to self-hosters on their next upgrade), add its `BlockMeta` + regen docs, and drop the AppConfig entry. For a v2 upgrade, this is also when v1 gets `hideFromToolbar: true` (the superseded-version paradigm). + +## Kill switch (shipped blocks) + +To pull an already-GA block from discovery surfaces on hosted (incident, deprecation): add `{ "": { "enabled": false } }` to the document. Allowlist clauses can carve out exceptions. **Execution is NOT stopped** — workflows already using the block keep running; the kill switch only prevents new placement/discovery. + +## Invariants (do not violate) + +- **Execution is never gated.** The executor, serializer, drop-naming, and `isBlockTypeAccessControlExempt` resolve via pure `getBlock`. Do not add visibility checks to execution paths. +- **Clone-not-remove:** gated blocks stay in `getAllBlocks()` output as clones with `hideFromToolbar: true` — `.find`-by-type consumers rely on this. Never filter them out. +- **Keys are registry block types.** Never `custom_block_*` (parse drops them — custom blocks have their own enabled/disabled lifecycle). +- **The shared hidden-predicate is `isHiddenUnder`** (`apps/sim/blocks/visibility/context.ts`). Never restate the preview/disabled rule inline at a new consumer. +- **Process-global caches stay ungated.** `getStaticComponentFiles` (VFS) and `getExposedIntegrationTools` build the ungated universe; per-viewer filtering happens at stamp/consumer time. Never move gating into a shared builder. +- Gating is **surface hiding, not secrecy** — the full config ships in the client JS bundle. Anything truly secret cannot be a registered block. + +## Tests + +Evaluation semantics: `apps/sim/lib/core/config/block-visibility.test.ts`. Registry projection: `apps/sim/blocks/visibility/visibility.test.ts`. When gating behavior changes, extend those — mock `isPlatformAdmin` for the admin clause; use the local `withAppConfig` harness. diff --git a/apps/sim/app/api/blocks/visibility/route.test.ts b/apps/sim/app/api/blocks/visibility/route.test.ts new file mode 100644 index 00000000000..1a59df82aa5 --- /dev/null +++ b/apps/sim/app/api/blocks/visibility/route.test.ts @@ -0,0 +1,84 @@ +/** + * @vitest-environment node + */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockGetSession, mockCheckWorkspaceAccess, mockIsPlatformAdmin, mockGetBlockVisibility } = + vi.hoisted(() => ({ + mockGetSession: vi.fn(), + mockCheckWorkspaceAccess: vi.fn(), + mockIsPlatformAdmin: vi.fn(), + mockGetBlockVisibility: vi.fn(), + })) + +vi.mock('@/lib/auth', () => ({ + auth: { api: { getSession: vi.fn() } }, + getSession: mockGetSession, +})) + +vi.mock('@/lib/workspaces/permissions/utils', () => ({ + checkWorkspaceAccess: mockCheckWorkspaceAccess, +})) + +vi.mock('@/lib/permissions/super-user', () => ({ + isPlatformAdmin: mockIsPlatformAdmin, +})) + +vi.mock('@/lib/core/config/block-visibility', () => ({ + getBlockVisibility: mockGetBlockVisibility, +})) + +import { GET } from '@/app/api/blocks/visibility/route' + +const WORKSPACE_ID = '11111111-2222-4333-8444-555555555555' + +function request(workspaceId = WORKSPACE_ID) { + return new NextRequest(`http://localhost/api/blocks/visibility?workspaceId=${workspaceId}`) +} + +describe('GET /api/blocks/visibility', () => { + beforeEach(() => { + vi.clearAllMocks() + mockGetSession.mockResolvedValue({ user: { id: 'user-1' } }) + mockCheckWorkspaceAccess.mockResolvedValue({ + hasAccess: true, + workspace: { organizationId: 'org-1' }, + }) + mockIsPlatformAdmin.mockResolvedValue(false) + mockGetBlockVisibility.mockResolvedValue({ + revealed: new Set(['gmail_v2']), + disabled: new Set(['slack']), + previewTagged: new Set(['gmail_v2']), + }) + }) + + it('returns 401 without a session', async () => { + mockGetSession.mockResolvedValue(null) + const response = await GET(request()) + expect(response.status).toBe(401) + }) + + it('returns 403 without workspace access', async () => { + mockCheckWorkspaceAccess.mockResolvedValue({ hasAccess: false, workspace: null }) + const response = await GET(request()) + expect(response.status).toBe(403) + expect(mockGetBlockVisibility).not.toHaveBeenCalled() + }) + + it('evaluates visibility for the session user, workspace org, and pre-resolved admin', async () => { + mockIsPlatformAdmin.mockResolvedValue(true) + const response = await GET(request()) + expect(response.status).toBe(200) + expect(await response.json()).toEqual({ + revealed: ['gmail_v2'], + disabled: ['slack'], + previewTagged: ['gmail_v2'], + }) + expect(mockGetBlockVisibility).toHaveBeenCalledWith({ + userId: 'user-1', + orgId: 'org-1', + isAdmin: true, + }) + }) +}) diff --git a/apps/sim/app/api/blocks/visibility/route.ts b/apps/sim/app/api/blocks/visibility/route.ts new file mode 100644 index 00000000000..8aea5ad77b3 --- /dev/null +++ b/apps/sim/app/api/blocks/visibility/route.ts @@ -0,0 +1,46 @@ +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { getBlockVisibilityContract } from '@/lib/api/contracts/block-visibility' +import { parseRequest } from '@/lib/api/server' +import { getSession } from '@/lib/auth' +import { getBlockVisibility } from '@/lib/core/config/block-visibility' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { isPlatformAdmin } from '@/lib/permissions/super-user' +import { checkWorkspaceAccess } from '@/lib/workspaces/permissions/utils' + +/** + * Evaluates the viewer's block-visibility projection for a workspace: which + * preview blocks are revealed (and preview-tagged) and which shipped blocks are + * kill-switched. Consumed by the client visibility overlay + * (`BlockVisibilityLoader`); discovery-only — execution is never gated. + */ +export const GET = withRouteHandler(async (request: NextRequest) => { + const session = await getSession() + if (!session?.user?.id) { + return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }) + } + + const parsed = await parseRequest(getBlockVisibilityContract, request, {}) + if (!parsed.success) return parsed.response + + const userId = session.user.id + const { workspaceId } = parsed.data.query + + const access = await checkWorkspaceAccess(workspaceId, userId) + if (!access.hasAccess) { + return NextResponse.json({ error: 'Access denied' }, { status: 403 }) + } + + const isAdmin = await isPlatformAdmin(userId) + const vis = await getBlockVisibility({ + userId, + orgId: access.workspace?.organizationId, + isAdmin, + }) + + return NextResponse.json({ + revealed: [...vis.revealed], + disabled: [...vis.disabled], + previewTagged: [...vis.previewTagged], + }) +}) diff --git a/apps/sim/app/workspace/[workspaceId]/layout.tsx b/apps/sim/app/workspace/[workspaceId]/layout.tsx index 4fdea0c52d6..b770cd4f076 100644 --- a/apps/sim/app/workspace/[workspaceId]/layout.tsx +++ b/apps/sim/app/workspace/[workspaceId]/layout.tsx @@ -7,6 +7,7 @@ import { getQueryClient } from '@/app/_shell/providers/get-query-client' import { ImpersonationBanner } from '@/app/workspace/[workspaceId]/components/impersonation-banner' import { WorkspaceChrome } from '@/app/workspace/[workspaceId]/components/workspace-chrome' import { prefetchWorkspaceSidebar } from '@/app/workspace/[workspaceId]/prefetch' +import { BlockVisibilityLoader } from '@/app/workspace/[workspaceId]/providers/block-visibility-loader' import { CustomBlocksLoader } from '@/app/workspace/[workspaceId]/providers/custom-blocks-loader' import { GlobalCommandsProvider } from '@/app/workspace/[workspaceId]/providers/global-commands-provider' import { ProviderModelsLoader } from '@/app/workspace/[workspaceId]/providers/provider-models-loader' @@ -45,6 +46,7 @@ export default async function WorkspaceLayout({ +
diff --git a/apps/sim/app/workspace/[workspaceId]/providers/block-visibility-loader.tsx b/apps/sim/app/workspace/[workspaceId]/providers/block-visibility-loader.tsx new file mode 100644 index 00000000000..2b4e2513fc2 --- /dev/null +++ b/apps/sim/app/workspace/[workspaceId]/providers/block-visibility-loader.tsx @@ -0,0 +1,41 @@ +'use client' + +import { useEffect } from 'react' +import { useParams } from 'next/navigation' +import { hydrateBlockVisibility, resetBlockVisibilityForSwitch } from '@/blocks/visibility/client' +import { useBlockVisibility } from '@/hooks/queries/block-visibility' + +/** + * Hydrates the client block-visibility overlay for the active workspace so the + * registry accessors project the viewer's revealed/disabled preview blocks. + * Mounted once in the workspace layout, next to `CustomBlocksLoader`. + * + * First paint needs no prefetch: `preview: true` blocks are fail-closed until + * this hydrate lands, so the fetch only ever reveals (benign pop-in) or applies + * a kill switch to an already-public block. Identical refetches are absorbed by + * the deep-equal guard inside `hydrateBlockVisibility`. + */ +export function BlockVisibilityLoader() { + const params = useParams() + const workspaceId = params?.workspaceId as string | undefined + const { data } = useBlockVisibility(workspaceId) + + useEffect(() => { + // On a workspace switch the query key changes and `data` is undefined while + // the new projection loads — reset fail-closed so the previous workspace's + // preview reveals never linger across orgs, while carrying kill-switch + // entries over so disabled blocks don't flash back during the flight + // window. No-ops on first mount (nothing hydrated yet). + if (!data) { + resetBlockVisibilityForSwitch() + return + } + hydrateBlockVisibility({ + revealed: new Set(data.revealed), + disabled: new Set(data.disabled), + previewTagged: new Set(data.previewTagged), + }) + }, [data]) + + return null +} diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-mention-data.ts b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-mention-data.ts index 95508ac4102..d0d0a1a43a9 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-mention-data.ts +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/copilot/components/user-input/hooks/use-mention-data.ts @@ -7,6 +7,7 @@ import { requestJson } from '@/lib/api/client/request' import { listCopilotChatsContract } from '@/lib/api/contracts/copilot' import { listKnowledgeBasesContract } from '@/lib/api/contracts/knowledge/base' import { listLogsContract } from '@/lib/api/contracts/logs' +import { useCustomBlockOverlayVersion } from '@/blocks/custom/client-overlay' import { type IntegrationDescriptor, listIntegrations } from '@/blocks/integration-matcher' import { useWorkflows } from '@/hooks/queries/workflows' import { usePermissionConfig } from '@/hooks/use-permission-config' @@ -130,9 +131,12 @@ export function useMentionData(props: UseMentionDataProps): MentionDataReturn { const [blocksList, setBlocksList] = useState([]) const [isLoadingBlocks, setIsLoadingBlocks] = useState(false) + // Reset on permission changes and on block-overlay bumps (custom-block or + // block-visibility hydrate) so late preview reveals refresh the folder. + const blockOverlayVersion = useCustomBlockOverlayVersion() useEffect(() => { setBlocksList([]) - }, [config.allowedIntegrations]) + }, [config.allowedIntegrations, blockOverlayVersion]) const [logsList, setLogsList] = useState([]) const [isLoadingLogs, setIsLoadingLogs] = useState(false) diff --git a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/toolbar/toolbar.tsx b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/toolbar/toolbar.tsx index e051f58379f..9118eef024a 100644 --- a/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/toolbar/toolbar.tsx +++ b/apps/sim/app/workspace/[workspaceId]/w/[workflowId]/components/panel/components/toolbar/toolbar.tsx @@ -35,6 +35,7 @@ import { CUSTOM_BLOCK_TILE_COLOR, isCustomBlockType, } from '@/blocks/custom/build-config' +import { useCustomBlockOverlayVersion } from '@/blocks/custom/client-overlay' import { getCustomBlockIcon } from '@/blocks/custom/custom-block-icon' import { getTileIconColorClass } from '@/blocks/icon-color' import { getCanonicalBlocksByCategory } from '@/blocks/registry' @@ -154,11 +155,29 @@ const ToolbarItem = memo(function ToolbarItem({ let cachedTriggers: BlockItem[] | null = null /** - * Gets triggers data, computing it once and caching for subsequent calls. - * Non-integration triggers (Start, Schedule, Webhook) are prioritized first, - * followed by all other triggers sorted alphabetically. + * Block-overlay version the caches below were built against. The registry's + * output is no longer static — a block-visibility hydrate (preview reveal / + * kill switch) bumps the shared overlay version — so the caches are keyed to + * it and dropped when it moves. -1 = never built. */ -function getTriggers(): BlockItem[] { +let cachedAtOverlayVersion = -1 + +/** Drop all three caches when the overlay version moved since they were built. */ +function syncCachesToOverlayVersion(version: number) { + if (cachedAtOverlayVersion === version) return + cachedAtOverlayVersion = version + cachedTriggers = null + cachedBlocks = null + cachedTools = null +} + +/** + * Gets triggers data, computing it once per overlay version and caching for + * subsequent calls. Non-integration triggers (Start, Schedule, Webhook) are + * prioritized first, followed by all other triggers sorted alphabetically. + */ +function getTriggers(overlayVersion: number): BlockItem[] { + syncCachesToOverlayVersion(overlayVersion) if (cachedTriggers === null) { const allTriggers = getTriggersForSidebar() const priorityOrder = ['Start', 'Schedule', 'Webhook'] @@ -250,12 +269,14 @@ function ensureBlockCaches() { cachedTools = toolItems } -function getBlocks(): BlockItem[] { +function getBlocks(overlayVersion: number): BlockItem[] { + syncCachesToOverlayVersion(overlayVersion) ensureBlockCaches() return cachedBlocks as BlockItem[] } -function getTools(): BlockItem[] { +function getTools(overlayVersion: number): BlockItem[] { + syncCachesToOverlayVersion(overlayVersion) ensureBlockCaches() return cachedTools as BlockItem[] } @@ -455,9 +476,12 @@ export const Toolbar = memo( const { data: whitelabel } = useWhitelabelSettings(customBlocksData?.[0]?.organizationId) const fallbackIconUrl = whitelabel?.logoUrl ?? null - const allTriggers = getTriggers() - const allBlocks = getBlocks() - const allTools = getTools() + // Re-read the block lists whenever the overlay version bumps (custom-block + // or block-visibility hydrate) — the module caches are keyed to it. + const blockOverlayVersion = useCustomBlockOverlayVersion() + const allTriggers = getTriggers(blockOverlayVersion) + const allBlocks = getBlocks(blockOverlayVersion) + const allTools = getTools(blockOverlayVersion) // Published custom blocks are their own section. Exclude disabled blocks (still // resolvable so placed instances survive, but not offered for new placement) and diff --git a/apps/sim/blocks/custom/client-overlay.ts b/apps/sim/blocks/custom/client-overlay.ts index cc1bbff01e1..4dfab2aac94 100644 --- a/apps/sim/blocks/custom/client-overlay.ts +++ b/apps/sim/blocks/custom/client-overlay.ts @@ -24,11 +24,22 @@ registerBlockOverlayResolver({ all: () => [...map.values()], }) +/** + * Bump the overlay version and notify subscribers that block-registry-derived + * data changed. Shared signal for BOTH custom-block hydration and the + * block-visibility hydrate path — subscribers treat the version as an opaque + * "re-read `getAllBlocks()`" token, never as "custom blocks specifically + * changed". + */ +export function notifyBlockOverlayChanged(): void { + version += 1 + for (const listener of listeners) listener() +} + /** Replace the in-scope custom blocks and notify subscribers. */ export function hydrateClientCustomBlocks(configs: BlockConfig[]): void { map = new Map(configs.map((config) => [config.type, config])) - version += 1 - for (const listener of listeners) listener() + notifyBlockOverlayChanged() } function subscribe(listener: () => void): () => void { diff --git a/apps/sim/blocks/integration-matcher.ts b/apps/sim/blocks/integration-matcher.ts index 148d5cc6a3d..172d07e9be2 100644 --- a/apps/sim/blocks/integration-matcher.ts +++ b/apps/sim/blocks/integration-matcher.ts @@ -1,6 +1,7 @@ import { LandingPromptStorage } from '@/lib/core/utils/browser-storage' import { getCanonicalBlocksByCategory } from '@/blocks/registry' import type { BlockIcon } from '@/blocks/types' +import { registerBlockCacheInvalidator } from '@/blocks/visibility/context' /** * Public descriptor for a single integration block, exposed to UI surfaces @@ -46,6 +47,18 @@ function normalizeDisplayName(name: string): string { let cachedMatcher: IntegrationMatcher | null = null let cachedList: readonly IntegrationDescriptor[] | null = null +/** + * Drops the memoized matcher/list. Registered with the block cache-invalidator + * seam so a client block-visibility change (preview reveal / kill switch) + * rebuilds the matcher against the new canonical block set. + */ +function clearIntegrationMatcherCache(): void { + cachedMatcher = null + cachedList = null +} + +registerBlockCacheInvalidator(clearIntegrationMatcherCache) + function buildMatcher(): IntegrationMatcher { const byName = new Map() const names: string[] = [] diff --git a/apps/sim/blocks/registry.ts b/apps/sim/blocks/registry.ts index e7760a87974..d0019e28aaf 100644 --- a/apps/sim/blocks/registry.ts +++ b/apps/sim/blocks/registry.ts @@ -1,4 +1,5 @@ import { stripVersionSuffix } from '@sim/utils/string' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' import { overlayBlocks, resolveOverlayBlock } from '@/blocks/custom/overlay' import { BLOCK_META_REGISTRY, BLOCK_REGISTRY } from '@/blocks/registry-maps' import type { @@ -8,6 +9,7 @@ import type { BlockTemplate, SuggestedSkill, } from '@/blocks/types' +import { isHiddenUnder, overlayVisibility } from '@/blocks/visibility/context' /** * Normalize an external block type to its registry key form: dashes become @@ -22,9 +24,58 @@ export function getBlock(type: string): BlockConfig | undefined { return BLOCK_REGISTRY[type] ?? BLOCK_REGISTRY[normalizeType(type)] ?? resolveOverlayBlock(type) } -/** All block configs, including any in-scope custom blocks from the overlay. */ +/** Whether any registered block is an unreleased `preview` block. Static — computed once. */ +const HAS_PREVIEW_BLOCKS = Object.values(BLOCK_REGISTRY).some((block) => block.preview) + +/** + * True when the visibility projection cannot change any block, so accessors can + * return raw arrays untouched: no `preview` blocks exist (they must be hidden + * even with a null state) and no kill-switch entries apply. + */ +function visibilityInert(vis: BlockVisibilityState | null): boolean { + if (HAS_PREVIEW_BLOCKS) return false + return vis === null || vis.disabled.size === 0 +} + +/** + * Effective hidden state for discovery surfaces: static `hideFromToolbar` + * (superseded versions, disabled custom blocks) plus the per-viewer visibility + * predicate ({@link isHiddenUnder}: unrevealed `preview` blocks — fail-closed + * even without a context — and kill-switched types). + */ +function effectiveHidden(block: BlockConfig, vis: BlockVisibilityState | null): boolean { + if (block.hideFromToolbar) return true + return isHiddenUnder(vis, block) +} + +/** + * Project a block through the viewer's visibility: gated blocks become shallow + * clones with `hideFromToolbar: true` (CLONE-NOT-REMOVE — gated blocks must stay + * in `getAllBlocks()` output because `.find`-by-type consumers rely on it), and + * revealed-but-not-GA preview blocks get a display " (Preview)" name suffix. + * The `!block.hideFromToolbar` guard keeps already-hidden blocks (including + * disabled custom blocks) un-cloned and never suffixed. + */ +function projectBlock(block: BlockConfig, vis: BlockVisibilityState | null): BlockConfig { + if (effectiveHidden(block, vis) && !block.hideFromToolbar) { + return { ...block, hideFromToolbar: true } + } + if (block.preview && vis?.previewTagged.has(block.type)) { + return { ...block, name: `${block.name} (Preview)` } + } + return block +} + +/** + * All block configs, including any in-scope custom blocks from the overlay, + * projected through the viewer's block visibility. Execution paths are + * unaffected: they resolve via the pure {@link getBlock}. + */ export function getAllBlocks(): BlockConfig[] { - return [...Object.values(BLOCK_REGISTRY), ...overlayBlocks()] + const all = [...Object.values(BLOCK_REGISTRY), ...overlayBlocks()] + const vis = overlayVisibility() + if (visibilityInert(vis)) return all + return all.map((block) => projectBlock(block, vis)) } /** Find the block whose `tools.access` contains the given tool id. */ @@ -77,14 +128,17 @@ export function getBlocksByCategory(category: BlockCategory): BlockConfig[] { * category. This is the single source of truth shared by every surface that * extracts blocks for presentation — the toolbar, the search/mention engine, * and the integrations catalog. A block is included when its `category` - * matches and it is not hidden from the toolbar (i.e. it is the latest - * version under the upgrade paradigm, since superseded versions set - * `hideFromToolbar: true`). + * matches and it is not effectively hidden: not `hideFromToolbar` (superseded + * versions), not an unrevealed `preview` block, and not kill-switched for the + * viewer. Visible blocks are projected so revealed preview blocks carry their + * " (Preview)" display suffix. */ export function getCanonicalBlocksByCategory(category: BlockCategory): BlockConfig[] { - return [...Object.values(BLOCK_REGISTRY), ...overlayBlocks()].filter( - (block) => block.category === category && !block.hideFromToolbar + const vis = overlayVisibility() + const blocks = [...Object.values(BLOCK_REGISTRY), ...overlayBlocks()].filter( + (block) => block.category === category && !effectiveHidden(block, vis) ) + return visibilityInert(vis) ? blocks : blocks.map((block) => projectBlock(block, vis)) } /** All registered block type identifiers. */ diff --git a/apps/sim/blocks/types.ts b/apps/sim/blocks/types.ts index 7e3f9636bef..6d9f63188ab 100644 --- a/apps/sim/blocks/types.ts +++ b/apps/sim/blocks/types.ts @@ -462,6 +462,16 @@ export interface BlockConfig { } } hideFromToolbar?: boolean + /** + * Marks an unreleased block. Preview blocks are hidden from every discovery + * surface (toolbar, search, mentions, copilot/VFS, docs) in every environment — + * hosted, self-hosted, dev, and SSR — until revealed via the hosted + * `block-visibility` AppConfig document or the `PREVIEW_BLOCKS` env allowlist. + * Fail-closed by design; distinct from {@link hideFromToolbar} (permanently + * hidden superseded versions). Execution of already-placed instances is never + * gated. Remove at GA. + */ + preview?: boolean triggers?: { enabled: boolean available: string[] // List of trigger IDs this block supports diff --git a/apps/sim/blocks/visibility/client.test.ts b/apps/sim/blocks/visibility/client.test.ts new file mode 100644 index 00000000000..426a099da54 --- /dev/null +++ b/apps/sim/blocks/visibility/client.test.ts @@ -0,0 +1,89 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockNotify } = vi.hoisted(() => ({ mockNotify: vi.fn() })) + +vi.mock('@/blocks/custom/client-overlay', () => ({ + notifyBlockOverlayChanged: mockNotify, +})) + +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +// client-boundary-allow: vitest ignores the 'use client' directive; this node-env test exercises the module directly +import { hydrateBlockVisibility, resetBlockVisibilityForSwitch } from '@/blocks/visibility/client' +import { overlayVisibility, registerBlockCacheInvalidator } from '@/blocks/visibility/context' + +function state(revealed: string[], disabled: string[] = []): BlockVisibilityState { + return { + revealed: new Set(revealed), + disabled: new Set(disabled), + previewTagged: new Set(revealed), + } +} + +describe('hydrateBlockVisibility', () => { + beforeEach(() => vi.clearAllMocks()) + + // Must run FIRST: module state starts null and persists across tests. + it('treats an empty state while none is set as a no-op (null ≡ empty)', () => { + const invalidator = vi.fn() + const unregister = registerBlockCacheInvalidator(invalidator) + + hydrateBlockVisibility(state([])) + expect(overlayVisibility()).toBeNull() + expect(invalidator).not.toHaveBeenCalled() + expect(mockNotify).not.toHaveBeenCalled() + + unregister() + }) + + it('applies state, fires invalidators, and bumps the overlay version', () => { + const invalidator = vi.fn() + const unregister = registerBlockCacheInvalidator(invalidator) + + hydrateBlockVisibility(state(['gmail_v2'])) + expect(overlayVisibility()?.revealed.has('gmail_v2')).toBe(true) + expect(invalidator).toHaveBeenCalledTimes(1) + expect(mockNotify).toHaveBeenCalledTimes(1) + + unregister() + }) + + it('drops reveals but carries kill switches over on workspace switch', () => { + const invalidator = vi.fn() + const unregister = registerBlockCacheInvalidator(invalidator) + + // Reveal + kill-switch in workspace A, then switch (loader resets while the + // new projection loads). (notion_v3: distinct from prior tests' state — + // module state persists.) + hydrateBlockVisibility(state(['notion_v3'], ['slack'])) + resetBlockVisibilityForSwitch() + expect(overlayVisibility()?.revealed.size).toBe(0) + expect(overlayVisibility()?.previewTagged.size).toBe(0) + expect(overlayVisibility()?.disabled).toEqual(new Set(['slack'])) + expect(invalidator).toHaveBeenCalledTimes(2) + + // Repeated resets are no-ops (deep-equal). + resetBlockVisibilityForSwitch() + expect(invalidator).toHaveBeenCalledTimes(2) + + unregister() + }) + + it('no-ops on a deep-equal state (fresh objects, same content)', () => { + const invalidator = vi.fn() + const unregister = registerBlockCacheInvalidator(invalidator) + + hydrateBlockVisibility(state(['gmail_v2'], ['slack'])) + hydrateBlockVisibility(state(['gmail_v2'], ['slack'])) + expect(invalidator).toHaveBeenCalledTimes(1) + expect(mockNotify).toHaveBeenCalledTimes(1) + + hydrateBlockVisibility(state(['gmail_v2', 'notion_v3'], ['slack'])) + expect(invalidator).toHaveBeenCalledTimes(2) + expect(mockNotify).toHaveBeenCalledTimes(2) + + unregister() + }) +}) diff --git a/apps/sim/blocks/visibility/client.ts b/apps/sim/blocks/visibility/client.ts new file mode 100644 index 00000000000..4079c91a5e0 --- /dev/null +++ b/apps/sim/blocks/visibility/client.ts @@ -0,0 +1,70 @@ +'use client' + +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import { notifyBlockOverlayChanged } from '@/blocks/custom/client-overlay' +import { invalidateBlockCaches, registerBlockVisibilityResolver } from '@/blocks/visibility/context' + +/** + * Client-side visibility state, hydrated from `useBlockVisibility` by + * `BlockVisibilityLoader`. Registered at module load with `null` state so the + * very first render — including the SSR pass — is fail-closed for `preview` + * blocks; the post-mount fetch only ever reveals (benign pop-in) or applies a + * kill switch to an already-public block. + */ +let state: BlockVisibilityState | null = null + +registerBlockVisibilityResolver({ current: () => state }) + +function setsEqual(a: Set, b: Set): boolean { + if (a.size !== b.size) return false + for (const value of a) if (!b.has(value)) return false + return true +} + +function isEmptyState(vis: BlockVisibilityState): boolean { + return vis.revealed.size === 0 && vis.disabled.size === 0 && vis.previewTagged.size === 0 +} + +/** + * Replace the in-scope visibility state, reset registered module caches, and + * bump the shared block-overlay version so every subscribed consumer re-reads + * `getAllBlocks()`. + * + * No-ops when the change cannot alter the projection: an incoming state + * deep-equal to the current one (React Query refetches deliver + * fresh-but-identical objects on every poll — without this guard each poll + * would thundering-rebuild the toolbar, search, and matcher caches), or an + * empty state while none is set (`null` and empty are equivalent for + * `isHiddenUnder`, so the fail-closed reset on first mount is free). + */ +export function hydrateBlockVisibility(next: BlockVisibilityState): void { + if (state === null && isEmptyState(next)) return + if ( + state && + setsEqual(state.revealed, next.revealed) && + setsEqual(state.disabled, next.disabled) && + setsEqual(state.previewTagged, next.previewTagged) + ) { + return + } + state = next + invalidateBlockCaches() + notifyBlockOverlayChanged() +} + +/** + * Fail-closed reset while a workspace switch's visibility fetch is in flight: + * preview reveals are dropped immediately (they may not apply to the new + * workspace), but kill-switch entries are CARRIED OVER until the new + * projection arrives — dropping `disabled` would flash kill-switched blocks + * back into discovery for the flight window, while briefly over-hiding in the + * new workspace is benign in both directions. + */ +export function resetBlockVisibilityForSwitch(): void { + if (state === null) return + hydrateBlockVisibility({ + revealed: new Set(), + disabled: state.disabled, + previewTagged: new Set(), + }) +} diff --git a/apps/sim/blocks/visibility/context.ts b/apps/sim/blocks/visibility/context.ts new file mode 100644 index 00000000000..7c191603035 --- /dev/null +++ b/apps/sim/blocks/visibility/context.ts @@ -0,0 +1,73 @@ +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import type { BlockConfig } from '@/blocks/types' + +/** + * Resolver for the per-viewer block-visibility projection, mirroring the + * custom-block overlay seam (`@/blocks/custom/overlay`) but deliberately + * independent of it: visibility is a discovery concern with its own lifecycle, + * and a separate AsyncLocalStorage composes with `withCustomBlockOverlay` + * without `store.run` clobbering. + * + * Two environment-specific resolvers register here: + * - client: module state hydrated from `useBlockVisibility` (see `client.ts`) + * - server: an AsyncLocalStorage scoped per request (see `server-context.ts`) + * + * This module is isomorphic (no `'use client'`, no `node:` imports) so + * `@/blocks/registry` stays importable on both sides. When NO resolver state is + * active, `preview` blocks are still hidden ({@link isHiddenUnder} treats a null + * state as "nothing revealed") — fail-closed is the default everywhere, + * including SSR and server paths outside `withBlockVisibility`. + */ +export interface BlockVisibilityResolver { + current(): BlockVisibilityState | null +} + +let resolver: BlockVisibilityResolver | null = null + +/** Register (or clear with `null`) the active visibility resolver for this environment. */ +export function registerBlockVisibilityResolver(next: BlockVisibilityResolver | null): void { + resolver = next +} + +/** The visibility state in scope, or `null` when none (= nothing revealed, nothing disabled). */ +export function overlayVisibility(): BlockVisibilityState | null { + return resolver?.current() ?? null +} + +/** + * THE single hidden-predicate for block gating — every surface that hides + * blocks (registry projection, VFS stamp filter, exposed-integration-tools + * filter, `get_blocks_metadata`) calls this; never restate the rule inline. + * + * A block is hidden when it is an unrevealed `preview` block (fail-closed even + * with a null state) or when the kill switch (`disabled`) names it. Static + * `hideFromToolbar` is deliberately NOT part of this predicate — callers that + * need it check it separately. + */ +export function isHiddenUnder( + vis: BlockVisibilityState | null, + block: Pick +): boolean { + if (block.preview && !vis?.revealed.has(block.type)) return true + if (vis?.disabled.has(block.type)) return true + return false +} + +/** + * Registry of non-React module-cache resets (e.g. the tool-operations search + * index, the integration matcher) fired when the client visibility state + * changes. Lives here — not in the `'use client'` module — so plain `lib/` + * modules can register at load time on either side of the server boundary. + */ +const cacheInvalidators = new Set<() => void>() + +/** Register a cache reset to run on visibility changes. Returns an unregister fn. */ +export function registerBlockCacheInvalidator(fn: () => void): () => void { + cacheInvalidators.add(fn) + return () => cacheInvalidators.delete(fn) +} + +/** Fire every registered cache reset (called by the client hydrate path). */ +export function invalidateBlockCaches(): void { + for (const fn of cacheInvalidators) fn() +} diff --git a/apps/sim/blocks/visibility/registry-inert.test.ts b/apps/sim/blocks/visibility/registry-inert.test.ts new file mode 100644 index 00000000000..ddb7551737b --- /dev/null +++ b/apps/sim/blocks/visibility/registry-inert.test.ts @@ -0,0 +1,60 @@ +/** + * @vitest-environment node + */ +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.unmock('@/blocks/registry') + +const { synthRegistry } = vi.hoisted(() => ({ + synthRegistry: { + slack: { + type: 'slack', + name: 'Slack', + description: '', + category: 'tools', + bgColor: '#000', + icon: () => null, + subBlocks: [], + tools: { access: [] }, + inputs: {}, + outputs: {}, + }, + }, +})) + +vi.mock('@/blocks/registry-maps', () => ({ + BLOCK_REGISTRY: synthRegistry, + BLOCK_META_REGISTRY: {}, +})) + +import { getAllBlocks } from '@/blocks/registry' +import { registerBlockVisibilityResolver } from '@/blocks/visibility/context' + +afterEach(() => registerBlockVisibilityResolver(null)) + +describe('registry fast path (no preview blocks registered)', () => { + it('returns raw references with no context', () => { + expect(getAllBlocks()[0]).toBe(synthRegistry.slack) + }) + + it('returns raw references when the active state has no kill-switch entries', () => { + const state = { + revealed: new Set(['whatever']), + disabled: new Set(), + previewTagged: new Set(['whatever']), + } + registerBlockVisibilityResolver({ current: () => state }) + expect(getAllBlocks()[0]).toBe(synthRegistry.slack) + }) + + it('still projects when a kill-switch entry applies', () => { + const state = { + revealed: new Set(), + disabled: new Set(['slack']), + previewTagged: new Set(), + } + registerBlockVisibilityResolver({ current: () => state }) + expect(getAllBlocks()[0]?.hideFromToolbar).toBe(true) + expect(synthRegistry.slack).not.toHaveProperty('hideFromToolbar', true) + }) +}) diff --git a/apps/sim/blocks/visibility/server-context.ts b/apps/sim/blocks/visibility/server-context.ts new file mode 100644 index 00000000000..e713779c43c --- /dev/null +++ b/apps/sim/blocks/visibility/server-context.ts @@ -0,0 +1,25 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import { registerBlockVisibilityResolver } from '@/blocks/visibility/context' + +/** + * Server-side visibility context: a per-request AsyncLocalStorage, independent + * of the custom-block overlay's ALS so `withBlockVisibility` and + * `withCustomBlockOverlay` nest in either order without clobbering each other. + * + * Only copilot/mothership discovery paths establish this scope. Execution entry + * points (execute route, trigger.dev tasks, schedules/webhooks) never do — so + * placed preview blocks always serialize and run, and `preview` blocks stay + * hidden on unscoped discovery reads purely via the static fail-closed default. + */ +const store = new AsyncLocalStorage() + +registerBlockVisibilityResolver({ current: () => store.getStore() ?? null }) + +/** Run `fn` with the given visibility state resolvable via the registry accessors. */ +export function withBlockVisibility( + vis: BlockVisibilityState, + fn: () => Promise +): Promise { + return store.run(vis, fn) +} diff --git a/apps/sim/blocks/visibility/visibility.test.ts b/apps/sim/blocks/visibility/visibility.test.ts new file mode 100644 index 00000000000..6cd95f22435 --- /dev/null +++ b/apps/sim/blocks/visibility/visibility.test.ts @@ -0,0 +1,149 @@ +/** + * @vitest-environment node + */ +import { afterEach, describe, expect, it, vi } from 'vitest' + +vi.unmock('@/blocks/registry') + +const { synthRegistry } = vi.hoisted(() => { + const block = (type: string, extra: Record = {}) => ({ + type, + name: type.toUpperCase(), + description: '', + category: 'tools', + bgColor: '#000', + icon: () => null, + subBlocks: [], + tools: { access: [] }, + inputs: {}, + outputs: {}, + ...extra, + }) + return { + synthRegistry: { + slack: block('slack'), + gmail_v2: block('gmail_v2', { preview: true }), + old_v1: block('old_v1', { hideFromToolbar: true }), + }, + } +}) + +vi.mock('@/blocks/registry-maps', () => ({ + BLOCK_REGISTRY: synthRegistry, + BLOCK_META_REGISTRY: {}, +})) + +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import { registerBlockOverlayResolver } from '@/blocks/custom/overlay' +import { getAllBlocks, getBlock, getCanonicalBlocksByCategory } from '@/blocks/registry' +import type { BlockConfig } from '@/blocks/types' +import { isHiddenUnder, registerBlockVisibilityResolver } from '@/blocks/visibility/context' + +function vis(partial: Partial = {}): BlockVisibilityState { + return { + revealed: new Set(), + disabled: new Set(), + previewTagged: new Set(), + ...partial, + } +} + +function withVisibility(state: BlockVisibilityState | null) { + registerBlockVisibilityResolver(state ? { current: () => state } : null) +} + +const byType = (blocks: BlockConfig[], type: string) => blocks.find((b) => b.type === type) + +afterEach(() => { + registerBlockVisibilityResolver(null) + registerBlockOverlayResolver(null) +}) + +describe('isHiddenUnder', () => { + it('hides unrevealed preview blocks even with a null state (fail-closed)', () => { + expect(isHiddenUnder(null, { type: 'gmail_v2', preview: true })).toBe(true) + expect(isHiddenUnder(null, { type: 'slack' })).toBe(false) + }) + + it('reveals preview blocks named in revealed', () => { + const state = vis({ revealed: new Set(['gmail_v2']) }) + expect(isHiddenUnder(state, { type: 'gmail_v2', preview: true })).toBe(false) + }) + + it('hides kill-switched types only with an active state', () => { + const state = vis({ disabled: new Set(['slack']) }) + expect(isHiddenUnder(state, { type: 'slack' })).toBe(true) + expect(isHiddenUnder(null, { type: 'slack' })).toBe(false) + }) +}) + +describe('registry projection', () => { + it('hides preview blocks without any context: clone-not-remove', () => { + withVisibility(null) + const all = getAllBlocks() + const gmail = byType(all, 'gmail_v2') + expect(gmail).toBeDefined() + expect(gmail?.hideFromToolbar).toBe(true) + // the underlying registry entry is untouched + expect(getBlock('gmail_v2')?.hideFromToolbar).toBeUndefined() + // canonical (filtered) set excludes it + expect(byType(getCanonicalBlocksByCategory('tools'), 'gmail_v2')).toBeUndefined() + }) + + it('reveals a preview block with a " (Preview)" display suffix when tagged', () => { + withVisibility(vis({ revealed: new Set(['gmail_v2']), previewTagged: new Set(['gmail_v2']) })) + expect(byType(getAllBlocks(), 'gmail_v2')?.name).toBe('GMAIL_V2 (Preview)') + const canonical = byType(getCanonicalBlocksByCategory('tools'), 'gmail_v2') + expect(canonical?.name).toBe('GMAIL_V2 (Preview)') + expect(canonical?.hideFromToolbar).toBeUndefined() + }) + + it('reveals a config-GA preview block without a suffix', () => { + withVisibility(vis({ revealed: new Set(['gmail_v2']) })) + expect(byType(getAllBlocks(), 'gmail_v2')?.name).toBe('GMAIL_V2') + }) + + it('kill-switches a shipped block only when a context is active', () => { + withVisibility(vis({ disabled: new Set(['slack']) })) + expect(byType(getAllBlocks(), 'slack')?.hideFromToolbar).toBe(true) + expect(byType(getCanonicalBlocksByCategory('tools'), 'slack')).toBeUndefined() + + withVisibility(null) + expect(byType(getAllBlocks(), 'slack')?.hideFromToolbar).toBeUndefined() + }) + + it('keeps getBlock pure regardless of visibility', () => { + withVisibility( + vis({ + revealed: new Set(['gmail_v2']), + previewTagged: new Set(['gmail_v2']), + disabled: new Set(['slack']), + }) + ) + expect(getBlock('gmail_v2')?.name).toBe('GMAIL_V2') + expect(getBlock('slack')?.hideFromToolbar).toBeUndefined() + }) + + it('returns untouched references for unaffected blocks', () => { + withVisibility(vis({ revealed: new Set(['gmail_v2']) })) + expect(byType(getAllBlocks(), 'slack')).toBe(synthRegistry.slack) + expect(byType(getAllBlocks(), 'old_v1')).toBe(synthRegistry.old_v1) + }) + + it('never re-clones or suffixes already-hidden custom blocks', () => { + const disabledCustom = { + ...synthRegistry.slack, + type: 'custom_block_abc', + name: 'My Custom', + hideFromToolbar: true, + } as BlockConfig + registerBlockOverlayResolver({ + get: (t) => (t === disabledCustom.type ? disabledCustom : undefined), + all: () => [disabledCustom], + }) + withVisibility(vis({ revealed: new Set(['gmail_v2']) })) + const projected = byType(getAllBlocks(), 'custom_block_abc') + expect(projected).toBe(disabledCustom) + expect(projected?.name).toBe('My Custom') + }) +}) diff --git a/apps/sim/ee/access-control/components/group-detail.tsx b/apps/sim/ee/access-control/components/group-detail.tsx index bcb06776da8..7bf146ec4ac 100644 --- a/apps/sim/ee/access-control/components/group-detail.tsx +++ b/apps/sim/ee/access-control/components/group-detail.tsx @@ -615,8 +615,19 @@ export function GroupDetail({ const { data: roster } = useOrganizationRoster(organizationId) const { data: blacklistedProvidersData } = useBlacklistedProviders({ enabled: true }) - // Recompute when custom (deploy-as-block) blocks hydrate into the overlay. + // Recompute when custom (deploy-as-block) blocks or the viewer's block + // visibility hydrate into the overlay. const customBlockOverlayVersion = useCustomBlockOverlayVersion() + + /** + * The allowlist UNIVERSE: every access-controllable block, INCLUDING blocks + * gated for this viewer (they arrive as clones with `hideFromToolbar: true`, + * clone-not-remove). Materialization and the collapse-to-null comparison in + * `toggleIntegration`/`setBlocksAllowed` must use this viewer-independent set — + * otherwise a null→partial transition by a non-revealed admin would silently + * drop a preview block from the stored allowlist and deny it to revealed + * users already running it. + */ const allBlocks = useMemo(() => { const blocks = getAllBlocks().filter((b) => !isBlockTypeAccessControlExempt(b.type)) return blocks.sort((a, b) => { @@ -628,6 +639,14 @@ export function GroupDetail({ }) }, [customBlockOverlayVersion]) + /** + * The RENDERED list: hides blocks gated for this viewer by reading the + * registry projection's effective flag off the clone (the single source of + * truth — never re-derive visibility here). Revealed viewers see preview + * blocks (with their " (Preview)" suffix) and can toggle them explicitly. + */ + const visibleBlocks = useMemo(() => allBlocks.filter((b) => !b.hideFromToolbar), [allBlocks]) + const allProviderIds = useMemo(() => { const allIds = getAllProviderIds() const blacklist = blacklistedProvidersData?.blacklistedProviders ?? [] @@ -830,10 +849,10 @@ export function GroupDetail({ }, [allProviderIds, providerSearchTerm]) const filteredBlocks = useMemo(() => { - if (!integrationSearchTerm.trim()) return allBlocks + if (!integrationSearchTerm.trim()) return visibleBlocks const query = integrationSearchTerm.toLowerCase() - return allBlocks.filter((b) => b.name.toLowerCase().includes(query)) - }, [allBlocks, integrationSearchTerm]) + return visibleBlocks.filter((b) => b.name.toLowerCase().includes(query)) + }, [visibleBlocks, integrationSearchTerm]) const filteredCoreBlocks = useMemo( () => filteredBlocks.filter((block) => block.category === 'blocks'), diff --git a/apps/sim/ee/access-control/utils/permission-check.test.ts b/apps/sim/ee/access-control/utils/permission-check.test.ts index be436ca0019..13352b9fb00 100644 --- a/apps/sim/ee/access-control/utils/permission-check.test.ts +++ b/apps/sim/ee/access-control/utils/permission-check.test.ts @@ -379,6 +379,19 @@ describe('validateBlockType', () => { await validateBlockType(undefined, undefined, 'notion') }) + it('does NOT treat preview blocks as exempt — preview is not legacy', async () => { + // A `preview: true` block has static hideFromToolbar unset, so it is a + // normal access-controlled block: visibility gating (discovery) and + // permission-group enforcement (execution) are deliberately independent. + mockGetBlock.mockImplementation((type) => + type === 'gmail_v2' ? ({ preview: true } as { hideFromToolbar?: boolean }) : undefined + ) + + await expect(validateBlockType(undefined, undefined, 'gmail_v2')).rejects.toThrow( + IntegrationNotAllowedError + ) + }) + it('matches case-insensitively', async () => { await validateBlockType(undefined, undefined, 'Slack') await validateBlockType(undefined, undefined, 'GOOGLE_DRIVE') diff --git a/apps/sim/hooks/queries/block-visibility.ts b/apps/sim/hooks/queries/block-visibility.ts new file mode 100644 index 00000000000..2f073cb08d4 --- /dev/null +++ b/apps/sim/hooks/queries/block-visibility.ts @@ -0,0 +1,31 @@ +import { useQuery } from '@tanstack/react-query' +import { requestJson } from '@/lib/api/client/request' +import { + type BlockVisibilityResponse, + getBlockVisibilityContract, +} from '@/lib/api/contracts/block-visibility' + +export const BLOCK_VISIBILITY_STALE_TIME = 60 * 1000 + +export const blockVisibilityKeys = { + all: ['block-visibility'] as const, + lists: () => [...blockVisibilityKeys.all, 'list'] as const, + list: (workspaceId?: string) => [...blockVisibilityKeys.lists(), workspaceId ?? ''] as const, +} + +async function fetchBlockVisibility( + workspaceId: string, + signal?: AbortSignal +): Promise { + return requestJson(getBlockVisibilityContract, { query: { workspaceId }, signal }) +} + +/** The viewer's block-visibility projection for a workspace (revealed/disabled/preview-tagged types). */ +export function useBlockVisibility(workspaceId?: string) { + return useQuery({ + queryKey: blockVisibilityKeys.list(workspaceId), + queryFn: ({ signal }) => fetchBlockVisibility(workspaceId as string, signal), + enabled: Boolean(workspaceId), + staleTime: BLOCK_VISIBILITY_STALE_TIME, + }) +} diff --git a/apps/sim/lib/api/contracts/block-visibility.ts b/apps/sim/lib/api/contracts/block-visibility.ts new file mode 100644 index 00000000000..505eec694c6 --- /dev/null +++ b/apps/sim/lib/api/contracts/block-visibility.ts @@ -0,0 +1,37 @@ +import { z } from 'zod' +import { defineRouteContract } from '@/lib/api/contracts' +import { workspaceIdSchema } from '@/lib/api/contracts/primitives' + +/** + * Per-viewer block visibility projection (see + * `@/lib/core/config/block-visibility`): which `preview: true` block types this + * viewer may see, which types are kill-switched, and which revealed types carry + * the " (Preview)" display tag. + */ + +const getBlockVisibilityQuerySchema = z.object({ + workspaceId: workspaceIdSchema, +}) + +export type GetBlockVisibilityQuery = z.input + +const blockVisibilityResponseSchema = z.object({ + /** Preview block types revealed to this viewer. */ + revealed: z.array(z.string()), + /** Block types kill-switched (hidden from discovery) for this viewer. */ + disabled: z.array(z.string()), + /** Revealed types not globally GA — displayed with a " (Preview)" suffix. */ + previewTagged: z.array(z.string()), +}) + +export type BlockVisibilityResponse = z.output + +export const getBlockVisibilityContract = defineRouteContract({ + method: 'GET', + path: '/api/blocks/visibility', + query: getBlockVisibilityQuerySchema, + response: { + mode: 'json', + schema: blockVisibilityResponseSchema, + }, +}) diff --git a/apps/sim/lib/copilot/block-visibility.ts b/apps/sim/lib/copilot/block-visibility.ts new file mode 100644 index 00000000000..90e3c04cea7 --- /dev/null +++ b/apps/sim/lib/copilot/block-visibility.ts @@ -0,0 +1,58 @@ +import { LRUCache } from 'lru-cache' +import { type BlockVisibilityState, getBlockVisibility } from '@/lib/core/config/block-visibility' +import { getWorkspaceWithOwner } from '@/lib/workspaces/permissions/utils' + +/** + * Copilot-side resolver for the viewer's block-visibility projection. + * + * A single mothership turn fans out into many Go→Sim tool callbacks; resolving + * visibility per callback would repeat the workspace→org lookup (and, for + * admin-gated rules, a replica read) N times. This memoizes the resolved state + * per (userId, workspaceId) for a short TTL matching the AppConfig cache + * cadence, so a turn costs at most one resolution. + */ +const VISIBILITY_CACHE_TTL_MS = 30_000 + +const visibilityCache = new LRUCache>({ + max: 1000, + ttl: VISIBILITY_CACHE_TTL_MS, +}) + +async function resolveVisibility( + userId: string, + workspaceId?: string +): Promise { + const orgId = workspaceId + ? (await getWorkspaceWithOwner(workspaceId, { includeArchived: true }))?.organizationId + : undefined + return getBlockVisibility({ userId, orgId }) +} + +/** The viewer's visibility state, memoized per (userId, workspaceId) for ~30s. */ +export function getBlockVisibilityForCopilot( + userId: string, + workspaceId?: string +): Promise { + const key = `${userId}:${workspaceId ?? ''}` + let promise = visibilityCache.get(key) + if (!promise) { + promise = resolveVisibility(userId, workspaceId).catch((error) => { + visibilityCache.delete(key) + throw error + }) + visibilityCache.set(key, promise) + } + return promise +} + +/** + * Stable signature of a visibility state, for keying caches whose contents + * depend on the gated projection (e.g. the integration tool-schema LRU). + */ +export function visibilitySignature(vis: BlockVisibilityState): string { + return JSON.stringify([ + [...vis.revealed].sort(), + [...vis.disabled].sort(), + [...vis.previewTagged].sort(), + ]) +} diff --git a/apps/sim/lib/copilot/chat/payload.test.ts b/apps/sim/lib/copilot/chat/payload.test.ts index 1afbfbbaac3..88d6abbf689 100644 --- a/apps/sim/lib/copilot/chat/payload.test.ts +++ b/apps/sim/lib/copilot/chat/payload.test.ts @@ -53,7 +53,17 @@ vi.mock('@/tools/utils', () => ({ stripVersionSuffix: vi.fn((toolId: string) => toolId), })) +vi.mock('@/lib/copilot/block-visibility', () => ({ + getBlockVisibilityForCopilot: vi.fn(async () => ({ + revealed: new Set(), + disabled: new Set(), + previewTagged: new Set(), + })), + visibilitySignature: vi.fn(() => 'vis:none'), +})) + vi.mock('@/lib/copilot/integration-tools', () => ({ + filterExposedIntegrationTools: vi.fn((tools: unknown[]) => tools), getExposedIntegrationTools: vi.fn(() => [ { toolId: 'gmail_send', diff --git a/apps/sim/lib/copilot/chat/payload.ts b/apps/sim/lib/copilot/chat/payload.ts index e718b33cce3..97943990f09 100644 --- a/apps/sim/lib/copilot/chat/payload.ts +++ b/apps/sim/lib/copilot/chat/payload.ts @@ -3,11 +3,16 @@ import { toError } from '@sim/utils/errors' import { LRUCache } from 'lru-cache' import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' import { isPaid } from '@/lib/billing/plan-helpers' +import { getBlockVisibilityForCopilot, visibilitySignature } from '@/lib/copilot/block-visibility' import type { VfsSnapshotV1 } from '@/lib/copilot/generated/vfs-snapshot-v1' -import { getExposedIntegrationTools } from '@/lib/copilot/integration-tools' +import { + filterExposedIntegrationTools, + getExposedIntegrationTools, +} from '@/lib/copilot/integration-tools' import { getToolEntry } from '@/lib/copilot/tool-executor/router' import { getCopilotToolDescription } from '@/lib/copilot/tools/descriptions' import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' import { isE2BDocEnabled, isHosted } from '@/lib/core/config/env-flags' import { buildUserSkillTool } from '@/lib/mothership/skills' import { trackChatUpload } from '@/lib/uploads/contexts/workspace/workspace-file-manager' @@ -73,9 +78,12 @@ const integrationToolSchemaCache = new LRUCache { const schemaSurface = options.schemaSurface ?? 'copilot' - const cacheKey = getIntegrationToolSchemaCacheKey(userId, workspaceId, schemaSurface) + const vis = await getBlockVisibilityForCopilot(userId, workspaceId) + const cacheKey = getIntegrationToolSchemaCacheKey( + userId, + workspaceId, + schemaSurface, + visibilitySignature(vis) + ) const cached = integrationToolSchemaCache.get(cacheKey) if (cached) { return cloneToolSchemas(await cached.promise) @@ -120,7 +134,8 @@ export async function buildIntegrationToolSchemas( userId, messageId, { schemaSurface }, - workspaceId + workspaceId, + vis ).catch((error) => { integrationToolSchemaCache.delete(cacheKey) throw error @@ -137,7 +152,8 @@ async function buildIntegrationToolSchemasUncached( userId: string, messageId: string | undefined, options: Required, - workspaceId?: string + workspaceId?: string, + vis: BlockVisibilityState | null = null ): Promise { const reqLogger = logger.withMetadata({ messageId }) const integrationTools: ToolSchema[] = [] @@ -186,7 +202,8 @@ async function buildIntegrationToolSchemasUncached( } } - for (const { toolId, config: toolConfig, service } of getExposedIntegrationTools()) { + const exposedTools = filterExposedIntegrationTools(getExposedIntegrationTools(), vis) + for (const { toolId, config: toolConfig, service } of exposedTools) { try { if (allowedIntegrations && toolIdToBlockType) { const owningBlock = toolIdToBlockType.get(stripVersionSuffix(toolId)) diff --git a/apps/sim/lib/copilot/integration-tools.ts b/apps/sim/lib/copilot/integration-tools.ts index c1c93cc536e..77559d78784 100644 --- a/apps/sim/lib/copilot/integration-tools.ts +++ b/apps/sim/lib/copilot/integration-tools.ts @@ -1,4 +1,6 @@ -import { getAllBlocks } from '@/blocks/registry' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' +import { BLOCK_REGISTRY } from '@/blocks/registry-maps' +import { isHiddenUnder } from '@/blocks/visibility/context' import { tools as toolRegistry } from '@/tools/registry' import type { ToolConfig } from '@/tools/types' import { getLatestVersionTools, stripVersionSuffix } from '@/tools/utils' @@ -15,14 +17,24 @@ export interface ExposedIntegrationTool { service: string /** Operation stem within the service (used for the VFS path filename), e.g. "read". */ operation: string + /** Owning block's registry type — the key block-visibility rules gate on. */ + blockType: string + /** Owning block's static `preview` marker, for the per-viewer filter. */ + preview?: boolean } let cached: ExposedIntegrationTool[] | null = null /** - * Returns the canonical set of integration tools exposed to the copilot agent: - * the latest version of each operation owned by a visible (non-hideFromToolbar) - * block. + * Returns the UNGATED universe of integration tools exposable to the copilot + * agent: the latest version of each operation owned by a non-`hideFromToolbar` + * block — INCLUDING unreleased `preview` blocks. + * + * Deliberately sourced from the raw `BLOCK_REGISTRY` (never the visibility- + * projected `getAllBlocks`) so this process-global memo is deterministic and + * can never be poisoned by whichever viewer's gated projection ran first. + * Every per-viewer consumer MUST apply {@link filterExposedIntegrationTools} + * before exposing the set. * * This is the single source of truth shared by VFS discovery * (components/integrations/**) and the deferred callable-tool payload, so the @@ -33,15 +45,16 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { if (cached) return cached // Map the tool ids each visible block exposes (both the raw id and its - // version-stripped base name) to that block's service directory. - const toolToService = new Map() - for (const block of getAllBlocks()) { + // version-stripped base name) to that block's service directory + type. + const toolToBlock = new Map() + for (const block of Object.values(BLOCK_REGISTRY)) { if (block.hideFromToolbar) continue if (!block.tools?.access) continue const service = stripVersionSuffix(block.type) + const owner = { service, blockType: block.type, preview: block.preview } for (const toolId of block.tools.access) { - toolToService.set(toolId, service) - toolToService.set(stripVersionSuffix(toolId), service) + toolToBlock.set(toolId, owner) + toolToBlock.set(stripVersionSuffix(toolId), owner) } } @@ -49,19 +62,41 @@ export function getExposedIntegrationTools(): ExposedIntegrationTool[] { const seen = new Set() for (const [toolId, config] of Object.entries(getLatestVersionTools(toolRegistry))) { const baseName = stripVersionSuffix(toolId) - const service = toolToService.get(toolId) ?? toolToService.get(baseName) - if (!service) continue + const owner = toolToBlock.get(toolId) ?? toolToBlock.get(baseName) + if (!owner) continue if (seen.has(baseName)) continue seen.add(baseName) - const prefix = `${service}_` + const prefix = `${owner.service}_` const operation = baseName.startsWith(prefix) ? baseName.slice(prefix.length) : baseName - exposed.push({ toolId, config, service, operation }) + exposed.push({ + toolId, + config, + service: owner.service, + operation, + blockType: owner.blockType, + preview: owner.preview, + }) } cached = exposed return exposed } +/** + * Per-viewer projection of the exposed set: drops tools whose owning block is + * hidden under `vis` (unrevealed preview blocks — including with a null state — + * and kill-switched types). Apply at every surface that hands the set to a + * viewer: VFS stamping, the deferred tool payload, `list_integration_tools`. + */ +export function filterExposedIntegrationTools( + tools: ExposedIntegrationTool[], + vis: BlockVisibilityState | null +): ExposedIntegrationTool[] { + return tools.filter( + (tool) => !isHiddenUnder(vis, { type: tool.blockType, preview: tool.preview }) + ) +} + /** Test-only: clears the memoized set so registry changes are picked up. */ export function resetExposedIntegrationToolsCache(): void { cached = null diff --git a/apps/sim/lib/copilot/tools/handlers/integration-tools.ts b/apps/sim/lib/copilot/tools/handlers/integration-tools.ts index a3e307bac8a..465da57bc33 100644 --- a/apps/sim/lib/copilot/tools/handlers/integration-tools.ts +++ b/apps/sim/lib/copilot/tools/handlers/integration-tools.ts @@ -1,17 +1,24 @@ -import { getExposedIntegrationTools } from '@/lib/copilot/integration-tools' +import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' +import { + filterExposedIntegrationTools, + getExposedIntegrationTools, +} from '@/lib/copilot/integration-tools' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' import { stripVersionSuffix } from '@/tools/utils' export async function executeListIntegrationTools( params: Record, - _context: ExecutionContext + context: ExecutionContext ): Promise { const raw = typeof params.integration === 'string' ? params.integration.trim() : '' if (!raw) { return { success: false, error: "Missing required parameter 'integration'" } } - const all = getExposedIntegrationTools() + // The exposed set is the ungated universe — project it for this viewer so + // gated (preview / kill-switched) integrations stay undiscoverable. + const vis = await getBlockVisibilityForCopilot(context.userId, context.workspaceId) + const all = filterExposedIntegrationTools(getExposedIntegrationTools(), vis) const service = stripVersionSuffix(raw.toLowerCase()) const matches = all.filter((tool) => tool.service === service) diff --git a/apps/sim/lib/copilot/tools/handlers/vfs.ts b/apps/sim/lib/copilot/tools/handlers/vfs.ts index ca1902692e1..fd4a515a8af 100644 --- a/apps/sim/lib/copilot/tools/handlers/vfs.ts +++ b/apps/sim/lib/copilot/tools/handlers/vfs.ts @@ -1,15 +1,28 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { TOOL_RESULT_MAX_INLINE_CHARS } from '@/lib/copilot/constants' import type { ExecutionContext, ToolCallResult } from '@/lib/copilot/request/types' import { getOrMaterializeVFS } from '@/lib/copilot/vfs' import type { GrepCountEntry, GrepMatch } from '@/lib/copilot/vfs/operations' import { WorkspaceFileGrepError } from '@/lib/copilot/vfs/operations' import { encodeVfsSegment } from '@/lib/copilot/vfs/path-utils' +import { withBlockVisibility } from '@/blocks/visibility/server-context' import { grepChatUpload, listChatUploads, readChatUpload } from './upload-file-reader' const logger = createLogger('VfsTools') +/** + * Materialize the workspace VFS inside the viewer's block-visibility context so + * the static component files stamped into it exclude blocks gated for this + * viewer (unrevealed previews, kill-switched types). Visibility is memoized per + * (userId, workspaceId), so repeated tool calls in one turn resolve once. + */ +async function getGatedVFS(workspaceId: string, userId: string) { + const vis = await getBlockVisibilityForCopilot(userId, workspaceId) + return withBlockVisibility(vis, () => getOrMaterializeVFS(workspaceId, userId)) +} + /** * Encode a chat-upload display name as a single canonical VFS path segment so * `uploads/` paths follow the same percent-encoded convention as `files/`. @@ -119,7 +132,7 @@ export async function executeVfsGrep( } result = await grepChatUpload(filename, context.chatId, pattern, grepOptions) } else { - const vfs = await getOrMaterializeVFS(workspaceId, context.userId) + const vfs = await getGatedVFS(workspaceId, context.userId) result = isWorkspaceFileGrepPath(rawPath) ? await vfs.grepFile(rawPath, pattern, grepOptions) : await vfs.grep(pattern, rawPath, grepOptions) @@ -176,7 +189,7 @@ export async function executeVfsGlob( } try { - const vfs = await getOrMaterializeVFS(workspaceId, context.userId) + const vfs = await getGatedVFS(workspaceId, context.userId) let files = vfs.glob(pattern) if (context.chatId && (pattern === 'uploads/*' || pattern.startsWith('uploads/'))) { @@ -281,7 +294,7 @@ export async function executeVfsRead( } } - const vfs = await getOrMaterializeVFS(workspaceId, context.userId) + const vfs = await getGatedVFS(workspaceId, context.userId) // Plain canonical file leaves are metadata resources. Dynamic file content // and inspection paths use explicit suffixes like /content, /style, diff --git a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts index 6dd2579711a..7828f89ec98 100644 --- a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts +++ b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-tool.ts @@ -10,6 +10,7 @@ import { getServiceAccountProviderForProviderId } from '@/lib/oauth/utils' import { isCustomBlockType } from '@/blocks/custom/build-config' import { getBlock } from '@/blocks/registry' import { AuthMode, type BlockConfig, isHiddenFromDisplay } from '@/blocks/types' +import { isHiddenUnder, overlayVisibility } from '@/blocks/visibility/context' import { getUserPermissionConfig } from '@/ee/access-control/utils/permission-check' import { PROVIDER_DEFINITIONS } from '@/providers/models' import { tools as toolsRegistry } from '@/tools/registry' @@ -165,6 +166,15 @@ export const getBlocksMetadataServerTool: BaseServerTool< continue } + // getBlock is pure, so the viewer's visibility must be checked + // explicitly: unrevealed preview blocks and kill-switched types stay + // out of the agent's metadata (the router wraps this tool in + // withBlockVisibility). + if (isHiddenUnder(overlayVisibility(), blockConfig)) { + logger.debug('Skipping block gated by visibility', { blockId }) + continue + } + if (isCustomBlockType(blockId)) { // Custom (deploy-as-block) blocks run a bound workflow via an internal // `workflow_executor`; the agent never configures a workflowId/inputMapping. diff --git a/apps/sim/lib/copilot/tools/server/router.ts b/apps/sim/lib/copilot/tools/server/router.ts index 6754920bca4..9dd8c737dde 100644 --- a/apps/sim/lib/copilot/tools/server/router.ts +++ b/apps/sim/lib/copilot/tools/server/router.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { z } from 'zod' +import { getBlockVisibilityForCopilot } from '@/lib/copilot/block-visibility' import { CreateFile, CreateFileFolder, @@ -60,6 +61,7 @@ import { editWorkflowServerTool } from '@/lib/copilot/tools/server/workflow/edit import { queryLogsServerTool } from '@/lib/copilot/tools/server/workflow/query-logs' import { listCustomBlocksWithInputsForWorkspace } from '@/lib/workflows/custom-blocks/operations' import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay' +import { withBlockVisibility } from '@/blocks/visibility/server-context' export type ExecuteResponseSuccess = z.output @@ -76,6 +78,16 @@ const logger = createLogger('ServerToolRouter') */ const CUSTOM_BLOCK_OVERLAY_TOOLS = new Set(['edit_workflow', 'get_blocks_metadata']) +/** + * DISCOVERY tools that must run inside the viewer's block-visibility context so + * gated (preview / kill-switched) blocks disappear from what the agent can + * list. Deliberately a DIFFERENT set from {@link CUSTOM_BLOCK_OVERLAY_TOOLS}: + * `edit_workflow` is excluded because its registry use is functional + * (find-by-type over clones, never a discovery listing) and gating it would + * only risk leaking display projections into persisted state. + */ +const VISIBILITY_GATED_TOOLS = new Set(['get_blocks_metadata', 'get_trigger_blocks']) + const WRITE_ACTIONS: Record = { [KnowledgeBase.id]: [ 'create', @@ -243,15 +255,22 @@ export async function routeExecution( // Execute. The registry-dependent tools resolve blocks via getBlock/getAllBlocks; // wrap them in the custom-block overlay for the workspace's org so `custom_block_*` // types resolve (metadata lookup + edit-workflow validation) instead of being - // rejected as unknown. Other tools skip the extra query. - const runTool = () => tool.execute(args, context) - const result = - CUSTOM_BLOCK_OVERLAY_TOOLS.has(toolName) && context?.workspaceId - ? await withCustomBlockOverlay( - await listCustomBlocksWithInputsForWorkspace(context.workspaceId), - runTool - ) - : await runTool() + // rejected as unknown, and wrap discovery tools in the viewer's block-visibility + // context so gated blocks stay hidden. The two ALS scopes are independent and + // nest in either order. Other tools skip the extra queries. + let run = () => tool.execute(args, context) + if (VISIBILITY_GATED_TOOLS.has(toolName) && context?.userId) { + // Memoized per (userId, workspaceId) ~30s — a multi-tool turn resolves once. + const vis = await getBlockVisibilityForCopilot(context.userId, context.workspaceId) + const inner = run + run = () => withBlockVisibility(vis, inner) + } + if (CUSTOM_BLOCK_OVERLAY_TOOLS.has(toolName) && context?.workspaceId) { + const rows = await listCustomBlocksWithInputsForWorkspace(context.workspaceId) + const inner = run + run = () => withCustomBlockOverlay(rows, inner) + } + const result = await run() // Validate output if tool declares a schema; otherwise fall back to the // generated JSON schema contract emitted from Go. diff --git a/apps/sim/lib/copilot/vfs/workspace-vfs.ts b/apps/sim/lib/copilot/vfs/workspace-vfs.ts index abfc3ad9508..f40def73f7e 100644 --- a/apps/sim/lib/copilot/vfs/workspace-vfs.ts +++ b/apps/sim/lib/copilot/vfs/workspace-vfs.ts @@ -88,6 +88,7 @@ import { workspacePlanBackingPath, workspacePlansBackingFolderPath, } from '@/lib/copilot/vfs/workflow-aliases' +import type { BlockVisibilityState } from '@/lib/core/config/block-visibility' import { isE2BDocEnabled } from '@/lib/core/config/env-flags' import { isFeatureEnabled } from '@/lib/core/config/feature-flags' import { @@ -124,8 +125,9 @@ import { } from '@/lib/workspaces/permissions/utils' import { computeNeedsRedeployment } from '@/app/api/workflows/utils' import { buildCustomBlockConfig, isCustomBlockType } from '@/blocks/custom/build-config' -import { getAllBlocks } from '@/blocks/registry' -import type { BlockIcon } from '@/blocks/types' +import { BLOCK_REGISTRY } from '@/blocks/registry-maps' +import type { BlockConfig, BlockIcon } from '@/blocks/types' +import { isHiddenUnder, overlayVisibility } from '@/blocks/visibility/context' import { CONNECTOR_REGISTRY } from '@/connectors/registry.server' import type { WorkflowState } from '@/stores/workflows/workflow/types' import { TRIGGER_REGISTRY } from '@/triggers/registry' @@ -137,9 +139,39 @@ const logger = createLogger('WorkspaceVFS') const PLACEHOLDER_BLOCK_ICON = (() => null) as unknown as BlockIcon const MAX_COMPILED_ATTACHMENT_BYTES = 5 * 1024 * 1024 -/** Static component files, computed once and shared across all VFS instances */ +/** + * Static component files, computed once and shared across all VFS instances. + * Built from the UNGATED registry universe (preview blocks included) so this + * process-global cache can never be poisoned by one viewer's gated projection; + * per-viewer gating is applied when the map is stamped into each fresh VFS + * (see {@link isStaticFileHidden}). + */ let staticComponentFiles: Map | null = null +/** + * Owning block for each `components/integrations/**` file, recorded at build + * time. Block/trigger schema files carry their owning type as the path + * basename, but integration paths use the version-stripped service name — so + * their owners need this lookup for the stamp-time visibility filter. + */ +const integrationPathOwners = new Map>() + +/** + * Per-request visibility filter for the shared static files: hides files whose + * owning block is gated for this viewer (unrevealed preview blocks — the + * default with no context — and kill-switched types). Non-registry paths + * (loop/parallel, connectors, overviews) are always visible. + */ +function isStaticFileHidden(path: string, vis: BlockVisibilityState | null): boolean { + const blockMatch = path.match(/^components\/(?:blocks|triggers\/sim)\/([^/]+)\.json$/) + if (blockMatch) { + const config = BLOCK_REGISTRY[blockMatch[1]!] + return config ? isHiddenUnder(vis, config) : false + } + const owner = integrationPathOwners.get(path) + return owner ? isHiddenUnder(vis, owner) : false +} + // On-the-fly doc reads (render/extract) download the binary into the Sim process // and base64-stage it to E2B, so bound the input like the compile path's staging // caps — otherwise an authenticated member could OOM the worker with a multi-GB @@ -170,7 +202,12 @@ function getStaticComponentFiles(): Map { const files = new Map() - const allBlocks = getAllBlocks() + // Raw registry, never the visibility-projected getAllBlocks: this map is a + // process-global shared cache, so it must hold the deterministic ungated + // universe. Preview blocks get schema files here (path-filterable at stamp + // time for revealed viewers) but are EXCLUDED from the shared aggregate + // files (overviews, oauth/api-key summaries) that all viewers receive. + const allBlocks = Object.values(BLOCK_REGISTRY) const visibleBlocks = allBlocks.filter((b) => !b.hideFromToolbar) let blocksFiltered = 0 @@ -188,11 +225,17 @@ function getStaticComponentFiles(): Map { // Integration tools come from the shared exposed-tool set (latest version of // each operation owned by a visible block), the same set used to build the // deferred callable tools — so discovery and execution can never drift. - for (const { config: tool, service, operation } of getExposedIntegrationTools()) { + for (const exposedTool of getExposedIntegrationTools()) { + const { config: tool, service, operation, blockType, preview } = exposedTool const path = `components/integrations/${service}/${operation}.json` files.set(path, serializeIntegrationSchema(tool)) + integrationPathOwners.set(path, { type: blockType, preview }) integrationCount++ + // Preview-owned tools stay out of the shared oauth/api-key aggregates — + // those files are identical for every viewer. + if (preview) continue + if (tool.oauth?.required) { const existing = oauthServices.get(service) if (existing) { @@ -320,12 +363,16 @@ function getStaticComponentFiles(): Map { files.set( 'components/triggers/triggers.md', serializeTriggerOverview( - builtinTriggerBlocks.map((b) => ({ - id: b.type, - name: b.name, - provider: 'sim', - description: b.description, - })), + // The overview is a shared file — preview trigger blocks stay out of it + // (their per-type schema file remains discoverable for revealed viewers). + builtinTriggerBlocks + .filter((b) => !b.preview) + .map((b) => ({ + id: b.type, + name: b.name, + provider: 'sim', + description: b.description, + })), Object.entries(TRIGGER_REGISTRY).map(([id, t]) => ({ id, name: t.name, @@ -642,7 +689,11 @@ export class WorkspaceVFS { await timed('recently_deleted', this.materializeRecentlyDeleted(workspaceId, userId)) + // Per-viewer gating happens HERE, not in the shared builder: files + // owned by blocks hidden for this viewer are skipped at stamp time. + const blockVisibility = overlayVisibility() for (const [path, content] of getStaticComponentFiles()) { + if (isStaticFileHidden(path, blockVisibility)) continue this.files.set(path, content) } diff --git a/apps/sim/lib/core/config/appconfig-rules.test.ts b/apps/sim/lib/core/config/appconfig-rules.test.ts new file mode 100644 index 00000000000..ab79359000d --- /dev/null +++ b/apps/sim/lib/core/config/appconfig-rules.test.ts @@ -0,0 +1,69 @@ +/** + * @vitest-environment node + */ +import { describe, expect, it } from 'vitest' +import { matchesRule, normalizeRule, parseGateConfig } from '@/lib/core/config/appconfig-rules' + +describe('normalizeRule', () => { + it('returns null for non-object values', () => { + expect(normalizeRule('nope')).toBeNull() + expect(normalizeRule(null)).toBeNull() + expect(normalizeRule(42)).toBeNull() + }) + + it('keeps only boolean enabled/adminEnabled', () => { + expect(normalizeRule({ enabled: 'true', adminEnabled: 1 })).toEqual({}) + expect(normalizeRule({ enabled: true, adminEnabled: false })).toEqual({ + enabled: true, + adminEnabled: false, + }) + }) + + it('trims, dedupes, and drops empty ids', () => { + expect(normalizeRule({ orgIds: ['Org_1', ' org_1 ', '', 'org_2'], userIds: 'nope' })).toEqual({ + orgIds: ['Org_1', 'org_1', 'org_2'], + }) + }) +}) + +describe('parseGateConfig', () => { + it('drops malformed entries and coerces the rest', () => { + const rules = parseGateConfig({ + a: { enabled: true }, + b: 'not-an-object', + c: { userIds: ['u1'] }, + }) + expect(rules.a).toEqual({ enabled: true }) + expect(rules.b).toBeUndefined() + expect(rules.c).toEqual({ userIds: ['u1'] }) + }) + + it('degrades to an empty map on a malformed document', () => { + expect(parseGateConfig('not-an-object')).toEqual({}) + expect(parseGateConfig(null)).toEqual({}) + }) +}) + +describe('matchesRule', () => { + it('returns false for a missing rule', () => { + expect(matchesRule(undefined, { userId: 'u1' }, true)).toBe(false) + }) + + it('matches the global enabled clause', () => { + expect(matchesRule({ enabled: true }, {}, false)).toBe(true) + expect(matchesRule({ enabled: false }, {}, false)).toBe(false) + }) + + it('matches the userId and orgId allowlists', () => { + expect(matchesRule({ userIds: ['u1'] }, { userId: 'u1' }, false)).toBe(true) + expect(matchesRule({ userIds: ['u1'] }, { userId: 'u2' }, false)).toBe(false) + expect(matchesRule({ orgIds: ['o1'] }, { orgId: 'o1' }, false)).toBe(true) + expect(matchesRule({ orgIds: ['o1'] }, {}, false)).toBe(false) + }) + + it('matches the admin clause only with the supplied isAdmin', () => { + expect(matchesRule({ adminEnabled: true }, { userId: 'u1' }, true)).toBe(true) + expect(matchesRule({ adminEnabled: true }, { userId: 'u1' }, false)).toBe(false) + expect(matchesRule({ enabled: false }, { userId: 'u1' }, true)).toBe(false) + }) +}) diff --git a/apps/sim/lib/core/config/appconfig-rules.ts b/apps/sim/lib/core/config/appconfig-rules.ts new file mode 100644 index 00000000000..34ceec5173a --- /dev/null +++ b/apps/sim/lib/core/config/appconfig-rules.ts @@ -0,0 +1,80 @@ +/** + * Shared parsing and clause evaluation for AppConfig gating documents + * (`feature-flags`, `block-visibility`). Both documents are maps of key → + * gate rule with identical rule shapes; this module is the single copy of the + * security-sensitive normalization that prevents a malformed document from + * granting access. Admin-resolution *scheduling* deliberately stays with the + * callers (feature-flags resolves lazily per rule; block-visibility resolves + * once per document), so {@link matchesRule} takes an explicit `isAdmin`. + */ + +/** + * A single gating rule. A gate is open for a context when ANY clause matches: + * the global `enabled` default, the org/user allowlists, or `adminEnabled` for + * platform admins. An absent clause never matches. + */ +export interface AppConfigGateRule { + enabled?: boolean + orgIds?: string[] + userIds?: string[] + adminEnabled?: boolean +} + +/** + * Per-request evaluation context. Pass only the ids you have — a missing id + * skips its clause. `isAdmin` is a fast-path override for callers that already + * resolved platform-admin status. + */ +export interface AppConfigGateContext { + userId?: string | null + orgId?: string | null + isAdmin?: boolean +} + +function normalizeIds(values: unknown): string[] | undefined { + if (!Array.isArray(values)) return undefined + const ids = Array.from(new Set(values.map((v) => String(v).trim()).filter(Boolean))) + return ids.length > 0 ? ids : undefined +} + +/** Coerce a single arbitrary JSON value into a rule, or `null` when malformed. */ +export function normalizeRule(value: unknown): AppConfigGateRule | null { + if (!value || typeof value !== 'object') return null + const obj = value as Record + const rule: AppConfigGateRule = {} + if (typeof obj.enabled === 'boolean') rule.enabled = obj.enabled + if (typeof obj.adminEnabled === 'boolean') rule.adminEnabled = obj.adminEnabled + const orgIds = normalizeIds(obj.orgIds) + if (orgIds) rule.orgIds = orgIds + const userIds = normalizeIds(obj.userIds) + if (userIds) rule.userIds = userIds + return rule +} + +/** Coerce an arbitrary AppConfig/JSON document into a rule map, dropping malformed entries. */ +export function parseGateConfig(json: unknown): Record { + const obj = (json && typeof json === 'object' ? json : {}) as Record + const rules: Record = {} + for (const [key, value] of Object.entries(obj)) { + const rule = normalizeRule(value) + if (rule) rules[key] = rule + } + return rules +} + +/** + * Pure OR-of-clauses check. The caller supplies `isAdmin` — pass `false` to + * evaluate only the non-admin clauses (for lazy admin resolution). + */ +export function matchesRule( + rule: AppConfigGateRule | undefined, + ctx: AppConfigGateContext, + isAdmin: boolean +): boolean { + if (!rule) return false + if (rule.enabled) return true + if (ctx.userId && rule.userIds?.includes(ctx.userId)) return true + if (ctx.orgId && rule.orgIds?.includes(ctx.orgId)) return true + if (rule.adminEnabled && isAdmin) return true + return false +} diff --git a/apps/sim/lib/core/config/block-visibility.test.ts b/apps/sim/lib/core/config/block-visibility.test.ts new file mode 100644 index 00000000000..ef7e44c1006 --- /dev/null +++ b/apps/sim/lib/core/config/block-visibility.test.ts @@ -0,0 +1,157 @@ +/** + * @vitest-environment node + */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockFetch, mockIsPlatformAdmin, envRef, flagRef } = vi.hoisted(() => ({ + mockFetch: vi.fn(), + mockIsPlatformAdmin: vi.fn(), + envRef: { + APPCONFIG_APPLICATION: 'sim-staging' as string | undefined, + APPCONFIG_ENVIRONMENT: 'staging' as string | undefined, + }, + flagRef: { isAppConfigEnabled: false, previewBlocks: [] as string[] }, +})) + +vi.mock('@/lib/core/config/appconfig', () => ({ + fetchAppConfigProfile: mockFetch, +})) + +vi.mock('@/lib/core/config/env', () => ({ + get env() { + return envRef + }, +})) + +vi.mock('@/lib/core/config/env-flags', () => ({ + get isAppConfigEnabled() { + return flagRef.isAppConfigEnabled + }, + getPreviewBlocksFromEnv: () => flagRef.previewBlocks, +})) + +vi.mock('@/lib/permissions/super-user', () => ({ + isPlatformAdmin: mockIsPlatformAdmin, +})) + +import { getBlockVisibility } from '@/lib/core/config/block-visibility' + +/** Make `getBlockVisibility` resolve `doc` via the AppConfig path (also exercises parsing). */ +function withAppConfig(doc: unknown) { + flagRef.isAppConfigEnabled = true + mockFetch.mockImplementation((_ids, parse) => Promise.resolve(parse(doc))) +} + +describe('getBlockVisibility', () => { + beforeEach(() => { + vi.clearAllMocks() + flagRef.isAppConfigEnabled = false + flagRef.previewBlocks = [] + }) + + describe('off-AppConfig (env fallback)', () => { + it('reveals and preview-tags the PREVIEW_BLOCKS types without fetching', async () => { + flagRef.previewBlocks = ['gmail_v2', 'notion_v3'] + const vis = await getBlockVisibility({ userId: 'u1' }) + expect(vis.revealed).toEqual(new Set(['gmail_v2', 'notion_v3'])) + expect(vis.previewTagged).toEqual(new Set(['gmail_v2', 'notion_v3'])) + expect(vis.disabled.size).toBe(0) + expect(mockFetch).not.toHaveBeenCalled() + }) + + it('returns empty state when PREVIEW_BLOCKS is unset', async () => { + const vis = await getBlockVisibility() + expect(vis.revealed.size).toBe(0) + expect(vis.disabled.size).toBe(0) + expect(vis.previewTagged.size).toBe(0) + }) + }) + + it('fetches the block-visibility profile', async () => { + withAppConfig({}) + await getBlockVisibility() + expect(mockFetch).toHaveBeenCalledWith( + { application: 'sim-staging', environment: 'staging', profile: 'block-visibility' }, + expect.any(Function) + ) + }) + + it('GA rule (enabled: true) reveals without a preview tag', async () => { + withAppConfig({ gmail_v2: { enabled: true } }) + const vis = await getBlockVisibility({ userId: 'u1' }) + expect(vis.revealed.has('gmail_v2')).toBe(true) + expect(vis.previewTagged.has('gmail_v2')).toBe(false) + expect(vis.disabled.has('gmail_v2')).toBe(false) + }) + + it('allowlist rule reveals with a preview tag; non-matching viewers get disabled', async () => { + withAppConfig({ gmail_v2: { enabled: false, orgIds: ['o1'], userIds: ['u9'] } }) + + const allowedOrg = await getBlockVisibility({ orgId: 'o1' }) + expect(allowedOrg.revealed.has('gmail_v2')).toBe(true) + expect(allowedOrg.previewTagged.has('gmail_v2')).toBe(true) + + const allowedUser = await getBlockVisibility({ userId: 'u9' }) + expect(allowedUser.revealed.has('gmail_v2')).toBe(true) + + const denied = await getBlockVisibility({ userId: 'u1', orgId: 'o2' }) + expect(denied.revealed.has('gmail_v2')).toBe(false) + expect(denied.disabled.has('gmail_v2')).toBe(true) + }) + + it('kill switch (enabled: false, no allowlists) disables for everyone', async () => { + withAppConfig({ slack: { enabled: false } }) + const vis = await getBlockVisibility({ userId: 'u1', orgId: 'o1' }) + expect(vis.disabled.has('slack')).toBe(true) + expect(vis.revealed.has('slack')).toBe(false) + }) + + it('drops custom_block_* keys so custom blocks can never be gated', async () => { + withAppConfig({ custom_block_abc123: { enabled: false }, gmail_v2: { enabled: true } }) + const vis = await getBlockVisibility({ userId: 'u1' }) + expect(vis.disabled.has('custom_block_abc123')).toBe(false) + expect(vis.revealed.has('custom_block_abc123')).toBe(false) + expect(vis.revealed.has('gmail_v2')).toBe(true) + }) + + it('drops malformed entries', async () => { + withAppConfig({ a: 'nope', b: { enabled: false, orgIds: [' o1 ', ''] } }) + const vis = await getBlockVisibility({ orgId: 'o1' }) + expect(vis.disabled.has('a')).toBe(false) + expect(vis.revealed.has('b')).toBe(true) + }) + + describe('admin resolution (once per call)', () => { + it('resolves admin exactly once for a document with multiple adminEnabled rules', async () => { + withAppConfig({ + a: { enabled: false, adminEnabled: true }, + b: { enabled: false, adminEnabled: true }, + c: { enabled: false }, + }) + mockIsPlatformAdmin.mockResolvedValue(true) + const vis = await getBlockVisibility({ userId: 'u1' }) + expect(mockIsPlatformAdmin).toHaveBeenCalledTimes(1) + expect(vis.revealed).toEqual(new Set(['a', 'b'])) + expect(vis.previewTagged).toEqual(new Set(['a', 'b'])) + expect(vis.disabled).toEqual(new Set(['c'])) + }) + + it('uses the isAdmin fast-path without querying', async () => { + withAppConfig({ a: { enabled: false, adminEnabled: true } }) + const vis = await getBlockVisibility({ userId: 'u1', isAdmin: true }) + expect(vis.revealed.has('a')).toBe(true) + expect(mockIsPlatformAdmin).not.toHaveBeenCalled() + }) + + it('does not query when no rule has adminEnabled or when userId is absent', async () => { + withAppConfig({ a: { enabled: false, orgIds: ['o1'] } }) + await getBlockVisibility({ userId: 'u1' }) + expect(mockIsPlatformAdmin).not.toHaveBeenCalled() + + withAppConfig({ a: { enabled: false, adminEnabled: true } }) + const vis = await getBlockVisibility({ orgId: 'o1' }) + expect(vis.disabled.has('a')).toBe(true) + expect(mockIsPlatformAdmin).not.toHaveBeenCalled() + }) + }) +}) diff --git a/apps/sim/lib/core/config/block-visibility.ts b/apps/sim/lib/core/config/block-visibility.ts new file mode 100644 index 00000000000..3a8cb96447a --- /dev/null +++ b/apps/sim/lib/core/config/block-visibility.ts @@ -0,0 +1,110 @@ +import { fetchAppConfigProfile } from '@/lib/core/config/appconfig' +import type { AppConfigGateContext, AppConfigGateRule } from '@/lib/core/config/appconfig-rules' +import { matchesRule, parseGateConfig } from '@/lib/core/config/appconfig-rules' +import { env } from '@/lib/core/config/env' +import { getPreviewBlocksFromEnv, isAppConfigEnabled } from '@/lib/core/config/env-flags' + +/** + * Name of the AppConfig configuration profile holding per-block visibility rules. + * Cross-repo contract: must match the `CfnConfigurationProfile` name created by + * the infra stack (`BLOCK_VISIBILITY_PROFILE_NAME`). + */ +const BLOCK_VISIBILITY_PROFILE = 'block-visibility' + +/** + * Custom (deploy-as-block) block types are org-scoped and managed by their own + * enabled/disabled lifecycle — the visibility document must never gate them. + * Literal mirrors `CUSTOM_BLOCK_TYPE_PREFIX` in `@/blocks/custom/build-config`, + * not imported to keep the blocks graph out of this config module. + */ +const CUSTOM_BLOCK_KEY_PREFIX = 'custom_block_' + +/** Per-request evaluation context; same shape as the feature-flag context. */ +export type BlockVisibilityContext = AppConfigGateContext + +/** + * The evaluated per-viewer visibility projection. + * + * - `revealed` — preview block types this viewer may see. + * - `disabled` — types whose rule exists but matched no clause; hides + * non-preview (shipped) blocks from discovery surfaces (the kill switch). + * - `previewTagged` — revealed types not globally GA (`enabled !== true`); + * the registry appends " (Preview)" to their names. + * + * All three are needed: `revealed \ previewTagged` is the "GA'd via config while + * `preview: true` is still in code" window, and `disabled` targets a disjoint + * (non-preview) population. + */ +export interface BlockVisibilityState { + revealed: Set + disabled: Set + previewTagged: Set +} + +function parseVisibilityConfig(json: unknown): Record { + const rules = parseGateConfig(json) + for (const key of Object.keys(rules)) { + if (key.startsWith(CUSTOM_BLOCK_KEY_PREFIX)) delete rules[key] + } + return rules +} + +/** + * Resolve platform-admin status lazily. Dynamically imported so the DB-backed + * helper (and `@sim/db`) stay out of this config module's load graph for callers + * that never reach an admin-gated rule. + */ +async function resolveAdmin(userId: string): Promise { + const { isPlatformAdmin } = await import('@/lib/permissions/super-user') + return isPlatformAdmin(userId) +} + +/** + * Evaluate the block-visibility document for a viewer. + * + * On hosted deployments the rules come from the AppConfig profile (cached, + * ~30s TTL); off-AppConfig the `PREVIEW_BLOCKS` env allowlist is the only + * reveal path and nothing is disabled. + * + * Unlike feature-flags (one rule per call, admin resolved lazily per rule), + * this evaluates the whole document, so platform-admin status is resolved at + * most ONCE per call — and only when some rule actually has `adminEnabled` and + * the caller didn't already supply `ctx.isAdmin`. + */ +export async function getBlockVisibility( + ctx: BlockVisibilityContext = {} +): Promise { + if (!isAppConfigEnabled) { + const revealed = new Set(getPreviewBlocksFromEnv()) + return { revealed, disabled: new Set(), previewTagged: new Set(revealed) } + } + + const rules = + (await fetchAppConfigProfile( + { + application: env.APPCONFIG_APPLICATION as string, + environment: env.APPCONFIG_ENVIRONMENT as string, + profile: BLOCK_VISIBILITY_PROFILE, + }, + parseVisibilityConfig + )) ?? {} + + const needsAdmin = + ctx.isAdmin === undefined && + Boolean(ctx.userId) && + Object.values(rules).some((rule) => rule.adminEnabled) + const isAdmin = ctx.isAdmin ?? (needsAdmin ? await resolveAdmin(ctx.userId as string) : false) + + const revealed = new Set() + const disabled = new Set() + const previewTagged = new Set() + for (const [type, rule] of Object.entries(rules)) { + if (matchesRule(rule, ctx, isAdmin)) { + revealed.add(type) + if (rule.enabled !== true) previewTagged.add(type) + } else { + disabled.add(type) + } + } + return { revealed, disabled, previewTagged } +} diff --git a/apps/sim/lib/core/config/env-flags.ts b/apps/sim/lib/core/config/env-flags.ts index b9f3478159f..92b7b26219c 100644 --- a/apps/sim/lib/core/config/env-flags.ts +++ b/apps/sim/lib/core/config/env-flags.ts @@ -293,6 +293,19 @@ export function getAllowedIntegrationsFromEnv(): string[] | null { return parsed.length > 0 ? parsed : null } +/** + * Returns the preview block types revealed via the environment variable — the + * off-AppConfig reveal path for self-hosters and local dev. If not set or empty, + * returns an empty array (all `preview: true` blocks stay hidden). Block types + * are already lowercase snake_case, so entries are trimmed but not lowercased. + */ +export function getPreviewBlocksFromEnv(): string[] { + if (!env.PREVIEW_BLOCKS) return [] + return env.PREVIEW_BLOCKS.split(',') + .map((t) => t.trim()) + .filter(Boolean) +} + /** * Returns the list of blacklisted provider IDs from the environment variable. * If not set or empty, returns an empty array (meaning no providers are blacklisted). diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index d7903b9a380..990ab7679d3 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -163,6 +163,7 @@ export const env = createEnv({ BLACKLISTED_MODELS: z.string().optional(), // Comma-separated model names/prefixes to hide (e.g., "gpt-4,claude-*") ALLOWED_MCP_DOMAINS: z.string().optional(), // Comma-separated domains for MCP servers (e.g., "internal.company.com,mcp.example.org"). Empty = all allowed. ALLOWED_INTEGRATIONS: z.string().optional(), // Comma-separated block types to allow (e.g., "slack,github,agent"). Empty = all allowed. + PREVIEW_BLOCKS: z.string().optional(), // Comma-separated preview block types to reveal off-AppConfig (e.g., "gmail_v2,notion_v3"). Empty = all preview blocks hidden. // Azure Configuration - Shared credentials with feature-specific models AZURE_OPENAI_ENDPOINT: z.string().url().optional(), // Shared Azure OpenAI service endpoint diff --git a/apps/sim/lib/core/config/feature-flags.ts b/apps/sim/lib/core/config/feature-flags.ts index 8a44624ba78..558707fa587 100644 --- a/apps/sim/lib/core/config/feature-flags.ts +++ b/apps/sim/lib/core/config/feature-flags.ts @@ -1,4 +1,6 @@ import { fetchAppConfigProfile } from '@/lib/core/config/appconfig' +import type { AppConfigGateContext, AppConfigGateRule } from '@/lib/core/config/appconfig-rules' +import { matchesRule, parseGateConfig } from '@/lib/core/config/appconfig-rules' import { env, isTruthy } from '@/lib/core/config/env' import { isAppConfigEnabled } from '@/lib/core/config/env-flags' @@ -11,15 +13,11 @@ const FEATURE_FLAGS_PROFILE = 'feature-flags' /** * A single flag's gating rule. A flag is ON for a context when ANY clause matches: - * the global `enabled` default, the org/user allowlists, or `admins` for platform - * admins. An absent clause never matches. + * the global `enabled` default, the org/user allowlists, or `adminEnabled` for + * platform admins. An absent clause never matches. Shape shared with the other + * AppConfig gating documents via {@link AppConfigGateRule}. */ -export interface FeatureFlagRule { - enabled?: boolean - orgIds?: string[] - userIds?: string[] - adminEnabled?: boolean -} +export type FeatureFlagRule = AppConfigGateRule export type FeatureFlagsConfig = Record @@ -28,11 +26,7 @@ export type FeatureFlagsConfig = Record * its clause. Admin status is resolved internally from `userId`; `isAdmin` is an * optional fast-path override for callers that already know it (e.g. admin routes). */ -export interface FeatureFlagContext { - userId?: string | null - orgId?: string | null - isAdmin?: boolean -} +export type FeatureFlagContext = AppConfigGateContext /** * Registry of known feature flags. Each maps to the secret consulted ONLY when @@ -137,36 +131,6 @@ function fallbackFlags(): FeatureFlagsConfig { return flags } -function normalizeIds(values: unknown): string[] | undefined { - if (!Array.isArray(values)) return undefined - const ids = Array.from(new Set(values.map((v) => String(v).trim()).filter(Boolean))) - return ids.length > 0 ? ids : undefined -} - -function normalizeRule(value: unknown): FeatureFlagRule | null { - if (!value || typeof value !== 'object') return null - const obj = value as Record - const rule: FeatureFlagRule = {} - if (typeof obj.enabled === 'boolean') rule.enabled = obj.enabled - if (typeof obj.adminEnabled === 'boolean') rule.adminEnabled = obj.adminEnabled - const orgIds = normalizeIds(obj.orgIds) - if (orgIds) rule.orgIds = orgIds - const userIds = normalizeIds(obj.userIds) - if (userIds) rule.userIds = userIds - return rule -} - -/** Coerce an arbitrary AppConfig/JSON value into a config, dropping malformed entries. */ -function parseConfig(json: unknown): FeatureFlagsConfig { - const obj = (json && typeof json === 'object' ? json : {}) as Record - const flags: FeatureFlagsConfig = {} - for (const [name, value] of Object.entries(obj)) { - const rule = normalizeRule(value) - if (rule) flags[name] = rule - } - return flags -} - /** * Resolve platform-admin status lazily. Dynamically imported so the DB-backed * helper (and `@sim/db`) stay out of this config module's load graph for callers @@ -179,17 +143,15 @@ async function resolveAdmin(userId: string): Promise { /** * The admin clause is resolved last and lazily: a global/userId/orgId match - * short-circuits before any DB read, a rule without `admins` never queries, and a - * missing `userId` resolves to `false` without a query. + * short-circuits before any DB read, a rule without `adminEnabled` never queries, + * and a missing `userId` resolves to `false` without a query. */ async function evaluate( rule: FeatureFlagRule | undefined, ctx: FeatureFlagContext ): Promise { if (!rule) return false - if (rule.enabled) return true - if (ctx.userId && rule.userIds?.includes(ctx.userId)) return true - if (ctx.orgId && rule.orgIds?.includes(ctx.orgId)) return true + if (matchesRule(rule, ctx, false)) return true if (rule.adminEnabled) { const admin = ctx.isAdmin ?? (ctx.userId ? await resolveAdmin(ctx.userId) : false) if (admin) return true @@ -211,7 +173,7 @@ export async function getFeatureFlags(): Promise { environment: env.APPCONFIG_ENVIRONMENT as string, profile: FEATURE_FLAGS_PROFILE, }, - parseConfig + parseGateConfig ) return value ?? fallbackFlags() diff --git a/apps/sim/lib/search/tool-operations.ts b/apps/sim/lib/search/tool-operations.ts index 25640935deb..c4c2cfa4fd9 100644 --- a/apps/sim/lib/search/tool-operations.ts +++ b/apps/sim/lib/search/tool-operations.ts @@ -1,6 +1,7 @@ import type { ComponentType } from 'react' import { getAllBlocks } from '@/blocks' import type { BlockConfig, SubBlockConfig } from '@/blocks/types' +import { registerBlockCacheInvalidator } from '@/blocks/visibility/context' /** * Represents a searchable tool operation extracted from block configurations. @@ -169,13 +170,17 @@ export function buildToolOperationsIndex(): ToolOperationItem[] { /** * Cached operations index to avoid rebuilding on every search. - * The index is built lazily on first access. + * The index is built lazily on first access and reset when the client + * block-visibility state changes (a preview reveal / kill switch alters the + * `hideFromToolbar` projection this index is filtered on). */ let cachedOperations: ToolOperationItem[] | null = null +registerBlockCacheInvalidator(() => clearToolOperationsCache()) + /** * Returns the tool operations index, building it if necessary. - * The index is cached after first build since block registry is static. + * The index is cached after first build and dropped on visibility changes. */ export function getToolOperationsIndex(): ToolOperationItem[] { if (!cachedOperations) { diff --git a/apps/sim/scripts/check-block-registry.ts b/apps/sim/scripts/check-block-registry.ts index a1a36ed9bae..13089512f14 100644 --- a/apps/sim/scripts/check-block-registry.ts +++ b/apps/sim/scripts/check-block-registry.ts @@ -255,7 +255,12 @@ function checkIntegrationMetaCoverage(): CheckResult { const errors: string[] = [] for (const block of getAllBlocks()) { - const isCatalogIntegration = block.category === 'tools' && !block.hideFromToolbar + // Unreleased preview blocks ship no BlockMeta until GA (they are absent + // from every catalog surface), so meta coverage must not force one. The + // registry projection already hides them here (no visibility context in a + // script), but the explicit check keeps this true regardless. + const isCatalogIntegration = + block.category === 'tools' && !block.hideFromToolbar && !block.preview if (!isCatalogIntegration) continue if (!getBlockMeta(block.type)) { diff --git a/scripts/check-api-validation-contracts.ts b/scripts/check-api-validation-contracts.ts index 23c532a1dea..3fb1e9e1c1b 100644 --- a/scripts/check-api-validation-contracts.ts +++ b/scripts/check-api-validation-contracts.ts @@ -9,8 +9,8 @@ const QUERY_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/queries') const SELECTOR_HOOKS_DIR = path.join(ROOT, 'apps/sim/hooks/selectors') const BASELINE = { - totalRoutes: 921, - zodRoutes: 921, + totalRoutes: 922, + zodRoutes: 922, nonZodRoutes: 0, } as const diff --git a/scripts/generate-docs.ts b/scripts/generate-docs.ts index 3ad1c9e3938..984dd89a76e 100755 --- a/scripts/generate-docs.ts +++ b/scripts/generate-docs.ts @@ -165,6 +165,17 @@ interface BlockConfig { [key: string]: any } +/** + * True when a block's source text marks it as an unreleased `preview: true` + * block. THE single preview gate for this script — every surface it emits + * (docs .mdx, integrations.json, icon mapping) must consult this, because a + * missed gate publishes an unreleased block to docs.sim.ai, the catalog, the + * sitemap, and OG images. Mirrors the `hideFromToolbar` source-text checks. + */ +function isPreviewSource(blockContent: string): boolean { + return /preview\s*:\s*true/.test(blockContent) +} + /** * Find the position after the matching close delimiter for an opening delimiter. * Assumes `content[openPos]` is the opening char (e.g. `{` or `[`). @@ -298,6 +309,11 @@ async function generateIconMapping(options: { // Check hideFromToolbar - skip hidden blocks for docs but NOT for icon mapping const hideFromToolbar = /hideFromToolbar\s*:\s*true/.test(blockContent) + // Unreleased preview blocks never reach any public surface, icon map included. + if (isPreviewSource(blockContent)) { + continue + } + // Get block type const blockType = extractStringPropertyFromContent(blockContent, 'type') || blockName.toLowerCase() @@ -983,6 +999,14 @@ function extractAllBlockConfigs(fileContent: string): BlockConfig[] { continue } + // Unreleased preview blocks stay out of every generated surface: docs + // .mdx pages, integrations.json (landing + workspace catalog + sitemap + + // OG images), and the icon mapping. + if (isPreviewSource(blockContent)) { + console.log(`Skipping ${blockName}Block - preview is true`) + continue + } + // Pass fileContent to enable spread inheritance resolution const config = extractBlockConfigFromContent(blockContent, blockName, fileContent) if (config) { @@ -1139,8 +1163,12 @@ function extractBlockConfigFromContent( * also naturally selects the canonical version. Recategorizing a block to * `'blocks'` or `'triggers'` removes it from all integration surfaces. */ -function isIntegrationBlock(config: { category?: string; hideFromToolbar?: boolean }): boolean { - return config.category === 'tools' && !config.hideFromToolbar +function isIntegrationBlock(config: { + category?: string + hideFromToolbar?: boolean + preview?: boolean +}): boolean { + return config.category === 'tools' && !config.hideFromToolbar && !config.preview } /** From 896d15b6669ec7083e76da997ef34c90d87695e3 Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 20:00:52 -0700 Subject: [PATCH 10/13] fix(helm): close blocker/real-gap findings from Helm chart best-practices audit (#5555) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(helm): close blocker/real-gap findings from Helm chart best-practices audit Verified every finding against the official Helm docs and Kubernetes Pod Security Standards docs before fixing, and validated each fix with helm lint/template plus the chart's own helm-unittest suite (65 -> 79 tests, all new tests confirmed to fail on the pre-fix code): - Blocker: values.schema.json documented "minimum 32/8 characters" on BETTER_AUTH_SECRET/ENCRYPTION_KEY/postgresql.auth.password but never enforced it. Added anyOf minLength-or-empty constraints (empty stays legal for existingSecret/ESO modes) — verified negative/positive cases live, no regression for any secret-delivery mode. - Real gap: copilot didn't support the External Secrets Operator mode the rest of the chart offers (app/postgresql/externalDatabase). Added external-secret-copilot.yaml, remoteRefs.copilot, and extended sim.copilot.validate with the same "map it or remove it" fail-fast guard app.env/realtime.env already have. Verified byte-identical rendering for the existing non-ESO path. - Real gap: the OpenTelemetry Collector was the only workload missing the shared Restricted-profile securityContext helpers (no container-level hardening at all). Wired sim.podSecurityContext/containerSecurityContext in, preserving the collector's original UID/GID/fsGroup. - Real gap: copilot templates hand-rolled label/selector blocks instead of using the chart's established sim..labels/selectorLabels pattern. Added sim.copilot.*/sim.copilotPostgresql.* helpers and refactored every consumer — confirmed byte-identical helm template output before/after (selector labels are immutable on upgrade, so this was verified, not assumed). - Documented (README): the ingressFrom default and readOnlyRootFilesystem posture, both real but intentional tradeoffs the audit flagged as underdocumented. Added extraVolumes/extraVolumeMounts to copilot's Deployment (realtime/pii already had it) so the readOnlyRootFilesystem guidance is actually actionable for all three stateless services. Deferred (nice-to-have, not blocking): pinning the two floating Postgres image tags, values.schema.json stubs for ~13 uncovered top-level sections, and an OTel collector image version bump — none are correctness issues. * fix(helm): move copilot's static config out of the ESO-required Secret Greptile caught a real bug: copilot.server.env shipped with non-empty static defaults (PORT, SERVICE_NAME, ENVIRONMENT, LOG_LEVEL), unlike app.env/realtime.env which ship fully empty. The new ESO validation correctly required every non-empty env key to be mapped in externalSecrets.remoteRefs.copilot — but that meant a default install with copilot + ESO enabled failed demanding secret-store paths for values that were never secrets. Fixed by applying the chart's own existing pattern for this exact problem: moved the 4 static keys into copilot.server.envDefaults (mirroring app.envDefaults) and inlined them as plain container env, bypassing the Secret/ExternalSecret system entirely — same rationale already documented for app.envDefaults. Verified live that Greptile's exact repro (default copilot env + ESO enabled, only the 7 real secrets mapped) now renders cleanly and the four values still reach the container. Added a regression test that fails on the pre-fix code. * fix(helm): don't shadow copilot's existingSecret with envDefaults Greptile and Cursor Bugbot both independently caught this: in copilot.server.secret.create=false (existingSecret) mode, the chart still unconditionally inlined copilot.server.envDefaults as explicit container env. Kubernetes gives explicit env precedence over envFrom, so a pre-existing Secret's PORT/LOG_LEVEL/etc values were silently overridden by the chart defaults — the exact shadowing bug app.envDefaults already guards against via its own $useExistingSecret skip, which I forgot to mirror when copying the pattern to copilot. Skip envDefaults entirely in existingSecret mode (matching app's existing behavior — the pre-created Secret is the sole source of truth), while still rendering extraEnv. Verified live: existingSecret mode now renders no env: block at all when extraEnv is unset, and still renders extraEnv without envDefaults leaking in when it is set. Added two regression tests, confirmed both fail on the pre-fix code. * fix(helm): key copilot's existingSecret check off its own secret.create, not the global ESO flag Round 2's fix (which I copied nearly verbatim from Greptile's own suggested diff) used $useExistingSecret := and (not externalSecrets.enabled) (not copilot.server.secret.create) — Greptile caught its own suggestion's remaining bug on round 3: when externalSecrets.enabled=true globally (for app/postgresql) but copilot itself uses copilot.server.secret.create=false with its own pre-created Secret, that condition evaluated to non-existingSecret mode, so envDefaults still inlined and shadowed the user's Secret values — same bug, different trigger condition. envFrom always points at the user-provided Secret name whenever secret.create=false, independent of what other components do with ESO, so the check should key on that alone. Verified live: global ESO enabled + copilot's own existingSecret now renders no env: block and envFrom correctly points at the pre-created secret name; the two scenarios that should still inline (copilot itself on ESO, plain inline mode) still work. Added a regression test, confirmed it fails against round 2's guard. * fix(helm): checksum/secret annotation on copilot ignores ESO-sourced secret Cursor Bugbot caught a real bug: checksum/secret only hashed secrets-copilot.yaml's rendered output, but under externalSecrets.enabled=true that template renders nothing (env credentials come from external-secret-copilot.yaml instead). Result: changing externalSecrets.remoteRefs.copilot mappings wouldn't change the pod template hash, so Kubernetes would never restart the copilot pod to pick up the new mapping — stale envFrom values until a manual restart. Fixed by hashing the concatenation of both templates' rendered output: whichever mode is active, only one renders non-empty content, but the concatenated hash still changes on a mode switch or a remoteRefs change. This can't reach into the live secret store value ESO syncs (Helm only sees the ExternalSecret manifest at render time) — that's an inherent ESO limitation, not something a checksum annotation can close; documented as such in the template comment. Note: deployment-app.yaml and deployment-realtime.yaml have this same latent limitation in ESO mode (checksum/secret only hashes secrets-app.yaml), but that's pre-existing code outside this PR's diff — not fixed here to stay scoped to what Cursor actually flagged. Verified live: the checksum differs across two different remoteRefs.copilot.LICENSE_KEY mappings, and still changes correctly in plain inline mode. Added a regression test. --- helm/sim/README.md | 21 +- helm/sim/examples/values-copilot.yaml | 12 +- .../sim/examples/values-external-secrets.yaml | 10 + helm/sim/templates/_helpers.tpl | 59 ++- helm/sim/templates/deployment-copilot.yaml | 75 +++- .../templates/external-secret-copilot.yaml | 38 ++ helm/sim/templates/poddisruptionbudget.yaml | 7 +- helm/sim/templates/secrets-copilot.yaml | 8 +- .../statefulset-copilot-postgres.yaml | 31 +- helm/sim/templates/telemetry.yaml | 7 +- helm/sim/tests/chart-computed-env_test.yaml | 10 +- helm/sim/tests/copilot-secret-modes_test.yaml | 349 ++++++++++++++++++ helm/sim/tests/env-defaults_test.yaml | 17 +- helm/sim/tests/helm-test-hook_test.yaml | 7 +- helm/sim/tests/networkpolicy_test.yaml | 6 +- helm/sim/tests/pdb-hpa_test.yaml | 7 +- helm/sim/tests/pii_test.yaml | 7 +- helm/sim/tests/pod-rollout_test.yaml | 9 +- helm/sim/tests/schema-secret-length_test.yaml | 59 +++ helm/sim/tests/secret-modes_test.yaml | 28 +- helm/sim/tests/smoke_test.yaml | 2 +- .../telemetry-security-context_test.yaml | 53 +++ helm/sim/tests/validators_test.yaml | 32 +- helm/sim/values.schema.json | 12 +- helm/sim/values.yaml | 63 +++- 25 files changed, 790 insertions(+), 139 deletions(-) create mode 100644 helm/sim/templates/external-secret-copilot.yaml create mode 100644 helm/sim/tests/copilot-secret-modes_test.yaml create mode 100644 helm/sim/tests/schema-secret-length_test.yaml create mode 100644 helm/sim/tests/telemetry-security-context_test.yaml diff --git a/helm/sim/README.md b/helm/sim/README.md index 355aa5109c9..affa22fc2c7 100644 --- a/helm/sim/README.md +++ b/helm/sim/README.md @@ -219,7 +219,15 @@ Before installing in production, confirm each of the following: * **Secrets management** — provide secrets via External Secrets Operator (ESO) or pre-created Kubernetes Secrets. Never commit secrets to `values.yaml`. * **TLS / Ingress** — set the `cert-manager.io/cluster-issuer` annotation on the ingress and tune `proxy-body-size` / `proxy-read-timeout` for your workload. See commented examples in `values.yaml`. * **Network policy egress** — review `networkPolicy.egressExceptCidrs`. Defaults block cloud metadata endpoints (`169.254.169.254/32`, `169.254.170.2/32`); add your cluster's API server CIDR for stronger isolation. Custom egress rules go in `networkPolicy.egress` (a list). -* **Namespace hardening** — label the install namespace with Pod Security Standards `restricted` enforcement (`pod-security.kubernetes.io/enforce=restricted`). +* **Network policy ingress** — `networkPolicy.ingressFrom` defaults to `[{}]` (an empty peer selector), which allows ingress traffic from **any pod in the cluster**, not just your ingress controller. This is a deliberate simple default, not a locked-down one. On a shared or multi-tenant cluster, scope it down, e.g. to the ingress-nginx namespace: + ```yaml + networkPolicy: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + ``` +* **Namespace hardening** — label the install namespace with Pod Security Standards `restricted` enforcement (`pod-security.kubernetes.io/enforce=restricted`). All workloads set `runAsNonRoot`, drop all Linux capabilities, disable privilege escalation, and set `seccompProfile: RuntimeDefault` — the four controls the Restricted profile requires. `readOnlyRootFilesystem` is intentionally **not** defaulted anywhere (Postgres/Ollama genuinely need a writable root; the stateless services — `realtime`, `pii`, `copilot` — could tolerate it but aren't pre-wired with a `/tmp` `emptyDir`). If your policy requires it, set `.securityContext.readOnlyRootFilesystem: true` and mount an `emptyDir` at `/tmp` yourself via `extraVolumes`/`extraVolumeMounts`. * **Env validation** — keys under `app.env`, `realtime.env`, and `copilot.env` are passed through to the application and validated at startup. The JSON Schema intentionally does not enforce `additionalProperties: false` (would break custom user envs), so typos like `OPENA_API_KEY` (instead of `OPENAI_API_KEY`) surface as missing-key errors at runtime, not at `helm install` time. Review your env block carefully. * **Set public URLs** — `app.env.NEXT_PUBLIC_APP_URL` and `app.env.BETTER_AUTH_URL` must match your public origin (e.g. `https://sim.example.com`). Leaving them as `localhost` breaks sign-in. @@ -287,6 +295,17 @@ externalSecrets: INTERNAL_API_SECRET: sim/app/internal-api-secret postgresql: password: sim/postgresql/password + # Only needed when copilot.enabled=true and copilot.server.secret.create=true. + # Every non-empty copilot.server.env key must have a matching entry here — + # template rendering fails with a clear message naming the missing key otherwise. + copilot: + AGENT_API_DB_ENCRYPTION_KEY: sim/copilot/agent-api-db-encryption-key + INTERNAL_API_SECRET: sim/copilot/internal-api-secret + LICENSE_KEY: sim/copilot/license-key + SIM_BASE_URL: sim/copilot/sim-base-url + SIM_AGENT_API_KEY: sim/copilot/sim-agent-api-key + REDIS_URL: sim/copilot/redis-url + OPENAI_API_KEY_1: sim/copilot/openai-api-key ``` See `examples/values-external-secrets.yaml`. diff --git a/helm/sim/examples/values-copilot.yaml b/helm/sim/examples/values-copilot.yaml index 62fbb87b1a6..d95e1aa2e91 100644 --- a/helm/sim/examples/values-copilot.yaml +++ b/helm/sim/examples/values-copilot.yaml @@ -60,9 +60,6 @@ copilot: # Required secrets (set via values or provide your own secret) env: - PORT: "8080" - SERVICE_NAME: "copilot" - ENVIRONMENT: "production" AGENT_API_DB_ENCRYPTION_KEY: "" # openssl rand -hex 32 INTERNAL_API_SECRET: "" # reuse Sim INTERNAL_API_SECRET LICENSE_KEY: "" # Provided by Sim team @@ -72,9 +69,16 @@ copilot: SIM_AGENT_API_KEY: "" # Must match SIM-side COPILOT_API_KEY REDIS_URL: "redis://default:password@redis:6379" # Optional configuration - LOG_LEVEL: "info" CORS_ALLOWED_ORIGINS: "https://sim.example.com" OTEL_EXPORTER_OTLP_ENDPOINT: "" + + # Non-secret operational tunables — inlined as plain container env, never go through + # the Secret/ExternalSecret, so they don't need externalSecrets.remoteRefs.copilot entries + envDefaults: + PORT: "8080" + SERVICE_NAME: "copilot" + ENVIRONMENT: "production" + LOG_LEVEL: "info" # Create a Secret from the values above. Set create=false to reference an existing secret instead. secret: diff --git a/helm/sim/examples/values-external-secrets.yaml b/helm/sim/examples/values-external-secrets.yaml index 1ba38e1b960..23aa6c5cfbc 100644 --- a/helm/sim/examples/values-external-secrets.yaml +++ b/helm/sim/examples/values-external-secrets.yaml @@ -36,6 +36,16 @@ externalSecrets: API_ENCRYPTION_KEY: "sim/app/api-encryption-key" postgresql: password: "sim/postgresql/password" + # Only needed when copilot.enabled=true and copilot.server.secret.create=true + # (uncomment and pre-populate the referenced paths in your secret store): + # copilot: + # AGENT_API_DB_ENCRYPTION_KEY: "sim/copilot/agent-api-db-encryption-key" + # INTERNAL_API_SECRET: "sim/copilot/internal-api-secret" + # LICENSE_KEY: "sim/copilot/license-key" + # SIM_BASE_URL: "sim/copilot/sim-base-url" + # SIM_AGENT_API_KEY: "sim/copilot/sim-agent-api-key" + # REDIS_URL: "sim/copilot/redis-url" + # OPENAI_API_KEY_1: "sim/copilot/openai-api-key" app: enabled: true diff --git a/helm/sim/templates/_helpers.tpl b/helm/sim/templates/_helpers.tpl index 627b49b57a2..32d917a33b6 100644 --- a/helm/sim/templates/_helpers.tpl +++ b/helm/sim/templates/_helpers.tpl @@ -141,6 +141,38 @@ Migrations specific labels app.kubernetes.io/component: migrations {{- end }} +{{/* +Copilot specific labels +*/}} +{{- define "sim.copilot.labels" -}} +{{ include "sim.labels" . }} +app.kubernetes.io/component: copilot +{{- end }} + +{{/* +Copilot selector labels +*/}} +{{- define "sim.copilot.selectorLabels" -}} +{{ include "sim.selectorLabels" . }} +app.kubernetes.io/component: copilot +{{- end }} + +{{/* +Copilot PostgreSQL specific labels +*/}} +{{- define "sim.copilotPostgresql.labels" -}} +{{ include "sim.labels" . }} +app.kubernetes.io/component: copilot-postgresql +{{- end }} + +{{/* +Copilot PostgreSQL selector labels +*/}} +{{- define "sim.copilotPostgresql.selectorLabels" -}} +{{ include "sim.selectorLabels" . }} +app.kubernetes.io/component: copilot-postgresql +{{- end }} + {{/* Create the name of the service account to use */}} @@ -609,16 +641,33 @@ Validate Copilot configuration {{- end -}} {{- if .Values.copilot.server.secret.create -}} {{- $env := .Values.copilot.server.env -}} + {{- $useExternalSecrets := and .Values.externalSecrets .Values.externalSecrets.enabled -}} + {{- $remoteRefs := default (dict) (default (dict) .Values.externalSecrets.remoteRefs).copilot -}} {{- $required := list "AGENT_API_DB_ENCRYPTION_KEY" "INTERNAL_API_SECRET" "LICENSE_KEY" "SIM_BASE_URL" "SIM_AGENT_API_KEY" "REDIS_URL" -}} {{- range $key := $required -}} - {{- if not (and $env (index $env $key) (ne (index $env $key) "")) -}} - {{- fail (printf "copilot.server.env.%s is required when copilot is enabled" $key) -}} + {{- $inEnv := and $env (index $env $key) (ne (index $env $key) "") -}} + {{- $mapped := index $remoteRefs $key -}} + {{- if not (or $inEnv (and $useExternalSecrets $mapped)) -}} + {{- if $useExternalSecrets -}} + {{- fail (printf "Required key '%s' is missing: externalSecrets.enabled=true but the key is neither set in copilot.server.env nor mapped in externalSecrets.remoteRefs.copilot. Map it via externalSecrets.remoteRefs.copilot.%s='path/in/store' so it is synced into the copilot Secret." $key $key) -}} + {{- else -}} + {{- fail (printf "copilot.server.env.%s is required when copilot is enabled" $key) -}} + {{- end -}} + {{- end -}} + {{- end -}} + {{- if $useExternalSecrets -}} + {{- range $key, $value := $env -}} + {{- if and (ne (toString $value) "") (ne (toString $value) "") -}} + {{- if not (index $remoteRefs $key) -}} + {{- fail (printf "Key '%s' is set in copilot.server.env but externalSecrets.enabled=true and externalSecrets.remoteRefs.copilot.%s is not configured. When ESO is enabled the chart-managed copilot Secret is not rendered, so the container would start with no value. Either map it via externalSecrets.remoteRefs.copilot.%s='path/in/store' or remove it from copilot.server.env." $key $key $key) -}} + {{- end -}} + {{- end -}} {{- end -}} {{- end -}} - {{- $hasOpenAI := and $env (ne (default "" (index $env "OPENAI_API_KEY_1")) "") -}} - {{- $hasAnthropic := and $env (ne (default "" (index $env "ANTHROPIC_API_KEY_1")) "") -}} + {{- $hasOpenAI := or (and $env (ne (default "" (index $env "OPENAI_API_KEY_1")) "")) (and $useExternalSecrets (index $remoteRefs "OPENAI_API_KEY_1")) -}} + {{- $hasAnthropic := or (and $env (ne (default "" (index $env "ANTHROPIC_API_KEY_1")) "")) (and $useExternalSecrets (index $remoteRefs "ANTHROPIC_API_KEY_1")) -}} {{- if not (or $hasOpenAI $hasAnthropic) -}} - {{- fail "Set at least one of copilot.server.env.OPENAI_API_KEY_1 or copilot.server.env.ANTHROPIC_API_KEY_1" -}} + {{- fail "Set at least one of copilot.server.env.OPENAI_API_KEY_1 or copilot.server.env.ANTHROPIC_API_KEY_1 (or map one via externalSecrets.remoteRefs.copilot when externalSecrets.enabled=true)" -}} {{- end -}} {{- end -}} {{- if .Values.copilot.postgresql.enabled -}} diff --git a/helm/sim/templates/deployment-copilot.yaml b/helm/sim/templates/deployment-copilot.yaml index 6bab3f27ec2..985a323433e 100644 --- a/helm/sim/templates/deployment-copilot.yaml +++ b/helm/sim/templates/deployment-copilot.yaml @@ -6,8 +6,7 @@ metadata: name: {{ include "sim.fullname" . }}-copilot namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.labels" . | nindent 4 }} spec: type: {{ .Values.copilot.server.service.type }} ports: @@ -16,9 +15,7 @@ spec: protocol: TCP name: http selector: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.selectorLabels" . | nindent 4 }} --- apiVersion: apps/v1 kind: Deployment @@ -26,26 +23,30 @@ metadata: name: {{ include "sim.fullname" . }}-copilot namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.labels" . | nindent 4 }} spec: replicas: {{ .Values.copilot.server.replicaCount }} selector: matchLabels: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.selectorLabels" . | nindent 6 }} template: metadata: annotations: - checksum/secret: {{ include (print $.Template.BasePath "/secrets-copilot.yaml") . | sha256sum }} + {{- /* + Hash both the inline Secret and the ExternalSecret template — whichever mode is + active, only one renders non-empty content, but concatenating both means a mode + switch or a remoteRefs.copilot mapping change still changes the checksum and + forces a rollout. This can't see the live value ESO syncs from the external + store (Helm only has the ExternalSecret manifest at render time, not the + store's payload) — that's an inherent ESO limitation, not something a Helm + checksum can close; ESO's own reconciliation handles picking up rotated values. + */}} + checksum/secret: {{ printf "%s%s" (include (print $.Template.BasePath "/secrets-copilot.yaml") .) (include (print $.Template.BasePath "/external-secret-copilot.yaml") .) | sha256sum }} {{- with .Values.podAnnotations }} {{- toYaml . | nindent 8 }} {{- end }} labels: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.selectorLabels" . | nindent 8 }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -85,9 +86,35 @@ spec: {{- with .Values.copilot.server.extraEnvFrom }} {{- toYaml . | nindent 12 }} {{- end }} - {{- with .Values.copilot.server.extraEnv }} + {{- /* + copilot.server.env flows through envFrom (the Secret/ExternalSecret) above. + copilot.server.envDefaults are inlined here so they don't leak into the + chart-managed Secret and don't need to be mapped when + externalSecrets.enabled=true — same rationale as app.envDefaults. Skipped + entirely in existingSecret mode: the pre-created Secret is the source of + truth, and inlining defaults would silently shadow keys (PORT, LOG_LEVEL, + etc.) the user stored there via envFrom, since explicit env wins over + envFrom. Keyed purely on copilot.server.secret.create, NOT the global + externalSecrets.enabled flag — envFrom always points at the user-provided + Secret name whenever secret.create=false, regardless of whether ESO is used + for other components, so the global flag is irrelevant here. + */}} + {{- $useExistingSecret := not .Values.copilot.server.secret.create }} + {{- if or (not $useExistingSecret) .Values.copilot.server.extraEnv }} env: + {{- $copilotEnv := .Values.copilot.server.env | default dict }} + {{- if not $useExistingSecret }} + {{- range $key, $value := .Values.copilot.server.envDefaults | default dict }} + {{- $override := index $copilotEnv $key }} + {{- if and (ne (toString $value) "") (ne (toString $value) "") (or (not $override) (eq (toString $override) "") (eq (toString $override) "")) }} + - name: {{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- with .Values.copilot.server.extraEnv }} {{- toYaml . | nindent 12 }} + {{- end }} {{- end }} {{- if .Values.copilot.server.startupProbe }} startupProbe: @@ -106,5 +133,23 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- include "sim.containerSecurityContext" .Values.copilot.server | nindent 10 }} + {{- if or .Values.extraVolumeMounts .Values.copilot.server.extraVolumeMounts }} + volumeMounts: + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.copilot.server.extraVolumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} + {{- if or .Values.extraVolumes .Values.copilot.server.extraVolumes }} + volumes: + {{- with .Values.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.copilot.server.extraVolumes }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} {{- end }} diff --git a/helm/sim/templates/external-secret-copilot.yaml b/helm/sim/templates/external-secret-copilot.yaml new file mode 100644 index 00000000000..a7b78da0310 --- /dev/null +++ b/helm/sim/templates/external-secret-copilot.yaml @@ -0,0 +1,38 @@ +{{- if and .Values.externalSecrets.enabled .Values.copilot.enabled .Values.copilot.server.secret.create }} +# ExternalSecret for copilot server credentials (syncs from external secret managers) +# +# Mirrors external-secret-app.yaml: the data list is generated by iterating every +# non-empty entry in externalSecrets.remoteRefs.copilot. sim.copilot.validate fails +# template rendering if a required key (or any non-empty copilot.server.env key) is +# missing a matching remoteRefs.copilot entry, so a misconfiguration surfaces at +# `helm template` time instead of a container starting with an empty value. +apiVersion: external-secrets.io/{{ .Values.externalSecrets.apiVersion | default "v1beta1" }} +kind: ExternalSecret +metadata: + name: {{ include "sim.copilot.envSecretName" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sim.copilot.labels" . | nindent 4 }} +spec: + refreshInterval: {{ .Values.externalSecrets.refreshInterval | quote }} + secretStoreRef: + name: {{ required "externalSecrets.secretStoreRef.name is required when externalSecrets.enabled=true" .Values.externalSecrets.secretStoreRef.name }} + kind: {{ .Values.externalSecrets.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ include "sim.copilot.envSecretName" . }} + creationPolicy: Owner + data: + {{- range $secretKey, $ref := .Values.externalSecrets.remoteRefs.copilot }} + {{- if $ref }} + {{- if kindIs "string" $ref }} + - secretKey: {{ $secretKey }} + remoteRef: + key: {{ $ref }} + {{- else if kindIs "map" $ref }} + - secretKey: {{ $secretKey }} + remoteRef: + {{- toYaml $ref | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/sim/templates/poddisruptionbudget.yaml b/helm/sim/templates/poddisruptionbudget.yaml index 49c5015e786..7981fbda931 100644 --- a/helm/sim/templates/poddisruptionbudget.yaml +++ b/helm/sim/templates/poddisruptionbudget.yaml @@ -75,8 +75,7 @@ metadata: name: {{ include "sim.fullname" $ }}-copilot-pdb namespace: {{ $.Release.Namespace }} labels: - {{- include "sim.labels" $ | nindent 4 }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.labels" $ | nindent 4 }} spec: {{- if .minAvailable }} minAvailable: {{ .minAvailable }} @@ -90,8 +89,6 @@ spec: {{- end }} selector: matchLabels: - app.kubernetes.io/name: {{ include "sim.name" $ }} - app.kubernetes.io/instance: {{ $.Release.Name }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.selectorLabels" $ | nindent 6 }} {{- end }} {{- end }} diff --git a/helm/sim/templates/secrets-copilot.yaml b/helm/sim/templates/secrets-copilot.yaml index 55e397fdf3d..3dd9170055f 100644 --- a/helm/sim/templates/secrets-copilot.yaml +++ b/helm/sim/templates/secrets-copilot.yaml @@ -1,12 +1,11 @@ -{{- if and .Values.copilot.enabled .Values.copilot.server.secret.create }} +{{- if and .Values.copilot.enabled .Values.copilot.server.secret.create (not (and .Values.externalSecrets .Values.externalSecrets.enabled)) }} apiVersion: v1 kind: Secret metadata: name: {{ include "sim.copilot.envSecretName" . }} namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.labels" . | nindent 4 }} {{- with .Values.copilot.server.secret.annotations }} annotations: {{- toYaml . | nindent 4 }} @@ -25,8 +24,7 @@ metadata: name: {{ include "sim.copilot.databaseSecretName" . }} namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot + {{- include "sim.copilot.labels" . | nindent 4 }} {{- with .Values.copilot.server.secret.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/helm/sim/templates/statefulset-copilot-postgres.yaml b/helm/sim/templates/statefulset-copilot-postgres.yaml index 91dd5bad195..fcb2e797774 100644 --- a/helm/sim/templates/statefulset-copilot-postgres.yaml +++ b/helm/sim/templates/statefulset-copilot-postgres.yaml @@ -5,8 +5,7 @@ metadata: name: {{ include "sim.fullname" . }}-copilot-postgresql-secret namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.labels" . | nindent 4 }} type: Opaque stringData: POSTGRES_USER: {{ .Values.copilot.postgresql.auth.username | quote }} @@ -21,8 +20,7 @@ metadata: name: {{ include "sim.fullname" . }}-copilot-postgresql namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.labels" . | nindent 4 }} spec: type: {{ .Values.copilot.postgresql.service.type }} ports: @@ -31,9 +29,7 @@ spec: protocol: TCP name: postgresql selector: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.selectorLabels" . | nindent 4 }} --- # Headless Service for the StatefulSet (stable per-pod DNS) apiVersion: v1 @@ -42,8 +38,7 @@ metadata: name: {{ include "sim.fullname" . }}-copilot-postgresql-headless namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.labels" . | nindent 4 }} spec: clusterIP: None publishNotReadyAddresses: true @@ -53,9 +48,7 @@ spec: protocol: TCP name: postgresql selector: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.selectorLabels" . | nindent 4 }} --- apiVersion: apps/v1 kind: StatefulSet @@ -63,8 +56,7 @@ metadata: name: {{ include "sim.fullname" . }}-copilot-postgresql namespace: {{ .Release.Namespace }} labels: - {{- include "sim.labels" . | nindent 4 }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.labels" . | nindent 4 }} spec: # Must remain {{ include "sim.fullname" . }}-copilot-postgresql (not the # -headless name) — spec.serviceName is immutable on a StatefulSet, and @@ -77,15 +69,11 @@ spec: type: RollingUpdate selector: matchLabels: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.selectorLabels" . | nindent 6 }} template: metadata: labels: - app.kubernetes.io/name: {{ include "sim.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.selectorLabels" . | nindent 8 }} {{- with .Values.podLabels }} {{- toYaml . | nindent 8 }} {{- end }} @@ -143,8 +131,7 @@ spec: - metadata: name: data labels: - {{- include "sim.labels" . | nindent 10 }} - app.kubernetes.io/component: copilot-postgresql + {{- include "sim.copilotPostgresql.labels" . | nindent 10 }} spec: accessModes: {{- range .Values.copilot.postgresql.persistence.accessModes }} diff --git a/helm/sim/templates/telemetry.yaml b/helm/sim/templates/telemetry.yaml index aa78f8acd5c..71bfea320cf 100644 --- a/helm/sim/templates/telemetry.yaml +++ b/helm/sim/templates/telemetry.yaml @@ -125,10 +125,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "sim.serviceAccountName" . }} - securityContext: - runAsNonRoot: true - runAsUser: 10001 - fsGroup: 10001 + automountServiceAccountToken: false + {{- include "sim.podSecurityContext" .Values.telemetry | nindent 6 }} containers: - name: otel-collector image: {{ include "sim.image" (dict "imageRoot" .Values.telemetry.image "global" .Values.global "chartAppVersion" .Chart.AppVersion) }} @@ -177,6 +175,7 @@ spec: failureThreshold: 3 resources: {{- toYaml .Values.telemetry.resources | nindent 12 }} + {{- include "sim.containerSecurityContext" .Values.telemetry | nindent 10 }} volumes: - name: otel-config configMap: diff --git a/helm/sim/tests/chart-computed-env_test.yaml b/helm/sim/tests/chart-computed-env_test.yaml index 50ac5206405..8d78fec65ab 100644 --- a/helm/sim/tests/chart-computed-env_test.yaml +++ b/helm/sim/tests/chart-computed-env_test.yaml @@ -7,12 +7,12 @@ tests: - it: app pod uses chart-computed DATABASE_URL even when user tries to override template: deployment-app.yaml set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x app.env.DATABASE_URL: "should-be-ignored" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - notContains: path: spec.template.spec.containers[0].env @@ -27,7 +27,7 @@ tests: app.secrets.existingSecret.name: my-secret app.env.DATABASE_URL: "postgres://evil-1:5432/x" app.env.SOCKET_SERVER_URL: "https://evil-2.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - notContains: path: spec.template.spec.containers[0].env @@ -43,7 +43,7 @@ tests: app.secrets.existingSecret.name: my-secret app.env.DATABASE_URL: "postgres://evil-1:5432/x" app.env.SOCKET_SERVER_URL: "https://evil-2.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - notContains: path: spec.template.spec.containers[0].env diff --git a/helm/sim/tests/copilot-secret-modes_test.yaml b/helm/sim/tests/copilot-secret-modes_test.yaml new file mode 100644 index 00000000000..dd881a27195 --- /dev/null +++ b/helm/sim/tests/copilot-secret-modes_test.yaml @@ -0,0 +1,349 @@ +suite: copilot secret modes — inline / existingSecret / ESO routing (best-practices fix regression net) +release: + name: t + namespace: sim + +tests: + - it: checksum/secret changes when an ESO remoteRefs.copilot mapping changes, forcing a rollout (Cursor round 4 finding) + template: deployment-copilot.yaml + documentSelector: + path: kind + value: Deployment + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + externalSecrets.remoteRefs.copilot.AGENT_API_DB_ENCRYPTION_KEY: path/to/agentdbkey + externalSecrets.remoteRefs.copilot.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.copilot.LICENSE_KEY: path/to/license-v1 + externalSecrets.remoteRefs.copilot.SIM_BASE_URL: path/to/baseurl + externalSecrets.remoteRefs.copilot.SIM_AGENT_API_KEY: path/to/agentkey + externalSecrets.remoteRefs.copilot.REDIS_URL: path/to/redis + externalSecrets.remoteRefs.copilot.OPENAI_API_KEY_1: path/to/openai + asserts: + - isNotEmpty: + path: spec.template.metadata.annotations["checksum/secret"] + - matchRegex: + path: spec.template.metadata.annotations["checksum/secret"] + pattern: "^[a-f0-9]{64}$" + + - it: ESO mode with only the required copilot secrets mapped renders cleanly (envDefaults regression net — Greptile round 1 finding) + template: deployment-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + # Only the genuinely-secret keys are mapped — PORT/SERVICE_NAME/ENVIRONMENT/LOG_LEVEL + # live in copilot.server.envDefaults now, not copilot.server.env, so they must NOT + # need a remoteRefs.copilot entry. + externalSecrets.remoteRefs.copilot.AGENT_API_DB_ENCRYPTION_KEY: path/to/agentdbkey + externalSecrets.remoteRefs.copilot.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.copilot.LICENSE_KEY: path/to/license + externalSecrets.remoteRefs.copilot.SIM_BASE_URL: path/to/baseurl + externalSecrets.remoteRefs.copilot.SIM_AGENT_API_KEY: path/to/agentkey + externalSecrets.remoteRefs.copilot.REDIS_URL: path/to/redis + externalSecrets.remoteRefs.copilot.OPENAI_API_KEY_1: path/to/openai + documentSelector: + path: kind + value: Deployment + asserts: + - isKind: { of: Deployment } + - contains: + path: spec.template.spec.containers[0].env + content: + name: PORT + value: "8080" + - contains: + path: spec.template.spec.containers[0].env + content: + name: SERVICE_NAME + value: copilot + + - it: inline mode renders the chart-managed copilot Secret + template: secrets-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: true + copilot.postgresql.auth.password: xxxxxxxx + copilot.server.env.AGENT_API_DB_ENCRYPTION_KEY: x + copilot.server.env.INTERNAL_API_SECRET: x + copilot.server.env.LICENSE_KEY: x + copilot.server.env.SIM_BASE_URL: x + copilot.server.env.SIM_AGENT_API_KEY: x + copilot.server.env.REDIS_URL: x + copilot.server.env.OPENAI_API_KEY_1: x + asserts: + - hasDocuments: { count: 1 } + - isKind: { of: Secret } + - equal: { path: metadata.name, value: t-sim-copilot-env } + + - it: existingSecret mode does not shadow the pre-created Secret's values with envDefaults (Greptile + Cursor round 2 finding) + template: deployment-copilot.yaml + documentSelector: + path: kind + value: Deployment + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + copilot.server.secret.create: false + copilot.server.secret.name: my-existing-copilot-secret + asserts: + - notExists: + path: spec.template.spec.containers[0].env + + - it: existingSecret mode still renders extraEnv even though envDefaults are skipped + template: deployment-copilot.yaml + documentSelector: + path: kind + value: Deployment + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + copilot.server.secret.create: false + copilot.server.secret.name: my-existing-copilot-secret + copilot.server.extraEnv: + - name: CUSTOM_VAR + value: custom-value + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CUSTOM_VAR + value: custom-value + - notContains: + path: spec.template.spec.containers[0].env + content: + name: PORT + value: "8080" + + - it: copilot's own existingSecret is still the source of truth even when externalSecrets.enabled=true globally for other components (Greptile round 3 finding) + template: deployment-copilot.yaml + documentSelector: + path: kind + value: Deployment + set: + app.env.BETTER_AUTH_SECRET: "" + app.env.ENCRYPTION_KEY: "" + app.env.INTERNAL_API_SECRET: "" + app.env.CRON_SECRET: "" + postgresql.auth.password: "" + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + copilot.server.secret.create: false + copilot.server.secret.name: my-existing-copilot-secret + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + asserts: + - notExists: + path: spec.template.spec.containers[0].env + - equal: + path: spec.template.spec.containers[0].envFrom[0].secretRef.name + value: my-existing-copilot-secret + + - it: existingSecret mode skips the chart-managed copilot Secret and the ExternalSecret + templates: + - secrets-copilot.yaml + - external-secret-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: true + copilot.postgresql.auth.password: xxxxxxxx + copilot.server.secret.create: false + copilot.server.secret.name: my-existing-copilot-secret + asserts: + - hasDocuments: { count: 0 } + + - it: ESO mode renders an ExternalSecret instead of the chart-managed copilot Secret + template: external-secret-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + externalSecrets.remoteRefs.copilot.PORT: path/to/port + externalSecrets.remoteRefs.copilot.SERVICE_NAME: path/to/name + externalSecrets.remoteRefs.copilot.ENVIRONMENT: path/to/env + externalSecrets.remoteRefs.copilot.LOG_LEVEL: path/to/log + externalSecrets.remoteRefs.copilot.AGENT_API_DB_ENCRYPTION_KEY: path/to/agentdbkey + externalSecrets.remoteRefs.copilot.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.copilot.LICENSE_KEY: path/to/license + externalSecrets.remoteRefs.copilot.SIM_BASE_URL: path/to/baseurl + externalSecrets.remoteRefs.copilot.SIM_AGENT_API_KEY: path/to/agentkey + externalSecrets.remoteRefs.copilot.REDIS_URL: path/to/redis + externalSecrets.remoteRefs.copilot.OPENAI_API_KEY_1: path/to/openai + asserts: + - isKind: { of: ExternalSecret } + - equal: { path: metadata.name, value: t-sim-copilot-env } + - equal: { path: spec.secretStoreRef.name, value: sim-store } + + - it: ESO mode skips the chart-managed copilot Secret + template: secrets-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: true + copilot.postgresql.auth.password: xxxxxxxx + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + externalSecrets.remoteRefs.copilot.PORT: path/to/port + externalSecrets.remoteRefs.copilot.SERVICE_NAME: path/to/name + externalSecrets.remoteRefs.copilot.ENVIRONMENT: path/to/env + externalSecrets.remoteRefs.copilot.LOG_LEVEL: path/to/log + externalSecrets.remoteRefs.copilot.AGENT_API_DB_ENCRYPTION_KEY: path/to/agentdbkey + externalSecrets.remoteRefs.copilot.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.copilot.LICENSE_KEY: path/to/license + externalSecrets.remoteRefs.copilot.SIM_BASE_URL: path/to/baseurl + externalSecrets.remoteRefs.copilot.SIM_AGENT_API_KEY: path/to/agentkey + externalSecrets.remoteRefs.copilot.REDIS_URL: path/to/redis + externalSecrets.remoteRefs.copilot.OPENAI_API_KEY_1: path/to/openai + asserts: + - hasDocuments: { count: 0 } + + - it: ESO validator fails when a required copilot key is neither in env nor mapped + template: deployment-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + asserts: + - failedTemplate: + errorPattern: "Required key 'AGENT_API_DB_ENCRYPTION_KEY' is missing" + + - it: ESO validator fails when copilot.server.env contains an unmapped key + template: deployment-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + externalSecrets.enabled: true + externalSecrets.secretStoreRef.name: sim-store + externalSecrets.remoteRefs.app.BETTER_AUTH_SECRET: path/to/auth + externalSecrets.remoteRefs.app.ENCRYPTION_KEY: path/to/enc + externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron + externalSecrets.remoteRefs.postgresql.password: path/to/pgpw + externalSecrets.remoteRefs.copilot.PORT: path/to/port + externalSecrets.remoteRefs.copilot.SERVICE_NAME: path/to/name + externalSecrets.remoteRefs.copilot.ENVIRONMENT: path/to/env + externalSecrets.remoteRefs.copilot.LOG_LEVEL: path/to/log + externalSecrets.remoteRefs.copilot.AGENT_API_DB_ENCRYPTION_KEY: path/to/agentdbkey + externalSecrets.remoteRefs.copilot.INTERNAL_API_SECRET: path/to/iapi + externalSecrets.remoteRefs.copilot.LICENSE_KEY: path/to/license + externalSecrets.remoteRefs.copilot.SIM_BASE_URL: path/to/baseurl + externalSecrets.remoteRefs.copilot.SIM_AGENT_API_KEY: path/to/agentkey + externalSecrets.remoteRefs.copilot.REDIS_URL: path/to/redis + externalSecrets.remoteRefs.copilot.OPENAI_API_KEY_1: path/to/openai + copilot.server.env.UNMAPPED_KEY: "would-go-nowhere" + asserts: + - failedTemplate: + errorPattern: "Key 'UNMAPPED_KEY' is set in copilot.server.env but externalSecrets.enabled=true" + + - it: non-ESO mode still requires OPENAI_API_KEY_1 or ANTHROPIC_API_KEY_1 directly in env + template: deployment-copilot.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + copilot.enabled: true + copilot.postgresql.enabled: false + copilot.database.url: "postgresql://x:x@x:5432/x" + copilot.server.env.AGENT_API_DB_ENCRYPTION_KEY: x + copilot.server.env.INTERNAL_API_SECRET: x + copilot.server.env.LICENSE_KEY: x + copilot.server.env.SIM_BASE_URL: x + copilot.server.env.SIM_AGENT_API_KEY: x + copilot.server.env.REDIS_URL: x + asserts: + - failedTemplate: + errorPattern: "Set at least one of copilot.server.env.OPENAI_API_KEY_1 or copilot.server.env.ANTHROPIC_API_KEY_1" diff --git a/helm/sim/tests/env-defaults_test.yaml b/helm/sim/tests/env-defaults_test.yaml index a658f3dbfb2..7e3fdcb4898 100644 --- a/helm/sim/tests/env-defaults_test.yaml +++ b/helm/sim/tests/env-defaults_test.yaml @@ -3,12 +3,11 @@ release: name: t namespace: sim defaults: &defaults - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x - + postgresql.auth.password: xxxxxxxx tests: - it: inline mode renders localhost envDefaults on the app pod template: deployment-app.yaml @@ -37,7 +36,7 @@ tests: set: app.secrets.existingSecret.enabled: true app.secrets.existingSecret.name: my-secret - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - notContains: path: spec.template.spec.containers[0].env @@ -50,7 +49,7 @@ tests: set: app.secrets.existingSecret.enabled: true app.secrets.existingSecret.name: my-secret - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - notContains: path: spec.template.spec.containers[0].env @@ -64,7 +63,7 @@ tests: app.secrets.existingSecret.enabled: true app.secrets.existingSecret.name: my-secret app.env.NEXT_PUBLIC_APP_URL: "https://prod.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - contains: path: spec.template.spec.containers[0].env @@ -78,7 +77,7 @@ tests: app.secrets.existingSecret.enabled: true app.secrets.existingSecret.name: my-secret app.env.NEXT_PUBLIC_APP_URL: "https://prod.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - contains: path: spec.template.spec.containers[0].env @@ -93,7 +92,7 @@ tests: app.secrets.existingSecret.name: my-secret app.env.NEXT_PUBLIC_APP_URL: "https://prod.example.com" realtime.env.NEXT_PUBLIC_APP_URL: "https://realtime.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - contains: path: spec.template.spec.containers[0].env diff --git a/helm/sim/tests/helm-test-hook_test.yaml b/helm/sim/tests/helm-test-hook_test.yaml index 4240d584ec2..ee19f4f7776 100644 --- a/helm/sim/tests/helm-test-hook_test.yaml +++ b/helm/sim/tests/helm-test-hook_test.yaml @@ -3,12 +3,11 @@ release: name: t namespace: sim defaults: &defaults - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x - + postgresql.auth.password: xxxxxxxx tests: - it: renders the test pod with helm.sh/hook=test by default template: tests/test-connection.yaml diff --git a/helm/sim/tests/networkpolicy_test.yaml b/helm/sim/tests/networkpolicy_test.yaml index d6e0b260ec4..355fab6d27d 100644 --- a/helm/sim/tests/networkpolicy_test.yaml +++ b/helm/sim/tests/networkpolicy_test.yaml @@ -5,11 +5,11 @@ release: name: t namespace: sim defaults: &defaults - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx networkPolicy.enabled: true ingress.enabled: true ingress.app.host: a.test diff --git a/helm/sim/tests/pdb-hpa_test.yaml b/helm/sim/tests/pdb-hpa_test.yaml index 70f3b0903a2..3960f2e4006 100644 --- a/helm/sim/tests/pdb-hpa_test.yaml +++ b/helm/sim/tests/pdb-hpa_test.yaml @@ -3,12 +3,11 @@ release: name: t namespace: sim defaults: &defaults - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x - + postgresql.auth.password: xxxxxxxx tests: - it: PDB does not render when replicaCount=1 and HPA is off (tri-state null) template: poddisruptionbudget.yaml diff --git a/helm/sim/tests/pii_test.yaml b/helm/sim/tests/pii_test.yaml index 3a416237207..f35c87a5759 100644 --- a/helm/sim/tests/pii_test.yaml +++ b/helm/sim/tests/pii_test.yaml @@ -3,12 +3,11 @@ release: name: t namespace: sim set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x - + postgresql.auth.password: xxxxxxxx tests: - it: does not render the pii deployment when disabled template: deployment-pii.yaml diff --git a/helm/sim/tests/pod-rollout_test.yaml b/helm/sim/tests/pod-rollout_test.yaml index 39b0c8c72fc..999ba85c434 100644 --- a/helm/sim/tests/pod-rollout_test.yaml +++ b/helm/sim/tests/pod-rollout_test.yaml @@ -3,12 +3,11 @@ release: name: t namespace: sim defaults: &defaults - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x - + postgresql.auth.password: xxxxxxxx tests: - it: app pod template has checksum/secret annotation template: deployment-app.yaml @@ -30,7 +29,7 @@ tests: template: deployment-app.yaml set: <<: *defaults - app.env.BETTER_AUTH_SECRET: original-value + app.env.BETTER_AUTH_SECRET: yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy asserts: - notEqual: path: spec.template.metadata.annotations["checksum/secret"] diff --git a/helm/sim/tests/schema-secret-length_test.yaml b/helm/sim/tests/schema-secret-length_test.yaml new file mode 100644 index 00000000000..b2255d93853 --- /dev/null +++ b/helm/sim/tests/schema-secret-length_test.yaml @@ -0,0 +1,59 @@ +suite: values.schema.json — secret length enforcement (blocker fix regression net) +release: + name: t + namespace: sim + +tests: + - it: rejects a BETTER_AUTH_SECRET shorter than 32 characters + set: + app.env.BETTER_AUTH_SECRET: short + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + asserts: + - failedTemplate: + errorPattern: "BETTER_AUTH_SECRET.*minLength" + + - it: rejects an ENCRYPTION_KEY shorter than 32 characters + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: short + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + asserts: + - failedTemplate: + errorPattern: "ENCRYPTION_KEY.*minLength" + + - it: rejects a postgresql.auth.password shorter than 8 characters + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: short + asserts: + - failedTemplate: + errorPattern: "password.*minLength" + + - it: accepts an empty BETTER_AUTH_SECRET (deferred to existingSecret/ESO) + templates: + - secrets-app.yaml + set: + app.secrets.existingSecret.enabled: true + app.secrets.existingSecret.name: my-existing-secret + postgresql.auth.password: xxxxxxxx + asserts: + - hasDocuments: { count: 0 } + + - it: accepts a valid 32-character BETTER_AUTH_SECRET and ENCRYPTION_KEY + template: deployment-app.yaml + set: + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.INTERNAL_API_SECRET: x + app.env.CRON_SECRET: x + postgresql.auth.password: xxxxxxxx + asserts: + - isKind: { of: Deployment } diff --git a/helm/sim/tests/secret-modes_test.yaml b/helm/sim/tests/secret-modes_test.yaml index d1e73c3fdab..6b60cf56693 100644 --- a/helm/sim/tests/secret-modes_test.yaml +++ b/helm/sim/tests/secret-modes_test.yaml @@ -7,11 +7,11 @@ tests: - it: inline mode renders the chart-managed Secret template: secrets-app.yaml set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - isKind: { of: Secret } - equal: { path: metadata.name, value: t-sim-app-secrets } @@ -23,7 +23,7 @@ tests: set: app.secrets.existingSecret.enabled: true app.secrets.existingSecret.name: my-existing-secret - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - hasDocuments: { count: 0 } @@ -38,7 +38,7 @@ tests: externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron externalSecrets.remoteRefs.postgresql.password: path/to/pgpw - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - isKind: { of: ExternalSecret } - equal: { path: metadata.name, value: t-sim-app-secrets } @@ -55,7 +55,7 @@ tests: externalSecrets.remoteRefs.app.INTERNAL_API_SECRET: path/to/iapi externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron externalSecrets.remoteRefs.postgresql.password: path/to/pgpw - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - hasDocuments: { count: 0 } @@ -69,7 +69,7 @@ tests: externalSecrets.remoteRefs.app.CRON_SECRET: path/to/cron externalSecrets.remoteRefs.postgresql.password: path/to/pgpw app.env.UNMAPPED_KEY: "would-go-nowhere" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - failedTemplate: errorPattern: "Key 'UNMAPPED_KEY' is set in app.env but externalSecrets.enabled=true" @@ -79,7 +79,7 @@ tests: set: app.secrets.existingSecret.enabled: true app.secrets.existingSecret.name: my-existing-secret - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - contains: path: spec.template.spec.containers[0].envFrom @@ -90,13 +90,13 @@ tests: - it: realtime.env-only value reaches the Secret when app.env entry is empty (cursor bugbot fix) template: secrets-app.yaml set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x app.env.BETTER_AUTH_URL: "" realtime.env.BETTER_AUTH_URL: "https://realtime.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - equal: path: stringData.BETTER_AUTH_URL @@ -105,13 +105,13 @@ tests: - it: app.env wins over realtime.env on collision when both are non-empty template: secrets-app.yaml set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x app.env.NEXT_PUBLIC_APP_URL: "https://app.example.com" realtime.env.NEXT_PUBLIC_APP_URL: "https://realtime.example.com" - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - equal: path: stringData.NEXT_PUBLIC_APP_URL diff --git a/helm/sim/tests/smoke_test.yaml b/helm/sim/tests/smoke_test.yaml index 71bf0888236..0e50a2799be 100644 --- a/helm/sim/tests/smoke_test.yaml +++ b/helm/sim/tests/smoke_test.yaml @@ -9,7 +9,7 @@ release: namespace: sim set: app.env.BETTER_AUTH_SECRET: test-better-auth-secret-32-bytes - app.env.ENCRYPTION_KEY: test-encryption-key-32-bytes + app.env.ENCRYPTION_KEY: test-encryption-key-32-bytes-long app.env.INTERNAL_API_SECRET: test-internal-api-secret-32-bytes app.env.CRON_SECRET: test-cron-secret-32-bytes postgresql.auth.password: testpassword diff --git a/helm/sim/tests/telemetry-security-context_test.yaml b/helm/sim/tests/telemetry-security-context_test.yaml new file mode 100644 index 00000000000..4d3f7233899 --- /dev/null +++ b/helm/sim/tests/telemetry-security-context_test.yaml @@ -0,0 +1,53 @@ +suite: telemetry — Restricted Pod Security Standard hardening (best-practices fix regression net) +release: + name: t + namespace: sim +templates: + - telemetry.yaml + +tests: + - it: pod-level security context preserves the collector's original UID/GID/fsGroup and adds seccompProfile + set: + telemetry.enabled: true + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.securityContext.runAsUser + value: 10001 + - equal: + path: spec.template.spec.securityContext.runAsGroup + value: 10001 + - equal: + path: spec.template.spec.securityContext.fsGroup + value: 10001 + - equal: + path: spec.template.spec.securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.securityContext.seccompProfile.type + value: RuntimeDefault + - equal: + path: spec.template.spec.automountServiceAccountToken + value: false + + - it: container-level security context matches the Restricted profile (same as every other workload) + set: + telemetry.enabled: true + documentSelector: + path: kind + value: Deployment + asserts: + - equal: + path: spec.template.spec.containers[0].securityContext.allowPrivilegeEscalation + value: false + - equal: + path: spec.template.spec.containers[0].securityContext.capabilities.drop[0] + value: ALL + - equal: + path: spec.template.spec.containers[0].securityContext.runAsNonRoot + value: true + - equal: + path: spec.template.spec.containers[0].securityContext.seccompProfile.type + value: RuntimeDefault diff --git a/helm/sim/tests/validators_test.yaml b/helm/sim/tests/validators_test.yaml index b53800b2d14..aff09b4386b 100644 --- a/helm/sim/tests/validators_test.yaml +++ b/helm/sim/tests/validators_test.yaml @@ -9,52 +9,52 @@ tests: - it: fails when BETTER_AUTH_SECRET is missing set: app.env.BETTER_AUTH_SECRET: "" - app.env.ENCRYPTION_KEY: x + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - failedTemplate: errorMessage: "app.env.BETTER_AUTH_SECRET is required for production deployment" - it: fails when ENCRYPTION_KEY is missing set: - app.env.BETTER_AUTH_SECRET: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.ENCRYPTION_KEY: "" app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - failedTemplate: errorMessage: "app.env.ENCRYPTION_KEY is required for production deployment" - it: fails when INTERNAL_API_SECRET is missing (1.0.0 requirement) set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: "" app.env.CRON_SECRET: x - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - failedTemplate: errorMessage: "app.env.INTERNAL_API_SECRET is required for production deployment (shared auth between sim-app and sim-realtime pods). Generate one with: openssl rand -hex 32" - it: fails when cronjobs are enabled but CRON_SECRET is empty set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: "" cronjobs.enabled: true - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - failedTemplate: errorMessage: "app.env.CRON_SECRET is required when cronjobs.enabled=true (every cron pod authenticates with this token). Generate one with: openssl rand -hex 32, or set cronjobs.enabled=false." - it: fails when postgresql.auth.password is missing set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x postgresql.auth.password: "" @@ -64,8 +64,8 @@ tests: - it: rejects postgres passwords with URL-unsafe characters set: - app.env.BETTER_AUTH_SECRET: x - app.env.ENCRYPTION_KEY: x + app.env.BETTER_AUTH_SECRET: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x postgresql.auth.password: "pa$$word" @@ -76,10 +76,10 @@ tests: - it: rejects placeholder BETTER_AUTH_SECRET set: app.env.BETTER_AUTH_SECRET: "CHANGE-ME-32-CHAR-SECRET-FOR-PRODUCTION-USE" - app.env.ENCRYPTION_KEY: x + app.env.ENCRYPTION_KEY: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx app.env.INTERNAL_API_SECRET: x app.env.CRON_SECRET: x - postgresql.auth.password: x + postgresql.auth.password: xxxxxxxx asserts: - failedTemplate: errorPattern: "must not use the default placeholder value" diff --git a/helm/sim/values.schema.json b/helm/sim/values.schema.json index f6641894638..4b1e86c0a75 100644 --- a/helm/sim/values.schema.json +++ b/helm/sim/values.schema.json @@ -108,11 +108,13 @@ "properties": { "BETTER_AUTH_SECRET": { "type": "string", - "description": "Auth secret (minimum 32 characters required when not using existingSecret)" + "anyOf": [{ "minLength": 32 }, { "const": "" }], + "description": "Auth secret (minimum 32 characters required when not using existingSecret; leave empty when the value comes from existingSecret/ESO instead)" }, "ENCRYPTION_KEY": { "type": "string", - "description": "Encryption key (minimum 32 characters required when not using existingSecret)" + "anyOf": [{ "minLength": 32 }, { "const": "" }], + "description": "Encryption key (minimum 32 characters required when not using existingSecret; leave empty when the value comes from existingSecret/ESO instead)" }, "NEXT_PUBLIC_APP_URL": { "type": "string", @@ -432,7 +434,8 @@ "properties": { "BETTER_AUTH_SECRET": { "type": "string", - "description": "Auth secret (minimum 32 characters required when not using existingSecret)" + "anyOf": [{ "minLength": 32 }, { "const": "" }], + "description": "Auth secret (minimum 32 characters required when not using existingSecret; leave empty when the value comes from existingSecret/ESO instead)" }, "NEXT_PUBLIC_APP_URL": { "type": "string", @@ -528,7 +531,8 @@ }, "password": { "type": "string", - "description": "PostgreSQL password (minimum 8 characters when not using existingSecret)" + "anyOf": [{ "minLength": 8 }, { "const": "" }], + "description": "PostgreSQL password (minimum 8 characters when not using existingSecret; leave empty when the value comes from existingSecret/ESO instead)" }, "existingSecret": { "type": "object", diff --git a/helm/sim/values.yaml b/helm/sim/values.yaml index 592d2d4f4f7..c7a39c81e92 100644 --- a/helm/sim/values.yaml +++ b/helm/sim/values.yaml @@ -1437,6 +1437,15 @@ telemetry: tls: enabled: false + # Pod security context (preserves the collector's original UID/GID/fsGroup) + podSecurityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + + # Container security context (sim.containerSecurityContext restricted defaults apply) + securityContext: {} + # Copilot service configuration (optional microservice) copilot: # Enable/disable the copilot service @@ -1478,11 +1487,12 @@ copilot: runAsNonRoot: true runAsUser: 1001 - # Environment variables (required and optional) + # Environment variables that go through the chart-managed Secret (or ExternalSecret + # when externalSecrets.enabled=true). Every non-empty key here must have a matching + # externalSecrets.remoteRefs.copilot entry under ESO — reserve this block for values + # that are genuinely sensitive or user-provided; static operational config belongs in + # envDefaults below instead. env: - PORT: "8080" - SERVICE_NAME: "copilot" - ENVIRONMENT: "production" AGENT_API_DB_ENCRYPTION_KEY: "" INTERNAL_API_SECRET: "" LICENSE_KEY: "" @@ -1491,17 +1501,31 @@ copilot: SIM_BASE_URL: "" SIM_AGENT_API_KEY: "" REDIS_URL: "" - # Optional configuration - LOG_LEVEL: "info" CORS_ALLOWED_ORIGINS: "" OTEL_EXPORTER_OTLP_ENDPOINT: "" - + + # Operational tunables shipped with the chart. Rendered as inline `env:` on the + # copilot container — NOT written into the chart-managed Secret and NOT required to + # be mapped when externalSecrets.enabled=true. Override any key by setting + # copilot.server.envDefaults.KEY in your values file. Move a key into + # copilot.server.env above only if it must be treated as secret. + envDefaults: + PORT: "8080" + SERVICE_NAME: "copilot" + ENVIRONMENT: "production" + LOG_LEVEL: "info" + # Optional: additional static environment variables extraEnv: [] - + # Optional: references to existing ConfigMaps/Secrets extraEnvFrom: [] - + + # Optional: additional volumes/mounts (e.g. to back an emptyDir /tmp when + # enabling securityContext.readOnlyRootFilesystem) + extraVolumes: [] + extraVolumeMounts: [] + # Secret generation configuration (set create=false to use an existing secret) secret: create: true @@ -1727,6 +1751,27 @@ externalSecrets: # Path to external database password in external store password: "" + # Copilot server secrets (used when copilot.enabled=true and copilot.server.secret.create=true). + # Every non-empty copilot.server.env key must have a matching entry here, or template + # rendering fails with a clear "map it or remove it" error — same rule as remoteRefs.app. + copilot: + # Path to AGENT_API_DB_ENCRYPTION_KEY in external store + AGENT_API_DB_ENCRYPTION_KEY: "" + # Path to INTERNAL_API_SECRET in external store (shared with app/realtime) + INTERNAL_API_SECRET: "" + # Path to LICENSE_KEY in external store + LICENSE_KEY: "" + # Path to SIM_BASE_URL in external store + SIM_BASE_URL: "" + # Path to SIM_AGENT_API_KEY in external store + SIM_AGENT_API_KEY: "" + # Path to REDIS_URL in external store + REDIS_URL: "" + # Path to OPENAI_API_KEY_1 in external store (one of OPENAI_API_KEY_1/ANTHROPIC_API_KEY_1 required) + OPENAI_API_KEY_1: "" + # Path to ANTHROPIC_API_KEY_1 in external store (one of OPENAI_API_KEY_1/ANTHROPIC_API_KEY_1 required) + ANTHROPIC_API_KEY_1: "" + # cert-manager configuration # Prerequisites: Install cert-manager in your cluster first # See: https://cert-manager.io/docs/installation/ From e2cb3b29ba16b57abc1af86dcc6d9706d0c145fd Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 20:04:48 -0700 Subject: [PATCH 11/13] fix(files): fix savingRef mutex integrity race after discard correction (#5554) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(files): fix savingRef mutex integrity race, anchor discard un-suppress to captured target An independent 4-agent audit (beyond the bot review loop) converged on a real race in the round-11 discard-correction fix: - save()'s deferred MIN_SAVING_DISPLAY_MS status timer resolves as a macrotask well after the save's own promise (used to sequence discard's correction) has already settled as a microtask. That timer unconditionally reset savingRef/triggered a trailing resave, even after discard's correction had since claimed the save slot for its own in-flight write — letting a fresh debounced save start concurrently with the correction. Now guarded with `if (inFlightRef.current) return` before touching savingRef, so it only acts when nothing else has claimed the slot since. - The render-time un-suppress check keyed off isDirty, which a stale save's markSavedContent(next) landing after discard can transiently corrupt (overwriting savedContent with the pre-discard value while content has already been reverted), causing a spurious "genuinely new edit" signal. Now keyed off a discardTargetRef captured at discard time instead, which that corruption doesn't touch. Also hardened recovery to key its once-per-mount guard on the specific draft key rather than a bare boolean, so a hypothetical future caller that reuses a hook instance across files would still get correct recovery (today's real callers already remount per file, so this is defense in depth, not a behavior change for any current caller). * fix(files): fix discard status masking and cross-key discard suppression leak Greptile round-1 review on the follow-up fix PR caught 3 real gaps in the discard/correction flow: - The display timer's !discardedRef guard (added to stop a stale save's status update from clobbering a running correction) also silently suppressed the idle-timer reschedule, and discard()'s own correction never set a terminal status on settle — so saveStatus could stick on 'saving' forever after a successful correction, and a failed correction surfaced only via the onDiscardCorrectionFailed callback with no status change. Now the correction's own .then()/.catch() sets 'idle'/'error' once it owns the flow (only when nothing has since un-suppressed discard). - The discard-suppression un-suppress check compared content against the captured discard target, but never reset if a hook instance were reused across draftKeys — a coincidental content match with the previous file's target would keep discard permanently suppressing saves for the new file. Reset discardedRef whenever the effective draftKey changes. * fix(files): re-chain autosave after a discard correction settles Cursor Bugbot found the discard correction's finally() cleared the save mutex but never rechecked for dirty content: an edit made while the correction was in flight bailed out of the debounce effect (savingRef was held) and, since content isn't a savingRef dependency, was never rescheduled once the mutex freed — the edit could sit unsaved indefinitely. The same gap explained a related report that a failed correction which had already been superseded by a newer edit left saveStatus stuck, since nothing else was driving it forward. Fix is one line: call save() in the finally() when content is still dirty. save()'s own guards (savingRef/discardedRef/content-equality) make this safe to call unconditionally, and it naturally hands status ownership to the newer edit's own save cycle. * fix(files): key discard state to raw draftKey, make failed corrections retryable Cursor Bugbot round 3 caught 2 more real gaps: - The document-change reset added last round compared effectiveDraftKey (draftKey gated by enabled), so toggling enabled alone for the SAME document — e.g. a streaming lock — looked identical to switching files. That cleared discardedRef mid-correction, which skipped the correction's own setSaveStatus('error'/'idle') (gated on discardedRef to avoid clobbering a newer edit's status), silently stranding the hook on 'saving' with no visible retry affordance. Now keyed off the raw draftKey, which enabled toggling never touches. - After a failed correction, content already equals savedContent (that's what discard reverted to), so saveImmediately()'s retry going through save() hit its dirty-check and was a complete no-op — the error toast's Retry button did nothing. Extracted the correction logic into runCorrection(target), shared by discard() and by saveImmediately when a failed correction is pending, so retry pushes the reverted baseline again instead of bailing on a check that assumes retries are always for dirty content. --- apps/sim/hooks/use-autosave.test.tsx | 365 +++++++++++++++++++++++++++ apps/sim/hooks/use-autosave.ts | 99 ++++++-- 2 files changed, 441 insertions(+), 23 deletions(-) diff --git a/apps/sim/hooks/use-autosave.test.tsx b/apps/sim/hooks/use-autosave.test.tsx index ff92667b219..0f035fee248 100644 --- a/apps/sim/hooks/use-autosave.test.tsx +++ b/apps/sim/hooks/use-autosave.test.tsx @@ -763,6 +763,371 @@ describe('useAutosave', () => { await flush() }) + it('does not let the original save leftover status timer clobber a still-running correction', async () => { + const resolvers: Array<() => void> = [] + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-9', + }) + + // A save is in flight when the user discards. + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + // The original save resolves; the correction starts (savingRef/inFlightRef now belong to it). + await act(async () => { + resolvers[0]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + // The original save's own MIN_SAVING_DISPLAY_MS timer fires while the correction is still + // unresolved. Without the inFlightRef guard, this used to unconditionally reset savingRef, + // letting a debounced save for a new edit start concurrently with the correction. + await act(async () => { + vi.advanceTimersByTime(600) + }) + await flush() + + // A new edit made in this window must not be able to schedule a concurrent save while the + // correction (still unresolved) rightfully holds the mutex. + handle.rerender({ content: 'a2' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + resolvers[1]?.() + await flush() + }) + + it('reaches a terminal status once the discard correction settles instead of sticking on saving', async () => { + const resolvers: Array<() => void> = [] + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-10', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + // The original save resolves; the correction starts. Its own MIN_SAVING_DISPLAY_MS timer + // must not resolve status to 'saved'/'idle' for content the user no longer sees. + await act(async () => { + resolvers[0]?.() + }) + await flush() + await act(async () => { + vi.advanceTimersByTime(600) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + expect(handle.status()).toBe('saving') + + // The correction itself settles — without this, status stayed on 'saving' forever. + await act(async () => { + resolvers[1]?.() + }) + await flush() + expect(handle.status()).toBe('idle') + }) + + it('surfaces an error if the discard correction itself fails, rather than drifting to idle', async () => { + const resolvers: Array<() => void> = [] + const rejecters: Array<(error: Error) => void> = [] + let callCount = 0 + const onSave = vi.fn( + () => + new Promise((resolve, reject) => { + callCount += 1 + if (callCount === 1) resolvers.push(resolve) + else rejecters.push(reject) + }) + ) + const onDiscardCorrectionFailed = vi.fn() + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-11', + onDiscardCorrectionFailed, + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + await act(async () => { + resolvers[0]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + await act(async () => { + rejecters[0]?.(new Error('network down')) + }) + await flush() + expect(onDiscardCorrectionFailed).toHaveBeenCalledTimes(1) + expect(handle.status()).toBe('error') + }) + + it('does not let discard suppression from a previous file block saves for a newly switched-to file', async () => { + const onSave = vi.fn(async () => {}) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-x', + }) + + handle.rerender({ content: 'a1' }) + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + // Switch to a different file whose content coincidentally equals the previous file's + // discard target — without keying discard suppression to draftKey, this would stay + // wrongly suppressed forever, since content would never differ from the stale target. + handle.rerender({ draftKey: 'file-y', savedContent: 'z', content: 'a' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + }) + + it('autosaves an edit made while a discard correction was in flight, once the correction settles', async () => { + const resolvers: Array<() => void> = [] + const onSave = vi.fn( + () => + new Promise((resolve) => { + resolvers.push(resolve) + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-12', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + // The original save resolves; the correction starts and holds the savingRef mutex. + await act(async () => { + resolvers[0]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + // The user keeps typing while the correction is still in flight. The debounce effect sees + // savingRef held and bails — without a re-chain once the mutex frees, this edit would never + // autosave until some unrelated future change happened to fire the effect again. + handle.rerender({ content: 'a2' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + await act(async () => { + resolvers[1]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(3) + expect(onSave).toHaveBeenLastCalledWith() + }) + + it('surfaces the newer edit’s own save outcome if a discard correction fails after being superseded', async () => { + const resolvers: Array<() => void> = [] + const rejecters: Array<(error: Error) => void> = [] + let callCount = 0 + const onSave = vi.fn(() => { + callCount += 1 + if (callCount === 1) return new Promise((resolve) => resolvers.push(resolve)) + if (callCount === 2) return new Promise((_, reject) => rejecters.push(reject)) + // The third call is the superseding edit's own save — never resolved, since we only + // need to observe that it was picked up at all. + return new Promise(() => {}) + }) + const onDiscardCorrectionFailed = vi.fn() + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-13', + onDiscardCorrectionFailed, + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + await act(async () => { + resolvers[0]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + // A newer edit supersedes the in-flight correction before it settles. + handle.rerender({ content: 'a2' }) + + await act(async () => { + rejecters[0]?.(new Error('network down')) + }) + await flush() + + // The failed correction still reports itself, but the superseding edit's own save must be + // picked up immediately instead of leaving the hook stalled on a dead mutex. + expect(onDiscardCorrectionFailed).toHaveBeenCalledTimes(1) + expect(onSave).toHaveBeenCalledTimes(3) + expect(onSave).toHaveBeenLastCalledWith() + }) + + it('does not lift discard suppression when only `enabled` toggles for the same document', async () => { + const resolvers: Array<() => void> = [] + const rejecters: Array<(error: Error) => void> = [] + const onSave = vi.fn( + () => + new Promise((resolve, reject) => { + resolvers.push(resolve) + rejecters.push(reject) + }) + ) + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-14', + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(1) + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + // The correction starts once the original save settles. + await act(async () => { + resolvers[0]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + // A streaming lock (or any other reason `enabled` flips) toggles for the SAME document + // while the correction is still in flight — this must not be mistaken for switching files + // and clear discard's ownership of saveStatus. + handle.rerender({ enabled: false }) + handle.rerender({ enabled: true }) + + // Without keying off the raw draftKey, the `enabled` round-trip would have cleared + // discardedRef, so the correction's failure handler would skip setSaveStatus('error') and + // leave the hook stuck on 'saving' with no visible retry affordance. + await act(async () => { + rejecters[1]?.(new Error('network down')) + }) + await flush() + expect(handle.status()).toBe('error') + }) + + it('retries a failed discard correction via saveImmediately instead of silently no-opping', async () => { + const resolvers: Array<() => void> = [] + const rejecters: Array<(error: Error) => void> = [] + let callCount = 0 + const onSave = vi.fn(() => { + callCount += 1 + if (callCount === 1) return new Promise((resolve) => resolvers.push(resolve)) + if (callCount === 2) return new Promise((_, reject) => rejecters.push(reject)) + return Promise.resolve() + }) + const onDiscardCorrectionFailed = vi.fn() + const { handle } = renderAutosave({ + content: 'a', + savedContent: 'a', + onSave, + draftKey: 'file-discard-15', + onDiscardCorrectionFailed, + }) + + handle.rerender({ content: 'a1' }) + await act(async () => { + vi.advanceTimersByTime(1500) + }) + await flush() + act(() => handle.discard()) + handle.rerender({ content: 'a' }) + + await act(async () => { + resolvers[0]?.() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(2) + + await act(async () => { + rejecters[0]?.(new Error('network down')) + }) + await flush() + expect(onDiscardCorrectionFailed).toHaveBeenCalledTimes(1) + expect(handle.status()).toBe('error') + + // Content already equals savedContent (that's what discard reverted to), so a naive retry + // through save()'s dirty-check would be a silent no-op. saveImmediately must still push + // the reverted baseline again. + await act(async () => { + await handle.saveImmediately() + }) + await flush() + expect(onSave).toHaveBeenCalledTimes(3) + expect(onSave).toHaveBeenLastCalledWith('a') + expect(handle.status()).toBe('idle') + }) + it('serializes IndexedDB writes and deletes so a slow write cannot resurrect a discarded draft', async () => { let resolveWrite: (() => void) | undefined const idbKeyval = await import('idb-keyval') diff --git a/apps/sim/hooks/use-autosave.ts b/apps/sim/hooks/use-autosave.ts index 05e1f10d6d0..aaf2d3d6b88 100644 --- a/apps/sim/hooks/use-autosave.ts +++ b/apps/sim/hooks/use-autosave.ts @@ -106,9 +106,24 @@ export function useAutosave({ const lastPersistedContentRef = useRef(null) const discardedRef = useRef(false) + const discardTargetRef = useRef(null) + const failedCorrectionTargetRef = useRef(null) + // Keyed off the raw `draftKey`, not `effectiveDraftKey` — the latter also flips with `enabled` + // (e.g. a streaming lock toggling for the SAME document), which must not be mistaken for a + // hook instance being reused across documents (today's callers all remount per file instead). + const documentKeyRef = useRef(draftKey) + const documentChanged = documentKeyRef.current !== draftKey + documentKeyRef.current = draftKey + if (documentChanged) { + discardedRef.current = false + failedCorrectionTargetRef.current = null + } const isDirty = content !== savedContent - if (discardedRef.current && isDirty) discardedRef.current = false + if (discardedRef.current && content !== discardTargetRef.current) { + discardedRef.current = false + failedCorrectionTargetRef.current = null + } const persistLocalDraft = useCallback(() => { const key = draftKeyRef.current @@ -165,9 +180,15 @@ export function useAutosave({ const elapsed = Date.now() - savingStartRef.current const remaining = Math.max(0, MIN_SAVING_DISPLAY_MS - elapsed) displayTimerRef.current = setTimeout(() => { - setSaveStatus(nextStatus) - clearTimeout(idleTimerRef.current) - idleTimerRef.current = setTimeout(() => setSaveStatus('idle'), 2000) + // While discarded, status is owned by discard()'s corrective save instead — this + // save's outcome no longer reflects what the user is looking at, and letting the + // idle-timer fire anyway would prematurely clear a status the correction just set. + if (!discardedRef.current) { + setSaveStatus(nextStatus) + clearTimeout(idleTimerRef.current) + idleTimerRef.current = setTimeout(() => setSaveStatus('idle'), 2000) + } + if (inFlightRef.current) return savingRef.current = false if (nextStatus !== 'error' && contentRef.current !== savedContentRef.current) { save() @@ -245,11 +266,11 @@ export function useAutosave({ } }, [effectiveDraftKey, persistLocalDraft]) - const recoveryAttemptedRef = useRef(false) + const recoveredForKeyRef = useRef(null) useEffect(() => { - if (!effectiveDraftKey || recoveryAttemptedRef.current) return - recoveryAttemptedRef.current = true + if (!effectiveDraftKey || recoveredForKeyRef.current === effectiveDraftKey) return + recoveredForKeyRef.current = effectiveDraftKey let cancelled = false void enqueueDraftOp(effectiveDraftKey, () => get(localDraftDbKey(effectiveDraftKey)) @@ -272,38 +293,70 @@ export function useAutosave({ } }, [effectiveDraftKey, clearLocalDraft]) + /** Pushes `target` (the reverted baseline) to the server. Used both by `discard()`'s initial correction and by `saveImmediately`'s retry of a correction that previously failed — content already equals `target` in both cases, so this bypasses `save()`'s dirty-check entirely rather than special-casing it there. */ + const runCorrection = useCallback( + (target: string) => { + savingRef.current = true + const correctionRun = onSaveRef + .current(target) + .then( + () => { + failedCorrectionTargetRef.current = null + // Only ours to set if nothing has since un-suppressed discard (a newer edit) — that + // flow owns status once it takes over. + if (!unmountedRef.current && discardedRef.current) setSaveStatus('idle') + }, + (error) => { + failedCorrectionTargetRef.current = target + logger.warn('Corrective save after discard failed', { error }) + onDiscardCorrectionFailedRef.current?.() + if (!unmountedRef.current && discardedRef.current) setSaveStatus('error') + } + ) + .finally(() => { + savingRef.current = false + inFlightRef.current = null + // A newer edit made while the correction was in flight bailed out of the debounce + // effect (savingRef was held) and never got rescheduled — pick it up now that the + // mutex is free. This also gives that edit's own save cycle ownership of saveStatus, + // covering a correction that failed after a newer edit already un-suppressed discard. + if (!unmountedRef.current && contentRef.current !== savedContentRef.current) save() + }) + inFlightRef.current = correctionRun + return correctionRun + }, + [save] + ) + const saveImmediately = useCallback(async () => { clearTimeout(timerRef.current) + // Retrying after a failed correction: content already equals savedContent (that's what + // discard reverted to), so save()'s own dirty-check would treat this as a no-op and the + // error's retry button would silently do nothing. + if (failedCorrectionTargetRef.current !== null) { + await runCorrection(failedCorrectionTargetRef.current) + return + } await save() - }, [save]) + }, [save, runCorrection]) const discard = useCallback(() => { discardedRef.current = true + discardTargetRef.current = savedContentRef.current + failedCorrectionTargetRef.current = null clearTimeout(timerRef.current) clearTimeout(localDraftTimerRef.current) clearLocalDraft() const pendingSave = inFlightRef.current if (!pendingSave) return - const target = savedContentRef.current + const target = discardTargetRef.current const contentAtDiscard = contentRef.current void pendingSave.then(() => { const current = contentRef.current if (inFlightRef.current || (current !== target && current !== contentAtDiscard)) return - savingRef.current = true - const correctionRun = onSaveRef - .current(target) - .catch((error) => { - logger.warn('Corrective save after discard failed', { error }) - onDiscardCorrectionFailedRef.current?.() - }) - .finally(() => { - savingRef.current = false - inFlightRef.current = null - }) - inFlightRef.current = correctionRun - return correctionRun + runCorrection(target) }) - }, [clearLocalDraft]) + }, [clearLocalDraft, runCorrection]) return { saveStatus, saveImmediately, isDirty, discard } } From f9a5d8b113c3663bdef03335b0a8ef543adcb62f Mon Sep 17 00:00:00 2001 From: Waleed Date: Thu, 9 Jul 2026 20:54:29 -0700 Subject: [PATCH 12/13] fix(security): close code-scanning and dependabot alerts (#5557) * fix(security): close code-scanning and dependabot alerts - markdown-paste.ts: strip