fix(logs): keep provenance sidecars intact through log PII redaction

apps/sim/app/api/v2/knowledge/search/route.application.test.ts

@@ -276,7 +276,9 @@ describe('Knowledge search provenance through the V2 route and reranker HTTP bou
     expect(body.data.rerankerStatus).toBe('applied')
     expect(providerPayload().documents).toEqual([CONTENT.replace(SECRET, '{{TOKEN}}')])
     expect(body.data.results[0].content).toBe(CONTENT)
-    expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledWith('synthetic-encrypted-token')
+    expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledWith('synthetic-encrypted-token', {
+      logFailure: false,
+    })
     expect(mockGenerateSearchEmbedding).toHaveBeenCalledWith(
       requestInput.query,
       expect.anything(),

apps/sim/executor/utils/resolved-secret-trace-registry.ts

@@ -417,6 +417,18 @@ export interface ImportResolvedSecretTraceProvenanceOptions {
    * `source-provenance-incomplete` from a symptom into an address.
    */
   origin?: string
+  /**
+   * Takes the import's entry decrypt-failure summary in place of the registry's own error line,
+   * for a caller that reports the failure itself with the context the registry lacks.
+   */
+  onDecryptFailure?: (failure: ProvenanceDecryptFailure) => void
+}
+
+/** Entry decrypt failures of one provenance import, summarised. */
+export interface ProvenanceDecryptFailure {
+  error: string
+  failedEntryCount: number
+  totalEntryCount: number
 }
 
 export interface ExportResolvedSecretTraceProvenanceForValueOptions {
@@ -966,7 +978,7 @@ export class ResolvedSecretTraceRegistry {
    * A staged registry filters values for one operation and is then discarded. Its caller owns the
    * reporting and says it with strictly more context — the real input path for a value filter, the
    * execution for a display read — so the registry's own summary lines would only restate it.
-   * Entry-level detail still logs — the caller cannot reconstruct it.
+   * Entry decrypt failures still log unless the caller takes them through `onDecryptFailure`.
    */
   private readonly staged: boolean
   /**
@@ -1694,11 +1706,10 @@ export class ResolvedSecretTraceRegistry {
 
     const sameScope = scopesMatch(provenance.scope, this.scope)
     let importedAll = true
-    let decryptFailures = 0
-    let firstDecryptError: string | undefined
+    let decryptFailure: ProvenanceDecryptFailure | undefined
     for (const entry of provenance.entries) {
       try {
-        const { decrypted } = await decryptSecret(entry.encryptedValue)
+        const { decrypted } = await decryptSecret(entry.encryptedValue, { logFailure: false })
         this.addActiveEntry(
           {
             name: entry.name ?? '',
@@ -1710,8 +1721,12 @@ export class ResolvedSecretTraceRegistry {
         )
       } catch (error) {
         importedAll = false
-        decryptFailures += 1
-        firstDecryptError ??= getErrorMessage(error, 'Unknown error')
+        decryptFailure ??= {
+          error: getErrorMessage(error, 'Unknown error'),
+          failedEntryCount: 0,
+          totalEntryCount: provenance.entries.length,
+        }
+        decryptFailure.failedEntryCount += 1
         this.markIncomplete('entry-decrypt-failed', { origin: options.origin })
       }
     }
@@ -1720,13 +1735,15 @@ export class ResolvedSecretTraceRegistry {
      * Summarised rather than logged per entry: one rotated or corrupt key fails every entry in the
      * bundle, and a bundle may carry up to MAX_PROVENANCE_ENTRIES of them.
      */
-    if (decryptFailures > 0) {
-      logger.error('Provenance entries could not be decrypted', {
-        error: firstDecryptError,
-        failedEntryCount: decryptFailures,
-        totalEntryCount: provenance.entries.length,
-        scopeWorkspaceId: this.scope?.workspaceId,
-      })
+    if (decryptFailure) {
+      if (options.onDecryptFailure) {
+        options.onDecryptFailure(decryptFailure)
+      } else {
+        logger.error('Provenance entries could not be decrypted', {
+          ...decryptFailure,
+          scopeWorkspaceId: this.scope?.workspaceId,
+        })
+      }
     }
 
     return importedAll

apps/sim/lib/logs/execution/logger.test.ts

@@ -23,6 +23,7 @@ import {
   billingUsageMonitorMock,
   billingUsageMonitorMockFns,
 } from '@sim/testing/mocks/billing-usage-monitor.mock'
+import { maskClientMock, maskClientMockFns } from '@sim/testing/mocks/mask-client.mock'
 import { isPlainRecord } from '@sim/utils/object'
 import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest'
 import { recordUsage } from '@/lib/billing/core/usage-log'
@@ -95,6 +96,8 @@ vi.mock('@/lib/billing/core/usage', () => billingUsageMock)
 
 vi.mock('@/lib/billing/core/usage-log', () => billingUsageLogMock)
 
+vi.mock('@/lib/guardrails/mask-client', () => maskClientMock)
+
 vi.mock('@/lib/billing/threshold-billing', () => ({
   checkAndBillOverageThreshold: vi.fn(() => Promise.resolve()),
   checkAndBillPayerOverageThreshold: vi.fn(() => Promise.resolve()),
@@ -270,6 +273,9 @@ describe('ExecutionLogger', () => {
     async function completeAndReadWrite(params: {
       executionState?: SerializableExecutionState
       redactedState?: SerializableExecutionState
+      /** Runs the real log-stage PII pass instead of standing in for it. */
+      realRedaction?: boolean
+      finalOutput?: Record<string, unknown>
     }) {
       const startedAt = new Date('2026-08-11T00:00:00.000Z')
       queueTableRows(workflowExecutionLogs, [
@@ -299,12 +305,14 @@ describe('ExecutionLogger', () => {
           costTotalRefined: boolean
         }>
       }
-      vi.spyOn(internals, 'applyPiiRedaction').mockImplementation(
-        async (_workspaceId: string, payload: Record<string, unknown>) =>
-          Object.hasOwn(params, 'redactedState')
-            ? { ...payload, executionState: params.redactedState }
-            : payload
-      )
+      if (!params.realRedaction) {
+        vi.spyOn(internals, 'applyPiiRedaction').mockImplementation(
+          async (_workspaceId: string, payload: Record<string, unknown>) =>
+            Object.hasOwn(params, 'redactedState')
+              ? { ...payload, executionState: params.redactedState }
+              : payload
+        )
+      }
       vi.spyOn(internals, 'recordExecutionUsage').mockResolvedValue({
         recordedIncrement: 0,
         costTotalRefined: false,
@@ -324,7 +332,7 @@ describe('ExecutionLogger', () => {
           baseExecutionCharge: 0,
           models: {},
         },
-        finalOutput: { completed: true },
+        finalOutput: params.finalOutput ?? { completed: true },
         traceSpans: [],
         ...(params.executionState ? { executionState: params.executionState } : {}),
       })
@@ -433,6 +441,65 @@ describe('ExecutionLogger', () => {
       expect(preserved?.blockStates).toEqual(redactedState.blockStates)
     })
 
+    test('keeps encrypted provenance sidecars intact through log PII redaction', async () => {
+      queueTableRows(workspace, [
+        {
+          orgSettings: {
+            piiRedaction: { rules: [{ workspaceId: null, entityTypes: ['PHONE_NUMBER'] }] },
+          },
+        },
+      ])
+      maskClientMockFns.mockMaskPIIBatchViaHttp.mockImplementation(async (texts: string[]) => ({
+        masked: texts.map((text) => text.replace(/\d{7,}/g, '<PHONE_NUMBER>')),
+        scrubbedCount: 0,
+      }))
+      const sidecar = (encryptedValue: string) => ({
+        version: 1 as const,
+        complete: true,
+        entries: [{ name: 'API_SECRET', encryptedValue }],
+      })
+      const finalOutputProvenance = sidecar('9f3c2a5551234567e0:4b18093327716650aa:0c1d')
+      const workflowInputProvenance = sidecar('1e2f5550001111e0:77a0384460007799bb:9d8e')
+      const loopInputProvenance = sidecar('3a4b5552223333c0:66b0129930004455cc:5f6a')
+      const parallelOutputProvenance = sidecar('7c8d5554445555a0:11b0567780009911dd:2e3f')
+
+      const written = await completeAndReadWrite({
+        realRedaction: true,
+        finalOutput: { reply: 'Call me at 5551234567' },
+        executionState: {
+          ...EMPTY_STATE,
+          sourceExecutionId: 'execution-20260811123456789',
+          resolvedSecretTraceProvenance: RUN_PROVENANCE,
+          resolvedSecretTraceCheckpointVersion: 1,
+          finalOutputResolvedSecretTraceProvenance: finalOutputProvenance,
+          workflowInputResolvedSecretTraceProvenance: workflowInputProvenance,
+          loopExecutions: {
+            loop1: {
+              iteration: 0,
+              items: ['Call 5559876543'],
+              inputResolvedSecretTraceProvenance: loopInputProvenance,
+            },
+          },
+          parallelExecutions: {
+            parallel1: { resolvedSecretTraceProvenance: parallelOutputProvenance },
+          },
+        } as SerializableExecutionState,
+      })
+
+      const state = written?.executionState as SerializableExecutionState | undefined
+      expect(written?.finalOutput).toEqual({ reply: 'Call me at <PHONE_NUMBER>' })
+      expect(state?.loopExecutions?.loop1?.items).toEqual(['Call <PHONE_NUMBER>'])
+      expect(state?.finalOutputResolvedSecretTraceProvenance).toEqual(finalOutputProvenance)
+      expect(state?.workflowInputResolvedSecretTraceProvenance).toEqual(workflowInputProvenance)
+      expect(state?.loopExecutions?.loop1?.inputResolvedSecretTraceProvenance).toEqual(
+        loopInputProvenance
+      )
+      expect(state?.parallelExecutions?.parallel1?.resolvedSecretTraceProvenance).toEqual(
+        parallelOutputProvenance
+      )
+      expect(state?.sourceExecutionId).toBe('execution-20260811123456789')
+    })
+
     test('summarizes oversized execution data before storage', () => {
       const loggerInstance = new ExecutionLogger() as any
       const largePayload = 'x'.repeat(1_100_000)

apps/sim/lib/logs/execution/logger.ts

@@ -15,6 +15,7 @@ import {
   getPostgresErrorCode,
 } from '@sim/utils/errors'
 import { generateId } from '@sim/utils/id'
+import { filterUndefined, isRecordLike } from '@sim/utils/object'
 import { and, eq, inArray, sql } from 'drizzle-orm'
 import { checkUsageStatus as checkResolvedUsageStatus } from '@/lib/billing/calculations/usage-monitor'
 import { type BillingAttributionSnapshot, toBillingContext } from '@/lib/billing/core/attribution'
@@ -259,6 +260,27 @@ function summarizeTraceSpansForExecutionData(traceSpans?: TraceSpan[]): TraceSpa
   })
 }
 
+/** Puts each loop or parallel scope's own provenance back onto its masked copy. */
+function restoreSubflowScopeProvenance(
+  redactedScopes: Record<string, unknown> | undefined,
+  originalScopes: Record<string, unknown> | undefined
+): Record<string, unknown> | undefined {
+  if (!redactedScopes) return redactedScopes
+  const scopes = { ...redactedScopes }
+  for (const [scopeId, originalScope] of Object.entries(originalScopes ?? {})) {
+    const redactedScope = scopes[scopeId]
+    if (!isRecordLike(redactedScope) || !isRecordLike(originalScope)) continue
+    scopes[scopeId] = {
+      ...redactedScope,
+      ...filterUndefined({
+        inputResolvedSecretTraceProvenance: originalScope.inputResolvedSecretTraceProvenance,
+        resolvedSecretTraceProvenance: originalScope.resolvedSecretTraceProvenance,
+      }),
+    }
+  }
+  return scopes
+}
+
 function summarizeExecutionState(executionState?: SerializableExecutionState) {
   if (!executionState) {
     return undefined
@@ -777,18 +799,19 @@ export class ExecutionLogger {
     })
   }
 
-  /** Restores server-only lifecycle metadata after broad execution-state PII masking. */
+  /**
+   * Restores server-only lifecycle metadata after broad execution-state PII masking. The masker
+   * walks every string leaf, so it can rewrite a substring of an encrypted provenance entry or an
+   * id and make it unusable; these fields carry no user content, so the originals go back verbatim.
+   */
   private preservePrivateExecutionStateMetadata(
     redactedState: SerializableExecutionState | undefined,
     originalState: SerializableExecutionState | undefined
   ): SerializableExecutionState | undefined {
-    if (!redactedState) return redactedState
+    if (!redactedState || !originalState) return redactedState
 
-    const provenance = originalState?.resolvedSecretTraceProvenance
-    const provenanceCheckpointVersion = originalState?.resolvedSecretTraceCheckpointVersion
-    const trustedLargeValueAccess = originalState?.trustedLargeValueAccess
     const blockStates = { ...redactedState.blockStates }
-    for (const [blockId, originalBlockState] of Object.entries(originalState?.blockStates ?? {})) {
+    for (const [blockId, originalBlockState] of Object.entries(originalState.blockStates)) {
       const redactedBlockState = blockStates[blockId]
       if (redactedBlockState && originalBlockState.resolvedSecretTraceProvenance) {
         blockStates[blockId] = {
@@ -798,8 +821,7 @@ export class ExecutionLogger {
       }
     }
     const blockLogs = redactedState.blockLogs.map((log, index) => {
-      const displayProvenance =
-        originalState?.blockLogs[index]?.displayResolvedSecretTraceProvenance
+      const displayProvenance = originalState.blockLogs[index]?.displayResolvedSecretTraceProvenance
       return displayProvenance
         ? { ...log, displayResolvedSecretTraceProvenance: displayProvenance }
         : log
@@ -809,17 +831,26 @@ export class ExecutionLogger {
       ...redactedState,
       blockStates,
       blockLogs,
-      ...(provenance !== undefined ? { resolvedSecretTraceProvenance: provenance } : {}),
-      ...(originalState?.workflowVariableResolvedSecretTraceProvenance !== undefined
-        ? {
-            workflowVariableResolvedSecretTraceProvenance:
-              originalState.workflowVariableResolvedSecretTraceProvenance,
-          }
-        : {}),
-      ...(provenanceCheckpointVersion !== undefined
-        ? { resolvedSecretTraceCheckpointVersion: provenanceCheckpointVersion }
-        : {}),
-      ...(trustedLargeValueAccess !== undefined ? { trustedLargeValueAccess } : {}),
+      ...filterUndefined({
+        loopExecutions: restoreSubflowScopeProvenance(
+          redactedState.loopExecutions,
+          originalState.loopExecutions
+        ),
+        parallelExecutions: restoreSubflowScopeProvenance(
+          redactedState.parallelExecutions,
+          originalState.parallelExecutions
+        ),
+        sourceExecutionId: originalState.sourceExecutionId,
+        trustedLargeValueAccess: originalState.trustedLargeValueAccess,
+        resolvedSecretTraceProvenance: originalState.resolvedSecretTraceProvenance,
+        workflowVariableResolvedSecretTraceProvenance:
+          originalState.workflowVariableResolvedSecretTraceProvenance,
+        workflowInputResolvedSecretTraceProvenance:
+          originalState.workflowInputResolvedSecretTraceProvenance,
+        finalOutputResolvedSecretTraceProvenance:
+          originalState.finalOutputResolvedSecretTraceProvenance,
+        resolvedSecretTraceCheckpointVersion: originalState.resolvedSecretTraceCheckpointVersion,
+      }),
     }
   }