diff --git a/apps/sim/app/api/v2/knowledge/search/route.application.test.ts b/apps/sim/app/api/v2/knowledge/search/route.application.test.ts index e65fe3637a9..414110c301b 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.application.test.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.application.test.ts @@ -276,7 +276,9 @@ describe('Knowledge search provenance through the V2 route and reranker HTTP bou expect(body.data.rerankerStatus).toBe('applied') expect(providerPayload().documents).toEqual([CONTENT.replace(SECRET, '{{TOKEN}}')]) expect(body.data.results[0].content).toBe(CONTENT) - expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledWith('synthetic-encrypted-token') + expect(encryptionMockFns.mockDecryptSecret).toHaveBeenCalledWith('synthetic-encrypted-token', { + logFailure: false, + }) expect(mockGenerateSearchEmbedding).toHaveBeenCalledWith( requestInput.query, expect.anything(), diff --git a/apps/sim/executor/utils/resolved-secret-trace-registry.ts b/apps/sim/executor/utils/resolved-secret-trace-registry.ts index 04f804a32ad..56a475e816d 100644 --- a/apps/sim/executor/utils/resolved-secret-trace-registry.ts +++ b/apps/sim/executor/utils/resolved-secret-trace-registry.ts @@ -417,6 +417,18 @@ export interface ImportResolvedSecretTraceProvenanceOptions { * `source-provenance-incomplete` from a symptom into an address. */ origin?: string + /** + * Takes the import's entry decrypt-failure summary in place of the registry's own error line, + * for a caller that reports the failure itself with the context the registry lacks. + */ + onDecryptFailure?: (failure: ProvenanceDecryptFailure) => void +} + +/** Entry decrypt failures of one provenance import, summarised. */ +export interface ProvenanceDecryptFailure { + error: string + failedEntryCount: number + totalEntryCount: number } export interface ExportResolvedSecretTraceProvenanceForValueOptions { @@ -966,7 +978,7 @@ export class ResolvedSecretTraceRegistry { * A staged registry filters values for one operation and is then discarded. Its caller owns the * reporting and says it with strictly more context — the real input path for a value filter, the * execution for a display read — so the registry's own summary lines would only restate it. - * Entry-level detail still logs — the caller cannot reconstruct it. + * Entry decrypt failures still log unless the caller takes them through `onDecryptFailure`. */ private readonly staged: boolean /** @@ -1694,11 +1706,10 @@ export class ResolvedSecretTraceRegistry { const sameScope = scopesMatch(provenance.scope, this.scope) let importedAll = true - let decryptFailures = 0 - let firstDecryptError: string | undefined + let decryptFailure: ProvenanceDecryptFailure | undefined for (const entry of provenance.entries) { try { - const { decrypted } = await decryptSecret(entry.encryptedValue) + const { decrypted } = await decryptSecret(entry.encryptedValue, { logFailure: false }) this.addActiveEntry( { name: entry.name ?? '', @@ -1710,8 +1721,12 @@ export class ResolvedSecretTraceRegistry { ) } catch (error) { importedAll = false - decryptFailures += 1 - firstDecryptError ??= getErrorMessage(error, 'Unknown error') + decryptFailure ??= { + error: getErrorMessage(error, 'Unknown error'), + failedEntryCount: 0, + totalEntryCount: provenance.entries.length, + } + decryptFailure.failedEntryCount += 1 this.markIncomplete('entry-decrypt-failed', { origin: options.origin }) } } @@ -1720,13 +1735,15 @@ export class ResolvedSecretTraceRegistry { * Summarised rather than logged per entry: one rotated or corrupt key fails every entry in the * bundle, and a bundle may carry up to MAX_PROVENANCE_ENTRIES of them. */ - if (decryptFailures > 0) { - logger.error('Provenance entries could not be decrypted', { - error: firstDecryptError, - failedEntryCount: decryptFailures, - totalEntryCount: provenance.entries.length, - scopeWorkspaceId: this.scope?.workspaceId, - }) + if (decryptFailure) { + if (options.onDecryptFailure) { + options.onDecryptFailure(decryptFailure) + } else { + logger.error('Provenance entries could not be decrypted', { + ...decryptFailure, + scopeWorkspaceId: this.scope?.workspaceId, + }) + } } return importedAll diff --git a/apps/sim/lib/logs/execution/logger.test.ts b/apps/sim/lib/logs/execution/logger.test.ts index d09ac80ebfc..b0ab40c2350 100644 --- a/apps/sim/lib/logs/execution/logger.test.ts +++ b/apps/sim/lib/logs/execution/logger.test.ts @@ -23,6 +23,7 @@ import { billingUsageMonitorMock, billingUsageMonitorMockFns, } from '@sim/testing/mocks/billing-usage-monitor.mock' +import { maskClientMock, maskClientMockFns } from '@sim/testing/mocks/mask-client.mock' import { isPlainRecord } from '@sim/utils/object' import { afterAll, beforeEach, describe, expect, test, vi } from 'vitest' import { recordUsage } from '@/lib/billing/core/usage-log' @@ -95,6 +96,8 @@ vi.mock('@/lib/billing/core/usage', () => billingUsageMock) vi.mock('@/lib/billing/core/usage-log', () => billingUsageLogMock) +vi.mock('@/lib/guardrails/mask-client', () => maskClientMock) + vi.mock('@/lib/billing/threshold-billing', () => ({ checkAndBillOverageThreshold: vi.fn(() => Promise.resolve()), checkAndBillPayerOverageThreshold: vi.fn(() => Promise.resolve()), @@ -270,6 +273,9 @@ describe('ExecutionLogger', () => { async function completeAndReadWrite(params: { executionState?: SerializableExecutionState redactedState?: SerializableExecutionState + /** Runs the real log-stage PII pass instead of standing in for it. */ + realRedaction?: boolean + finalOutput?: Record }) { const startedAt = new Date('2026-08-11T00:00:00.000Z') queueTableRows(workflowExecutionLogs, [ @@ -299,12 +305,14 @@ describe('ExecutionLogger', () => { costTotalRefined: boolean }> } - vi.spyOn(internals, 'applyPiiRedaction').mockImplementation( - async (_workspaceId: string, payload: Record) => - Object.hasOwn(params, 'redactedState') - ? { ...payload, executionState: params.redactedState } - : payload - ) + if (!params.realRedaction) { + vi.spyOn(internals, 'applyPiiRedaction').mockImplementation( + async (_workspaceId: string, payload: Record) => + Object.hasOwn(params, 'redactedState') + ? { ...payload, executionState: params.redactedState } + : payload + ) + } vi.spyOn(internals, 'recordExecutionUsage').mockResolvedValue({ recordedIncrement: 0, costTotalRefined: false, @@ -324,7 +332,7 @@ describe('ExecutionLogger', () => { baseExecutionCharge: 0, models: {}, }, - finalOutput: { completed: true }, + finalOutput: params.finalOutput ?? { completed: true }, traceSpans: [], ...(params.executionState ? { executionState: params.executionState } : {}), }) @@ -433,6 +441,65 @@ describe('ExecutionLogger', () => { expect(preserved?.blockStates).toEqual(redactedState.blockStates) }) + test('keeps encrypted provenance sidecars intact through log PII redaction', async () => { + queueTableRows(workspace, [ + { + orgSettings: { + piiRedaction: { rules: [{ workspaceId: null, entityTypes: ['PHONE_NUMBER'] }] }, + }, + }, + ]) + maskClientMockFns.mockMaskPIIBatchViaHttp.mockImplementation(async (texts: string[]) => ({ + masked: texts.map((text) => text.replace(/\d{7,}/g, '')), + scrubbedCount: 0, + })) + const sidecar = (encryptedValue: string) => ({ + version: 1 as const, + complete: true, + entries: [{ name: 'API_SECRET', encryptedValue }], + }) + const finalOutputProvenance = sidecar('9f3c2a5551234567e0:4b18093327716650aa:0c1d') + const workflowInputProvenance = sidecar('1e2f5550001111e0:77a0384460007799bb:9d8e') + const loopInputProvenance = sidecar('3a4b5552223333c0:66b0129930004455cc:5f6a') + const parallelOutputProvenance = sidecar('7c8d5554445555a0:11b0567780009911dd:2e3f') + + const written = await completeAndReadWrite({ + realRedaction: true, + finalOutput: { reply: 'Call me at 5551234567' }, + executionState: { + ...EMPTY_STATE, + sourceExecutionId: 'execution-20260811123456789', + resolvedSecretTraceProvenance: RUN_PROVENANCE, + resolvedSecretTraceCheckpointVersion: 1, + finalOutputResolvedSecretTraceProvenance: finalOutputProvenance, + workflowInputResolvedSecretTraceProvenance: workflowInputProvenance, + loopExecutions: { + loop1: { + iteration: 0, + items: ['Call 5559876543'], + inputResolvedSecretTraceProvenance: loopInputProvenance, + }, + }, + parallelExecutions: { + parallel1: { resolvedSecretTraceProvenance: parallelOutputProvenance }, + }, + } as SerializableExecutionState, + }) + + const state = written?.executionState as SerializableExecutionState | undefined + expect(written?.finalOutput).toEqual({ reply: 'Call me at ' }) + expect(state?.loopExecutions?.loop1?.items).toEqual(['Call ']) + expect(state?.finalOutputResolvedSecretTraceProvenance).toEqual(finalOutputProvenance) + expect(state?.workflowInputResolvedSecretTraceProvenance).toEqual(workflowInputProvenance) + expect(state?.loopExecutions?.loop1?.inputResolvedSecretTraceProvenance).toEqual( + loopInputProvenance + ) + expect(state?.parallelExecutions?.parallel1?.resolvedSecretTraceProvenance).toEqual( + parallelOutputProvenance + ) + expect(state?.sourceExecutionId).toBe('execution-20260811123456789') + }) + test('summarizes oversized execution data before storage', () => { const loggerInstance = new ExecutionLogger() as any const largePayload = 'x'.repeat(1_100_000) diff --git a/apps/sim/lib/logs/execution/logger.ts b/apps/sim/lib/logs/execution/logger.ts index 850e817fd48..0c26186d937 100644 --- a/apps/sim/lib/logs/execution/logger.ts +++ b/apps/sim/lib/logs/execution/logger.ts @@ -15,6 +15,7 @@ import { getPostgresErrorCode, } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' +import { filterUndefined, isRecordLike } from '@sim/utils/object' import { and, eq, inArray, sql } from 'drizzle-orm' import { checkUsageStatus as checkResolvedUsageStatus } from '@/lib/billing/calculations/usage-monitor' import { type BillingAttributionSnapshot, toBillingContext } from '@/lib/billing/core/attribution' @@ -259,6 +260,27 @@ function summarizeTraceSpansForExecutionData(traceSpans?: TraceSpan[]): TraceSpa }) } +/** Puts each loop or parallel scope's own provenance back onto its masked copy. */ +function restoreSubflowScopeProvenance( + redactedScopes: Record | undefined, + originalScopes: Record | undefined +): Record | undefined { + if (!redactedScopes) return redactedScopes + const scopes = { ...redactedScopes } + for (const [scopeId, originalScope] of Object.entries(originalScopes ?? {})) { + const redactedScope = scopes[scopeId] + if (!isRecordLike(redactedScope) || !isRecordLike(originalScope)) continue + scopes[scopeId] = { + ...redactedScope, + ...filterUndefined({ + inputResolvedSecretTraceProvenance: originalScope.inputResolvedSecretTraceProvenance, + resolvedSecretTraceProvenance: originalScope.resolvedSecretTraceProvenance, + }), + } + } + return scopes +} + function summarizeExecutionState(executionState?: SerializableExecutionState) { if (!executionState) { return undefined @@ -777,18 +799,19 @@ export class ExecutionLogger { }) } - /** Restores server-only lifecycle metadata after broad execution-state PII masking. */ + /** + * Restores server-only lifecycle metadata after broad execution-state PII masking. The masker + * walks every string leaf, so it can rewrite a substring of an encrypted provenance entry or an + * id and make it unusable; these fields carry no user content, so the originals go back verbatim. + */ private preservePrivateExecutionStateMetadata( redactedState: SerializableExecutionState | undefined, originalState: SerializableExecutionState | undefined ): SerializableExecutionState | undefined { - if (!redactedState) return redactedState + if (!redactedState || !originalState) return redactedState - const provenance = originalState?.resolvedSecretTraceProvenance - const provenanceCheckpointVersion = originalState?.resolvedSecretTraceCheckpointVersion - const trustedLargeValueAccess = originalState?.trustedLargeValueAccess const blockStates = { ...redactedState.blockStates } - for (const [blockId, originalBlockState] of Object.entries(originalState?.blockStates ?? {})) { + for (const [blockId, originalBlockState] of Object.entries(originalState.blockStates)) { const redactedBlockState = blockStates[blockId] if (redactedBlockState && originalBlockState.resolvedSecretTraceProvenance) { blockStates[blockId] = { @@ -798,8 +821,7 @@ export class ExecutionLogger { } } const blockLogs = redactedState.blockLogs.map((log, index) => { - const displayProvenance = - originalState?.blockLogs[index]?.displayResolvedSecretTraceProvenance + const displayProvenance = originalState.blockLogs[index]?.displayResolvedSecretTraceProvenance return displayProvenance ? { ...log, displayResolvedSecretTraceProvenance: displayProvenance } : log @@ -809,17 +831,26 @@ export class ExecutionLogger { ...redactedState, blockStates, blockLogs, - ...(provenance !== undefined ? { resolvedSecretTraceProvenance: provenance } : {}), - ...(originalState?.workflowVariableResolvedSecretTraceProvenance !== undefined - ? { - workflowVariableResolvedSecretTraceProvenance: - originalState.workflowVariableResolvedSecretTraceProvenance, - } - : {}), - ...(provenanceCheckpointVersion !== undefined - ? { resolvedSecretTraceCheckpointVersion: provenanceCheckpointVersion } - : {}), - ...(trustedLargeValueAccess !== undefined ? { trustedLargeValueAccess } : {}), + ...filterUndefined({ + loopExecutions: restoreSubflowScopeProvenance( + redactedState.loopExecutions, + originalState.loopExecutions + ), + parallelExecutions: restoreSubflowScopeProvenance( + redactedState.parallelExecutions, + originalState.parallelExecutions + ), + sourceExecutionId: originalState.sourceExecutionId, + trustedLargeValueAccess: originalState.trustedLargeValueAccess, + resolvedSecretTraceProvenance: originalState.resolvedSecretTraceProvenance, + workflowVariableResolvedSecretTraceProvenance: + originalState.workflowVariableResolvedSecretTraceProvenance, + workflowInputResolvedSecretTraceProvenance: + originalState.workflowInputResolvedSecretTraceProvenance, + finalOutputResolvedSecretTraceProvenance: + originalState.finalOutputResolvedSecretTraceProvenance, + resolvedSecretTraceCheckpointVersion: originalState.resolvedSecretTraceCheckpointVersion, + }), } } diff --git a/apps/sim/lib/logs/execution/trace-store.ts b/apps/sim/lib/logs/execution/trace-store.ts index d2334e83800..48c86eb92a2 100644 --- a/apps/sim/lib/logs/execution/trace-store.ts +++ b/apps/sim/lib/logs/execution/trace-store.ts @@ -14,6 +14,7 @@ import { traceSpansHaveHandledErrors } from '@/lib/logs/execution/trace-spans/ha import type { TraceSpan } from '@/lib/logs/types' import { isResolvedSecretTraceProvenanceV1, + type ProvenanceDecryptFailure, ResolvedSecretTraceRegistry, } from '@/executor/utils/resolved-secret-trace-registry' @@ -446,7 +447,7 @@ export async function materializeExecutionDataForDisplayWithBlockOutputs( executionState?.[RESOLVED_SECRET_PROVENANCE_KEY], 'traceStore.blockOutputRunProvenance' ) - const provenanceFaults = new Map() + const provenanceFaults = new Map() if (runImport.fault) provenanceFaults.set('run', runImport.fault) const blockOutputs = new Map() const projectionStore = createReadOnlyProjectionStore(context) @@ -494,9 +495,15 @@ function readRecord(value: unknown): Record | undefined { type StoredDisplayProvenanceFault = 'incomplete' | 'malformed' | 'undecryptable' +interface StoredDisplayFault { + kind: StoredDisplayProvenanceFault + /** Set for `undecryptable`: the entry detail the registry would otherwise log on its own line. */ + decryptFailure?: ProvenanceDecryptFailure +} + interface StoredDisplayEnvelopeImport { registry: ResolvedSecretTraceRegistry | undefined - fault: StoredDisplayProvenanceFault | undefined + fault: StoredDisplayFault | undefined } /** @@ -517,15 +524,27 @@ async function importStoredDisplayEnvelope( ): Promise { if (provenance === undefined) return { registry: undefined, fault: undefined } if (!isResolvedSecretTraceProvenanceV1(provenance)) { - return { registry: undefined, fault: 'malformed' } + return { registry: undefined, fault: { kind: 'malformed' } } } const registry = new ResolvedSecretTraceRegistry([], provenance.scope, { staged: true }) - await registry.importProvenance(provenance, { trusted: true, origin }) - const fault = !provenance.complete - ? 'incomplete' + let decryptFailure: ProvenanceDecryptFailure | undefined + await registry.importProvenance(provenance, { + trusted: true, + origin, + // Only a complete envelope can classify as `undecryptable`, whose line carries this detail. + ...(provenance.complete + ? { + onDecryptFailure: (failure: ProvenanceDecryptFailure) => { + decryptFailure = failure + }, + } + : {}), + }) + const fault: StoredDisplayFault | undefined = !provenance.complete + ? { kind: 'incomplete' } : registry.isPermanentlyIncomplete() - ? 'undecryptable' + ? { kind: 'undecryptable', decryptFailure } : undefined return { registry, fault } } @@ -538,7 +557,7 @@ const STORED_PROVENANCE_FAULT_REPORTS = { message: 'Stored execution provenance cannot vouch for display content', }, malformed: { level: 'error', message: 'Stored execution provenance is malformed' }, - /** The entry-level decrypt error already logs its counts; this adds the execution it hit. */ + /** Carries the entry decrypt detail too, which the registry hands over instead of logging. */ undecryptable: { level: 'error', message: 'Stored execution provenance could not be decrypted' }, } as const satisfies Record< StoredDisplayProvenanceFault, @@ -563,7 +582,7 @@ const STORED_PROVENANCE_FAULT_REPORTS = { function reportStoredDisplayProvenanceFaults( site: string, context: TraceStoreReadContext, - faults: ReadonlyMap + faults: ReadonlyMap ): void { if (faults.size === 0) return const details = { @@ -576,13 +595,23 @@ function reportStoredDisplayProvenanceFaults( StoredDisplayProvenanceFault, (typeof STORED_PROVENANCE_FAULT_REPORTS)[StoredDisplayProvenanceFault], ][]) { - const parts = [...faults].filter(([, fault]) => fault === kind).map(([part]) => part) - if (parts.length === 0) continue + const matching = [...faults].filter(([, fault]) => fault.kind === kind) + if (matching.length === 0) continue + const decryptFailures = matching.flatMap(([, fault]) => + fault.decryptFailure ? [fault.decryptFailure] : [] + ) logger[report.level](report.message, { ...details, fault: kind, - parts: parts.slice(0, MAX_REPORTED_PROVENANCE_FAULT_PARTS), - partCount: parts.length, + parts: matching.slice(0, MAX_REPORTED_PROVENANCE_FAULT_PARTS).map(([part]) => part), + partCount: matching.length, + ...(decryptFailures.length > 0 + ? { + error: decryptFailures[0].error, + failedEntryCount: decryptFailures.reduce((sum, f) => sum + f.failedEntryCount, 0), + totalEntryCount: decryptFailures.reduce((sum, f) => sum + f.totalEntryCount, 0), + } + : {}), }) } } @@ -625,7 +654,7 @@ export async function projectExecutionDataForDisplay( return projectLegacyExecutionDataForDisplay(executionData) } - const provenanceFaults = new Map() + const provenanceFaults = new Map() const runImport = await importStoredDisplayEnvelope(provenance, 'traceStore.spanProvenance') const registry = runImport.registry if (runImport.fault) provenanceFaults.set('traceSpans', runImport.fault) diff --git a/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts b/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts index 96a707dbcfa..a7fad652c5f 100644 --- a/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts +++ b/apps/sim/lib/mothership/agent-cli/file-provenance.test.ts @@ -458,7 +458,7 @@ describe('file provenance at the actual CLI and model-result boundary', () => { const output = await response.text() expect(output).toContain(content) expect(trace.isPermanentlyIncomplete()).toBe(false) - expect(mocks.decrypt).toHaveBeenCalledWith('fixture-ciphertext') + expect(mocks.decrypt).toHaveBeenCalledWith('fixture-ciphertext', { logFailure: false }) expect( JSON.stringify( inspectToolResultForCopilot({ success: true, output }, trace, 'sim_cli').result diff --git a/apps/sim/lib/mothership/request/tools/client.test.ts b/apps/sim/lib/mothership/request/tools/client.test.ts index 80d5fe38d69..7ba230faee4 100644 --- a/apps/sim/lib/mothership/request/tools/client.test.ts +++ b/apps/sim/lib/mothership/request/tools/client.test.ts @@ -607,7 +607,7 @@ describe('workflow client tool completion', () => { registry, }) - expect(decryptSecret).toHaveBeenCalledWith('encrypted-child-secret') + expect(decryptSecret).toHaveBeenCalledWith('encrypted-child-secret', { logFailure: false }) expect(completion?.data).toEqual({ success: true, workflowId: 'workflow-1',