Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions apps/docs/openapi-v2-logs.json
Original file line number Diff line number Diff line change
Expand Up @@ -1448,6 +1448,11 @@
"type": "number",
"description": "Offset from the root span in milliseconds."
},
"contentWithheld": {
"type": "boolean",
"const": true,
"description": "Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned."
},
"toolCalls": {
"type": "array",
"items": {
Expand Down
7 changes: 7 additions & 0 deletions apps/sim/lib/api/contracts/logs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,7 @@ export type LogTraceSpan = {
tokens?: number | { total?: number; input?: number; output?: number }
cost?: { total?: number; input?: number; output?: number; toolCost?: number }
relativeStartMs?: number
contentWithheld?: true
toolCalls?: Array<z.output<typeof toolCallSchema>>
children?: LogTraceSpan[]
}
Expand Down Expand Up @@ -273,6 +274,12 @@ export const traceSpanSchema: z.ZodType<LogTraceSpan> = z
.number()
.describe('Offset from the root span in milliseconds.')
.optional(),
contentWithheld: z
.literal(true)
.describe(
"Present when the log projection withheld some or all of this span's content (input, output, thinking, error message, tool-call or provider-segment content) because it could not be proven safe to display. Status, timing, and usage remain. A field missing from a span carrying this marker may exist on the run but was not returned."
)
.optional(),
toolCalls: z.array(toolCallSchema).describe('Tool calls recorded by the span.').optional(),
children: z.array(traceSpanSchema).describe('Nested child trace spans.').optional(),
})
Expand Down
79 changes: 79 additions & 0 deletions apps/sim/lib/logs/execution/trace-secret-projection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -502,6 +502,85 @@ describe('projectTraceSpansForSecrets', () => {
expect(span.children?.[0]).not.toHaveProperty('output')
})

it('marks every span of a structural-only fallback as content-withheld', async () => {
const source = [
createSpan({
output: { secret: 'raw' },
children: [createSpan({ id: 'child-1', output: { secret: 'raw' } })],
}),
]

const incomplete = await projectTraceSpansForSecrets(source, {
registry: createRegistry([], false),
store: STORE,
})
const failedMatcher = await projectTraceSpansForSecrets(source, {
registry: createRegistry([
{ plaintext: 's'.repeat(64 * 1024 + 1), replacement: '{{TOO_LARGE}}' },
]),
store: STORE,
})
const failedInvariant = await enforceTraceSpanSecretInvariant(
[createSpan({ output: { apiKey: '[REDACTED]' } })],
{
registry: createRegistry([{ plaintext: 'REDACTED', replacement: '{{X}}' }]),
store: STORE,
}
)

for (const [span] of [incomplete, failedMatcher]) {
expect(span.contentWithheld).toBe(true)
expect(span.children?.[0].contentWithheld).toBe(true)
}
expect(failedInvariant[0].contentWithheld).toBe(true)
})

it('marks a span whose content field was omitted and leaves fully projected spans unmarked', async () => {
const [omitted, intact] = await projectTraceSpansForSecrets(
[
createSpan({ input: { safe: 'keep' }, output: { secretval: 1, '{{TOKEN}}': 2 } }),
createSpan({ id: 'span-2', output: { value: 'secretval' } }),
],
{
registry: createRegistry([{ plaintext: 'secretval', replacement: '{{TOKEN}}' }]),
store: STORE,
}
)

expect(omitted.input).toEqual({ safe: 'keep' })
expect(omitted.contentWithheld).toBe(true)
expect(intact.output).toEqual({ value: '{{TOKEN}}' })
expect(intact).not.toHaveProperty('contentWithheld')
})

it('marks a span whose tool-call content was omitted', async () => {
const [span] = await projectTraceSpansForSecrets(
[
createSpan({
output: { safe: 'keep' },
toolCalls: [
{
name: 'legacy',
duration: 1,
startTime: '2026-07-01T00:00:00.000Z',
endTime: '2026-07-01T00:00:00.001Z',
status: 'success',
output: { secretval: 1, '{{TOKEN}}': 2 },
},
],
}),
],
{
registry: createRegistry([{ plaintext: 'secretval', replacement: '{{TOKEN}}' }]),
store: STORE,
}
)

expect(span.output).toEqual({ safe: 'keep' })
expect(span.toolCalls?.[0]).not.toHaveProperty('output')
expect(span.contentWithheld).toBe(true)
})

it('fails closed when a later log transform reintroduces an active literal', async () => {
const source = [createSpan({ output: { apiKey: '[REDACTED]' } })]

Expand Down
8 changes: 8 additions & 0 deletions apps/sim/lib/logs/execution/trace-secret-projection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,8 @@ interface ProjectionContext {
sourceLargeValueBytes: number
storedLargeValueBytes: number
storedLargeValueCount: number
/** Content fields omitted so far; a span compares it across its own fields to set `contentWithheld`. */
omittedContentFields: number
}

interface OversizedHydrationGate {
Expand Down Expand Up @@ -162,6 +164,7 @@ function createProjectionContext(
manifestIds: new WeakMap<object, string>(),
nextManifestId: 0,
largeValueCount: 0,
omittedContentFields: 0,
sourceLargeValueBytes: 0,
storedLargeValueBytes: 0,
storedLargeValueCount: 0,
Expand Down Expand Up @@ -840,6 +843,7 @@ async function sanitizeContentField(
logger.warn('Omitting trace content that could not be sanitized', {
failure: describeProjectionFailure(error),
})
context.omittedContentFields += 1
return OMIT
}
}
Expand Down Expand Up @@ -959,6 +963,7 @@ async function sanitizeTraceSpan(
}

let projected = omit(span, ['displayResolvedSecretTraceProvenance']) as TraceSpan
const omittedBefore = context.omittedContentFields

for (const key of ['input', 'output', 'thinking', 'errorMessage'] as const) {
const value = span[key]
Expand Down Expand Up @@ -993,6 +998,8 @@ async function sanitizeTraceSpan(
projected.providerTiming = { ...span.providerTiming, segments }
}

if (context.omittedContentFields > omittedBefore) projected.contentWithheld = true

assertContentIsSafe(projected, depth)

if (span.children !== undefined) {
Expand Down Expand Up @@ -1106,6 +1113,7 @@ function structuralOnlySpan(

return {
...structural,
contentWithheld: true,
...(modelToolCalls
? {
modelToolCalls: projectBoundedArray(modelToolCalls, (call) =>
Expand Down
8 changes: 8 additions & 0 deletions apps/sim/lib/logs/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,14 @@ export interface TraceSpan {
* one read, not the run.
*/
childTraceAccess?: 'granted' | 'disabled' | 'missing' | 'truncated'
/**
* Set by secret projection when it omitted content it could not prove safe from
* this span (`input`, `output`, `thinking`, `errorMessage`, or its tool-call and
* provider-segment content), or reduced the span to structure only. Status, timing,
* and usage remain. Without it a withheld field is
* indistinguishable from one the block never produced.
*/
contentWithheld?: true
model?: string
cost?: {
input?: number
Expand Down
62 changes: 62 additions & 0 deletions apps/sim/lib/mothership/agent-cli/engines.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,68 @@ describe('logs query', () => {
})
})

const WITHHELD_RUNS = {
[RUNS_PATH]: {
data: [
{ runId: 'run-1', status: 'completed' },
{ runId: 'run-2', status: 'completed' },
],
},
'/api/v2/logs/run-1': {
data: {
traceSpans: [{ name: 'Count rows', status: 'success', output: { result: { count: 6 } } }],
},
},
// The log projection withheld this span's content; its structure survived.
'/api/v2/logs/run-2': {
data: {
traceSpans: [{ name: 'Count rows', status: 'success', duration: 12, contentWithheld: true }],
},
},
}

async function logsQueryWithheld(flags: Record<string, string>) {
const result = await runEngine('logs query', ['wf-1'], runtimeWith(WITHHELD_RUNS), {
block: 'Count rows',
...flags,
})
expect(result.exitCode).toBe(0)
return JSON.parse(result.stdout)
}

describe('logs query on a span whose content was withheld', () => {
it('reports withheld content instead of a missing path, and counts it', async () => {
const report = await logsQueryWithheld({ field: 'result.count' })
expect(report.rows[0]).toMatchObject({ runId: 'run-1', value: 6 })
expect(report.rows[1]).toMatchObject({ runId: 'run-2', hits: 1, value: null })
expect(report.rows[1].note).not.toBe('path not found on span')
expect(report.rows[1].note).toMatch(/withheld/)
expect(report.contentWithheld).toBe(1)
})

it('still answers structural paths on a withheld span', async () => {
const report = await logsQueryWithheld({ field: 'duration' })
expect(report.rows[1]).toMatchObject({ runId: 'run-2', value: 12 })
expect(report.rows[1].note).toBeUndefined()
expect(report.contentWithheld).toBeUndefined()
})

it('keeps a withheld run under --where instead of counting it as filtered out', async () => {
const report = await logsQueryWithheld({ where: 'result.count=6' })
expect(report.filteredOut).toBe(0)
expect(report.rows.map((row: { runId: string }) => row.runId)).toEqual(['run-1', 'run-2'])
expect(report.rows[1].note).toMatch(/withheld/)
expect(report.contentWithheld).toBe(1)
})

it('still resolves a readable --field when --where hits withheld content', async () => {
const report = await logsQueryWithheld({ field: 'duration', where: 'result.count=6' })
expect(report.rows[1]).toMatchObject({ runId: 'run-2', value: 12 })
expect(report.rows[1].note).toMatch(/--where/)
expect(report.contentWithheld).toBe(1)
})
})

const DEPS_STATE = {
blocks: {
'trigger-1': { type: 'starter', name: 'Start' },
Expand Down
37 changes: 29 additions & 8 deletions apps/sim/lib/mothership/agent-cli/engines/query.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ const MAX_LIMIT = 50
const TRACE_FETCH_CONCURRENCY = 5
const VALUE_MAX_CHARS = 600
const PATH_NOT_FOUND_NOTE = 'path not found on span'
/**
* The log projection withheld this span's content rather than return what it could not
* prove safe to display, so an unresolved content path says nothing about the run.
*/
const CONTENT_WITHHELD_NOTE = 'span content withheld by the log projection; value unknown'
const WHERE_WITHHELD_NOTE =
'span content withheld by the log projection; --where could not be checked, run kept'

/**
* Keys that live on the span itself. A path whose head is anything else is read
Expand Down Expand Up @@ -56,6 +63,7 @@ interface QueryTraceSpan {
duration?: number
input?: Record<string, unknown>
output?: Record<string, unknown>
contentWithheld?: true
children?: QueryTraceSpan[]
}

Expand Down Expand Up @@ -158,6 +166,7 @@ export const logsQueryCommand: AgentCliEngine = {

let filteredOut = 0
let missingTrace = 0
let contentWithheld = 0
let fieldResolvedUnder: 'output' | undefined
let whereResolvedUnder: 'output' | undefined
const rows = await mapWithConcurrency(
Expand Down Expand Up @@ -193,23 +202,34 @@ export const logsQueryCommand: AgentCliEngine = {
}
return { ...base, hits: 0, value: null }
}
const matchedRow = {
...base,
hits: matches.length,
blockStatus: last.status ?? 'success',
}
let whereWithheld = false
if (wherePath) {
const matched = resolveSpanPathWithFallback(last, wherePath)
if (matched.resolvedUnder) whereResolvedUnder = matched.resolvedUnder
if (String(matched.value) !== whereValue) {
// Filtering on a withheld value would silently drop a run that may well match.
whereWithheld = matched.value === undefined && last.contentWithheld === true
if (!whereWithheld && String(matched.value) !== whereValue) {
filteredOut++
return undefined
}
}
const resolved = resolveSpanPathWithFallback(last, field)
if (resolved.resolvedUnder) fieldResolvedUnder = resolved.resolvedUnder
return {
...base,
hits: matches.length,
blockStatus: last.status ?? 'success',
value: clipValue(resolved.value),
...(resolved.value === undefined ? { note: PATH_NOT_FOUND_NOTE } : {}),
}
const fieldWithheld = resolved.value === undefined && last.contentWithheld === true
if (whereWithheld || fieldWithheld) contentWithheld++
const note = fieldWithheld
? CONTENT_WITHHELD_NOTE
: resolved.value === undefined
? PATH_NOT_FOUND_NOTE
: whereWithheld
? WHERE_WITHHELD_NOTE
: undefined
return { ...matchedRow, value: clipValue(resolved.value), ...(note ? { note } : {}) }
}
)

Expand All @@ -230,6 +250,7 @@ export const logsQueryCommand: AgentCliEngine = {
}
: {}),
...(missingTrace ? { missingTrace } : {}),
...(contentWithheld ? { contentWithheld } : {}),
rows: kept,
},
null,
Expand Down
2 changes: 2 additions & 0 deletions packages/sim-cli/src/generated/v2-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6244,6 +6244,7 @@ type GetLogResponseRef1 = {
toolCost?: number
}
relativeStartMs?: number
contentWithheld?: true
toolCalls?: Array<{
id?: string
name?: string
Expand Down Expand Up @@ -9037,6 +9038,7 @@ type ListLogsResponseRef2 = {
toolCost?: number
}
relativeStartMs?: number
contentWithheld?: true
toolCalls?: Array<{
id?: string
name?: string
Expand Down