diff --git a/src/agents/hooks/deferred-tools.ts b/src/agents/hooks/deferred-tools.ts index cc4aa620..9df8039f 100644 --- a/src/agents/hooks/deferred-tools.ts +++ b/src/agents/hooks/deferred-tools.ts @@ -17,10 +17,15 @@ export const deferredToolsHook: Hook = { } if (ctx.systemPrompt.hasOverride) { + // Custom systemPrompt path: every tool ships inline in the catalog, + // there is no ToolSearch split. Mirror that into the prompt-cache + // fields so downstream renderers see a self-consistent shape. ctx.setTurnCatalog({ tools: ctx.allTools, deferred: [], deferredEnabled: false, + inlineTools: ctx.allTools, + discoveredCatalogTools: [], }) return } diff --git a/src/agents/hooks/forward-progress-to-channel.test.ts b/src/agents/hooks/forward-progress-to-channel.test.ts index 0f7afb70..113536c0 100644 --- a/src/agents/hooks/forward-progress-to-channel.test.ts +++ b/src/agents/hooks/forward-progress-to-channel.test.ts @@ -158,7 +158,7 @@ function hookContext(onAssistantTurn: (text: string) => void): HookContext { hasOverride: false, renderEffective: () => '', }, - turnCatalog: { tools: [], deferred: [], deferredEnabled: false }, + turnCatalog: { tools: [], deferred: [], deferredEnabled: false, inlineTools: [], discoveredCatalogTools: [] }, setTurnCatalog() {}, mergeUsage() {}, markDidCompact() {}, diff --git a/src/agents/hooks/split-render.ts b/src/agents/hooks/split-render.ts index d3080466..382a5793 100644 --- a/src/agents/hooks/split-render.ts +++ b/src/agents/hooks/split-render.ts @@ -21,6 +21,12 @@ export const splitRenderHook: Hook = { const sessionCtx = getCurrentSessionContext() const rendered = renderSystemPromptSplit(ctx.systemPrompt.template, getTodos(), { tools: ctx.turnCatalog.tools, + // 2026-05-26 cache anchoring: render only the always-loaded subset into + // the stable `## Tool Catalog`; route already-discovered deferred tools + // through the variable suffix so promoting a new tool via ToolSearch no + // longer extends the stable system prompt and breaks OpenAI prefix-cache. + inlineCatalogTools: ctx.turnCatalog.inlineTools, + discoveredCatalogTools: ctx.turnCatalog.discoveredCatalogTools, deferredTools: ctx.turnCatalog.deferred, discoveredTools: sessionCtx?.discoveredTools, }) diff --git a/src/prompt.test.ts b/src/prompt.test.ts index 6e91ca40..7b788a6a 100644 --- a/src/prompt.test.ts +++ b/src/prompt.test.ts @@ -74,6 +74,111 @@ describe('renderSystemPromptSplit — cache anchoring', () => { }) assert.equal(variable, '') }) + + // Regression: 2026-05-26 dogfood §cache hit rate root-cause. + // Even after fix/cache-suffix-relocate moved TodoList + deferred-reminder + // out of `instructions`, the `## Tool Catalog` section still listed every + // discovered tool. Each ToolSearch promotion grew that section a few bytes + // and broke OpenAI's prefix-cache fingerprint anywhere after it. Result: + // dogfood cache hit ~11% instead of the expected 60-80% — see usage.jsonl + // 1,536-token-truncation pattern. The fix routes already-discovered tool + // descriptions into the variable suffix so the stable prefix is invariant + // under discoveredTools changes. + describe('cache anchoring under discovered tool promotion', () => { + it('keeps stable prefix byte-identical when discoveredTools grows', () => { + const allTools = [ + fakeTool('Bash'), + fakeTool('Read'), + fakeTool('ToolSearch'), + fakeTool('WebSearch'), + fakeTool('Dispatch'), + ] + const inlineCatalogTools = [ + fakeTool('Bash'), + fakeTool('Read'), + fakeTool('ToolSearch'), + ] + + // Turn N: only Bash / Read / ToolSearch are callable. + const beforePromotion = renderSystemPromptSplit(template, [], { + tools: inlineCatalogTools, + inlineCatalogTools, + discoveredCatalogTools: [], + deferredTools: [fakeTool('WebSearch'), fakeTool('Dispatch')], + discoveredTools: new Map(), + }) + + // Turn N+k: model called ToolSearch and promoted WebSearch + Dispatch. + const afterPromotion = renderSystemPromptSplit(template, [], { + tools: allTools, + inlineCatalogTools, + discoveredCatalogTools: [fakeTool('WebSearch'), fakeTool('Dispatch')], + deferredTools: [fakeTool('WebSearch'), fakeTool('Dispatch')], + discoveredTools: new Map([ + ['WebSearch', 5], + ['Dispatch', 5], + ]), + }) + + assert.equal( + beforePromotion.stable, + afterPromotion.stable, + 'stable prefix must not change when ToolSearch promotes deferred tools', + ) + assert.notEqual( + beforePromotion.variable, + afterPromotion.variable, + 'variable suffix must carry the discovered tool descriptions', + ) + }) + + it('renders discovered tool descriptions in the variable suffix only', () => { + const inlineCatalogTools = [fakeTool('Bash'), fakeTool('ToolSearch')] + const { stable, variable } = renderSystemPromptSplit(template, [], { + tools: [...inlineCatalogTools, fakeTool('WebSearch')], + inlineCatalogTools, + discoveredCatalogTools: [fakeTool('WebSearch')], + deferredTools: [fakeTool('WebSearch')], + discoveredTools: new Map([['WebSearch', 1]]), + }) + assert.doesNotMatch(stable, /WebSearch/) + assert.match(variable, /WebSearch/) + // The discovered-tools reminder is its own block, + // distinct from the undiscovered-deferred reminder (no overlap with + // the existing test above). + assert.doesNotMatch( + variable, + /are now available via ToolSearch\. Their schemas are NOT loaded/, + ) + }) + + it('omits the discovered-tools section when nothing has been promoted yet', () => { + const inlineCatalogTools = [fakeTool('Bash'), fakeTool('ToolSearch')] + const { variable } = renderSystemPromptSplit(template, [], { + tools: inlineCatalogTools, + inlineCatalogTools, + discoveredCatalogTools: [], + deferredTools: [fakeTool('WebSearch')], + discoveredTools: new Map(), + }) + // Variable still carries the undiscovered-deferred reminder; only the + // new discovered-tools block should be absent. + assert.doesNotMatch(variable, /were loaded via ToolSearch earlier/) + }) + + it('falls back to rendering full tools[] when inlineCatalogTools is omitted', () => { + // Backward-compat path: callers that haven't updated still get the + // pre-fix behavior of rendering the entire tools array in the catalog. + // This keeps unrelated callers (tests, future custom systemPrompt + // shapes) from breaking at the type level. + const { stable } = renderSystemPromptSplit(template, [], { + tools: [fakeTool('Bash'), fakeTool('Read'), fakeTool('WebSearch')], + }) + assert.match(stable, /^- Bash$/m) + assert.match(stable, /^- Read$/m) + assert.match(stable, /^- WebSearch$/m) + }) + }) }) function fakeTool(name: string): Tool { diff --git a/src/prompt.ts b/src/prompt.ts index 2f815df1..4992de74 100644 --- a/src/prompt.ts +++ b/src/prompt.ts @@ -96,6 +96,14 @@ export type SystemPromptRenderOptions = { tools: Tool[] deferredTools?: Tool[] discoveredTools?: ReadonlyMap + // 2026-05-26: cache anchoring. When `inlineCatalogTools` is provided, the + // stable `## Tool Catalog` section renders only that subset, and the + // remainder of `tools` (discovered via ToolSearch this session) is rendered + // separately into the variable suffix. Callers that don't pass this still + // fall back to rendering the full `tools` array in the catalog — preserves + // backward compat for tests / custom systemPrompt shapes. + inlineCatalogTools?: Tool[] + discoveredCatalogTools?: Tool[] } function formatSkillsSection(role: Role): string { @@ -572,8 +580,13 @@ export function renderSystemPromptSplit( todos: TodoItem[], options?: SystemPromptRenderOptions, ): RenderedSystemPrompt { - const tools = options?.tools ?? [] - const toolDescriptions = formatToolCatalog(tools) + // Stable catalog: prefer the caller-supplied inlineCatalogTools subset (the + // always-loaded tools, whose membership is fixed for the query loop). Fall + // back to the full tools[] for callers that haven't opted into the split + // (tests, future custom systemPrompt shapes). The discovered subset is + // routed into the variable suffix below. + const stableCatalogTools = options?.inlineCatalogTools ?? options?.tools ?? [] + const toolDescriptions = formatToolCatalog(stableCatalogTools) const toolSection = [ '## Tool Catalog', 'Available tools (full schemas / usage live in the tools API description field):', @@ -588,6 +601,12 @@ export function renderSystemPromptSplit( if (todoSection) { variableParts.push(todoSection) } + const discoveredReminder = buildDiscoveredToolsReminder( + options?.discoveredCatalogTools ?? [], + ) + if (discoveredReminder) { + variableParts.push(discoveredReminder) + } const deferredReminder = buildDeferredToolsReminder( options?.deferredTools ?? [], options?.discoveredTools ?? new Map(), @@ -682,3 +701,27 @@ ${undiscovered .join('\n')} ` } + +// 2026-05-26: discovered (already-promoted) deferred tools used to be listed +// in the stable `## Tool Catalog` section, which made the stable system +// prompt grow each time ToolSearch promoted a new tool. Under OpenAI's +// auto-cache (wire-byte prefix fingerprint), this broke prefix cache for +// every subsequent turn — dogfood §cache hit rate measured ~11% instead of +// the expected 60-80%. The fix renders these tools in the variable suffix +// (injected into the last user message) so the stable section is invariant +// under discoveredTools changes. The block name lists the tool names + their +// whenToUse so the model has the same one-line hint it had pre-fix; full +// schemas still live in the provider's tools API field. +function buildDiscoveredToolsReminder( + discoveredCatalogTools: readonly Tool[], +): string { + if (discoveredCatalogTools.length === 0) { + return '' + } + return ` +The following deferred tools were loaded via ToolSearch earlier in this session and are now callable. Full schemas live in the tools API description field — call them by name: +${discoveredCatalogTools + .map(tool => (tool.whenToUse ? `- ${tool.name}: ${tool.whenToUse}` : `- ${tool.name}`)) + .join('\n')} +` +} diff --git a/src/query.ts b/src/query.ts index b2e19369..1f269a3c 100644 --- a/src/query.ts +++ b/src/query.ts @@ -491,6 +491,8 @@ export async function query(params: QueryParams): Promise<{ tools: params.tools, deferred: [], deferredEnabled: false, + inlineTools: params.tools, + discoveredCatalogTools: [], } const makeHookContext = (messagesSnapshot?: Message[]): HookContext => ({ @@ -580,6 +582,8 @@ export async function query(params: QueryParams): Promise<{ tools: params.tools, deferred: [], deferredEnabled: false, + inlineTools: params.tools, + discoveredCatalogTools: [], } for (const hook of lifecycleHooks) { await hook.beforeTurn?.(makeHookContext()) diff --git a/src/tools/deferred-loading.ts b/src/tools/deferred-loading.ts index 07d6016b..3caae1e2 100644 --- a/src/tools/deferred-loading.ts +++ b/src/tools/deferred-loading.ts @@ -8,6 +8,17 @@ export type TurnToolCatalog = { tools: Tool[] deferred: Tool[] deferredEnabled: boolean + // 2026-05-26: prompt-cache split. `inlineTools` is the always-loaded subset + // (alwaysLoad-tagged + ToolSearch when deferred loading is on); its + // membership is fixed for the lifetime of one query loop, so rendering it + // into the `## Tool Catalog` stable section keeps that section byte-stable. + // `discoveredCatalogTools` is the this-turn promoted-via-ToolSearch subset; + // its membership changes between turns, so the prompt renders it inside the + // variable suffix (injected into the last user message) instead of the + // stable system section. Both are still in `tools` so the provider's tools + // array carries every callable schema this turn. + inlineTools: Tool[] + discoveredCatalogTools: Tool[] } export function buildTurnToolCatalog(input: { @@ -17,20 +28,26 @@ export function buildTurnToolCatalog(input: { }): TurnToolCatalog { const { allTools, discoveredTools, config } = input if (!shouldEnableDeferredLoading(config, allTools)) { + const all = [...allTools] return { - tools: [...allTools], + tools: all, deferred: [], deferredEnabled: false, + inlineTools: all, + discoveredCatalogTools: [], } } const { alwaysLoaded, deferred } = partitionTools(allTools) const discovered = deferred.filter(tool => discoveredTools.has(tool.name)) - const tools = [...alwaysLoaded, toolSearchTool, ...discovered] + const inlineTools = dedupeTools([...alwaysLoaded, toolSearchTool]) + const tools = dedupeTools([...inlineTools, ...discovered]) return { - tools: dedupeTools(tools), + tools, deferred, deferredEnabled: true, + inlineTools, + discoveredCatalogTools: discovered, } }