From db3299dc03468f232a560df4c60b5d9925f9086f Mon Sep 17 00:00:00 2001 From: ozymandiashh <234437643+ozymandiashh@users.noreply.github.com> Date: Tue, 19 May 2026 01:37:38 +0300 Subject: [PATCH] Add MCP project profile advisor --- CHANGELOG.md | 4 + src/optimize.ts | 254 +++++++++++++++++++++++++++++++++++++ tests/mcp-coverage.test.ts | 216 ++++++++++++++++++++++++++++++- 3 files changed, 472 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b1f1dcc..0419cb00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ ## Unreleased ### Added (CLI) +- **MCP project profile advisor in `optimize`.** CodeBurn now flags MCP + servers that are useful in one project but loaded into other projects where + they are never invoked, with a project-scoping prompt that preserves the hot + workflow while reducing cold-project schema overhead. - **Agent and subagent tracking coverage.** Gemini sessions now emit one provider call per assistant message with token usage instead of one aggregate call per session, preserving per-message tools, bash commands, timestamps, diff --git a/src/optimize.ts b/src/optimize.ts index c672bac0..4e1a8046 100644 --- a/src/optimize.ts +++ b/src/optimize.ts @@ -61,6 +61,12 @@ const MCP_COVERAGE_MIN_TOOLS = 10 const MCP_COVERAGE_MIN_SESSIONS = 2 const MCP_COVERAGE_LOW_THRESHOLD = 0.20 const MCP_COVERAGE_HIGH_IMPACT_TOKENS = 200_000 +const MCP_PROFILE_MIN_PROJECTS = 3 +const MCP_PROFILE_MIN_HOT_INVOCATIONS = 2 +const MCP_PROFILE_HOT_INVOCATION_SHARE = 0.80 +const MCP_PROFILE_MIN_COLD_LOADED_SESSIONS = 2 +const MCP_PROFILE_HIGH_IMPACT_TOKENS = 200_000 +const MCP_PROFILE_PREVIEW = 3 // Anthropic prices cache writes at 125% of base input and cache reads at // roughly 10% of base input. We use these to keep overhead estimates honest: // most MCP schema bytes live in the cached prefix and only get charged at @@ -895,6 +901,253 @@ export function detectMcpToolCoverage( } } +type McpProjectProfileStats = { + project: string + projectKey: string + projectPath: string + loadedSessions: number + invocations: number +} + +type McpProfileCandidate = { + server: string + toolsAvailable: number + hotProjects: McpProjectProfileStats[] + coldProjects: McpProjectProfileStats[] + coldProjectKeys: Set + loadedProjects: number + loadedSessions: number + invocations: number + hotShare: number + estimatedTokensSaved: number +} + +function projectProfileLabel(project: ProjectSummary): string { + return project.projectPath || project.project +} + +function projectProfileKey(project: ProjectSummary): string { + return projectProfileLabel(project) +} + +function sessionLoadedMcpServer( + session: ProjectSummary['sessions'][number], + server: string, +): boolean { + for (const fqn of session.mcpInventory ?? []) { + const parts = fqn.split('__') + if (parts.length >= 3 && parts[0] === 'mcp' && parts[1] === server) return true + } + return false +} + +function lowCoverageMcpServers(coverage: McpServerCoverage[]): Set { + return new Set( + coverage + .filter(c => + c.toolsAvailable > MCP_COVERAGE_MIN_TOOLS + && c.loadedSessions >= MCP_COVERAGE_MIN_SESSIONS + && c.coverageRatio < MCP_COVERAGE_LOW_THRESHOLD, + ) + .map(c => c.server), + ) +} + +function estimateMcpProfileColdSchemaCost( + projects: ProjectSummary[], + serverToolCounts: Map, + coldProjectKeysByServer: Map>, +): McpSchemaCostEstimate { + if (serverToolCounts.size === 0 || coldProjectKeysByServer.size === 0) { + return { cacheWriteTokens: 0, cacheReadTokens: 0, effectiveInputTokens: 0 } + } + + let cacheWriteTokens = 0 + let cacheReadTokens = 0 + for (const project of projects) { + const projectKey = projectProfileKey(project) + for (const session of project.sessions) { + let schemaTokens = 0 + for (const [server, toolsAvailable] of serverToolCounts) { + if (!coldProjectKeysByServer.get(server)?.has(projectKey)) continue + if (!sessionLoadedMcpServer(session, server)) continue + schemaTokens += toolsAvailable * TOKENS_PER_MCP_TOOL + } + if (schemaTokens === 0) continue + + for (const turn of session.turns) { + for (const call of turn.assistantCalls) { + if (call.usage.cacheCreationInputTokens > 0) { + cacheWriteTokens += Math.min(schemaTokens, call.usage.cacheCreationInputTokens) + } + if (call.usage.cacheReadInputTokens > 0) { + cacheReadTokens += Math.min(schemaTokens, call.usage.cacheReadInputTokens) + } + } + } + } + } + + const effectiveInputTokens = cacheWriteTokens * CACHE_WRITE_MULTIPLIER + cacheReadTokens * CACHE_READ_DISCOUNT + return { cacheWriteTokens, cacheReadTokens, effectiveInputTokens } +} + +function collectMcpProjectProfiles( + projects: ProjectSummary[], + coverage: McpServerCoverage[], +): McpProfileCandidate[] { + const suppressedServers = lowCoverageMcpServers(coverage) + const coverageByServer = new Map(coverage.map(c => [c.server, c])) + const byServer = new Map>() + + function getProjectStats(server: string, project: ProjectSummary): McpProjectProfileStats { + let serverProjects = byServer.get(server) + if (!serverProjects) { + serverProjects = new Map() + byServer.set(server, serverProjects) + } + const key = projectProfileKey(project) + let stats = serverProjects.get(key) + if (!stats) { + stats = { + project: project.project, + projectKey: key, + projectPath: projectProfileLabel(project), + loadedSessions: 0, + invocations: 0, + } + serverProjects.set(key, stats) + } + return stats + } + + for (const project of projects) { + for (const session of project.sessions) { + const loadedServers = new Set() + for (const fqn of session.mcpInventory ?? []) { + const parts = fqn.split('__') + if (parts.length >= 3 && parts[0] === 'mcp' && parts[1]) loadedServers.add(parts[1]) + } + for (const server of loadedServers) { + getProjectStats(server, project).loadedSessions++ + } + for (const [server, data] of Object.entries(session.mcpBreakdown)) { + getProjectStats(server, project).invocations += data.calls + } + } + } + + const candidates: McpProfileCandidate[] = [] + for (const [server, projectStats] of byServer) { + if (suppressedServers.has(server)) continue + const coverageStats = coverageByServer.get(server) + if (!coverageStats) continue + if (coverageStats.toolsAvailable === 0) continue + + const loaded = Array.from(projectStats.values()).filter(p => p.loadedSessions > 0) + if (loaded.length < MCP_PROFILE_MIN_PROJECTS) continue + const invocations = loaded.reduce((sum, p) => sum + p.invocations, 0) + if (invocations < MCP_PROFILE_MIN_HOT_INVOCATIONS) continue + + loaded.sort((a, b) => + b.invocations - a.invocations + || b.loadedSessions - a.loadedSessions + || a.projectPath.localeCompare(b.projectPath), + ) + const invokedProjects = loaded.filter(p => p.invocations > 0) + if (invokedProjects.length === 0) continue + const hotProjects = invokedProjects.slice(0, 2) + const hotInvocations = hotProjects.reduce((sum, p) => sum + p.invocations, 0) + const hotShare = hotInvocations / invocations + if (hotShare < MCP_PROFILE_HOT_INVOCATION_SHARE) continue + + const coldProjects = loaded.filter(p => p.invocations === 0) + const coldLoadedSessions = coldProjects.reduce((sum, p) => sum + p.loadedSessions, 0) + if (coldLoadedSessions < MCP_PROFILE_MIN_COLD_LOADED_SESSIONS) continue + + const coldProjectKeys = new Set(coldProjects.map(project => project.projectKey)) + const cost = estimateMcpProfileColdSchemaCost( + projects, + new Map([[server, coverageStats.toolsAvailable]]), + new Map([[server, coldProjectKeys]]), + ) + + candidates.push({ + server, + toolsAvailable: coverageStats.toolsAvailable, + hotProjects, + coldProjects, + coldProjectKeys, + loadedProjects: loaded.length, + loadedSessions: loaded.reduce((sum, p) => sum + p.loadedSessions, 0), + invocations, + hotShare, + estimatedTokensSaved: Math.round(cost.effectiveInputTokens), + }) + } + + candidates.sort((a, b) => + b.estimatedTokensSaved - a.estimatedTokensSaved + || b.coldProjects.length - a.coldProjects.length + || b.loadedSessions - a.loadedSessions + || a.server.localeCompare(b.server), + ) + return candidates +} + +export function detectMcpProfileAdvisor( + projects: ProjectSummary[], + coverage = aggregateMcpCoverage(projects), +): WasteFinding | null { + const candidates = collectMcpProjectProfiles(projects, coverage) + if (candidates.length === 0) return null + + const preview = candidates.slice(0, MCP_PROFILE_PREVIEW) + const lines = preview.map(candidate => { + const hot = candidate.hotProjects + .slice(0, 2) + .map(p => `${p.projectPath} (${p.invocations} call${p.invocations === 1 ? '' : 's'})`) + .join(', ') + const cold = candidate.coldProjects + .slice(0, 3) + .map(p => `${p.projectPath} (${p.loadedSessions} loaded session${p.loadedSessions === 1 ? '' : 's'})`) + .join(', ') + const coldExtra = candidate.coldProjects.length > 3 ? `, +${candidate.coldProjects.length - 3} more` : '' + return `${candidate.server}: ${Math.round(candidate.hotShare * 100)}% of ${candidate.invocations} calls in ${hot}; loaded but unused in ${cold}${coldExtra}` + }) + const extra = candidates.length > preview.length ? `; +${candidates.length - preview.length} more` : '' + const serverToolCounts = new Map(candidates.map(c => [c.server, c.toolsAvailable])) + const coldProjectKeysByServer = new Map(candidates.map(c => [c.server, c.coldProjectKeys])) + const combinedCost = estimateMcpProfileColdSchemaCost(projects, serverToolCounts, coldProjectKeysByServer) + const tokensSaved = Math.round(combinedCost.effectiveInputTokens) + const impact: Impact = tokensSaved >= MCP_PROFILE_HIGH_IMPACT_TOKENS + || candidates.length >= UNUSED_MCP_HIGH_THRESHOLD + ? 'high' + : 'medium' + + return { + title: `${candidates.length} MCP server${candidates.length === 1 ? '' : 's'} should be project-scoped`, + explanation: + `These MCP servers look useful in a small set of projects but are loaded into other projects where they are not invoked. ` + + `Project-scoping them keeps the hot-project workflow while avoiding schema overhead elsewhere. ${lines.join('; ')}${extra}.`, + impact, + tokensSaved, + fix: { + type: 'paste', + destination: 'prompt', + label: 'Ask Claude to turn this into a project-scoped MCP profile:', + text: [ + `Review these MCP profile recommendations before changing config (${preview.length} of ${candidates.length} shown):`, + ...preview.map(candidate => { + const hot = candidate.hotProjects.map(p => p.projectPath).join(', ') + const cold = candidate.coldProjects.slice(0, 3).map(p => p.projectPath).join(', ') + return `- Keep ${candidate.server} available for ${hot}; remove or project-scope it away from ${cold}. Re-add it only in projects that actually need it.` + }), + ].join('\n'), + }, + } +} + export function detectUnusedMcp( calls: ToolCall[], projects: ProjectSummary[], @@ -1800,6 +2053,7 @@ export async function scanAndDetect( () => detectDuplicateReads(toolCalls, dateRange), () => detectUnusedMcp(toolCalls, projects, projectCwds, mcpCoverage), () => detectMcpToolCoverage(projects, mcpCoverage), + () => detectMcpProfileAdvisor(projects, mcpCoverage), () => detectLowWorthSessions(projects), () => detectContextBloat(projects, lowWorthSessionIds), () => detectSessionOutliers(projects, outlierExclusions), diff --git a/tests/mcp-coverage.test.ts b/tests/mcp-coverage.test.ts index c2a45950..66181688 100644 --- a/tests/mcp-coverage.test.ts +++ b/tests/mcp-coverage.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect } from 'vitest' import { aggregateMcpCoverage, + detectMcpProfileAdvisor, detectMcpToolCoverage, estimateMcpSchemaCost, } from '../src/optimize.js' @@ -100,9 +101,13 @@ function makeSession(opts: { } function project(sessions: SessionSummary[]): ProjectSummary { + return projectNamed('p', sessions) +} + +function projectNamed(name: string, sessions: SessionSummary[]): ProjectSummary { return { - project: 'p', - projectPath: '/tmp/p', + project: name, + projectPath: `/tmp/${name}`, sessions, totalCostUSD: 0, totalApiCalls: sessions.reduce((s, ses) => s + ses.apiCalls, 0), @@ -448,3 +453,210 @@ describe('detectMcpToolCoverage', () => { expect((finding!.fix as { text: string }).text.split('\n')).toHaveLength(2) }) }) + +// --------------------------------------------------------------------------- +// detectMcpProfileAdvisor — project-scoping recommendations +// --------------------------------------------------------------------------- + +describe('detectMcpProfileAdvisor', () => { + const smallInventory = Array.from({ length: 4 }, (_, i) => `mcp__github__t${i}`) + + it('flags a server loaded across projects but invoked only in one hot project', () => { + const hotTurns = [makeTurn([ + makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }), + makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }), + ])] + const coldTurns = [makeTurn([makeCall({ cacheCreation: 10_000 })])] + const projects = [ + projectNamed('api', [ + makeSession({ inventory: smallInventory, turns: hotTurns, mcpBreakdown: { github: { calls: 2 } } }), + ]), + projectNamed('web', [ + makeSession({ inventory: smallInventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } }), + ]), + projectNamed('docs', [ + makeSession({ inventory: smallInventory, turns: coldTurns, mcpBreakdown: { github: { calls: 0 } } }), + ]), + ] + + const finding = detectMcpProfileAdvisor(projects) + expect(finding).not.toBeNull() + expect(finding!.title).toContain('project-scoped') + expect(finding!.explanation).toContain('github') + expect(finding!.explanation).toContain('/tmp/api') + expect(finding!.explanation).toContain('/tmp/web') + expect(finding!.explanation).toContain('/tmp/docs') + // Cold projects each loaded 4 tools. 4 * 400 = 1,600 schema tokens; + // cache write pricing is 1.25x, across two cold sessions = 4,000. + expect(finding!.tokensSaved).toBe(4000) + expect(finding!.fix.type).toBe('paste') + if (finding!.fix.type === 'paste') { + expect(finding!.fix.destination).toBe('prompt') + expect(finding!.fix.text).toContain('Keep github available for /tmp/api') + expect(finding!.fix.text).toContain('/tmp/web') + expect(finding!.fix.text).toContain('/tmp/docs') + } + }) + + it('does not flag servers used evenly across loaded projects', () => { + const projects = ['api', 'web', 'docs'].map(name => projectNamed(name, [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 })])], + mcpBreakdown: { github: { calls: 2 } }, + }), + ])) + + expect(detectMcpProfileAdvisor(projects)).toBeNull() + }) + + it('allows a hot profile shared by two projects', () => { + const projects = [ + projectNamed('api', [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([ + makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }), + makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }), + ])], + mcpBreakdown: { github: { calls: 2 } }, + }), + ]), + projectNamed('web', [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([ + makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 }), + makeCall({ tools: ['mcp__github__t1'], cacheCreation: 10_000 }), + ])], + mcpBreakdown: { github: { calls: 2 } }, + }), + ]), + projectNamed('docs', [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([makeCall({ cacheCreation: 10_000 })])], + mcpBreakdown: { github: { calls: 0 } }, + }), + ]), + projectNamed('playground', [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([makeCall({ cacheCreation: 10_000 })])], + mcpBreakdown: { github: { calls: 0 } }, + }), + ]), + ] + + const finding = detectMcpProfileAdvisor(projects) + expect(finding).not.toBeNull() + expect(finding!.explanation).toContain('/tmp/api') + expect(finding!.explanation).toContain('/tmp/web') + expect(finding!.explanation).toContain('/tmp/docs') + expect(finding!.explanation).toContain('/tmp/playground') + }) + + it('caps profile savings once when multiple candidate servers share cold sessions', () => { + const githubInventory = Array.from({ length: 4 }, (_, i) => `mcp__github__t${i}`) + const slackInventory = Array.from({ length: 4 }, (_, i) => `mcp__slack__t${i}`) + const inventory = [...githubInventory, ...slackInventory] + const projects = [ + projectNamed('api', [ + makeSession({ + inventory, + turns: [makeTurn([ + makeCall({ tools: ['mcp__github__t0'] }), + makeCall({ tools: ['mcp__github__t1'] }), + makeCall({ tools: ['mcp__slack__t0'] }), + makeCall({ tools: ['mcp__slack__t1'] }), + ])], + mcpBreakdown: { github: { calls: 2 }, slack: { calls: 2 } }, + }), + ]), + projectNamed('web', [ + makeSession({ + inventory, + turns: [makeTurn([makeCall({ cacheCreation: 2_000 })])], + mcpBreakdown: { github: { calls: 0 }, slack: { calls: 0 } }, + }), + ]), + projectNamed('docs', [ + makeSession({ + inventory, + turns: [makeTurn([makeCall({ cacheCreation: 2_000 })])], + mcpBreakdown: { github: { calls: 0 }, slack: { calls: 0 } }, + }), + ]), + ] + + const finding = detectMcpProfileAdvisor(projects) + expect(finding).not.toBeNull() + expect(finding!.title).toContain('2 MCP servers') + expect(finding!.explanation).toContain('github') + expect(finding!.explanation).toContain('slack') + // Each server has 4 tools (1,600 schema tokens), but both servers share + // the same cold session calls. The combined budget is 3,200 per cold + // call capped by the 2,000-token cache bucket, not 1,600 + 1,600. + // 2 cold calls * 2,000 capped write tokens * 1.25 = 5,000. + expect(finding!.tokensSaved).toBe(5000) + }) + + it('requires at least three loaded projects before recommending a profile', () => { + const projects = [ + projectNamed('api', [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([makeCall({ tools: ['mcp__github__t0'], cacheCreation: 10_000 })])], + mcpBreakdown: { github: { calls: 2 } }, + }), + ]), + projectNamed('web', [ + makeSession({ + inventory: smallInventory, + turns: [makeTurn([makeCall({ cacheCreation: 10_000 })])], + mcpBreakdown: { github: { calls: 0 } }, + }), + ]), + ] + + expect(detectMcpProfileAdvisor(projects)).toBeNull() + }) + + it('does not duplicate low tool coverage findings for the same server', () => { + const inventory = Array.from({ length: 12 }, (_, i) => `mcp__huge__t${i}`) + const projects = [ + projectNamed('api', [ + makeSession({ + inventory, + turns: [makeTurn([makeCall({ tools: ['mcp__huge__t0'], cacheCreation: 20_000 })])], + mcpBreakdown: { huge: { calls: 3 } }, + }), + ]), + projectNamed('web', [ + makeSession({ + inventory, + turns: [makeTurn([makeCall({ cacheCreation: 20_000 })])], + mcpBreakdown: { huge: { calls: 0 } }, + }), + ]), + projectNamed('docs', [ + makeSession({ + inventory, + turns: [makeTurn([makeCall({ cacheCreation: 20_000 })])], + mcpBreakdown: { huge: { calls: 0 } }, + }), + ]), + ] + const coverage = [{ + server: 'huge', + toolsAvailable: 12, + toolsInvoked: 1, + unusedTools: Array.from({ length: 11 }, (_, i) => `mcp__huge__t${i + 1}`), + invocations: 3, + loadedSessions: 3, + coverageRatio: 1 / 12, + }] + + expect(detectMcpProfileAdvisor(projects, coverage)).toBeNull() + }) +})