feat: implement act apply-model and report baseline tripwire (#607)

src/act/cli.ts

@@ -65,6 +65,44 @@ export function registerActCommands(program: Command): void {
       }
     })
 
+  act
+    .command('apply-model <project>')
+    .description('Apply the model default recommendation for a project')
+    .action(async (project: string) => {
+      try {
+        const { parseAllSessions, filterProjectsByName } = await import('../parser.js')
+        const { recommendModelDefault, buildApplyModelDefaultPlan } = await import('./model-defaults.js')
+        const { runAction } = await import('./apply.js')
+        const chalk = (await import('chalk')).default
+
+        const projects = filterProjectsByName(await parseAllSessions(), [project])
+        const p = projects[0]
+        if (!p) {
+          console.error(`Project "${project}" not found in session history.`)
+          process.exitCode = 1
+          return
+        }
+
+        const recommendation = recommendModelDefault(p)
+        if (!recommendation) {
+          console.error(`No default model recommendation available for ${project} at this time.`)
+          process.exitCode = 1
+          return
+        }
+
+        const plan = await buildApplyModelDefaultPlan(recommendation)
+        const record = await runAction(plan)
+
+        console.log(`Applied default model ${chalk.green(recommendation.candidateModel)} for ${project}`)
+        console.log(chalk.dim(`  Evidence: ${recommendation.candidateEditTurns} turns, ${(recommendation.candidateOneShotRate * 100).toFixed(1)}% one-shot, $${recommendation.candidateCostPerEdit.toFixed(3)}/edit`))
+        console.log(chalk.dim(`  Undo anytime: codeburn act undo ${shortId(record.id)}`))
+        console.log(chalk.dim(`  Per-session override: --model <name>`))
+      } catch (err) {
+        console.error(err instanceof Error ? err.message : String(err))
+        process.exitCode = 1
+      }
+    })
+
   act
     .command('report')
     .description('Realized vs estimated savings for applied actions older than 3 days')

src/act/model-defaults.ts

@@ -0,0 +1,170 @@
+import { readFile } from 'node:fs/promises'
+import { join } from 'node:path'
+
+import { aggregateModelStats, type ModelStats } from '../compare-stats.js'
+import type { ProjectSummary } from '../types.js'
+import { sha256File } from './backup.js'
+import type { ActionPlan } from './types.js'
+
+const MIN_EDIT_TURNS = 30
+const MAX_COST_RATIO = 0.6
+const ONE_SHOT_TOLERANCE = 0.03
+const DEBUGGING_HEAVY_THRESHOLD = 0.4
+const RECENCY_DAYS = 14
+const MS_PER_DAY = 24 * 60 * 60 * 1000
+
+export type ModelDefaultRecommendation = {
+  project: string
+  projectPath: string
+  currentModel: string
+  candidateModel: string
+  provider: string
+  currentEditTurns: number
+  candidateEditTurns: number
+  currentOneShotRate: number
+  candidateOneShotRate: number
+  currentCostPerEdit: number
+  candidateCostPerEdit: number
+  savingsPct: number
+  debuggingHeavy: boolean
+}
+
+function oneShotRate(s: ModelStats): number {
+  return s.editTurns > 0 ? s.oneShotTurns / s.editTurns : 0
+}
+
+function costPerEdit(s: ModelStats): number {
+  return s.editTurns > 0 ? s.editCost / s.editTurns : Number.POSITIVE_INFINITY
+}
+
+function isRecent(lastSeen: string, now: Date): boolean {
+  if (!lastSeen) return false
+  const seen = new Date(lastSeen)
+  if (Number.isNaN(seen.getTime())) return false
+  return now.getTime() - seen.getTime() <= RECENCY_DAYS * MS_PER_DAY
+}
+
+function providerByModel(project: ProjectSummary): Map<string, string> {
+  const providers = new Map<string, string>()
+  for (const session of project.sessions) {
+    for (const turn of session.turns) {
+      const primary = turn.assistantCalls[0]
+      if (!primary || primary.model === '<synthetic>') continue
+      if (!providers.has(primary.model)) providers.set(primary.model, primary.provider)
+      for (const call of turn.assistantCalls) {
+        if (call.model === '<synthetic>') continue
+        if (!providers.has(call.model)) providers.set(call.model, call.provider)
+      }
+    }
+  }
+  return providers
+}
+
+function isDebuggingHeavy(project: ProjectSummary): boolean {
+  let debuggingEditTurns = 0
+  let totalEditTurns = 0
+  for (const session of project.sessions) {
+    for (const breakdown of Object.values(session.categoryBreakdown)) {
+      totalEditTurns += breakdown.editTurns
+    }
+    debuggingEditTurns += session.categoryBreakdown.debugging?.editTurns ?? 0
+  }
+  return totalEditTurns > 0 && debuggingEditTurns / totalEditTurns > DEBUGGING_HEAVY_THRESHOLD
+}
+
+export function recommendModelDefault(project: ProjectSummary, opts: { now?: Date } = {}): ModelDefaultRecommendation | null {
+  const now = opts.now ?? new Date()
+  const stats = aggregateModelStats([project])
+    .filter(s => s.model !== '<synthetic>' && s.editTurns >= MIN_EDIT_TURNS)
+    .sort((a, b) => b.editTurns - a.editTurns || b.editCost - a.editCost)
+
+  const current = stats[0]
+  if (!current) return null
+
+  const providers = providerByModel(project)
+  const provider = providers.get(current.model)
+  if (!provider || !isRecent(current.lastSeen, now)) return null
+
+  const currentRate = oneShotRate(current)
+  const currentCost = costPerEdit(current)
+  if (!Number.isFinite(currentCost) || currentCost <= 0) return null
+
+  const debuggingHeavy = isDebuggingHeavy(project)
+  const tolerance = debuggingHeavy ? 0 : ONE_SHOT_TOLERANCE
+
+  const candidates = stats
+    .slice(1)
+    .filter(candidate => providers.get(candidate.model) === provider)
+    .filter(candidate => isRecent(candidate.lastSeen, now))
+    .map(candidate => ({
+      candidate,
+      candidateRate: oneShotRate(candidate),
+      candidateCost: costPerEdit(candidate),
+    }))
+    .filter(({ candidateRate }) => candidateRate >= currentRate - tolerance)
+    .filter(({ candidateCost }) => candidateCost <= currentCost * MAX_COST_RATIO)
+    .sort((a, b) => {
+      const savingsA = 1 - a.candidateCost / currentCost
+      const savingsB = 1 - b.candidateCost / currentCost
+      return savingsB - savingsA || b.candidateRate - a.candidateRate
+    })
+
+  const best = candidates[0]
+  if (!best) return null
+
+  return {
+    project: project.project,
+    projectPath: project.projectPath,
+    currentModel: current.model,
+    candidateModel: best.candidate.model,
+    provider,
+    currentEditTurns: current.editTurns,
+    candidateEditTurns: best.candidate.editTurns,
+    currentOneShotRate: currentRate,
+    candidateOneShotRate: best.candidateRate,
+    currentCostPerEdit: currentCost,
+    candidateCostPerEdit: best.candidateCost,
+    savingsPct: (1 - best.candidateCost / currentCost) * 100,
+    debuggingHeavy,
+  }
+}
+
+export async function buildApplyModelDefaultPlan(recommendation: ModelDefaultRecommendation): Promise<ActionPlan> {
+  const settingsPath = join(recommendation.projectPath, '.claude', 'settings.json')
+  let settings: Record<string, unknown> = {}
+  let expectedHash: string | null = null
+
+  try {
+    const raw = await readFile(settingsPath, 'utf-8')
+    expectedHash = await sha256File(settingsPath)
+    settings = JSON.parse(raw) as Record<string, unknown>
+    if (!settings || Array.isArray(settings) || typeof settings !== 'object') settings = {}
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException).code
+    if (code !== 'ENOENT') throw err
+  }
+
+  settings.model = recommendation.candidateModel
+
+  return {
+    kind: 'model-default',
+    findingId: `model-default:${recommendation.project}`,
+    description: `Set Claude Code default model to ${recommendation.candidateModel} for ${recommendation.project}`,
+    changes: [{
+      op: 'edit',
+      path: settingsPath,
+      content: JSON.stringify(settings, null, 2) + '\n',
+      expectedHash,
+    }],
+    baseline: {
+      windowDays: 30,
+      capturedAt: new Date().toISOString(),
+      estimatedTokens: 0,
+      sessions: recommendation.currentEditTurns + recommendation.candidateEditTurns,
+      metrics: {
+        [recommendation.candidateModel]: recommendation.candidateOneShotRate,
+        [recommendation.currentModel]: recommendation.currentOneShotRate,
+      },
+    },
+  }
+}

src/act/report.ts

@@ -284,6 +284,53 @@ async function guardRow(
   }
 }
 
+async function modelDefaultRow(
+  base: ActReportRow, rec: ActionRecord, sessions: SessionSummary[],
+  baseline: ActionBaseline, afterStart: Date, now: Date,
+): Promise<ActReportRow> {
+  const models = Object.keys(baseline.metrics)
+  if (models.length < 2) return { ...base, note: 'not measurable: invalid baseline' }
+  const candidateModel = models[0]!
+  const preApplyRate = baseline.metrics[candidateModel]!
+
+  const mockProject: ProjectSummary = {
+    project: 'mock',
+    projectPath: 'mock',
+    totalCostUSD: 0,
+    totalSavingsUSD: 0,
+    totalApiCalls: 0,
+    totalProxiedCostUSD: 0,
+    sessions,
+  }
+
+  const { aggregateModelStats } = await import('../compare-stats.js')
+  const stats = aggregateModelStats([mockProject]).find(s => s.model === candidateModel)
+
+  if (!stats || stats.editTurns < 20) {
+    return { ...base, note: `not measurable: < 20 edit turns for ${candidateModel} since apply` }
+  }
+
+  const postApplyRate = stats.oneShotTurns / stats.editTurns
+
+  if (postApplyRate < preApplyRate - 0.05) {
+    return {
+      ...base,
+      status: 'measured',
+      realizedTokens: 0,
+      confidence: 'low',
+      note: `quality regression, consider undo: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%`
+    }
+  }
+
+  return {
+    ...base,
+    status: 'measured',
+    realizedTokens: 0,
+    confidence: 'normal',
+    note: `correlation, not attribution: one-shot rate ${(preApplyRate * 100).toFixed(1)}% -> ${(postApplyRate * 100).toFixed(1)}%`
+  }
+}
+
 async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterStart: Date, now: Date, opts: ActReportOptions): Promise<ActReportRow> {
   const estimatedAtApply = rec.baseline?.estimatedTokens ?? 0
   const base: ActReportRow = {
@@ -307,6 +354,7 @@ async function computeRow(rec: ActionRecord, sessions: SessionSummary[], afterSt
   if (rec.kind === 'claude-md-rule') return readEditRow(base, sessions, baseline, afterStart, now)
   if (rec.kind === 'shell-config') return { ...base, note: 'not measurable: bash result token sizes are not retained in the summary' }
   if (rec.kind === 'guard-install') return guardRow(base, afterStart, now, baseline, opts)
+  if (rec.kind === 'model-default') return modelDefaultRow(base, rec, sessions, baseline, afterStart, now)
   return { ...base, note: 'not measurable: kind is not tracked by act report' }
 }
 

src/compare.tsx

@@ -8,6 +8,7 @@ import { parseAllSessions } from './parser.js'
 import { getAllProviders } from './providers/index.js'
 import type { ProjectSummary, DateRange } from './types.js'
 import { patchStdoutForWindows } from './ink-win.js'
+import { recommendModelDefault, type ModelDefaultRecommendation } from './act/model-defaults.js'
 
 const ORANGE = '#FF8C42'
 const GREEN = '#5BF5A0'
@@ -51,11 +52,12 @@ function barWidth(rate: number): number {
 
 type ModelSelectorProps = {
   models: ModelStats[]
+  recommendations: ModelDefaultRecommendation[]
   onSelect: (a: ModelStats, b: ModelStats) => void
   onBack: () => void
 }
 
-function ModelSelector({ models, onSelect, onBack }: ModelSelectorProps) {
+function ModelSelector({ models, recommendations, onSelect, onBack }: ModelSelectorProps) {
   const { exit } = useApp()
   const [cursor, setCursor] = useState(0)
   const [selected, setSelected] = useState<Set<number>>(new Set())
@@ -126,6 +128,26 @@ function ModelSelector({ models, onSelect, onBack }: ModelSelectorProps) {
         <Text color={ORANGE} bold>[esc]</Text><Text dimColor> back  </Text>
         <Text color={ORANGE} bold>[q]</Text><Text dimColor> quit</Text>
       </Text>
+
+      {recommendations.length > 0 && (
+        <Box flexDirection="column" marginTop={1} borderStyle="round" borderColor={ORANGE} paddingX={1}>
+          <Text bold color={ORANGE}>Model defaults recommendation</Text>
+          <Text> </Text>
+          {recommendations.map(rec => (
+            <Box flexDirection="column" key={rec.project} marginBottom={1}>
+              <Text>
+                <Text>{rec.project}: </Text>
+                <Text bold>{rec.currentModel}</Text>
+                <Text>{' -> '}</Text>
+                <Text bold color={GREEN}>{rec.candidateModel}</Text>
+              </Text>
+              <Text color={DIM}>  Current:  {(rec.currentOneShotRate*100).toFixed(1)}% one-shot over {rec.currentEditTurns} edits, {formatCost(rec.currentCostPerEdit)}/edit</Text>
+              <Text color={DIM}>  Candidate: {(rec.candidateOneShotRate*100).toFixed(1)}% one-shot over {rec.candidateEditTurns} edits, {formatCost(rec.candidateCostPerEdit)}/edit</Text>
+              <Text>  To apply: <Text color="#00FFFF">codeburn act apply-model {rec.project}</Text></Text>
+            </Box>
+          ))}
+        </Box>
+      )}
     </Box>
   )
 }
@@ -317,6 +339,14 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
   const { exit } = useApp()
   const [phase, setPhase] = useState<'select' | 'loading' | 'results'>('select')
   const [models, setModels] = useState<ModelStats[]>(() => aggregateModelStats(projects))
+  const [recommendations, setRecommendations] = useState<ModelDefaultRecommendation[]>(() => {
+    const recs: ModelDefaultRecommendation[] = []
+    for (const p of projects) {
+      const rec = recommendModelDefault(p)
+      if (rec) recs.push(rec)
+    }
+    return recs
+  })
   const [pickedNames, setPickedNames] = useState<[string, string] | null>(null)
   const [selectedA, setSelectedA] = useState<ModelStats | null>(null)
   const [selectedB, setSelectedB] = useState<ModelStats | null>(null)
@@ -331,6 +361,13 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
     const newModels = aggregateModelStats(projects)
     setModels(newModels)
 
+    const recs: ModelDefaultRecommendation[] = []
+    for (const p of projects) {
+      const rec = recommendModelDefault(p)
+      if (rec) recs.push(rec)
+    }
+    setRecommendations(recs)
+
     if (!pickedNames) return
     const hasA = newModels.some(m => m.model === pickedNames[0])
     const hasB = newModels.some(m => m.model === pickedNames[1])
@@ -460,6 +497,7 @@ export function CompareView({ projects, onBack }: CompareViewProps) {
   return (
     <ModelSelector
       models={models}
+      recommendations={recommendations}
       onSelect={handleSelect}
       onBack={onBack}
     />