From ea8dec628ef6da1b555ce9816d9bed41edda157b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 24 Oct 2026 04:45:03 +0000 Subject: [PATCH] Add Lite, Medium, and Heavy harness tiers Step 1 of #874 loads three budgets on the Step 0 harness and routes one triage label (harness:lite, harness:medium, harness:heavy). The process harness stays default. HarnessPro checks a tier offline against CI, core/dev/eval, and rsi-baseline. References #874 Co-authored-by: Hongsheng Liu --- .github/labels.json | 15 ++ .github/scripts/sync-labels.test.mjs | 2 +- core/src/host/harness.js | 204 +++++++++++++++++- core/src/host/usage.js | 53 +++-- core/test/harness.test.js | 124 ++++++++++- core/test/usage-fields.test.js | 36 +++- docs/subsystems/core-host.md | 19 +- patches/ds-bot/0001-core-fixes.patch | 64 ++++-- patches/ds-bot/files/src/host/harness.js | 204 +++++++++++++++++- patches/ds-bot/files/test/harness.test.js | 124 ++++++++++- .../ds-bot/files/test/usage-fields.test.js | 36 +++- 11 files changed, 818 insertions(+), 63 deletions(-) diff --git a/.github/labels.json b/.github/labels.json index f6bc13f5..ef6920c2 100644 --- a/.github/labels.json +++ b/.github/labels.json @@ -63,5 +63,20 @@ "name": "pass", "color": "006B75", "description": "Reviewer PASS verdict at the merged head; set by pass-label, never removed" + }, + { + "name": "harness:lite", + "color": "BFDADC", + "description": "Triage routes this issue to the Lite harness tier (#874)" + }, + { + "name": "harness:medium", + "color": "F9D0C4", + "description": "Triage routes this issue to the Medium harness tier (#874)" + }, + { + "name": "harness:heavy", + "color": "6F42C1", + "description": "Triage routes this issue to the Heavy harness tier (#874)" } ] diff --git a/.github/scripts/sync-labels.test.mjs b/.github/scripts/sync-labels.test.mjs index 0c0ebf9c..092a7f0c 100644 --- a/.github/scripts/sync-labels.test.mjs +++ b/.github/scripts/sync-labels.test.mjs @@ -16,7 +16,7 @@ test('plan: create missing, keep equal, update changed, ignore others', () => { test('labels.json: valid, unique, agent-ready is the approval label', () => { const names = wanted.map((l) => l.name.toLowerCase()) // #702 retired the approval label; #739 added `pass`, which records that a verdict was a PASS. - assert.equal(wanted.length, 13, 'the 13 live labels') + assert.equal(wanted.length, 16, 'the 16 live labels') assert.equal(new Set(names).size, names.length) for (const l of wanted) { assert.match(l.color, /^[0-9A-F]{6}$/i) diff --git a/core/src/host/harness.js b/core/src/host/harness.js index 80e25191..b652a804 100644 --- a/core/src/host/harness.js +++ b/core/src/host/harness.js @@ -1,13 +1,65 @@ -// Step 0 (#903 / #874 Option B): one declarative harness layer. -// Today's setup is the single `default` harness. Behavior-neutral: the runtime still -// runs the same modules; this file names them as one config so Step 1 can add tiers later. +// Declarative harness layer (#874 Option B). // -// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` -// (not one of the three mirrors ds-bot/ · patches/ds-bot/ · core/). That injection stays -// outside this harness file; eval `P3-tree-split` must keep passing. +// Step 0 (#903): today's setup is the `default` harness. It names the modules the +// runtime already runs. It does not replace them. +// +// Step 1: Lite, Medium, and Heavy are three more configs on that same layer. +// Each one points at the same modules and adds a budget (prompt size, tool cap +// and allow-list, model output cap, step/retry cap). `applied` is false: loading +// a tier does not change prompt.js, tools.js, models.js, or turns.js. The +// default path stays the process harness. Triage picks a tier with one label +// (`harness:lite`, `harness:medium`, `harness:heavy`); `risk:T*` is not a harness. +// +// HarnessPro is the offline check for that config. It names CI, core/dev/eval, +// and rsi-baseline. It does not run them and it does not call a model. +// +// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under +// `plugins/tree` (not one of the three mirrors). That injection stays outside +// this file; eval `P3-tree-split` must keep passing. export const DEFAULT_HARNESS_ID = 'default' +// Triage sets exactly one of these. The controller does not write them, and +// they are not risk tiers. +export const HARNESS_TIER_LABELS = Object.freeze({ + lite: 'harness:lite', + medium: 'harness:medium', + heavy: 'harness:heavy', +}) + +export const HARNESS_PRO_VERIFIERS = Object.freeze(['ci', 'core/dev/eval', 'rsi-baseline']) + +const TIER_BUDGETS = { + lite: { + prompt: { maxChars: 8_000 }, + tools: { + maxCalls: 8, + allow: ['list_bots', 'list_models', 'recall', 'read_own_chat', 'read_group_chat', 'list_secrets'], + }, + model: { maxTokens: 2_048 }, + stepRetry: { maxSteps: 4, maxRetries: 0 }, + }, + medium: { + prompt: { maxChars: 16_000 }, + tools: { + maxCalls: 24, + allow: [ + 'list_bots', 'list_models', 'recall', 'read_own_chat', 'read_group_chat', 'list_secrets', + 'message_bot', 'ask_user', 'remember', 'forget', 'list_capabilities', 'bot_schedules', + 'read_browser', 'request_secret', 'post_to_group', 'update_bot', 'record_decision', + ], + }, + model: { maxTokens: 8_192 }, + stepRetry: { maxSteps: 12, maxRetries: 1 }, + }, + heavy: { + prompt: { maxChars: 48_000 }, + tools: { maxCalls: 64, allow: null }, + model: { maxTokens: 16_384 }, + stepRetry: { maxSteps: 32, maxRetries: 1 }, + }, +} + export function defaultHarness() { return { id: DEFAULT_HARNESS_ID, @@ -26,12 +78,148 @@ export function defaultHarness() { } } +function tierHarness(id) { + const budget = TIER_BUDGETS[id] + const base = defaultHarness() + return { + ...base, + id, + prompt: { ...base.prompt, maxChars: budget.prompt.maxChars }, + tools: { + ...base.tools, + maxCalls: budget.tools.maxCalls, + allow: budget.tools.allow === null ? null : [...budget.tools.allow], + }, + model: { ...base.model, maxTokens: budget.model.maxTokens }, + stepRetry: { + ...base.stepRetry, + maxSteps: budget.stepRetry.maxSteps, + maxRetries: budget.stepRetry.maxRetries, + }, + verifiers: { + offline: [...HARNESS_PRO_VERIFIERS], + note: 'HarnessPro checks this config offline against CI, core/dev/eval, and rsi-baseline. No model call.', + }, + // Declared budget only. The named modules still run as they do for default. + applied: false, + } +} + +const TIERS = { + lite: () => tierHarness('lite'), + medium: () => tierHarness('medium'), + heavy: () => tierHarness('heavy'), +} + export function resolveHarness(id = DEFAULT_HARNESS_ID) { - if (id !== DEFAULT_HARNESS_ID) throw new Error(`unknown harness: ${id}`) - return defaultHarness() + if (id === DEFAULT_HARNESS_ID) return defaultHarness() + const build = TIERS[id] + if (!build) throw new Error(`unknown harness: ${id}`) + return build() +} + +function labelNames(input) { + const list = Array.isArray(input) ? input : (input?.labels ?? []) + const names = [] + for (const entry of list) { + const name = typeof entry === 'string' ? entry : entry?.name + if (typeof name === 'string' && name.trim() !== '') names.push(name.trim().toLowerCase()) + } + return names +} + +export function routeHarness(input) { + const byLabel = new Map(Object.entries(HARNESS_TIER_LABELS).map(([id, label]) => [label, id])) + const hits = [] + for (const name of labelNames(input)) { + if (!name.startsWith('harness:')) continue + const id = byLabel.get(name) + if (id === undefined) throw new Error(`unknown harness label: ${name}`) + if (!hits.includes(id)) hits.push(id) + } + if (hits.length > 1) throw new Error(`one harness tier label, got ${hits.map(id => HARNESS_TIER_LABELS[id]).join(', ')}`) + return resolveHarness(hits[0] ?? DEFAULT_HARNESS_ID) +} + +function secretAt(value, path) { + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + const found = secretAt(value[index], `${path}[${index}]`) + if (found) return found + } + return '' + } + if (value !== null && typeof value === 'object') { + for (const [key, child] of Object.entries(value)) { + const here = path === '' ? key : `${path}.${key}` + if (/api[_-]?key/i.test(key) || /^(token|secret|password|authorization)$/i.test(key)) return here + const found = secretAt(child, here) + if (found) return found + } + return '' + } + if (typeof value === 'string' && (/\bsk-[A-Za-z0-9]{8,}\b/.test(value) || value.includes('DEEPSEEK_API_KEY') || value.includes('GITHUB_TOKEN'))) { + return path === '' ? 'value' : path + } + return '' +} + +// Offline shape check. A tier must keep the default modules and name the three +// HarnessPro verifiers. This function does not run CI, eval, or rsi-baseline. +export function harnessPro(harness) { + const failures = [] + const base = defaultHarness() + if (!harness || typeof harness.id !== 'string') failures.push('id') + else if (harness.id !== DEFAULT_HARNESS_ID && TIERS[harness.id] === undefined) failures.push('id') + for (const key of ['prompt', 'tools', 'model', 'stepRetry']) { + if (harness?.[key]?.module !== base[key].module) failures.push(`${key}.module`) + } + if (harness?.id && harness.id !== DEFAULT_HARNESS_ID) { + const offline = harness.verifiers?.offline + const same = Array.isArray(offline) + && offline.length === HARNESS_PRO_VERIFIERS.length + && HARNESS_PRO_VERIFIERS.every((name, index) => offline[index] === name) + if (!same) failures.push('verifiers.offline') + if (!(harness.prompt?.maxChars > 0)) failures.push('prompt.maxChars') + if (!(harness.tools?.maxCalls > 0)) failures.push('tools.maxCalls') + if (harness.tools?.allow !== null && !Array.isArray(harness.tools?.allow)) failures.push('tools.allow') + if (!(harness.model?.maxTokens > 0)) failures.push('model.maxTokens') + if (!(harness.stepRetry?.maxSteps > 0)) failures.push('stepRetry.maxSteps') + if (!Number.isInteger(harness.stepRetry?.maxRetries) || harness.stepRetry.maxRetries < 0) failures.push('stepRetry.maxRetries') + } + const secret = secretAt(harness, '') + if (secret) failures.push(`secret at ${secret}`) + return { ok: failures.length === 0, failures, verifiers: [...HARNESS_PRO_VERIFIERS] } +} + +export function harnessTierOrder() { + const lite = resolveHarness('lite') + const medium = resolveHarness('medium') + const heavy = resolveHarness('heavy') + const axes = { + 'prompt.maxChars': [lite.prompt.maxChars, medium.prompt.maxChars, heavy.prompt.maxChars], + 'tools.maxCalls': [lite.tools.maxCalls, medium.tools.maxCalls, heavy.tools.maxCalls], + 'model.maxTokens': [lite.model.maxTokens, medium.model.maxTokens, heavy.model.maxTokens], + 'stepRetry.maxSteps': [lite.stepRetry.maxSteps, medium.stepRetry.maxSteps, heavy.stepRetry.maxSteps], + 'stepRetry.maxRetries': [lite.stepRetry.maxRetries, medium.stepRetry.maxRetries, heavy.stepRetry.maxRetries], + } + const failures = [] + for (const [name, [low, mid, high]] of Object.entries(axes)) { + if (!(low <= mid && mid <= high) || low === high) failures.push(name) + } + if (!Array.isArray(lite.tools.allow) || !Array.isArray(medium.tools.allow)) failures.push('allow') + else { + for (const tool of lite.tools.allow) { + if (!medium.tools.allow.includes(tool)) failures.push(`allow ${tool}`) + } + if (lite.tools.allow.length >= medium.tools.allow.length) failures.push('allow size') + } + if (heavy.tools.allow !== null) failures.push('heavy.allow') + return { ok: failures.length === 0, failures, axes } } export function install(rt) { rt.harness = resolveHarness(DEFAULT_HARNESS_ID) rt.resolveHarness = resolveHarness + rt.routeHarness = routeHarness } diff --git a/core/src/host/usage.js b/core/src/host/usage.js index dcf9324a..04cb39df 100644 --- a/core/src/host/usage.js +++ b/core/src/host/usage.js @@ -13,6 +13,7 @@ import { randomUUID } from 'node:crypto' import { mkdir, unlink } from 'node:fs/promises' import { join } from 'node:path' +import { resolveHarness, routeHarness } from './harness.js' const LEDGER_VERSION = 3 const PAGE_MESSAGES = 400 @@ -107,6 +108,9 @@ function noteIssuePr(state, event) { // request header named. A fork's events up to its inherited cut are the parent's, so // the cut drops the rows before it (`reset`). export function foldCalls(events, from = {}) { + // A routed tier is a row field. It stays out of `state`: marks.fold is the + // resume cursor (provider, model, retry) and a pre-Step-0 mark has no harness. + const harness = typeof from.harness === 'string' && from.harness !== '' ? resolveHarness(from.harness).id : undefined const state = { provider: from.provider ?? '', model: from.model ?? '', @@ -167,7 +171,7 @@ export function foldCalls(events, from = {}) { toolCalls: state.turnTools, wallMs, issuePr: state.issuePr, - })) + }, { harness })) break } default: @@ -335,6 +339,19 @@ export function install(rt) { if (normalized === '') issuePrBySession.delete(sessionId) else issuePrBySession.set(sessionId, normalized) } + // Tier chosen at triage for this Session. Unset stays on the process default. + const harnessBySession = new Map() + const noteUsageHarness = (sessionId, ref) => { + if (typeof sessionId !== 'string' || sessionId === '') return 'default' + if (ref === undefined || ref === '' || ref === 'default') { + harnessBySession.delete(sessionId) + return 'default' + } + const id = typeof ref === 'string' ? resolveHarness(ref).id : routeHarness(ref).id + if (id === 'default') harnessBySession.delete(sessionId) + else harnessBySession.set(sessionId, id) + return harnessBySession.get(sessionId) ?? 'default' + } function openLedger() { if (ledger !== undefined) return Promise.resolve(ledger) const job = opening ??= (async () => { @@ -469,7 +486,12 @@ export function install(rt) { } events.reverse() if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`) - const fold = foldCalls(events, { ...(mark?.fold ?? {}), issuePr: issuePrBySession.get(sessionId) ?? '' }) + const notedHarness = harnessBySession.get(sessionId) + const fold = foldCalls(events, { + ...(mark?.fold ?? {}), + issuePr: issuePrBySession.get(sessionId) ?? '', + ...(notedHarness && { harness: notedHarness }), + }) ledger.commit(sessionId, { rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' }, { issuePr: row.issuePr || issuePrBySession.get(sessionId) || '' })), reset: fold.reset, @@ -699,20 +721,23 @@ export function install(rt) { yield chunk } } finally { - if (usage !== undefined) record({ - sessionId: tag?.sessionId ?? options.sessionId, - kind: tag?.kind ?? options.purpose ?? 'other', - provider: options.provider, - model: options.model, - usage, - issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(tag?.sessionId ?? options.sessionId) ?? '', - toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0), - wallMs: Math.max(0, Date.now() - started), - harness: tag?.harness ?? rt.harness?.id ?? 'default', - }) + if (usage !== undefined) { + const sessionKey = tag?.sessionId ?? options.sessionId + record({ + sessionId: sessionKey, + kind: tag?.kind ?? options.purpose ?? 'other', + provider: options.provider, + model: options.model, + usage, + issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(sessionKey) ?? '', + toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0), + wallMs: Math.max(0, Date.now() - started), + harness: tag?.harness ?? harnessBySession.get(sessionKey) ?? rt.harness?.id ?? 'default', + }) + } } })() }) - Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr }) + Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr, noteUsageHarness }) } diff --git a/core/test/harness.test.js b/core/test/harness.test.js index 798b5e75..0f097142 100644 --- a/core/test/harness.test.js +++ b/core/test/harness.test.js @@ -1,6 +1,21 @@ import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' import { test } from 'node:test' -import { DEFAULT_HARNESS_ID, defaultHarness, resolveHarness } from '../src/host/harness.js' +import { fileURLToPath } from 'node:url' +import { + DEFAULT_HARNESS_ID, + HARNESS_PRO_VERIFIERS, + HARNESS_TIER_LABELS, + defaultHarness, + harnessPro, + harnessTierOrder, + install, + resolveHarness, + routeHarness, +} from '../src/host/harness.js' + +const here = dirname(fileURLToPath(import.meta.url)) test('Step 0 default harness names prompt, tools, model, step·retry, and verifiers', () => { const harness = defaultHarness() @@ -12,12 +27,115 @@ test('Step 0 default harness names prompt, tools, model, step·retry, and verifi assert.ok(harness.verifiers.note) }) -test('resolveHarness only accepts default in Step 0', () => { +test('default harness keeps the Step 0 shape', () => { + const harness = defaultHarness() + assert.deepEqual(Object.keys(harness), ['id', 'prompt', 'tools', 'model', 'stepRetry', 'verifiers', 'exceptions']) + assert.deepEqual(Object.keys(harness.prompt), ['module', 'section']) + assert.deepEqual(Object.keys(harness.tools), ['module']) + assert.deepEqual(Object.keys(harness.model), ['module', 'note']) + assert.deepEqual(Object.keys(harness.stepRetry), ['module', 'note']) + assert.deepEqual(Object.keys(harness.verifiers), ['note']) + assert.equal(harness.id, 'default') assert.equal(resolveHarness().id, 'default') - assert.throws(() => resolveHarness('lite'), /unknown harness/) + assert.equal(resolveHarness('default').id, 'default') +}) + +test('unknown harness ids still throw', () => { + assert.throws(() => resolveHarness('magpie'), /unknown harness/) + assert.throws(() => resolveHarness('harness:lite'), /unknown harness/) }) test('tree prompt exception is listed outside the default harness file', () => { const tree = defaultHarness().exceptions.find(entry => entry.id === 'tree-prompt') assert.equal(tree.path, 'plugins/tree') + for (const id of ['lite', 'medium', 'heavy']) { + const tier = resolveHarness(id).exceptions.find(entry => entry.id === 'tree-prompt') + assert.equal(tier.path, 'plugins/tree', id) + } +}) + +test('lite, medium, and heavy load with distinct ordered budgets', () => { + const order = harnessTierOrder() + assert.deepEqual(order.failures, [], order.failures.join('; ')) + assert.equal(order.ok, true) + const lite = resolveHarness('lite') + const medium = resolveHarness('medium') + const heavy = resolveHarness('heavy') + assert.equal(lite.applied, false) + assert.equal(medium.applied, false) + assert.equal(heavy.applied, false) + assert.ok(lite.prompt.maxChars < medium.prompt.maxChars) + assert.ok(medium.prompt.maxChars < heavy.prompt.maxChars) + assert.ok(lite.tools.maxCalls < medium.tools.maxCalls) + assert.ok(medium.tools.maxCalls < heavy.tools.maxCalls) + assert.ok(lite.model.maxTokens < medium.model.maxTokens) + assert.ok(medium.model.maxTokens < heavy.model.maxTokens) + assert.ok(lite.stepRetry.maxSteps < medium.stepRetry.maxSteps) + assert.ok(medium.stepRetry.maxSteps < heavy.stepRetry.maxSteps) + assert.ok(lite.stepRetry.maxRetries < medium.stepRetry.maxRetries) + assert.ok(medium.stepRetry.maxRetries <= heavy.stepRetry.maxRetries) + assert.ok(lite.tools.allow.length < medium.tools.allow.length) + assert.equal(heavy.tools.allow, null) + for (const tool of lite.tools.allow) assert.ok(medium.tools.allow.includes(tool), tool) + for (const blocked of ['create_bot', 'delete_group', 'stop_bot']) { + assert.equal(medium.tools.allow.includes(blocked), false, blocked) + } +}) + +test('tier tool budgets name tools the host registers', () => { + const files = [ + 'tools.js', 'memory.js', 'schedules.js', 'secrets.js', 'browser.js', 'decisions.js', 'groups.js', + ] + const names = new Set() + for (const file of files) { + const text = readFileSync(join(here, '../src/host', file), 'utf8') + for (const match of text.matchAll(/defineTool\(\{[\s\S]*?name: '([a-z0-9_]+)'/g)) names.add(match[1]) + } + for (const id of ['lite', 'medium']) { + for (const tool of resolveHarness(id).tools.allow) assert.ok(names.has(tool), `${id} ${tool}`) + } +}) + +test('triage routes one harness label and leaves every other label on default', () => { + assert.deepEqual(HARNESS_TIER_LABELS, { + lite: 'harness:lite', + medium: 'harness:medium', + heavy: 'harness:heavy', + }) + assert.equal(routeHarness([]).id, 'default') + assert.equal(routeHarness(['agent-ready', 'risk:T2']).id, 'default') + assert.equal(routeHarness(['harness:lite']).id, 'lite') + assert.equal(routeHarness([{ name: 'harness:medium' }, { name: 'bug' }]).id, 'medium') + assert.equal(routeHarness({ labels: [{ name: 'Harness:Heavy' }] }).id, 'heavy') + assert.equal(routeHarness(['harness:lite', 'harness:lite']).id, 'lite') + assert.throws(() => routeHarness(['harness:lite', 'harness:heavy']), /one harness tier label/) + assert.throws(() => routeHarness(['harness:turbo']), /unknown harness label/) +}) + +test('HarnessPro checks a tier offline against CI, eval, and rsi-baseline', () => { + assert.deepEqual([...HARNESS_PRO_VERIFIERS], ['ci', 'core/dev/eval', 'rsi-baseline']) + for (const id of ['default', 'lite', 'medium', 'heavy']) { + const report = harnessPro(resolveHarness(id)) + assert.equal(report.ok, true, `${id} ${report.failures.join('; ')}`) + } + const drifted = resolveHarness('lite') + drifted.prompt = { ...drifted.prompt, module: './other-prompt.js' } + assert.equal(harnessPro(drifted).ok, false) + const unverified = resolveHarness('medium') + unverified.verifiers = { offline: ['ci'] } + const missing = harnessPro(unverified) + assert.equal(missing.ok, false) + assert.ok(missing.failures.includes('verifiers.offline')) + const leaked = resolveHarness('heavy') + leaked.model = { ...leaked.model, apiKey: 'sk-live-secret' } + assert.equal(harnessPro(leaked).ok, false) +}) + +test('install keeps the process harness on default and exposes routing', () => { + const rt = {} + install(rt) + assert.equal(rt.harness.id, 'default') + assert.equal(rt.resolveHarness('heavy').id, 'heavy') + assert.equal(rt.routeHarness(['harness:lite']).id, 'lite') + assert.equal(rt.harness.id, 'default') }) diff --git a/core/test/usage-fields.test.js b/core/test/usage-fields.test.js index 79510fae..9f2f0a4b 100644 --- a/core/test/usage-fields.test.js +++ b/core/test/usage-fields.test.js @@ -3,7 +3,7 @@ import { spawnSync } from 'node:child_process' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { test } from 'node:test' -import { extractIssuePr, foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' +import { extractIssuePr, foldCalls, install as installUsage, missingUsageFields, withUsageFields } from '../src/host/usage.js' const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } }) const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } }) @@ -46,6 +46,40 @@ test('foldCalls counts tool/call events and wall clock from the attempt', () => assert.equal(rows[0].issuePr, '#903') }) +test('foldCalls records a routed harness and leaves the resume state unchanged', () => { + const { rows, state } = foldCalls([ + { type: 'turn/start', time: 1000, data: { turn: 1 } }, + header('deepseek', 'v4-flash'), + { type: 'user', time: 1001, data: { text: 'see PR #908 for #874' } }, + { type: 'tool/call', time: 1100, data: { name: 'read_file' } }, + message(1, 0, usage(10, 2), 1600), + ], { harness: 'lite' }) + assert.equal(rows.length, 1) + assert.deepEqual(missingUsageFields(rows[0]), []) + assert.equal(rows[0].harness, 'lite') + assert.equal(rows[0].issuePr, 'PR #908') + assert.equal(rows[0].toolCalls, 1) + assert.equal(rows[0].wallMs, 600) + assert.deepEqual(state, { provider: 'deepseek', model: 'v4-flash', retry: null }) + assert.throws(() => foldCalls([message(1, 0, usage(1, 1), 1)], { harness: 'turbo' }), /unknown harness/) +}) + +test('noteUsageHarness keeps a triage label and clears back to default', () => { + const rt = { + home: '/tmp/unused-usage-home', + state: { bots: {}, rooms: {}, agents: {} }, + ctx: { effect() {}, on() {} }, + warn() {}, + } + installUsage(rt) + assert.equal(rt.noteUsageHarness('s1', ['harness:medium']), 'medium') + assert.equal(rt.noteUsageHarness('s1', { labels: [{ name: 'harness:heavy' }] }), 'heavy') + assert.equal(rt.noteUsageHarness('s1', 'default'), 'default') + assert.equal(rt.noteUsageHarness('s1', ['agent-ready']), 'default') + assert.throws(() => rt.noteUsageHarness('s1', 'turbo'), /unknown harness/) + assert.throws(() => rt.noteUsageHarness('s1', ['harness:lite', 'harness:heavy']), /one harness tier label/) +}) + test('extractIssuePr prefers PR refs over bare issue numbers', () => { assert.equal(extractIssuePr('see PR #908 for #903'), 'PR #908') assert.equal(extractIssuePr('https://github.com/ThinkFlowLab/FlyBot/issues/903'), '#903') diff --git a/docs/subsystems/core-host.md b/docs/subsystems/core-host.md index 7dbe5b4a..8d565aa3 100644 --- a/docs/subsystems/core-host.md +++ b/docs/subsystems/core-host.md @@ -45,8 +45,8 @@ The server side of the patched `ds-bot` core (`core/`): the part of [ds-bot-core | `core/src/host/schedules.js` | Scheduled tasks through DSH's schedule service: list, create, edit, pause/resume, retry and delete for Settings → Scheduled tasks, each task's last run, and moving a Bot's tasks to its fresh part (#354, #473, see **Scheduled tasks: edit, pause, missed runs** and **Scheduled tasks: last run and Retry**). Also the `bot_schedules` tool: the Bot-wide list DSH's own `schedule_list` cannot give, group chats' own Sessions included (#628, see **Scheduled tasks: the Bot-wide list**). Tests: `core/test/schedules.test.js`. | | `core/src/host/automation-default.js` | Automation tasks on by default (#378): once per profile, `offerBundle` asks DSH's plugin manager to switch on `@deepseek-ai/dsh-experimental-schedule-bundle` and writes the offer to `dsh-defaults.json` next to the team. Tests: `core/test/automation-default.test.js`. | | `core/src/host/notify.js` | Out-of-app delivery, first half (#355): a transport-agnostic adapter with Slack's `chat.postMessage` shape, one key per reminder so a retried send cannot duplicate, exponential backoff that honours `Retry-After`, three attempts at most, a structural failure not retried, and zero transport calls while the channel is off. Ships `fakeSlack` for tests; nothing calls it until the real Slack transport lands (see **Out-of-app delivery, first half**). Tests: `core/test/notify.test.js`. | -| `core/src/host/harness.js` | Step 0 declarative `default` harness (#903 / #874 Option B): names today's prompt / tools / model / step·retry / verifiers modules as one config (`rt.harness`, `rt.resolveHarness`); behavior-neutral. Tree prompt exception: `@flybot/tree` keeps its marked prompt under `plugins/tree` (outside the three mirrors); listed on the harness, injection stays outside this file. Installed in `MODULES` right after `store`. Tests: `core/test/harness.test.js`. | -| `core/src/host/usage.js` | Usage ledger at `$DSH_HOME/bot/usage.sqlite` (`calls`). Ledger v3 (#903): every row carries `harness`, `issue_pr`, `tool_calls`, `wall_ms`, plus existing token columns (`withUsageFields` / `missingUsageFields`). Session turns are metered via `foldCalls` over turn events (`tool/call` count, wall time from attempt start, `issuePr` from turn text or `rt.noteUsageIssuePr(sessionId, ref)`), not from llm/stream alone. Sample row fixture: `core/test/fixtures/usage-sample.sqlite`. Tests: `core/test/usage-fields.test.js`, `core/test/usage.test.js`. | +| `core/src/host/harness.js` | Declarative harness (#874 Option B). Step 0 (#903): `default` names today's prompt / tools / model / step·retry / verifiers (`rt.harness`, `rt.resolveHarness`); behavior-neutral. Step 1: `lite` / `medium` / `heavy` load through the same resolver with distinct declared budgets; `rt.routeHarness` reads one triage label (`harness:lite`, `harness:medium`, `harness:heavy`). `applied` stays false, so the named modules are not rewritten. `harnessPro` checks a config offline against CI, `core/dev/eval`, and `rsi-baseline` and does not call a model. Tree prompt exception: `@flybot/tree` keeps its marked prompt under `plugins/tree` (outside the three mirrors); listed on the harness, injection stays outside this file. Installed in `MODULES` right after `store`. Tests: `core/test/harness.test.js`. | +| `core/src/host/usage.js` | Usage ledger at `$DSH_HOME/bot/usage.sqlite` (`calls`). Ledger v3 (#903): every row carries `harness`, `issue_pr`, `tool_calls`, `wall_ms`, plus existing token columns (`withUsageFields` / `missingUsageFields`). Session turns are metered via `foldCalls` over turn events (`tool/call` count, wall time from attempt start, `issuePr` from turn text or `rt.noteUsageIssuePr(sessionId, ref)`), not from llm/stream alone. A triage tier is stamped with `rt.noteUsageHarness(sessionId, idOrLabels)` and lands on the row's `harness` field; the resume cursor (`marks.fold`) stays provider / model / retry. Sample row fixture: `core/test/fixtures/usage-sample.sqlite`. Tests: `core/test/usage-fields.test.js`, `core/test/usage.test.js`. | | `core/src/host/tool-schema.js` | The tool definitions the model sees: its `defineTool` wraps DSH's and completes the compiled schema with `explicitObjectShape`, which fills `properties: {}` and `required: []` on every object node (recursing through `properties`, `items`, `oneOf`/`anyOf`/`allOf`, `$defs`, …). A strict OpenAI-compatible gateway reads an absent member as null and rejects the whole request (`null is not of type "array"`), so one argument-less tool would break every turn; argument validation keeps using DSH's schema unchanged. Every host module registers tools through this wrapper. Tests: `core/test/tool-schema.test.js`. | | `core/src/shared/remote-label.js` | Owner-badge labels for a friend's Bot and “Shared room from …” titles. `friendBotTitle(bot)` / `friendBotOwner(bot)` name a friend's chat from the roster (`Chief (SelfB1233)`) and its owner (`SelfB1233`, the person an approval card names) with one formatter for every client surface (#498). `remoteBotName` / `isAutoOwnerName` / `seatOwnerName` keep group and face-room labels in that format too (#526): Bot Social's ` · ` member label is not a Bot name, and an owner name made up from an install id (`FlyBot 682b8a43`) gives way to the username in the home handle (`Reviewer (flybot682b8a43)`). `isListedRemoteChat` / `listedChatBots` keep gone, hidden, and home-level (`@something.home`) remotes out of the sidebar, the **发给** picker, mentions, and `list_bots` friends lines; history stays on disk. | @@ -58,11 +58,20 @@ Add a new invariant under the heading it belongs to (or a new heading), not at t Step 0 of #874 Option B is behavior-neutral: one declarative harness and five usage fields that every `calls` row must carry. -- `src/host/harness.js` installs as `rt.harness` / `rt.resolveHarness` after `store` in `MODULES` (`core/index.js`). Today's setup is the single `default` harness: it names the existing prompt / tools / model / step·retry modules and notes that verifiers live in CI / merge-gate / `core/dev/eval`, not as an in-process Step 0 list. Unknown harness ids throw. Lite/Medium/Heavy and Magpie stay out of scope; nothing under `core/dev/harness/` is edited here. -- Tree exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` (not one of `ds-bot/` · `patches/ds-bot/` · `core/`). The default harness lists that exception; injection stays outside the harness file so eval `P3-tree-split` still passes. -- Usage ledger `calls` is version 3 (`LEDGER_VERSION`): columns include `harness`, `issue_pr`, `tool_calls`, `wall_ms` beside the existing token fields. `withUsageFields` fills defaults (`harness` → `default`, the rest empty/0); `missingUsageFields` fails a row that lacks any of the five. `foldCalls` counts `tool/call` events, computes `wallMs` from the attempt start, and takes `issuePr` from turn text (`extractIssuePr`) or from `rt.noteUsageIssuePr` when a caller tags the session. Rows without those turn signals stay `''` / `0` by design, not as a silent shell that never wires turn data. +- `src/host/harness.js` installs as `rt.harness` / `rt.resolveHarness` / `rt.routeHarness` after `store` in `MODULES` (`core/index.js`). `rt.harness` is always the `default` harness: it names the existing prompt / tools / model / step·retry modules and notes that verifiers live in CI / merge-gate / `core/dev/eval`, not as an in-process Step 0 list. Unknown harness ids throw. Magpie stays out of scope; nothing under `core/dev/harness/` is edited here. +- Tree exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` (not one of `ds-bot/` · `patches/ds-bot/` · `core/`). Every harness lists that exception; injection stays outside the harness file so eval `P3-tree-split` still passes. +- Usage ledger `calls` is version 3 (`LEDGER_VERSION`): columns include `harness`, `issue_pr`, `tool_calls`, `wall_ms` beside the existing token fields. `withUsageFields` fills defaults (`harness` → `default`, the rest empty/0); `missingUsageFields` fails a row that lacks any of the five. `foldCalls` counts `tool/call` events, computes `wallMs` from the attempt start, and takes `issuePr` from turn text (`extractIssuePr`) or from `rt.noteUsageIssuePr` when a caller tags the session. Rows without those turn signals stay `''` / `0` by design, not as a silent shell that never wires turn data. `foldCalls` also copies `from.harness` onto the row when a caller passes a known id, and leaves `state` as `{ provider, model, retry }`. - Fixture for acceptance: `core/test/fixtures/usage-sample.sqlite` holds one real `calls` row. Covered by `core/test/harness.test.js` and `core/test/usage-fields.test.js`. +### Harness tiers (#874 Step 1) + +Lite, Medium, and Heavy are three configs on the Step 0 harness, not a second runtime. + +- `resolveHarness('lite' | 'medium' | 'heavy')` returns the same module pointers as `default` plus a declared budget: `prompt.maxChars`, `tools.maxCalls` and `tools.allow` (`null` on Heavy means the full set the host registers), `model.maxTokens`, `stepRetry.maxSteps` and `stepRetry.maxRetries`. The numbers rise from Lite to Heavy. `applied` is false: prompt, tools, model, and turn code keep running as they do for `default`. +- Triage selects a tier with exactly one label: `harness:lite`, `harness:medium`, `harness:heavy` (catalogued in `.github/labels.json`; a maintainer creates them with `sync-labels.mjs --apply`). No such label, or any other label including `risk:T*`, routes to `default`. Two tier labels, or an unknown `harness:*` label, throw. The controller does not write these labels. +- `harnessPro(config)` is the offline check an RSI edit of a tier has to pass: the config still names `./prompt.js`, `./tools.js`, `./models.js`, and `./turns.js`, its `verifiers.offline` list is `ci`, `core/dev/eval`, `rsi-baseline`, and it carries no key or token field. The function does not run those three checks and does not call a model. Running them is the pull request's own CI, `node core/dev/eval/run.mjs --list`, and `node --test .github/scripts/rsi-baseline.test.mjs`. +- `rt.noteUsageHarness(sessionId, idOrLabels)` stores the chosen tier for that Session. Later `foldCalls` rows and live meter rows record it in `harness`. Clearing it records `default` again. Issue/PR, tool-call count, and wall time stay the Step 0 fields. + ### Room execution boundary `groups.js` owns room scheduling, membership checks, private-brief selection, diff --git a/patches/ds-bot/0001-core-fixes.patch b/patches/ds-bot/0001-core-fixes.patch index c9c77c4c..8f97fd2a 100644 --- a/patches/ds-bot/0001-core-fixes.patch +++ b/patches/ds-bot/0001-core-fixes.patch @@ -4488,16 +4488,18 @@ diff --git a/src/host/turns.js b/src/host/turns.js diff --git a/src/host/usage.js b/src/host/usage.js --- a/src/host/usage.js +++ b/src/host/usage.js -@@ -14,7 +14,7 @@ import { randomUUID } from 'node:crypto' +@@ -13,8 +13,9 @@ + import { randomUUID } from 'node:crypto' import { mkdir, unlink } from 'node:fs/promises' import { join } from 'node:path' ++import { resolveHarness, routeHarness } from './harness.js' -const LEDGER_VERSION = 2 +const LEDGER_VERSION = 3 const PAGE_MESSAGES = 400 const PAGE_TIMEOUT_MS = 20_000 const SCAN_TIMEOUT_MS = 180_000 -@@ -45,30 +45,104 @@ const tokensOf = usage => ({ +@@ -45,30 +45,107 @@ const tokensOf = usage => ({ output: count(usage.outputTokens), }) @@ -4564,6 +4566,9 @@ diff --git a/src/host/usage.js b/src/host/usage.js // the cut drops the rows before it (`reset`). export function foldCalls(events, from = {}) { - const state = { provider: from.provider ?? '', model: from.model ?? '', retry: from.retry ?? null } ++ // A routed tier is a row field. It stays out of `state`: marks.fold is the ++ // resume cursor (provider, model, retry) and a pre-Step-0 mark has no harness. ++ const harness = typeof from.harness === 'string' && from.harness !== '' ? resolveHarness(from.harness).id : undefined + const state = { + provider: from.provider ?? '', + model: from.model ?? '', @@ -4619,7 +4624,7 @@ diff --git a/src/host/usage.js b/src/host/usage.js + toolCalls: state.turnTools, + wallMs, + issuePr: state.issuePr, -+ })) ++ }, { harness })) break } default: @@ -4665,7 +4670,7 @@ diff --git a/src/host/usage.js b/src/host/usage.js return { mark(session) { const row = getMark.get(session) -@@ -233,6 +233,15 @@ export function install(rt) { +@@ -233,6 +233,28 @@ export function install(rt) { let opening // Metered calls that finish before the ledger opens. const early = [] @@ -4677,6 +4682,19 @@ diff --git a/src/host/usage.js b/src/host/usage.js + const normalized = extractIssuePr(typeof ref === 'string' ? ref : '') || (typeof ref === 'string' ? ref.trim() : '') + if (normalized === '') issuePrBySession.delete(sessionId) + else issuePrBySession.set(sessionId, normalized) ++ } ++ // Tier chosen at triage for this Session. Unset stays on the process default. ++ const harnessBySession = new Map() ++ const noteUsageHarness = (sessionId, ref) => { ++ if (typeof sessionId !== 'string' || sessionId === '') return 'default' ++ if (ref === undefined || ref === '' || ref === 'default') { ++ harnessBySession.delete(sessionId) ++ return 'default' ++ } ++ const id = typeof ref === 'string' ? resolveHarness(ref).id : routeHarness(ref).id ++ if (id === 'default') harnessBySession.delete(sessionId) ++ else harnessBySession.set(sessionId, id) ++ return harnessBySession.get(sessionId) ?? 'default' + } function openLedger() { if (ledger !== undefined) return Promise.resolve(ledger) @@ -4690,12 +4708,17 @@ diff --git a/src/host/usage.js b/src/host/usage.js return sqliteLedger(DatabaseSync, path) } catch (error) { if (fresh || !damaged(error)) { -@@ -367,9 +367,9 @@ export function install(rt) { +@@ -367,9 +367,14 @@ export function install(rt) { } events.reverse() if (events.length > 0 && events[0].seq !== from + 1) throw new Error(`the history of ${sessionId} came back with a gap`) - const fold = foldCalls(events, mark?.fold) -+ const fold = foldCalls(events, { ...(mark?.fold ?? {}), issuePr: issuePrBySession.get(sessionId) ?? '' }) ++ const notedHarness = harnessBySession.get(sessionId) ++ const fold = foldCalls(events, { ++ ...(mark?.fold ?? {}), ++ issuePr: issuePrBySession.get(sessionId) ?? '', ++ ...(notedHarness && { harness: notedHarness }), ++ }) ledger.commit(sessionId, { - rows: fold.rows.map(row => ({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' })), + rows: fold.rows.map(row => withUsageFields({ ...row, key: `${sessionId}#${row.key}`, session: sessionId, kind: 'turn' }, { issuePr: row.issuePr || issuePrBySession.get(sessionId) || '' })), @@ -4730,28 +4753,31 @@ diff --git a/src/host/usage.js b/src/host/usage.js try { for await (const chunk of next()) { if (chunk?.type === 'usage' && chunk.usage && typeof chunk === 'object' && !counted.has(chunk)) { -@@ -588,10 +588,20 @@ export function install(rt) { +@@ -588,10 +588,23 @@ export function install(rt) { yield chunk } } finally { - if (usage !== undefined) record({ sessionId: tag?.sessionId ?? options.sessionId, kind: tag?.kind ?? options.purpose ?? 'other', provider: options.provider, model: options.model, usage }) -+ if (usage !== undefined) record({ -+ sessionId: tag?.sessionId ?? options.sessionId, -+ kind: tag?.kind ?? options.purpose ?? 'other', -+ provider: options.provider, -+ model: options.model, -+ usage, -+ issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(tag?.sessionId ?? options.sessionId) ?? '', -+ toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0), -+ wallMs: Math.max(0, Date.now() - started), -+ harness: tag?.harness ?? rt.harness?.id ?? 'default', -+ }) ++ if (usage !== undefined) { ++ const sessionKey = tag?.sessionId ?? options.sessionId ++ record({ ++ sessionId: sessionKey, ++ kind: tag?.kind ?? options.purpose ?? 'other', ++ provider: options.provider, ++ model: options.model, ++ usage, ++ issuePr: tag?.issuePr ?? options.issuePr ?? issuePrBySession.get(sessionKey) ?? '', ++ toolCalls: Number.isFinite(tag?.toolCalls) ? tag.toolCalls : (Number.isFinite(options.toolCalls) ? options.toolCalls : 0), ++ wallMs: Math.max(0, Date.now() - started), ++ harness: tag?.harness ?? harnessBySession.get(sessionKey) ?? rt.harness?.id ?? 'default', ++ }) ++ } } })() }) - Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest }) -+ Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr }) ++ Object.assign(rt, { usageView, tagUsage, carryUsageTag, ownRequest, noteUsageIssuePr, noteUsageHarness }) } diff --git a/src/shared/mentions.js b/src/shared/mentions.js --- a/src/shared/mentions.js diff --git a/patches/ds-bot/files/src/host/harness.js b/patches/ds-bot/files/src/host/harness.js index 80e25191..b652a804 100644 --- a/patches/ds-bot/files/src/host/harness.js +++ b/patches/ds-bot/files/src/host/harness.js @@ -1,13 +1,65 @@ -// Step 0 (#903 / #874 Option B): one declarative harness layer. -// Today's setup is the single `default` harness. Behavior-neutral: the runtime still -// runs the same modules; this file names them as one config so Step 1 can add tiers later. +// Declarative harness layer (#874 Option B). // -// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under `plugins/tree` -// (not one of the three mirrors ds-bot/ · patches/ds-bot/ · core/). That injection stays -// outside this harness file; eval `P3-tree-split` must keep passing. +// Step 0 (#903): today's setup is the `default` harness. It names the modules the +// runtime already runs. It does not replace them. +// +// Step 1: Lite, Medium, and Heavy are three more configs on that same layer. +// Each one points at the same modules and adds a budget (prompt size, tool cap +// and allow-list, model output cap, step/retry cap). `applied` is false: loading +// a tier does not change prompt.js, tools.js, models.js, or turns.js. The +// default path stays the process harness. Triage picks a tier with one label +// (`harness:lite`, `harness:medium`, `harness:heavy`); `risk:T*` is not a harness. +// +// HarnessPro is the offline check for that config. It names CI, core/dev/eval, +// and rsi-baseline. It does not run them and it does not call a model. +// +// Tree prompt exception: `@flybot/tree` keeps its own marked prompt under +// `plugins/tree` (not one of the three mirrors). That injection stays outside +// this file; eval `P3-tree-split` must keep passing. export const DEFAULT_HARNESS_ID = 'default' +// Triage sets exactly one of these. The controller does not write them, and +// they are not risk tiers. +export const HARNESS_TIER_LABELS = Object.freeze({ + lite: 'harness:lite', + medium: 'harness:medium', + heavy: 'harness:heavy', +}) + +export const HARNESS_PRO_VERIFIERS = Object.freeze(['ci', 'core/dev/eval', 'rsi-baseline']) + +const TIER_BUDGETS = { + lite: { + prompt: { maxChars: 8_000 }, + tools: { + maxCalls: 8, + allow: ['list_bots', 'list_models', 'recall', 'read_own_chat', 'read_group_chat', 'list_secrets'], + }, + model: { maxTokens: 2_048 }, + stepRetry: { maxSteps: 4, maxRetries: 0 }, + }, + medium: { + prompt: { maxChars: 16_000 }, + tools: { + maxCalls: 24, + allow: [ + 'list_bots', 'list_models', 'recall', 'read_own_chat', 'read_group_chat', 'list_secrets', + 'message_bot', 'ask_user', 'remember', 'forget', 'list_capabilities', 'bot_schedules', + 'read_browser', 'request_secret', 'post_to_group', 'update_bot', 'record_decision', + ], + }, + model: { maxTokens: 8_192 }, + stepRetry: { maxSteps: 12, maxRetries: 1 }, + }, + heavy: { + prompt: { maxChars: 48_000 }, + tools: { maxCalls: 64, allow: null }, + model: { maxTokens: 16_384 }, + stepRetry: { maxSteps: 32, maxRetries: 1 }, + }, +} + export function defaultHarness() { return { id: DEFAULT_HARNESS_ID, @@ -26,12 +78,148 @@ export function defaultHarness() { } } +function tierHarness(id) { + const budget = TIER_BUDGETS[id] + const base = defaultHarness() + return { + ...base, + id, + prompt: { ...base.prompt, maxChars: budget.prompt.maxChars }, + tools: { + ...base.tools, + maxCalls: budget.tools.maxCalls, + allow: budget.tools.allow === null ? null : [...budget.tools.allow], + }, + model: { ...base.model, maxTokens: budget.model.maxTokens }, + stepRetry: { + ...base.stepRetry, + maxSteps: budget.stepRetry.maxSteps, + maxRetries: budget.stepRetry.maxRetries, + }, + verifiers: { + offline: [...HARNESS_PRO_VERIFIERS], + note: 'HarnessPro checks this config offline against CI, core/dev/eval, and rsi-baseline. No model call.', + }, + // Declared budget only. The named modules still run as they do for default. + applied: false, + } +} + +const TIERS = { + lite: () => tierHarness('lite'), + medium: () => tierHarness('medium'), + heavy: () => tierHarness('heavy'), +} + export function resolveHarness(id = DEFAULT_HARNESS_ID) { - if (id !== DEFAULT_HARNESS_ID) throw new Error(`unknown harness: ${id}`) - return defaultHarness() + if (id === DEFAULT_HARNESS_ID) return defaultHarness() + const build = TIERS[id] + if (!build) throw new Error(`unknown harness: ${id}`) + return build() +} + +function labelNames(input) { + const list = Array.isArray(input) ? input : (input?.labels ?? []) + const names = [] + for (const entry of list) { + const name = typeof entry === 'string' ? entry : entry?.name + if (typeof name === 'string' && name.trim() !== '') names.push(name.trim().toLowerCase()) + } + return names +} + +export function routeHarness(input) { + const byLabel = new Map(Object.entries(HARNESS_TIER_LABELS).map(([id, label]) => [label, id])) + const hits = [] + for (const name of labelNames(input)) { + if (!name.startsWith('harness:')) continue + const id = byLabel.get(name) + if (id === undefined) throw new Error(`unknown harness label: ${name}`) + if (!hits.includes(id)) hits.push(id) + } + if (hits.length > 1) throw new Error(`one harness tier label, got ${hits.map(id => HARNESS_TIER_LABELS[id]).join(', ')}`) + return resolveHarness(hits[0] ?? DEFAULT_HARNESS_ID) +} + +function secretAt(value, path) { + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + const found = secretAt(value[index], `${path}[${index}]`) + if (found) return found + } + return '' + } + if (value !== null && typeof value === 'object') { + for (const [key, child] of Object.entries(value)) { + const here = path === '' ? key : `${path}.${key}` + if (/api[_-]?key/i.test(key) || /^(token|secret|password|authorization)$/i.test(key)) return here + const found = secretAt(child, here) + if (found) return found + } + return '' + } + if (typeof value === 'string' && (/\bsk-[A-Za-z0-9]{8,}\b/.test(value) || value.includes('DEEPSEEK_API_KEY') || value.includes('GITHUB_TOKEN'))) { + return path === '' ? 'value' : path + } + return '' +} + +// Offline shape check. A tier must keep the default modules and name the three +// HarnessPro verifiers. This function does not run CI, eval, or rsi-baseline. +export function harnessPro(harness) { + const failures = [] + const base = defaultHarness() + if (!harness || typeof harness.id !== 'string') failures.push('id') + else if (harness.id !== DEFAULT_HARNESS_ID && TIERS[harness.id] === undefined) failures.push('id') + for (const key of ['prompt', 'tools', 'model', 'stepRetry']) { + if (harness?.[key]?.module !== base[key].module) failures.push(`${key}.module`) + } + if (harness?.id && harness.id !== DEFAULT_HARNESS_ID) { + const offline = harness.verifiers?.offline + const same = Array.isArray(offline) + && offline.length === HARNESS_PRO_VERIFIERS.length + && HARNESS_PRO_VERIFIERS.every((name, index) => offline[index] === name) + if (!same) failures.push('verifiers.offline') + if (!(harness.prompt?.maxChars > 0)) failures.push('prompt.maxChars') + if (!(harness.tools?.maxCalls > 0)) failures.push('tools.maxCalls') + if (harness.tools?.allow !== null && !Array.isArray(harness.tools?.allow)) failures.push('tools.allow') + if (!(harness.model?.maxTokens > 0)) failures.push('model.maxTokens') + if (!(harness.stepRetry?.maxSteps > 0)) failures.push('stepRetry.maxSteps') + if (!Number.isInteger(harness.stepRetry?.maxRetries) || harness.stepRetry.maxRetries < 0) failures.push('stepRetry.maxRetries') + } + const secret = secretAt(harness, '') + if (secret) failures.push(`secret at ${secret}`) + return { ok: failures.length === 0, failures, verifiers: [...HARNESS_PRO_VERIFIERS] } +} + +export function harnessTierOrder() { + const lite = resolveHarness('lite') + const medium = resolveHarness('medium') + const heavy = resolveHarness('heavy') + const axes = { + 'prompt.maxChars': [lite.prompt.maxChars, medium.prompt.maxChars, heavy.prompt.maxChars], + 'tools.maxCalls': [lite.tools.maxCalls, medium.tools.maxCalls, heavy.tools.maxCalls], + 'model.maxTokens': [lite.model.maxTokens, medium.model.maxTokens, heavy.model.maxTokens], + 'stepRetry.maxSteps': [lite.stepRetry.maxSteps, medium.stepRetry.maxSteps, heavy.stepRetry.maxSteps], + 'stepRetry.maxRetries': [lite.stepRetry.maxRetries, medium.stepRetry.maxRetries, heavy.stepRetry.maxRetries], + } + const failures = [] + for (const [name, [low, mid, high]] of Object.entries(axes)) { + if (!(low <= mid && mid <= high) || low === high) failures.push(name) + } + if (!Array.isArray(lite.tools.allow) || !Array.isArray(medium.tools.allow)) failures.push('allow') + else { + for (const tool of lite.tools.allow) { + if (!medium.tools.allow.includes(tool)) failures.push(`allow ${tool}`) + } + if (lite.tools.allow.length >= medium.tools.allow.length) failures.push('allow size') + } + if (heavy.tools.allow !== null) failures.push('heavy.allow') + return { ok: failures.length === 0, failures, axes } } export function install(rt) { rt.harness = resolveHarness(DEFAULT_HARNESS_ID) rt.resolveHarness = resolveHarness + rt.routeHarness = routeHarness } diff --git a/patches/ds-bot/files/test/harness.test.js b/patches/ds-bot/files/test/harness.test.js index 798b5e75..0f097142 100644 --- a/patches/ds-bot/files/test/harness.test.js +++ b/patches/ds-bot/files/test/harness.test.js @@ -1,6 +1,21 @@ import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' import { test } from 'node:test' -import { DEFAULT_HARNESS_ID, defaultHarness, resolveHarness } from '../src/host/harness.js' +import { fileURLToPath } from 'node:url' +import { + DEFAULT_HARNESS_ID, + HARNESS_PRO_VERIFIERS, + HARNESS_TIER_LABELS, + defaultHarness, + harnessPro, + harnessTierOrder, + install, + resolveHarness, + routeHarness, +} from '../src/host/harness.js' + +const here = dirname(fileURLToPath(import.meta.url)) test('Step 0 default harness names prompt, tools, model, step·retry, and verifiers', () => { const harness = defaultHarness() @@ -12,12 +27,115 @@ test('Step 0 default harness names prompt, tools, model, step·retry, and verifi assert.ok(harness.verifiers.note) }) -test('resolveHarness only accepts default in Step 0', () => { +test('default harness keeps the Step 0 shape', () => { + const harness = defaultHarness() + assert.deepEqual(Object.keys(harness), ['id', 'prompt', 'tools', 'model', 'stepRetry', 'verifiers', 'exceptions']) + assert.deepEqual(Object.keys(harness.prompt), ['module', 'section']) + assert.deepEqual(Object.keys(harness.tools), ['module']) + assert.deepEqual(Object.keys(harness.model), ['module', 'note']) + assert.deepEqual(Object.keys(harness.stepRetry), ['module', 'note']) + assert.deepEqual(Object.keys(harness.verifiers), ['note']) + assert.equal(harness.id, 'default') assert.equal(resolveHarness().id, 'default') - assert.throws(() => resolveHarness('lite'), /unknown harness/) + assert.equal(resolveHarness('default').id, 'default') +}) + +test('unknown harness ids still throw', () => { + assert.throws(() => resolveHarness('magpie'), /unknown harness/) + assert.throws(() => resolveHarness('harness:lite'), /unknown harness/) }) test('tree prompt exception is listed outside the default harness file', () => { const tree = defaultHarness().exceptions.find(entry => entry.id === 'tree-prompt') assert.equal(tree.path, 'plugins/tree') + for (const id of ['lite', 'medium', 'heavy']) { + const tier = resolveHarness(id).exceptions.find(entry => entry.id === 'tree-prompt') + assert.equal(tier.path, 'plugins/tree', id) + } +}) + +test('lite, medium, and heavy load with distinct ordered budgets', () => { + const order = harnessTierOrder() + assert.deepEqual(order.failures, [], order.failures.join('; ')) + assert.equal(order.ok, true) + const lite = resolveHarness('lite') + const medium = resolveHarness('medium') + const heavy = resolveHarness('heavy') + assert.equal(lite.applied, false) + assert.equal(medium.applied, false) + assert.equal(heavy.applied, false) + assert.ok(lite.prompt.maxChars < medium.prompt.maxChars) + assert.ok(medium.prompt.maxChars < heavy.prompt.maxChars) + assert.ok(lite.tools.maxCalls < medium.tools.maxCalls) + assert.ok(medium.tools.maxCalls < heavy.tools.maxCalls) + assert.ok(lite.model.maxTokens < medium.model.maxTokens) + assert.ok(medium.model.maxTokens < heavy.model.maxTokens) + assert.ok(lite.stepRetry.maxSteps < medium.stepRetry.maxSteps) + assert.ok(medium.stepRetry.maxSteps < heavy.stepRetry.maxSteps) + assert.ok(lite.stepRetry.maxRetries < medium.stepRetry.maxRetries) + assert.ok(medium.stepRetry.maxRetries <= heavy.stepRetry.maxRetries) + assert.ok(lite.tools.allow.length < medium.tools.allow.length) + assert.equal(heavy.tools.allow, null) + for (const tool of lite.tools.allow) assert.ok(medium.tools.allow.includes(tool), tool) + for (const blocked of ['create_bot', 'delete_group', 'stop_bot']) { + assert.equal(medium.tools.allow.includes(blocked), false, blocked) + } +}) + +test('tier tool budgets name tools the host registers', () => { + const files = [ + 'tools.js', 'memory.js', 'schedules.js', 'secrets.js', 'browser.js', 'decisions.js', 'groups.js', + ] + const names = new Set() + for (const file of files) { + const text = readFileSync(join(here, '../src/host', file), 'utf8') + for (const match of text.matchAll(/defineTool\(\{[\s\S]*?name: '([a-z0-9_]+)'/g)) names.add(match[1]) + } + for (const id of ['lite', 'medium']) { + for (const tool of resolveHarness(id).tools.allow) assert.ok(names.has(tool), `${id} ${tool}`) + } +}) + +test('triage routes one harness label and leaves every other label on default', () => { + assert.deepEqual(HARNESS_TIER_LABELS, { + lite: 'harness:lite', + medium: 'harness:medium', + heavy: 'harness:heavy', + }) + assert.equal(routeHarness([]).id, 'default') + assert.equal(routeHarness(['agent-ready', 'risk:T2']).id, 'default') + assert.equal(routeHarness(['harness:lite']).id, 'lite') + assert.equal(routeHarness([{ name: 'harness:medium' }, { name: 'bug' }]).id, 'medium') + assert.equal(routeHarness({ labels: [{ name: 'Harness:Heavy' }] }).id, 'heavy') + assert.equal(routeHarness(['harness:lite', 'harness:lite']).id, 'lite') + assert.throws(() => routeHarness(['harness:lite', 'harness:heavy']), /one harness tier label/) + assert.throws(() => routeHarness(['harness:turbo']), /unknown harness label/) +}) + +test('HarnessPro checks a tier offline against CI, eval, and rsi-baseline', () => { + assert.deepEqual([...HARNESS_PRO_VERIFIERS], ['ci', 'core/dev/eval', 'rsi-baseline']) + for (const id of ['default', 'lite', 'medium', 'heavy']) { + const report = harnessPro(resolveHarness(id)) + assert.equal(report.ok, true, `${id} ${report.failures.join('; ')}`) + } + const drifted = resolveHarness('lite') + drifted.prompt = { ...drifted.prompt, module: './other-prompt.js' } + assert.equal(harnessPro(drifted).ok, false) + const unverified = resolveHarness('medium') + unverified.verifiers = { offline: ['ci'] } + const missing = harnessPro(unverified) + assert.equal(missing.ok, false) + assert.ok(missing.failures.includes('verifiers.offline')) + const leaked = resolveHarness('heavy') + leaked.model = { ...leaked.model, apiKey: 'sk-live-secret' } + assert.equal(harnessPro(leaked).ok, false) +}) + +test('install keeps the process harness on default and exposes routing', () => { + const rt = {} + install(rt) + assert.equal(rt.harness.id, 'default') + assert.equal(rt.resolveHarness('heavy').id, 'heavy') + assert.equal(rt.routeHarness(['harness:lite']).id, 'lite') + assert.equal(rt.harness.id, 'default') }) diff --git a/patches/ds-bot/files/test/usage-fields.test.js b/patches/ds-bot/files/test/usage-fields.test.js index 79510fae..9f2f0a4b 100644 --- a/patches/ds-bot/files/test/usage-fields.test.js +++ b/patches/ds-bot/files/test/usage-fields.test.js @@ -3,7 +3,7 @@ import { spawnSync } from 'node:child_process' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { test } from 'node:test' -import { extractIssuePr, foldCalls, missingUsageFields, withUsageFields } from '../src/host/usage.js' +import { extractIssuePr, foldCalls, install as installUsage, missingUsageFields, withUsageFields } from '../src/host/usage.js' const header = (provider, model) => ({ type: 'request/header', data: { header: { config: { provider, model } }, reason: 'change' } }) const message = (turn, step, usage, time = 0) => ({ type: 'assistant/message', time, data: { turn, step, message: { role: 'assistant', content: [] }, stream: [], usage } }) @@ -46,6 +46,40 @@ test('foldCalls counts tool/call events and wall clock from the attempt', () => assert.equal(rows[0].issuePr, '#903') }) +test('foldCalls records a routed harness and leaves the resume state unchanged', () => { + const { rows, state } = foldCalls([ + { type: 'turn/start', time: 1000, data: { turn: 1 } }, + header('deepseek', 'v4-flash'), + { type: 'user', time: 1001, data: { text: 'see PR #908 for #874' } }, + { type: 'tool/call', time: 1100, data: { name: 'read_file' } }, + message(1, 0, usage(10, 2), 1600), + ], { harness: 'lite' }) + assert.equal(rows.length, 1) + assert.deepEqual(missingUsageFields(rows[0]), []) + assert.equal(rows[0].harness, 'lite') + assert.equal(rows[0].issuePr, 'PR #908') + assert.equal(rows[0].toolCalls, 1) + assert.equal(rows[0].wallMs, 600) + assert.deepEqual(state, { provider: 'deepseek', model: 'v4-flash', retry: null }) + assert.throws(() => foldCalls([message(1, 0, usage(1, 1), 1)], { harness: 'turbo' }), /unknown harness/) +}) + +test('noteUsageHarness keeps a triage label and clears back to default', () => { + const rt = { + home: '/tmp/unused-usage-home', + state: { bots: {}, rooms: {}, agents: {} }, + ctx: { effect() {}, on() {} }, + warn() {}, + } + installUsage(rt) + assert.equal(rt.noteUsageHarness('s1', ['harness:medium']), 'medium') + assert.equal(rt.noteUsageHarness('s1', { labels: [{ name: 'harness:heavy' }] }), 'heavy') + assert.equal(rt.noteUsageHarness('s1', 'default'), 'default') + assert.equal(rt.noteUsageHarness('s1', ['agent-ready']), 'default') + assert.throws(() => rt.noteUsageHarness('s1', 'turbo'), /unknown harness/) + assert.throws(() => rt.noteUsageHarness('s1', ['harness:lite', 'harness:heavy']), /one harness tier label/) +}) + test('extractIssuePr prefers PR refs over bare issue numbers', () => { assert.equal(extractIssuePr('see PR #908 for #903'), 'PR #908') assert.equal(extractIssuePr('https://github.com/ThinkFlowLab/FlyBot/issues/903'), '#903')