From 27d2d42d8b383052e96a7dcbdfab59c47dd105e6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 13 Jul 2026 07:29:23 +0000 Subject: [PATCH] feat(providers): remap tiers onto a custom gateway's real model IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit model_tiers.json pins public Anthropic IDs (claude-haiku-4-5-…, claude-sonnet-5, …). A self-hosted LiteLLM/proxy gateway usually serves its own model names, so a stock ID sent verbatim 404s. New src/gateway_model_map.js asks the gateway what it actually serves (GET /v1/models, once per process) and scores each advertised model against every tier's family — the family word (haiku/sonnet/opus/fable) gates the match, the setOverlap name-token coefficient picks the best id, ties break toward the id closest to the canonical name — then remaps each tier onto a real gateway model. - Zero breaking change: the MODELS export shape is untouched; this is a resolution-time layer. resolveModel (providers) and buildRunner (adjudicate) consult it only when the resolved id is a stock Anthropic ID, so an explicit .forge/providers.json alias or ANTHROPIC_MODEL override always wins. - Fail-safe: no gateway, unreachable /v1/models, or no family match returns the stock id unchanged. Direct api.anthropic.com sessions never probe and are byte-identical. - forge doctor surfaces the resolved tier→model mapping under a "gateway models" row so users can verify it and pin explicit IDs if a family scored wrong. - The key travels via the child's env, never argv (llm.js pattern); zero runtime deps. Adds 9 tests and documents the behavior in GUIDE, ARCHITECTURE, and CHANGELOG. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_019PXuKmJp92Gdo2FudNjSx2 --- ARCHITECTURE.md | 14 +++ CHANGELOG.md | 13 +++ docs/GUIDE.md | 12 ++ src/adjudicate.js | 13 ++- src/doctor.js | 38 +++++++ src/gateway_model_map.js | 195 +++++++++++++++++++++++++++++++++ src/providers.js | 32 +++++- test/doctor.test.js | 17 +++ test/gateway_model_map.test.js | 151 +++++++++++++++++++++++++ 9 files changed, 478 insertions(+), 7 deletions(-) create mode 100644 src/gateway_model_map.js create mode 100644 test/gateway_model_map.test.js diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3d17306..ce55df0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -250,6 +250,20 @@ runs `bump.mjs auto`: it releases only when a `feat`/`fix`/`perf`/breaking commi when none exist, and exits `3` (a clean skip, not a failure) otherwise — so releases cut themselves without a chore/docs merge spamming the registry. +**Custom-gateway model remap (`src/gateway_model_map.js`).** The tier table (`model_tiers.json`) +pins public Anthropic IDs, but a self-hosted LiteLLM/proxy gateway serves its own model names, so a +stock ID sent verbatim 404s. When a non-default gateway base URL is configured, the module fetches +`GET /v1/models` **once per process** (a spawned-node child with the key in env, never argv — the +`llm.js` pattern) and scores each advertised id against every tier's family: the family word +(haiku/sonnet/opus/fable) is a hard gate, the `setOverlap` coefficient of the tier's name tokens +picks the best match, ties break toward the id closest to the canonical name. `resolveModel` +(providers) and `buildRunner` (adjudicate) consult it only when the resolved id is a _stock_ ID — +an explicit `.forge/providers.json` alias or `ANTHROPIC_MODEL` override is never touched — and it +fails safe to the stock ID on no gateway / unreachable `/v1/models` / no family match, so direct +`api.anthropic.com` users are byte-identical. `forge doctor`'s **gateway models** row prints the +resolved `tier→model` mapping for verification. The `MODELS` export shape is unchanged: this is a +resolution-time layer, not a table edit. + **Intent cards (`src/intent.js`).** Prompt → intent by the same exemplar k-NN math as model routing — a labeled bank (English + Hinglish rows) under overlap similarity with a confidence gate, NOT a keyword DFA. Note `intentGrams` ≠ `contentGrams`: route.js stops diff --git a/CHANGELOG.md b/CHANGELOG.md index 3376992..18b0f66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- Custom-gateway model remap (`src/gateway_model_map.js`). The tier table pins public + Anthropic IDs that a self-hosted LiteLLM/proxy gateway may not serve; when a non-default + gateway base URL is set, Forge fetches `GET /v1/models` once per process and scores each + advertised model against every tier's family (family-word gate + `setOverlap` name-token + score, deterministic tie-break) to remap `haiku/sonnet/opus/fable` onto the gateway's real + IDs. `forge doctor` surfaces the resolved `tier→model` mapping under a **gateway models** row. + Zero breaking change — the `MODELS` export shape is unchanged, it fails safe to the stock ID + on no gateway / unreachable `/v1/models` / no family match, and an explicit + `.forge/providers.json` alias or `ANTHROPIC_MODEL` override always wins. Direct + `api.anthropic.com` sessions never probe and are byte-identical. + ## [0.12.4] - 2026-07-11 ### Fixed diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 4010137..3621801 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -226,6 +226,18 @@ Corporate gateway environments work out of the box: with `ANTHROPIC_BASE_URL` + `ANTHROPIC_AUTH_TOKEN` set (LiteLLM-style gateways), detection classifies the gateway, auth uses the token as a Bearer credential, and `ANTHROPIC_MODEL` pins the model. +**Custom gateways that rename models.** The tier table ships public Anthropic IDs +(`claude-haiku-4-5-…`, `claude-sonnet-5`, …), but a self-hosted gateway often serves its +own names (`bedrock-claude-haiku`, `prod-sonnet-5`). When a non-default gateway base URL is +set, Forge asks it once per process (`GET /v1/models`) and scores each advertised model +against every tier's family — the family word (haiku/sonnet/opus/fable) gates the match, the +overlap score picks the best id — then remaps each tier onto a real gateway model. It is a +silent, zero-config fallback: no gateway, an unreachable `/v1/models`, or no family match and +the stock IDs are used unchanged; direct `api.anthropic.com` sessions never probe. An explicit +model in `.forge/providers.json` (or `ANTHROPIC_MODEL`) always wins over the remap. `forge +doctor` prints the resolved `tier→model` mapping under **gateway models** so you can verify it +and pin explicit IDs if a family scored wrong. + ### `forge impact ` — what will this edit break? Reverse-dependency blast radius from the atlas graph. Run `forge atlas build` first. diff --git a/src/adjudicate.js b/src/adjudicate.js index a835624..4689aa0 100644 --- a/src/adjudicate.js +++ b/src/adjudicate.js @@ -11,6 +11,7 @@ // - ZERO-DEP. Access is a `claude -p` CLI shell-out; the runner is injectable so the pure // prompt/parse/verify logic is fully testable without the CLI or the network. import { execFileSync, spawnSync } from "node:child_process"; +import { gatewayModelId } from "./gateway_model_map.js"; import { buildHttpRunner as httpRunner } from "./llm.js"; import { MODELS } from "./model_tiers.js"; import { envModelOverride } from "./providers.js"; @@ -31,7 +32,11 @@ function hasClaude() { if (!_claudeChecked) { _claudeChecked = true; try { - const r = spawnSync("which", ["claude"], { encoding: "utf8", timeout: 2000, stdio: "pipe" }); + const r = spawnSync("which", ["claude"], { + encoding: "utf8", + timeout: 2000, + stdio: "pipe", + }); _claudeAvail = r.status === 0; } catch { _claudeAvail = false; @@ -43,7 +48,11 @@ function hasClaude() { /** Build an injectable LLM runner. Tries direct HTTP when `claude` CLI is unavailable * or when FORGE_LLM_HTTP=1. Falls back to `claude -p` otherwise. */ export function buildRunner({ model = "haiku", timeoutMs = 20000 } = {}) { - const resolvedModel = envModelOverride() || MODELS[model]?.id || model; + const override = envModelOverride(); + const stock = override || MODELS[model]?.id || model; + // A forced override is honored verbatim; otherwise remap the tier's stock id onto a custom + // gateway's real model when one is advertised (no-op for direct Anthropic — see gateway_model_map). + const resolvedModel = override ? stock : gatewayModelId(model, stock); if (process.env.FORGE_LLM_HTTP === "1" || !hasClaude()) { return httpRunner({ model: resolvedModel, timeoutMs }); } diff --git a/src/doctor.js b/src/doctor.js index f742c12..393059d 100644 --- a/src/doctor.js +++ b/src/doctor.js @@ -8,6 +8,7 @@ import { BRAND } from "./brand.js"; import { summary as cortexSummary } from "./cortex.js"; import { docsCheck } from "./docs_check.js"; import { extractHash, hashContent } from "./emit/_shared.js"; +import { gatewayBase, gatewayModelMap } from "./gateway_model_map.js"; import { verify as ledgerVerify, repoLedger } from "./ledger_store.js"; import { PRICING_VERIFIED } from "./model_tiers.js"; import { activeProvider, envModelOverride } from "./providers.js"; @@ -309,6 +310,42 @@ function checkProvider(out, targetRoot) { } } +// Custom-gateway model mapping: stock Anthropic ids can 404 on a self-hosted gateway that +// serves its own names. Surface the tier→gateway-model remap so the user can VERIFY it (and +// pin explicit ids if a family scored wrong). Only speaks for a non-default gateway base URL — +// direct api.anthropic.com sessions never probe, so this stays silent and network-free there. +function checkGateway(out) { + const base = gatewayBase(); + if (!base) return; // direct Anthropic or no gateway configured — nothing to remap + let m; + try { + m = gatewayModelMap({ base }); + } catch { + m = null; + } + if (!m || m.reachable === false) { + out.push( + warn( + "gateway models", + `${base} — /v1/models unreachable; using stock IDs (may 404 if this gateway renames models)`, + ), + ); + return; + } + const entries = Object.entries(m.models); + if (!entries.length) { + out.push( + warn( + "gateway models", + `${base} serves ${m.catalog.length} model(s) but none matched a tier family — set explicit IDs via \`${BRAND.cli} config provider add\``, + ), + ); + return; + } + const summary = entries.map(([tier, v]) => `${tier}→${v.id}`).join(", "); + out.push(ok("gateway models", `${base}: ${summary}`)); +} + // Docs↔code drift — a self-check of the forge package's own docs, so it only runs // when doctor is pointed at the forge repo itself (contributors + CI), never at a // host project whose README rightly says nothing about forge commands. @@ -342,6 +379,7 @@ export function doctor({ targetRoot = process.cwd() } = {}) { const results = []; checkNode(results); checkProvider(results, targetRoot); + checkGateway(results); checkBrandConsistency(results); checkLayers(results); checkGuardsExecutable(results); diff --git a/src/gateway_model_map.js b/src/gateway_model_map.js new file mode 100644 index 0000000..5a14678 --- /dev/null +++ b/src/gateway_model_map.js @@ -0,0 +1,195 @@ +// forge gateway model map — remap complexity tiers onto a CUSTOM gateway's real model IDs. +// +// The problem: model_tiers.json pins public Anthropic IDs (claude-haiku-4-5-20251001, …). +// A self-hosted LiteLLM/proxy gateway rarely exposes those exact names — it advertises its +// OWN ids (e.g. "bedrock-claude-haiku", "prod-sonnet", "claude-3-5-sonnet-v2"). Sending a +// stock id straight to such a gateway 404s. So we ask the gateway what it actually serves +// (GET /v1/models, once per process) and SCORE each advertised id against every tier's family +// — the same DATA-is-a-table / DECISION-is-a-formula rule the rest of forge follows: the tier +// families are data, the pick is a graded overlap score (src/math.js setOverlap), inspectable +// and testable. +// +// Contract (zero breaking change): +// - Only engages for a NON-default gateway base URL. Direct api.anthropic.com → no-op, no net. +// - FAIL-SAFE. No gateway, unreachable, unparseable, or no family match → returns the stock +// id unchanged. Callers are byte-identical to before when there is nothing to remap. +// - The MODELS export shape is untouched; nothing here mutates model_tiers. +import { spawnSync } from "node:child_process"; +import { setOverlap } from "./math.js"; +import { MODELS, TIER_ORDER } from "./model_tiers.js"; + +const ANTHROPIC_DEFAULT = "https://api.anthropic.com"; + +// GET {base}/v1/models in a spawned node child so this module stays synchronous like every +// other forge faculty (embed.js / llm.js pattern). The auth key travels via the child's env +// (_FORGE_LLM_KEY) — never in argv, never logged. Accepts both OpenAI-shaped ({data:[{id}]}) +// and Anthropic-shaped ({data:[{id}]}) catalogs; both key the list under data[].id. +const FETCH_CHILD = `let raw="";process.stdin.on("data",(d)=>{raw+=d;});process.stdin.on("end",async()=>{try{const{url,timeoutMs}=JSON.parse(raw);const key=process.env._FORGE_LLM_KEY||"";const headers={"anthropic-version":"2023-06-01"};if(key.startsWith("Bearer ")){headers.authorization=key;}else if(key){headers["x-api-key"]=key;headers.authorization="Bearer "+key;}const ac=new AbortController();const timer=setTimeout(()=>ac.abort(),timeoutMs||5000);let res;try{res=await fetch(url,{headers,signal:ac.signal});}finally{clearTimeout(timer);}if(!res.ok){process.stderr.write("http "+res.status);process.exit(1);}const data=await res.json();const rows=Array.isArray(data)?data:Array.isArray(data&&data.data)?data.data:[];const ids=rows.map((m)=>(typeof m==="string"?m:m&&m.id)).filter((x)=>typeof x==="string"&&x);process.stdout.write(JSON.stringify(ids));}catch(e){process.stderr.write(String((e&&e.message)||e));process.exit(1);}});`; + +// Process-lifetime cache: base URL -> string[] (advertised ids) | null (fetched, none usable). +// "Once per process" is the whole point — the ambient LLM path must not re-probe on every call. +const _catalogCache = new Map(); + +/** Clear the per-process /v1/models cache (tests only). */ +export function _resetGatewayCache() { + _catalogCache.clear(); +} + +/** + * The active gateway base URL to remap against, or null when there is nothing to remap. + * Mirrors llm.js resolution (LITELLM_BASE_URL wins, then ANTHROPIC_BASE_URL). The default + * Anthropic endpoint returns null so direct-API users never trigger a probe or a remap. + * @returns {string|null} + */ +export function gatewayBase() { + const url = (process.env.LITELLM_BASE_URL || process.env.ANTHROPIC_BASE_URL || "").replace( + /\/+$/, + "", + ); + if (!url) return null; + if (url.toLowerCase() === ANTHROPIC_DEFAULT) return null; // direct Anthropic — stock ids are correct + return url; +} + +function apiKey() { + return ( + process.env.ANTHROPIC_API_KEY || + process.env.ANTHROPIC_AUTH_TOKEN || + process.env.LITELLM_API_KEY || + "" + ); +} + +function spawnFetch(base, timeoutMs) { + const r = spawnSync(process.execPath, ["-e", FETCH_CHILD], { + input: JSON.stringify({ url: `${base}/v1/models`, timeoutMs }), + encoding: "utf8", + timeout: timeoutMs + 1000, + maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, _FORGE_LLM_KEY: apiKey() }, + stdio: ["pipe", "pipe", "pipe"], + }); + if (r.error || r.status !== 0 || !r.stdout) return null; + const ids = JSON.parse(r.stdout); + return Array.isArray(ids) ? ids : null; +} + +/** + * Fetch (and cache once per process) the model ids a gateway advertises at /v1/models. + * @param {string} base gateway base URL (no trailing slash) + * @param {{timeoutMs?: number, fetchImpl?: (base:string)=>string[]}} [opts] fetchImpl is injectable for tests + * @returns {string[]|null} advertised ids, or null on any failure + */ +export function fetchModelIds(base, { timeoutMs = 5000, fetchImpl } = {}) { + if (!base) return null; + if (_catalogCache.has(base)) return _catalogCache.get(base); + let ids = null; + try { + ids = fetchImpl ? fetchImpl(base) : spawnFetch(base, timeoutMs); + } catch { + ids = null; + } + const clean = Array.isArray(ids) + ? [...new Set(ids.filter((x) => typeof x === "string" && x))] + : null; + const result = clean && clean.length ? clean : null; + _catalogCache.set(base, result); + return result; +} + +const tokenize = (s) => + new Set( + String(s) + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean), + ); + +/** Reference token set for a tier: the family key plus its marketing-name tokens (e.g. haiku → {haiku,4,5}). */ +export function familyTokens(tier) { + return tokenize(`${tier} ${MODELS[tier]?.name ?? ""}`); +} + +/** + * Score how well a gateway model id belongs to a tier family, in [0,1]. + * The family word itself (haiku/sonnet/opus/fable) is a HARD gate — absent it, the id is not a + * candidate for that tier (score 0), so an unrelated model can never be mis-assigned. Present it, + * the score is the overlap coefficient of the tier's reference tokens with the id's tokens, which + * rewards a version match ("claude-sonnet-5" scores 1.0 for sonnet; "prod-sonnet" scores lower). + * @param {string} modelId + * @param {string} tier + * @returns {number} + */ +export function familyScore(modelId, tier) { + const toks = tokenize(modelId); + if (!toks.has(tier)) return 0; // family word MUST be present + return setOverlap(familyTokens(tier), toks); +} + +// Deterministic tie-break among equal-scoring candidates: prefer the id closest to the canonical +// name (fewest tokens — less vendor/deployment noise), then lexicographic for stability. +function tieBreak(a, b) { + const na = tokenize(a).size; + const nb = tokenize(b).size; + if (na !== nb) return na - nb; + return a < b ? -1 : a > b ? 1 : 0; +} + +/** + * Pure: given a gateway's advertised ids, pick the best id per tier by family score. + * @param {string[]} ids + * @returns {Record} only tiers with a family match appear + */ +export function buildGatewayMap(ids = []) { + const list = [...new Set((ids || []).filter((x) => typeof x === "string" && x))]; + /** @type {Record} */ + const map = {}; + for (const tier of TIER_ORDER) { + let best = null; + for (const id of list) { + const score = familyScore(id, tier); + if (score <= 0) continue; + if (!best || score > best.score || (score === best.score && tieBreak(id, best.id) < 0)) { + best = { id, score }; + } + } + if (best) map[tier] = best; + } + return map; +} + +/** + * The tier→gateway-model mapping for the active gateway. Fetches /v1/models (cached) and scores. + * @param {{base?: string, fetchImpl?: (base:string)=>string[], timeoutMs?: number}} [opts] + * @returns {{active:boolean, base:(string|null), reachable?:boolean, catalog?:string[], models:Record}} + */ +export function gatewayModelMap({ base, fetchImpl, timeoutMs } = {}) { + const b = base ?? gatewayBase(); + if (!b) return { active: false, base: null, models: {} }; + const ids = fetchModelIds(b, { fetchImpl, timeoutMs }); + if (!ids) return { active: true, base: b, reachable: false, models: {} }; + return { + active: true, + base: b, + reachable: true, + catalog: ids, + models: buildGatewayMap(ids), + }; +} + +/** + * Resolve a tier to a gateway model id, or return `fallbackId` unchanged (silent fallback). + * This is the one function callers reach for: it never throws and never blocks a direct-API user. + * @param {string} tier + * @param {string} fallbackId the stock id to use when there is nothing to remap + * @param {{base?: string, fetchImpl?: (base:string)=>string[], timeoutMs?: number}} [opts] + * @returns {string} + */ +export function gatewayModelId(tier, fallbackId, opts = {}) { + try { + const m = gatewayModelMap(opts); + return m.models?.[tier]?.id ?? fallbackId; + } catch { + return fallbackId; + } +} diff --git a/src/providers.js b/src/providers.js index 3cbb919..bf19239 100644 --- a/src/providers.js +++ b/src/providers.js @@ -6,6 +6,7 @@ import { execFileSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; +import { gatewayModelId } from "./gateway_model_map.js"; import { MODELS } from "./model_tiers.js"; const PROVIDERS_FILE = "providers.json"; @@ -133,12 +134,22 @@ export function envModelOverride() { return process.env.ANTHROPIC_MODEL?.trim() || process.env.FORGE_MODEL?.trim() || null; } -/** Resolve a tier key (haiku/sonnet/opus/fable) to the active provider's model ID. */ +/** True when `id` is a stock Anthropic public id from model_tiers (i.e. the passthrough default, + * not a deliberately-configured gateway alias). Only stock ids are candidates for a gateway remap. */ +function isStockId(id) { + return id != null && Object.values(MODELS).some((m) => m.id === id); +} + +/** Resolve a tier key (haiku/sonnet/opus/fable) to the active provider's model ID. + * When the provider is a custom gateway and the resolved id is a stock Anthropic public id the + * gateway may not serve, remap it onto a real advertised model (gateway_model_map). Explicit + * aliases and direct-Anthropic setups are returned untouched — silent, zero-breaking fallback. */ export function resolveModel(root, tierKey) { const override = envModelOverride(); if (override) return override; const provider = activeProvider(root); - return provider.models?.[tierKey] ?? MODELS[tierKey]?.id ?? null; + const configured = provider.models?.[tierKey] ?? MODELS[tierKey]?.id ?? null; + return isStockId(configured) ? gatewayModelId(tierKey, configured) : configured; } /** Switch the active provider. Returns the new active config. */ @@ -224,11 +235,19 @@ export function autoDetectProvider() { } if (process.env.OPENROUTER_API_KEY) { - return { name: "openrouter", ...BUILTIN_PROVIDERS.openrouter, source: "OPENROUTER_API_KEY" }; + return { + name: "openrouter", + ...BUILTIN_PROVIDERS.openrouter, + source: "OPENROUTER_API_KEY", + }; } if (process.env.ANTHROPIC_API_KEY) { - return { name: "anthropic", ...BUILTIN_PROVIDERS.anthropic, source: "ANTHROPIC_API_KEY" }; + return { + name: "anthropic", + ...BUILTIN_PROVIDERS.anthropic, + source: "ANTHROPIC_API_KEY", + }; } if (process.env.ANTHROPIC_AUTH_TOKEN) { @@ -349,7 +368,10 @@ export function providerStatus(root = process.cwd()) { { key: "ANTHROPIC_BASE_URL", set: Boolean(process.env.ANTHROPIC_BASE_URL) }, { key: "OPENROUTER_API_KEY", set: Boolean(process.env.OPENROUTER_API_KEY) }, { key: "ANTHROPIC_API_KEY", set: Boolean(process.env.ANTHROPIC_API_KEY) }, - { key: "ANTHROPIC_AUTH_TOKEN", set: Boolean(process.env.ANTHROPIC_AUTH_TOKEN) }, + { + key: "ANTHROPIC_AUTH_TOKEN", + set: Boolean(process.env.ANTHROPIC_AUTH_TOKEN), + }, { key: "ANTHROPIC_MODEL", set: Boolean(process.env.ANTHROPIC_MODEL) }, ]; diff --git a/test/doctor.test.js b/test/doctor.test.js index b112a6c..58474d1 100644 --- a/test/doctor.test.js +++ b/test/doctor.test.js @@ -55,6 +55,23 @@ test("doctor surfaces the new tooling / guards-exec / atlas / pricing checks", ( } }); +test("doctor stays silent about gateway models when no custom gateway is configured", () => { + const save = { + l: process.env.LITELLM_BASE_URL, + a: process.env.ANTHROPIC_BASE_URL, + }; + try { + // Direct Anthropic (default endpoint) or nothing configured → no probe, no "gateway models" row. + process.env.LITELLM_BASE_URL = ""; + process.env.ANTHROPIC_BASE_URL = "https://api.anthropic.com"; + const labels = doctor({ targetRoot: fixture() }).results.map((r) => r.label); + assert.ok(!labels.includes("gateway models"), "no gateway check for a direct-API session"); + } finally { + process.env.LITELLM_BASE_URL = save.l ?? ""; + process.env.ANTHROPIC_BASE_URL = save.a ?? ""; + } +}); + test("doctor checks plugin manifests and hook compatibility", () => { const results = doctor({ targetRoot: fixture() }).results; const claude = results.find((r) => r.label === "Claude plugin hooks"); diff --git a/test/gateway_model_map.test.js b/test/gateway_model_map.test.js new file mode 100644 index 0000000..9715f6a --- /dev/null +++ b/test/gateway_model_map.test.js @@ -0,0 +1,151 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + _resetGatewayCache, + buildGatewayMap, + familyScore, + familyTokens, + fetchModelIds, + gatewayBase, + gatewayModelId, + gatewayModelMap, +} from "../src/gateway_model_map.js"; + +// --------------------------------------------------------------------------- +// familyScore — the family word is a hard gate; overlap rewards a version match. +// --------------------------------------------------------------------------- + +test("familyScore requires the family word and scores higher on a version match", () => { + // "haiku" absent → not a candidate for the haiku tier, no matter what else matches. + assert.equal(familyScore("claude-sonnet-5", "haiku"), 0); + assert.equal(familyScore("gpt-4o", "opus"), 0); + // Family word present → scored by overlap of {haiku,4,5} with the id's tokens. + const exact = familyScore("claude-haiku-4-5", "haiku"); // {haiku,4,5} ⊆ id → 1.0 + const bare = familyScore("prod-haiku", "haiku"); // only "haiku" shared → 1/3 + assert.ok(Math.abs(exact - 1) < 1e-9, "full name+version match saturates at 1"); + assert.ok(bare > 0 && bare < exact, "a bare family match still scores, but below an exact one"); +}); + +test("familyTokens carries the tier key plus its marketing-name tokens", () => { + assert.deepEqual([...familyTokens("sonnet")].sort(), ["5", "sonnet"]); + assert.deepEqual([...familyTokens("haiku")].sort(), ["4", "5", "haiku"]); +}); + +// --------------------------------------------------------------------------- +// buildGatewayMap — pure assignment of tiers to a gateway's advertised ids. +// --------------------------------------------------------------------------- + +test("buildGatewayMap maps each tier to the best family-matching advertised id", () => { + const ids = [ + "bedrock-claude-haiku-4-5", + "prod-sonnet-5", + "claude-opus-4-8-internal", + "gpt-4o", // unrelated — never assigned + ]; + const map = buildGatewayMap(ids); + assert.equal(map.haiku.id, "bedrock-claude-haiku-4-5"); + assert.equal(map.sonnet.id, "prod-sonnet-5"); + assert.equal(map.opus.id, "claude-opus-4-8-internal"); + assert.ok(!map.fable, "no fable model advertised → tier omitted (caller keeps the stock id)"); +}); + +test("buildGatewayMap breaks ties toward the id closest to the canonical name", () => { + // Both contain "sonnet" + "5" → equal overlap score; the shorter, less-noisy id wins. + const map = buildGatewayMap(["vendor-region-prod-sonnet-5-preview", "claude-sonnet-5"]); + assert.equal(map.sonnet.id, "claude-sonnet-5"); +}); + +test("buildGatewayMap never assigns an unrelated model and tolerates junk input", () => { + assert.deepEqual(buildGatewayMap(["gpt-4o", "mixtral", "llama-3"]), {}); + assert.deepEqual(buildGatewayMap([]), {}); + assert.deepEqual(buildGatewayMap([null, 42, "", "haiku"]).haiku.id, "haiku"); +}); + +// --------------------------------------------------------------------------- +// gatewayBase — direct Anthropic never engages the remap. +// --------------------------------------------------------------------------- + +test("gatewayBase returns null for the default Anthropic endpoint and when unset", () => { + const save = { + l: process.env.LITELLM_BASE_URL, + a: process.env.ANTHROPIC_BASE_URL, + }; + try { + process.env.LITELLM_BASE_URL = ""; + process.env.ANTHROPIC_BASE_URL = "https://api.anthropic.com/"; + assert.equal(gatewayBase(), null, "trailing-slash default is still the default"); + process.env.ANTHROPIC_BASE_URL = ""; + assert.equal(gatewayBase(), null, "nothing configured → null"); + process.env.LITELLM_BASE_URL = "http://gw.internal:4000/"; + assert.equal( + gatewayBase(), + "http://gw.internal:4000", + "gateway url wins, trailing slash trimmed", + ); + } finally { + process.env.LITELLM_BASE_URL = save.l ?? ""; + process.env.ANTHROPIC_BASE_URL = save.a ?? ""; + } +}); + +// --------------------------------------------------------------------------- +// fetch caching + fail-safe resolution (fetch is injected — no network in tests). +// --------------------------------------------------------------------------- + +test("fetchModelIds caches once per process and returns null on failure", () => { + _resetGatewayCache(); + let calls = 0; + const fetchImpl = () => { + calls++; + return ["claude-haiku-4-5", "claude-sonnet-5"]; + }; + const a = fetchModelIds("http://gw:4000", { fetchImpl }); + const b = fetchModelIds("http://gw:4000", { fetchImpl }); + assert.deepEqual(a, ["claude-haiku-4-5", "claude-sonnet-5"]); + assert.deepEqual(b, a); + assert.equal(calls, 1, "second lookup is served from the process cache"); + + _resetGatewayCache(); + const throwing = () => { + throw new Error("connrefused"); + }; + assert.equal(fetchModelIds("http://down:4000", { fetchImpl: throwing }), null); +}); + +test("gatewayModelMap reports reachability and the scored mapping", () => { + _resetGatewayCache(); + const m = gatewayModelMap({ + base: "http://gw:4000", + fetchImpl: () => ["my-haiku", "my-sonnet-5", "gpt-4o"], + }); + assert.equal(m.active, true); + assert.equal(m.reachable, true); + assert.equal(m.models.haiku.id, "my-haiku"); + assert.equal(m.models.sonnet.id, "my-sonnet-5"); + + _resetGatewayCache(); + const down = gatewayModelMap({ + base: "http://down:4000", + fetchImpl: () => null, + }); + assert.equal(down.reachable, false); + assert.deepEqual(down.models, {}); +}); + +test("gatewayModelId remaps a matched tier and falls back silently otherwise", () => { + _resetGatewayCache(); + const opts = { + base: "http://gw:4000", + fetchImpl: () => ["renamed-haiku-4-5", "prod-sonnet-5"], + }; + assert.equal(gatewayModelId("haiku", "claude-haiku-4-5-20251001", opts), "renamed-haiku-4-5"); + // fable isn't advertised → the caller's stock id is returned unchanged (silent fallback). + assert.equal(gatewayModelId("fable", "claude-fable-5", opts), "claude-fable-5"); + + // No gateway base at all (default fetchImpl never invoked) → always the fallback, never a throw. + _resetGatewayCache(); + assert.equal( + gatewayModelId("haiku", "claude-haiku-4-5-20251001", { base: null }), + "claude-haiku-4-5-20251001", + ); +});