Skip to content

Commit 84dd3fb

Browse files
konardclaude
andcommitted
feat(core): add multi-account pool with automatic rate-limit failover
Implement the ability to register multiple accounts per AI provider (Claude, Codex, Gemini) and automatically switch between them when one hits API rate limits. Core changes: - Add account pool domain types (AccountEntry, ProviderAccountPool, AccountPoolState) - Add pure account pool management functions (add, remove, select, cooldown) - Add rate-limit detection from agent output streams - Add API account-pool service with disk persistence - Add REST endpoints: GET/POST /account-pool/*, /account-pool/add, /remove, /next, /clear-cooldown - Integrate rate-limit monitoring into agent log consumption pipeline - Auto-restart agents with next available account on rate-limit detection - Bounded failover (max 10 consecutive restarts per agent key) Tests: 33 new tests covering pool logic and rate-limit detection Closes #213 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 9061e25 commit 84dd3fb

11 files changed

Lines changed: 1258 additions & 2 deletions

File tree

packages/api/src/api/contracts.ts

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,32 @@ export type FederationInboxResult =
282282
readonly subscription: FollowSubscription
283283
}
284284

285+
// CHANGE: add account pool API types for multi-account management
286+
// WHY: expose account pool CRUD and status via the API
287+
// QUOTE(ТЗ): "Сделать возможность регистрировать много аккаунтов"
288+
// REF: issue-213
289+
// PURITY: CORE
290+
// COMPLEXITY: O(1)
291+
export type AccountPoolProvider = "claude" | "codex" | "gemini"
292+
293+
export type AddAccountRequest = {
294+
readonly provider: AccountPoolProvider
295+
readonly label: string
296+
}
297+
298+
export type RemoveAccountRequest = {
299+
readonly provider: AccountPoolProvider
300+
readonly label: string
301+
}
302+
303+
export type AccountPoolSummary = {
304+
readonly provider: AccountPoolProvider
305+
readonly total: number
306+
readonly available: number
307+
readonly coolingDown: number
308+
readonly activeLabel: string | undefined
309+
}
310+
285311
export type ApiEventType =
286312
| "snapshot"
287313
| "project.created"

packages/api/src/api/schema.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,26 @@ export const CreateAgentRequestSchema = Schema.Struct({
9898
label: OptionalString
9999
})
100100

101+
// CHANGE: add account pool request schemas
102+
// WHY: validate API requests for multi-account pool management
103+
// REF: issue-213
104+
// PURITY: CORE
105+
// COMPLEXITY: O(1)
106+
export const AccountPoolProviderSchema = Schema.Literal("claude", "codex", "gemini")
107+
108+
export const AddAccountRequestSchema = Schema.Struct({
109+
provider: AccountPoolProviderSchema,
110+
label: Schema.String
111+
})
112+
113+
export const RemoveAccountRequestSchema = Schema.Struct({
114+
provider: AccountPoolProviderSchema,
115+
label: Schema.String
116+
})
117+
118+
export type AddAccountRequestInput = Schema.Schema.Type<typeof AddAccountRequestSchema>
119+
export type RemoveAccountRequestInput = Schema.Schema.Type<typeof RemoveAccountRequestSchema>
120+
101121
export const CreateFollowRequestSchema = Schema.Struct({
102122
actor: OptionalString,
103123
object: Schema.String,

packages/api/src/http.ts

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import { renderError, type AppError } from "@effect-template/lib/usecases/errors
1212

1313
import { ApiAuthRequiredError, ApiBadRequestError, ApiConflictError, ApiInternalError, ApiNotFoundError, describeUnknown } from "./api/errors.js"
1414
import {
15+
AddAccountRequestSchema,
1516
ApplyAllRequestSchema,
1617
CodexAuthImportRequestSchema,
1718
CodexAuthLoginRequestSchema,
@@ -21,6 +22,7 @@ import {
2122
CreateProjectRequestSchema,
2223
GithubAuthLoginRequestSchema,
2324
GithubAuthLogoutRequestSchema,
25+
RemoveAccountRequestSchema,
2426
StateCommitRequestSchema,
2527
StateInitRequestSchema,
2628
StateSyncRequestSchema,
@@ -36,6 +38,15 @@ import {
3638
readGithubAuthStatus
3739
} from "./services/auth.js"
3840
import { streamCodexAuthLogin } from "./services/auth-codex-login-stream.js"
41+
import {
42+
addPoolAccount,
43+
clearAccountCooldown,
44+
getPoolSummary,
45+
listAllPoolAccounts,
46+
listPoolAccounts,
47+
removePoolAccount,
48+
selectNextPoolAccount
49+
} from "./services/account-pool.js"
3950
import { getAgent, getAgentAttachInfo, listAgents, readAgentLogs, startAgent, stopAgent } from "./services/agents.js"
4051
import { latestProjectCursor, listProjectEventsSince } from "./services/events.js"
4152
import {
@@ -415,7 +426,64 @@ export const makeRouter = () => {
415426
)
416427
)
417428

418-
const withState = base.pipe(
429+
const withAccountPool = base.pipe(
430+
HttpRouter.get(
431+
"/account-pool",
432+
Effect.sync(() => ({ accounts: listAllPoolAccounts() })).pipe(
433+
Effect.flatMap((payload) => jsonResponse(payload, 200)),
434+
Effect.catchAll(errorResponse)
435+
)
436+
),
437+
HttpRouter.get(
438+
"/account-pool/:provider",
439+
HttpRouter.schemaParams(Schema.Struct({ provider: Schema.String })).pipe(
440+
Effect.flatMap(({ provider }) => {
441+
const p = provider as "claude" | "codex" | "gemini"
442+
return jsonResponse({
443+
accounts: listPoolAccounts(p),
444+
summary: getPoolSummary(p)
445+
}, 200)
446+
}),
447+
Effect.catchAll(errorResponse)
448+
)
449+
),
450+
HttpRouter.post(
451+
"/account-pool/add",
452+
Effect.gen(function*(_) {
453+
const request = yield* _(HttpServerRequest.schemaBodyJson(AddAccountRequestSchema))
454+
const state = addPoolAccount(request.provider, request.label)
455+
return yield* _(jsonResponse({ ok: true, state }, 201))
456+
}).pipe(Effect.catchAll(errorResponse))
457+
),
458+
HttpRouter.post(
459+
"/account-pool/remove",
460+
Effect.gen(function*(_) {
461+
const request = yield* _(HttpServerRequest.schemaBodyJson(RemoveAccountRequestSchema))
462+
const state = removePoolAccount(request.provider, request.label)
463+
return yield* _(jsonResponse({ ok: true, state }, 200))
464+
}).pipe(Effect.catchAll(errorResponse))
465+
),
466+
HttpRouter.post(
467+
"/account-pool/next",
468+
Effect.gen(function*(_) {
469+
const request = yield* _(HttpServerRequest.schemaBodyJson(
470+
Schema.Struct({ provider: Schema.Literal("claude", "codex", "gemini") })
471+
))
472+
const account = selectNextPoolAccount(request.provider)
473+
return yield* _(jsonResponse({ account: account ?? null }, 200))
474+
}).pipe(Effect.catchAll(errorResponse))
475+
),
476+
HttpRouter.post(
477+
"/account-pool/clear-cooldown",
478+
Effect.gen(function*(_) {
479+
const request = yield* _(HttpServerRequest.schemaBodyJson(RemoveAccountRequestSchema))
480+
const state = clearAccountCooldown(request.provider, request.label)
481+
return yield* _(jsonResponse({ ok: true, state }, 200))
482+
}).pipe(Effect.catchAll(errorResponse))
483+
)
484+
)
485+
486+
const withState = withAccountPool.pipe(
419487
HttpRouter.get(
420488
"/state/path",
421489
readStatePathOutput().pipe(

packages/api/src/program.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import { Console, Effect, Layer, Option } from "effect"
44
import { createServer } from "node:http"
55

66
import { makeRouter } from "./http.js"
7+
import { initializeAccountPool } from "./services/account-pool.js"
78
import { initializeAgentState } from "./services/agents.js"
89
import { startOutboxPolling } from "./services/federation.js"
910

@@ -49,6 +50,11 @@ export const program = (() => {
4950
return Effect.scoped(
5051
Console.log(`docker-git api boot port=${port}`).pipe(
5152
Effect.zipRight(initializeAgentState()),
53+
Effect.zipRight(
54+
Effect.tryPromise({ try: () => initializeAccountPool(), catch: () => new Error("account pool init failed") }).pipe(
55+
Effect.catchAll(() => Effect.void)
56+
)
57+
),
5258
Effect.zipRight(
5359
Console.log(`docker-git outbox polling interval=${pollingInterval}ms`)
5460
),
Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// CHANGE: add API-level account pool service with persistence and rate-limit monitoring
2+
// WHY: enable automatic switching between registered accounts when one hits API rate limits
3+
// QUOTE(ТЗ): "Сделать возможность регистрировать много аккаунтов codex, claude code и когда на одном лимиты закаончиваются он переходит на другой аккаунт"
4+
// REF: issue-213
5+
// SOURCE: n/a
6+
// FORMAT THEOREM: ∀op ∈ PoolOperation: op(state) → persist(nextState) ∧ consistent(nextState)
7+
// PURITY: SHELL
8+
// EFFECT: Effect<Result, ApiError>
9+
// INVARIANT: pool state is persisted to disk after every mutation; in-memory state is source of truth
10+
// COMPLEXITY: O(n) per operation where n = total accounts
11+
12+
import { defaultProjectsRoot } from "@effect-template/lib/usecases/path-helpers"
13+
import type {
14+
AccountPoolProvider,
15+
AccountPoolState,
16+
AccountEntry,
17+
RateLimitEvent
18+
} from "@effect-template/lib/core/account-pool-domain"
19+
import {
20+
addAccount,
21+
removeAccount,
22+
markRateLimited,
23+
clearCooldown,
24+
selectNextAvailable,
25+
advanceActiveIndex,
26+
listAccounts,
27+
listAllAccounts,
28+
poolSummary,
29+
emptyPoolState
30+
} from "@effect-template/lib/usecases/account-pool"
31+
import { detectRateLimit } from "@effect-template/lib/usecases/rate-limit-detector"
32+
import { promises as fs } from "node:fs"
33+
import { join } from "node:path"
34+
35+
let poolState: AccountPoolState = emptyPoolState(new Date().toISOString())
36+
let initialized = false
37+
38+
const nowIso = (): string => new Date().toISOString()
39+
40+
const stateFilePath = (): string =>
41+
join(defaultProjectsRoot(process.cwd()), ".orch", "state", "account-pool.json")
42+
43+
const persistState = async (): Promise<void> => {
44+
const filePath = stateFilePath()
45+
await fs.mkdir(join(filePath, ".."), { recursive: true })
46+
await fs.writeFile(filePath, JSON.stringify(poolState, null, 2), "utf8")
47+
}
48+
49+
const persistBestEffort = (): void => {
50+
void persistState().catch(() => {
51+
// best effort
52+
})
53+
}
54+
55+
export const initializeAccountPool = async (): Promise<void> => {
56+
if (initialized) {
57+
return
58+
}
59+
60+
const filePath = stateFilePath()
61+
const exists = await fs.stat(filePath).then(() => true).catch(() => false)
62+
if (exists) {
63+
const raw = await fs.readFile(filePath, "utf8")
64+
const parsed = JSON.parse(raw) as AccountPoolState
65+
poolState = {
66+
pools: parsed.pools ?? [],
67+
updatedAt: parsed.updatedAt ?? nowIso()
68+
}
69+
}
70+
71+
initialized = true
72+
}
73+
74+
export const addPoolAccount = (
75+
provider: AccountPoolProvider,
76+
label: string
77+
): AccountPoolState => {
78+
const now = nowIso()
79+
poolState = addAccount(poolState, provider, label, now)
80+
persistBestEffort()
81+
return poolState
82+
}
83+
84+
export const removePoolAccount = (
85+
provider: AccountPoolProvider,
86+
label: string
87+
): AccountPoolState => {
88+
const now = nowIso()
89+
poolState = removeAccount(poolState, provider, label, now)
90+
persistBestEffort()
91+
return poolState
92+
}
93+
94+
export const markAccountRateLimited = (
95+
event: RateLimitEvent
96+
): AccountPoolState => {
97+
const now = nowIso()
98+
poolState = markRateLimited(poolState, event, now)
99+
persistBestEffort()
100+
return poolState
101+
}
102+
103+
export const clearAccountCooldown = (
104+
provider: AccountPoolProvider,
105+
label: string
106+
): AccountPoolState => {
107+
const now = nowIso()
108+
poolState = clearCooldown(poolState, provider, label, now)
109+
persistBestEffort()
110+
return poolState
111+
}
112+
113+
export const selectNextPoolAccount = (
114+
provider: AccountPoolProvider
115+
): AccountEntry | undefined => {
116+
const now = nowIso()
117+
const account = selectNextAvailable(poolState, provider, now)
118+
if (account !== undefined) {
119+
poolState = advanceActiveIndex(poolState, provider, now)
120+
persistBestEffort()
121+
}
122+
return account
123+
}
124+
125+
export const listPoolAccounts = (
126+
provider: AccountPoolProvider
127+
): ReadonlyArray<AccountEntry> =>
128+
listAccounts(poolState, provider)
129+
130+
export const listAllPoolAccounts = (): ReadonlyArray<AccountEntry> =>
131+
listAllAccounts(poolState)
132+
133+
export const getPoolSummary = (
134+
provider: AccountPoolProvider
135+
): {
136+
readonly total: number
137+
readonly available: number
138+
readonly coolingDown: number
139+
readonly activeLabel: string | undefined
140+
} => poolSummary(poolState, provider, nowIso())
141+
142+
export const getPoolState = (): AccountPoolState => poolState
143+
144+
/**
145+
* Check an agent output line for rate-limit signals.
146+
* If a rate-limit is detected, marks the account as rate-limited
147+
* and returns the event for the caller to act upon.
148+
*
149+
* @effect mutates poolState on detection
150+
*/
151+
export const checkLineForRateLimit = (
152+
provider: AccountPoolProvider,
153+
label: string,
154+
line: string
155+
): RateLimitEvent | undefined => {
156+
const now = nowIso()
157+
const event = detectRateLimit(provider, label, line, now)
158+
if (event !== undefined) {
159+
markAccountRateLimited(event)
160+
}
161+
return event
162+
}

0 commit comments

Comments
 (0)