From 0abf18dcef3279627e1d544692d39494f71a5d6d Mon Sep 17 00:00:00 2001 From: Lex <14139451+LeXwDeX@users.noreply.github.com> Date: Tue, 27 Oct 2026 22:57:31 +0800 Subject: [PATCH 1/4] fix: retry circuit-breaker rejections from upstream gateways Gateways with circuit breakers reject with text like "all targets recovering (half-open probe in flight)" and a non-5xx status, which no retry rule matched, so parallel subagents failed instead of backing off. Issue: #900 --- packages/opencode/src/session/retry.ts | 2 ++ packages/opencode/test/session/retry.test.ts | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index 829dab4af..dbcc3dfd1 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -69,6 +69,8 @@ const RETRYABLE_MESSAGE_PATTERNS = [ /^timeout$|\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b/i, /try your request again|retry your request|resource exhausted|resource_exhausted/i, /\btry again (?:later|in\b)|\b(?:currently|temporarily) at capacity\b/i, + // Gateway circuit breakers reject while every upstream target is tripped, often with a 4xx status. + /\ball targets recovering\b|\bhalf[-_\s]open\b|\bcircuit(?:[-_\s]breaker)?(?:\s+is)?[-_\s]open\b/i, ] function cap(ms: number) { diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index 03f671cbf..430fae4b8 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -222,6 +222,11 @@ describe("session.retry.retryable", () => { "The model is currently at capacity due to high demand", "The service is temporarily at capacity", "upstream returned status 524", + "all targets recovering (half-open probe in flight)", + "upstream target is half-open", + "circuit breaker open", + "Circuit breaker is open for this channel", + "circuit open", ])("retries matching API error text: %s", (message) => { expect(SessionRetry.retryable(wrap(message), retryProvider)).toEqual({ message }) }) @@ -244,6 +249,19 @@ describe("session.retry.retryable", () => { expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Request failed" }) }) + test("retries 4xx circuit-breaker rejections from gateways", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "all targets recovering (half-open probe in flight)", + isRetryable: false, + statusCode: 400, + }).toObject(), + ) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ + message: "all targets recovering (half-open probe in flight)", + }) + }) + test("retries transport timeout errors", () => { const request = MessageV2.fromError(new ProviderError.HeaderTimeoutError(10000), { providerID }) expect(SessionV1.APIError.isInstance(request)).toBe(true) From 144a744806db1fd5e24e3fd68f8225de9b79bc8d Mon Sep 17 00:00:00 2001 From: Lex <14139451+LeXwDeX@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:09:16 +0800 Subject: [PATCH 2/4] fix: reconnect after unrecognised request failures A failure that matches no retry rule is now retried up to 3 times with 3s, 6s, 6s waits. Aborts, context overflow, auth, output-length, structured-output, content-filter and reasoning-state failures stay terminal. Issue: #900 --- packages/opencode/src/session/retry.ts | 42 +++++++++++++-- .../test/session/processor-effect.test.ts | 10 ++-- packages/opencode/test/session/retry.test.ts | 52 ++++++++++++++++++- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index dbcc3dfd1..4d311fa96 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -3,6 +3,7 @@ import { SessionV1 } from "@opencode-ai/core/v1/session" import { Cause, Clock, Duration, Effect, Schedule, Schema } from "effect" import { iife } from "@/util/iife" import { isRecord } from "@/util/record" +import { REASONING_STATE_MESSAGE } from "@/provider/error" export type Err = ReturnType @@ -60,6 +61,11 @@ export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout export const RETRY_MAX_RETRIES = 5 +// Catch-all reconnect for failures no rule recognises: 3s, 6s, 6s. +export const FALLBACK_INITIAL_DELAY = 3000 +export const FALLBACK_MAX_DELAY = 6000 +export const FALLBACK_MAX_RETRIES = 3 + const RETRYABLE_MESSAGE_PATTERNS = [ // Bounded so token counts or IDs in a 4xx body ("150000 tokens") never read as a status. /\b(?:429|500|502|503|504|524)\b/i, @@ -110,6 +116,10 @@ export function delay(attempt: number, error?: SessionV1.APIError, random = Math return cap(Math.min(exponential(attempt, random), RETRY_MAX_DELAY_NO_HEADERS)) } +export function fallbackDelay(attempt: number) { + return Math.min(FALLBACK_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), FALLBACK_MAX_DELAY) +} + function exponential(attempt: number, random: number) { const base = RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1) return Math.ceil(base + base * RETRY_JITTER_FACTOR * random) @@ -196,6 +206,29 @@ export function retryable(error: Err, provider: string) { return undefined } +/** + * Unrecognised failures get a bounded reconnect attempt. Failures with a known + * terminal meaning stay terminal: retrying them cannot change the outcome. + */ +export function fallback(error: Err): Retryable | undefined { + if ( + SessionV1.AbortedError.isInstance(error) || + SessionV1.ContextOverflowError.isInstance(error) || + SessionV1.AuthError.isInstance(error) || + SessionV1.OutputLengthError.isInstance(error) || + SessionV1.StructuredOutputError.isInstance(error) || + SessionV1.ContentFilterError.isInstance(error) + ) + return undefined + if (SessionV1.APIError.isInstance(error)) { + const status = error.data.statusCode + if (status === 401 || status === 403) return undefined + if (error.data.message === REASONING_STATE_MESSAGE) return undefined + } + const message = isRecord(error.data) ? error.data.message : undefined + return { message: typeof message === "string" && message ? message : "Unknown error, reconnecting" } +} + function matchesRetryableMessage(value: unknown) { return typeof value === "string" && RETRYABLE_MESSAGE_PATTERNS.some((pattern) => pattern.test(value)) } @@ -233,11 +266,14 @@ export function policy(opts: { Effect.succeed((meta: Schedule.InputMetadata) => { if (opts.shouldRetry?.(meta.input) === false) return Cause.done(meta.attempt) const error = opts.parse(meta.input) - const retry = retryable(error, opts.provider) + const known = retryable(error, opts.provider) + const retry = known ?? fallback(error) if (!retry) return Cause.done(meta.attempt) - if (meta.attempt > RETRY_MAX_RETRIES) return Cause.done(meta.attempt) + if (meta.attempt > (known ? RETRY_MAX_RETRIES : FALLBACK_MAX_RETRIES)) return Cause.done(meta.attempt) return Effect.gen(function* () { - const wait = delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined) + const wait = known + ? delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined) + : fallbackDelay(meta.attempt) const now = yield* Clock.currentTimeMillis yield* opts.set({ attempt: meta.attempt, diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 0a3bbac96..ea5cd9ecd 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -14,6 +14,7 @@ import { Session } from "@/session/session" import { LLM } from "../../src/session/llm" import { MessageV2 } from "../../src/session/message-v2" import { SessionProcessor } from "../../src/session/processor" +import { SessionRetry } from "../../src/session/retry" import { MessageID, PartID, SessionID } from "../../src/session/schema" import { SessionStatus } from "../../src/session/status" import { SessionSummary } from "../../src/session/summary" @@ -554,13 +555,14 @@ it.live("session.processor effect tests reset reasoning state across retries", ( ), ) -it.live("session.processor effect tests do not retry unknown json errors", () => +it.live("session.processor effect tests reconnect unknown json errors three times before stopping", () => provideTmpdirServer( ({ dir, llm }) => Effect.gen(function* () { const { processors, session, provider } = yield* boot() - yield* llm.error(400, { error: { message: "no_kv_space" } }) + for (let i = 0; i <= SessionRetry.FALLBACK_MAX_RETRIES; i++) + yield* llm.error(400, { error: { message: "no_kv_space" } }) const chat = yield* session.create({}) const parent = yield* user(chat.id, "json") @@ -590,11 +592,12 @@ it.live("session.processor effect tests do not retry unknown json errors", () => }) expect(value).toBe("stop") - expect(yield* llm.calls).toBe(1) + expect(yield* llm.calls).toBe(SessionRetry.FALLBACK_MAX_RETRIES + 1) expect(handle.message.error?.name).toBe("APIError") }), { config: (url) => providerCfg(url) }, ), + 30_000, ) it.live("session.processor effect tests retry recognized structured json errors", () => @@ -1606,6 +1609,7 @@ itFragmentFailure.live("session.processor effect tests flush partial v2 fragment }), { config: cfg }, ), + 30_000, ) const displayLLM = Layer.succeed( diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index 430fae4b8..c49d16b02 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -3,7 +3,8 @@ import { SessionV1 } from "@opencode-ai/core/v1/session" import type { NamedError } from "@opencode-ai/core/util/error" import { APICallError } from "ai" import { setTimeout as sleep } from "node:timers/promises" -import { Effect, Schedule, Schema } from "effect" +import { Clock, Effect, Exit, Fiber, Schedule, Schema } from "effect" +import * as TestClock from "effect/testing/TestClock" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { SessionRetry } from "../../src/session/retry" @@ -146,6 +147,55 @@ describe("session.retry.delay", () => { expect(attempts).toStrictEqual([1, 2, 3, 4, 5]) }), ) + + test("policy reconnects unrecognised failures up to three times", () => + Effect.gen(function* () { + const waits: number[] = [] + let calls = 0 + const fiber = yield* Effect.suspend(() => { + calls++ + return Effect.fail(new Error("socket went sideways")) + }).pipe( + Effect.retry( + SessionRetry.policy({ + provider: "test", + parse: (error) => MessageV2.fromError(error, { providerID }), + set: (info) => + Effect.gen(function* () { + waits.push(info.next - (yield* Clock.currentTimeMillis)) + }), + }), + ), + Effect.exit, + Effect.forkChild, + ) + yield* TestClock.adjust("30 seconds") + const exit = yield* Fiber.join(fiber) + + expect(Exit.isFailure(exit)).toBe(true) + expect(calls).toBe(SessionRetry.FALLBACK_MAX_RETRIES + 1) + expect(waits).toStrictEqual([3000, 6000, 6000]) + }).pipe(Effect.provide(TestClock.layer()), Effect.runPromise), + ) + + test("keeps terminal failures out of the fallback", () => { + const terminal = [ + new SessionV1.AbortedError({ message: "Aborted" }).toObject(), + new SessionV1.ContextOverflowError({ message: "too long" }).toObject(), + new SessionV1.AuthError({ providerID: "test", message: "bad key" }).toObject(), + new SessionV1.ContentFilterError({ message: "filtered" }).toObject(), + new SessionV1.APIError({ message: "denied", isRetryable: false, statusCode: 401 }).toObject(), + new SessionV1.APIError({ message: ProviderError.REASONING_STATE_MESSAGE, isRetryable: false }).toObject(), + ] + expect(terminal.map(SessionRetry.fallback)).toStrictEqual(terminal.map(() => undefined)) + expect(SessionRetry.fallback(wrap("socket went sideways"))).toEqual({ message: "socket went sideways" }) + }) +}) + +describe("session.retry.fallbackDelay", () => { + test("backs off from 3 seconds and caps at 6 seconds", () => { + expect([1, 2, 3].map(SessionRetry.fallbackDelay)).toStrictEqual([3000, 6000, 6000]) + }) }) describe("session.retry.retryable", () => { From cddc83c996f5550f138bc445f7b9d62ff4d8e310 Mon Sep 17 00:00:00 2001 From: Lex <14139451+LeXwDeX@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:09:43 +0800 Subject: [PATCH 3/4] docs: add GraphAgent 1.0.90 release notes Issue: #900 --- .github/releases/v1.0.90.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/releases/v1.0.90.md diff --git a/.github/releases/v1.0.90.md b/.github/releases/v1.0.90.md new file mode 100644 index 000000000..4c6277888 --- /dev/null +++ b/.github/releases/v1.0.90.md @@ -0,0 +1,36 @@ +## opencode 1.0.90 + +Stable release from `main` branch. Retry gateway circuit-breaker rejections and reconnect after unrecognised request failures. + +--- + +### ๐Ÿ› Bug Fixes + +- **Retry gateway circuit-breaker rejections (#900).** Gateways that trip every upstream target reply with text such as `all targets recovering (half-open probe in flight)`, often with a 4xx status. These replies now count as transient and use the normal retry backoff, so parallel subagents and DAG workers wait for the breaker to close instead of failing at once. `half-open`, `circuit open` and `circuit breaker is open` are matched too. +- **Reconnect after unrecognised failures (#900).** A failure that matches no retry rule, such as an unexpected WebSocket or transport error, is now retried up to 3 times. Each retry sends a new request after 3 s, then 6 s, then 6 s. Failures with a known final outcome still stop at once: user aborts, compaction timeouts, context overflow, authentication errors (including 401 and 403), output-length and structured-output errors, content filtering, and rejected reasoning state. The existing check that blocks a retry once a tool has run, or may have run, applies first and is unchanged. A request the provider rejects with an unrecognised 4xx error now fails after about 15 seconds instead of at once. + +--- + +### ๐Ÿงช Test Summary + +```text +Local toolchain: Bun 1.4.3 (pinned 1.4.2; CI uses the pin) +Session tests: 993 passed, 1 failed + (the signed-reasoning prefix test failed once under full-suite load and passes alone, with or without this change) +Retry unit tests: 74 passed, 0 failed +Processor effect tests: 29 passed, 0 failed +Runtime typecheck: passed +``` + +--- + +### ๐Ÿ” Verification + +- Required merge checks are Typecheck, Unit Tests (linux), E2E Tests (linux), and E2E Tests (windows). +- The release pipeline validates template provenance, three-platform CLI builds, executable and install smoke checks, release notes, and SHA256SUMS before publication. +- No dependency, runtime-pin, HTTP, event, or database changes are included. Retry limits, delays and `Retry-After` handling for errors that a rule recognises are unchanged. +- Regression coverage includes circuit-breaker text in messages and in a 400 response, the 3-retry cap with 3 s/6 s/6 s waits checked under a test clock, and terminal errors that stay out of the fallback. Live gateway behaviour was not exercised. + +--- + +**Full changelog:** [`graphagent-v1.0.89`...`graphagent-v1.0.90`](https://github.com/LeXwDeX/GraphAgent/compare/graphagent-v1.0.89...graphagent-v1.0.90) From e5e44e573742003c9ce7a2b387d4003c8605eaab Mon Sep 17 00:00:00 2001 From: Lex <14139451+LeXwDeX@users.noreply.github.com> Date: Tue, 27 Oct 2026 23:37:39 +0800 Subject: [PATCH 4/4] fix: keep 4xx responses out of the reconnect fallback A 4xx is the provider's answer to the request; resending cannot change it and delayed StopFailure hooks. The fallback now covers failures without an HTTP status, such as dropped WebSockets and transport errors. Issue: #900 --- .github/releases/v1.0.90.md | 6 ++++-- packages/opencode/src/session/retry.ts | 5 +++-- .../opencode/test/session/processor-effect.test.ts | 10 ++++------ packages/opencode/test/session/retry.test.ts | 1 + 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/releases/v1.0.90.md b/.github/releases/v1.0.90.md index 4c6277888..192ef733c 100644 --- a/.github/releases/v1.0.90.md +++ b/.github/releases/v1.0.90.md @@ -7,7 +7,7 @@ Stable release from `main` branch. Retry gateway circuit-breaker rejections and ### ๐Ÿ› Bug Fixes - **Retry gateway circuit-breaker rejections (#900).** Gateways that trip every upstream target reply with text such as `all targets recovering (half-open probe in flight)`, often with a 4xx status. These replies now count as transient and use the normal retry backoff, so parallel subagents and DAG workers wait for the breaker to close instead of failing at once. `half-open`, `circuit open` and `circuit breaker is open` are matched too. -- **Reconnect after unrecognised failures (#900).** A failure that matches no retry rule, such as an unexpected WebSocket or transport error, is now retried up to 3 times. Each retry sends a new request after 3 s, then 6 s, then 6 s. Failures with a known final outcome still stop at once: user aborts, compaction timeouts, context overflow, authentication errors (including 401 and 403), output-length and structured-output errors, content filtering, and rejected reasoning state. The existing check that blocks a retry once a tool has run, or may have run, applies first and is unchanged. A request the provider rejects with an unrecognised 4xx error now fails after about 15 seconds instead of at once. +- **Reconnect after unrecognised failures (#900).** A failure that matches no retry rule and has no HTTP status, such as a dropped WebSocket, a broken stream or an unexpected transport error, is now retried up to 3 times. Each retry sends a new request after 3 s, then 6 s, then 6 s. Failures with a known final outcome still stop at once: any 4xx response, user aborts, compaction timeouts, context overflow, authentication errors, output-length and structured-output errors, content filtering, and rejected reasoning state. The existing check that blocks a retry once a tool has run, or may have run, applies first and is unchanged. --- @@ -19,6 +19,8 @@ Session tests: 993 passed, 1 failed (the signed-reasoning prefix test failed once under full-suite load and passes alone, with or without this change) Retry unit tests: 74 passed, 0 failed Processor effect tests: 29 passed, 0 failed +StopFailure hook tests (lifecycle, admission, subagent routing): 24 passed, 1 failed + (the Stop asyncRewake chain test hits its 5 s timeout locally without this change too; it passes in CI) Runtime typecheck: passed ``` @@ -29,7 +31,7 @@ Runtime typecheck: passed - Required merge checks are Typecheck, Unit Tests (linux), E2E Tests (linux), and E2E Tests (windows). - The release pipeline validates template provenance, three-platform CLI builds, executable and install smoke checks, release notes, and SHA256SUMS before publication. - No dependency, runtime-pin, HTTP, event, or database changes are included. Retry limits, delays and `Retry-After` handling for errors that a rule recognises are unchanged. -- Regression coverage includes circuit-breaker text in messages and in a 400 response, the 3-retry cap with 3 s/6 s/6 s waits checked under a test clock, and terminal errors that stay out of the fallback. Live gateway behaviour was not exercised. +- Regression coverage includes circuit-breaker text in messages and in a 400 response, the 3-retry cap with 3 s/6 s/6 s waits checked under a test clock, and terminal errors, including a 400, that stay out of the fallback. StopFailure hooks still receive an immediate 400 failure. Live gateway behaviour was not exercised. --- diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index 4d311fa96..ae3ec81f7 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -208,7 +208,8 @@ export function retryable(error: Err, provider: string) { /** * Unrecognised failures get a bounded reconnect attempt. Failures with a known - * terminal meaning stay terminal: retrying them cannot change the outcome. + * terminal meaning stay terminal, and so does any 4xx: the provider answered + * the request and a resend cannot change that answer. */ export function fallback(error: Err): Retryable | undefined { if ( @@ -222,7 +223,7 @@ export function fallback(error: Err): Retryable | undefined { return undefined if (SessionV1.APIError.isInstance(error)) { const status = error.data.statusCode - if (status === 401 || status === 403) return undefined + if (status !== undefined && status >= 400 && status < 500) return undefined if (error.data.message === REASONING_STATE_MESSAGE) return undefined } const message = isRecord(error.data) ? error.data.message : undefined diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index ea5cd9ecd..9beaea5a7 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -14,7 +14,6 @@ import { Session } from "@/session/session" import { LLM } from "../../src/session/llm" import { MessageV2 } from "../../src/session/message-v2" import { SessionProcessor } from "../../src/session/processor" -import { SessionRetry } from "../../src/session/retry" import { MessageID, PartID, SessionID } from "../../src/session/schema" import { SessionStatus } from "../../src/session/status" import { SessionSummary } from "../../src/session/summary" @@ -555,14 +554,13 @@ it.live("session.processor effect tests reset reasoning state across retries", ( ), ) -it.live("session.processor effect tests reconnect unknown json errors three times before stopping", () => +it.live("session.processor effect tests do not retry unknown json errors", () => provideTmpdirServer( ({ dir, llm }) => Effect.gen(function* () { const { processors, session, provider } = yield* boot() - for (let i = 0; i <= SessionRetry.FALLBACK_MAX_RETRIES; i++) - yield* llm.error(400, { error: { message: "no_kv_space" } }) + yield* llm.error(400, { error: { message: "no_kv_space" } }) const chat = yield* session.create({}) const parent = yield* user(chat.id, "json") @@ -592,12 +590,11 @@ it.live("session.processor effect tests reconnect unknown json errors three time }) expect(value).toBe("stop") - expect(yield* llm.calls).toBe(SessionRetry.FALLBACK_MAX_RETRIES + 1) + expect(yield* llm.calls).toBe(1) expect(handle.message.error?.name).toBe("APIError") }), { config: (url) => providerCfg(url) }, ), - 30_000, ) it.live("session.processor effect tests retry recognized structured json errors", () => @@ -1609,6 +1606,7 @@ itFragmentFailure.live("session.processor effect tests flush partial v2 fragment }), { config: cfg }, ), + // The unclassified provider error goes through the 3s/6s/6s reconnect fallback first. 30_000, ) diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index c49d16b02..7f9938a3d 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -185,6 +185,7 @@ describe("session.retry.delay", () => { new SessionV1.AuthError({ providerID: "test", message: "bad key" }).toObject(), new SessionV1.ContentFilterError({ message: "filtered" }).toObject(), new SessionV1.APIError({ message: "denied", isRetryable: false, statusCode: 401 }).toObject(), + new SessionV1.APIError({ message: "no_kv_space", isRetryable: false, statusCode: 400 }).toObject(), new SessionV1.APIError({ message: ProviderError.REASONING_STATE_MESSAGE, isRetryable: false }).toObject(), ] expect(terminal.map(SessionRetry.fallback)).toStrictEqual(terminal.map(() => undefined))