diff --git a/.github/releases/v1.0.90.md b/.github/releases/v1.0.90.md new file mode 100644 index 000000000..192ef733c --- /dev/null +++ b/.github/releases/v1.0.90.md @@ -0,0 +1,38 @@ +## opencode 1.0.90 + +Stable release from `main` branch. Retry gateway circuit-breaker rejections and reconnect after unrecognised request failures. + +--- + +### ๐Ÿ› Bug Fixes + +- **Retry gateway circuit-breaker rejections (#900).** Gateways that trip every upstream target reply with text such as `all targets recovering (half-open probe in flight)`, often with a 4xx status. These replies now count as transient and use the normal retry backoff, so parallel subagents and DAG workers wait for the breaker to close instead of failing at once. `half-open`, `circuit open` and `circuit breaker is open` are matched too. +- **Reconnect after unrecognised failures (#900).** A failure that matches no retry rule and has no HTTP status, such as a dropped WebSocket, a broken stream or an unexpected transport error, is now retried up to 3 times. Each retry sends a new request after 3 s, then 6 s, then 6 s. Failures with a known final outcome still stop at once: any 4xx response, user aborts, compaction timeouts, context overflow, authentication errors, output-length and structured-output errors, content filtering, and rejected reasoning state. The existing check that blocks a retry once a tool has run, or may have run, applies first and is unchanged. + +--- + +### ๐Ÿงช Test Summary + +```text +Local toolchain: Bun 1.4.3 (pinned 1.4.2; CI uses the pin) +Session tests: 993 passed, 1 failed + (the signed-reasoning prefix test failed once under full-suite load and passes alone, with or without this change) +Retry unit tests: 74 passed, 0 failed +Processor effect tests: 29 passed, 0 failed +StopFailure hook tests (lifecycle, admission, subagent routing): 24 passed, 1 failed + (the Stop asyncRewake chain test hits its 5 s timeout locally without this change too; it passes in CI) +Runtime typecheck: passed +``` + +--- + +### ๐Ÿ” Verification + +- Required merge checks are Typecheck, Unit Tests (linux), E2E Tests (linux), and E2E Tests (windows). +- The release pipeline validates template provenance, three-platform CLI builds, executable and install smoke checks, release notes, and SHA256SUMS before publication. +- No dependency, runtime-pin, HTTP, event, or database changes are included. Retry limits, delays and `Retry-After` handling for errors that a rule recognises are unchanged. +- Regression coverage includes circuit-breaker text in messages and in a 400 response, the 3-retry cap with 3 s/6 s/6 s waits checked under a test clock, and terminal errors, including a 400, that stay out of the fallback. StopFailure hooks still receive an immediate 400 failure. Live gateway behaviour was not exercised. + +--- + +**Full changelog:** [`graphagent-v1.0.89`...`graphagent-v1.0.90`](https://github.com/LeXwDeX/GraphAgent/compare/graphagent-v1.0.89...graphagent-v1.0.90) diff --git a/packages/opencode/src/session/retry.ts b/packages/opencode/src/session/retry.ts index 829dab4af..ae3ec81f7 100644 --- a/packages/opencode/src/session/retry.ts +++ b/packages/opencode/src/session/retry.ts @@ -3,6 +3,7 @@ import { SessionV1 } from "@opencode-ai/core/v1/session" import { Cause, Clock, Duration, Effect, Schedule, Schema } from "effect" import { iife } from "@/util/iife" import { isRecord } from "@/util/record" +import { REASONING_STATE_MESSAGE } from "@/provider/error" export type Err = ReturnType @@ -60,6 +61,11 @@ export const RETRY_MAX_DELAY_NO_HEADERS = 30_000 // 30 seconds export const RETRY_MAX_DELAY = 2_147_483_647 // max 32-bit signed integer for setTimeout export const RETRY_MAX_RETRIES = 5 +// Catch-all reconnect for failures no rule recognises: 3s, 6s, 6s. +export const FALLBACK_INITIAL_DELAY = 3000 +export const FALLBACK_MAX_DELAY = 6000 +export const FALLBACK_MAX_RETRIES = 3 + const RETRYABLE_MESSAGE_PATTERNS = [ // Bounded so token counts or IDs in a 4xx body ("150000 tokens") never read as a status. /\b(?:429|500|502|503|504|524)\b/i, @@ -69,6 +75,8 @@ const RETRYABLE_MESSAGE_PATTERNS = [ /^timeout$|\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b/i, /try your request again|retry your request|resource exhausted|resource_exhausted/i, /\btry again (?:later|in\b)|\b(?:currently|temporarily) at capacity\b/i, + // Gateway circuit breakers reject while every upstream target is tripped, often with a 4xx status. + /\ball targets recovering\b|\bhalf[-_\s]open\b|\bcircuit(?:[-_\s]breaker)?(?:\s+is)?[-_\s]open\b/i, ] function cap(ms: number) { @@ -108,6 +116,10 @@ export function delay(attempt: number, error?: SessionV1.APIError, random = Math return cap(Math.min(exponential(attempt, random), RETRY_MAX_DELAY_NO_HEADERS)) } +export function fallbackDelay(attempt: number) { + return Math.min(FALLBACK_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1), FALLBACK_MAX_DELAY) +} + function exponential(attempt: number, random: number) { const base = RETRY_INITIAL_DELAY * Math.pow(RETRY_BACKOFF_FACTOR, attempt - 1) return Math.ceil(base + base * RETRY_JITTER_FACTOR * random) @@ -194,6 +206,30 @@ export function retryable(error: Err, provider: string) { return undefined } +/** + * Unrecognised failures get a bounded reconnect attempt. Failures with a known + * terminal meaning stay terminal, and so does any 4xx: the provider answered + * the request and a resend cannot change that answer. + */ +export function fallback(error: Err): Retryable | undefined { + if ( + SessionV1.AbortedError.isInstance(error) || + SessionV1.ContextOverflowError.isInstance(error) || + SessionV1.AuthError.isInstance(error) || + SessionV1.OutputLengthError.isInstance(error) || + SessionV1.StructuredOutputError.isInstance(error) || + SessionV1.ContentFilterError.isInstance(error) + ) + return undefined + if (SessionV1.APIError.isInstance(error)) { + const status = error.data.statusCode + if (status !== undefined && status >= 400 && status < 500) return undefined + if (error.data.message === REASONING_STATE_MESSAGE) return undefined + } + const message = isRecord(error.data) ? error.data.message : undefined + return { message: typeof message === "string" && message ? message : "Unknown error, reconnecting" } +} + function matchesRetryableMessage(value: unknown) { return typeof value === "string" && RETRYABLE_MESSAGE_PATTERNS.some((pattern) => pattern.test(value)) } @@ -231,11 +267,14 @@ export function policy(opts: { Effect.succeed((meta: Schedule.InputMetadata) => { if (opts.shouldRetry?.(meta.input) === false) return Cause.done(meta.attempt) const error = opts.parse(meta.input) - const retry = retryable(error, opts.provider) + const known = retryable(error, opts.provider) + const retry = known ?? fallback(error) if (!retry) return Cause.done(meta.attempt) - if (meta.attempt > RETRY_MAX_RETRIES) return Cause.done(meta.attempt) + if (meta.attempt > (known ? RETRY_MAX_RETRIES : FALLBACK_MAX_RETRIES)) return Cause.done(meta.attempt) return Effect.gen(function* () { - const wait = delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined) + const wait = known + ? delay(meta.attempt, SessionV1.APIError.isInstance(error) ? error : undefined) + : fallbackDelay(meta.attempt) const now = yield* Clock.currentTimeMillis yield* opts.set({ attempt: meta.attempt, diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 0a3bbac96..9beaea5a7 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -1606,6 +1606,8 @@ itFragmentFailure.live("session.processor effect tests flush partial v2 fragment }), { config: cfg }, ), + // The unclassified provider error goes through the 3s/6s/6s reconnect fallback first. + 30_000, ) const displayLLM = Layer.succeed( diff --git a/packages/opencode/test/session/retry.test.ts b/packages/opencode/test/session/retry.test.ts index 03f671cbf..7f9938a3d 100644 --- a/packages/opencode/test/session/retry.test.ts +++ b/packages/opencode/test/session/retry.test.ts @@ -3,7 +3,8 @@ import { SessionV1 } from "@opencode-ai/core/v1/session" import type { NamedError } from "@opencode-ai/core/util/error" import { APICallError } from "ai" import { setTimeout as sleep } from "node:timers/promises" -import { Effect, Schedule, Schema } from "effect" +import { Clock, Effect, Exit, Fiber, Schedule, Schema } from "effect" +import * as TestClock from "effect/testing/TestClock" import { LayerNode } from "@opencode-ai/core/effect/layer-node" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { SessionRetry } from "../../src/session/retry" @@ -146,6 +147,56 @@ describe("session.retry.delay", () => { expect(attempts).toStrictEqual([1, 2, 3, 4, 5]) }), ) + + test("policy reconnects unrecognised failures up to three times", () => + Effect.gen(function* () { + const waits: number[] = [] + let calls = 0 + const fiber = yield* Effect.suspend(() => { + calls++ + return Effect.fail(new Error("socket went sideways")) + }).pipe( + Effect.retry( + SessionRetry.policy({ + provider: "test", + parse: (error) => MessageV2.fromError(error, { providerID }), + set: (info) => + Effect.gen(function* () { + waits.push(info.next - (yield* Clock.currentTimeMillis)) + }), + }), + ), + Effect.exit, + Effect.forkChild, + ) + yield* TestClock.adjust("30 seconds") + const exit = yield* Fiber.join(fiber) + + expect(Exit.isFailure(exit)).toBe(true) + expect(calls).toBe(SessionRetry.FALLBACK_MAX_RETRIES + 1) + expect(waits).toStrictEqual([3000, 6000, 6000]) + }).pipe(Effect.provide(TestClock.layer()), Effect.runPromise), + ) + + test("keeps terminal failures out of the fallback", () => { + const terminal = [ + new SessionV1.AbortedError({ message: "Aborted" }).toObject(), + new SessionV1.ContextOverflowError({ message: "too long" }).toObject(), + new SessionV1.AuthError({ providerID: "test", message: "bad key" }).toObject(), + new SessionV1.ContentFilterError({ message: "filtered" }).toObject(), + new SessionV1.APIError({ message: "denied", isRetryable: false, statusCode: 401 }).toObject(), + new SessionV1.APIError({ message: "no_kv_space", isRetryable: false, statusCode: 400 }).toObject(), + new SessionV1.APIError({ message: ProviderError.REASONING_STATE_MESSAGE, isRetryable: false }).toObject(), + ] + expect(terminal.map(SessionRetry.fallback)).toStrictEqual(terminal.map(() => undefined)) + expect(SessionRetry.fallback(wrap("socket went sideways"))).toEqual({ message: "socket went sideways" }) + }) +}) + +describe("session.retry.fallbackDelay", () => { + test("backs off from 3 seconds and caps at 6 seconds", () => { + expect([1, 2, 3].map(SessionRetry.fallbackDelay)).toStrictEqual([3000, 6000, 6000]) + }) }) describe("session.retry.retryable", () => { @@ -222,6 +273,11 @@ describe("session.retry.retryable", () => { "The model is currently at capacity due to high demand", "The service is temporarily at capacity", "upstream returned status 524", + "all targets recovering (half-open probe in flight)", + "upstream target is half-open", + "circuit breaker open", + "Circuit breaker is open for this channel", + "circuit open", ])("retries matching API error text: %s", (message) => { expect(SessionRetry.retryable(wrap(message), retryProvider)).toEqual({ message }) }) @@ -244,6 +300,19 @@ describe("session.retry.retryable", () => { expect(SessionRetry.retryable(error, retryProvider)).toEqual({ message: "Request failed" }) }) + test("retries 4xx circuit-breaker rejections from gateways", () => { + const error = Schema.decodeUnknownSync(SessionV1.APIError.Schema)( + new SessionV1.APIError({ + message: "all targets recovering (half-open probe in flight)", + isRetryable: false, + statusCode: 400, + }).toObject(), + ) + expect(SessionRetry.retryable(error, retryProvider)).toEqual({ + message: "all targets recovering (half-open probe in flight)", + }) + }) + test("retries transport timeout errors", () => { const request = MessageV2.fromError(new ProviderError.HeaderTimeoutError(10000), { providerID }) expect(SessionV1.APIError.isInstance(request)).toBe(true)