diff --git a/CLAUDE.md b/CLAUDE.md index 22fa724..a098fa3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -135,6 +135,14 @@ kill rm -f ~/.nxd/nxd.lock ~/.nxd/events.jsonl ~/.nxd/nxd.db ``` +## Current State (2026-06-24) — Ollama capacity clean pause + +- **Transient Ollama overload no longer burns the escalation chain.** Running several concurrent builds against one Ollama instance, the server returns transient overload/loading/OOM conditions (HTTP 429/503, "server busy", "no slots available", "model is loading", "out of memory", "context deadline exceeded", "connection refused"). Previously these were treated as story-quality failures and the requirement only paused *after* burning reset → manager → tech-lead re-plan, with misleading messages ("agent produced no code changes", "re-plan failed"). +- **Fix (`internal/llm/errors.go` + `internal/engine/capacity_pause.go`):** + - `llm.IsCapacityError(err)` classifies 429/503/529 (typed `*APIError`) **and** stringified Ollama HTTP-client errors carrying a capacity signature. `llm.ContainsCapacitySignature(string)` is the shared vocabulary. Capacity is distinct from `IsFatalAPIError` (401/403/billing — permanent) and from a 404 model-not-found (operator config error). Signatures are the strings `internal/llm/ollama.go` and the Ollama server actually emit. + - `Monitor.pauseIfCapacity(storyID, stage, err)` pauses the requirement **without consuming an escalation attempt or advancing the tier**. Wired at: reviewer, merge/conflict-resolution, manager diagnosis, tech-lead re-plan. `agentCompletionHasCapacityError(storyID)` scans the latest `STORY_COMPLETED` payload's recorded `error` field so a native (Gemma) agent that hit an overload (empty diff) pauses cleanly instead of escalating as "no code changes". The pause reason states the cause is transient — resume after the server recovers. +- Tests: `internal/llm/capacity_test.go`, the overload case in `internal/llm/ollama_test.go`, and `internal/engine/capacity_pause_test.go` (unit + post-execution review/empty-diff paths). + ## Current State (2026-06-24) — audit hardening - **Planner degenerate-plan guard**: `Planner.plan` (`internal/engine/planner.go`) now rejects an empty story list and any story with an empty id/title *before* emitting REQ_PLANNED/STORY_CREATED, so a requirement can no longer be stranded with nothing to dispatch. Tests: `internal/engine/planner_validation_test.go`. diff --git a/internal/engine/capacity_pause.go b/internal/engine/capacity_pause.go new file mode 100644 index 0000000..742cbcf --- /dev/null +++ b/internal/engine/capacity_pause.go @@ -0,0 +1,64 @@ +package engine + +import ( + "fmt" + "log" + + "github.com/tzone85/nexus-dispatch/internal/llm" + "github.com/tzone85/nexus-dispatch/internal/state" +) + +// capacityPauseReason builds the standardized pause reason for a transient +// capacity / overload exhaustion at a given pipeline stage. The phrasing tells +// the operator the failure is transient and that resuming after the Ollama +// server recovers will continue from where it left off. +func capacityPauseReason(stage string, err error) string { + return fmt.Sprintf( + "Ollama capacity/overload during %s — transient, resume after the server recovers (free a GPU slot / let the model finish loading / free memory): %v", + stage, err, + ) +} + +// pauseIfCapacity inspects an LLM-call error. If it is a transient capacity +// exhaustion (HTTP 429/503/529, server busy, no slots, model loading, OOM, +// connection refused, context deadline), it pauses the requirement cleanly and +// returns true — WITHOUT consuming an escalation attempt or advancing the tier, +// since the failure has nothing to do with the story's quality and will succeed +// once the Ollama server recovers. +// +// Returns false for every other error so the caller's normal handling runs. +func (m *Monitor) pauseIfCapacity(storyID, stage string, err error) bool { + if !llm.IsCapacityError(err) { + return false + } + log.Printf("[pipeline] Ollama capacity/overload during %s for %s — pausing without escalation: %v", + stage, storyID, err) + m.pauseRequirement(storyID, capacityPauseReason(stage, err)) + return true +} + +// agentCompletionHasCapacityError reports whether the most recent +// STORY_COMPLETED event for a story carries a capacity/overload signature in +// its recorded error envelope. Native (Gemma) agents run in-process and call +// Ollama directly; when the server is overloaded the agent's completion call +// fails, the executor records the error in the STORY_COMPLETED payload, and the +// agent produces an empty diff. Without this scan a capacity-limited agent looks +// identical to a lazy agent and the story is wrongly escalated as "produced no +// code changes". The error envelope is the only evidence of the transient cause. +func (m *Monitor) agentCompletionHasCapacityError(storyID string) bool { + events, err := m.eventStore.List(state.EventFilter{ + Type: state.EventStoryCompleted, + StoryID: storyID, + }) + if err != nil || len(events) == 0 { + return false + } + // Scan the latest completion event's recorded error. + latest := events[len(events)-1] + payload := state.DecodePayload(latest.Payload) + errText, _ := payload["error"].(string) + if errText == "" { + return false + } + return llm.ContainsCapacitySignature(errText) +} diff --git a/internal/engine/capacity_pause_test.go b/internal/engine/capacity_pause_test.go new file mode 100644 index 0000000..f4af34c --- /dev/null +++ b/internal/engine/capacity_pause_test.go @@ -0,0 +1,283 @@ +package engine + +import ( + "context" + "fmt" + "os" + "os/exec" + "testing" + + "github.com/tzone85/nexus-dispatch/internal/config" + "github.com/tzone85/nexus-dispatch/internal/llm" + "github.com/tzone85/nexus-dispatch/internal/state" +) + +// capacityTestStores builds in-memory event + projection stores for the +// capacity-pause unit tests. +func capacityTestStores(t *testing.T) (state.EventStore, state.ProjectionStore) { + t.Helper() + es, err := state.NewFileStore(t.TempDir() + "/events.jsonl") + if err != nil { + t.Fatalf("event store: %v", err) + } + ps, err := state.NewSQLiteStore(":memory:") + if err != nil { + t.Fatalf("proj store: %v", err) + } + t.Cleanup(func() { + es.Close() + ps.Close() + }) + return es, ps +} + +func seedCapacityStory(t *testing.T, es state.EventStore, ps state.ProjectionStore, reqID, storyID string) { + t.Helper() + reqEvt := state.NewEvent(state.EventReqSubmitted, "cli", "", map[string]any{ + "id": reqID, "title": "Req", "description": "desc", + }) + if err := es.Append(reqEvt); err != nil { + t.Fatal(err) + } + if err := ps.Project(reqEvt); err != nil { + t.Fatal(err) + } + storyEvt := state.NewEvent(state.EventStoryCreated, "tl", storyID, map[string]any{ + "id": storyID, "req_id": reqID, "title": "Task", "description": "d", "complexity": 3, + }) + if err := es.Append(storyEvt); err != nil { + t.Fatal(err) + } + if err := ps.Project(storyEvt); err != nil { + t.Fatal(err) + } +} + +// pauseIfCapacity must pause the requirement WITHOUT emitting any escalation / +// review-failed event when the error is a transient capacity exhaustion, and +// must return false (leaving the caller's normal handling to run) otherwise. +func TestPauseIfCapacity(t *testing.T) { + t.Run("capacity error pauses cleanly", func(t *testing.T) { + es, ps := capacityTestStores(t) + seedCapacityStory(t, es, ps, "r-cap", "s-cap") + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps) + + capErr := fmt.Errorf(`ollama API error (status 503): {"error":"server overloaded, please retry shortly"}`) + if !m.pauseIfCapacity("s-cap", "review", capErr) { + t.Fatal("pauseIfCapacity returned false for a capacity error") + } + + paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused}) + if len(paused) != 1 { + t.Errorf("expected 1 REQ_PAUSED, got %d", len(paused)) + } + // Must NOT burn an escalation tier. + failed, _ := es.List(state.EventFilter{Type: state.EventStoryReviewFailed, StoryID: "s-cap"}) + if len(failed) != 0 { + t.Errorf("capacity pause must not emit STORY_REVIEW_FAILED; got %d", len(failed)) + } + esc, _ := es.List(state.EventFilter{Type: state.EventStoryEscalated, StoryID: "s-cap"}) + if len(esc) != 0 { + t.Errorf("capacity pause must not emit STORY_ESCALATED; got %d", len(esc)) + } + }) + + t.Run("ordinary error returns false", func(t *testing.T) { + es, ps := capacityTestStores(t) + seedCapacityStory(t, es, ps, "r-ord", "s-ord") + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps) + + if m.pauseIfCapacity("s-ord", "review", fmt.Errorf("undefined: Foo")) { + t.Fatal("pauseIfCapacity returned true for an ordinary error") + } + paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused}) + if len(paused) != 0 { + t.Errorf("ordinary error must not pause; got %d REQ_PAUSED", len(paused)) + } + }) + + t.Run("nil error returns false", func(t *testing.T) { + es, ps := capacityTestStores(t) + seedCapacityStory(t, es, ps, "r-nil", "s-nil") + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps) + if m.pauseIfCapacity("s-nil", "review", nil) { + t.Fatal("pauseIfCapacity returned true for a nil error") + } + }) +} + +// agentCompletionHasCapacityError scans the latest STORY_COMPLETED event's +// recorded error envelope. A native (Gemma) agent that hit an Ollama overload +// still emits STORY_COMPLETED (with the error in the payload) and produces an +// empty diff — without this scan it would look identical to a lazy agent and be +// wrongly escalated as "produced no code changes". +func TestAgentCompletionHasCapacityError(t *testing.T) { + t.Run("detects capacity error in completion payload", func(t *testing.T) { + es, ps := capacityTestStores(t) + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps) + evt := state.NewEvent(state.EventStoryCompleted, "agent-1", "s-1", map[string]any{ + "native": true, + "error": "llm completion (iteration 2): ollama API error (status 503): server busy, please try again", + }) + es.Append(evt) + ps.Project(evt) + + if !m.agentCompletionHasCapacityError("s-1") { + t.Error("expected capacity error to be detected in completion payload") + } + }) + + t.Run("ignores ordinary completion error", func(t *testing.T) { + es, ps := capacityTestStores(t) + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps) + evt := state.NewEvent(state.EventStoryCompleted, "agent-1", "s-2", map[string]any{ + "native": true, + "error": "llm completion (iteration 1): undefined: Foo", + }) + es.Append(evt) + ps.Project(evt) + + if m.agentCompletionHasCapacityError("s-2") { + t.Error("ordinary completion error must not be detected as capacity") + } + }) + + t.Run("no completion event", func(t *testing.T) { + es, ps := capacityTestStores(t) + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps) + if m.agentCompletionHasCapacityError("s-missing") { + t.Error("missing completion event must not be detected as capacity") + } + }) +} + +// makeWorktreeWithDiff builds a git repo with a committed change on a feature +// branch so the post-execution pipeline sees a non-empty diff and proceeds to +// the review stage. +func makeWorktreeWithDiff(t *testing.T) string { + t.Helper() + dir := t.TempDir() + run := func(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v (%s)", args, err, out) + } + } + run("init", "-b", "main") + run("config", "user.email", "t@t.t") + run("config", "user.name", "t") + if err := os.WriteFile(dir+"/base.txt", []byte("base\n"), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", "base") + run("checkout", "-b", "nxd/s-pe-cap") + if err := os.WriteFile(dir+"/feature.txt", []byte("feature\n"), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", "feature") + return dir +} + +// makeEmptyWorktree builds a git repo with a feature branch identical to main +// (no diff), simulating a native agent that produced nothing. +func makeEmptyWorktree(t *testing.T, branch string) string { + t.Helper() + dir := t.TempDir() + run := func(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = dir + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %v: %v (%s)", args, err, out) + } + } + run("init", "-b", "main") + run("config", "user.email", "t@t.t") + run("config", "user.name", "t") + if err := os.WriteFile(dir+"/base.txt", []byte("base\n"), 0o644); err != nil { + t.Fatal(err) + } + run("add", ".") + run("commit", "-m", "base") + run("checkout", "-b", branch) + return dir +} + +// TestPostExecutionPipeline_EmptyDiffCapacity verifies that when a native agent +// produces no diff AND its STORY_COMPLETED payload records an Ollama capacity +// error, the requirement pauses cleanly instead of resetting to draft (which +// would burn an escalation attempt as "produced no code changes"). +func TestPostExecutionPipeline_EmptyDiffCapacity(t *testing.T) { + es, ps := capacityTestStores(t) + seedCapacityStory(t, es, ps, "r-empty", "s-empty") + + // Native agent emitted STORY_COMPLETED carrying an Ollama overload error. + completed := state.NewEvent(state.EventStoryCompleted, "agent-1", "s-empty", map[string]any{ + "native": true, + "error": "llm completion (iteration 3): ollama API error (status 503): server busy, please try again. maximum pending requests exceeded", + }) + es.Append(completed) + ps.Project(completed) + + worktree := makeEmptyWorktree(t, "nxd/s-empty") + m := NewMonitor(nil, nil, nil, nil, nil, config.Config{ + Routing: config.RoutingConfig{MaxRetriesBeforeEscalation: 2}, + }, es, ps) + + ag := ActiveAgent{ + Assignment: Assignment{ + StoryID: "s-empty", AgentID: "agent-1", + SessionName: "nxd-test-empty", Branch: "nxd/s-empty", + }, + WorktreePath: worktree, + } + + m.postExecutionPipeline(context.Background(), ag, worktree) + + paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused}) + if len(paused) < 1 { + t.Error("expected REQ_PAUSED for empty-diff capacity error") + } + failed, _ := es.List(state.EventFilter{Type: state.EventStoryReviewFailed, StoryID: "s-empty"}) + if len(failed) != 0 { + t.Errorf("empty-diff capacity must not emit STORY_REVIEW_FAILED; got %d", len(failed)) + } +} + +// TestPostExecutionPipeline_ReviewCapacity verifies a capacity error during +// review PAUSES the requirement (resume after the server recovers) rather than +// resetting to draft — which would burn an escalation attempt on a transient +// overload the story never had a chance to avoid. +func TestPostExecutionPipeline_ReviewCapacity(t *testing.T) { + es, ps := capacityTestStores(t) + seedCapacityStory(t, es, ps, "r-cap", "s-pe-cap") + + worktree := makeWorktreeWithDiff(t) + + capErr := fmt.Errorf(`reviewer LLM call: ollama API error (status 503): {"error":"server overloaded, please retry shortly"}`) + reviewer := NewReviewer(llm.NewErrorClient(capErr), "ollama", "gemma4", 4000, es, ps) + + cfg := config.Config{Routing: config.RoutingConfig{MaxRetriesBeforeEscalation: 2}} + m := NewMonitor(nil, nil, reviewer, nil, nil, cfg, es, ps) + + ag := ActiveAgent{ + Assignment: Assignment{ + StoryID: "s-pe-cap", AgentID: "agent-1", + SessionName: "nxd-test-cap", Branch: "nxd/s-pe-cap", + }, + WorktreePath: worktree, + } + + m.postExecutionPipeline(context.Background(), ag, worktree) + + paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused}) + if len(paused) < 1 { + t.Error("expected REQ_PAUSED for capacity error during review") + } + failed, _ := es.List(state.EventFilter{Type: state.EventStoryReviewFailed, StoryID: "s-pe-cap"}) + if len(failed) != 0 { + t.Errorf("capacity error must not emit STORY_REVIEW_FAILED (burns escalation); got %d", len(failed)) + } +} diff --git a/internal/engine/monitor.go b/internal/engine/monitor.go index 5aff484..be9e3bc 100644 --- a/internal/engine/monitor.go +++ b/internal/engine/monitor.go @@ -475,6 +475,18 @@ func (m *Monitor) postExecutionPipeline(ctx context.Context, ag ActiveAgent, rep return } if diff == "" { + // A native (Gemma) agent that hit an Ollama overload still emits + // STORY_COMPLETED with the error recorded in its payload and an empty + // diff. Without this check it looks identical to a lazy agent and would + // be escalated as "produced no code changes" — burning a tier on a + // transient limit. Pause cleanly instead so a resume after the server + // recovers re-dispatches the story. + if m.agentCompletionHasCapacityError(storyID) { + log.Printf("[pipeline] %s produced no diff but its agent hit an Ollama capacity limit — pausing without escalation", storyID) + outcomeForRelease = devdb.OutcomePaused + m.pauseRequirement(storyID, capacityPauseReason("agent execution", fmt.Errorf("agent completion recorded an Ollama capacity/overload error"))) + return + } log.Printf("[pipeline] no changes produced for %s, resetting to draft for re-dispatch", storyID) m.resetStoryToDraft(storyID, "monitor", "agent produced no code changes") return @@ -529,6 +541,15 @@ func (m *Monitor) postExecutionPipeline(ctx context.Context, ag ActiveAgent, rep result, err := m.reviewer.Review(ctx, storyID, storyTitle, storyAC, diff, blastRadius, fileTree) if err != nil { EmitStageCompleted(m.eventStore, m.projStore, "monitor", storyID, "review", "failure", reviewStart) + // Transient Ollama capacity/overload (429/503, server busy, no + // slots, model loading, OOM) — pause cleanly WITHOUT burning an + // escalation attempt, since the story never had a chance to fail + // on its own merits and review will succeed once the server + // recovers. + if m.pauseIfCapacity(storyID, "review", err) { + outcomeForRelease = devdb.OutcomePaused + return + } // Fatal API errors (auth failures, billing exhaustion, // permission denied) will never succeed on retry -- pause // the entire requirement to stop the infinite loop. @@ -672,6 +693,12 @@ func (m *Monitor) postExecutionPipeline(ctx context.Context, ag ActiveAgent, rep if err != nil { EmitStageCompleted(m.eventStore, m.projStore, "monitor", storyID, "merge", "failure", mergeStart) + // Transient Ollama capacity/overload during LLM conflict + // resolution — pause cleanly without burning an escalation tier. + if m.pauseIfCapacity(storyID, "merge/conflict-resolution", err) { + outcomeForRelease = devdb.OutcomePaused + return + } // Fatal API errors during conflict resolution (credits exhausted, // auth failure) must pause the requirement immediately. if llm.IsFatalAPIError(err) { @@ -1151,6 +1178,11 @@ func (m *Monitor) handleManagerEscalation(ctx context.Context, story PlannedStor action, err := m.manager.Diagnose(ctx, dc) if err != nil { log.Printf("[manager] diagnosis failed for %s: %v", storyID, err) + // Transient Ollama capacity/overload during manager diagnosis — + // pause cleanly without burning the tier. + if m.pauseIfCapacity(storyID, "manager-diagnosis", err) { + return + } if llm.IsFatalAPIError(err) { m.pauseRequirement(storyID, fmt.Sprintf("fatal API error in manager: %v", err)) return @@ -1384,6 +1416,13 @@ func (m *Monitor) handleTechLeadEscalation(ctx context.Context, story PlannedSto replacements, err := m.planner.RePlan(ctx, storyID, rc.ReqID, failureContext.String()) if err != nil { log.Printf("[tech-lead] re-plan failed for %s: %v", storyID, err) + // Transient Ollama capacity/overload during re-plan — the pause is + // already clean (re-plan failure pauses), but route it through + // pauseIfCapacity so the operator sees the accurate transient reason + // instead of a misleading "re-plan failed". + if m.pauseIfCapacity(storyID, "tech-lead-replan", err) { + return + } m.pauseRequirement(storyID, fmt.Sprintf("tech lead re-plan failed: %v", err)) return } diff --git a/internal/llm/capacity_test.go b/internal/llm/capacity_test.go new file mode 100644 index 0000000..2e73f38 --- /dev/null +++ b/internal/llm/capacity_test.go @@ -0,0 +1,106 @@ +package llm_test + +import ( + "fmt" + "testing" + + "github.com/tzone85/nexus-dispatch/internal/llm" +) + +// IsCapacityError must recognise the transient capacity / overload conditions +// an Ollama backend produces — HTTP 429/503 (typed *APIError) and the +// stringified error envelopes the Ollama HTTP client emits (server busy, no +// slots, model loading, OOM, connection refused, context deadline). These are +// transient: the request succeeds once the server has a free slot / has loaded +// the model / has memory again, so the pipeline must pause-and-resume rather +// than burn the escalation chain or fail the story. +func TestIsCapacityError(t *testing.T) { + tests := []struct { + name string + err error + expect bool + }{ + // Typed APIError status codes from the Ollama HTTP layer. + {name: "429 typed", err: &llm.APIError{StatusCode: 429, Message: "rate limited"}, expect: true}, + {name: "503 typed", err: &llm.APIError{StatusCode: 503, Message: "service unavailable"}, expect: true}, + {name: "529 typed", err: &llm.APIError{StatusCode: 529, Message: "overloaded"}, expect: true}, + {name: "wrapped 503 typed", err: fmt.Errorf("reviewer: %w", &llm.APIError{StatusCode: 503, Message: "x"}), expect: true}, + + // Stringified Ollama HTTP-client errors — the real production failures. + // ollama.go wraps a non-200 status as "ollama API error (status NNN): ". + {name: "ollama 503 string", err: fmt.Errorf(`ollama API error (status 503): {"error":"server overloaded, please retry shortly"}`), expect: true}, + {name: "ollama 429 string", err: fmt.Errorf(`ollama API error (status 429): too many requests`), expect: true}, + {name: "server busy", err: fmt.Errorf(`ollama API error (status 503): server busy, please try again. maximum pending requests exceeded`), expect: true}, + {name: "llm busy no slots", err: fmt.Errorf(`unexpected server status: llm busy - no slots available`), expect: true}, + {name: "server overloaded", err: fmt.Errorf("Server overloaded, please retry shortly"), expect: true}, + {name: "model loading", err: fmt.Errorf("ollama: model is loading"), expect: true}, + {name: "loading model", err: fmt.Errorf("error: loading model gemma4:e4b"), expect: true}, + {name: "out of memory", err: fmt.Errorf(`ollama API error (status 500): {"error":"model requires more system memory than is available"}`), expect: true}, + {name: "out of memory phrase", err: fmt.Errorf("out of memory while loading model"), expect: true}, + {name: "context deadline", err: fmt.Errorf("ollama http request: context deadline exceeded"), expect: true}, + {name: "connection refused", err: fmt.Errorf("ollama connection refused at http://localhost:11434: is Ollama running?"), expect: true}, + {name: "dial tcp", err: fmt.Errorf("ollama http request: dial tcp 127.0.0.1:11434: connect: connection refused"), expect: true}, + {name: "too many requests", err: fmt.Errorf("too many requests"), expect: true}, + {name: "overloaded generic", err: fmt.Errorf("the backend is currently overloaded"), expect: true}, + + // Must NOT classify as capacity — fatal or ordinary failures. + {name: "401 auth typed", err: &llm.APIError{StatusCode: 401, Message: "unauthorized"}, expect: false}, + {name: "400 billing typed", err: &llm.APIError{StatusCode: 400, Message: "credit balance too low"}, expect: false}, + {name: "404 model not found", err: fmt.Errorf(`ollama model "gemma4" not found: pull it with 'ollama pull gemma4'`), expect: false}, + {name: "ordinary compile error", err: fmt.Errorf("undefined: Foo"), expect: false}, + {name: "nil", err: nil, expect: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := llm.IsCapacityError(tc.err); got != tc.expect { + t.Errorf("IsCapacityError(%v) = %v, want %v", tc.err, got, tc.expect) + } + }) + } +} + +// ContainsCapacitySignature is the shared vocabulary scan; it must match the +// raw transient signatures and ignore fatal / ordinary text. +func TestContainsCapacitySignature(t *testing.T) { + hits := []string{ + "server busy", + "no slots available", + "model is loading", + "out of memory", + "connection refused", + "context deadline exceeded", + "OVERLOADED", + "Too Many Requests", + } + for _, s := range hits { + if !llm.ContainsCapacitySignature(s) { + t.Errorf("ContainsCapacitySignature(%q) = false, want true", s) + } + } + + misses := []string{ + "credit balance too low", + "authentication failed", + "undefined: Foo", + "model not found: pull it with 'ollama pull'", + "", + } + for _, s := range misses { + if llm.ContainsCapacitySignature(s) { + t.Errorf("ContainsCapacitySignature(%q) = true, want false", s) + } + } +} + +// A capacity error must never be misclassified as fatal — they take different +// pipeline paths (fatal = give up; capacity = pause-and-resume-after-reset). +func TestCapacityErrorIsNotFatal(t *testing.T) { + overloaded := fmt.Errorf(`ollama API error (status 503): {"error":"server overloaded, please retry shortly"}`) + if llm.IsFatalAPIError(overloaded) { + t.Error("Ollama overload must not be classified as fatal") + } + if !llm.IsCapacityError(overloaded) { + t.Error("Ollama overload must be classified as capacity") + } +} diff --git a/internal/llm/errors.go b/internal/llm/errors.go index 9d94428..90fc533 100644 --- a/internal/llm/errors.go +++ b/internal/llm/errors.go @@ -62,6 +62,71 @@ func IsOverloaded(err error) bool { return apiErr.StatusCode == 529 } +// capacitySignatures are substrings that indicate a transient capacity / +// resource-exhaustion condition the caller should back off from rather than +// treat as a story-quality failure. These are the strings the Ollama HTTP +// client (internal/llm/ollama.go) and the Ollama server itself actually +// produce. Kept lowercase; match against a lowercased error string. +var capacitySignatures = []string{ + "too many requests", // HTTP 429 canonical text + "rate limit", // generic rate limiting + "overloaded", // Ollama "server overloaded, please retry shortly" + "server busy", // Ollama "server busy, please try again" + "llm busy", // Ollama "unexpected server status: llm busy" + "no slots available", // Ollama queue full (OLLAMA_NUM_PARALLEL saturated) + "maximum pending requests exceeded", // Ollama queue depth (OLLAMA_MAX_QUEUE) + "model is loading", // model warm-up in progress + "loading model", // alternate warm-up wording + "out of memory", // OOM — succeeds once another model unloads + "more system memory than is available", // Ollama OOM body + "context deadline exceeded", // per-iteration timeout under server load + "connection refused", // Ollama not reachable / overwhelmed + "dial tcp", // dial failure to the Ollama socket + "(status 503)", // ollama.go envelope: "ollama API error (status 503): ..." + "(status 429)", // ollama.go envelope: rate limited + "(status 529)", // generic overloaded status +} + +// ContainsCapacitySignature reports whether a raw string carries a transient +// capacity / overload / resource-exhaustion signal. Exported so the engine's +// pause path can scan an agent's recorded error envelope with the exact same +// vocabulary the predicate uses. +func ContainsCapacitySignature(s string) bool { + lower := strings.ToLower(s) + for _, sig := range capacitySignatures { + if strings.Contains(lower, sig) { + return true + } + } + return false +} + +// IsCapacityError returns true when the error is a transient capacity / +// resource exhaustion — HTTP 429 (rate limit), 503 (service unavailable / +// overloaded), or 529 — whether it arrived as a typed *APIError or as a +// stringified Ollama error that never got classified. +// +// This is distinct from IsFatalAPIError (401/403/billing — permanent): a +// capacity error WILL succeed once the server frees a slot / loads the model / +// reclaims memory, so the pipeline should pause-and-resume rather than burn the +// escalation chain or fail the story. A 404 (model not found) is NOT capacity — +// it is an operator config error that retrying won't fix. +func IsCapacityError(err error) bool { + if err == nil { + return false + } + var apiErr *APIError + if errors.As(err, &apiErr) { + if apiErr.StatusCode == 429 || apiErr.StatusCode == 503 || apiErr.StatusCode == 529 { + return true + } + } + // Fall back to scanning the error text: the Ollama HTTP client stringifies + // the server's overload / busy / loading / OOM envelope into a plain error + // before it reaches a decision point. + return ContainsCapacitySignature(err.Error()) +} + // IsRetryable returns true when the error is transient and the request can be retried. func IsRetryable(err error) bool { var apiErr *APIError diff --git a/internal/llm/ollama_test.go b/internal/llm/ollama_test.go index 4ce93ce..1b4d8d2 100644 --- a/internal/llm/ollama_test.go +++ b/internal/llm/ollama_test.go @@ -130,6 +130,38 @@ func TestOllamaClient_ModelNotFound(t *testing.T) { } } +// A 503 (overloaded) / 429 (rate limited) from the Ollama server must be +// classified as a transient capacity error so the pipeline pauses-and-resumes +// instead of treating it as a story-quality failure. +func TestOllamaClient_OverloadIsCapacityError(t *testing.T) { + for _, status := range []int{http.StatusServiceUnavailable, http.StatusTooManyRequests} { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + w.Write([]byte(`{"error": "server overloaded, please retry shortly"}`)) + })) + + client := llm.NewOllamaClient("test-model", + llm.WithOllamaBaseURL(server.URL), + llm.WithOllamaTimeout(2*time.Second), + ) + _, err := client.Complete(context.Background(), llm.CompletionRequest{ + Model: "test-model", + Messages: []llm.Message{{Role: llm.RoleUser, Content: "Hi"}}, + }) + server.Close() + + if err == nil { + t.Fatalf("status %d: expected error", status) + } + if !llm.IsCapacityError(err) { + t.Errorf("status %d: IsCapacityError = false, want true (err=%v)", status, err) + } + if llm.IsFatalAPIError(err) { + t.Errorf("status %d: IsFatalAPIError = true, want false (err=%v)", status, err) + } + } +} + func TestOllamaClient_ConnectionRefused(t *testing.T) { // Use a port that nothing is listening on client := llm.NewOllamaClient("test-model",