fix: clean pause on transient Ollama capacity/overload without burning escalation

CLAUDE.md

@@ -135,6 +135,14 @@ kill <stale-pid>
 rm -f ~/.nxd/nxd.lock ~/.nxd/events.jsonl ~/.nxd/nxd.db
 ```
 
+## Current State (2026-06-24) — Ollama capacity clean pause
+
+- **Transient Ollama overload no longer burns the escalation chain.** Running several concurrent builds against one Ollama instance, the server returns transient overload/loading/OOM conditions (HTTP 429/503, "server busy", "no slots available", "model is loading", "out of memory", "context deadline exceeded", "connection refused"). Previously these were treated as story-quality failures and the requirement only paused *after* burning reset → manager → tech-lead re-plan, with misleading messages ("agent produced no code changes", "re-plan failed").
+- **Fix (`internal/llm/errors.go` + `internal/engine/capacity_pause.go`):**
+  - `llm.IsCapacityError(err)` classifies 429/503/529 (typed `*APIError`) **and** stringified Ollama HTTP-client errors carrying a capacity signature. `llm.ContainsCapacitySignature(string)` is the shared vocabulary. Capacity is distinct from `IsFatalAPIError` (401/403/billing — permanent) and from a 404 model-not-found (operator config error). Signatures are the strings `internal/llm/ollama.go` and the Ollama server actually emit.
+  - `Monitor.pauseIfCapacity(storyID, stage, err)` pauses the requirement **without consuming an escalation attempt or advancing the tier**. Wired at: reviewer, merge/conflict-resolution, manager diagnosis, tech-lead re-plan. `agentCompletionHasCapacityError(storyID)` scans the latest `STORY_COMPLETED` payload's recorded `error` field so a native (Gemma) agent that hit an overload (empty diff) pauses cleanly instead of escalating as "no code changes". The pause reason states the cause is transient — resume after the server recovers.
+- Tests: `internal/llm/capacity_test.go`, the overload case in `internal/llm/ollama_test.go`, and `internal/engine/capacity_pause_test.go` (unit + post-execution review/empty-diff paths).
+
 ## Current State (2026-06-24) — audit hardening
 
 - **Planner degenerate-plan guard**: `Planner.plan` (`internal/engine/planner.go`) now rejects an empty story list and any story with an empty id/title *before* emitting REQ_PLANNED/STORY_CREATED, so a requirement can no longer be stranded with nothing to dispatch. Tests: `internal/engine/planner_validation_test.go`.

internal/engine/capacity_pause.go

@@ -0,0 +1,64 @@
+package engine
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/tzone85/nexus-dispatch/internal/llm"
+	"github.com/tzone85/nexus-dispatch/internal/state"
+)
+
+// capacityPauseReason builds the standardized pause reason for a transient
+// capacity / overload exhaustion at a given pipeline stage. The phrasing tells
+// the operator the failure is transient and that resuming after the Ollama
+// server recovers will continue from where it left off.
+func capacityPauseReason(stage string, err error) string {
+	return fmt.Sprintf(
+		"Ollama capacity/overload during %s — transient, resume after the server recovers (free a GPU slot / let the model finish loading / free memory): %v",
+		stage, err,
+	)
+}
+
+// pauseIfCapacity inspects an LLM-call error. If it is a transient capacity
+// exhaustion (HTTP 429/503/529, server busy, no slots, model loading, OOM,
+// connection refused, context deadline), it pauses the requirement cleanly and
+// returns true — WITHOUT consuming an escalation attempt or advancing the tier,
+// since the failure has nothing to do with the story's quality and will succeed
+// once the Ollama server recovers.
+//
+// Returns false for every other error so the caller's normal handling runs.
+func (m *Monitor) pauseIfCapacity(storyID, stage string, err error) bool {
+	if !llm.IsCapacityError(err) {
+		return false
+	}
+	log.Printf("[pipeline] Ollama capacity/overload during %s for %s — pausing without escalation: %v",
+		stage, storyID, err)
+	m.pauseRequirement(storyID, capacityPauseReason(stage, err))
+	return true
+}
+
+// agentCompletionHasCapacityError reports whether the most recent
+// STORY_COMPLETED event for a story carries a capacity/overload signature in
+// its recorded error envelope. Native (Gemma) agents run in-process and call
+// Ollama directly; when the server is overloaded the agent's completion call
+// fails, the executor records the error in the STORY_COMPLETED payload, and the
+// agent produces an empty diff. Without this scan a capacity-limited agent looks
+// identical to a lazy agent and the story is wrongly escalated as "produced no
+// code changes". The error envelope is the only evidence of the transient cause.
+func (m *Monitor) agentCompletionHasCapacityError(storyID string) bool {
+	events, err := m.eventStore.List(state.EventFilter{
+		Type:    state.EventStoryCompleted,
+		StoryID: storyID,
+	})
+	if err != nil || len(events) == 0 {
+		return false
+	}
+	// Scan the latest completion event's recorded error.
+	latest := events[len(events)-1]
+	payload := state.DecodePayload(latest.Payload)
+	errText, _ := payload["error"].(string)
+	if errText == "" {
+		return false
+	}
+	return llm.ContainsCapacitySignature(errText)
+}

internal/engine/capacity_pause_test.go

@@ -0,0 +1,283 @@
+package engine
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+
+	"github.com/tzone85/nexus-dispatch/internal/config"
+	"github.com/tzone85/nexus-dispatch/internal/llm"
+	"github.com/tzone85/nexus-dispatch/internal/state"
+)
+
+// capacityTestStores builds in-memory event + projection stores for the
+// capacity-pause unit tests.
+func capacityTestStores(t *testing.T) (state.EventStore, state.ProjectionStore) {
+	t.Helper()
+	es, err := state.NewFileStore(t.TempDir() + "/events.jsonl")
+	if err != nil {
+		t.Fatalf("event store: %v", err)
+	}
+	ps, err := state.NewSQLiteStore(":memory:")
+	if err != nil {
+		t.Fatalf("proj store: %v", err)
+	}
+	t.Cleanup(func() {
+		es.Close()
+		ps.Close()
+	})
+	return es, ps
+}
+
+func seedCapacityStory(t *testing.T, es state.EventStore, ps state.ProjectionStore, reqID, storyID string) {
+	t.Helper()
+	reqEvt := state.NewEvent(state.EventReqSubmitted, "cli", "", map[string]any{
+		"id": reqID, "title": "Req", "description": "desc",
+	})
+	if err := es.Append(reqEvt); err != nil {
+		t.Fatal(err)
+	}
+	if err := ps.Project(reqEvt); err != nil {
+		t.Fatal(err)
+	}
+	storyEvt := state.NewEvent(state.EventStoryCreated, "tl", storyID, map[string]any{
+		"id": storyID, "req_id": reqID, "title": "Task", "description": "d", "complexity": 3,
+	})
+	if err := es.Append(storyEvt); err != nil {
+		t.Fatal(err)
+	}
+	if err := ps.Project(storyEvt); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// pauseIfCapacity must pause the requirement WITHOUT emitting any escalation /
+// review-failed event when the error is a transient capacity exhaustion, and
+// must return false (leaving the caller's normal handling to run) otherwise.
+func TestPauseIfCapacity(t *testing.T) {
+	t.Run("capacity error pauses cleanly", func(t *testing.T) {
+		es, ps := capacityTestStores(t)
+		seedCapacityStory(t, es, ps, "r-cap", "s-cap")
+		m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps)
+
+		capErr := fmt.Errorf(`ollama API error (status 503): {"error":"server overloaded, please retry shortly"}`)
+		if !m.pauseIfCapacity("s-cap", "review", capErr) {
+			t.Fatal("pauseIfCapacity returned false for a capacity error")
+		}
+
+		paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused})
+		if len(paused) != 1 {
+			t.Errorf("expected 1 REQ_PAUSED, got %d", len(paused))
+		}
+		// Must NOT burn an escalation tier.
+		failed, _ := es.List(state.EventFilter{Type: state.EventStoryReviewFailed, StoryID: "s-cap"})
+		if len(failed) != 0 {
+			t.Errorf("capacity pause must not emit STORY_REVIEW_FAILED; got %d", len(failed))
+		}
+		esc, _ := es.List(state.EventFilter{Type: state.EventStoryEscalated, StoryID: "s-cap"})
+		if len(esc) != 0 {
+			t.Errorf("capacity pause must not emit STORY_ESCALATED; got %d", len(esc))
+		}
+	})
+
+	t.Run("ordinary error returns false", func(t *testing.T) {
+		es, ps := capacityTestStores(t)
+		seedCapacityStory(t, es, ps, "r-ord", "s-ord")
+		m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps)
+
+		if m.pauseIfCapacity("s-ord", "review", fmt.Errorf("undefined: Foo")) {
+			t.Fatal("pauseIfCapacity returned true for an ordinary error")
+		}
+		paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused})
+		if len(paused) != 0 {
+			t.Errorf("ordinary error must not pause; got %d REQ_PAUSED", len(paused))
+		}
+	})
+
+	t.Run("nil error returns false", func(t *testing.T) {
+		es, ps := capacityTestStores(t)
+		seedCapacityStory(t, es, ps, "r-nil", "s-nil")
+		m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps)
+		if m.pauseIfCapacity("s-nil", "review", nil) {
+			t.Fatal("pauseIfCapacity returned true for a nil error")
+		}
+	})
+}
+
+// agentCompletionHasCapacityError scans the latest STORY_COMPLETED event's
+// recorded error envelope. A native (Gemma) agent that hit an Ollama overload
+// still emits STORY_COMPLETED (with the error in the payload) and produces an
+// empty diff — without this scan it would look identical to a lazy agent and be
+// wrongly escalated as "produced no code changes".
+func TestAgentCompletionHasCapacityError(t *testing.T) {
+	t.Run("detects capacity error in completion payload", func(t *testing.T) {
+		es, ps := capacityTestStores(t)
+		m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps)
+		evt := state.NewEvent(state.EventStoryCompleted, "agent-1", "s-1", map[string]any{
+			"native": true,
+			"error":  "llm completion (iteration 2): ollama API error (status 503): server busy, please try again",
+		})
+		es.Append(evt)
+		ps.Project(evt)
+
+		if !m.agentCompletionHasCapacityError("s-1") {
+			t.Error("expected capacity error to be detected in completion payload")
+		}
+	})
+
+	t.Run("ignores ordinary completion error", func(t *testing.T) {
+		es, ps := capacityTestStores(t)
+		m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps)
+		evt := state.NewEvent(state.EventStoryCompleted, "agent-1", "s-2", map[string]any{
+			"native": true,
+			"error":  "llm completion (iteration 1): undefined: Foo",
+		})
+		es.Append(evt)
+		ps.Project(evt)
+
+		if m.agentCompletionHasCapacityError("s-2") {
+			t.Error("ordinary completion error must not be detected as capacity")
+		}
+	})
+
+	t.Run("no completion event", func(t *testing.T) {
+		es, ps := capacityTestStores(t)
+		m := NewMonitor(nil, nil, nil, nil, nil, config.Config{}, es, ps)
+		if m.agentCompletionHasCapacityError("s-missing") {
+			t.Error("missing completion event must not be detected as capacity")
+		}
+	})
+}
+
+// makeWorktreeWithDiff builds a git repo with a committed change on a feature
+// branch so the post-execution pipeline sees a non-empty diff and proceeds to
+// the review stage.
+func makeWorktreeWithDiff(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	run := func(args ...string) {
+		cmd := exec.Command("git", args...)
+		cmd.Dir = dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			t.Fatalf("git %v: %v (%s)", args, err, out)
+		}
+	}
+	run("init", "-b", "main")
+	run("config", "user.email", "t@t.t")
+	run("config", "user.name", "t")
+	if err := os.WriteFile(dir+"/base.txt", []byte("base\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	run("add", ".")
+	run("commit", "-m", "base")
+	run("checkout", "-b", "nxd/s-pe-cap")
+	if err := os.WriteFile(dir+"/feature.txt", []byte("feature\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	run("add", ".")
+	run("commit", "-m", "feature")
+	return dir
+}
+
+// makeEmptyWorktree builds a git repo with a feature branch identical to main
+// (no diff), simulating a native agent that produced nothing.
+func makeEmptyWorktree(t *testing.T, branch string) string {
+	t.Helper()
+	dir := t.TempDir()
+	run := func(args ...string) {
+		cmd := exec.Command("git", args...)
+		cmd.Dir = dir
+		if out, err := cmd.CombinedOutput(); err != nil {
+			t.Fatalf("git %v: %v (%s)", args, err, out)
+		}
+	}
+	run("init", "-b", "main")
+	run("config", "user.email", "t@t.t")
+	run("config", "user.name", "t")
+	if err := os.WriteFile(dir+"/base.txt", []byte("base\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	run("add", ".")
+	run("commit", "-m", "base")
+	run("checkout", "-b", branch)
+	return dir
+}
+
+// TestPostExecutionPipeline_EmptyDiffCapacity verifies that when a native agent
+// produces no diff AND its STORY_COMPLETED payload records an Ollama capacity
+// error, the requirement pauses cleanly instead of resetting to draft (which
+// would burn an escalation attempt as "produced no code changes").
+func TestPostExecutionPipeline_EmptyDiffCapacity(t *testing.T) {
+	es, ps := capacityTestStores(t)
+	seedCapacityStory(t, es, ps, "r-empty", "s-empty")
+
+	// Native agent emitted STORY_COMPLETED carrying an Ollama overload error.
+	completed := state.NewEvent(state.EventStoryCompleted, "agent-1", "s-empty", map[string]any{
+		"native": true,
+		"error":  "llm completion (iteration 3): ollama API error (status 503): server busy, please try again. maximum pending requests exceeded",
+	})
+	es.Append(completed)
+	ps.Project(completed)
+
+	worktree := makeEmptyWorktree(t, "nxd/s-empty")
+	m := NewMonitor(nil, nil, nil, nil, nil, config.Config{
+		Routing: config.RoutingConfig{MaxRetriesBeforeEscalation: 2},
+	}, es, ps)
+
+	ag := ActiveAgent{
+		Assignment: Assignment{
+			StoryID: "s-empty", AgentID: "agent-1",
+			SessionName: "nxd-test-empty", Branch: "nxd/s-empty",
+		},
+		WorktreePath: worktree,
+	}
+
+	m.postExecutionPipeline(context.Background(), ag, worktree)
+
+	paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused})
+	if len(paused) < 1 {
+		t.Error("expected REQ_PAUSED for empty-diff capacity error")
+	}
+	failed, _ := es.List(state.EventFilter{Type: state.EventStoryReviewFailed, StoryID: "s-empty"})
+	if len(failed) != 0 {
+		t.Errorf("empty-diff capacity must not emit STORY_REVIEW_FAILED; got %d", len(failed))
+	}
+}
+
+// TestPostExecutionPipeline_ReviewCapacity verifies a capacity error during
+// review PAUSES the requirement (resume after the server recovers) rather than
+// resetting to draft — which would burn an escalation attempt on a transient
+// overload the story never had a chance to avoid.
+func TestPostExecutionPipeline_ReviewCapacity(t *testing.T) {
+	es, ps := capacityTestStores(t)
+	seedCapacityStory(t, es, ps, "r-cap", "s-pe-cap")
+
+	worktree := makeWorktreeWithDiff(t)
+
+	capErr := fmt.Errorf(`reviewer LLM call: ollama API error (status 503): {"error":"server overloaded, please retry shortly"}`)
+	reviewer := NewReviewer(llm.NewErrorClient(capErr), "ollama", "gemma4", 4000, es, ps)
+
+	cfg := config.Config{Routing: config.RoutingConfig{MaxRetriesBeforeEscalation: 2}}
+	m := NewMonitor(nil, nil, reviewer, nil, nil, cfg, es, ps)
+
+	ag := ActiveAgent{
+		Assignment: Assignment{
+			StoryID: "s-pe-cap", AgentID: "agent-1",
+			SessionName: "nxd-test-cap", Branch: "nxd/s-pe-cap",
+		},
+		WorktreePath: worktree,
+	}
+
+	m.postExecutionPipeline(context.Background(), ag, worktree)
+
+	paused, _ := es.List(state.EventFilter{Type: state.EventReqPaused})
+	if len(paused) < 1 {
+		t.Error("expected REQ_PAUSED for capacity error during review")
+	}
+	failed, _ := es.List(state.EventFilter{Type: state.EventStoryReviewFailed, StoryID: "s-pe-cap"})
+	if len(failed) != 0 {
+		t.Errorf("capacity error must not emit STORY_REVIEW_FAILED (burns escalation); got %d", len(failed))
+	}
+}