From 2df00af840bf0a3f45c9136cbf633aa94e73e5b8 Mon Sep 17 00:00:00 2001 From: Sai Krishna Rallabandi Date: Thu, 23 Jul 2026 15:57:27 -0500 Subject: [PATCH] judge: clamp max_tokens to the model's output cap The judge hard-codes max_tokens=32768 at two call sites, above gpt-4o-mini's 16384 completion limit. ollama silently ignores an oversized max_tokens; OpenAI returns 400, the 20x retry loop then hangs grading until the batch drops the result with no error surfaced. Presents exactly like an invalid API key. --- src/claw_anything/graders/llm_judge.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/claw_anything/graders/llm_judge.py b/src/claw_anything/graders/llm_judge.py index 3f96958..d9a036f 100644 --- a/src/claw_anything/graders/llm_judge.py +++ b/src/claw_anything/graders/llm_judge.py @@ -152,7 +152,11 @@ def _chat_with_retries( {"role": "user", "content": user}, ], "temperature": 0.0, - "max_tokens": max_tokens, + # Clamp to the model's real output cap. The default 32768 is + # above gpt-4o-mini's 16384 limit; ollama silently ignores an + # oversized max_tokens but OpenAI 400s, and the 20x retry loop + # then hangs grading until the batch drops the result. + "max_tokens": min(max_tokens, 16384), } resp = self.client.chat.completions.create(**kwargs) log_llm_call( @@ -334,7 +338,7 @@ def evaluate( {"role": "user", "content": user_msg}, ], "temperature": 0.0, - "max_tokens": 32768, + "max_tokens": 16384, # gpt-4o-mini output cap; see the note above } resp = self.client.chat.completions.create(**_judge_kwargs) log_llm_call(