feat: add max_retries to the messages handler factories

packages/claude-messages/api-report/launchdarkly_ai_claude_messages.txt

@@ -1,3 +1,3 @@
 # Public API of launchdarkly_ai_claude_messages. Generated by `make api-report`; do not edit.
 claude_messages(config_key: str, user_input: str, context: LDContext, **kwargs: Any) -> Any
-create_claude_messages_handler(*, capture_content: bool | None = None) -> ProviderHandler
+create_claude_messages_handler(*, capture_content: bool | None = None, max_retries: int | None = None) -> ProviderHandler

packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py

@@ -529,7 +529,7 @@ def _is_coroutine(fn: Any) -> bool:
 
 
 def create_claude_messages_handler(
-    *, capture_content: bool | None = None
+    *, capture_content: bool | None = None, max_retries: int | None = None
 ) -> ProviderHandler:
     """Creates a ``ProviderHandler`` for Anthropic Claude (messages API).
 
@@ -541,22 +541,35 @@ def create_claude_messages_handler(
     for this handler. Conversation content is PII, so a run emits only metadata, meaning
     models, token counts, timings and tool names, until capture is turned on. Turning this
     on sends the text of every request and response to whatever collector the SDK is pointed at.
+
+    Set *max_retries* to change how many times the ``AsyncAnthropic`` client retries a failed
+    request (a rate limit, an overload, a timeout, a 5xx) before the error reaches the handler.
+    ``None`` (the default) leaves the client's own default of 2 in place. Raise it when many calls
+    share one rate-limit window, as an evaluation run with many concurrent workers does: two
+    retries is thin when a whole batch hits the same window at once. The retries happen inside the
+    client, so a retried turn is still one ``chat`` span.
     """
     report_usage(
         "claude-messages.createClaudeMessagesHandler", PACKAGE_NAME, __version__
     )
-    return _create_claude_messages_handler(capture_content=capture_content)
+    return _create_claude_messages_handler(
+        capture_content=capture_content, max_retries=max_retries
+    )
 
 
 def _create_claude_messages_handler(
-    *, capture_content: bool | None = None
+    *, capture_content: bool | None = None, max_retries: int | None = None
 ) -> ProviderHandler:
     """Non-reporting :func:`create_claude_messages_handler`, used by this package's wrappers."""
     content_override = capture_content
     import importlib
 
     anthropic_mod = importlib.import_module("anthropic")
-    client = anthropic_mod.AsyncAnthropic()
+    # Passed only when set, so an omitted value keeps the vendor default rather than a copy of it.
+    client_options: dict[str, Any] = {}
+    if max_retries is not None:
+        client_options["max_retries"] = max_retries
+    client = anthropic_mod.AsyncAnthropic(**client_options)
 
     async def _call_impl(
         config: AiConfigRep,
@@ -950,14 +963,17 @@ def claude_messages(
     **kwargs: Any,
 ) -> Any:
     """Convenience wrapper: creates a handler and calls config(...).invoke()."""
-    # Both are lifted out of kwargs: capture_content configures the handler, variables belong to
-    # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller
-    # asking for content on spans got a TypeError instead of content.
+    # All three are lifted out of kwargs: capture_content and max_retries configure the handler,
+    # variables belong to the invocation. Leaving any in would pass it to config(), which takes
+    # none of them, so a caller asking for content on spans got a TypeError instead of content.
     report_usage("claude-messages.claudeMessages", PACKAGE_NAME, __version__)
     variables = kwargs.pop("variables", None)
     capture_content = kwargs.pop("capture_content", None)
+    max_retries = kwargs.pop("max_retries", None)
     return _config(
         key=config_key,
-        handler=_create_claude_messages_handler(capture_content=capture_content),
+        handler=_create_claude_messages_handler(
+            capture_content=capture_content, max_retries=max_retries
+        ),
         **kwargs,
     ).invoke(user_input, context, variables=variables)

packages/claude-messages/tests/test_handler.py

@@ -3127,3 +3127,65 @@ async def test_effort_still_lands_in_output_config_when_output_config_is_malform
         await create_claude_messages_handler()(config, "q", {}, {})
         kwargs = mock_anthropic.messages.create.call_args.kwargs
         assert kwargs["output_config"] == {"effort": "low"}
+
+
+class TestMaxRetries:
+    """`max_retries` reaches the `AsyncAnthropic` client the factory builds, and only when set.
+
+    The client's own retries are the right layer for rate limits, so an omitted value must leave the
+    vendor default in place rather than replace it, and nothing may ever turn retries off.
+    """
+
+    def _build(
+        self, monkeypatch: pytest.MonkeyPatch, **kwargs: Any
+    ) -> tuple[Any, MagicMock]:
+        import anthropic
+
+        from launchdarkly_ai_claude_messages import create_claude_messages_handler
+
+        monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key")
+        real_client = anthropic.AsyncAnthropic
+        built: list[Any] = []
+
+        def _construct(**options: Any) -> Any:
+            built.append(real_client(**options))
+            return built[-1]
+
+        with patch("anthropic.AsyncAnthropic", side_effect=_construct) as ctor:
+            create_claude_messages_handler(**kwargs)
+        return built[0], ctor
+
+    def test_value_reaches_the_constructed_client(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        client, _ctor = self._build(monkeypatch, max_retries=7)
+        assert client.max_retries == 7
+
+    def test_omitted_value_keeps_the_vendor_default(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        import anthropic
+
+        client, ctor = self._build(monkeypatch)
+        assert "max_retries" not in ctor.call_args.kwargs
+        assert client.max_retries == anthropic.DEFAULT_MAX_RETRIES
+
+    def test_explicit_zero_is_the_callers_choice(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        client, _ctor = self._build(monkeypatch, max_retries=0)
+        assert client.max_retries == 0
+
+    def test_convenience_wrapper_forwards_it_to_the_factory(self) -> None:
+        import launchdarkly_ai_claude_messages.handler as handler_mod
+
+        factory = MagicMock()
+        fake_config = MagicMock()
+        with (
+            patch.object(handler_mod, "_create_claude_messages_handler", factory),
+            patch.object(handler_mod, "_config", fake_config),
+        ):
+            handler_mod.claude_messages("k", "q", {}, max_retries=6)
+        assert factory.call_args.kwargs["max_retries"] == 6
+        # config() takes no such argument, so leaving it in kwargs would raise TypeError.
+        assert "max_retries" not in fake_config.call_args.kwargs

packages/langchain-messages/api-report/launchdarkly_ai_langchain_messages.txt

@@ -1,3 +1,3 @@
 # Public API of launchdarkly_ai_langchain_messages. Generated by `make api-report`; do not edit.
-create_langchain_messages_handler(llm: Any = None, *, capture_content: bool | None = None) -> ProviderHandler
+create_langchain_messages_handler(llm: Any = None, *, capture_content: bool | None = None, max_retries: int | None = None) -> ProviderHandler
 langchain_messages(config_key: str, user_input: str, context: LDContext, llm: Any = None, **kwargs: Any) -> Any

packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py

@@ -591,6 +591,7 @@ def _model_constructor_kwargs(
     *,
     mapping_keys: frozenset[str] = frozenset(),
     renames: Mapping[str, str] | None = None,
+    max_retries: int | None = None,
 ) -> dict[str, Any]:
     params = model_parameters(config)
     for alias, name in (renames or {}).items():
@@ -601,6 +602,10 @@ def _model_constructor_kwargs(
         params, forwarded_keys, mapping_keys=mapping_keys
     )
     parameters["model"] = _resolved_model_name(config, fallback_name)
+    # From the handler's options, never from model.parameters, which excludes it above. Set only
+    # when given, so an omitted value keeps each chat model's own default.
+    if max_retries is not None:
+        parameters["max_retries"] = max_retries
     return parameters
 
 
@@ -609,13 +614,16 @@ def _is_model_factory(llm: Any) -> bool:
     return callable(llm) and not hasattr(llm, "invoke") and not hasattr(llm, "ainvoke")
 
 
-def _make_default_chat_model(config: AiConfigRep, importlib: Any) -> Any:
+def _make_default_chat_model(
+    config: AiConfigRep, importlib: Any, max_retries: int | None = None
+) -> Any:
     """
     Instantiate the appropriate LangChain chat model based on ``config.provider.name``.
     Falls back to ``ChatOpenAI`` when the provider is not recognised.
     Requires the matching ``langchain-<provider>`` integration package to be installed.
     ``model.parameters`` are filtered to that class's forwarded list first. ``tools`` is never on
-    that list, since tools are bound separately from ``config["tools"]``.
+    that list, since tools are bound separately from ``config["tools"]``. *max_retries*, when set,
+    is passed to whichever class is built.
     """
     provider = config.get("provider", {}).get("name", "openai").lower()
     if provider == "anthropic":
@@ -629,6 +637,7 @@ def _make_default_chat_model(config: AiConfigRep, importlib: Any) -> Any:
                         _CHAT_ANTHROPIC_FORWARDED_KEYS,
                         mapping_keys=_CHAT_ANTHROPIC_MAPPING_KEYS,
                         renames=_CHAT_ANTHROPIC_RENAMES,
+                        max_retries=max_retries,
                     )
                 )
             )
@@ -648,6 +657,7 @@ def _make_default_chat_model(config: AiConfigRep, importlib: Any) -> Any:
                 _CHAT_BEDROCK_CONVERSE_FORWARDED_KEYS,
                 mapping_keys=_CHAT_BEDROCK_CONVERSE_MAPPING_KEYS,
                 renames=_CHAT_BEDROCK_CONVERSE_RENAMES,
+                max_retries=max_retries,
             )
         )
     lc_openai = importlib.import_module("langchain_openai")
@@ -657,14 +667,17 @@ def _make_default_chat_model(config: AiConfigRep, importlib: Any) -> Any:
             "gpt-4o",
             _CHAT_OPENAI_FORWARDED_KEYS,
             mapping_keys=_CHAT_OPENAI_MAPPING_KEYS,
+            max_retries=max_retries,
         )
     )
 
 
-async def _resolve_base_model(config: AiConfigRep, llm: Any, importlib: Any) -> Any:
+async def _resolve_base_model(
+    config: AiConfigRep, llm: Any, importlib: Any, max_retries: int | None = None
+) -> Any:
     invocation = _config_for_model_call(config)
     if llm is None:
-        return _make_default_chat_model(invocation, importlib)
+        return _make_default_chat_model(invocation, importlib, max_retries)
     if _is_model_factory(llm):
         model = llm(invocation)
         if asyncio.iscoroutine(model):
@@ -762,7 +775,10 @@ async def _run_structured_turn(
 
 
 def create_langchain_messages_handler(
-    llm: Any = None, *, capture_content: bool | None = None
+    llm: Any = None,
+    *,
+    capture_content: bool | None = None,
+    max_retries: int | None = None,
 ) -> ProviderHandler:
     """
     Creates a ``ProviderHandler`` for LangChain (chat models).
@@ -777,15 +793,36 @@ def create_langchain_messages_handler(
     which is off until ``configure`` sets it. ``True`` or ``False`` overrides that default
     for this handler. Conversation content is PII, so a run emits only metadata, meaning
     models, token counts, timings and tool names, until capture is turned on.
+
+    Set *max_retries* to change how many times the default chat model retries a failed request (a
+    rate limit, a timeout, a 5xx) before the error reaches the handler. It is passed to the
+    ``ChatOpenAI``, ``ChatAnthropic`` or ``ChatBedrockConverse`` the handler builds. ``None`` (the
+    default) leaves that class's own default in place, which is 2 for OpenAI and Anthropic. Raise
+    it when many calls share one rate-limit window, as an evaluation run with many concurrent
+    workers does: two retries is thin when a whole batch hits the same window at once. It cannot
+    be combined with *llm*, because the handler does not build that model: set ``max_retries`` on
+    the model instead.
     """
     report_usage("langchain-messages.createLangChainHandler", PACKAGE_NAME, __version__)
-    return _create_langchain_messages_handler(llm, capture_content=capture_content)
+    return _create_langchain_messages_handler(
+        llm, capture_content=capture_content, max_retries=max_retries
+    )
 
 
 def _create_langchain_messages_handler(
-    llm: Any = None, *, capture_content: bool | None = None
+    llm: Any = None,
+    *,
+    capture_content: bool | None = None,
+    max_retries: int | None = None,
 ) -> ProviderHandler:
     """Non-reporting :func:`create_langchain_messages_handler`, used by this package's wrappers."""
+    # Refused rather than ignored: the handler never constructs a model it is given, so the value
+    # would go nowhere and the caller would believe their retries had been raised.
+    if llm is not None and max_retries is not None:
+        raise ValueError(
+            "max_retries applies only to the default chat model. Set max_retries on the model "
+            "passed as llm instead."
+        )
     content_override = capture_content
 
     async def _call_impl(
@@ -826,7 +863,7 @@ async def _call_impl(
                     system_instructions=system_instructions,
                     messages=span_messages,
                 )
-            base_model = await _resolve_base_model(config, llm, importlib)
+            base_model = await _resolve_base_model(config, llm, importlib, max_retries)
 
             tool_defs = _build_tools(config.get("tools") or {})
             output_format = config.get("outputFormat")
@@ -1097,6 +1134,7 @@ def _stream_impl(
             variables or {},
             history,
             capture_content=effective_capture_content(content_override),
+            max_retries=max_retries,
         )
 
     return _create_handler(
@@ -1116,6 +1154,7 @@ async def _stream_gen(
     history: list[dict[str, Any]] | None = None,
     *,
     capture_content: bool = False,
+    max_retries: int | None = None,
 ) -> AsyncGenerator[dict[str, Any], None]:
     """Streams the run, emitting the same span tree as the blocking path.
 
@@ -1138,7 +1177,7 @@ async def _stream_gen(
     """
     import importlib
 
-    base_model = await _resolve_base_model(config, llm, importlib)
+    base_model = await _resolve_base_model(config, llm, importlib, max_retries)
 
     span = start_root_span(config, variables)
     parent = parent_context_of(span)
@@ -1501,16 +1540,17 @@ def langchain_messages(
     **kwargs: Any,
 ) -> Any:
     """Convenience wrapper: creates a handler and calls config(...).invoke()."""
-    # Both are lifted out of kwargs: capture_content configures the handler, variables belong to
-    # the invocation. Leaving either in would pass it to config(), which takes neither, so a caller
-    # asking for content on spans got a TypeError instead of content.
+    # All three are lifted out of kwargs: capture_content and max_retries configure the handler,
+    # variables belong to the invocation. Leaving any in would pass it to config(), which takes
+    # none of them, so a caller asking for content on spans got a TypeError instead of content.
     report_usage("langchain-messages.langchainMessages", PACKAGE_NAME, __version__)
     variables = kwargs.pop("variables", None)
     capture_content = kwargs.pop("capture_content", None)
+    max_retries = kwargs.pop("max_retries", None)
     return _config(
         key=config_key,
         handler=_create_langchain_messages_handler(
-            llm, capture_content=capture_content
+            llm, capture_content=capture_content, max_retries=max_retries
         ),
         **kwargs,
     ).invoke(user_input, context, variables=variables)