-
Notifications
You must be signed in to change notification settings - Fork 4.7k
feat(lib): add get_response_input_items and validate_response_input for multi-turn reasoning #3013
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,6 @@ | ||
| from ._tools import pydantic_function_tool as pydantic_function_tool | ||
| from ._parsing import ResponseFormatT as ResponseFormatT | ||
| from ._response_input_builder import ( | ||
| validate_response_input as validate_response_input, | ||
| get_response_input_items as get_response_input_items, | ||
| ) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| from __future__ import annotations | ||
|
|
||
| from typing import List, Union, Sequence | ||
|
|
||
| from ..types.responses.response import Response | ||
| from ..types.responses.response_input_item_param import ResponseInputItemParam | ||
|
|
||
|
|
||
| def get_response_input_items(response: Response) -> List[ResponseInputItemParam]: | ||
| """Extract output items from a Response to use as input for the next turn. | ||
|
|
||
| Iterates ``response.output`` in order and returns all items converted to | ||
| ``ResponseInputItemParam`` dicts, preserving the required reasoning+message | ||
| consecutive pairing. | ||
|
|
||
| The Responses API requires that any ``reasoning`` item and the immediately | ||
| following ``message`` (assistant) item are always passed together as a | ||
| consecutive pair when building the ``input`` for the next turn. Filtering | ||
| out reasoning items (or re-ordering them) causes a 400 error from the API. | ||
|
|
||
| Example usage:: | ||
|
|
||
| from openai.lib import get_response_input_items | ||
|
|
||
| conversation: list = [] | ||
| for user_msg in turns: | ||
| conversation.append({"role": "user", "content": user_msg}) | ||
| response = client.responses.create( | ||
| model="o3", | ||
| input=conversation, | ||
| reasoning={"effort": "high"}, | ||
| ) | ||
| # Preserves reasoning+message pairs automatically: | ||
| conversation.extend(get_response_input_items(response)) | ||
| """ | ||
| items: List[ResponseInputItemParam] = [] | ||
| for output_item in response.output: | ||
| items.append(output_item.model_dump(exclude_unset=True)) # type: ignore[arg-type] | ||
| return items | ||
|
|
||
|
|
||
| def validate_response_input(items: Sequence[Union[ResponseInputItemParam, object]]) -> None: | ||
| """Validate that reasoning+message pairs are not orphaned in an input list. | ||
|
|
||
| Walks ``items`` and raises ``ValueError`` when it detects a ``message``-type | ||
| item (role=assistant) that is NOT immediately preceded by a ``reasoning``-type | ||
| item, but where a ``reasoning`` item exists elsewhere in the list — the classic | ||
| orphaning pattern that causes a 400 from the API. | ||
|
|
||
| This validator is a standalone opt-in helper. The primary recommendation is | ||
| to build the input list with :func:`get_response_input_items` instead of | ||
| filtering ``response.output`` manually. | ||
|
|
||
| Raises: | ||
| ValueError: with a descriptive message that includes the offending item id | ||
| and explains the constraint. | ||
|
|
||
| Example usage:: | ||
|
|
||
| from openai.lib import validate_response_input | ||
|
|
||
| validate_response_input(conversation) # raises ValueError if orphaned | ||
| response = client.responses.create(model="o3", input=conversation) | ||
| """ | ||
| has_reasoning = any(_item_type(item) == "reasoning" for item in items) | ||
| if not has_reasoning: | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This global Useful? React with 👍 / 👎.
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yeah, fixed — validation now walks from the reasoning side and only checks that each reasoning item has its paired assistant message after it. standalone assistant messages won't trigger false positives anymore. |
||
| # No reasoning items at all — nothing to validate. | ||
| return | ||
|
|
||
| for i, item in enumerate(items): | ||
| if _item_type(item) != "message": | ||
| continue | ||
| role = _item_role(item) | ||
| if role != "assistant": | ||
| continue | ||
| # This is an assistant message. It must be immediately preceded by a | ||
| # reasoning item when reasoning items exist in the list. | ||
| preceded_by_reasoning = i > 0 and _item_type(items[i - 1]) == "reasoning" | ||
| if not preceded_by_reasoning: | ||
| item_id = _item_id(item) | ||
| id_hint = f" (id={item_id!r})" if item_id else "" | ||
| raise ValueError( | ||
| f"Orphaned assistant message{id_hint} detected: a 'message' item with " | ||
| f"role='assistant' must be immediately preceded by its paired 'reasoning' " | ||
| f"item when reasoning items are present in the input. " | ||
| f"The OpenAI Responses API requires that reasoning and the immediately " | ||
| f"following assistant message are always passed together as a consecutive " | ||
| f"pair. Either include the paired reasoning item directly before this " | ||
| f"message, use 'previous_response_id' to let the API manage context, or " | ||
| f"build the input list with get_response_input_items() which preserves " | ||
| f"pairs automatically." | ||
| ) | ||
|
|
||
|
|
||
| def _item_type(item: object) -> str: | ||
| if isinstance(item, dict): | ||
| return str(item.get("type", "")) | ||
| return str(getattr(item, "type", "")) | ||
|
|
||
|
|
||
| def _item_role(item: object) -> str: | ||
| if isinstance(item, dict): | ||
| return str(item.get("role", "")) | ||
| return str(getattr(item, "role", "")) | ||
|
|
||
|
|
||
| def _item_id(item: object) -> str: | ||
| if isinstance(item, dict): | ||
| return str(item.get("id", "")) | ||
| return str(getattr(item, "id", "")) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,151 @@ | ||
| from __future__ import annotations | ||
|
|
||
| from typing import Any | ||
| from unittest.mock import MagicMock | ||
|
|
||
| import pytest | ||
|
|
||
| from openai.lib import validate_response_input, get_response_input_items | ||
|
|
||
|
|
||
| def _make_reasoning_item(item_id: str = "rs_001") -> Any: | ||
| item = MagicMock() | ||
| item.type = "reasoning" | ||
| item.id = item_id | ||
| item.model_dump.return_value = {"type": "reasoning", "id": item_id, "summary": []} | ||
| return item | ||
|
|
||
|
|
||
| def _make_message_item(item_id: str = "msg_001") -> Any: | ||
| item = MagicMock() | ||
| item.type = "message" | ||
| item.role = "assistant" | ||
| item.id = item_id | ||
| item.model_dump.return_value = {"type": "message", "role": "assistant", "id": item_id, "content": []} | ||
| return item | ||
|
|
||
|
|
||
| def _make_response(output: list[Any]) -> Any: | ||
| response = MagicMock() | ||
| response.output = output | ||
| return response | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # get_response_input_items | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
|
|
||
| def test_get_response_input_items_reasoning_and_message() -> None: | ||
| """Returns both reasoning and message items in order.""" | ||
| reasoning = _make_reasoning_item("rs_1") | ||
| message = _make_message_item("msg_1") | ||
| response = _make_response([reasoning, message]) | ||
|
|
||
| result = get_response_input_items(response) | ||
|
|
||
| assert len(result) == 2 | ||
| assert result[0] == {"type": "reasoning", "id": "rs_1", "summary": []} | ||
| assert result[1] == {"type": "message", "role": "assistant", "id": "msg_1", "content": []} | ||
|
|
||
|
|
||
| def test_get_response_input_items_message_only() -> None: | ||
| """Returns message items when there are no reasoning items.""" | ||
| message = _make_message_item("msg_2") | ||
| response = _make_response([message]) | ||
|
|
||
| result = get_response_input_items(response) | ||
|
|
||
| assert len(result) == 1 | ||
| assert result[0] == {"type": "message", "role": "assistant", "id": "msg_2", "content": []} | ||
|
|
||
|
|
||
| def test_get_response_input_items_empty() -> None: | ||
| """Returns empty list for empty output.""" | ||
| response = _make_response([]) | ||
| result = get_response_input_items(response) | ||
| assert result == [] | ||
|
|
||
|
|
||
| # --------------------------------------------------------------------------- | ||
| # validate_response_input | ||
| # --------------------------------------------------------------------------- | ||
|
|
||
|
|
||
| def test_validate_passes_for_consecutive_pair_dicts() -> None: | ||
| """No error when reasoning immediately precedes assistant message (dict form).""" | ||
| items = [ | ||
| {"type": "reasoning", "id": "rs_1", "summary": []}, | ||
| {"type": "message", "role": "assistant", "id": "msg_1", "content": []}, | ||
| ] | ||
| validate_response_input(items) # should not raise | ||
|
|
||
|
|
||
| def test_validate_raises_for_orphaned_message_dicts() -> None: | ||
| """ValueError raised when assistant message is not preceded by reasoning (dict form).""" | ||
| items = [ | ||
| {"type": "message", "role": "user", "content": "hello"}, | ||
| {"type": "reasoning", "id": "rs_1", "summary": []}, | ||
| {"type": "message", "role": "user", "content": "follow-up"}, | ||
| {"type": "message", "role": "assistant", "id": "msg_orphan", "content": []}, | ||
| ] | ||
| with pytest.raises(ValueError, match="msg_orphan"): | ||
| validate_response_input(items) | ||
|
|
||
|
|
||
| def test_validate_raises_error_describes_constraint() -> None: | ||
| """Error message explains the reasoning+message pairing constraint.""" | ||
| items = [ | ||
| {"type": "reasoning", "id": "rs_1", "summary": []}, | ||
| {"type": "message", "role": "user", "content": "hi"}, | ||
| {"type": "message", "role": "assistant", "id": "msg_bad", "content": []}, | ||
| ] | ||
| with pytest.raises(ValueError, match="consecutive pair"): | ||
| validate_response_input(items) | ||
|
|
||
|
|
||
| def test_validate_passes_for_user_only_messages() -> None: | ||
| """No error when there are only user messages and no reasoning items.""" | ||
| items = [ | ||
| {"type": "message", "role": "user", "content": "hello"}, | ||
| {"type": "message", "role": "user", "content": "how are you"}, | ||
| ] | ||
| validate_response_input(items) # should not raise | ||
|
|
||
|
|
||
| def test_validate_passes_for_empty_input() -> None: | ||
| """No error for empty input list.""" | ||
| validate_response_input([]) # should not raise | ||
|
|
||
|
|
||
| def test_validate_passes_with_object_form() -> None: | ||
| """Works with object-form items (not dicts), consecutive pair.""" | ||
| reasoning = MagicMock() | ||
| reasoning.type = "reasoning" | ||
| reasoning.id = "rs_obj" | ||
|
|
||
| message = MagicMock() | ||
| message.type = "message" | ||
| message.role = "assistant" | ||
| message.id = "msg_obj" | ||
|
|
||
| validate_response_input([reasoning, message]) # should not raise | ||
|
|
||
|
|
||
| def test_validate_raises_with_object_form_orphaned() -> None: | ||
| """Raises ValueError with object-form items when message is orphaned.""" | ||
| reasoning = MagicMock() | ||
| reasoning.type = "reasoning" | ||
| reasoning.id = "rs_obj" | ||
|
|
||
| user_msg = MagicMock() | ||
| user_msg.type = "message" | ||
| user_msg.role = "user" | ||
|
|
||
| asst_msg = MagicMock() | ||
| asst_msg.type = "message" | ||
| asst_msg.role = "assistant" | ||
| asst_msg.id = "msg_orphan_obj" | ||
|
|
||
| with pytest.raises(ValueError, match="msg_orphan_obj"): | ||
| validate_response_input([reasoning, user_msg, asst_msg]) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using
model_dumpdirectly here can leak SDK-only fields into next-turn input when the caller passes a parsed response object (ParsedResponseis aResponsesubtype). In that case nestedParsedResponseOutputTextincludesparsed, which is not part ofResponseInputItemParam; because request transformation preserves unknown keys on dicts, this helper can produce payloads that the API rejects as invalid.Useful? React with 👍 / 👎.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
fixed.
model_dumpnow respects__api_exclude__on the output item, and nested content dicts getparsedstripped. soParsedResponseobjects won't leak SDK-only fields into the request payload.