From ffc43612395e98edb25c14112fcbad403fbee9cf Mon Sep 17 00:00:00 2001 From: Slava Primenko Date: Wed, 29 Apr 2026 16:38:38 +0200 Subject: [PATCH 1/4] Configure UserSimulatorProvider --- src/google/adk/evaluation/evaluation_generator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index f8fb6795aa..f00be022ce 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -46,6 +46,7 @@ from .eval_case import SessionInput from .eval_set import EvalSet from .request_intercepter_plugin import _RequestIntercepterPlugin +from .simulation.user_simulator import BaseUserSimulatorConfig from .simulation.user_simulator import Status as UserSimulatorStatus from .simulation.user_simulator import UserSimulator from .simulation.user_simulator_provider import UserSimulatorProvider @@ -75,6 +76,7 @@ async def generate_responses( agent_module_path: str, repeat_num: int = 3, agent_name: str = None, + user_simulator_config: Optional[BaseUserSimulatorConfig] = None, ) -> list[EvalCaseResponses]: """Returns evaluation responses for the given dataset and agent. @@ -90,7 +92,10 @@ async def generate_responses( for eval_case in eval_set.eval_cases: # assume only static conversations are needed - user_simulator = UserSimulatorProvider().provide(eval_case) + user_simulator = UserSimulatorProvider( + user_simulator_config=user_simulator_config + ).provide(eval_case) + responses = [] for _ in range(repeat_num): response_invocations = await EvaluationGenerator._process_query( From caf69f2cb963c8e637600d91a2bea75c013ffb12 Mon Sep 17 00:00:00 2001 From: Slava Primenko Date: Wed, 29 Apr 2026 17:30:17 +0200 Subject: [PATCH 2/4] Test UserSimulatorProvider configuration --- .../evaluation/test_evaluation_generator.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/tests/unittests/evaluation/test_evaluation_generator.py b/tests/unittests/evaluation/test_evaluation_generator.py index a4aa8691fd..271e91fd59 100644 --- a/tests/unittests/evaluation/test_evaluation_generator.py +++ b/tests/unittests/evaluation/test_evaluation_generator.py @@ -16,8 +16,11 @@ from google.adk.evaluation.app_details import AgentDetails from google.adk.evaluation.app_details import AppDetails +from google.adk.evaluation.eval_case import EvalCase +from google.adk.evaluation.eval_set import EvalSet from google.adk.evaluation.evaluation_generator import EvaluationGenerator from google.adk.evaluation.request_intercepter_plugin import _RequestIntercepterPlugin +from google.adk.evaluation.simulation.llm_backed_user_simulator import LlmBackedUserSimulatorConfig from google.adk.evaluation.simulation.user_simulator import NextUserMessage from google.adk.evaluation.simulation.user_simulator import Status as UserSimulatorStatus from google.adk.evaluation.simulation.user_simulator import UserSimulator @@ -479,3 +482,45 @@ async def mock_generate_inferences_side_effect( mock_generate_inferences.assert_called_once() called_with_content = mock_generate_inferences.call_args.args[3] assert called_with_content.parts[0].text == "message 1" + + +class TestGenerateResponses: + """Test cases for EvaluationGenerator.generate_responses method.""" + + @pytest.mark.asyncio + async def test_generate_responses_forwards_llm_backed_user_simulator_config( + self, mocker + ): + """Tests that an LlmBackedUserSimulatorConfig is forwarded to the provider verbatim.""" + mock_provider_cls = mocker.patch( + "google.adk.evaluation.evaluation_generator.UserSimulatorProvider" + ) + mocker.patch( + "google.adk.evaluation.evaluation_generator.EvaluationGenerator._process_query", + new_callable=mocker.AsyncMock, + return_value=[], + ) + + user_simulator_config = LlmBackedUserSimulatorConfig( + model="test-model", + max_allowed_invocations=5, + ) + eval_set = EvalSet( + eval_set_id="test_set", + eval_cases=[EvalCase(eval_id="case_0", conversation=[])], + ) + + await EvaluationGenerator.generate_responses( + eval_set=eval_set, + agent_module_path="some.agent.module", + repeat_num=1, + user_simulator_config=user_simulator_config, + ) + + mock_provider_cls.assert_called_once_with( + user_simulator_config=user_simulator_config + ) + assert ( + mock_provider_cls.call_args.kwargs["user_simulator_config"] + is user_simulator_config + ) From 780a640b3c6723d60967e6c209497ca04a75e7bb Mon Sep 17 00:00:00 2001 From: Slava Primenko Date: Wed, 29 Apr 2026 17:43:20 +0200 Subject: [PATCH 3/4] Add docstring for user_simulator_config param --- src/google/adk/evaluation/evaluation_generator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index f00be022ce..eb0833d35b 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -87,6 +87,11 @@ async def generate_responses( usually done to remove uncertainty that a single run may bring. agent_name: The name of the agent that should be evaluated. This is usually the sub-agent. + user_simulator_config: Optional configuration for the user simulator. + Only relevant for eval cases that use a `conversation_scenario` (which + are driven by `LlmBackedUserSimulator`); ignored for static + conversations. Pass an `LlmBackedUserSimulatorConfig` to override the + user-simulation model, max invocations, or custom instructions. """ results = [] From 2301b07f9d2d98717c4f16ecc4faabe694d5607d Mon Sep 17 00:00:00 2001 From: Slava Primenko Date: Sun, 17 May 2026 16:03:51 +0200 Subject: [PATCH 4/4] Remove the stale evaluation_generator comment --- src/google/adk/evaluation/evaluation_generator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/google/adk/evaluation/evaluation_generator.py b/src/google/adk/evaluation/evaluation_generator.py index 87c5f6fcac..794c9cf259 100644 --- a/src/google/adk/evaluation/evaluation_generator.py +++ b/src/google/adk/evaluation/evaluation_generator.py @@ -284,7 +284,6 @@ async def generate_responses( results = [] for eval_case in eval_set.eval_cases: - # assume only static conversations are needed user_simulator = UserSimulatorProvider( user_simulator_config=user_simulator_config ).provide(eval_case)