diff --git a/benchmark/runtime/sforge.py b/benchmark/runtime/sforge.py index 62c9d2a943..1218ba3a3c 100644 --- a/benchmark/runtime/sforge.py +++ b/benchmark/runtime/sforge.py @@ -32,6 +32,7 @@ # Experiment profiles, local to the research runner; not new LoopX modes. PROFILES = ("official", "single", "native-goal", "heartbeat-resume", "heartbeat-explore") DEFAULT_TIMEOUT_SECONDS = 18 * 60 * 60 +_RUNTIME_CREDENTIAL_PLACEHOLDER = "runtime-managed-credential" class SForgeEnvironment: @@ -233,10 +234,27 @@ def _install_worker(self, backend, handle, log_dir, logger): effort = self._config.agent_effort common = dict(logs_dir=log_dir / "worker", model_name=self._config.agent_model, reasoning_effort=effort) + runtime_base_url = self._config.agent_api_base_url + runtime_api_key = self._config.agent_api_key if self._config.agent_api_base_url: + route = backend.exec_run( + handle, + ["/bin/bash", "-c", 'printf "%s" "${OPENAI_BASE_URL:-}"'], + user="agent", + ) + runtime_base_url = (getattr(route, "output", "") or "").strip() + if route.exit_code or not runtime_base_url: + raise RuntimeError("Could not read the effective model route from the trial") + if runtime_base_url != self._config.agent_api_base_url: + # SForge may replace a provider route with a host credential + # gateway after constructing this Agent. Keep the provider + # credential on the host and give child runtimes a non-secret + # value; SForge supplies the same effective route to the outer + # solver process. + runtime_api_key = _RUNTIME_CREDENTIAL_PLACEHOLDER common["extra_env"] = { - "OPENAI_BASE_URL": self._config.agent_api_base_url, - "OPENAI_API_KEY": self._config.agent_api_key, + "OPENAI_BASE_URL": runtime_base_url, + "OPENAI_API_KEY": runtime_api_key, "CODEX_AUTH_JSON_PATH": "", } if self.profile in {"official", "single"}: @@ -253,8 +271,8 @@ def _install_worker(self, backend, handle, log_dir, logger): prepare_codex_home( home, execution=Execution(mode="plain"), workspace=Path(self.cwd), model=self._config.agent_model, effort=effort, - base_url=self._config.agent_api_base_url or "", - api_key=self._config.agent_api_key or "", wire_api="responses", skills=None, + base_url=runtime_base_url or "", + api_key=runtime_api_key or "", wire_api="responses", skills=None, model_catalog=offline._model_catalog_source, model_catalog_destination=(Path(MODEL_CATALOG_REMOTE) if offline._model_catalog_staged else None), diff --git a/benchmark/tests/test_sforge_runtime.py b/benchmark/tests/test_sforge_runtime.py index 31e0cb84b8..60dae42b34 100644 --- a/benchmark/tests/test_sforge_runtime.py +++ b/benchmark/tests/test_sforge_runtime.py @@ -170,7 +170,8 @@ def copy(handle, source, target): if source.name == "config.toml": observed.update(tomllib.loads(source.read_text())) backend = SimpleNamespace(copy_to_container=copy, - exec_run=lambda *args, **kwargs: SimpleNamespace(exit_code=0)) + exec_run=lambda *args, **kwargs: SimpleNamespace( + exit_code=0, output="http://example.invalid/v1")) worker = SForgeWorker(SForgeConfig(agent_model="fixture", agent_effort="max", agent_api_base_url="http://example.invalid/v1", agent_api_key="fixture"), profile="official", cwd="/task") @@ -1155,7 +1156,8 @@ async def installed(self, environment): copied = {} def copy(handle, source, target): copied[str(target)] = source.read_text() - backend = SimpleNamespace(exec_run=lambda *a, **k: SimpleNamespace(exit_code=0), + backend = SimpleNamespace(exec_run=lambda *a, **k: SimpleNamespace( + exit_code=0, output=config.agent_api_base_url), copy_to_container=copy) config = SForgeConfig(agent_model='fixture', agent_effort='xhigh', agent_api_base_url='http://127.0.0.1:9876/v1', @@ -1185,6 +1187,61 @@ def copy(handle, source, target): SForgeWorker(config, profile=profile, cwd='/task') +@pytest.mark.parametrize('profile', ['official', 'native-goal']) +def test_custom_provider_preserves_sforge_runtime_gateway(tmp_path, monkeypatch, profile): + pytest.importorskip('sforge') + pytest.importorskip('harbor') + import tomllib + from types import SimpleNamespace + from sforge.harness.agent.codex import CodexAgent + from sforge.harness.config import SForgeConfig + from benchmark.runtime.sforge import SForgeWorker, CodexOffline, BenchmarkCodex + + provider_url = 'https://provider.invalid/v1' + provider_key = 'synthetic-provider-secret' + runtime_url = 'http://host.docker.internal:43123' + monkeypatch.setenv('CODEX_AUTH_JSON_PATH', str(tmp_path / 'must-not-read-auth.json')) + monkeypatch.setenv('OPENAI_API_KEY', provider_key) + monkeypatch.setenv('OPENAI_BASE_URL', provider_url) + + async def installed(self, environment): + pass + + monkeypatch.setattr(CodexOffline, 'install', installed) + monkeypatch.setattr(BenchmarkCodex, 'install', installed) + monkeypatch.setattr(CodexAgent, 'install_stop_hook', lambda *args: None) + copied = {} + + def execute(handle, command, **kwargs): + output = runtime_url if isinstance(command, list) and 'OPENAI_BASE_URL' in command[-1] else '' + return SimpleNamespace(exit_code=0, output=output) + + def copy(handle, source, target): + copied[str(target)] = source.read_text() + + backend = SimpleNamespace(exec_run=execute, copy_to_container=copy) + config = SForgeConfig( + agent_model='fixture', agent_effort='xhigh', + agent_api_base_url=provider_url, agent_api_key=provider_key, + ) + worker = SForgeWorker(config, profile=profile, cwd='/task') + worker.install_stop_hook(backend, None, tmp_path, None) + + if profile == 'official': + settings = tomllib.loads(copied['/home/agent/.codex/config.toml']) + assert settings['model_providers']['harbor']['base_url'] == runtime_url + else: + env = worker.runtime._worker_env(cwd='/task') + assert env['OPENAI_BASE_URL'] == runtime_url + assert env['OPENAI_API_KEY'] == 'runtime-managed-credential' + worker.prepared = True + command = worker.format_run_cmd('/task/prompt.md') + assert provider_url not in command + assert provider_key not in command + assert all(provider_url not in contents for contents in copied.values()) + assert all(provider_key not in contents for contents in copied.values()) + + def test_custom_provider_without_key_fails_before_creating_attempt(tmp_path, monkeypatch): pytest.importorskip('sforge') pytest.importorskip('harbor')