diff --git a/examples/sdk_evolution_agent/behavior.py b/examples/sdk_evolution_agent/behavior.py index 5594972..11d9d2f 100644 --- a/examples/sdk_evolution_agent/behavior.py +++ b/examples/sdk_evolution_agent/behavior.py @@ -6,6 +6,7 @@ import importlib.metadata import inspect import json +import re import subprocess import sys import tempfile @@ -34,6 +35,10 @@ "start_params", } ) +_SENSITIVE_PATH_START_RE = re.compile( + r"(?P^|[^A-Za-z0-9_.-])" + r"(?P(?:[A-Za-z][A-Za-z0-9+.-]*://|[A-Za-z]:[\\/]|\\\\|/))" +) def collect_behavior_evidence( @@ -848,8 +853,11 @@ def _probe_codex(*, version: str | None, scope: str) -> BehaviorProbeResult: def _probe_codex_cli_bin(*, version: str | None, scope: str) -> BehaviorProbeResult: package = "openai-codex-cli-bin" + module_name = DEFAULT_MODULES[package] try: installed = importlib.metadata.version(package) + module = importlib.import_module(module_name) + _require_bundled_codex_file(module) except Exception as exc: return _failed(package, version, scope, "binary-distribution", exc) return BehaviorProbeResult( @@ -858,11 +866,37 @@ def _probe_codex_cli_bin(*, version: str | None, scope: str) -> BehaviorProbeRes scope=scope, probe="binary-distribution", status="pass", - summary="Codex CLI binary distribution metadata is available.", - details={"installed_version": installed}, + summary="Codex CLI binary distribution exposes a bundled executable.", + details={ + "installed_version": installed, + "module": module_name, + "bundled_binary": "regular-file", + }, ) +def _require_bundled_codex_file(module: Any) -> None: + """Require the Codex wheel helper to resolve to a regular file without retaining its path.""" + + try: + raw_path = module.bundled_codex_path() + except Exception as exc: + detail = _safe_exception_detail(exc) + raise RuntimeError(f"codex_cli_bin.bundled_codex_path() failed ({detail})") from exc + try: + bundled_path = Path(raw_path) + is_file = bundled_path.is_file() + except (OSError, TypeError, ValueError) as exc: + detail = _safe_exception_detail(exc) + raise RuntimeError( + f"codex_cli_bin.bundled_codex_path() did not return a usable path ({detail})" + ) from exc + if not is_file: + raise RuntimeError( + "codex_cli_bin.bundled_codex_path() did not return an existing regular file" + ) + + def _probe_antigravity(*, version: str | None, scope: str) -> BehaviorProbeResult: package = "google-antigravity" try: @@ -955,7 +989,7 @@ def _failed( probe: str, exc: Exception, ) -> BehaviorProbeResult: - error = _bounded_text(exc) + error = _safe_exception_detail(exc) if package == "openai-codex-cli-bin" else _bounded_text(exc) return BehaviorProbeResult( package=package, version=version, @@ -1048,6 +1082,15 @@ def _bounded_text(value: object, *, limit: int = 480) -> str: return " ".join(text.split())[:limit] +def _safe_exception_detail(exc: Exception, *, limit: int = 240) -> str: + """Keep useful failure context while removing path-bearing message suffixes.""" + + message = " ".join(str(exc).split()) + match = _SENSITIVE_PATH_START_RE.search(message) + redacted = f"{message[: match.start('path')]}" if match is not None else message + return _bounded_text(f"{type(exc).__name__}: {redacted}", limit=limit) + + def _string_or_none(value: object) -> str | None: if value is None: return None @@ -1056,14 +1099,30 @@ def _string_or_none(value: object) -> str | None: _PROBE_SCRIPT = textwrap.dedent( - """ + r""" import importlib import importlib.metadata import inspect import json + import re import sys + from pathlib import Path package, version, scope = sys.argv[1:4] + sensitive_path_start_re = re.compile( + r"(?P^|[^A-Za-z0-9_.-])" + r"(?P(?:[A-Za-z][A-Za-z0-9+.-]*://|[A-Za-z]:[\\/]|\\\\|/))" + ) + + def safe_exception_detail(exc, limit=240): + message = " ".join(str(exc).split()) + match = sensitive_path_start_re.search(message) + redacted = ( + f"{message[:match.start('path')]}" + if match is not None + else message + ) + return f"{type(exc).__name__}: {redacted}"[:limit] def fields(cls): if hasattr(cls, "model_fields"): @@ -1076,14 +1135,19 @@ def fields(cls): return set() def failed(probe, exc): + detail = ( + safe_exception_detail(exc) + if package == "openai-codex-cli-bin" + else str(exc) + ) return { "package": package, "version": version, "scope": scope, "probe": probe, "status": "fail", - "summary": str(exc), - "details": {"error": str(exc)}, + "summary": detail, + "details": {"error": detail}, } def result(probe, status, summary, details): @@ -1142,11 +1206,36 @@ def result(probe, status, summary, details): )] elif package == "openai-codex-cli-bin": installed = importlib.metadata.version(package) + module = importlib.import_module("codex_cli_bin") + try: + raw_path = module.bundled_codex_path() + except Exception as exc: + detail = safe_exception_detail(exc) + raise RuntimeError( + f"codex_cli_bin.bundled_codex_path() failed ({detail})" + ) from exc + try: + bundled_path = Path(raw_path) + is_file = bundled_path.is_file() + except (OSError, TypeError, ValueError) as exc: + detail = safe_exception_detail(exc) + raise RuntimeError( + "codex_cli_bin.bundled_codex_path() did not return a usable path " + f"({detail})" + ) from exc + if not is_file: + raise RuntimeError( + "codex_cli_bin.bundled_codex_path() did not return an existing regular file" + ) payload = [result( "binary-distribution", "pass", - "Codex CLI binary distribution metadata is available.", - {"installed_version": installed}, + "Codex CLI binary distribution exposes a bundled executable.", + { + "installed_version": installed, + "module": "codex_cli_bin", + "bundled_binary": "regular-file", + }, )] elif package == "google-antigravity": importlib.import_module("google.antigravity") @@ -1178,7 +1267,8 @@ def result(probe, status, summary, details): else: payload = [result("package-import", "skip", "No behavior probe is defined.", {})] except Exception as exc: - payload = [failed("adapter-contract", exc)] + probe = "binary-distribution" if package == "openai-codex-cli-bin" else "adapter-contract" + payload = [failed(probe, exc)] print(json.dumps(payload, sort_keys=True)) """ diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py index 31f80e8..dbca816 100644 --- a/examples/sdk_evolution_agent/snapshots.py +++ b/examples/sdk_evolution_agent/snapshots.py @@ -19,7 +19,7 @@ DEFAULT_MODULES = { "claude-agent-sdk": "claude_agent_sdk", "openai-codex": "openai_codex", - "openai-codex-cli-bin": "openai_codex_cli_bin", + "openai-codex-cli-bin": "codex_cli_bin", "google-antigravity": "google.antigravity", } diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 6250f59..4fd3eb4 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -28,12 +28,14 @@ CodexAgentRuntime, ) from examples.sdk_evolution_agent import auth as auth_module +from examples.sdk_evolution_agent import behavior as behavior_module from examples.sdk_evolution_agent.auth import CodexAuthResult, ensure_codex_sdk_auth from examples.sdk_evolution_agent.behavior import ( assess_behavior_payload, collect_behavior_evidence, diff_behavior_results, probe_candidate_in_venv, + probe_current_package, summarize_behavior, ) from examples.sdk_evolution_agent.cli import RunOptions, _collect_snapshots, parse_args, run_agent @@ -69,6 +71,7 @@ validate_mapping, ) from examples.sdk_evolution_agent.snapshots import ( + DEFAULT_MODULES, diff_snapshots, snapshot_candidate_in_venv, snapshot_current_api, @@ -945,6 +948,335 @@ def fake_run(args: Any, **kwargs: Any) -> Any: assert len(result.summary) <= 560 +@pytest.mark.parametrize( + ("private_path", "secret_suffix"), + [ + ("/tmp/private folder/POSIX_SECRET_SUFFIX", "POSIX_SECRET_SUFFIX"), + (r"C:\Program Files\Private\DRIVE_SECRET_SUFFIX", "DRIVE_SECRET_SUFFIX"), + (r"\\server\Private Share\UNC_SECRET_SUFFIX", "UNC_SECRET_SUFFIX"), + ( + "https://example.invalid/private folder/URL_SECRET_SUFFIX", + "URL_SECRET_SUFFIX", + ), + ], +) +def test_codex_cli_exception_redaction_discards_entire_path_suffix( + private_path: str, + secret_suffix: str, +) -> None: + detail = behavior_module._safe_exception_detail( + ValueError( + f"REDACTION_SENTINEL at {private_path} trailing {secret_suffix} " + + ("x" * 1_000) + ) + ) + + assert detail == "ValueError: REDACTION_SENTINEL at " + assert private_path not in detail + assert secret_suffix not in detail + assert len(detail) <= 240 + + +@pytest.mark.parametrize( + ("failure", "expected_summary"), + [ + ("metadata", "RuntimeError: METADATA_SENTINEL"), + ("module", "ModuleNotFoundError: MODULE_SENTINEL"), + ("helper-missing", "bundled_codex_path() failed"), + ("helper-error", "ValueError: HELPER_SENTINEL"), + ("invalid-return", "TypeError"), + ("missing-file", "existing regular file"), + ("directory", "existing regular file"), + ], +) +def test_codex_cli_binary_probe_contains_each_failure( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str, + expected_summary: str, +) -> None: + metadata_path = tmp_path / "private metadata" / "METADATA_SECRET_SUFFIX" + drive_path = r"C:\Program Files\Private\MODULE_SECRET_SUFFIX" + unc_path = r"\\server\Private Share\HELPER_SECRET_SUFFIX" + + def metadata_version(package: str) -> str: + assert package == "openai-codex-cli-bin" + if failure == "metadata": + raise RuntimeError( + f"METADATA_SENTINEL at {metadata_path} " + ("m" * 1_000) + ) + return "1.2.3" + + def helper_error() -> Path: + raise ValueError( + f"HELPER_SENTINEL at {unc_path} " + ("h" * 1_000) + ) + + module = types.SimpleNamespace(bundled_codex_path=lambda: tmp_path / "codex") + if failure == "helper-missing": + module = types.SimpleNamespace() + elif failure == "helper-error": + module = types.SimpleNamespace(bundled_codex_path=helper_error) + elif failure == "invalid-return": + module = types.SimpleNamespace(bundled_codex_path=lambda: None) + elif failure == "directory": + module = types.SimpleNamespace(bundled_codex_path=lambda: tmp_path) + + def import_module(name: str) -> Any: + assert name == "codex_cli_bin" + if failure == "module": + raise ModuleNotFoundError( + f"MODULE_SENTINEL: No module named codex_cli_bin at {drive_path} " + + ("i" * 1_000) + ) + return module + + monkeypatch.setattr(behavior_module.importlib.metadata, "version", metadata_version) + monkeypatch.setattr(behavior_module.importlib, "import_module", import_module) + + (result,) = probe_current_package("openai-codex-cli-bin", version="1.2.3") + + assert result.probe == "binary-distribution" + assert result.status == "fail" + assert expected_summary in result.summary + assert str(tmp_path) not in result.summary + assert drive_path not in result.summary + assert unc_path not in result.summary + serialized_details = json.dumps(result.details) + assert str(tmp_path) not in serialized_details + assert drive_path not in serialized_details + assert unc_path not in serialized_details + for secret in ( + "METADATA_SECRET_SUFFIX", + "MODULE_SECRET_SUFFIX", + "HELPER_SECRET_SUFFIX", + ): + assert secret not in result.summary + assert secret not in serialized_details + assert result.details["error"] == result.summary + assert len(result.summary) <= 240 + assert len(result.details["error"]) <= 240 + if failure in {"metadata", "module", "helper-error"}: + assert "" in result.summary + + +@pytest.mark.parametrize( + ("failure", "expected_summary", "secret_suffix"), + [ + ("conversion", "ValueError: CONVERSION_SENTINEL", "CONVERSION_SECRET_SUFFIX"), + ("file-check", "OSError: FILE_CHECK_SENTINEL", "FILE_CHECK_SECRET_SUFFIX"), + ], +) +def test_codex_cli_binary_probe_redacts_path_conversion_and_file_check_errors( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + failure: str, + expected_summary: str, + secret_suffix: str, +) -> None: + conversion_url = ( + "https://example.invalid/private folder/CONVERSION_SECRET_SUFFIX" + ) + file_check_path = tmp_path / "private file check" / "FILE_CHECK_SECRET_SUFFIX" + + class InvalidPath: + def __fspath__(self) -> str: + raise ValueError( + f"CONVERSION_SENTINEL at {conversion_url} " + ("c" * 1_000) + ) + + if failure == "file-check": + + def fail_file_check(self: Path) -> bool: + del self + raise OSError( + f"FILE_CHECK_SENTINEL at {file_check_path} " + ("f" * 1_000) + ) + + monkeypatch.setattr(Path, "is_file", fail_file_check) + + raw_path: object = InvalidPath() if failure == "conversion" else tmp_path / "codex" + module = types.SimpleNamespace(bundled_codex_path=lambda: raw_path) + monkeypatch.setattr( + behavior_module.importlib.metadata, + "version", + lambda package: "1.2.3", + ) + monkeypatch.setattr( + behavior_module.importlib, + "import_module", + lambda name: module, + ) + + (result,) = probe_current_package("openai-codex-cli-bin", version="1.2.3") + + assert result.probe == "binary-distribution" + assert result.status == "fail" + assert expected_summary in result.summary + assert "" in result.summary + assert secret_suffix not in result.summary + assert secret_suffix not in json.dumps(result.details) + assert result.details["error"] == result.summary + assert len(result.summary) <= 240 + assert len(result.details["error"]) <= 240 + + +def test_codex_cli_binary_probe_requires_only_a_regular_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + binary = tmp_path / "codex" + binary.write_bytes(b"bundled binary") + module = types.SimpleNamespace(bundled_codex_path=lambda: binary) + monkeypatch.setattr( + behavior_module.importlib.metadata, + "version", + lambda package: "1.2.3", + ) + monkeypatch.setattr( + behavior_module.importlib, + "import_module", + lambda name: module, + ) + + (result,) = probe_current_package("openai-codex-cli-bin", version="1.2.3") + + assert result.status == "pass" + assert result.probe == "binary-distribution" + assert result.details == { + "installed_version": "1.2.3", + "module": "codex_cli_bin", + "bundled_binary": "regular-file", + } + assert str(binary) not in json.dumps(result.details) + + +@pytest.mark.parametrize( + ("failure", "expected_summary", "secret_suffix"), + [ + ( + "metadata", + "RuntimeError: EMBEDDED_METADATA_SENTINEL", + "EMBEDDED_METADATA_SECRET_SUFFIX", + ), + ( + "module", + "ModuleNotFoundError: EMBEDDED_MODULE_SENTINEL", + "EMBEDDED_MODULE_SECRET_SUFFIX", + ), + ( + "helper-error", + "ValueError: EMBEDDED_HELPER_SENTINEL", + "EMBEDDED_HELPER_SECRET_SUFFIX", + ), + ( + "conversion-error", + "ValueError: EMBEDDED_CONVERSION_SENTINEL", + "EMBEDDED_CONVERSION_SECRET_SUFFIX", + ), + ( + "file-check-error", + "OSError: EMBEDDED_FILE_CHECK_SENTINEL", + "EMBEDDED_FILE_CHECK_SECRET_SUFFIX", + ), + ("missing-file", "existing regular file", None), + ], +) +def test_embedded_codex_cli_failures_keep_binary_probe_label( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + failure: str, + expected_summary: str, + secret_suffix: str | None, +) -> None: + metadata_path = ( + tmp_path / "private metadata" / "EMBEDDED_METADATA_SECRET_SUFFIX" + ) + drive_path = r"D:\Program Files\Private\EMBEDDED_MODULE_SECRET_SUFFIX" + unc_path = r"\\candidate-host\Private Share\EMBEDDED_HELPER_SECRET_SUFFIX" + conversion_url = ( + "https://example.invalid/private folder/EMBEDDED_CONVERSION_SECRET_SUFFIX" + ) + file_check_path = ( + tmp_path / "private file check" / "EMBEDDED_FILE_CHECK_SECRET_SUFFIX" + ) + + def fail_metadata(package: str) -> str: + if failure == "metadata": + raise RuntimeError( + f"EMBEDDED_METADATA_SENTINEL for {package} at {metadata_path} " + + ("m" * 1_000) + ) + return "1.2.3" + + def helper_error() -> Path: + raise ValueError( + f"EMBEDDED_HELPER_SENTINEL at {unc_path} " + ("h" * 1_000) + ) + + class InvalidPath: + def __fspath__(self) -> str: + raise ValueError( + f"EMBEDDED_CONVERSION_SENTINEL at {conversion_url} " + + ("c" * 1_000) + ) + + def bundled_path() -> object: + if failure == "helper-error": + return helper_error() + if failure == "conversion-error": + return InvalidPath() + return tmp_path / "missing" + + if failure == "file-check-error": + + def fail_file_check(self: Path) -> bool: + del self + raise OSError( + f"EMBEDDED_FILE_CHECK_SENTINEL at {file_check_path} " + + ("f" * 1_000) + ) + + monkeypatch.setattr(Path, "is_file", fail_file_check) + + def import_module(name: str) -> Any: + assert name == "codex_cli_bin" + if failure == "module": + raise ModuleNotFoundError( + f"EMBEDDED_MODULE_SENTINEL at {drive_path} " + ("i" * 1_000) + ) + return module + + helper = bundled_path + module = types.SimpleNamespace(bundled_codex_path=helper) + + monkeypatch.setattr(behavior_module.importlib.metadata, "version", fail_metadata) + monkeypatch.setattr(behavior_module.importlib, "import_module", import_module) + monkeypatch.setattr( + sys, + "argv", + ["probe", "openai-codex-cli-bin", "1.2.3", "candidate"], + ) + + exec(behavior_module._PROBE_SCRIPT, {}) + + [result] = json.loads(capsys.readouterr().out) + assert result["probe"] == "binary-distribution" + assert result["status"] == "fail" + assert expected_summary in result["summary"] + assert str(tmp_path) not in result["summary"] + assert drive_path not in result["summary"] + assert unc_path not in result["summary"] + if secret_suffix is not None: + assert secret_suffix not in result["summary"] + assert secret_suffix not in json.dumps(result["details"]) + assert "" in result["summary"] + assert result["details"]["error"] == result["summary"] + assert len(result["summary"]) <= 240 + assert len(result["details"]["error"]) <= 240 + + def test_behavior_summary_accepts_valid_no_update_and_unchanged_evidence() -> None: baseline = _probe( "claude-agent-sdk", @@ -1535,6 +1867,73 @@ def test_report_persists_and_renders_recomputed_behavior_summary(tmp_path: Path) assert "failed the required contract" in report +def test_snapshot_module_mapping_preserves_distribution_identities() -> None: + expected = { + "claude-agent-sdk": "claude_agent_sdk", + "openai-codex": "openai_codex", + "openai-codex-cli-bin": "codex_cli_bin", + "google-antigravity": "google.antigravity", + } + + assert {package: DEFAULT_MODULES[package] for package in expected} == expected + + +def test_current_codex_cli_snapshot_uses_real_import_module( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = types.ModuleType("codex_cli_bin") + module.bundled_codex_path = lambda: Path("unused") + monkeypatch.setitem(sys.modules, "codex_cli_bin", module) + + snapshot = snapshot_current_api("openai-codex-cli-bin", version="1.2.3") + + assert snapshot.package == "openai-codex-cli-bin" + assert snapshot.module == "codex_cli_bin" + assert snapshot.import_error is None + + +def test_isolated_codex_cli_snapshot_receives_real_import_module( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[Any, ...]] = [] + + def fake_run(args: Any, **kwargs: Any) -> Any: + del kwargs + calls.append(tuple(args)) + payload = ( + '{"package":"openai-codex-cli-bin","version":"1.2.3",' + '"module":"codex_cli_bin","members":[],"import_error":null}' + ) + return types.SimpleNamespace( + stdout=payload if len(calls) == 3 else "", + stderr="", + returncode=0, + ) + + monkeypatch.setattr("examples.sdk_evolution_agent.snapshots.subprocess.run", fake_run) + + snapshot = snapshot_candidate_in_venv("openai-codex-cli-bin", "1.2.3") + + assert calls[1][-1] == "openai-codex-cli-bin==1.2.3" + assert calls[2][-1] == "codex_cli_bin" + assert snapshot.package == "openai-codex-cli-bin" + assert snapshot.module == "codex_cli_bin" + assert snapshot.import_error is None + + +def test_real_codex_cli_snapshot_and_probe_when_extra_is_installed() -> None: + pytest.importorskip("codex_cli_bin", reason="Codex extra is not installed") + + snapshot = snapshot_current_api("openai-codex-cli-bin") + (probe,) = probe_current_package("openai-codex-cli-bin", version=snapshot.version) + + assert snapshot.module == "codex_cli_bin" + assert snapshot.import_error is None + assert probe.probe == "binary-distribution" + assert probe.status == "pass" + assert probe.details["bundled_binary"] == "regular-file" + + def test_snapshot_and_diff_public_api(monkeypatch: pytest.MonkeyPatch) -> None: module = types.ModuleType("fake_sdk")