diff --git a/examples/sdk_evolution_agent/behavior.py b/examples/sdk_evolution_agent/behavior.py index 96317b4..b45a264 100644 --- a/examples/sdk_evolution_agent/behavior.py +++ b/examples/sdk_evolution_agent/behavior.py @@ -41,16 +41,14 @@ def collect_behavior_evidence( continue locked_version = _string_or_none(package.get("locked_version")) installed_version = _string_or_none(package.get("installed_version")) - current_version = locked_version or installed_version if ( inspect_candidates and locked_version - and installed_version and locked_version != installed_version ): results.extend(probe_candidate_in_venv(name, locked_version, scope="current-baseline")) else: - results.extend(probe_current_package(name, version=current_version)) + results.extend(probe_current_package(name, version=installed_version)) candidate = update_versions.get(name) if candidate: if inspect_candidates: diff --git a/examples/sdk_evolution_agent/cli.py b/examples/sdk_evolution_agent/cli.py index b776339..87461df 100644 --- a/examples/sdk_evolution_agent/cli.py +++ b/examples/sdk_evolution_agent/cli.py @@ -526,10 +526,10 @@ def _collect_snapshots(evidence: dict[str, Any], *, inspect_candidates: bool = F locked = package.get("locked_version") installed = package.get("installed_version") baseline = locked or installed - if inspect_candidates and locked and installed and locked != installed: + if inspect_candidates and locked and locked != installed: snapshots.append(snapshot_candidate_in_venv(name, str(locked))) else: - snapshots.append(snapshot_current_api(name, version=baseline)) + snapshots.append(snapshot_current_api(name, version=installed)) if not inspect_candidates: continue candidate = update_versions.get(name) diff --git a/examples/sdk_evolution_agent/snapshots.py b/examples/sdk_evolution_agent/snapshots.py index f198536..31f80e8 100644 --- a/examples/sdk_evolution_agent/snapshots.py +++ b/examples/sdk_evolution_agent/snapshots.py @@ -128,51 +128,111 @@ def snapshot_candidate_in_venv( """Inspect a candidate version in an isolated temporary virtualenv.""" module_name = DEFAULT_MODULES.get(package, package.replace("-", "_")) - with tempfile.TemporaryDirectory(prefix="ark-sdk-snapshot-") as directory: - venv = Path(directory) / ".venv" - # Scrub the environment for every subprocess that touches freshly downloaded - # upstream code: give it a throwaway HOME and only PATH, so a malicious or - # buggy candidate package cannot read the caller's credentials/config. - env = isolated_env(Path(directory)) - subprocess.run( - (python, "-m", "venv", str(venv)), check=True, timeout=timeout, env=env + step = "virtual environment creation" + try: + with tempfile.TemporaryDirectory(prefix="ark-sdk-snapshot-") as directory: + venv = Path(directory) / ".venv" + # Scrub the environment for every subprocess that touches freshly downloaded + # upstream code: give it a throwaway HOME and only PATH, so a malicious or + # buggy candidate package cannot read the caller's credentials/config. + env = isolated_env(Path(directory)) + subprocess.run( + (python, "-m", "venv", str(venv)), + check=True, + timeout=timeout, + env=env, + ) + bin_dir = "Scripts" if sys.platform == "win32" else "bin" + venv_python = venv / bin_dir / "python" + step = "package installation" + subprocess.run( + (str(venv_python), "-m", "pip", "install", f"{package}=={version}"), + check=True, + text=True, + capture_output=True, + timeout=timeout, + env=env, + ) + step = "snapshot execution" + completed = subprocess.run( + ( + str(venv_python), + "-c", + _SNAPSHOT_SCRIPT, + package, + version, + module_name, + ), + check=True, + text=True, + capture_output=True, + timeout=timeout, + env=env, + ) + except subprocess.TimeoutExpired as exc: + return _failed_isolated_snapshot( + package, + version, + module_name, + f"{step} timed out after {exc.timeout}s", ) - bin_dir = "Scripts" if sys.platform == "win32" else "bin" - venv_python = venv / bin_dir / "python" - subprocess.run( - (str(venv_python), "-m", "pip", "install", f"{package}=={version}"), - check=True, - text=True, - capture_output=True, - timeout=timeout, - env=env, + except subprocess.CalledProcessError as exc: + detail = _bounded_failure_detail(exc.stderr or exc.stdout or str(exc)) + return _failed_isolated_snapshot( + package, + version, + module_name, + f"{step} failed: {detail}", ) - completed = subprocess.run( - ( - str(venv_python), - "-c", - _SNAPSHOT_SCRIPT, - package, - version, - module_name, - ), - check=True, - text=True, - capture_output=True, - timeout=timeout, - env=env, + except OSError as exc: + return _failed_isolated_snapshot( + package, + version, + module_name, + f"{step} failed: {_bounded_failure_detail(exc)}", ) - raw = json.loads(completed.stdout) + + try: + raw = json.loads(completed.stdout) + return ApiSnapshot( + package=raw["package"], + version=raw["version"], + module=raw["module"], + members=tuple(ApiMember(**item) for item in raw.get("members", ())), + import_error=raw.get("import_error"), + source="isolated-venv", + ) + except (json.JSONDecodeError, KeyError, TypeError, ValueError) as exc: + output = _bounded_failure_detail(completed.stdout) + detail = f"malformed snapshot output: {exc}" + if output: + detail += f"; stdout={output}" + return _failed_isolated_snapshot(package, version, module_name, detail) + + +def _failed_isolated_snapshot( + package: str, + version: str, + module_name: str, + error: str, +) -> ApiSnapshot: return ApiSnapshot( - package=raw["package"], - version=raw["version"], - module=raw["module"], - members=tuple(ApiMember(**item) for item in raw.get("members", ())), - import_error=raw.get("import_error"), + package=package, + version=version, + module=module_name, + import_error=_bounded_failure_detail(error, limit=560), source="isolated-venv", ) +def _bounded_failure_detail(value: object, *, limit: int = 480) -> str: + if isinstance(value, bytes): + text = value.decode(errors="replace") + else: + text = str(value) + return " ".join(text.split())[:limit] + + def _member_kind(value: Any) -> str: if inspect.isclass(value): return "class" diff --git a/tests/test_sdk_evolution_agent.py b/tests/test_sdk_evolution_agent.py index 81b8817..b3dfec2 100644 --- a/tests/test_sdk_evolution_agent.py +++ b/tests/test_sdk_evolution_agent.py @@ -2,6 +2,7 @@ import os import stat +import subprocess import sys import types from pathlib import Path @@ -53,7 +54,11 @@ SchemaValidationError, validate_mapping, ) -from examples.sdk_evolution_agent.snapshots import diff_snapshots, snapshot_current_api +from examples.sdk_evolution_agent.snapshots import ( + diff_snapshots, + snapshot_candidate_in_venv, + snapshot_current_api, +) from examples.sdk_evolution_agent.stages import ( SDK_EVOLUTION_CODEX_HOME, SDK_EVOLUTION_CODEX_MODEL, @@ -492,6 +497,156 @@ def isolated(package: str, version: str, *, scope: str = "candidate"): assert behavior["diffs"][0].severity == "changed" +def test_behavior_evidence_uses_locked_baseline_when_sdk_not_installed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str, str]] = [] + + def isolated(package: str, version: str, *, scope: str = "candidate"): + calls.append((package, version, scope)) + return (_probe(package, version, scope, "pass", {"scope": scope}),) + + def current(package: str, *, version: str | None = None): + raise AssertionError(f"ambient probe must not represent {package} {version}") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_candidate_in_venv", + isolated, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_current_package", + current, + ) + + collect_behavior_evidence( + [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.106", + "installed_version": None, + } + ], + {"claude-agent-sdk": "0.2.110"}, + inspect_candidates=True, + ) + + assert calls == [ + ("claude-agent-sdk", "0.2.106", "current-baseline"), + ("claude-agent-sdk", "0.2.110", "candidate"), + ] + + +def test_behavior_evidence_without_opt_in_reports_actual_ambient_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str | None]] = [] + + def current(package: str, *, version: str | None = None): + calls.append((package, version)) + return (_probe(package, version, "current-environment", "fail", {}),) + + def isolated(package: str, version: str, *, scope: str = "candidate"): + raise AssertionError(f"isolated probe must not run for {package} {version} {scope}") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_current_package", + current, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_candidate_in_venv", + isolated, + ) + + behavior = collect_behavior_evidence( + [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.106", + "installed_version": None, + } + ], + {"claude-agent-sdk": "0.2.110"}, + ) + + assert calls == [("claude-agent-sdk", None)] + assert [result.status for result in behavior["results"]] == ["fail", "skip"] + + +def test_behavior_evidence_without_opt_in_uses_drifted_installed_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str | None]] = [] + + def current(package: str, *, version: str | None = None): + calls.append((package, version)) + return (_probe(package, version, "current-environment", "pass", {}),) + + def isolated(package: str, version: str, *, scope: str = "candidate"): + raise AssertionError(f"isolated probe must not run for {package} {version} {scope}") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_current_package", + current, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_candidate_in_venv", + isolated, + ) + + collect_behavior_evidence( + [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.96", + "installed_version": "0.2.106", + } + ], + {"claude-agent-sdk": "0.2.110"}, + ) + + assert calls == [("claude-agent-sdk", "0.2.106")] + + +def test_behavior_evidence_reuses_matching_ambient_baseline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str, str | None]] = [] + + def current(package: str, *, version: str | None = None): + calls.append(("current", package, version)) + return (_probe(package, version, "current-environment", "pass", {}),) + + def isolated(package: str, version: str, *, scope: str = "candidate"): + calls.append((scope, package, version)) + return (_probe(package, version, scope, "pass", {}),) + + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_current_package", + current, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.behavior.probe_candidate_in_venv", + isolated, + ) + + collect_behavior_evidence( + [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.106", + "installed_version": "0.2.106", + } + ], + {"claude-agent-sdk": "0.2.110"}, + inspect_candidates=True, + ) + + assert calls == [ + ("current", "claude-agent-sdk", "0.2.106"), + ("candidate", "claude-agent-sdk", "0.2.110"), + ] + + def test_behavior_candidate_probes_are_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: # Probing a candidate pip-installs and imports freshly downloaded upstream # code; without --inspect-candidates that must never happen, and the gap @@ -646,6 +801,111 @@ def run_new(value: str, *, verbose: bool = False) -> str: assert diff.changed == ("run",) +@pytest.mark.parametrize( + ("failure_call", "detail"), + [ + (1, "venv creation failed"), + (2, "package install failed"), + (3, "snapshot execution failed"), + ], +) +def test_candidate_snapshot_records_subprocess_failures( + monkeypatch: pytest.MonkeyPatch, + failure_call: int, + detail: str, +) -> None: + calls = 0 + + def fake_run(args: Any, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + if calls == failure_call: + raise subprocess.CalledProcessError( + returncode=1, + cmd=args, + stderr=(detail + " ") * 100, + ) + return types.SimpleNamespace(stdout="{}", stderr="", returncode=0) + + monkeypatch.setattr("examples.sdk_evolution_agent.snapshots.subprocess.run", fake_run) + + snapshot = snapshot_candidate_in_venv("claude-agent-sdk", "0.2.110") + + assert snapshot.package == "claude-agent-sdk" + assert snapshot.version == "0.2.110" + assert snapshot.source == "isolated-venv" + assert snapshot.import_error is not None + assert detail in snapshot.import_error + assert len(snapshot.import_error) <= 600 + + +def test_candidate_snapshot_records_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + def fake_run(args: Any, **kwargs: Any) -> Any: + raise subprocess.TimeoutExpired(cmd=args, timeout=kwargs["timeout"]) + + monkeypatch.setattr("examples.sdk_evolution_agent.snapshots.subprocess.run", fake_run) + + snapshot = snapshot_candidate_in_venv("claude-agent-sdk", "0.2.110", timeout=7) + + assert snapshot.import_error is not None + assert "timed out after 7s" in snapshot.import_error + assert snapshot.source == "isolated-venv" + + +def test_candidate_snapshot_records_malformed_output(monkeypatch: pytest.MonkeyPatch) -> None: + calls = 0 + + def fake_run(args: Any, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + return types.SimpleNamespace( + stdout="not-json" if calls == 3 else "", + stderr="", + returncode=0, + ) + + monkeypatch.setattr("examples.sdk_evolution_agent.snapshots.subprocess.run", fake_run) + + snapshot = snapshot_candidate_in_venv("claude-agent-sdk", "0.2.110") + + assert snapshot.import_error is not None + assert "malformed snapshot output" in snapshot.import_error + assert snapshot.source == "isolated-venv" + + +def test_candidate_snapshot_scrubs_subprocess_env( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured_envs: list[dict[str, str] | None] = [] + calls = 0 + + def fake_run(args: Any, **kwargs: Any) -> Any: + nonlocal calls + calls += 1 + captured_envs.append(kwargs.get("env")) + payload = ( + '{"package":"claude-agent-sdk","version":"0.2.110",' + '"module":"claude_agent_sdk","members":[],"import_error":null}' + ) + return types.SimpleNamespace( + stdout=payload if calls == 3 else "", + stderr="", + returncode=0, + ) + + monkeypatch.setenv("ARK_FAKE_SECRET", "not-for-snapshots") + monkeypatch.setattr("examples.sdk_evolution_agent.snapshots.subprocess.run", fake_run) + + snapshot = snapshot_candidate_in_venv("claude-agent-sdk", "0.2.110") + + assert snapshot.import_error is None + assert len(captured_envs) == 3 + for env in captured_envs: + assert env is not None + assert "ARK_FAKE_SECRET" not in env + assert env.get("HOME") != os.environ.get("HOME") + + def test_parse_args_candidate_inspection_is_opt_in() -> None: # Candidate inspection pip-installs+imports upstream code, so it is off by # default and only enabled with the explicit flag. @@ -803,6 +1063,56 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot: ] +def test_collect_snapshots_uses_locked_baseline_when_sdk_not_installed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str, str | None]] = [] + + def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot: + raise AssertionError(f"ambient snapshot must not represent {package} {version}") + + def isolated_snapshot(package: str, version: str) -> ApiSnapshot: + calls.append(("isolated", package, version)) + return ApiSnapshot( + package=package, + version=version, + module=package.replace("-", "_"), + source="isolated-venv", + ) + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + current_snapshot, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", + isolated_snapshot, + ) + + _collect_snapshots( + { + "packages": [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.106", + "installed_version": None, + "latest_version": "0.2.110", + }, + ], + "refresh_preview": { + "stdout": "", + "stderr": "Update claude-agent-sdk v0.2.106 -> v0.2.110\n", + }, + }, + inspect_candidates=True, + ) + + assert calls == [ + ("isolated", "claude-agent-sdk", "0.2.106"), + ("isolated", "claude-agent-sdk", "0.2.110"), + ] + + def test_collect_snapshots_without_opt_in_never_installs_even_when_lock_drifted( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -846,7 +1156,55 @@ def isolated_snapshot(package: str, version: str) -> ApiSnapshot: inspect_candidates=False, ) - assert calls == [("current", "claude-agent-sdk", "0.2.96")] + assert calls == [("current", "claude-agent-sdk", "0.2.106")] + + +def test_collect_snapshots_without_opt_in_reports_missing_ambient_sdk( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[tuple[str, str, str | None]] = [] + + def current_snapshot(package: str, *, version: str | None = None) -> ApiSnapshot: + calls.append(("current", package, version)) + return ApiSnapshot( + package=package, + version=version, + module=package.replace("-", "_"), + import_error="not installed", + ) + + def isolated_snapshot(package: str, version: str) -> ApiSnapshot: + raise AssertionError(f"isolated snapshot must not run for {package} {version}") + + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_current_api", + current_snapshot, + ) + monkeypatch.setattr( + "examples.sdk_evolution_agent.cli.snapshot_candidate_in_venv", + isolated_snapshot, + ) + + snapshots = _collect_snapshots( + { + "packages": [ + { + "name": "claude-agent-sdk", + "locked_version": "0.2.106", + "installed_version": None, + "latest_version": "0.2.110", + } + ], + "refresh_preview": { + "stdout": "", + "stderr": "Update claude-agent-sdk v0.2.106 -> v0.2.110\n", + }, + }, + inspect_candidates=False, + ) + + assert calls == [("current", "claude-agent-sdk", None)] + assert snapshots[0].import_error == "not installed" def test_candidate_api_diff_guard_blocks_missing_update_diff() -> None: