From f2229d1187d978bf97df31d621cb0fa45c01db84 Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Thu, 22 Oct 2026 13:07:59 -0700 Subject: [PATCH] feat(compat): add opt-in AlphaEvolve bundle launcher --- examples/alphaevolve_interop/README.md | 63 +++- openevolve/compat/alphaevolve_bundle.py | 70 +++- openevolve/compat/alphaevolve_launch.py | 257 ++++++++++++++ tests/test_alphaevolve_bundle.py | 1 + tests/test_alphaevolve_launch.py | 438 ++++++++++++++++++++++++ 5 files changed, 814 insertions(+), 15 deletions(-) create mode 100644 openevolve/compat/alphaevolve_launch.py create mode 100644 tests/test_alphaevolve_launch.py diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 7b2c71af1..4178d2de4 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -229,6 +229,9 @@ python -m openevolve.compat.alphaevolve_bundle \ # Review/install the evaluator's dependencies in your environment first. python ./alphaevolve-bundle/check.py + +# Offline preflight; does not import the SDK/evaluator or execute either source. +python ./alphaevolve-bundle/run_alphaevolve.py ``` The destination must not exist, and its parent must already exist. Exporting @@ -251,6 +254,7 @@ alphaevolve-bundle/ openevolve_evaluator.py # Original evaluator bytes evaluate.py # Reverse adapter entry point check.py + run_alphaevolve.py # Offline preflight by default; explicit --run for cloud use README.md requirements.txt # Only when explicitly supplied ``` @@ -275,19 +279,64 @@ evaluation to stdout and redirects Python-level evaluator/import stdout to stderr. Evaluators must capture their own subprocess output to keep stdout clean. The check also works by absolute path from another directory. Import/contract errors and adapter-reported failures produce a nonzero exit -status. The check executes -trusted code with the current process's privileges; it adds no sandbox or -timeout and cannot recognize failures silently converted into numeric scores. +status. The check executes trusted code with the current process's privileges; +it adds no sandbox or timeout and cannot recognize failures silently converted +into numeric scores. + +### Preflight and opt-in cloud launch + +`run_alphaevolve.py` defaults to a basic offline JSON summary of the bundle, +models and generation budget. It checks the exported configuration subset, +evaluation options and Python sources without importing the SDK or evaluator, +executing a seed, looking up credentials or making requests. `--dry-run` is an +explicit alias for this default. Dependency installation, full SDK schemas, +permissions and live model availability are not checked in this mode. + +For cloud use, install the official SDK from a reviewed checkout of +[Google's repository](https://github.com/Google-Cloud-AI/alphaevolve-on-googlecloud) +and follow its README for ADC, project, engine and service access setup. Review +the generation budget and select currently available models explicitly during +export; the Gemini-name check is not a live model catalog. Then opt in: + +```bash +python ./alphaevolve-bundle/run_alphaevolve.py --run \ + --project-id YOUR_PROJECT_ID --engine YOUR_ENGINE_ID \ + --max-programs-evaluated 1 +``` -This is an offline-checkable export slice. It does not create a cloud client, -start an experiment, enforce a cost limit or reproduce the OpenEvolve evaluation -pipeline and search behavior. Managed-service authentication and execution -remain separate operations that may incur charges. +The project, engine and positive evaluation target are required for `--run`. +`--location` accepts `global` (default), `us` or `eu`. No project/engine values +are inferred from `.env` or environment variables; Google credentials follow the +SDK's ADC behavior only after cloud use is requested. There is no endpoint override. + +The launcher validates the configuration with the real SDK and evaluates the +seed locally before constructing a client. It creates a new experiment, uploads +the seed with its actual scores, checks seed creation and start responses, then +invokes the SDK controller with one sampler and one synchronous evaluator. This +avoids requiring thread-safe or signal-free evaluators. The generated entry point +resolves the bundle relative to itself and works from another directory. Existing +bundles can also use `python -m openevolve.compat.alphaevolve_launch /path/to/bundle`. + +The final run JSON includes the experiment name, SDK statistics and whether the +evaluation target was reached; failures or unmet targets exit nonzero. The target +cannot exceed the exported generation-attempt budget, but it is not a strict +evaluation cap: SDK polling/queued work can overshoot it, and seed evaluation is +additional. The positive `--idle-timeout-seconds` (default 120) is an idle threshold, +not a wall-clock timeout or a cost limit. Running trusted sources adds no sandbox. + +Cloud operations may incur charges. Every `--run` creates a new experiment; +reruns do not resume existing work. Failure, interruption or local controller exit +does not automatically stop or delete remote resources. Inspect the reported +experiment in your project. Export and preflight never start cloud work, and +offline validation is not evidence of a managed-service run. This remains a +limited #468 export slice and does not reproduce OpenEvolve's evaluator pipeline, +search behavior or the broader SDK shim. Run the offline bundle tests with: ```bash python -m unittest discover -s tests -p 'test_alphaevolve_bundle.py' -v +python -m unittest discover -s tests -p 'test_alphaevolve_launch.py' -v ``` ## Import a local Python experiment configuration diff --git a/openevolve/compat/alphaevolve_bundle.py b/openevolve/compat/alphaevolve_bundle.py index 7b7cc683a..f3625113f 100644 --- a/openevolve/compat/alphaevolve_bundle.py +++ b/openevolve/compat/alphaevolve_bundle.py @@ -63,6 +63,16 @@ def main(): raise SystemExit(main()) ''' +_LAUNCH = '''"""Offline preflight by default; --run explicitly opts in to cloud use.""" + +from pathlib import Path + +from openevolve.compat.alphaevolve_launch import main + +if __name__ == "__main__": + raise SystemExit(main(bundle_dir=Path(__file__).resolve().parent)) +''' + _README = """# Minimal AlphaEvolve evaluator bundle This directory packages a single Python seed and a synchronous @@ -74,12 +84,13 @@ def main(): - `evaluation.json`: the objective metric and finite failure score. - `experiment.json`: the limited AlphaEvolve experiment configuration. - `check.py`: an explicit local seed-evaluation check. +- `run_alphaevolve.py`: offline preflight or an explicitly requested cloud launch. - `requirements.txt`, only if explicitly supplied during export. ## Install dependencies Use Python 3.10 or newer and an OpenEvolve checkout containing -`openevolve.compat.alphaevolve_reverse`. Install that checkout in your chosen +`openevolve.compat.alphaevolve_launch`. Install that checkout in your chosen environment, for example `python -m pip install /path/to/openevolve`. The exporter does not infer, install or vendor dependencies. Review and install your evaluator's packages; if `requirements.txt` is included, review it before @@ -120,11 +131,52 @@ def main(): # Pass `evaluate` as evaluator_function to your configured AlphaEvolveExperiment. ``` -Cloud authentication, client construction, experiment creation and controller -execution are separate, explicit operations and may incur charges. This bundle -does not perform them or claim a managed-service run. Generation limits are not -cost or evaluation limits. Configure evaluation workers separately from service -generation concurrency and check model availability/mixture rules yourself. +## Preflight or explicitly launch an experiment + +Run `python run_alphaevolve.py` (or add `--dry-run`) for an offline JSON summary. +The script resolves the bundle relative to itself, including from another working +directory. It checks the exported configuration subset, evaluation options and +source files without importing the SDK or evaluator, executing the seed, +discovering credentials or making requests. This basic check does not validate +installed dependencies, SDK schemas, permissions or live model availability. +Use `check.py` separately to execute and test your trusted evaluator locally. + +For cloud use, install the official SDK from a reviewed checkout of +https://github.com/Google-Cloud-AI/alphaevolve-on-googlecloud, for example +`python -m pip install /path/to/alphaevolve-on-googlecloud`. Follow that project's +README for Google application-default credentials, the GCP project, Gemini +Enterprise engine and access requirements. Choose models currently supported +by your service; the exporter's Gemini-name syntax check is not a model catalog. + +After reviewing the exported generation budget, explicitly opt in: + +```bash +python run_alphaevolve.py --run \\ + --project-id YOUR_PROJECT_ID --engine YOUR_ENGINE_ID \\ + --max-programs-evaluated 1 +``` + +Only `--run` imports the optional SDK and trusted evaluator. It validates with +the real SDK and evaluates the seed before constructing a client, then creates +a new experiment, uploads the seed with its actual scores, starts generation +and invokes the SDK controller with one sampler and one synchronous evaluator. +`--location` accepts `global` (default), `us` or `eu`. Credentials use the SDK's +normal Google ADC flow; no key, endpoint or .env discovery is added by the runner. + +The evaluation target is required for `--run` and cannot exceed the exported +generation-attempt budget. It is a controller stopping target, not a strict +evaluation cap: SDK polling/queued work can overshoot it, and seed evaluation +is additional. Generation limits are not cost limits. The positive +`--idle-timeout-seconds` (default 120) is an idle threshold, not a wall-clock +timeout, sandbox, cloud cancellation or money limit. + +The final run JSON includes the experiment name, SDK statistics and whether the +evaluation target was reached. An unmet target or launch failure exits nonzero. +SDK diagnostics go to stderr. Cloud operations may incur charges. Each `--run` +creates a new experiment; reruns do not resume an earlier experiment. Failures +or interruptions do not delete or stop cloud resources automatically; inspect +the reported experiment in your project. No managed-service run is claimed by +the export or offline checks. ## Scope @@ -138,7 +190,7 @@ def main(): non-finite results, and removes temporary candidate files. It omits artifacts and does not reproduce OpenEvolve's evaluator pipeline, retries, LLM feedback, cascade stages, resource limits or search behavior. This is a limited export -slice of OpenEvolve #468, not a complete cloud exporter or SDK shim. +slice of OpenEvolve #468, not a complete SDK shim or search-behavior translation. """ @@ -207,7 +259,8 @@ def export_to_alphaevolve( the two named UTF-8 .py files and an optional explicit dependency file. Reject existing destinations (including symlinks); their parent must already exist. Validate inputs before creating output and clean up new output on write errors. - No dependencies, data, local modules, cloud runners or credentials are inferred. + No dependencies, data, local modules or credentials are inferred. The generated + launcher defaults to offline preflight; cloud use requires explicit --run. Read the generated README and explicitly run check.py after installing deps. """ destination = Path(out_dir).absolute() @@ -238,6 +291,7 @@ def export_to_alphaevolve( "src/__init__.py": b"", "src/evaluate.py": _EVALUATOR.encode("utf-8"), "check.py": _CHECK.encode("utf-8"), + "run_alphaevolve.py": _LAUNCH.encode("utf-8"), "README.md": _README.encode("utf-8"), } ) diff --git a/openevolve/compat/alphaevolve_launch.py b/openevolve/compat/alphaevolve_launch.py new file mode 100644 index 000000000..ef21c836b --- /dev/null +++ b/openevolve/compat/alphaevolve_launch.py @@ -0,0 +1,257 @@ +"""Preflight an exported Python bundle; explicitly opt in to a paid cloud run.""" + +import argparse +import asyncio +import contextlib +import importlib +import importlib.util +import json +import math +import re +import sys +import uuid +from pathlib import Path + +from openevolve.compat.alphaevolve_bundle import _read_python +from openevolve.compat.alphaevolve_export import _gemini_name, _positive_integer +from openevolve.compat.alphaevolve_reverse import adapt_openevolve_evaluator + + +def _object(value, required, optional=(), *, label): + if ( + not isinstance(value, dict) + or not set(required).issubset(value) + or set(value) - set(required) - set(optional) + ): + raise ValueError(f"{label} must contain only the supported bundle fields") + return value + + +def _read_json(path): + value = json.loads(path.read_text(encoding="utf-8-sig")) + json.dumps(value, allow_nan=False) # Reject non-standard NaN/Infinity input. + return value + + +def _prepare_bundle(root): + config = _object( + _read_json(root / "experiment.json"), + ("title", "problem_description", "program_language", "run_settings", "generation_settings"), + label="experiment.json", + ) + if config["program_language"] != "python": + raise ValueError("Only single-file Python bundles are supported") + for field in ("title", "problem_description"): + if not isinstance(config[field], str) or not config[field].strip(): + raise ValueError(f"{field} must be non-empty text") + run = _object(config["run_settings"], ("max_programs",), ("concurrency",), label="run_settings") + _positive_integer(run["max_programs"], "run_settings.max_programs") + if "concurrency" in run: + _positive_integer(run["concurrency"], "run_settings.concurrency") + generation = _object(config["generation_settings"], ("models",), label="generation_settings") + models = generation["models"] + if not isinstance(models, list) or not 1 <= len(models) <= 2: + raise ValueError("Bundle requires one or two explicitly named Gemini models") + weights = [] + for model in models: + _object(model, ("name", "weight"), label="generation model") + _gemini_name(model["name"], "generation model name") + weight = model["weight"] + if ( + isinstance(weight, bool) + or not isinstance(weight, (int, float)) + or not math.isfinite(weight) + or weight <= 0 + ): + raise ValueError("Generation model weights must be finite and positive") + weights.append(weight) + try: + if not math.isfinite(math.fsum(weights)): + raise ValueError("Generation model weights must have a finite sum") + except OverflowError: + raise ValueError("Generation model weights must have a finite sum") from None + + settings = _object( + _read_json(root / "evaluation.json"), + ("objective_metric", "failure_score"), + label="evaluation.json", + ) + adapt_openevolve_evaluator(lambda path: {}, **settings) + source = _read_python(root / "src/program.py").decode("utf-8") + _read_python(root / "src/openevolve_evaluator.py", evaluator=True) + for file in ("src/__init__.py", "src/evaluate.py"): + try: + compile((root / file).read_bytes(), file, "exec") + except SyntaxError: + raise ValueError(f"Bundle file must contain valid Python: {file}") from None + candidate = {"content": {"files": [{"path": "program.py", "content": source}]}} + return config, candidate + + +@contextlib.contextmanager +def _bundle_evaluator(root): + # A private package supports relative imports and dataclasses without + # replacing an application's existing `src` package or retaining old bundles. + name = "_openevolve_alphaevolve_" + uuid.uuid4().hex + spec = importlib.util.spec_from_file_location( + name, root / "src/__init__.py", submodule_search_locations=[str(root / "src")] + ) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + evaluate = importlib.import_module(name + ".evaluate").evaluate + if not callable(evaluate): + raise TypeError("Bundle entry point must expose a callable evaluate") + yield evaluate + finally: + for key in list(sys.modules): + if key == name or key.startswith(name + "."): + del sys.modules[key] + + +def _run(root, config, candidate, args): + # Optional SDK imports occur only after --run and all local input checks. + from alpha_evolve.client import AlphaEvolveClient + from alpha_evolve.controller import run_controller_loop + from alpha_evolve.experiment import AlphaEvolveExperiment + from alpha_evolve.models import AlphaEvolveExperimentConfig, AlphaEvolveProgramEvaluation + + AlphaEvolveExperimentConfig.model_validate(config) + stage = "local seed evaluation" + try: + with _bundle_evaluator(root) as evaluate: + seed_evaluation = evaluate(candidate) + AlphaEvolveProgramEvaluation.model_validate(seed_evaluation) + if any( + insight.get("label") == "OpenEvolve Evaluation Error" + for insight in seed_evaluation.get("insights", {}).get("insights", []) + ): + raise ValueError("Seed evaluation failed") + initial = {**candidate, "evaluation": seed_evaluation} + stage = "session/experiment creation" + client = AlphaEvolveClient( + project_id=args.project_id, engine=args.engine, location=args.location + ) + experiment = AlphaEvolveExperiment( + client, + evaluate, + max_programs_evaluated=args.max_programs_evaluated, + parallel_evaluation=False, + ) + experiment.create_experiment(config) + print(f"Cloud experiment: {experiment.experiment_name}", file=sys.stderr) + stage = "seed upload" + experiment.create_initial_program(initial) + if not experiment.initial_program_name: + raise RuntimeError("SDK did not confirm seed creation") + stage = "experiment start" + # The experiment helper discards this response; use the public + # client method so an HTTP failure cannot silently start polling. + if client.start_experiment(experiment.experiment_name) is None: + raise RuntimeError("SDK did not confirm experiment start") + stage = "controller execution" + asyncio.run( + run_controller_loop( + experiment, + num_samplers=1, + num_evaluators=1, + idle_timeout_s=args.idle_timeout_seconds, + ) + ) + reached = experiment.stats["num_programs_evaluated"] >= args.max_programs_evaluated + return { + "mode": "run", + "experiment_name": experiment.experiment_name, + "stats": experiment.stats, + "evaluation_target_reached": reached, + } + except Exception as exc: + raise RuntimeError( + f"Cloud launch failed during {stage} ({type(exc).__name__}). " + "Inspect SDK logs; any created cloud resources are not automatically deleted." + ) from None + + +def _positive_option(value): + try: + return _positive_integer(int(value), "option") + except ValueError: + raise argparse.ArgumentTypeError("must be a positive integer") from None + + +def _resource_id(value): + if not re.fullmatch(r"[A-Za-z0-9_-]+", value): + raise argparse.ArgumentTypeError( + "must be a non-empty resource ID, without slashes or spaces" + ) + return value + + +def main(argv=None, *, bundle_dir=None): + parser = argparse.ArgumentParser(description=__doc__) + if bundle_dir is None: + parser.add_argument("bundle_dir", type=Path) + mode = parser.add_mutually_exclusive_group() + mode.add_argument("--dry-run", action="store_true", help="Offline preflight only (default)") + mode.add_argument( + "--run", action="store_true", help="Create/start a cloud experiment; may incur charges" + ) + parser.add_argument("--project-id", type=_resource_id) + parser.add_argument("--engine", type=_resource_id) + parser.add_argument("--location", choices=("global", "us", "eu"), default="global") + parser.add_argument("--max-programs-evaluated", type=_positive_option) + parser.add_argument("--idle-timeout-seconds", type=_positive_option, default=120) + args = parser.parse_args(argv) + if args.run and not (args.project_id and args.engine and args.max_programs_evaluated): + parser.error("--run requires --project-id, --engine and --max-programs-evaluated") + root = Path(bundle_dir if bundle_dir is not None else args.bundle_dir).resolve() + try: + config, candidate = _prepare_bundle(root) + if ( + args.max_programs_evaluated is not None + and args.max_programs_evaluated > config["run_settings"]["max_programs"] + ): + raise ValueError( + "Evaluation target cannot exceed the exported generation-attempt budget" + ) + except (OSError, TypeError, ValueError, OverflowError) as exc: + parser.error(str(exc)) + if not args.run: + print( + json.dumps( + { + "mode": "dry-run", + "bundle_dir": str(root), + "title": config["title"], + "models": config["generation_settings"]["models"], + "max_generation_attempts": config["run_settings"]["max_programs"], + "max_programs_evaluated": args.max_programs_evaluated, + "project_id": args.project_id, + "engine": args.engine, + "location": args.location, + "sdk_and_evaluator_imported": False, + }, + ensure_ascii=False, + allow_nan=False, + indent=2, + ) + ) + return 0 + try: + # Redirect Python-level evaluator and SDK chatter away from JSON stdout. + with contextlib.redirect_stdout(sys.stderr): + result = _run(root, config, candidate, args) + except ImportError: + parser.exit( + 1, + "Install the official AlphaEvolve SDK and evaluator dependencies; see the bundle README.\n", + ) + except Exception as exc: + parser.exit(1, f"{exc}\n") + print(json.dumps(result, ensure_ascii=False, allow_nan=False, indent=2)) + return 0 if result["evaluation_target_reached"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_alphaevolve_bundle.py b/tests/test_alphaevolve_bundle.py index 3995ff271..de32852e4 100644 --- a/tests/test_alphaevolve_bundle.py +++ b/tests/test_alphaevolve_bundle.py @@ -116,6 +116,7 @@ def test_bundle_has_only_expected_files_and_preserves_source_bytes(self): "experiment.json", "evaluation.json", "check.py", + "run_alphaevolve.py", "README.md", }, ) diff --git a/tests/test_alphaevolve_launch.py b/tests/test_alphaevolve_launch.py new file mode 100644 index 000000000..fe3ceaea9 --- /dev/null +++ b/tests/test_alphaevolve_launch.py @@ -0,0 +1,438 @@ +"""Offline preflight, explicit cloud opt-in and launch failure-path tests.""" + +import contextlib +import copy +import io +import json +import os +import shutil +import subprocess +import sys +import tempfile +import types +import unittest +import warnings +from pathlib import Path +from unittest.mock import Mock, patch + +from openevolve.compat.alphaevolve_bundle import export_to_alphaevolve +from openevolve.compat.alphaevolve_launch import main +from openevolve.config import Config + + +class TestAlphaEvolveLaunch(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.inputs = self.root / "inputs" + self.inputs.mkdir() + self.program = self.inputs / "seed.py" + self.program.write_bytes("# 中文\r\n7\r\n".encode("utf-8")) + self.evaluator = self.inputs / "evaluator.py" + self.evaluator.write_text( + "from dataclasses import dataclass\nfrom pathlib import Path\n" + "@dataclass\nclass Score:\n value: int\n" + "def evaluate(path):\n print('evaluator diagnostic')\n" + " return {'score': Score(len(Path(path).read_bytes())).value}\n" + ) + self.bundle = self.root / "bundle" + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + export_to_alphaevolve( + self.program, + self.evaluator, + Config.from_dict( + { + "language": "python", + "max_iterations": 3, + "llm": {"models": [{"name": "gemini-3.5-flash", "weight": 1}]}, + } + ), + self.bundle, + title="Example 中文", + problem_description="Score the source.", + objective_metric="score", + ) + self.expected_score = len(self.program.read_bytes()) + self.stdout = io.StringIO() + self.stderr = io.StringIO() + self.run_options = [ + "--run", + "--project-id", + "review-project", + "--engine", + "review-engine", + "--max-programs-evaluated", + "1", + ] + + def launch(self, *options, bundle=None): + with contextlib.redirect_stdout(self.stdout), contextlib.redirect_stderr(self.stderr): + return main([str(bundle or self.bundle), *options]) + + @contextlib.contextmanager + def sdk(self): + self.client = Mock() + self.client.start_experiment.return_value = {} + self.client_type = Mock(return_value=self.client) + self.experiment = Mock() + self.experiment.experiment_name = "offline-session/alphaEvolveExperiments/example" + self.experiment.initial_program_name = "offline-seed" + self.experiment.stats = {"num_programs_generated": 1, "num_programs_evaluated": 1} + self.experiment_type = Mock(return_value=self.experiment) + self.config_schema = Mock() + self.evaluation_schema = Mock() + + async def controller(experiment, **options): + self.controller_options = options + evaluate = self.experiment_type.call_args.args[1] + result = evaluate({"content": self.initial["content"]}) + self.assertEqual(result["scores"]["scores"][0]["score"], self.expected_score) + + self.controller = Mock(side_effect=controller) + + def capture_seed(candidate): + self.initial = copy.deepcopy(candidate) + + self.experiment.create_initial_program.side_effect = capture_seed + modules = { + name: types.ModuleType(name) + for name in ( + "alpha_evolve", + "alpha_evolve.client", + "alpha_evolve.controller", + "alpha_evolve.experiment", + "alpha_evolve.models", + ) + } + modules["alpha_evolve"].__path__ = [] + modules["alpha_evolve.client"].AlphaEvolveClient = self.client_type + modules["alpha_evolve.controller"].run_controller_loop = self.controller + modules["alpha_evolve.experiment"].AlphaEvolveExperiment = self.experiment_type + modules["alpha_evolve.models"].AlphaEvolveExperimentConfig = self.config_schema + modules["alpha_evolve.models"].AlphaEvolveProgramEvaluation = self.evaluation_schema + with patch.dict(sys.modules, modules): + yield + + def assert_no_bundle_modules(self): + self.assertFalse(any(name.startswith("_openevolve_alphaevolve_") for name in sys.modules)) + + def test_default_preflight_never_imports_sdk_evaluator_or_executes_source(self): + marker = self.root / "must-not-exist" + self.bundle.joinpath("src/openevolve_evaluator.py").write_text( + f"from pathlib import Path\nPath({str(marker)!r}).touch()\n" + "def evaluate(path): return {'score': 1}\n" + ) + original_import = __import__ + + def guarded_import(name, *args, **kwargs): + if name.startswith(("alpha_evolve", "_openevolve_alphaevolve_")): + raise AssertionError("preflight imported executable code") + return original_import(name, *args, **kwargs) + + with ( + patch("builtins.__import__", side_effect=guarded_import), + patch( + "socket.socket.connect", side_effect=AssertionError("network forbidden") + ) as sockets, + ): + self.assertEqual(self.launch(), 0) + sockets.assert_not_called() + self.assertFalse(marker.exists()) + plan = json.loads(self.stdout.getvalue()) + self.assertEqual(plan["mode"], "dry-run") + self.assertFalse(plan["sdk_and_evaluator_imported"]) + self.assertEqual(plan["max_generation_attempts"], 3) + self.assertEqual(plan["title"], "Example 中文") + self.assertNotIn("content", plan) + self.assert_no_bundle_modules() + + def test_explicit_dry_run_ignores_cloud_environment_and_keeps_files_unchanged(self): + before = {p: p.read_bytes() for p in self.bundle.rglob("*") if p.is_file()} + with patch.dict(os.environ, {"PROJECT_ID": "not-selected", "GE_APP_ID": "not-selected"}): + self.assertEqual(self.launch("--dry-run"), 0) + self.assertIsNone(json.loads(self.stdout.getvalue())["project_id"]) + self.assertIsNone(json.loads(self.stdout.getvalue())["engine"]) + self.assertEqual(before, {p: p.read_bytes() for p in self.bundle.rglob("*") if p.is_file()}) + + def test_relocated_generated_entry_runs_from_unrelated_directory(self): + moved = self.root / "moved" + self.bundle.rename(moved) + shutil.rmtree(self.inputs) + elsewhere = self.root / "elsewhere" + elsewhere.mkdir() + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + completed = subprocess.run( + [sys.executable, str(moved / "run_alphaevolve.py")], + cwd=elsewhere, + env=environment, + capture_output=True, + text=True, + timeout=20, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertEqual(json.loads(completed.stdout)["bundle_dir"], str(moved.resolve())) + + def test_invalid_run_options_fail_before_loading_sdk(self): + cases = [ + ["--run"], + ["--run", "--project-id", "review-project"], + ["--run", "--dry-run"], + ["--max-programs-evaluated", "0"], + ["--max-programs-evaluated", "-1"], + ["--max-programs-evaluated", "1.5"], + ["--max-programs-evaluated", "4"], + ["--idle-timeout-seconds", "0"], + ["--project-id", "project/path"], + ["--engine", "with space"], + ["--location", "https://another-host"], + ] + with self.sdk(): + for options in cases: + with self.subTest(options=options), self.assertRaises(SystemExit) as caught: + self.launch(*options) + self.assertEqual(caught.exception.code, 2) + self.client_type.assert_not_called() + self.config_schema.model_validate.assert_not_called() + + def test_missing_bundle_files_fail_before_cloud_calls(self): + files = ( + "experiment.json", + "evaluation.json", + "src/program.py", + "src/openevolve_evaluator.py", + "src/evaluate.py", + "src/__init__.py", + ) + with self.sdk(): + for relative in files: + path = self.bundle / relative + data = path.read_bytes() + path.unlink() + try: + with self.subTest(file=relative), self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 2) + finally: + path.write_bytes(data) + self.client_type.assert_not_called() + + def test_invalid_configuration_subset_is_rejected(self): + path = self.bundle / "experiment.json" + original = json.loads(path.read_text()) + changes = [ + {"program_language": "cpp"}, + {"title": " "}, + {"run_settings": {}}, + {"run_settings": {"max_programs": True}}, + {"run_settings": {"max_programs": 3, "concurrency": 0}}, + {"generation_settings": {"models": []}}, + {"generation_settings": {"models": [{"name": "local-model", "weight": 1}]}}, + {"generation_settings": {"models": [{"name": "gemini-x", "weight": True}]}}, + {"generation_settings": {"models": [{"name": "gemini-x", "weight": -1}]}}, + {"generation_settings": {"models": [{"name": "gemini-x", "weight": float("nan")}]}}, + {"unrecognized_field": "must not be silently dropped"}, + ] + for change in changes: + path.write_text(json.dumps({**original, **change})) + with self.subTest(change=change), self.assertRaises(SystemExit) as caught: + self.launch() + self.assertEqual(caught.exception.code, 2) + + def test_invalid_evaluation_options_and_python_are_rejected(self): + path = self.bundle / "evaluation.json" + original = path.read_bytes() + for data in ( + "[", + "null", + '{"objective_metric":"error","failure_score":-1}', + '{"objective_metric":"score","failure_score":Infinity}', + ): + path.write_text(data) + with self.subTest(data=data), self.assertRaises(SystemExit) as caught: + self.launch() + self.assertEqual(caught.exception.code, 2) + path.write_bytes(original) + for file, source in ( + ("src/program.py", "return 1\n"), + ("src/openevolve_evaluator.py", "def other(): pass\n"), + ("src/evaluate.py", "def invalid(:\n"), + ("src/__init__.py", "return 1\n"), + ): + path = self.bundle / file + original = path.read_bytes() + path.write_text(source) + try: + with self.subTest(file=file), self.assertRaises(SystemExit) as caught: + self.launch() + self.assertEqual(caught.exception.code, 2) + finally: + path.write_bytes(original) + + def test_explicit_run_passes_real_seed_scores_and_separate_controller_options(self): + marker = self.root / "seed-evaluated" + path = self.bundle / "src/openevolve_evaluator.py" + path.write_text( + path.read_text().replace( + " print('evaluator diagnostic')", + f" Path({str(marker)!r}).touch()\n print('evaluator diagnostic')", + ) + ) + with self.sdk(): + + def require_seed_first(**options): + self.assertTrue(marker.exists()) + self.assertEqual( + options, + { + "project_id": "review-project", + "engine": "review-engine", + "location": "us", + }, + ) + return self.client + + self.client_type.side_effect = require_seed_first + self.assertEqual( + self.launch(*self.run_options, "--location", "us", "--idle-timeout-seconds", "17"), + 0, + ) + self.experiment_type.assert_called_once() + self.assertEqual( + self.experiment_type.call_args.kwargs, + { + "max_programs_evaluated": 1, + "parallel_evaluation": False, + }, + ) + self.experiment.create_experiment.assert_called_once_with( + json.loads((self.bundle / "experiment.json").read_text()) + ) + self.assertEqual( + self.initial["content"]["files"][0]["content"].encode("utf-8"), + self.program.read_bytes(), + ) + self.assertEqual( + self.initial["evaluation"]["scores"]["scores"], + [{"metric": "score", "score": self.expected_score}], + ) + self.assertEqual( + self.controller_options, + {"num_samplers": 1, "num_evaluators": 1, "idle_timeout_s": 17}, + ) + self.client.start_experiment.assert_called_once_with(self.experiment.experiment_name) + self.experiment.start_experiment.assert_not_called() + self.assertTrue(json.loads(self.stdout.getvalue())["evaluation_target_reached"]) + self.assertIn("evaluator diagnostic", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_missing_sdk_fails_without_loading_evaluator(self): + with patch.dict(sys.modules, {"alpha_evolve": None}): + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.assertIn("Install the official AlphaEvolve SDK", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_sdk_schema_rejection_happens_before_evaluator_or_client(self): + with self.sdk(): + self.config_schema.model_validate.side_effect = ValueError("SDK schema changed") + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.client_type.assert_not_called() + self.assertNotIn("evaluator diagnostic", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_failed_seed_never_creates_cloud_client_or_resources(self): + for source in ( + "def evaluate(path): raise RuntimeError('private-detail')\n", + "def evaluate(path): return {'wrong_objective': 1}\n", + ): + (self.bundle / "src/openevolve_evaluator.py").write_text(source) + with self.sdk(), self.subTest(source=source): + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.client_type.assert_not_called() + self.assert_no_bundle_modules() + self.assertNotIn("private-detail", self.stderr.getvalue()) + + def test_bundle_package_does_not_replace_src_or_reuse_an_earlier_bundle(self): + sentinel = types.ModuleType("src") + with patch.dict(sys.modules, {"src": sentinel}): + for score in (7, 9): + other = self.root / f"bundle-{score}" + shutil.copytree(self.bundle, other) + (other / "src/openevolve_evaluator.py").write_text( + f"def evaluate(path): return {{'score': {score}}}\n" + ) + with self.sdk(): + + async def finish(*args, **kwargs): + return None + + self.controller.side_effect = finish + self.launch(*self.run_options, bundle=other) + self.assertEqual( + self.initial["evaluation"]["scores"]["scores"][0]["score"], score + ) + self.assertIs(sys.modules["src"], sentinel) + self.assert_no_bundle_modules() + + def test_session_or_experiment_failure_does_not_upload_seed_or_start_controller(self): + with self.sdk(): + self.experiment.create_experiment.side_effect = RuntimeError("private-response") + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.experiment.create_initial_program.assert_not_called() + self.controller.assert_not_called() + self.assertIn("session/experiment creation", self.stderr.getvalue()) + self.assertNotIn("private-response", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_unconfirmed_seed_upload_does_not_start_generation(self): + with self.sdk(): + self.experiment.initial_program_name = None + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.client.start_experiment.assert_not_called() + self.controller.assert_not_called() + self.assertIn("seed upload", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_unconfirmed_start_does_not_enter_controller(self): + with self.sdk(): + self.client.start_experiment.return_value = None + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.controller.assert_not_called() + self.assertIn("experiment start", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_controller_failure_cleans_imports_and_returns_nonzero(self): + with self.sdk(): + self.controller.side_effect = RuntimeError("controller problem") + with self.assertRaises(SystemExit) as caught: + self.launch(*self.run_options) + self.assertEqual(caught.exception.code, 1) + self.assertIn("controller execution", self.stderr.getvalue()) + self.assert_no_bundle_modules() + + def test_controller_early_exit_is_reported_as_unmet_target(self): + with self.sdk(): + self.experiment.stats["num_programs_evaluated"] = 0 + self.assertEqual(self.launch(*self.run_options), 1) + report = json.loads(self.stdout.getvalue()) + self.assertFalse(report["evaluation_target_reached"]) + self.assertEqual(report["stats"]["num_programs_evaluated"], 0) + self.assert_no_bundle_modules() + + +if __name__ == "__main__": + unittest.main()