From 33c338fb4ca7c9a73f348176f611f961829a097c Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Wed, 21 Oct 2026 13:29:19 -0700 Subject: [PATCH 1/2] feat(compat): adapt OpenEvolve evaluators for AlphaEvolve candidates --- examples/alphaevolve_interop/README.md | 66 +++++- openevolve/compat/alphaevolve_reverse.py | 155 ++++++++++++++ tests/test_alphaevolve_reverse.py | 258 +++++++++++++++++++++++ 3 files changed, 475 insertions(+), 4 deletions(-) create mode 100644 openevolve/compat/alphaevolve_reverse.py create mode 100644 tests/test_alphaevolve_reverse.py diff --git a/examples/alphaevolve_interop/README.md b/examples/alphaevolve_interop/README.md index 53f27734d..414779106 100644 --- a/examples/alphaevolve_interop/README.md +++ b/examples/alphaevolve_interop/README.md @@ -1,11 +1,12 @@ -# AlphaEvolve local evaluator interoperability +# AlphaEvolve evaluator interoperability This is the evaluator-contract slice of [#468](https://github.com/algorithmicsuperintelligence/openevolve/issues/468). It runs the **unchanged** `program.py` and `evaluate.py` from Google's [AlphaEvolve examples](https://github.com/Google-Cloud-AI/alphaevolve-on-googlecloud) -through an explicit adapter. A separate opt-in helper imports a limited subset -of an experiment configuration for local Python runs. Neither helper replaces -the cloud SDK, exports bundles, or implements all of #468. +through an explicit adapter. A reverse adapter also lets a synchronous OpenEvolve +evaluator score AlphaEvolve candidates. Separate opt-in helpers import a limited +subset of experiment configuration for local Python runs. These helpers do not +replace the cloud SDK, export bundles, or implement all of #468. ## Evaluate an upstream seed locally @@ -83,6 +84,62 @@ The adapter provides **no sandbox**. Like the original local evaluators, it runs trusted Python code with the current process's privileges. Use a separate execution environment for untrusted candidates. +## Use an OpenEvolve evaluator with AlphaEvolve candidates + +The reverse adapter accepts AlphaEvolve's `content.files` candidate format and +calls an existing synchronous `evaluate(program_path)` function: + +```python +from my_evaluator import evaluate as evaluate_openevolve +from openevolve.compat.alphaevolve_reverse import adapt_openevolve_evaluator + +evaluate_candidate = adapt_openevolve_evaluator( + evaluate_openevolve, + objective_metric="combined_score", + failure_score=-1e12, +) +result = evaluate_candidate( + {"content": {"files": [{"path": "src/main.py", "content": "# candidate source\n"}]}} +) +# {"scores": {"scores": [{"metric": "combined_score", "score": ...}, ...]}} +``` + +Pass `evaluate_candidate` as `evaluator_function` to the upstream SDK's +`AlphaEvolveExperiment`. The nested dictionary result is accepted by its evaluator +wrapper and by `AlphaEvolveProgramEvaluation.model_validate(result)`. The adapter +itself does not import the Google SDK, create a client, authenticate, or make +requests. Managed-service setup and execution remain the caller's responsibility. + +Only one Python source file is supported, supplied as a dictionary or SDK-style +object. Its source is written unchanged as UTF-8 to a private temporary +`program.py`; candidate path metadata is ignored, and files are removed after +evaluation, including on exceptions. Evaluators requiring sibling files, a +particular original filename, asynchronous execution or files that outlive the +call are not supported. The adapter does not change the working directory, +provide a sandbox, or enforce a timeout; the trusted evaluator retains its +existing execution privileges and resource controls. + +Both a metrics dictionary and `EvaluationResult` are supported. Numeric metrics +keep their names and values. Boolean diagnostic flags and reserved `error` / +`timeout` fields are not exported as optimization metrics. `objective_metric` +must be present on success; it is not a request to rename metrics or configure +the service's optimization objective. + +Evaluator exceptions, explicit `error` in metrics/artifacts, truthy `timeout`, +and non-finite scores map **every available numeric metric and the objective** +to the finite `failure_score`. Generic error feedback is included without +copying exception text or artifact contents. Choose a failure value below all +valid scores for the problem, especially for negative objectives. Malformed +candidates, multi-file input, missing successful objectives and malformed return +contracts raise errors. Interrupts propagate after cleanup. +Zero or negative scores without an explicit failure marker remain valid scores; +the adapter cannot infer failures that an evaluator silently converts to numbers. + +Artifacts are omitted with a warning, apart from recognizing the `error` marker; +text, images, and `alphaevolve_insights` artifacts are not round-tripped. This is +only the reverse evaluator-contract slice of #468, not a cloud export bundle, +SDK shim, or an assertion that a managed-service experiment has been run. + ## Import a local Python experiment configuration `config_from_alphaevolve` imports an explicit experiment dictionary without @@ -263,6 +320,7 @@ python -m unittest discover -s tests -p test_alphaevolve_compat.py python -m unittest discover -s tests -p test_alphaevolve_config.py python -m unittest discover -s tests -p test_alphaevolve_env.py python -m unittest discover -s tests -p test_alphaevolve_runner.py +python -m unittest discover -s tests -p test_alphaevolve_reverse.py ``` The walkthrough additionally permits direct validation against an upstream diff --git a/openevolve/compat/alphaevolve_reverse.py b/openevolve/compat/alphaevolve_reverse.py new file mode 100644 index 000000000..af35f503f --- /dev/null +++ b/openevolve/compat/alphaevolve_reverse.py @@ -0,0 +1,155 @@ +"""Use synchronous OpenEvolve evaluators with the AlphaEvolve candidate contract.""" + +import inspect +import math +import tempfile +import warnings +from collections.abc import Mapping +from dataclasses import dataclass +from numbers import Real +from pathlib import Path +from typing import Any, Callable + +from openevolve.compat.alphaevolve import _field +from openevolve.evaluation_result import EvaluationResult + + +@dataclass(frozen=True) +class _OpenEvolveEvaluator: + evaluator: Callable + objective_metric: str + failure_score: float + + def _evaluation(self, scores: dict, failure: str = "") -> dict: + if failure: + scores = {name: self.failure_score for name in scores} + scores[self.objective_metric] = self.failure_score + result = { + "scores": { + "scores": [{"metric": name, "score": score} for name, score in scores.items()] + } + } + if failure: + result["insights"] = { + "insights": [{"label": "OpenEvolve Evaluation Error", "text": failure}] + } + return result + + def __call__(self, candidate: Any) -> dict: + files = _field(_field(candidate, "content"), "files") + if not isinstance(files, (list, tuple)) or len(files) != 1: + raise ValueError("AlphaEvolve candidate must contain exactly one source file") + source = _field(files[0], "content") + if not isinstance(source, str): + raise TypeError("AlphaEvolve source content must be a string") + if _field(files[0], "program_language") not in (None, "python"): + raise ValueError("Only single-file Python candidates are supported") + + # Never use the candidate's path for filesystem writes. Close the file + # before calling the evaluator, including on platforms with file locks. + with tempfile.TemporaryDirectory(prefix="openevolve-alphaevolve-") as directory: + path = Path(directory) / "program.py" + path.write_text(source, encoding="utf-8", newline="") + try: + result = self.evaluator(str(path)) + except Exception as exc: + # Do not copy exception text, local paths or credentials into + # feedback that a caller may later submit to a cloud service. + return self._evaluation({}, f"OpenEvolve evaluator raised {type(exc).__name__}") + + if inspect.isawaitable(result): + if inspect.iscoroutine(result): + result.close() + raise TypeError("OpenEvolve evaluator must return synchronously") + + artifacts = {} + if isinstance(result, EvaluationResult): + metrics, artifacts = result.metrics, result.artifacts + elif isinstance(result, Mapping): + metrics = result + else: + raise TypeError("OpenEvolve evaluator must return metrics or EvaluationResult") + if not isinstance(metrics, Mapping) or not metrics: + raise ValueError("OpenEvolve evaluation must contain a non-empty metrics mapping") + if not isinstance(artifacts, Mapping): + raise TypeError("OpenEvolve artifacts must be a mapping") + if any(name != "error" for name in artifacts): + warnings.warn( + "OpenEvolve artifacts are not exported by this adapter", + UserWarning, + stacklevel=2, + ) + + failed = "error" in metrics or "error" in artifacts or bool(metrics.get("timeout", False)) + failure = "OpenEvolve evaluation reported an error or timeout" if failed else "" + scores = {} + for name, score in metrics.items(): + if not isinstance(name, str) or not name.strip(): + raise ValueError("OpenEvolve metric names must be non-empty strings") + if name in ("error", "timeout"): + continue + if isinstance(score, bool): + if name == self.objective_metric: + raise TypeError("OpenEvolve objective must be numeric, not boolean") + continue + if not isinstance(score, Real): + raise TypeError(f"OpenEvolve metric {name!r} must be numeric") + try: + value = float(score) + except OverflowError: + value = math.inf + if not math.isfinite(value): + failure = "OpenEvolve evaluation returned a non-finite score" + scores[name] = value + + if not failure and self.objective_metric not in scores: + raise ValueError( + f"OpenEvolve evaluation is missing objective {self.objective_metric!r}" + ) + return self._evaluation(scores, failure) + + +def adapt_openevolve_evaluator( + evaluator: Callable, + *, + objective_metric: str, + failure_score: float = -1e12, +) -> Callable[[Any], dict]: + """Wrap a synchronous path-based evaluator for AlphaEvolve's dict contract. + + Accept a dictionary or SDK-style single-file Python candidate. Write its + UTF-8 source to a private temporary program.py, call evaluator(path), and + clean up on success or failure. Candidate path metadata is ignored. + + Export numeric metrics in scores.scores; omit boolean diagnostic flags and + the reserved error/timeout fields. Require objective_metric on success. + Exceptions, explicit error/timeout results and non-finite scores map all + available metrics plus the objective to failure_score with generic feedback. + Choose a finite failure_score below every valid score for the problem. + Malformed candidates/return contracts raise rather than being guessed. + + Artifacts are not exported (a warning contains no artifact names/values). + This does not configure the service's optimization objective, enforce a + timeout, sandbox execution, emulate the SDK or export a cloud-run bundle. + The trusted evaluator retains its existing privileges and dependencies. + No Google SDK, client or authentication is imported or created here. + """ + if not callable(evaluator): + raise TypeError("evaluator must be callable") + if inspect.iscoroutinefunction(evaluator) or inspect.iscoroutinefunction( + getattr(evaluator, "__call__", None) + ): + raise TypeError("OpenEvolve evaluator must be synchronous") + if not isinstance(objective_metric, str) or not objective_metric.strip(): + raise ValueError("objective_metric must be a non-empty string") + if objective_metric in ("error", "timeout"): + raise ValueError("error and timeout are reserved for failure handling") + if isinstance(failure_score, bool) or not isinstance(failure_score, Real): + raise ValueError("failure_score must be finite") + try: + failure_score = float(failure_score) + except OverflowError: + raise ValueError("failure_score must be finite") from None + if not math.isfinite(failure_score): + raise ValueError("failure_score must be finite") + return _OpenEvolveEvaluator(evaluator, objective_metric, failure_score) diff --git a/tests/test_alphaevolve_reverse.py b/tests/test_alphaevolve_reverse.py new file mode 100644 index 000000000..ac350ef66 --- /dev/null +++ b/tests/test_alphaevolve_reverse.py @@ -0,0 +1,258 @@ +"""Offline tests for OpenEvolve-to-AlphaEvolve evaluator interoperability.""" + +import copy +import inspect +import json +import tempfile +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import cloudpickle + +from openevolve.compat.alphaevolve import adapt_evaluator +from openevolve.compat.alphaevolve_reverse import adapt_openevolve_evaluator +from openevolve.evaluation_result import EvaluationResult + + +def candidate(source="x = 1\n", **file_fields): + return {"content": {"files": [{"path": "src/main.py", "content": source, **file_fields}]}} + + +def scores(result): + return {entry["metric"]: entry["score"] for entry in result["scores"]["scores"]} + + +class TestReverseAlphaEvolveAdapter(unittest.TestCase): + def adapt(self, evaluator, **options): + return adapt_openevolve_evaluator(evaluator, objective_metric="fitness", **options) + + def test_exact_source_private_file_and_cleanup_with_ignored_candidate_path(self): + paths = [] + source = "# 中文\r\nx = 1\r\n" + + def evaluate(path): + path = Path(path) + paths.append(path) + self.assertEqual(path.name, "program.py") + self.assertEqual(path.read_bytes(), source.encode("utf-8")) + return {"fitness": -3, "diversity": 2} + + for name in ("../../outside.py", "/absolute.py", "src/main.py"): + supplied = candidate(source, path=name) + original = copy.deepcopy(supplied) + self.assertEqual( + scores(self.adapt(evaluate)(supplied)), {"fitness": -3.0, "diversity": 2.0} + ) + self.assertEqual(supplied, original) + self.assertEqual(len({path.parent for path in paths}), 3) + for path in paths: + self.assertFalse(path.exists()) + self.assertFalse(path.parent.exists()) + + def test_sdk_style_candidate_objects(self): + supplied = SimpleNamespace( + content=SimpleNamespace( + files=[SimpleNamespace(content="pass\n", program_language="python")] + ) + ) + self.assertEqual( + scores(self.adapt(lambda path: {"fitness": 1})(supplied)), {"fitness": 1.0} + ) + + def test_malformed_or_multifile_candidates_do_not_call_evaluator(self): + evaluator = Mock() + for supplied in ( + None, + {}, + {"content": {}}, + {"content": {"files": []}}, + {"content": {"files": "code"}}, + {"content": {"files": [{"content": "a"}, {"content": "b"}]}}, + ): + with self.subTest(candidate=supplied), self.assertRaises(ValueError): + self.adapt(evaluator)(supplied) + evaluator.assert_not_called() + + def test_non_string_source_and_non_python_language_are_rejected(self): + for source in (None, b"pass", 10): + with self.subTest(source=source), self.assertRaises(TypeError): + self.adapt(lambda path: {"fitness": 1})(candidate(source)) + with self.assertRaisesRegex(ValueError, "Python"): + self.adapt(lambda path: {"fitness": 1})(candidate(program_language="cpp")) + + def test_empty_source_can_be_scored_by_the_evaluator(self): + result = self.adapt(lambda path: {"fitness": len(Path(path).read_bytes())})(candidate("")) + self.assertEqual(scores(result), {"fitness": 0.0}) + + def test_numeric_metrics_are_preserved_and_boolean_flags_are_not_objectives(self): + metrics = {"fitness": -10, "size": 3.5, "passed": True, "skipped": False, "timeout": False} + original = metrics.copy() + result = self.adapt(lambda path: metrics)(candidate()) + self.assertEqual(scores(result), {"fitness": -10.0, "size": 3.5}) + self.assertNotIn("insights", result) + self.assertEqual(metrics, original) + json.dumps(result, allow_nan=False) + + def test_evaluation_result_warns_without_exporting_text_or_binary_artifacts(self): + supplied = EvaluationResult( + {"fitness": 2}, {"private-key": "private-value", "plot": b"png"} + ) + original = copy.deepcopy(supplied) + with self.assertWarnsRegex(UserWarning, "artifacts are not exported") as caught: + result = self.adapt(lambda path: supplied)(candidate()) + self.assertEqual(scores(result), {"fitness": 2.0}) + for secret in ("private-key", "private-value", "plot"): + self.assertNotIn(secret, str(caught.warning)) + self.assertNotIn(secret, json.dumps(result)) + self.assertEqual(supplied, original) + + def test_explicit_failure_marks_every_available_metric_and_the_objective_failed(self): + for supplied in ( + {"fitness": 2, "size": 999, "error": 0.0}, + {"size": 999, "timeout": True}, + {"error": "private exception text"}, + EvaluationResult({"fitness": 2, "size": 999}, {"error": "private file path"}), + ): + with self.subTest(result=supplied): + original = copy.deepcopy(supplied) + result = self.adapt(lambda path: supplied)(candidate()) + self.assertEqual(scores(result)["fitness"], -1e12) + self.assertTrue(all(value == -1e12 for value in scores(result).values())) + self.assertNotIn("error", scores(result)) + self.assertNotIn("timeout", scores(result)) + self.assertIn("insights", result) + self.assertNotIn("private", json.dumps(result)) + self.assertEqual(supplied, original) + + def test_non_finite_auxiliary_or_objective_metrics_fail_all_scores(self): + for metric in ("fitness", "size"): + for value in (float("nan"), float("inf"), -float("inf"), 10**1000): + supplied = {"fitness": -1, "size": 10, metric: value} + with self.subTest(metric=metric, value=value): + result = self.adapt(lambda path: supplied)(candidate()) + self.assertEqual(scores(result), {"fitness": -1e12, "size": -1e12}) + json.dumps(result, allow_nan=False) + + def test_evaluator_exception_is_sanitized_and_temp_files_are_cleaned(self): + paths = [] + + def evaluate(path): + paths.append(Path(path)) + raise RuntimeError("private-credential /private/path") + + result = self.adapt(evaluate)(candidate()) + self.assertEqual(scores(result), {"fitness": -1e12}) + self.assertIn("RuntimeError", json.dumps(result)) + self.assertNotIn("private-credential", json.dumps(result)) + self.assertNotIn(str(paths[0]), json.dumps(result)) + self.assertFalse(paths[0].parent.exists()) + + def test_keyboard_interrupt_propagates_and_still_cleans_up(self): + paths = [] + + def evaluate(path): + paths.append(Path(path)) + raise KeyboardInterrupt + + with self.assertRaises(KeyboardInterrupt): + self.adapt(evaluate)(candidate()) + self.assertFalse(paths[0].parent.exists()) + + def test_malformed_result_contracts_raise_after_cleanup(self): + for supplied in ( + None, + [], + {}, + {"other": 1}, + {"fitness": None}, + {"fitness": "1"}, + {"fitness": True}, + {"fitness": 1, "": 2}, + {None: 1}, + EvaluationResult([], {}), + EvaluationResult({"fitness": 1}, []), + ): + paths = [] + + def evaluate(path): + paths.append(Path(path)) + return supplied + + with self.subTest(result=supplied), self.assertRaises((TypeError, ValueError)): + self.adapt(evaluate)(candidate()) + self.assertFalse(paths[0].parent.exists()) + + def test_async_functions_and_async_callables_are_rejected(self): + async def evaluate(path): + return {"fitness": 1} + + class AsyncEvaluator: + async def __call__(self, path): + return {"fitness": 1} + + for evaluator in (evaluate, AsyncEvaluator()): + with ( + self.subTest(evaluator=evaluator), + self.assertRaisesRegex(TypeError, "synchronous"), + ): + self.adapt(evaluator) + + def test_unexpected_awaitable_result_is_closed_and_rejected(self): + async def evaluate(): + return {"fitness": 1} + + coroutine = evaluate() + with self.assertRaisesRegex(TypeError, "synchronously"): + self.adapt(lambda path: coroutine)(candidate()) + self.assertEqual(inspect.getcoroutinestate(coroutine), inspect.CORO_CLOSED) + + def test_configuration_validation_and_custom_failure_score(self): + for objective in (None, "", " ", "error", "timeout"): + with self.subTest(objective=objective), self.assertRaises(ValueError): + adapt_openevolve_evaluator(lambda path: {}, objective_metric=objective) + for failure in (True, None, "-1", float("nan"), float("inf"), 10**1000): + with self.subTest(failure=failure), self.assertRaises(ValueError): + self.adapt(lambda path: {}, failure_score=failure) + with self.assertRaises(TypeError): + self.adapt(None) + result = self.adapt(lambda path: {"error": 0}, failure_score=-1e20)(candidate()) + self.assertEqual(scores(result), {"fitness": -1e20}) + + def test_parallel_calls_use_independent_temporary_files(self): + barrier = threading.Barrier(2) + paths = [] + + def evaluate(path): + paths.append(Path(path)) + barrier.wait(timeout=5) + return {"fitness": int(Path(path).read_text())} + + adapter = self.adapt(evaluate) + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(adapter, [candidate("1"), candidate("2")])) + self.assertEqual([scores(result)["fitness"] for result in results], [1.0, 2.0]) + self.assertEqual(len({path.parent for path in paths}), 2) + self.assertTrue(all(not path.parent.exists() for path in paths)) + + def test_callable_serialization_does_not_require_the_google_sdk(self): + adapter = cloudpickle.loads(cloudpickle.dumps(self.adapt(lambda path: {"fitness": 2}))) + self.assertEqual(scores(adapter(candidate())), {"fitness": 2.0}) + + def test_round_trip_through_existing_inbound_adapter(self): + reverse = self.adapt(lambda path: {"fitness": -3.0, "diversity": 2.0}) + forward = adapt_evaluator(reverse, objective_metric="fitness") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "initial.py" + path.write_text("pass\n", encoding="utf-8") + self.assertEqual( + forward(str(path)).metrics, + {"fitness": -3.0, "diversity": 2.0, "combined_score": -3.0}, + ) + + +if __name__ == "__main__": + unittest.main() From cea8a5a970599e46edb10c286e4ee71c7d865658 Mon Sep 17 00:00:00 2001 From: Yushu Li Date: Wed, 21 Oct 2026 13:40:07 -0700 Subject: [PATCH 2/2] test(database): make weighted sampling preference check deterministic The 200-draw Monte Carlo assertion can fail for a valid sampler when the set-backed population order changes. Inspect the distribution passed to random.choices and compare its expected score against uniform sampling instead. This retains the high-fitness preference check without random CI failures. --- tests/test_sample_from_island_ratios.py | 43 ++++++++++++++----------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/tests/test_sample_from_island_ratios.py b/tests/test_sample_from_island_ratios.py index 54eacb53c..def1fa9e2 100644 --- a/tests/test_sample_from_island_ratios.py +++ b/tests/test_sample_from_island_ratios.py @@ -7,6 +7,9 @@ import random import unittest +from math import fsum +from unittest.mock import patch + from openevolve.config import Config from openevolve.database import Program, ProgramDatabase @@ -189,25 +192,27 @@ def test_weighted_mode_favors_high_fitness(self): self.db.config.exploration_ratio = 0.0 self.db.config.exploitation_ratio = 0.0 - random.seed(42) - - # Sample many times and track which programs are selected - sampled_scores = [] - for _ in range(200): - parent, _ = self.db.sample_from_island(island_id=0) - sampled_scores.append(parent.metrics["score"]) - - # Average sampled score should be higher than mean score of all programs - # (mean represents what uniform random sampling would produce) - island_programs = [self.db.programs[pid] for pid in self.db.islands[0]] - all_scores = [p.metrics["score"] for p in island_programs] - mean_score = sum(all_scores) / len(all_scores) - avg_sampled_score = sum(sampled_scores) / len(sampled_scores) - - # Weighted sampling should favor higher scores (shift average upward from mean) - self.assertGreater(avg_sampled_score, mean_score, - f"Weighted sampling should favor high fitness: " - f"sampled_avg={avg_sampled_score:.4f} should be > mean={mean_score:.4f}") + # Inspect the actual distribution passed to the standard-library sampler. + # A small Monte Carlo sample can miss this bias, even with a fixed RNG + # seed, because the population comes from a hash-ordered set. + with patch("openevolve.database.random.choices", wraps=random.choices) as choose: + parent, _ = self.db.sample_from_island(island_id=0, num_inspirations=0) + + choose.assert_called_once() + population = choose.call_args.args[0] + weights = choose.call_args.kwargs["weights"] + self.assertEqual(choose.call_args.kwargs["k"], 1) + self.assertEqual({p.id for p in population}, self.db.islands[0]) + self.assertIn(parent.id, self.db.islands[0]) + self.assertEqual(len(weights), len(population)) + + all_scores = [p.metrics["score"] for p in population] + mean_score = fsum(all_scores) / len(all_scores) + expected_score = fsum(s * w for s, w in zip(all_scores, weights)) / fsum(weights) + + # Keep the fitness preference assertion, without random false failures. + # The tolerance also rejects effectively uniform weighting. + self.assertGreater(expected_score, mean_score + 1e-9) class TestSampleFromIslandEdgeCases(unittest.TestCase):