diff --git a/openevolve/database.py b/openevolve/database.py index ab1dad21b..0551e6c89 100644 --- a/openevolve/database.py +++ b/openevolve/database.py @@ -281,7 +281,8 @@ def add( target_island: Specific island to add to (auto-detects parent's island if None) Returns: - Program ID, or None when the admission strategy rejects the candidate + Program ID, or None when the admission strategy or the novelty check + rejects the candidate """ if self.population_strategy is None or not any( ( @@ -382,17 +383,25 @@ def _add( if iteration is not None: program.iteration_found = iteration self.last_iteration = max(self.last_iteration, iteration) + previous_entry = self.programs.get(program.id) self.programs[program.id] = program - # Calculate feature coordinates for MAP-Elites - feature_coords = self._calculate_feature_coords(program) - # Novelty check before adding if not self._is_novel(program.id, island_idx): logger.debug( f"Program {program.id} failed in novelty check and won't be added in the island {island_idx}" ) - return program.id # Do not add non-novel program + # Undo the provisional store so the rejected program cannot be sampled, + # counted against the population limit, or saved to checkpoints. + if previous_entry is None: + del self.programs[program.id] + else: + self.programs[program.id] = previous_entry + return None + + # Calculate feature coordinates for MAP-Elites (after the novelty check so + # rejected programs do not widen the feature scaling ranges) + feature_coords = self._calculate_feature_coords(program) # Add to island-specific feature map (replacing existing if better) feature_key = self._feature_coords_to_key(feature_coords) diff --git a/openevolve/process_parallel.py b/openevolve/process_parallel.py index 8c2b29adf..ee45f8447 100644 --- a/openevolve/process_parallel.py +++ b/openevolve/process_parallel.py @@ -959,7 +959,7 @@ async def run_evolution( break except _CandidateRejected as rejected: - logger.info("Admission strategy rejected program %s", rejected) + logger.info("Database rejected program %s", rejected) try: self._checkpoint_if_due(completed_iteration, checkpoint_callback) except Exception as e: diff --git a/tests/test_novelty_rejection.py b/tests/test_novelty_rejection.py new file mode 100644 index 000000000..2cbf21787 --- /dev/null +++ b/tests/test_novelty_rejection.py @@ -0,0 +1,89 @@ +"""Tests that programs rejected by the novelty check are not kept in the database.""" + +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from openevolve.config import Config +from openevolve.database import Program, ProgramDatabase + + +class NotNovelLLM: + async def generate_with_context(self, system_message: str, messages: list) -> str: + return "NOT_NOVEL" + + +class TestNoveltyRejection(unittest.TestCase): + @patch("openevolve.embedding.EmbeddingClient") + def setUp(self, mock_embedding_client_class) -> None: + # Identical embeddings so every candidate is sent to the (rejecting) LLM judge + mock_embedding_client_class.return_value = MagicMock( + get_embedding=MagicMock(return_value=[1.0, 0.0]) + ) + config = Config() + config.database.in_memory = True + config.database.db_path = None + config.database.num_islands = 1 + config.database.population_size = 3 + config.database.embedding_model = "text-embedding-3-small" + config.database.similarity_threshold = 0.5 + config.database.novelty_llm = NotNovelLLM() + self.db = ProgramDatabase(config.database) + + self.seed = Program(id="seed", code="x = 0", metrics={"combined_score": 0.1}) + self.assertEqual(self.db.add(self.seed), "seed") + + def add_rejected(self, n: int) -> list: + return [ + self.db.add( + Program( + id=f"dup{i}", + code=f"x = {i + 1}", + parent_id="seed", + metrics={"combined_score": 0.9}, + ) + ) + for i in range(n) + ] + + def test_rejected_program_is_not_stored(self) -> None: + results = self.add_rejected(5) + + self.assertEqual(results, [None] * 5) + self.assertEqual(set(self.db.programs), {"seed"}) + self.assertEqual(self.db.islands, [{"seed"}]) + self.assertLessEqual(len(self.db.programs), self.db.config.population_size) + + def test_rejected_program_is_never_sampled_or_ranked(self) -> None: + self.add_rejected(5) + + for _ in range(20): + self.assertEqual(self.db._sample_random_parent().id, "seed") + self.assertEqual([p.id for p in self.db.get_top_programs(n=10)], ["seed"]) + + def test_rejected_program_is_not_checkpointed(self) -> None: + self.add_rejected(2) + + with tempfile.TemporaryDirectory() as tmp: + self.db.save(tmp) + config = Config() + config.database.in_memory = True + config.database.db_path = None + config.database.num_islands = 1 + restored = ProgramDatabase(config.database) + restored.load(tmp) + + self.assertEqual(set(restored.programs), {"seed"}) + + def test_rejection_does_not_touch_feature_stats(self) -> None: + stats_before = { + k: dict(v, values=list(v["values"])) for k, v in self.db.feature_stats.items() + } + + self.db.add(Program(id="long", code="y = 1\n" * 500, metrics={"combined_score": 0.9})) + + self.assertEqual(self.db.feature_stats, stats_before) + + +if __name__ == "__main__": + unittest.main()