diff --git a/examples/circle_packing/README.md b/examples/circle_packing/README.md index 878b666f19..3167214435 100644 --- a/examples/circle_packing/README.md +++ b/examples/circle_packing/README.md @@ -193,6 +193,25 @@ Target ratio: 0.9997314619131079 (99.97% of AlphaEvolve's result) This demonstrates that OpenEvolve can successfully reproduce the results from the AlphaEvolve paper on this mathematical optimization problem. +## Reproducibility & the target value + +The evaluator uses a `TARGET_VALUE` (default `2.635`, the AlphaEvolve result for n=26) only to compute the `target_ratio` and `combined_score` metrics. It creates evolutionary pressure by scaling fitness — it does not leak a solution to the model, and any positive value works (see issue #117): + +> You can put any other value for the target like 3.0 and it will still work. The target is used to compute the metrics and create evolutionary pressure. It by itself doesn't give a solution. + +To reproduce the search without the AlphaEvolve prior, override it via the environment: + +```bash +export CIRCLE_PACKING_TARGET_VALUE=3.0 # or 1.0, 10.0, ... any positive value +``` + +Some empirical reference points for what OpenEvolve reaches on this problem without relying on the target as a hint: + +- With the default configuration, OpenEvolve reached a sum of radii of 2.634 in about 800 generations (see issue #117). +- Issue #156 reports 2.635977394746595 — surpassing the AlphaEvolve result — found at iteration 206 of a 450-iteration multi-phase run. + +Note that the system messages in `config_phase_1.yaml` (line 24), `config_phase_2.yaml` (line 24), `config_phase_1_anthropic.yaml` (line 21) and `config_phase_2_anthropic.yaml` (line 22) mention the 2.635 value to the model as an encouraging upper bound. For a fully prior-free run, edit or remove those sentences in your own configs as well. + ## Key Observations The evolution process demonstrated several interesting patterns: diff --git a/examples/circle_packing/evaluator.py b/examples/circle_packing/evaluator.py index 6773653574..ece8aebad0 100644 --- a/examples/circle_packing/evaluator.py +++ b/examples/circle_packing/evaluator.py @@ -13,6 +13,12 @@ import sys import pickle +# Target value from the AlphaEvolve paper, used to compute target_ratio and +# combined_score. It only provides evolutionary pressure (fitness scaling) and +# does not leak a solution; any positive value works (see issue #117). +# Override it via the environment, e.g. CIRCLE_PACKING_TARGET_VALUE=3.0 +TARGET_VALUE = float(os.environ.get("CIRCLE_PACKING_TARGET_VALUE", "2.635")) + class TimeoutError(Exception): pass @@ -193,9 +199,6 @@ def evaluate(program_path): Returns: Dictionary of metrics """ - # Target value from the paper - TARGET_VALUE = 2.635 # AlphaEvolve result for n=26 - try: # For constructor-based approaches, a single evaluation is sufficient # since the result is deterministic @@ -304,8 +307,8 @@ def evaluate_stage1(program_path): # Calculate sum actual_sum = np.sum(radii) if valid else 0.0 - # Target from paper - target = 2.635 + # Target from paper (shared module-level value) + target = TARGET_VALUE # Simple combined score for stage 1 combined_score = (actual_sum / target) if valid else 0.0 diff --git a/tests/test_circle_packing_target_value.py b/tests/test_circle_packing_target_value.py new file mode 100644 index 0000000000..ce89d61c82 --- /dev/null +++ b/tests/test_circle_packing_target_value.py @@ -0,0 +1,102 @@ +""" +Tests for the configurable circle packing target value (issue #117). + +The evaluator's TARGET_VALUE is only a scaling reference used to compute +`target_ratio`/`combined_score` (the evolutionary pressure). It does not leak a +solution, and any positive value works. These tests verify that: + +1. By default the evaluator keeps the historical AlphaEvolve target (2.635). +2. Setting CIRCLE_PACKING_TARGET_VALUE in the environment changes the scaling of + `target_ratio` but not the validity of a solution. +3. `evaluate()` and `evaluate_stage1()` share the same target. + +The evaluator module is loaded by path (it is not part of the openevolve package) +with a fresh module instance per test, so environment changes are picked up at +module import time and no global state leaks between tests. + +Each evaluation runs a tiny deterministic program (26 fixed circles on a grid, no +scipy), so the whole test module finishes in seconds. +""" + +import importlib.util +import os +import tempfile +import textwrap +import unittest +from unittest.mock import patch + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +EVALUATOR_PATH = os.path.join(REPO_ROOT, "examples", "circle_packing", "evaluator.py") + +DETERMINISTIC_PROGRAM = textwrap.dedent(""" + import numpy as np + + + def run_packing(): + centers = np.zeros((26, 2)) + radii = np.zeros(26) + k = 0 + for x in np.linspace(0.12, 0.88, 6): + for y in np.linspace(0.12, 0.88, 5): + if k < 26: + centers[k] = [x, y] + radii[k] = 0.075 + k += 1 + return centers, radii, float(radii.sum()) + """) + + +def load_evaluator(): + """Load a fresh evaluator module instance by path""" + spec = importlib.util.spec_from_file_location("circle_packing_evaluator", EVALUATOR_PATH) + evaluator = importlib.util.module_from_spec(spec) + spec.loader.exec_module(evaluator) + return evaluator + + +class TestCirclePackingTargetValue(unittest.TestCase): + """Check the evaluator target value is configurable and only affects scaling""" + + def setUp(self): + fd, self.program_path = tempfile.mkstemp(suffix=".py") + with os.fdopen(fd, "w") as f: + f.write(DETERMINISTIC_PROGRAM) + self.addCleanup(os.unlink, self.program_path) + + def test_default_target_value(self): + """Without the environment variable, the historical 2.635 target is used""" + evaluator = load_evaluator() + + self.assertEqual(evaluator.TARGET_VALUE, 2.635) + + metrics = evaluator.evaluate(self.program_path) + self.assertEqual(metrics["validity"], 1.0) + self.assertAlmostEqual(metrics["target_ratio"], metrics["sum_radii"] / 2.635) + + def test_target_value_from_environment(self): + """CIRCLE_PACKING_TARGET_VALUE rescales target_ratio but validity is unchanged""" + with patch.dict(os.environ, {"CIRCLE_PACKING_TARGET_VALUE": "3.0"}): + evaluator = load_evaluator() + + self.assertEqual(evaluator.TARGET_VALUE, 3.0) + + metrics = evaluator.evaluate(self.program_path) + + self.assertEqual(metrics["validity"], 1.0) + self.assertAlmostEqual(metrics["sum_radii"], 26 * 0.075) + self.assertAlmostEqual(metrics["target_ratio"], metrics["sum_radii"] / 3.0) + + def test_stage1_and_evaluate_share_target(self): + """evaluate() and evaluate_stage1() must use the same (configured) target""" + with patch.dict(os.environ, {"CIRCLE_PACKING_TARGET_VALUE": "3.0"}): + evaluator = load_evaluator() + metrics = evaluator.evaluate(self.program_path) + stage1 = evaluator.evaluate_stage1(self.program_path) + + self.assertEqual(metrics["validity"], stage1["validity"]) + self.assertAlmostEqual(metrics["sum_radii"], stage1["sum_radii"]) + self.assertAlmostEqual(metrics["target_ratio"], stage1["target_ratio"]) + + +if __name__ == "__main__": + unittest.main()