Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions examples/circle_packing/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,25 @@ Target ratio: 0.9997314619131079 (99.97% of AlphaEvolve's result)

This demonstrates that OpenEvolve can successfully reproduce the results from the AlphaEvolve paper on this mathematical optimization problem.

## Reproducibility & the target value

The evaluator uses a `TARGET_VALUE` (default `2.635`, the AlphaEvolve result for n=26) only to compute the `target_ratio` and `combined_score` metrics. It creates evolutionary pressure by scaling fitness — it does not leak a solution to the model, and any positive value works (see issue #117):

> You can put any other value for the target like 3.0 and it will still work. The target is used to compute the metrics and create evolutionary pressure. It by itself doesn't give a solution.

To reproduce the search without the AlphaEvolve prior, override it via the environment:

```bash
export CIRCLE_PACKING_TARGET_VALUE=3.0 # or 1.0, 10.0, ... any positive value
```

Some empirical reference points for what OpenEvolve reaches on this problem without relying on the target as a hint:

- With the default configuration, OpenEvolve reached a sum of radii of 2.634 in about 800 generations (see issue #117).
- Issue #156 reports 2.635977394746595 — surpassing the AlphaEvolve result — found at iteration 206 of a 450-iteration multi-phase run.

Note that the system messages in `config_phase_1.yaml` (line 24), `config_phase_2.yaml` (line 24), `config_phase_1_anthropic.yaml` (line 21) and `config_phase_2_anthropic.yaml` (line 22) mention the 2.635 value to the model as an encouraging upper bound. For a fully prior-free run, edit or remove those sentences in your own configs as well.

## Key Observations

The evolution process demonstrated several interesting patterns:
Expand Down
13 changes: 8 additions & 5 deletions examples/circle_packing/evaluator.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@
import sys
import pickle

# Target value from the AlphaEvolve paper, used to compute target_ratio and
# combined_score. It only provides evolutionary pressure (fitness scaling) and
# does not leak a solution; any positive value works (see issue #117).
# Override it via the environment, e.g. CIRCLE_PACKING_TARGET_VALUE=3.0
TARGET_VALUE = float(os.environ.get("CIRCLE_PACKING_TARGET_VALUE", "2.635"))


class TimeoutError(Exception):
pass
Expand Down Expand Up @@ -193,9 +199,6 @@ def evaluate(program_path):
Returns:
Dictionary of metrics
"""
# Target value from the paper
TARGET_VALUE = 2.635 # AlphaEvolve result for n=26

try:
# For constructor-based approaches, a single evaluation is sufficient
# since the result is deterministic
Expand Down Expand Up @@ -304,8 +307,8 @@ def evaluate_stage1(program_path):
# Calculate sum
actual_sum = np.sum(radii) if valid else 0.0

# Target from paper
target = 2.635
# Target from paper (shared module-level value)
target = TARGET_VALUE

# Simple combined score for stage 1
combined_score = (actual_sum / target) if valid else 0.0
Expand Down
102 changes: 102 additions & 0 deletions tests/test_circle_packing_target_value.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
"""
Tests for the configurable circle packing target value (issue #117).

The evaluator's TARGET_VALUE is only a scaling reference used to compute
`target_ratio`/`combined_score` (the evolutionary pressure). It does not leak a
solution, and any positive value works. These tests verify that:

1. By default the evaluator keeps the historical AlphaEvolve target (2.635).
2. Setting CIRCLE_PACKING_TARGET_VALUE in the environment changes the scaling of
`target_ratio` but not the validity of a solution.
3. `evaluate()` and `evaluate_stage1()` share the same target.

The evaluator module is loaded by path (it is not part of the openevolve package)
with a fresh module instance per test, so environment changes are picked up at
module import time and no global state leaks between tests.

Each evaluation runs a tiny deterministic program (26 fixed circles on a grid, no
scipy), so the whole test module finishes in seconds.
"""

import importlib.util
import os
import tempfile
import textwrap
import unittest
from unittest.mock import patch

REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
EVALUATOR_PATH = os.path.join(REPO_ROOT, "examples", "circle_packing", "evaluator.py")

DETERMINISTIC_PROGRAM = textwrap.dedent("""
import numpy as np


def run_packing():
centers = np.zeros((26, 2))
radii = np.zeros(26)
k = 0
for x in np.linspace(0.12, 0.88, 6):
for y in np.linspace(0.12, 0.88, 5):
if k < 26:
centers[k] = [x, y]
radii[k] = 0.075
k += 1
return centers, radii, float(radii.sum())
""")


def load_evaluator():
"""Load a fresh evaluator module instance by path"""
spec = importlib.util.spec_from_file_location("circle_packing_evaluator", EVALUATOR_PATH)
evaluator = importlib.util.module_from_spec(spec)
spec.loader.exec_module(evaluator)
return evaluator


class TestCirclePackingTargetValue(unittest.TestCase):
"""Check the evaluator target value is configurable and only affects scaling"""

def setUp(self):
fd, self.program_path = tempfile.mkstemp(suffix=".py")
with os.fdopen(fd, "w") as f:
f.write(DETERMINISTIC_PROGRAM)
self.addCleanup(os.unlink, self.program_path)

def test_default_target_value(self):
"""Without the environment variable, the historical 2.635 target is used"""
evaluator = load_evaluator()

self.assertEqual(evaluator.TARGET_VALUE, 2.635)

metrics = evaluator.evaluate(self.program_path)
self.assertEqual(metrics["validity"], 1.0)
self.assertAlmostEqual(metrics["target_ratio"], metrics["sum_radii"] / 2.635)

def test_target_value_from_environment(self):
"""CIRCLE_PACKING_TARGET_VALUE rescales target_ratio but validity is unchanged"""
with patch.dict(os.environ, {"CIRCLE_PACKING_TARGET_VALUE": "3.0"}):
evaluator = load_evaluator()

self.assertEqual(evaluator.TARGET_VALUE, 3.0)

metrics = evaluator.evaluate(self.program_path)

self.assertEqual(metrics["validity"], 1.0)
self.assertAlmostEqual(metrics["sum_radii"], 26 * 0.075)
self.assertAlmostEqual(metrics["target_ratio"], metrics["sum_radii"] / 3.0)

def test_stage1_and_evaluate_share_target(self):
"""evaluate() and evaluate_stage1() must use the same (configured) target"""
with patch.dict(os.environ, {"CIRCLE_PACKING_TARGET_VALUE": "3.0"}):
evaluator = load_evaluator()
metrics = evaluator.evaluate(self.program_path)
stage1 = evaluator.evaluate_stage1(self.program_path)

self.assertEqual(metrics["validity"], stage1["validity"])
self.assertAlmostEqual(metrics["sum_radii"], stage1["sum_radii"])
self.assertAlmostEqual(metrics["target_ratio"], stage1["target_ratio"])


if __name__ == "__main__":
unittest.main()
Loading