From 6ac5f277d8d69645b0708467c783d5875d691a23 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 12 Mar 2026 13:00:51 +0200 Subject: [PATCH] Rename iterations to num_candidates --- CHANGELOG.md | 6 +----- docs/tuning.md | 4 ++-- evaluations/README.md | 2 +- evaluations/evaluations/benchmark.py | 6 +++--- evaluations/evaluations/optimization.py | 11 ++++++----- evaluations/tests/test_optimization.py | 10 +++++----- 6 files changed, 18 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4f75a7..db8ada19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,14 +3,10 @@ ### Added -- **GEPA prompt optimization**: `evaluations optimize` command for automated QA system prompt improvement using evolutionary optimization with LLM-judged scoring +- **GEPA prompt optimization**: `evaluations optimize` command for automated QA system prompt improvement using evolutionary optimization with LLM-judged scoring. Cases are split 50/50 into train/val sets; GEPA budget is auto-computed from `--num-candidates` and dataset size. - **Tuning docs**: Added step 7 (Optimize QA Prompts) to the tuning workflow in `docs/tuning.md` - **Evaluations test coverage**: Tests for evaluators (MAP, MRR), config, benchmark helpers, dataset mappers/builders, and optimization -### Changed - -- **Optimization train/val split**: Cases are now split 50/50 into train and val sets. GEPA budget is auto-computed from `--iterations` and dataset size, replacing the broken `--max-calls` parameter that counted per-example evaluations - ### Fixed - **Read-only mode table creation**: `--read-only` no longer creates lance tables when pointed at an empty directory. `Store._init_tables()` now raises `ReadOnlyError` when tables are missing in read-only mode. diff --git a/docs/tuning.md b/docs/tuning.md index 04e7de0a..9a0de721 100644 --- a/docs/tuning.md +++ b/docs/tuning.md @@ -76,7 +76,7 @@ The `evaluations optimize` command uses GEPA (Generalized Evolutionary Prompt Al evaluations optimize wix # Constrained run -evaluations optimize repliqa --limit 40 --iterations 30 +evaluations optimize repliqa --limit 40 --num-candidates 30 # Save result evaluations optimize wix --output optimized_prompt.txt @@ -85,7 +85,7 @@ evaluations optimize wix --output optimized_prompt.txt | Option | Default | Description | |--------|---------|-------------| | `--limit` | all cases | QA cases to use (split 50/50 train/val) | -| `--iterations` | `50` | Optimization iterations | +| `--num-candidates` | `50` | Number of candidate prompts to evaluate | | `--output` | — | Save optimized prompt to file | | `--config` | auto | haiku.rag YAML config path | | `--db` | auto | Database path override | diff --git a/evaluations/README.md b/evaluations/README.md index 37c0941c..8781a1be 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -62,7 +62,7 @@ Optimize QA system prompts using GEPA (Generalized Evolutionary Prompt Algorithm ```bash evaluations optimize wix -evaluations optimize repliqa --limit 40 --iterations 30 +evaluations optimize repliqa --limit 40 --num-candidates 30 evaluations optimize wix --output optimized_prompt.txt ``` diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 81f44290..ea1cfb91 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -483,8 +483,8 @@ def optimize( limit: int | None = typer.Option( None, "--limit", help="Limit QA cases (split 50/50 into train/val)." ), - iterations: int = typer.Option( - 50, "--iterations", help="Number of optimization iterations." + num_candidates: int = typer.Option( + 50, "--num-candidates", help="Number of candidate prompts to evaluate." ), output: Path | None = typer.Option( None, "--output", help="Save optimized prompt to file." @@ -509,7 +509,7 @@ def optimize( spec=spec, config=app_config, cases=cases, - iterations=iterations, + num_candidates=num_candidates, db_path=db, output=output, ) diff --git a/evaluations/evaluations/optimization.py b/evaluations/evaluations/optimization.py index 20a4cf4c..39c066ad 100644 --- a/evaluations/evaluations/optimization.py +++ b/evaluations/evaluations/optimization.py @@ -205,6 +205,7 @@ class ReflectionLM: return result.output +# Cases per GEPA reflection minibatch (used for budget calculation) REFLECTION_MINIBATCH_SIZE = 3 @@ -212,7 +213,7 @@ def run_optimization( spec: DatasetSpec, config: AppConfig, cases: list[QACase], - iterations: int, + num_candidates: int, db_path: Path | None = None, output: Path | None = None, ) -> dict[str, Any]: @@ -239,16 +240,16 @@ def run_optimization( trainset = cases[:mid] valset = cases[mid:] - # Budget: initial valset eval + worst-case iterations - # (each iteration: 2 minibatch evals + full valset if accepted) - max_metric_calls = len(valset) + iterations * ( + # Budget: initial valset eval + per-candidate worst case + # (each candidate: 2 minibatch evals + full valset if accepted) + max_metric_calls = len(valset) + num_candidates * ( 2 * REFLECTION_MINIBATCH_SIZE + len(valset) ) console.print(f"Optimizing prompt for dataset: {spec.key}", style="bold magenta") console.print( f"Train: {len(trainset)}, Val: {len(valset)}, " - f"Iterations: {iterations}, GEPA budget: {max_metric_calls}" + f"Candidates: {num_candidates}, Budget: {max_metric_calls} eval calls" ) console.print(f"Seed prompt length: {len(seed_prompt)} chars") diff --git a/evaluations/tests/test_optimization.py b/evaluations/tests/test_optimization.py index 5522f964..16e65605 100644 --- a/evaluations/tests/test_optimization.py +++ b/evaluations/tests/test_optimization.py @@ -307,7 +307,7 @@ class TestRunOptimization: spec=spec, config=AppConfig(), cases=cases, - iterations=10, + num_candidates=10, db_path=tmp_path / "test.lancedb", ) @@ -337,7 +337,7 @@ class TestRunOptimization: spec=spec, config=AppConfig(), cases=cases, - iterations=5, + num_candidates=5, db_path=tmp_path / "test.lancedb", output=output_path, ) @@ -371,7 +371,7 @@ class TestRunOptimization: spec=spec, config=AppConfig(), cases=cases, - iterations=1, + num_candidates=1, db_path=tmp_path / "test.lancedb", ) @@ -404,12 +404,12 @@ class TestRunOptimization: spec=spec, config=AppConfig(), cases=cases, - iterations=5, + num_candidates=5, db_path=tmp_path / "test.lancedb", ) call_kwargs = mock_gepa.call_args[1] assert len(call_kwargs["trainset"]) == 5 assert len(call_kwargs["valset"]) == 5 - # Budget = valset_size + iterations * (2*minibatch + valset_size) + # Budget = valset_size + num_candidates * (2*minibatch + valset_size) assert call_kwargs["max_metric_calls"] == 5 + 5 * (2 * 3 + 5)