Rename iterations to num_candidates

CHANGELOG.md

@@ -3,14 +3,10 @@
 
 ### Added
 
-- **GEPA prompt optimization**: `evaluations optimize` command for automated QA system prompt improvement using evolutionary optimization with LLM-judged scoring
+- **GEPA prompt optimization**: `evaluations optimize` command for automated QA system prompt improvement using evolutionary optimization with LLM-judged scoring. Cases are split 50/50 into train/val sets; GEPA budget is auto-computed from `--num-candidates` and dataset size.
 - **Tuning docs**: Added step 7 (Optimize QA Prompts) to the tuning workflow in `docs/tuning.md`
 - **Evaluations test coverage**: Tests for evaluators (MAP, MRR), config, benchmark helpers, dataset mappers/builders, and optimization
 
-### Changed
-
-- **Optimization train/val split**: Cases are now split 50/50 into train and val sets. GEPA budget is auto-computed from `--iterations` and dataset size, replacing the broken `--max-calls` parameter that counted per-example evaluations
-
 ### Fixed
 
 - **Read-only mode table creation**: `--read-only` no longer creates lance tables when pointed at an empty directory. `Store._init_tables()` now raises `ReadOnlyError` when tables are missing in read-only mode.

docs/tuning.md

@@ -76,7 +76,7 @@ The `evaluations optimize` command uses GEPA (Generalized Evolutionary Prompt Al
 evaluations optimize wix
 
 # Constrained run
-evaluations optimize repliqa --limit 40 --iterations 30
+evaluations optimize repliqa --limit 40 --num-candidates 30
 
 # Save result
 evaluations optimize wix --output optimized_prompt.txt
@@ -85,7 +85,7 @@ evaluations optimize wix --output optimized_prompt.txt
 | Option | Default | Description |
 |--------|---------|-------------|
 | `--limit` | all cases | QA cases to use (split 50/50 train/val) |
-| `--iterations` | `50` | Optimization iterations |
+| `--num-candidates` | `50` | Number of candidate prompts to evaluate |
 | `--output` | — | Save optimized prompt to file |
 | `--config` | auto | haiku.rag YAML config path |
 | `--db` | auto | Database path override |

evaluations/README.md

@@ -62,7 +62,7 @@ Optimize QA system prompts using GEPA (Generalized Evolutionary Prompt Algorithm
 
 ```bash
 evaluations optimize wix
-evaluations optimize repliqa --limit 40 --iterations 30
+evaluations optimize repliqa --limit 40 --num-candidates 30
 evaluations optimize wix --output optimized_prompt.txt
 ```
 

evaluations/evaluations/benchmark.py

@@ -483,8 +483,8 @@ def optimize(
     limit: int | None = typer.Option(
         None, "--limit", help="Limit QA cases (split 50/50 into train/val)."
     ),
-    iterations: int = typer.Option(
-        50, "--iterations", help="Number of optimization iterations."
+    num_candidates: int = typer.Option(
+        50, "--num-candidates", help="Number of candidate prompts to evaluate."
     ),
     output: Path | None = typer.Option(
         None, "--output", help="Save optimized prompt to file."
@@ -509,7 +509,7 @@ def optimize(
         spec=spec,
         config=app_config,
         cases=cases,
-        iterations=iterations,
+        num_candidates=num_candidates,
         db_path=db,
         output=output,
     )

evaluations/evaluations/optimization.py

@@ -205,6 +205,7 @@ class ReflectionLM:
         return result.output
 
 
+# Cases per GEPA reflection minibatch (used for budget calculation)
 REFLECTION_MINIBATCH_SIZE = 3
 
 
@@ -212,7 +213,7 @@ def run_optimization(
     spec: DatasetSpec,
     config: AppConfig,
     cases: list[QACase],
-    iterations: int,
+    num_candidates: int,
     db_path: Path | None = None,
     output: Path | None = None,
 ) -> dict[str, Any]:
@@ -239,16 +240,16 @@ def run_optimization(
     trainset = cases[:mid]
     valset = cases[mid:]
 
-    # Budget: initial valset eval + worst-case iterations
-    # (each iteration: 2 minibatch evals + full valset if accepted)
-    max_metric_calls = len(valset) + iterations * (
+    # Budget: initial valset eval + per-candidate worst case
+    # (each candidate: 2 minibatch evals + full valset if accepted)
+    max_metric_calls = len(valset) + num_candidates * (
         2 * REFLECTION_MINIBATCH_SIZE + len(valset)
     )
 
     console.print(f"Optimizing prompt for dataset: {spec.key}", style="bold magenta")
     console.print(
         f"Train: {len(trainset)}, Val: {len(valset)}, "
-        f"Iterations: {iterations}, GEPA budget: {max_metric_calls}"
+        f"Candidates: {num_candidates}, Budget: {max_metric_calls} eval calls"
     )
     console.print(f"Seed prompt length: {len(seed_prompt)} chars")
 

evaluations/tests/test_optimization.py

@@ -307,7 +307,7 @@ class TestRunOptimization:
                 spec=spec,
                 config=AppConfig(),
                 cases=cases,
-                iterations=10,
+                num_candidates=10,
                 db_path=tmp_path / "test.lancedb",
             )
 
@@ -337,7 +337,7 @@ class TestRunOptimization:
                 spec=spec,
                 config=AppConfig(),
                 cases=cases,
-                iterations=5,
+                num_candidates=5,
                 db_path=tmp_path / "test.lancedb",
                 output=output_path,
             )
@@ -371,7 +371,7 @@ class TestRunOptimization:
                 spec=spec,
                 config=AppConfig(),
                 cases=cases,
-                iterations=1,
+                num_candidates=1,
                 db_path=tmp_path / "test.lancedb",
             )
 
@@ -404,12 +404,12 @@ class TestRunOptimization:
                 spec=spec,
                 config=AppConfig(),
                 cases=cases,
-                iterations=5,
+                num_candidates=5,
                 db_path=tmp_path / "test.lancedb",
             )
 
         call_kwargs = mock_gepa.call_args[1]
         assert len(call_kwargs["trainset"]) == 5
         assert len(call_kwargs["valset"]) == 5
-        # Budget = valset_size + iterations * (2*minibatch + valset_size)
+        # Budget = valset_size + num_candidates * (2*minibatch + valset_size)
         assert call_kwargs["max_metric_calls"] == 5 + 5 * (2 * 3 + 5)