cleanup

evaluations/evaluations/benchmark.py

@@ -28,6 +28,10 @@ load_dotenv(find_dotenv(usecwd=True))
 
 HF_REPO_ID = "ggozad/haiku-rag-eval-dbs"
 
+JUDGE_MODEL_CONFIG = ModelConfig(
+    provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0
+)
+
 logfire.configure(send_to_logfire="if-token-present", service_name="evals")
 logfire.instrument_pydantic_ai()
 configure_cli_logging()
@@ -278,10 +282,7 @@ async def run_qa_benchmark(
         for index, doc in enumerate(corpus, start=1)
     ]
 
-    judge_config = ModelConfig(
+    judge_model = get_model(JUDGE_MODEL_CONFIG, config)
-        provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0
-    )
-    judge_model = get_model(judge_config, config)
 
     evaluation_dataset = EvalDataset[str, str, dict[str, str]](
         name=spec.key,
@@ -314,7 +315,7 @@ async def run_qa_benchmark(
             dataset_key=spec.key,
             test_cases=len(cases),
             config=config,
-            judge_config=judge_config,
+            judge_config=JUDGE_MODEL_CONFIG,
         )
 
         report = await evaluation_dataset.evaluate(
@@ -334,11 +335,10 @@ async def run_qa_benchmark(
     total_processed = len(report.cases)
     failures = report.failures
 
-    total_cases = total_processed
+    accuracy = passing_cases / total_processed if total_processed > 0 else 0
-    accuracy = passing_cases / total_cases if total_cases > 0 else 0
 
     console.print("\n=== QA Benchmark Results ===", style="bold cyan")
-    console.print(f"Total questions: {total_cases}")
+    console.print(f"Total questions: {total_processed}")
     console.print(f"Correct answers: {passing_cases}")
     console.print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)")
 
@@ -420,6 +420,19 @@ def _resolve_dataset(dataset: str) -> DatasetSpec:
     return spec
 
 
+def _resolve_datasets(dataset: str) -> list[DatasetSpec]:
+    """Resolve 'all' or a single dataset key to a list of DatasetSpecs."""
+    if dataset.lower() == "all":
+        return list(DATASETS.values())
+    spec = DATASETS.get(dataset.lower())
+    if spec is None:
+        valid_datasets = ", ".join(sorted(DATASETS))
+        raise typer.BadParameter(
+            f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
+        )
+    return [spec]
+
+
 @app.command()
 def run(
     dataset: str = typer.Argument(..., help="Dataset key to evaluate."),
@@ -514,16 +527,7 @@ def download(
     force: bool = typer.Option(False, "--force", help="Overwrite existing database."),
 ) -> None:
     """Download pre-built evaluation database from HuggingFace."""
-    if dataset.lower() == "all":
+    specs = _resolve_datasets(dataset)
-        specs = list(DATASETS.values())
-    else:
-        spec = DATASETS.get(dataset.lower())
-        if spec is None:
-            valid_datasets = ", ".join(sorted(DATASETS))
-            raise typer.BadParameter(
-                f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
-            )
-        specs = [spec]
 
     for spec in specs:
         db = spec.db_path()
@@ -574,16 +578,7 @@ def upload(
     dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."),
 ) -> None:
     """Upload evaluation database to HuggingFace (maintainer only)."""
-    if dataset.lower() == "all":
+    specs = _resolve_datasets(dataset)
-        specs = list(DATASETS.values())
-    else:
-        spec = DATASETS.get(dataset.lower())
-        if spec is None:
-            valid_datasets = ", ".join(sorted(DATASETS))
-            raise typer.BadParameter(
-                f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
-            )
-        specs = [spec]
 
     api = HfApi()
 

evaluations/evaluations/datasets/__init__.py

@@ -2,12 +2,12 @@ from evaluations.config import DatasetSpec
 
 from .hotpotqa import HOTPOTQA_SPEC
 from .open_rag_bench import OPEN_RAG_BENCH_SPEC
-from .repliqa import REPLIQ_SPEC
+from .repliqa import REPLIQA_SPEC
 from .wix import WIX_SPEC
 
 DATASETS: dict[str, DatasetSpec] = {
     spec.key: spec
-    for spec in (REPLIQ_SPEC, WIX_SPEC, HOTPOTQA_SPEC, OPEN_RAG_BENCH_SPEC)
+    for spec in (REPLIQA_SPEC, WIX_SPEC, HOTPOTQA_SPEC, OPEN_RAG_BENCH_SPEC)
 }
 
 __all__ = ["DATASETS"]

evaluations/evaluations/datasets/repliqa.py

@@ -47,7 +47,7 @@ def build_repliqa_case(
     )
 
 
-REPLIQ_SPEC = DatasetSpec(
+REPLIQA_SPEC = DatasetSpec(
     key="repliqa",
     db_filename="repliqa.lancedb",
     document_loader=load_repliqa_corpus,

evaluations/evaluations/optimization.py

@@ -9,6 +9,7 @@ from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output_expected
 
 from gepa.core.adapter import EvaluationBatch
 
+from evaluations.benchmark import JUDGE_MODEL_CONFIG
 from evaluations.config import DatasetSpec
 from haiku.rag.agents.qa import get_qa_agent
 from haiku.rag.agents.qa.prompts import QA_SYSTEM_PROMPT
@@ -204,10 +205,7 @@ def run_optimization(
 
     console = Console()
 
-    judge_config = ModelConfig(
+    judge_model = get_model(JUDGE_MODEL_CONFIG, config)
-        provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0
-    )
-    judge_model = get_model(judge_config, config)
 
     db = spec.db_path(db_path)
     adapter = QAPromptAdapter(