Fixes

evaluations/evaluations/benchmark.py

@@ -303,7 +303,7 @@ async def run_qa_benchmark(
 
     db = spec.db_path(db_path)
     async with HaikuRAG(db, config=config) as rag:
-        qa = get_qa_agent(rag, system_prompt=spec.resolve_system_prompt(config))
+        qa = get_qa_agent(rag, config, system_prompt=spec.resolve_system_prompt(config))
 
         async def answer_question(question: str) -> str:
             answer, _ = await qa.answer(question)
@@ -424,13 +424,7 @@ def _resolve_datasets(dataset: str) -> list[DatasetSpec]:
     """Resolve 'all' or a single dataset key to a list of DatasetSpecs."""
     if dataset.lower() == "all":
         return list(DATASETS.values())
-    spec = DATASETS.get(dataset.lower())
-    if spec is None:
-        valid_datasets = ", ".join(sorted(DATASETS))
-        raise typer.BadParameter(
-            f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
-        )
-    return [spec]
+    return [_resolve_dataset(dataset)]
 
 
 @app.command()
@@ -441,7 +435,7 @@ def run(
     ),
     db: Path | None = typer.Option(None, "--db", help="Override the database path."),
     skip_db: bool = typer.Option(
-        False, "--skip-db", help="Skip updateing the evaluation db."
+        False, "--skip-db", help="Skip updating the evaluation db."
     ),
     skip_retrieval: bool = typer.Option(
         False, "--skip-retrieval", help="Skip retrieval benchmark."

evaluations/evaluations/optimization.py

@@ -1,9 +1,11 @@
 import asyncio
+import logging
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
+from pydantic_ai.models import Model
 from pydantic_evals import Case
 from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output_expected
 
@@ -17,6 +19,8 @@ from haiku.rag.client import HaikuRAG
 from haiku.rag.config.models import AppConfig, ModelConfig
 from haiku.rag.utils import get_model
 
+logger = logging.getLogger(__name__)
+
 
 OPTIMIZATION_SCORING_RUBRIC = """You are evaluating the quality of an answer to a question,
 comparing it against a reference answer.
@@ -65,7 +69,7 @@ class QAPromptAdapter:
 
     config: AppConfig
     db_path: Path
-    judge_model: Any
+    judge_model: Model
 
     def evaluate(
         self,
@@ -96,6 +100,9 @@ class QAPromptAdapter:
                 try:
                     answer, _ = await qa.answer(question)
                 except Exception:
+                    logger.warning(
+                        "QA agent failed for question: %s", question, exc_info=True
+                    )
                     answer = None
 
                 if answer is not None: