Merge pull request #153 from ggozad/feat/eval-naming

Name evaluation runs. Use only LLMJudge evaluator.

CHANGELOG.md

@@ -1,6 +1,13 @@
 # Changelog
 ## [Unreleased]
 
+### Changed
+
+- **Evaluations**: Improved evaluation dataset naming and simplified evaluator configuration
+  - `EvalDataset` now accepts dataset name for better organization in Logfire
+  - Added `--name` CLI parameter to override evaluation run names
+  - Removed `IsInstance` evaluator, using only `LLMJudge` for QA evaluation
+
 ### Fixed
 
 - **AG-UI Activity Events**: Activity events now correctly use structured dict content instead of strings

docs/benchmarks.md

@@ -26,6 +26,7 @@ You can also use command-line options:
 - `--skip-retrieval` - Skip retrieval benchmark
 - `--skip-qa` - Skip QA benchmark
 - `--qa-limit N` - Limit number of QA cases to evaluate
+- `--name NAME` - Override the evaluation name (defaults to `{dataset}_qa_evaluation`)
 
 ## Recall
 

evaluations/evaluations/benchmark.py

@@ -9,7 +9,7 @@ from dotenv import load_dotenv
 from pydantic_ai.models.openai import OpenAIChatModel
 from pydantic_ai.providers.ollama import OllamaProvider
 from pydantic_evals import Dataset as EvalDataset
-from pydantic_evals.evaluators import IsInstance, LLMJudge
+from pydantic_evals.evaluators import LLMJudge
 from pydantic_evals.reporting import ReportCaseFailure
 from rich.console import Console
 from rich.progress import Progress
@@ -162,7 +162,10 @@ async def run_retrieval_benchmark(
 
 
 async def run_qa_benchmark(
-    spec: DatasetSpec, config: AppConfig, qa_limit: int | None = None
+    spec: DatasetSpec,
+    config: AppConfig,
+    qa_limit: int | None = None,
+    name: str | None = None,
 ) -> ReportCaseFailure[str, str, dict[str, str]] | None:
     corpus = spec.qa_loader()
     if qa_limit is not None:
@@ -179,9 +182,9 @@ async def run_qa_benchmark(
     )
 
     evaluation_dataset = EvalDataset[str, str, dict[str, str]](
+        name=spec.key,
         cases=cases,
         evaluators=[
-            IsInstance(type_name="str"),
             LLMJudge(
                 rubric=ANSWER_EQUIVALENCE_RUBRIC,
                 include_input=True,
@@ -202,9 +205,10 @@ async def run_qa_benchmark(
         async def answer_question(question: str) -> str:
             return await qa.answer(question)
 
+        eval_name = name if name is not None else f"{spec.key}_qa_evaluation"
         report = await evaluation_dataset.evaluate(
             answer_question,
-            name=f"{spec.key}_qa_evaluation",
+            name=eval_name,
             max_concurrency=1,
             progress=True,
         )
@@ -244,6 +248,7 @@ async def evaluate_dataset(
     skip_retrieval: bool,
     skip_qa: bool,
     qa_limit: int | None,
+    name: str | None,
 ) -> None:
     if not skip_db:
         console.print(f"Using dataset: {spec.key}", style="bold magenta")
@@ -255,7 +260,7 @@ async def evaluate_dataset(
 
     if not skip_qa:
         console.print("\nRunning QA benchmarks...", style="bold yellow")
-        await run_qa_benchmark(spec, config, qa_limit=qa_limit)
+        await run_qa_benchmark(spec, config, qa_limit=qa_limit, name=name)
 
 
 app = typer.Typer(help="Run retrieval and QA benchmarks for configured datasets.")
@@ -277,6 +282,7 @@ def run(
     qa_limit: int | None = typer.Option(
         None, "--qa-limit", help="Limit number of QA cases."
     ),
+    name: str | None = typer.Option(None, "--name", help="Override evaluation name."),
 ) -> None:
     spec = DATASETS.get(dataset.lower())
     if spec is None:
@@ -311,6 +317,7 @@ def run(
             skip_retrieval=skip_retrieval,
             skip_qa=skip_qa,
             qa_limit=qa_limit,
+            name=name,
         )
     )