From 6812435ba04636db90a7544b92f2044c8b860fd8 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 9 Mar 2026 17:07:21 +0200 Subject: [PATCH] cleanup --- evaluations/evaluations/benchmark.py | 51 +++++++++----------- evaluations/evaluations/datasets/__init__.py | 4 +- evaluations/evaluations/datasets/repliqa.py | 2 +- evaluations/evaluations/optimization.py | 6 +-- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 76ea0d9b..bdbf985c 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -28,6 +28,10 @@ load_dotenv(find_dotenv(usecwd=True)) HF_REPO_ID = "ggozad/haiku-rag-eval-dbs" +JUDGE_MODEL_CONFIG = ModelConfig( + provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0 +) + logfire.configure(send_to_logfire="if-token-present", service_name="evals") logfire.instrument_pydantic_ai() configure_cli_logging() @@ -278,10 +282,7 @@ async def run_qa_benchmark( for index, doc in enumerate(corpus, start=1) ] - judge_config = ModelConfig( - provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0 - ) - judge_model = get_model(judge_config, config) + judge_model = get_model(JUDGE_MODEL_CONFIG, config) evaluation_dataset = EvalDataset[str, str, dict[str, str]]( name=spec.key, @@ -314,7 +315,7 @@ async def run_qa_benchmark( dataset_key=spec.key, test_cases=len(cases), config=config, - judge_config=judge_config, + judge_config=JUDGE_MODEL_CONFIG, ) report = await evaluation_dataset.evaluate( @@ -334,11 +335,10 @@ async def run_qa_benchmark( total_processed = len(report.cases) failures = report.failures - total_cases = total_processed - accuracy = passing_cases / total_cases if total_cases > 0 else 0 + accuracy = passing_cases / total_processed if total_processed > 0 else 0 console.print("\n=== QA Benchmark Results ===", style="bold cyan") - console.print(f"Total questions: {total_cases}") + console.print(f"Total questions: {total_processed}") console.print(f"Correct answers: {passing_cases}") console.print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)") @@ -420,6 +420,19 @@ def _resolve_dataset(dataset: str) -> DatasetSpec: return spec +def _resolve_datasets(dataset: str) -> list[DatasetSpec]: + """Resolve 'all' or a single dataset key to a list of DatasetSpecs.""" + if dataset.lower() == "all": + return list(DATASETS.values()) + spec = DATASETS.get(dataset.lower()) + if spec is None: + valid_datasets = ", ".join(sorted(DATASETS)) + raise typer.BadParameter( + f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" + ) + return [spec] + + @app.command() def run( dataset: str = typer.Argument(..., help="Dataset key to evaluate."), @@ -514,16 +527,7 @@ def download( force: bool = typer.Option(False, "--force", help="Overwrite existing database."), ) -> None: """Download pre-built evaluation database from HuggingFace.""" - if dataset.lower() == "all": - specs = list(DATASETS.values()) - else: - spec = DATASETS.get(dataset.lower()) - if spec is None: - valid_datasets = ", ".join(sorted(DATASETS)) - raise typer.BadParameter( - f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" - ) - specs = [spec] + specs = _resolve_datasets(dataset) for spec in specs: db = spec.db_path() @@ -574,16 +578,7 @@ def upload( dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."), ) -> None: """Upload evaluation database to HuggingFace (maintainer only).""" - if dataset.lower() == "all": - specs = list(DATASETS.values()) - else: - spec = DATASETS.get(dataset.lower()) - if spec is None: - valid_datasets = ", ".join(sorted(DATASETS)) - raise typer.BadParameter( - f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" - ) - specs = [spec] + specs = _resolve_datasets(dataset) api = HfApi() diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index b9b22896..51a598da 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -2,12 +2,12 @@ from evaluations.config import DatasetSpec from .hotpotqa import HOTPOTQA_SPEC from .open_rag_bench import OPEN_RAG_BENCH_SPEC -from .repliqa import REPLIQ_SPEC +from .repliqa import REPLIQA_SPEC from .wix import WIX_SPEC DATASETS: dict[str, DatasetSpec] = { spec.key: spec - for spec in (REPLIQ_SPEC, WIX_SPEC, HOTPOTQA_SPEC, OPEN_RAG_BENCH_SPEC) + for spec in (REPLIQA_SPEC, WIX_SPEC, HOTPOTQA_SPEC, OPEN_RAG_BENCH_SPEC) } __all__ = ["DATASETS"] diff --git a/evaluations/evaluations/datasets/repliqa.py b/evaluations/evaluations/datasets/repliqa.py index ba6a1ec8..981750fb 100644 --- a/evaluations/evaluations/datasets/repliqa.py +++ b/evaluations/evaluations/datasets/repliqa.py @@ -47,7 +47,7 @@ def build_repliqa_case( ) -REPLIQ_SPEC = DatasetSpec( +REPLIQA_SPEC = DatasetSpec( key="repliqa", db_filename="repliqa.lancedb", document_loader=load_repliqa_corpus, diff --git a/evaluations/evaluations/optimization.py b/evaluations/evaluations/optimization.py index 9e8910cb..bbd1984d 100644 --- a/evaluations/evaluations/optimization.py +++ b/evaluations/evaluations/optimization.py @@ -9,6 +9,7 @@ from pydantic_evals.evaluators.llm_as_a_judge import judge_input_output_expected from gepa.core.adapter import EvaluationBatch +from evaluations.benchmark import JUDGE_MODEL_CONFIG from evaluations.config import DatasetSpec from haiku.rag.agents.qa import get_qa_agent from haiku.rag.agents.qa.prompts import QA_SYSTEM_PROMPT @@ -204,10 +205,7 @@ def run_optimization( console = Console() - judge_config = ModelConfig( - provider="ollama", name="gpt-oss", enable_thinking=False, temperature=0.0 - ) - judge_model = get_model(judge_config, config) + judge_model = get_model(JUDGE_MODEL_CONFIG, config) db = spec.db_path(db_path) adapter = QAPromptAdapter(