From 9b166969e27a595ac1bc6277ca0f400416e6793a Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 26 Jan 2026 15:53:37 +0200 Subject: [PATCH 1/2] Introduce huggingface dataset for sharing evaluation dbs --- CHANGELOG.md | 9 +++ docs/benchmarks.md | 36 +++++++++- evaluations/evaluations/benchmark.py | 104 +++++++++++++++++++++++++++ evaluations/pyproject.toml | 1 + uv.lock | 2 + 5 files changed, 149 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92267f8e..a2c3705e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog ## [Unreleased] +### Added + +- **Evaluation Database Hosting**: Pre-built evaluation databases available on HuggingFace + - `evaluations download ` downloads pre-built databases from `ggozad/haiku-rag-eval-dbs` + - `evaluations upload ` uploads databases to HuggingFace (maintainer only) + - Supports `all` argument to download/upload all datasets at once + - Use `--force` flag to overwrite existing databases + - Avoids lengthy database rebuild times for users running benchmarks + ### Changed - **Dependencies**: Updated core dependencies diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 7649ef90..48034244 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -7,18 +7,48 @@ We evaluate `haiku.rag` on several datasets to measure both retrieval quality an You can run evaluations with the `evaluations` CLI: ```bash -evaluations repliqa -evaluations wix +evaluations run repliqa +evaluations run wix ``` The evaluation flow is orchestrated with [`pydantic-evals`](https://github.com/pydantic/pydantic-ai/tree/main/libs/pydantic-evals), which we leverage for dataset management, scoring, and report generation. +### Pre-built Databases + +Building evaluation databases from scratch can take a long time, especially for large datasets like OpenRAG Bench. Pre-built databases are available on HuggingFace: + +```bash +# Download a specific dataset +evaluations download repliqa + +# Download all datasets +evaluations download all + +# Force re-download (overwrite existing) +evaluations download repliqa --force +``` + +Available datasets: + +| Dataset | Size | +|---------|------| +| `repliqa` | ~18MB | +| `hotpotqa` | ~331MB | +| `wix` | ~511MB | +| `open_rag_bench` | ~14GB | + +After downloading, run benchmarks with `--skip-db` to use the pre-built database: + +```bash +evaluations run repliqa --skip-db +``` + ### Configuration The benchmark script accepts several options: ```bash -evaluations repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb +evaluations run repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb ``` **Options:** diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index b17756cd..f51e4853 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -1,4 +1,7 @@ import asyncio +import shutil +import tempfile +import zipfile from collections.abc import Mapping from pathlib import Path from typing import Any, cast @@ -6,6 +9,7 @@ from typing import Any, cast import logfire import typer from dotenv import find_dotenv, load_dotenv +from huggingface_hub import HfApi, hf_hub_download from pydantic_evals import Case, Dataset as EvalDataset from pydantic_evals.evaluators import LLMJudge from pydantic_evals.reporting import ReportCaseFailure @@ -24,6 +28,8 @@ from haiku.rag.utils import get_model load_dotenv(find_dotenv(usecwd=True)) +HF_REPO_ID = "ggozad/haiku-rag-eval-dbs" + logfire.configure(send_to_logfire="if-token-present", service_name="evals") logfire.instrument_pydantic_ai() configure_cli_logging() @@ -449,5 +455,103 @@ def run( ) +@app.command() +def download( + dataset: str = typer.Argument(..., help="Dataset key or 'all' to download all."), + force: bool = typer.Option(False, "--force", help="Overwrite existing database."), +) -> None: + """Download pre-built evaluation database from HuggingFace.""" + if dataset.lower() == "all": + specs = list(DATASETS.values()) + else: + spec = DATASETS.get(dataset.lower()) + if spec is None: + valid_datasets = ", ".join(sorted(DATASETS)) + raise typer.BadParameter( + f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" + ) + specs = [spec] + + for spec in specs: + db = spec.db_path() + if db.exists() and not force: + console.print( + f"[yellow]Skipping {spec.key}: database already exists at {db}[/yellow]" + ) + console.print("Use --force to overwrite.") + continue + + console.print(f"[blue]Downloading {spec.key}...[/blue]") + zip_filename = f"{spec.db_filename}.zip" + + try: + zip_path = hf_hub_download( + repo_id=HF_REPO_ID, + filename=zip_filename, + repo_type="dataset", + ) + except Exception as e: + console.print(f"[red]Failed to download {spec.key}: {e}[/red]") + continue + + # Remove existing database if force is set + if db.exists(): + shutil.rmtree(db) + + # Extract to the parent directory + db.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(db.parent) + + console.print(f"[green]Downloaded {spec.key} to {db}[/green]") + + +@app.command() +def upload( + dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."), +) -> None: + """Upload evaluation database to HuggingFace (maintainer only).""" + if dataset.lower() == "all": + specs = list(DATASETS.values()) + else: + spec = DATASETS.get(dataset.lower()) + if spec is None: + valid_datasets = ", ".join(sorted(DATASETS)) + raise typer.BadParameter( + f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" + ) + specs = [spec] + + api = HfApi() + + for spec in specs: + db = spec.db_path() + if not db.exists(): + console.print(f"[red]Database not found at {db}[/red]") + continue + + console.print(f"[blue]Uploading {spec.key}...[/blue]") + zip_filename = f"{spec.db_filename}.zip" + + with tempfile.TemporaryDirectory() as tmpdir: + zip_path = Path(tmpdir) / zip_filename + console.print("[dim]Creating zip archive...[/dim]") + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for file in db.rglob("*"): + if file.is_file(): + arcname = f"{spec.db_filename}/{file.relative_to(db)}" + zf.write(file, arcname) + + console.print("[dim]Uploading to HuggingFace...[/dim]") + api.upload_file( + path_or_fileobj=str(zip_path), + path_in_repo=zip_filename, + repo_id=HF_REPO_ID, + repo_type="dataset", + ) + + console.print(f"[green]Uploaded {spec.key} to {HF_REPO_ID}[/green]") + + if __name__ == "__main__": app() diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index fb4507ab..02493da0 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "haiku.rag-slim", "pydantic-ai-slim[evals,logfire]>=1.46.0", "datasets>=4.5.0", + "huggingface_hub>=0.20.0", "typer>=0.19.2,<0.20.0", "python-dotenv>=1.2.1", ] diff --git a/uv.lock b/uv.lock index bd93217e..a9db3520 100644 --- a/uv.lock +++ b/uv.lock @@ -1345,6 +1345,7 @@ source = { editable = "evaluations" } dependencies = [ { name = "datasets" }, { name = "haiku-rag-slim" }, + { name = "huggingface-hub" }, { name = "pydantic-ai-slim", extra = ["evals", "logfire"] }, { name = "python-dotenv" }, { name = "typer" }, @@ -1354,6 +1355,7 @@ dependencies = [ requires-dist = [ { name = "datasets", specifier = ">=4.5.0" }, { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, + { name = "huggingface-hub", specifier = ">=0.20.0" }, { name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.46.0" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "typer", specifier = ">=0.19.2,<0.20.0" }, From 369b3a4bf77488ce08d2d337b59f1d92d6513b07 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 26 Jan 2026 16:26:08 +0200 Subject: [PATCH 2/2] No zip --- docs/benchmarks.md | 2 +- evaluations/evaluations/benchmark.py | 38 ++++++++-------------------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 48034244..8d7b002c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -32,7 +32,7 @@ Available datasets: | Dataset | Size | |---------|------| -| `repliqa` | ~18MB | +| `repliqa` | ~30MB | | `hotpotqa` | ~331MB | | `wix` | ~511MB | | `open_rag_bench` | ~14GB | diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index f51e4853..aeb98ecf 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -1,7 +1,5 @@ import asyncio import shutil -import tempfile -import zipfile from collections.abc import Mapping from pathlib import Path from typing import Any, cast @@ -9,7 +7,7 @@ from typing import Any, cast import logfire import typer from dotenv import find_dotenv, load_dotenv -from huggingface_hub import HfApi, hf_hub_download +from huggingface_hub import HfApi, snapshot_download from pydantic_evals import Case, Dataset as EvalDataset from pydantic_evals.evaluators import LLMJudge from pydantic_evals.reporting import ReportCaseFailure @@ -482,13 +480,12 @@ def download( continue console.print(f"[blue]Downloading {spec.key}...[/blue]") - zip_filename = f"{spec.db_filename}.zip" try: - zip_path = hf_hub_download( + downloaded_path = snapshot_download( repo_id=HF_REPO_ID, - filename=zip_filename, repo_type="dataset", + allow_patterns=f"{spec.db_filename}/*", ) except Exception as e: console.print(f"[red]Failed to download {spec.key}: {e}[/red]") @@ -498,10 +495,9 @@ def download( if db.exists(): shutil.rmtree(db) - # Extract to the parent directory + # Copy from cache to target location db.parent.mkdir(parents=True, exist_ok=True) - with zipfile.ZipFile(zip_path, "r") as zf: - zf.extractall(db.parent) + shutil.copytree(Path(downloaded_path) / spec.db_filename, db) console.print(f"[green]Downloaded {spec.key} to {db}[/green]") @@ -531,24 +527,12 @@ def upload( continue console.print(f"[blue]Uploading {spec.key}...[/blue]") - zip_filename = f"{spec.db_filename}.zip" - - with tempfile.TemporaryDirectory() as tmpdir: - zip_path = Path(tmpdir) / zip_filename - console.print("[dim]Creating zip archive...[/dim]") - with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: - for file in db.rglob("*"): - if file.is_file(): - arcname = f"{spec.db_filename}/{file.relative_to(db)}" - zf.write(file, arcname) - - console.print("[dim]Uploading to HuggingFace...[/dim]") - api.upload_file( - path_or_fileobj=str(zip_path), - path_in_repo=zip_filename, - repo_id=HF_REPO_ID, - repo_type="dataset", - ) + api.upload_folder( + folder_path=str(db), + path_in_repo=spec.db_filename, + repo_id=HF_REPO_ID, + repo_type="dataset", + ) console.print(f"[green]Uploaded {spec.key} to {HF_REPO_ID}[/green]")