diff --git a/CHANGELOG.md b/CHANGELOG.md index 92267f8e..a2c3705e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog ## [Unreleased] +### Added + +- **Evaluation Database Hosting**: Pre-built evaluation databases available on HuggingFace + - `evaluations download ` downloads pre-built databases from `ggozad/haiku-rag-eval-dbs` + - `evaluations upload ` uploads databases to HuggingFace (maintainer only) + - Supports `all` argument to download/upload all datasets at once + - Use `--force` flag to overwrite existing databases + - Avoids lengthy database rebuild times for users running benchmarks + ### Changed - **Dependencies**: Updated core dependencies diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 7649ef90..8d7b002c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -7,18 +7,48 @@ We evaluate `haiku.rag` on several datasets to measure both retrieval quality an You can run evaluations with the `evaluations` CLI: ```bash -evaluations repliqa -evaluations wix +evaluations run repliqa +evaluations run wix ``` The evaluation flow is orchestrated with [`pydantic-evals`](https://github.com/pydantic/pydantic-ai/tree/main/libs/pydantic-evals), which we leverage for dataset management, scoring, and report generation. +### Pre-built Databases + +Building evaluation databases from scratch can take a long time, especially for large datasets like OpenRAG Bench. Pre-built databases are available on HuggingFace: + +```bash +# Download a specific dataset +evaluations download repliqa + +# Download all datasets +evaluations download all + +# Force re-download (overwrite existing) +evaluations download repliqa --force +``` + +Available datasets: + +| Dataset | Size | +|---------|------| +| `repliqa` | ~30MB | +| `hotpotqa` | ~331MB | +| `wix` | ~511MB | +| `open_rag_bench` | ~14GB | + +After downloading, run benchmarks with `--skip-db` to use the pre-built database: + +```bash +evaluations run repliqa --skip-db +``` + ### Configuration The benchmark script accepts several options: ```bash -evaluations repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb +evaluations run repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb ``` **Options:** diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index b17756cd..aeb98ecf 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -1,4 +1,5 @@ import asyncio +import shutil from collections.abc import Mapping from pathlib import Path from typing import Any, cast @@ -6,6 +7,7 @@ from typing import Any, cast import logfire import typer from dotenv import find_dotenv, load_dotenv +from huggingface_hub import HfApi, snapshot_download from pydantic_evals import Case, Dataset as EvalDataset from pydantic_evals.evaluators import LLMJudge from pydantic_evals.reporting import ReportCaseFailure @@ -24,6 +26,8 @@ from haiku.rag.utils import get_model load_dotenv(find_dotenv(usecwd=True)) +HF_REPO_ID = "ggozad/haiku-rag-eval-dbs" + logfire.configure(send_to_logfire="if-token-present", service_name="evals") logfire.instrument_pydantic_ai() configure_cli_logging() @@ -449,5 +453,89 @@ def run( ) +@app.command() +def download( + dataset: str = typer.Argument(..., help="Dataset key or 'all' to download all."), + force: bool = typer.Option(False, "--force", help="Overwrite existing database."), +) -> None: + """Download pre-built evaluation database from HuggingFace.""" + if dataset.lower() == "all": + specs = list(DATASETS.values()) + else: + spec = DATASETS.get(dataset.lower()) + if spec is None: + valid_datasets = ", ".join(sorted(DATASETS)) + raise typer.BadParameter( + f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" + ) + specs = [spec] + + for spec in specs: + db = spec.db_path() + if db.exists() and not force: + console.print( + f"[yellow]Skipping {spec.key}: database already exists at {db}[/yellow]" + ) + console.print("Use --force to overwrite.") + continue + + console.print(f"[blue]Downloading {spec.key}...[/blue]") + + try: + downloaded_path = snapshot_download( + repo_id=HF_REPO_ID, + repo_type="dataset", + allow_patterns=f"{spec.db_filename}/*", + ) + except Exception as e: + console.print(f"[red]Failed to download {spec.key}: {e}[/red]") + continue + + # Remove existing database if force is set + if db.exists(): + shutil.rmtree(db) + + # Copy from cache to target location + db.parent.mkdir(parents=True, exist_ok=True) + shutil.copytree(Path(downloaded_path) / spec.db_filename, db) + + console.print(f"[green]Downloaded {spec.key} to {db}[/green]") + + +@app.command() +def upload( + dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."), +) -> None: + """Upload evaluation database to HuggingFace (maintainer only).""" + if dataset.lower() == "all": + specs = list(DATASETS.values()) + else: + spec = DATASETS.get(dataset.lower()) + if spec is None: + valid_datasets = ", ".join(sorted(DATASETS)) + raise typer.BadParameter( + f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all" + ) + specs = [spec] + + api = HfApi() + + for spec in specs: + db = spec.db_path() + if not db.exists(): + console.print(f"[red]Database not found at {db}[/red]") + continue + + console.print(f"[blue]Uploading {spec.key}...[/blue]") + api.upload_folder( + folder_path=str(db), + path_in_repo=spec.db_filename, + repo_id=HF_REPO_ID, + repo_type="dataset", + ) + + console.print(f"[green]Uploaded {spec.key} to {HF_REPO_ID}[/green]") + + if __name__ == "__main__": app() diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index fb4507ab..02493da0 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "haiku.rag-slim", "pydantic-ai-slim[evals,logfire]>=1.46.0", "datasets>=4.5.0", + "huggingface_hub>=0.20.0", "typer>=0.19.2,<0.20.0", "python-dotenv>=1.2.1", ] diff --git a/uv.lock b/uv.lock index bd93217e..a9db3520 100644 --- a/uv.lock +++ b/uv.lock @@ -1345,6 +1345,7 @@ source = { editable = "evaluations" } dependencies = [ { name = "datasets" }, { name = "haiku-rag-slim" }, + { name = "huggingface-hub" }, { name = "pydantic-ai-slim", extra = ["evals", "logfire"] }, { name = "python-dotenv" }, { name = "typer" }, @@ -1354,6 +1355,7 @@ dependencies = [ requires-dist = [ { name = "datasets", specifier = ">=4.5.0" }, { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, + { name = "huggingface-hub", specifier = ">=0.20.0" }, { name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.46.0" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "typer", specifier = ">=0.19.2,<0.20.0" },