Merge pull request #260 from ggozad/feat/evals-huggingface

Introduce huggingface dataset for sharing evaluation dbs

CHANGELOG.md

@@ -1,6 +1,15 @@
 # Changelog
 ## [Unreleased]
 
+### Added
+
+- **Evaluation Database Hosting**: Pre-built evaluation databases available on HuggingFace
+  - `evaluations download <dataset>` downloads pre-built databases from `ggozad/haiku-rag-eval-dbs`
+  - `evaluations upload <dataset>` uploads databases to HuggingFace (maintainer only)
+  - Supports `all` argument to download/upload all datasets at once
+  - Use `--force` flag to overwrite existing databases
+  - Avoids lengthy database rebuild times for users running benchmarks
+
 ### Changed
 
 - **Dependencies**: Updated core dependencies

docs/benchmarks.md

@@ -7,18 +7,48 @@ We evaluate `haiku.rag` on several datasets to measure both retrieval quality an
 You can run evaluations with the `evaluations` CLI:
 
 ```bash
-evaluations repliqa
+evaluations run repliqa
-evaluations wix
+evaluations run wix
 ```
 
 The evaluation flow is orchestrated with [`pydantic-evals`](https://github.com/pydantic/pydantic-ai/tree/main/libs/pydantic-evals), which we leverage for dataset management, scoring, and report generation.
 
+### Pre-built Databases
+
+Building evaluation databases from scratch can take a long time, especially for large datasets like OpenRAG Bench. Pre-built databases are available on HuggingFace:
+
+```bash
+# Download a specific dataset
+evaluations download repliqa
+
+# Download all datasets
+evaluations download all
+
+# Force re-download (overwrite existing)
+evaluations download repliqa --force
+```
+
+Available datasets:
+
+| Dataset | Size |
+|---------|------|
+| `repliqa` | ~30MB |
+| `hotpotqa` | ~331MB |
+| `wix` | ~511MB |
+| `open_rag_bench` | ~14GB |
+
+After downloading, run benchmarks with `--skip-db` to use the pre-built database:
+
+```bash
+evaluations run repliqa --skip-db
+```
+
 ### Configuration
 
 The benchmark script accepts several options:
 
 ```bash
-evaluations repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb
+evaluations run repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb
 ```
 
 **Options:**

evaluations/evaluations/benchmark.py

@@ -1,4 +1,5 @@
 import asyncio
+import shutil
 from collections.abc import Mapping
 from pathlib import Path
 from typing import Any, cast
@@ -6,6 +7,7 @@ from typing import Any, cast
 import logfire
 import typer
 from dotenv import find_dotenv, load_dotenv
+from huggingface_hub import HfApi, snapshot_download
 from pydantic_evals import Case, Dataset as EvalDataset
 from pydantic_evals.evaluators import LLMJudge
 from pydantic_evals.reporting import ReportCaseFailure
@@ -24,6 +26,8 @@ from haiku.rag.utils import get_model
 
 load_dotenv(find_dotenv(usecwd=True))
 
+HF_REPO_ID = "ggozad/haiku-rag-eval-dbs"
+
 logfire.configure(send_to_logfire="if-token-present", service_name="evals")
 logfire.instrument_pydantic_ai()
 configure_cli_logging()
@@ -449,5 +453,89 @@ def run(
     )
 
 
+@app.command()
+def download(
+    dataset: str = typer.Argument(..., help="Dataset key or 'all' to download all."),
+    force: bool = typer.Option(False, "--force", help="Overwrite existing database."),
+) -> None:
+    """Download pre-built evaluation database from HuggingFace."""
+    if dataset.lower() == "all":
+        specs = list(DATASETS.values())
+    else:
+        spec = DATASETS.get(dataset.lower())
+        if spec is None:
+            valid_datasets = ", ".join(sorted(DATASETS))
+            raise typer.BadParameter(
+                f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
+            )
+        specs = [spec]
+
+    for spec in specs:
+        db = spec.db_path()
+        if db.exists() and not force:
+            console.print(
+                f"[yellow]Skipping {spec.key}: database already exists at {db}[/yellow]"
+            )
+            console.print("Use --force to overwrite.")
+            continue
+
+        console.print(f"[blue]Downloading {spec.key}...[/blue]")
+
+        try:
+            downloaded_path = snapshot_download(
+                repo_id=HF_REPO_ID,
+                repo_type="dataset",
+                allow_patterns=f"{spec.db_filename}/*",
+            )
+        except Exception as e:
+            console.print(f"[red]Failed to download {spec.key}: {e}[/red]")
+            continue
+
+        # Remove existing database if force is set
+        if db.exists():
+            shutil.rmtree(db)
+
+        # Copy from cache to target location
+        db.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(Path(downloaded_path) / spec.db_filename, db)
+
+        console.print(f"[green]Downloaded {spec.key} to {db}[/green]")
+
+
+@app.command()
+def upload(
+    dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."),
+) -> None:
+    """Upload evaluation database to HuggingFace (maintainer only)."""
+    if dataset.lower() == "all":
+        specs = list(DATASETS.values())
+    else:
+        spec = DATASETS.get(dataset.lower())
+        if spec is None:
+            valid_datasets = ", ".join(sorted(DATASETS))
+            raise typer.BadParameter(
+                f"Unknown dataset '{dataset}'. Choose from: {valid_datasets}, all"
+            )
+        specs = [spec]
+
+    api = HfApi()
+
+    for spec in specs:
+        db = spec.db_path()
+        if not db.exists():
+            console.print(f"[red]Database not found at {db}[/red]")
+            continue
+
+        console.print(f"[blue]Uploading {spec.key}...[/blue]")
+        api.upload_folder(
+            folder_path=str(db),
+            path_in_repo=spec.db_filename,
+            repo_id=HF_REPO_ID,
+            repo_type="dataset",
+        )
+
+        console.print(f"[green]Uploaded {spec.key} to {HF_REPO_ID}[/green]")
+
+
 if __name__ == "__main__":
     app()

evaluations/pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "haiku.rag-slim",
     "pydantic-ai-slim[evals,logfire]>=1.46.0",
     "datasets>=4.5.0",
+    "huggingface_hub>=0.20.0",
     "typer>=0.19.2,<0.20.0",
     "python-dotenv>=1.2.1",
 ]

uv.lock

@@ -1345,6 +1345,7 @@ source = { editable = "evaluations" }
 dependencies = [
     { name = "datasets" },
     { name = "haiku-rag-slim" },
+    { name = "huggingface-hub" },
     { name = "pydantic-ai-slim", extra = ["evals", "logfire"] },
     { name = "python-dotenv" },
     { name = "typer" },
@@ -1354,6 +1355,7 @@ dependencies = [
 requires-dist = [
     { name = "datasets", specifier = ">=4.5.0" },
     { name = "haiku-rag-slim", editable = "haiku_rag_slim" },
+    { name = "huggingface-hub", specifier = ">=0.20.0" },
     { name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.46.0" },
     { name = "python-dotenv", specifier = ">=1.2.1" },
     { name = "typer", specifier = ">=0.19.2,<0.20.0" },