From ce7201271f387810348e8e5570d27528830f1c30 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 6 May 2026 12:49:03 +0300 Subject: [PATCH] split open_rag_bench dataset into orb_text and orb_multimodal variants --- docs/benchmarks.md | 4 +-- evaluations/README.md | 10 ++++--- evaluations/evaluations/benchmark.py | 28 +++++++++++++---- evaluations/evaluations/datasets/__init__.py | 10 +++++-- .../evaluations/datasets/open_rag_bench.py | 30 +++++++++++++------ 5 files changed, 60 insertions(+), 22 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index e4a86110..15ea3531 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -35,8 +35,8 @@ Available datasets: | `repliqa` | ~30MB | | `hotpotqa` | ~331MB | | `wix` | ~511MB | -| `open_rag_bench` (text embedder + VLM picture descriptions) | ~15GB | -| `open_rag_bench` (multimodal embedder, override with `--db`) | ~16GB | +| `orb_text` — OpenRAG Bench, text embedder (`qwen3-embedding:4b`) with VLM picture descriptions baked into chunk content | ~15GB | +| `orb_multimodal` — OpenRAG Bench, multimodal embedder (`qwen3-vl-embedding-8b`); picture vectors live in the same space as text for cross-modal retrieval | ~16GB | After downloading, run benchmarks with `--skip-db` to use the pre-built database: diff --git a/evaluations/README.md b/evaluations/README.md index 8777c3c3..3a7e5042 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -8,10 +8,12 @@ This package is not published to PyPI and is only used for development and testi Contains evaluation scripts for benchmarking RAG retrieval and QA performance, plus GEPA-based prompt optimization. Available datasets: -- RepliQA -- WiX -- HotpotQA -- OpenRAG Bench +- RepliQA (`repliqa`) +- WiX (`wix`) +- HotpotQA (`hotpotqa`) +- OpenRAG Bench, two variants: + - `orb_text` — text embedder (`qwen3-embedding:4b`, 2560-dim) with VLM picture descriptions baked into chunk content at ingest. Use for text-only retrieval/QA against figure-rich corpora. + - `orb_multimodal` — multimodal embedder (`qwen3-vl-embedding-8b`, 4096-dim) with picture vectors in the same space as text. Use for cross-modal retrieval (text-as-query → figure hits, image-as-query) and vision QA where the figure itself is the answer. ## Usage diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 7849d04b..09d4a7cc 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -765,7 +765,15 @@ def download( def upload( dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."), ) -> None: - """Upload evaluation database to HuggingFace (maintainer only).""" + """Upload evaluation database to HuggingFace (maintainer only). + + Uses ``upload_large_folder`` for resumable, parallel transfer — important + for the multi-GB ORB databases which would otherwise abort on any transient + network failure under plain ``upload_folder``. + + The local folder basename equals ``spec.db_filename`` (see + ``DatasetSpec.db_path``), so the folder lands at that path on the Hub. + """ specs = _resolve_datasets(dataset) api = HfApi() @@ -776,13 +784,23 @@ def upload( console.print(f"[red]Database not found at {db}[/red]") continue - console.print(f"[blue]Uploading {spec.key}...[/blue]") - api.upload_folder( + # Wipe the existing remote path so we don't accumulate orphaned files + # from prior uploads. upload_large_folder doesn't accept delete_patterns, + # so we do this as a separate commit. Safe to run if the path is missing. + try: + api.delete_folder( + path_in_repo=spec.db_filename, + repo_id=HF_REPO_ID, + repo_type="dataset", + ) + except Exception: + pass + + console.print(f"[blue]Uploading {spec.key} ({db})...[/blue]") + api.upload_large_folder( folder_path=str(db), - path_in_repo=spec.db_filename, repo_id=HF_REPO_ID, repo_type="dataset", - delete_patterns="*", ) console.print(f"[green]Uploaded {spec.key} to {HF_REPO_ID}[/green]") diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index 51a598da..ca344dc4 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,13 +1,19 @@ from evaluations.config import DatasetSpec from .hotpotqa import HOTPOTQA_SPEC -from .open_rag_bench import OPEN_RAG_BENCH_SPEC +from .open_rag_bench import ORB_MULTIMODAL_SPEC, ORB_TEXT_SPEC from .repliqa import REPLIQA_SPEC from .wix import WIX_SPEC DATASETS: dict[str, DatasetSpec] = { spec.key: spec - for spec in (REPLIQA_SPEC, WIX_SPEC, HOTPOTQA_SPEC, OPEN_RAG_BENCH_SPEC) + for spec in ( + REPLIQA_SPEC, + WIX_SPEC, + HOTPOTQA_SPEC, + ORB_TEXT_SPEC, + ORB_MULTIMODAL_SPEC, + ) } __all__ = ["DATASETS"] diff --git a/evaluations/evaluations/datasets/open_rag_bench.py b/evaluations/evaluations/datasets/open_rag_bench.py index 35f07204..5514627f 100644 --- a/evaluations/evaluations/datasets/open_rag_bench.py +++ b/evaluations/evaluations/datasets/open_rag_bench.py @@ -216,14 +216,26 @@ def is_multimodal_query(source: str) -> bool: return "image" in source -OPEN_RAG_BENCH_SPEC = DatasetSpec( - key="orb", +def _orb_spec(key: str, db_filename: str) -> DatasetSpec: + return DatasetSpec( + key=key, + db_filename=db_filename, + document_loader=load_orb_corpus, + document_mapper=map_orb_document, + qa_loader=load_orb_qa, + qa_case_builder=build_orb_case, + retrieval_loader=load_orb_retrieval, + retrieval_mapper=map_orb_retrieval, + retrieval_evaluator=MAPEvaluator(), + ) + + +ORB_TEXT_SPEC = _orb_spec( + key="orb_text", db_filename="open_rag_bench_text.lancedb", - document_loader=load_orb_corpus, - document_mapper=map_orb_document, - qa_loader=load_orb_qa, - qa_case_builder=build_orb_case, - retrieval_loader=load_orb_retrieval, - retrieval_mapper=map_orb_retrieval, - retrieval_evaluator=MAPEvaluator(), +) + +ORB_MULTIMODAL_SPEC = _orb_spec( + key="orb_multimodal", + db_filename="open_rag_bench_multimodal.lancedb", )