split open_rag_bench dataset into orb_text and orb_multimodal variants

docs/benchmarks.md

@@ -35,8 +35,8 @@ Available datasets:
 | `repliqa` | ~30MB |
 | `hotpotqa` | ~331MB |
 | `wix` | ~511MB |
-| `open_rag_bench` (text embedder + VLM picture descriptions) | ~15GB |
-| `open_rag_bench` (multimodal embedder, override with `--db`) | ~16GB |
+| `orb_text` — OpenRAG Bench, text embedder (`qwen3-embedding:4b`) with VLM picture descriptions baked into chunk content | ~15GB |
+| `orb_multimodal` — OpenRAG Bench, multimodal embedder (`qwen3-vl-embedding-8b`); picture vectors live in the same space as text for cross-modal retrieval | ~16GB |
 
 After downloading, run benchmarks with `--skip-db` to use the pre-built database:
 

evaluations/README.md

@@ -8,10 +8,12 @@ This package is not published to PyPI and is only used for development and testi
 
 Contains evaluation scripts for benchmarking RAG retrieval and QA performance, plus GEPA-based prompt optimization. Available datasets:
 
-- RepliQA
-- WiX
-- HotpotQA
-- OpenRAG Bench
+- RepliQA (`repliqa`)
+- WiX (`wix`)
+- HotpotQA (`hotpotqa`)
+- OpenRAG Bench, two variants:
+  - `orb_text` — text embedder (`qwen3-embedding:4b`, 2560-dim) with VLM picture descriptions baked into chunk content at ingest. Use for text-only retrieval/QA against figure-rich corpora.
+  - `orb_multimodal` — multimodal embedder (`qwen3-vl-embedding-8b`, 4096-dim) with picture vectors in the same space as text. Use for cross-modal retrieval (text-as-query → figure hits, image-as-query) and vision QA where the figure itself is the answer.
 
 ## Usage
 

evaluations/evaluations/benchmark.py

@@ -765,7 +765,15 @@ def download(
 def upload(
     dataset: str = typer.Argument(..., help="Dataset key or 'all' to upload all."),
 ) -> None:
-    """Upload evaluation database to HuggingFace (maintainer only)."""
+    """Upload evaluation database to HuggingFace (maintainer only).
+
+    Uses ``upload_large_folder`` for resumable, parallel transfer — important
+    for the multi-GB ORB databases which would otherwise abort on any transient
+    network failure under plain ``upload_folder``.
+
+    The local folder basename equals ``spec.db_filename`` (see
+    ``DatasetSpec.db_path``), so the folder lands at that path on the Hub.
+    """
     specs = _resolve_datasets(dataset)
 
     api = HfApi()
@@ -776,13 +784,23 @@ def upload(
             console.print(f"[red]Database not found at {db}[/red]")
             continue
 
-        console.print(f"[blue]Uploading {spec.key}...[/blue]")
-        api.upload_folder(
+        # Wipe the existing remote path so we don't accumulate orphaned files
+        # from prior uploads. upload_large_folder doesn't accept delete_patterns,
+        # so we do this as a separate commit. Safe to run if the path is missing.
+        try:
+            api.delete_folder(
+                path_in_repo=spec.db_filename,
+                repo_id=HF_REPO_ID,
+                repo_type="dataset",
+            )
+        except Exception:
+            pass
+
+        console.print(f"[blue]Uploading {spec.key} ({db})...[/blue]")
+        api.upload_large_folder(
             folder_path=str(db),
-            path_in_repo=spec.db_filename,
             repo_id=HF_REPO_ID,
             repo_type="dataset",
-            delete_patterns="*",
         )
 
         console.print(f"[green]Uploaded {spec.key} to {HF_REPO_ID}[/green]")

evaluations/evaluations/datasets/__init__.py

@@ -1,13 +1,19 @@
 from evaluations.config import DatasetSpec
 
 from .hotpotqa import HOTPOTQA_SPEC
-from .open_rag_bench import OPEN_RAG_BENCH_SPEC
+from .open_rag_bench import ORB_MULTIMODAL_SPEC, ORB_TEXT_SPEC
 from .repliqa import REPLIQA_SPEC
 from .wix import WIX_SPEC
 
 DATASETS: dict[str, DatasetSpec] = {
     spec.key: spec
-    for spec in (REPLIQA_SPEC, WIX_SPEC, HOTPOTQA_SPEC, OPEN_RAG_BENCH_SPEC)
+    for spec in (
+        REPLIQA_SPEC,
+        WIX_SPEC,
+        HOTPOTQA_SPEC,
+        ORB_TEXT_SPEC,
+        ORB_MULTIMODAL_SPEC,
+    )
 }
 
 __all__ = ["DATASETS"]

evaluations/evaluations/datasets/open_rag_bench.py

@@ -216,14 +216,26 @@ def is_multimodal_query(source: str) -> bool:
     return "image" in source
 
 
-OPEN_RAG_BENCH_SPEC = DatasetSpec(
-    key="orb",
+def _orb_spec(key: str, db_filename: str) -> DatasetSpec:
+    return DatasetSpec(
+        key=key,
+        db_filename=db_filename,
+        document_loader=load_orb_corpus,
+        document_mapper=map_orb_document,
+        qa_loader=load_orb_qa,
+        qa_case_builder=build_orb_case,
+        retrieval_loader=load_orb_retrieval,
+        retrieval_mapper=map_orb_retrieval,
+        retrieval_evaluator=MAPEvaluator(),
+    )
+
+
+ORB_TEXT_SPEC = _orb_spec(
+    key="orb_text",
     db_filename="open_rag_bench_text.lancedb",
-    document_loader=load_orb_corpus,
-    document_mapper=map_orb_document,
-    qa_loader=load_orb_qa,
-    qa_case_builder=build_orb_case,
-    retrieval_loader=load_orb_retrieval,
-    retrieval_mapper=map_orb_retrieval,
-    retrieval_evaluator=MAPEvaluator(),
+)
+
+ORB_MULTIMODAL_SPEC = _orb_spec(
+    key="orb_multimodal",
+    db_filename="open_rag_bench_multimodal.lancedb",
 )