Restore hotpotqa evaluation dataset

evaluations/configs/hotpotqa.yaml

@@ -0,0 +1,27 @@
+# Reference config for the `hotpotqa` pre-built evaluation database.
+# HotpotQA (distractor validation split) multi-hop QA over wiki paragraphs.
+# Run: evaluations run hotpotqa --config configs/hotpotqa.yaml
+# base_url uses the `vllm` host serving each model over an OpenAI-compatible API.
+
+environment: development
+
+storage:
+  auto_vacuum: false
+
+embeddings:
+  model:
+    provider: openai
+    name: qwen3-embedding-4b
+    vector_dim: 2560
+    base_url: http://vllm:11431/v1
+
+reranking:
+  model:
+    provider: cross-encoder
+    name: mixedbread-ai/mxbai-rerank-base-v2
+
+qa:
+  model:
+    provider: openai
+    name: gemma4-26b
+    base_url: http://vllm:11432/v1

evaluations/evaluations/datasets/__init__.py

@@ -1,5 +1,6 @@
 from evaluations.config import DatasetSpec
 
+from .hotpotqa import HOTPOTQA_SPEC
 from .open_rag_bench import (
     ORB_MULTIMODAL_NEMOTRON_SPEC,
     ORB_MULTIMODAL_SPEC,
@@ -12,6 +13,7 @@ DATASETS: dict[str, DatasetSpec] = {
     spec.key: spec
     for spec in (
         WIX_SPEC,
+        HOTPOTQA_SPEC,
         ORB_TEXT_SPEC,
         ORB_MULTIMODAL_SPEC,
         ORB_MULTIMODAL_NEMOTRON_SPEC,

evaluations/evaluations/datasets/hotpotqa.py

@@ -0,0 +1,108 @@
+from collections.abc import Mapping
+from typing import Any, cast
+
+from datasets import Dataset, load_dataset
+from pydantic_evals import Case
+
+from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
+from evaluations.evaluators import MAPEvaluator
+
+
+def load_hotpotqa_validation() -> Dataset:
+    dataset_dict = load_dataset("hotpotqa/hotpot_qa", "distractor")
+    return dataset_dict["validation"]
+
+
+def extract_unique_documents(dataset: Dataset) -> list[dict[str, Any]]:
+    """Extract unique documents from all context paragraphs, deduplicated by title."""
+    seen_titles: set[str] = set()
+    documents: list[dict[str, Any]] = []
+
+    for sample in dataset:
+        sample = cast(Mapping[str, Any], sample)
+        context = sample["context"]
+        titles = context["title"]
+        sentences_list = context["sentences"]
+
+        for title, sentences in zip(titles, sentences_list):
+            if title in seen_titles:
+                continue
+            seen_titles.add(title)
+            content = " ".join(sentences)
+            documents.append({"title": title, "content": content})
+
+    return documents
+
+
+_cached_documents: list[dict[str, Any]] | None = None
+
+
+def load_hotpotqa_documents() -> list[dict[str, Any]]:
+    """Load and cache unique documents from HotpotQA."""
+    global _cached_documents
+    if _cached_documents is None:
+        dataset = load_hotpotqa_validation()
+        _cached_documents = extract_unique_documents(dataset)
+    return _cached_documents
+
+
+def document_loader() -> Dataset:
+    """Return documents as a Dataset-like iterable."""
+    docs = load_hotpotqa_documents()
+    return Dataset.from_list(docs)
+
+
+def map_hotpotqa_document(doc: Mapping[str, Any]) -> DocumentPayload:
+    return DocumentPayload(
+        uri=doc["title"],
+        content=doc["content"],
+        title=doc["title"],
+    )
+
+
+def map_hotpotqa_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None:
+    supporting_facts = doc["supporting_facts"]
+    titles = supporting_facts["title"]
+    if not titles:
+        return None
+
+    unique_titles = tuple(dict.fromkeys(titles))
+    return RetrievalSample(
+        question=doc["question"],
+        expected_uris=unique_titles,
+    )
+
+
+def build_hotpotqa_case(
+    index: int, doc: Mapping[str, Any]
+) -> Case[str, str, dict[str, str]]:
+    question_id = doc["id"]
+    question_type = doc["type"]
+    level = doc["level"]
+
+    case_name = f"{index}_{question_id}"
+
+    return Case(
+        name=case_name,
+        inputs=doc["question"],
+        expected_output=doc["answer"],
+        metadata={
+            "question_id": str(question_id),
+            "type": str(question_type),
+            "level": str(level),
+            "case_index": str(index),
+        },
+    )
+
+
+HOTPOTQA_SPEC = DatasetSpec(
+    key="hotpotqa",
+    db_filename="hotpotqa.lancedb",
+    document_loader=document_loader,
+    document_mapper=map_hotpotqa_document,
+    qa_loader=load_hotpotqa_validation,
+    qa_case_builder=build_hotpotqa_case,
+    retrieval_loader=load_hotpotqa_validation,
+    retrieval_mapper=map_hotpotqa_retrieval,
+    retrieval_evaluator=MAPEvaluator(),
+)

evaluations/tests/test_datasets.py

@@ -1,5 +1,11 @@
 from pathlib import Path
 
+from evaluations.datasets.hotpotqa import (
+    build_hotpotqa_case,
+    extract_unique_documents,
+    map_hotpotqa_document,
+    map_hotpotqa_retrieval,
+)
 from evaluations.datasets.open_rag_bench import (
     build_orb_case,
     download_pdf,
@@ -88,6 +94,78 @@ class TestWix:
         assert case.name == "case_1"
 
 
+class TestHotpotQA:
+    def test_map_document(self) -> None:
+        doc = {"title": "Albert Einstein", "content": "Was a physicist."}
+        payload = map_hotpotqa_document(doc)
+        assert payload.uri == "Albert Einstein"
+        assert payload.content == "Was a physicist."
+        assert payload.title == "Albert Einstein"
+
+    def test_map_retrieval(self) -> None:
+        doc = {
+            "question": "Who was Einstein?",
+            "supporting_facts": {"title": ["Albert Einstein", "Physics"]},
+        }
+        sample = map_hotpotqa_retrieval(doc)
+        assert sample is not None
+        assert sample.expected_uris == ("Albert Einstein", "Physics")
+
+    def test_map_retrieval_deduplicates_titles(self) -> None:
+        doc = {
+            "question": "Q?",
+            "supporting_facts": {"title": ["A", "B", "A"]},
+        }
+        sample = map_hotpotqa_retrieval(doc)
+        assert sample is not None
+        assert sample.expected_uris == ("A", "B")
+
+    def test_map_retrieval_no_titles(self) -> None:
+        doc = {"question": "Q?", "supporting_facts": {"title": []}}
+        assert map_hotpotqa_retrieval(doc) is None
+
+    def test_build_case(self) -> None:
+        doc = {
+            "id": "abc123",
+            "question": "What is X?",
+            "answer": "X is Y.",
+            "type": "comparison",
+            "level": "hard",
+        }
+        case = build_hotpotqa_case(5, doc)
+        assert case.name == "5_abc123"
+        assert case.inputs == "What is X?"
+        assert case.expected_output == "X is Y."
+        assert case.metadata == {
+            "question_id": "abc123",
+            "type": "comparison",
+            "level": "hard",
+            "case_index": "5",
+        }
+
+    def test_extract_unique_documents(self) -> None:
+        # Simulate a minimal dataset with context
+        dataset = [
+            {
+                "context": {
+                    "title": ["Doc A", "Doc B"],
+                    "sentences": [["Sentence 1."], ["Sentence 2.", " More."]],
+                }
+            },
+            {
+                "context": {
+                    "title": ["Doc A", "Doc C"],
+                    "sentences": [["Dupe."], ["Sentence 3."]],
+                }
+            },
+        ]
+        docs = extract_unique_documents(dataset)  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+        assert len(docs) == 3
+        titles = [d["title"] for d in docs]
+        assert titles == ["Doc A", "Doc B", "Doc C"]
+        assert docs[1]["content"] == "Sentence 2.  More."
+
+
 class TestOpenRAGBench:
     def test_map_document(self, tmp_path: Path) -> None:
         # Pre-create a cached PDF