From 0e8fa551f3dd8ea0c80f2571903d59c9f19b122c Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 15 Jul 2026 12:33:53 +0300 Subject: [PATCH] Restore hotpotqa evaluation dataset --- evaluations/configs/hotpotqa.yaml | 27 +++++ evaluations/evaluations/datasets/__init__.py | 2 + evaluations/evaluations/datasets/hotpotqa.py | 108 +++++++++++++++++++ evaluations/tests/test_datasets.py | 78 ++++++++++++++ 4 files changed, 215 insertions(+) create mode 100644 evaluations/configs/hotpotqa.yaml create mode 100644 evaluations/evaluations/datasets/hotpotqa.py diff --git a/evaluations/configs/hotpotqa.yaml b/evaluations/configs/hotpotqa.yaml new file mode 100644 index 00000000..05cfa23a --- /dev/null +++ b/evaluations/configs/hotpotqa.yaml @@ -0,0 +1,27 @@ +# Reference config for the `hotpotqa` pre-built evaluation database. +# HotpotQA (distractor validation split) multi-hop QA over wiki paragraphs. +# Run: evaluations run hotpotqa --config configs/hotpotqa.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: cross-encoder + name: mixedbread-ai/mxbai-rerank-base-v2 + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index 1e132d04..a2a58c2b 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,5 +1,6 @@ from evaluations.config import DatasetSpec +from .hotpotqa import HOTPOTQA_SPEC from .open_rag_bench import ( ORB_MULTIMODAL_NEMOTRON_SPEC, ORB_MULTIMODAL_SPEC, @@ -12,6 +13,7 @@ DATASETS: dict[str, DatasetSpec] = { spec.key: spec for spec in ( WIX_SPEC, + HOTPOTQA_SPEC, ORB_TEXT_SPEC, ORB_MULTIMODAL_SPEC, ORB_MULTIMODAL_NEMOTRON_SPEC, diff --git a/evaluations/evaluations/datasets/hotpotqa.py b/evaluations/evaluations/datasets/hotpotqa.py new file mode 100644 index 00000000..d47f939e --- /dev/null +++ b/evaluations/evaluations/datasets/hotpotqa.py @@ -0,0 +1,108 @@ +from collections.abc import Mapping +from typing import Any, cast + +from datasets import Dataset, load_dataset +from pydantic_evals import Case + +from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample +from evaluations.evaluators import MAPEvaluator + + +def load_hotpotqa_validation() -> Dataset: + dataset_dict = load_dataset("hotpotqa/hotpot_qa", "distractor") + return dataset_dict["validation"] + + +def extract_unique_documents(dataset: Dataset) -> list[dict[str, Any]]: + """Extract unique documents from all context paragraphs, deduplicated by title.""" + seen_titles: set[str] = set() + documents: list[dict[str, Any]] = [] + + for sample in dataset: + sample = cast(Mapping[str, Any], sample) + context = sample["context"] + titles = context["title"] + sentences_list = context["sentences"] + + for title, sentences in zip(titles, sentences_list): + if title in seen_titles: + continue + seen_titles.add(title) + content = " ".join(sentences) + documents.append({"title": title, "content": content}) + + return documents + + +_cached_documents: list[dict[str, Any]] | None = None + + +def load_hotpotqa_documents() -> list[dict[str, Any]]: + """Load and cache unique documents from HotpotQA.""" + global _cached_documents + if _cached_documents is None: + dataset = load_hotpotqa_validation() + _cached_documents = extract_unique_documents(dataset) + return _cached_documents + + +def document_loader() -> Dataset: + """Return documents as a Dataset-like iterable.""" + docs = load_hotpotqa_documents() + return Dataset.from_list(docs) + + +def map_hotpotqa_document(doc: Mapping[str, Any]) -> DocumentPayload: + return DocumentPayload( + uri=doc["title"], + content=doc["content"], + title=doc["title"], + ) + + +def map_hotpotqa_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None: + supporting_facts = doc["supporting_facts"] + titles = supporting_facts["title"] + if not titles: + return None + + unique_titles = tuple(dict.fromkeys(titles)) + return RetrievalSample( + question=doc["question"], + expected_uris=unique_titles, + ) + + +def build_hotpotqa_case( + index: int, doc: Mapping[str, Any] +) -> Case[str, str, dict[str, str]]: + question_id = doc["id"] + question_type = doc["type"] + level = doc["level"] + + case_name = f"{index}_{question_id}" + + return Case( + name=case_name, + inputs=doc["question"], + expected_output=doc["answer"], + metadata={ + "question_id": str(question_id), + "type": str(question_type), + "level": str(level), + "case_index": str(index), + }, + ) + + +HOTPOTQA_SPEC = DatasetSpec( + key="hotpotqa", + db_filename="hotpotqa.lancedb", + document_loader=document_loader, + document_mapper=map_hotpotqa_document, + qa_loader=load_hotpotqa_validation, + qa_case_builder=build_hotpotqa_case, + retrieval_loader=load_hotpotqa_validation, + retrieval_mapper=map_hotpotqa_retrieval, + retrieval_evaluator=MAPEvaluator(), +) diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index eda78bbf..3622697b 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -1,5 +1,11 @@ from pathlib import Path +from evaluations.datasets.hotpotqa import ( + build_hotpotqa_case, + extract_unique_documents, + map_hotpotqa_document, + map_hotpotqa_retrieval, +) from evaluations.datasets.open_rag_bench import ( build_orb_case, download_pdf, @@ -88,6 +94,78 @@ class TestWix: assert case.name == "case_1" +class TestHotpotQA: + def test_map_document(self) -> None: + doc = {"title": "Albert Einstein", "content": "Was a physicist."} + payload = map_hotpotqa_document(doc) + assert payload.uri == "Albert Einstein" + assert payload.content == "Was a physicist." + assert payload.title == "Albert Einstein" + + def test_map_retrieval(self) -> None: + doc = { + "question": "Who was Einstein?", + "supporting_facts": {"title": ["Albert Einstein", "Physics"]}, + } + sample = map_hotpotqa_retrieval(doc) + assert sample is not None + assert sample.expected_uris == ("Albert Einstein", "Physics") + + def test_map_retrieval_deduplicates_titles(self) -> None: + doc = { + "question": "Q?", + "supporting_facts": {"title": ["A", "B", "A"]}, + } + sample = map_hotpotqa_retrieval(doc) + assert sample is not None + assert sample.expected_uris == ("A", "B") + + def test_map_retrieval_no_titles(self) -> None: + doc = {"question": "Q?", "supporting_facts": {"title": []}} + assert map_hotpotqa_retrieval(doc) is None + + def test_build_case(self) -> None: + doc = { + "id": "abc123", + "question": "What is X?", + "answer": "X is Y.", + "type": "comparison", + "level": "hard", + } + case = build_hotpotqa_case(5, doc) + assert case.name == "5_abc123" + assert case.inputs == "What is X?" + assert case.expected_output == "X is Y." + assert case.metadata == { + "question_id": "abc123", + "type": "comparison", + "level": "hard", + "case_index": "5", + } + + def test_extract_unique_documents(self) -> None: + # Simulate a minimal dataset with context + dataset = [ + { + "context": { + "title": ["Doc A", "Doc B"], + "sentences": [["Sentence 1."], ["Sentence 2.", " More."]], + } + }, + { + "context": { + "title": ["Doc A", "Doc C"], + "sentences": [["Dupe."], ["Sentence 3."]], + } + }, + ] + docs = extract_unique_documents(dataset) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + assert len(docs) == 3 + titles = [d["title"] for d in docs] + assert titles == ["Doc A", "Doc B", "Doc C"] + assert docs[1]["content"] == "Sentence 2. More." + + class TestOpenRAGBench: def test_map_document(self, tmp_path: Path) -> None: # Pre-create a cached PDF