From 5c4799164c342399906b131d38a680d55b8a82d7 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 11 Dec 2025 10:17:45 +0200 Subject: [PATCH 1/3] Escape when getting by URI --- .../haiku/rag/store/repositories/document.py | 8 +++++- tests/test_document.py | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/haiku_rag_slim/haiku/rag/store/repositories/document.py b/haiku_rag_slim/haiku/rag/store/repositories/document.py index 52172ccf..f9a2c037 100644 --- a/haiku_rag_slim/haiku/rag/store/repositories/document.py +++ b/haiku_rag_slim/haiku/rag/store/repositories/document.py @@ -6,6 +6,11 @@ from haiku.rag.store.engine import DocumentRecord, Store from haiku.rag.store.models.document import Document +def _escape_sql_string(value: str) -> str: + """Escape single quotes in SQL string literals.""" + return value.replace("'", "''") + + class DocumentRepository: """Repository for Document operations.""" @@ -161,9 +166,10 @@ class DocumentRepository: async def get_by_uri(self, uri: str) -> Document | None: """Get a document by its URI.""" + escaped_uri = _escape_sql_string(uri) results = list( self.store.documents_table.search() - .where(f"uri = '{uri}'") + .where(f"uri = '{escaped_uri}'") .limit(1) .to_pydantic(DocumentRecord) ) diff --git a/tests/test_document.py b/tests/test_document.py index ba884696..1cf25678 100644 --- a/tests/test_document.py +++ b/tests/test_document.py @@ -173,3 +173,30 @@ def test_document_get_docling_document_no_id_no_cache(): # Each call parses fresh (different objects) assert doc1 is not doc2 + + +@pytest.mark.asyncio +async def test_document_get_by_uri_with_special_characters( + qa_corpus: Dataset, temp_db_path +): + """Test get_by_uri handles URIs with special characters like single quotes.""" + store = Store(temp_db_path, create=True) + doc_repo = DocumentRepository(store) + + first_doc = qa_corpus[0] + document_text = first_doc["document_extracted"] + + doc_with_quote = Document( + content=document_text, + uri="Hamish and Andy's Gap Year", + metadata={"source": "test"}, + ) + + created_doc = await doc_repo.create(doc_with_quote) + + retrieved = await doc_repo.get_by_uri("Hamish and Andy's Gap Year") + assert retrieved is not None + assert retrieved.id == created_doc.id + assert retrieved.uri == "Hamish and Andy's Gap Year" + + store.close() From 752673505979e0e6260d4e0bc0bd73242f1682ec Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 11 Dec 2025 10:18:44 +0200 Subject: [PATCH 2/3] hotpotqa adapter --- evaluations/evaluations/datasets/__init__.py | 5 +- evaluations/evaluations/datasets/hotpotqa.py | 108 +++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 evaluations/evaluations/datasets/hotpotqa.py diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index 94a00c34..54b92b3b 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,8 +1,11 @@ from evaluations.config import DatasetSpec +from .hotpotqa import HOTPOTQA_SPEC from .repliqa import REPLIQ_SPEC from .wix import WIX_SPEC -DATASETS: dict[str, DatasetSpec] = {spec.key: spec for spec in (REPLIQ_SPEC, WIX_SPEC)} +DATASETS: dict[str, DatasetSpec] = { + spec.key: spec for spec in (REPLIQ_SPEC, WIX_SPEC, HOTPOTQA_SPEC) +} __all__ = ["DATASETS"] diff --git a/evaluations/evaluations/datasets/hotpotqa.py b/evaluations/evaluations/datasets/hotpotqa.py new file mode 100644 index 00000000..2f0a9b1c --- /dev/null +++ b/evaluations/evaluations/datasets/hotpotqa.py @@ -0,0 +1,108 @@ +from collections.abc import Mapping +from typing import Any, cast + +from datasets import Dataset, DatasetDict, load_dataset +from pydantic_evals import Case + +from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample +from evaluations.evaluators import MAPEvaluator + + +def load_hotpotqa_validation() -> Dataset: + dataset_dict = cast(DatasetDict, load_dataset("hotpotqa/hotpot_qa", "distractor")) + return cast(Dataset, dataset_dict["validation"]) + + +def extract_unique_documents(dataset: Dataset) -> list[dict[str, Any]]: + """Extract unique documents from all context paragraphs, deduplicated by title.""" + seen_titles: set[str] = set() + documents: list[dict[str, Any]] = [] + + for sample in dataset: + sample = cast(Mapping[str, Any], sample) + context = sample["context"] + titles = context["title"] + sentences_list = context["sentences"] + + for title, sentences in zip(titles, sentences_list): + if title in seen_titles: + continue + seen_titles.add(title) + content = " ".join(sentences) + documents.append({"title": title, "content": content}) + + return documents + + +_cached_documents: list[dict[str, Any]] | None = None + + +def load_hotpotqa_documents() -> list[dict[str, Any]]: + """Load and cache unique documents from HotpotQA.""" + global _cached_documents + if _cached_documents is None: + dataset = load_hotpotqa_validation() + _cached_documents = extract_unique_documents(dataset) + return _cached_documents + + +def document_loader() -> Dataset: + """Return documents as a Dataset-like iterable.""" + docs = load_hotpotqa_documents() + return Dataset.from_list(docs) + + +def map_hotpotqa_document(doc: Mapping[str, Any]) -> DocumentPayload: + return DocumentPayload( + uri=doc["title"], + content=doc["content"], + title=doc["title"], + ) + + +def map_hotpotqa_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None: + supporting_facts = doc["supporting_facts"] + titles = supporting_facts["title"] + if not titles: + return None + + unique_titles = tuple(dict.fromkeys(titles)) + return RetrievalSample( + question=doc["question"], + expected_uris=unique_titles, + ) + + +def build_hotpotqa_case( + index: int, doc: Mapping[str, Any] +) -> Case[str, str, dict[str, str]]: + question_id = doc["id"] + question_type = doc["type"] + level = doc["level"] + + case_name = f"{index}_{question_id}" + + return Case( + name=case_name, + inputs=doc["question"], + expected_output=doc["answer"], + metadata={ + "question_id": str(question_id), + "type": str(question_type), + "level": str(level), + "case_index": str(index), + }, + ) + + +HOTPOTQA_SPEC = DatasetSpec( + key="hotpotqa", + db_filename="hotpotqa.lancedb", + document_loader=document_loader, + document_mapper=map_hotpotqa_document, + qa_loader=load_hotpotqa_validation, + qa_case_builder=build_hotpotqa_case, + retrieval_loader=load_hotpotqa_validation, + retrieval_mapper=map_hotpotqa_retrieval, + retrieval_evaluator=MAPEvaluator(), +) From f726229f60a73cc239633e4ec2cd528a0a527abc Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 12 Dec 2025 12:20:06 +0200 Subject: [PATCH 3/3] Reformat benchmarks and add hotpotqa placeholder --- CHANGELOG.md | 13 ++++ docs/benchmarks.md | 156 ++++++++++++++++++++++----------------------- 2 files changed, 88 insertions(+), 81 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3626f220..fee7f76b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog ## [Unreleased] +### Added + +- **HotpotQA Evaluation**: Added HotpotQA dataset adapter for multi-hop QA benchmarks + - Extracts unique documents from validation set context paragraphs + - Uses MAP for retrieval evaluation (multiple supporting documents per question) + - Run with `evaluations hotpotqa` + +### Changed + +- **Benchmarks Documentation**: Restructured benchmarks.md for clarity + - Added dedicated Methodology section explaining MRR, MAP, and QA Accuracy metrics + - Organized results by dataset with retrieval and QA subsections + ## [0.20.2] - 2025-12-12 ### Fixed diff --git a/docs/benchmarks.md b/docs/benchmarks.md index f0ee8c9f..350adfbe 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,18 +1,19 @@ # Benchmarks -We use the [repliqa](https://huggingface.co/datasets/ServiceNow/repliqa) dataset for the evaluation of `haiku.rag`. +We evaluate `haiku.rag` on several datasets to measure both retrieval quality and question-answering accuracy. -You can perform your own evaluations with the `evaluations` CLI command: +## Running Evaluations + +You can run evaluations with the `evaluations` CLI: ```bash evaluations repliqa +evaluations wix ``` -The evaluation flow is orchestrated with -[`pydantic-evals`](https://github.com/pydantic/pydantic-ai/tree/main/libs/pydantic-evals), -which we leverage for dataset management, scoring, and report generation. +The evaluation flow is orchestrated with [`pydantic-evals`](https://github.com/pydantic/pydantic-ai/tree/main/libs/pydantic-evals), which we leverage for dataset management, scoring, and report generation. -## Configuration +### Configuration The benchmark script accepts several options: @@ -20,109 +21,102 @@ The benchmark script accepts several options: evaluations repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.lancedb ``` -**Configuration options:** +**Options:** + - `--config PATH` - Specify a custom `haiku.rag.yaml` configuration file -- `--db PATH` - Override the database path (default: `~/.local/share/haiku.rag/evaluations/dbs/{dataset}.lancedb` on Linux, `~/Library/Application Support/haiku.rag/evaluations/dbs/{dataset}.lancedb` on macOS) +- `--db PATH` - Override the database path (default: platform-specific user data directory) - `--skip-db` - Skip updating the evaluation database - `--skip-retrieval` - Skip retrieval benchmark - `--skip-qa` - Skip QA benchmark -- `--limit N` - Limit number of test cases for both retrieval and QA -- `--name NAME` - Override the evaluation name (defaults to `{dataset}_retrieval_evaluation` or `{dataset}_qa_evaluation`) +- `--limit N` - Limit number of test cases +- `--name NAME` - Override the evaluation name -If no config file is specified, the script will search for a config file in the standard locations: -1. `./haiku.rag.yaml` (current directory) -2. User config directory -3. Falls back to default configuration +If no config file is specified, the script searches standard locations: `./haiku.rag.yaml`, user config directory, then falls back to defaults. -## RepliQA Retrieval +## Methodology -We use the [RepliQA](https://huggingface.co/datasets/ServiceNow/repliqa) dataset to evaluate retrieval performance. We load the `News Stories` from `repliqa_3` (1035 documents) and index them. Subsequently, we run a search over the `question` field for each row of the dataset and check whether we match the document that answers the question. Questions for which the answer cannot be found in the documents are ignored. +### Retrieval Metrics -For RepliQA, we use **Mean Reciprocal Rank (MRR)** as the primary metric since each query has exactly one relevant document. +**Mean Reciprocal Rank (MRR)** - Used when each query has exactly one relevant document. -**How MRR is calculated:** -- For each query, we retrieve the top-K documents and find the rank (position) of the first relevant document -- The reciprocal rank for that query is `1/rank` (e.g., if the relevant document is at position 3, the score is 1/3 ≈ 0.333) -- If no relevant document is found in the top-K results, the score is 0 -- MRR is the mean of these reciprocal ranks across all queries -- Scores range from 0 (never found) to 1 (always found at rank 1) +- For each query, find the rank (position) of the first relevant document in top-K results +- Reciprocal rank = `1/rank` (e.g., rank 3 → 1/3 ≈ 0.333) +- If not found in top-K, score is 0 +- MRR is the mean across all queries +- Range: 0 (never found) to 1 (always at rank 1) -**Example:** If we run 3 queries and the relevant documents are found at positions 1, 2, and not found: -- Query 1: 1/1 = 1.0 -- Query 2: 1/2 = 0.5 -- Query 3: 0 (not found) -- MRR = (1.0 + 0.5 + 0) / 3 = 0.5 +**Mean Average Precision (MAP)** - Used when queries have multiple relevant documents. -### MRR Results +- For each relevant document at position k, calculate precision@k = (relevant docs in top k) / k +- Average Precision (AP) = mean of these precision values / total relevant documents +- MAP is the mean of AP scores across all queries +- Range: 0 to 1; rewards ranking relevant documents higher -| Embedding Model | MRR | Reranker | -|---------------------------------------|-------|------------------------| -| Ollama / `qwen3-embedding:8b` | 0.91 | - | +### QA Accuracy -## Question/Answer evaluation +For question-answering evaluation, `pydantic-evals` coordinates an LLM judge (Ollama `qwen3`) to determine whether answers are correct. Accuracy is the fraction of correctly answered questions. -Again using the same dataset, we use a QA agent to answer the question. -`pydantic-evals` runs each case and coordinates an LLM judge (Ollama `qwen3`) to -determine whether the answer is correct. The obtained accuracy is as follows: +## RepliQA -| Embedding Model | QA Model | Accuracy | Reranker | -|------------------------------------|-----------------------------------|-----------|------------------------| -| Ollama / `qwen3-embedding:4b` | Ollama / `gpt-oss` - no thinking | 0.82 | None | -| Ollama / `qwen3-embedding:0.6b` | Ollama / `gpt-oss` - thinking | 0.89 | None | -| Ollama / `mxbai-embed-large` | Ollama / `qwen3` - thinking | 0.85 | None | -| Ollama / `mxbai-embed-large` | Ollama / `qwen3` - thinking | 0.87 | `mxbai-rerank-base-v2` | -| Ollama / `mxbai-embed-large` | Ollama / `qwen3:0.6b` | 0.28 | None | +[RepliQA](https://huggingface.co/datasets/ServiceNow/repliqa) contains synthetic news stories with question-answer pairs. We use `News Stories` from `repliqa_3` (1035 documents). Each question has exactly one relevant document, so we use MRR for retrieval evaluation. + +*Results from v0.19.6* + +### Retrieval (MRR) + +| Embedding Model | MRR | Reranker | +|-------------------------------|------|----------| +| Ollama / `qwen3-embedding:8b` | 0.91 | - | + +### QA Accuracy + +| Embedding Model | QA Model | Accuracy | Reranker | +|------------------------------|----------------------------------|----------|------------------------| +| Ollama / `qwen3-embedding:4b` | Ollama / `gpt-oss` - no thinking | 0.82 | None | +| Ollama / `qwen3-embedding:0.6b` | Ollama / `gpt-oss` - thinking | 0.89 | None | +| Ollama / `mxbai-embed-large` | Ollama / `qwen3` - thinking | 0.85 | None | +| Ollama / `mxbai-embed-large` | Ollama / `qwen3` - thinking | 0.87 | `mxbai-rerank-base-v2` | +| Ollama / `mxbai-embed-large` | Ollama / `qwen3:0.6b` | 0.28 | None | Note the significant degradation when very small models are used such as `qwen3:0.6b`. -## Wix Retrieval +## Wix -We also track retrieval performance on [WixQA](https://huggingface.co/datasets/Wix/WixQA), -a dataset of real customer support questions paired with curated answers from -Wix. The benchmark follows the evaluation protocol described in the -[WixQA paper](https://arxiv.org/abs/2505.08643) and gives us a view into how the -system handles conversational, product-specific support queries. +[WixQA](https://huggingface.co/datasets/Wix/WixQA) contains real customer support questions paired with curated answers from Wix. The benchmark follows the evaluation protocol from the [WixQA paper](https://arxiv.org/abs/2505.08643). Each query can have multiple relevant passages, so we use MAP for retrieval evaluation. -For retrieval evaluation, we index the reference answer passages shipped with the dataset and -run retrieval against each user question. Each sample supplies one or more -relevant passage URIs. +We benchmark both the plain text version (HTML stripped, no structure) and HTML version. Since HTML chunks are small (typically a phrase), we use `chunk_radius=2` to expand context. -For Wix, we use **Mean Average Precision (MAP)** as the primary metric since each query has multiple relevant documents. MAP accounts for both the presence and ranking of all relevant documents. +*Results from v0.20.0* -**How MAP is calculated:** -- For each query, we retrieve the top-K documents and identify which ones are relevant -- For each relevant document found at position k, we calculate precision@k = (number of relevant docs in top k) / k -- Average Precision (AP) for that query is the mean of these precision values, divided by the total number of relevant documents -- MAP is the mean of AP scores across all queries -- Scores range from 0 (no relevant documents found) to 1 (all relevant documents ranked at the top) +### Retrieval (MAP) -**Example:** If a query has 2 relevant documents (A and B), and we retrieve 5 documents [A, X, B, Y, Z]: -- A is at position 1: precision@1 = 1/1 = 1.0 (1 relevant out of top 1) -- B is at position 3: precision@3 = 2/3 ≈ 0.667 (2 relevant out of top 3) -- AP = (1.0 + 0.667) / 2 = 0.833 -- If we had another query with AP = 0.5, then MAP = (0.833 + 0.5) / 2 = 0.667 +| Embedding Model | Chunk size | MAP | Reranker | Notes | +|------------------------|------------|------|------------------------|------------------------------| +| `qwen3-embedding:4b` | 256 | 0.34 | None | html, `chunk-radius=2` | +| `qwen3-embedding:4b` | 256 | 0.39 | `mxbai-rerank-base-v2` | html, `chunk-radius=2` | +| `qwen3-embedding:4b` | 256 | 0.43 | None | plain text, `chunk-radius=0` | +| `qwen3-embedding:4b` | 512 | 0.45 | None | plain text, `chunk-radius=0` | -MAP rewards systems that rank relevant documents higher, not just finding them. +### QA Accuracy -In the following results the benchmarks have been run using both the plain text version of the Wix dataset (which has HTML tags stripped and no document structure) as well as the HTML version. -Since the chunks in the HTML version are very small (typically a phrase) we use `chunk_radius=2` to tune the retrieval. +| Embedding Model | Chunk size | QA Model | Accuracy | Notes | +|----------------------|------------|-----------------------------|----------|------------------------------| +| `qwen3-embedding:4b` | 256 | `gpt-oss:20b` - no thinking | 0.74 | plain text, `chunk-radius=0` | +| `qwen3-embedding:4b` | 256 | `gpt-oss:20b` - thinking | 0.79 | html, `chunk-radius=2` | +| `qwen3-embedding:4b` | 256 | `gpt-oss:20b` - thinking | 0.80 | html, `chunk-radius=2`, reranker=`mxbai-rerank-base-v2` | -### MAP Results +## HotpotQA -| Embedding Model | Chunk size | MAP | Reranker | Notes | -|----------------------------|------------|-------|------------------------|--------------------------------------------------------| -| `qwen3-embedding:4b` | 256 | 0.34 | None | html, `chunk-radius=2` | -| `qwen3-embedding:4b` | 256 | 0.39 | None | html, `chunk-radius=2`, reranker=`mxbai-rerank-base-v2`| -| `qwen3-embedding:4b` | 256 | 0.43 | None | plain text, `chunk-radius=0` | -| `qwen3-embedding:4b` | 512 | 0.45 | None | plain text, `chunk-radius=0` | +[HotpotQA](https://huggingface.co/datasets/hotpotqa/hotpot_qa) is a multi-hop question answering dataset requiring reasoning over multiple Wikipedia paragraphs. Each question requires evidence from 2+ documents, making it ideal for testing retrieval and reasoning capabilities. We use MAP for retrieval evaluation since queries have multiple relevant documents. +### Retrieval (MAP) -## QA Accuracy +| Embedding Model | MAP | Reranker | Notes | +|-----------------|-----|----------|-------| +| | | | | -And for QA accuracy, +### QA Accuracy -| Embedding Model | Chunk size | QA Model | Accuracy | Notes | -|----------------------------|------------|-----------------------------|----------|---------------------------------------------------------| -| `qwen3-embedding:4b` | 256 | `gpt-oss:20b` - no thinking | 0.74 | plain text, `chunk-radius=0` | -| `qwen3-embedding:4b` | 256 | `gpt-oss:20b` - thinking | 0.79 | html, `chunk-radius=2` | -| `qwen3-embedding:4b` | 256 | `gpt-oss:20b` - thinking | 0.80 | html, `chunk-radius=2`, reranker=`mxbai-rerank-base-v2` | +| Embedding Model | QA Model | Accuracy | Notes | +|-----------------|----------|----------|-------| +| | | | |