exclude corrupt mi_phone.pdf from MMLongBench-Doc

docs/configuration/qa.md

@@ -18,6 +18,11 @@ Context expansion is automatic and section-aware. For structured documents (with
 !!! note "Reranking behavior"
     When a reranker is configured, search automatically retrieves 10x the requested limit, then reranks to return the final count. This improves result quality without requiring you to adjust `limit`.
 
+!!! warning "Reranker compatibility with multimodal content"
+    Text-only rerankers (mxbai, cohere, jina, vllm, cross-encoder) score documents by their text content. Picture chunks emitted by docling have empty content, so the reranker scores them near zero and drops them from the top results.
+
+    If retrieval relies on visual content (charts, figures, screenshots with text rendered as pixels), disable reranking by leaving `reranking.model` empty so the multimodal embedder's ranking survives.
+
 ## Question Answering Configuration
 
 Configure the rag skill (used by `client.ask`, `haiku-rag ask`, and the MCP `ask_question` tool):

evaluations/evaluations/benchmark.py

@@ -406,6 +406,16 @@ async def run_qa_benchmark(
     skill_factory = _skill_factory_for_target(target)
     resolved_skill_model = get_model(skill_config, config)
 
+    from haiku.rag.tools.filters import build_multi_document_filter
+
+    question_to_filter: dict[str, str | None] = {}
+    for case in cases:
+        meta = case.metadata or {}
+        target_uri = meta.get("target_doc_uri")
+        question_to_filter[case.inputs] = (
+            build_multi_document_filter([target_uri]) if target_uri else None
+        )
+
     async def answer_question(question: str) -> str:
         result = await run_skill_question(
             skill_factory=skill_factory,
@@ -413,6 +423,7 @@ async def run_qa_benchmark(
             config=config,
             question=question,
             skill_model=resolved_skill_model,
+            document_filter=question_to_filter.get(question),
         )
         set_eval_attribute("cited_uris", result.cited_uris)
         return result.answer

evaluations/evaluations/datasets/__init__.py

@@ -1,5 +1,6 @@
 from evaluations.config import DatasetSpec
 
+from .mmlongbench import MMLONGBENCH_SPEC
 from .open_rag_bench import ORB_MULTIMODAL_SPEC, ORB_TEXT_SPEC
 from .wix import WIX_SPEC
 
@@ -9,6 +10,7 @@ DATASETS: dict[str, DatasetSpec] = {
         WIX_SPEC,
         ORB_TEXT_SPEC,
         ORB_MULTIMODAL_SPEC,
+        MMLONGBENCH_SPEC,
     )
 }
 

evaluations/evaluations/datasets/mmlongbench.py

@@ -0,0 +1,162 @@
+import ast
+import logging
+from collections.abc import Mapping
+from pathlib import Path
+from typing import Any
+
+from datasets import Dataset, load_dataset
+from huggingface_hub import snapshot_download
+from pydantic_evals import Case
+
+from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
+from evaluations.evaluators import MAPEvaluator
+
+logger = logging.getLogger(__name__)
+
+REPO_ID = "yubo2333/MMLongBench-Doc"
+PDF_SUBDIR = "documents"
+_LIST_FIELDS = ("evidence_pages", "evidence_sources")
+
+# The HF repo serves a blob for these files whose content does not match the
+# document the questions were written against (the LFS pointer and the served
+# bytes diverge, and the bytes are an unrelated PDF). They are unrecoverable
+# upstream, so the document and all its questions are dropped to keep the
+# benchmark answerable and reproducible.
+_EXCLUDED_DOCS = frozenset({"mi_phone.pdf"})
+
+
+def get_cache_dir() -> Path:
+    cache_dir = Path.home() / ".cache" / "haiku.rag" / "evaluations" / "mmlongbench"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir
+
+
+def ensure_pdfs_downloaded() -> Path:
+    cache_dir = get_cache_dir()
+    snapshot_download(
+        repo_id=REPO_ID,
+        repo_type="dataset",
+        allow_patterns=f"{PDF_SUBDIR}/*.pdf",
+        ignore_patterns=[f"{PDF_SUBDIR}/{name}" for name in _EXCLUDED_DOCS],
+        local_dir=cache_dir,
+    )
+    return cache_dir / PDF_SUBDIR
+
+
+def _parse_list_field(value: Any) -> list[Any]:
+    if isinstance(value, list):
+        return value
+    if not value:
+        return []
+    return ast.literal_eval(value)
+
+
+def _load_hf_qa_split() -> list[dict[str, Any]]:
+    dataset = load_dataset(REPO_ID, split="train")
+    return [dict(row) for row in dataset]
+
+
+_qa_records: list[dict[str, Any]] | None = None
+
+
+def load_qa_records() -> list[dict[str, Any]]:
+    global _qa_records
+    if _qa_records is not None:
+        return _qa_records
+    rows = [r for r in _load_hf_qa_split() if r.get("doc_id") not in _EXCLUDED_DOCS]
+    for row in rows:
+        for field in _LIST_FIELDS:
+            row[field] = _parse_list_field(row.get(field))
+    _qa_records = rows
+    return _qa_records
+
+
+def load_mmlb_corpus() -> Dataset:
+    pdf_dir = ensure_pdfs_downloaded()
+    records = load_qa_records()
+    doc_types: dict[str, str] = {}
+    for row in records:
+        doc_id = row["doc_id"]
+        if doc_id not in doc_types:
+            doc_types[doc_id] = row.get("doc_type", "")
+    corpus = [
+        {"doc_id": doc_id, "doc_type": doc_type}
+        for doc_id, doc_type in doc_types.items()
+        if (pdf_dir / doc_id).exists()
+    ]
+    return Dataset.from_list(corpus)
+
+
+def map_mmlb_document(doc: Mapping[str, Any]) -> DocumentPayload | None:
+    doc_id = doc["doc_id"]
+    pdf_path = get_cache_dir() / PDF_SUBDIR / doc_id
+    if not pdf_path.exists():
+        logger.warning(f"PDF not found in cache: {doc_id}")
+        return None
+    return DocumentPayload(
+        uri=doc_id,
+        source_path=pdf_path,
+        title=doc_id,
+        metadata={"doc_type": doc.get("doc_type", "")},
+    )
+
+
+def load_mmlb_qa() -> Dataset:
+    return Dataset.from_list(load_qa_records())
+
+
+def build_mmlb_case(
+    index: int, doc: Mapping[str, Any]
+) -> Case[str, str, dict[str, str]]:
+    evidence_pages = list(doc.get("evidence_pages") or [])
+    evidence_sources = list(doc.get("evidence_sources") or [])
+    metadata: dict[str, str] = {
+        "case_index": str(index),
+        "doc_id": doc["doc_id"],
+        "target_doc_uri": doc["doc_id"],
+        "doc_type": doc.get("doc_type", ""),
+        "answer_format": doc.get("answer_format", ""),
+        "evidence_pages": str(evidence_pages),
+        "evidence_sources": ",".join(str(s) for s in evidence_sources),
+    }
+    return Case(
+        name=f"{index}_{doc['doc_id']}",
+        inputs=doc["question"],
+        expected_output=doc["answer"],
+        metadata=metadata,
+    )
+
+
+def load_mmlb_retrieval() -> Dataset:
+    records = []
+    for row in load_qa_records():
+        if not row.get("evidence_pages"):
+            continue
+        records.append(row)
+    return Dataset.from_list(records)
+
+
+def map_mmlb_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None:
+    evidence_pages = doc.get("evidence_pages") or []
+    if not evidence_pages:
+        return None
+    sources = doc.get("evidence_sources") or []
+    source_type = ",".join(str(s) for s in sources) if sources else None
+    return RetrievalSample(
+        question=doc["question"],
+        expected_uris=(doc["doc_id"],),
+        source_type=source_type,
+    )
+
+
+MMLONGBENCH_SPEC = DatasetSpec(
+    key="mmlongbench",
+    db_filename="mmlongbench.lancedb",
+    document_loader=load_mmlb_corpus,
+    document_mapper=map_mmlb_document,
+    qa_loader=load_mmlb_qa,
+    qa_case_builder=build_mmlb_case,
+    retrieval_loader=load_mmlb_retrieval,
+    retrieval_mapper=map_mmlb_retrieval,
+    retrieval_evaluator=MAPEvaluator(),
+)

evaluations/tests/test_datasets.py

@@ -1,5 +1,11 @@
 from pathlib import Path
 
+from evaluations.datasets.mmlongbench import (
+    build_mmlb_case,
+    load_qa_records,
+    map_mmlb_document,
+    map_mmlb_retrieval,
+)
 from evaluations.datasets.open_rag_bench import (
     build_orb_case,
     download_pdf,
@@ -158,3 +164,162 @@ class TestOpenRAGBench:
         assert is_multimodal_query("image") is True
         assert is_multimodal_query("image_table") is True
         assert is_multimodal_query("text") is False
+
+
+class TestMMLongBenchDoc:
+    def test_map_document(self, tmp_path: Path) -> None:
+        pdf_dir = tmp_path / "documents"
+        pdf_dir.mkdir()
+        pdf_path = pdf_dir / "report.pdf"
+        pdf_path.write_bytes(b"%PDF-fake")
+
+        from unittest.mock import patch
+
+        with patch(
+            "evaluations.datasets.mmlongbench.get_cache_dir",
+            return_value=tmp_path,
+        ):
+            payload = map_mmlb_document(
+                {"doc_id": "report.pdf", "doc_type": "Financial report"}
+            )
+
+        assert payload is not None
+        assert payload.uri == "report.pdf"
+        assert payload.title == "report.pdf"
+        assert payload.source_path == pdf_path
+        assert payload.metadata == {"doc_type": "Financial report"}
+
+    def test_map_document_missing_pdf(self, tmp_path: Path) -> None:
+        from unittest.mock import patch
+
+        with patch(
+            "evaluations.datasets.mmlongbench.get_cache_dir",
+            return_value=tmp_path,
+        ):
+            payload = map_mmlb_document(
+                {"doc_id": "missing.pdf", "doc_type": "Brochure"}
+            )
+
+        assert payload is None
+
+    def test_map_retrieval(self) -> None:
+        doc = {
+            "question": "What is the revenue?",
+            "doc_id": "NIKE_2021_10K.pdf",
+            "evidence_pages": [3, 5],
+            "evidence_sources": ["Table", "Pure-text"],
+        }
+        sample = map_mmlb_retrieval(doc)
+        assert sample is not None
+        assert sample.question == "What is the revenue?"
+        assert sample.expected_uris == ("NIKE_2021_10K.pdf",)
+        assert sample.source_type == "Table,Pure-text"
+
+    def test_map_retrieval_skips_unanswerable(self) -> None:
+        doc = {
+            "question": "What does the document say about Mars?",
+            "doc_id": "NIKE_2021_10K.pdf",
+            "evidence_pages": [],
+            "evidence_sources": [],
+        }
+        assert map_mmlb_retrieval(doc) is None
+
+    def test_build_case(self) -> None:
+        doc = {
+            "doc_id": "report.pdf",
+            "doc_type": "Financial report",
+            "question": "What is the net income?",
+            "answer": "42",
+            "evidence_pages": [5],
+            "evidence_sources": ["Table"],
+            "answer_format": "Int",
+        }
+        case = build_mmlb_case(7, doc)
+        assert case.name == "7_report.pdf"
+        assert case.inputs == "What is the net income?"
+        assert case.expected_output == "42"
+        assert case.metadata is not None
+        assert case.metadata["doc_id"] == "report.pdf"
+        assert case.metadata["target_doc_uri"] == "report.pdf"
+        assert case.metadata["doc_type"] == "Financial report"
+        assert case.metadata["answer_format"] == "Int"
+        assert case.metadata["evidence_pages"] == "[5]"
+        assert case.metadata["evidence_sources"] == "Table"
+        assert case.metadata["case_index"] == "7"
+
+    def test_load_qa_records_parses_list_fields(self) -> None:
+        from unittest.mock import patch
+
+        raw_rows = [
+            {
+                "doc_id": "a.pdf",
+                "doc_type": "Brochure",
+                "question": "Q1?",
+                "answer": "A1",
+                "evidence_pages": "[3, 5]",
+                "evidence_sources": "['Table', 'Pure-text']",
+                "answer_format": "Str",
+            },
+            {
+                "doc_id": "b.pdf",
+                "doc_type": "Academic paper",
+                "question": "Q2?",
+                "answer": "Not answerable",
+                "evidence_pages": "[]",
+                "evidence_sources": "[]",
+                "answer_format": "None",
+            },
+        ]
+
+        import evaluations.datasets.mmlongbench as m
+
+        m._qa_records = None
+        with patch(
+            "evaluations.datasets.mmlongbench._load_hf_qa_split",
+            return_value=raw_rows,
+        ):
+            records = load_qa_records()
+        m._qa_records = None
+
+        assert records[0]["evidence_pages"] == [3, 5]
+        assert records[0]["evidence_sources"] == ["Table", "Pure-text"]
+        assert records[1]["evidence_pages"] == []
+        assert records[1]["evidence_sources"] == []
+
+    def test_load_qa_records_drops_excluded_docs(self) -> None:
+        from unittest.mock import patch
+
+        import evaluations.datasets.mmlongbench as m
+
+        raw_rows = [
+            {
+                "doc_id": "mi_phone.pdf",
+                "doc_type": "Guidebook",
+                "question": "Q?",
+                "answer": "A",
+                "evidence_pages": "[1]",
+                "evidence_sources": "['Pure-text']",
+                "answer_format": "Str",
+            },
+            {
+                "doc_id": "keep.pdf",
+                "doc_type": "Brochure",
+                "question": "Q2?",
+                "answer": "A2",
+                "evidence_pages": "[2]",
+                "evidence_sources": "['Table']",
+                "answer_format": "Str",
+            },
+        ]
+
+        m._qa_records = None
+        with patch(
+            "evaluations.datasets.mmlongbench._load_hf_qa_split",
+            return_value=raw_rows,
+        ):
+            records = load_qa_records()
+        m._qa_records = None
+
+        doc_ids = {r["doc_id"] for r in records}
+        assert "mi_phone.pdf" not in doc_ids
+        assert "keep.pdf" in doc_ids

haiku_rag_slim/haiku/rag/skills/rag-analysis/SKILL.md

@@ -112,6 +112,14 @@ Search results include `doc_item_refs` (e.g. `["#/texts/48", "#/tables/0"]`) tha
 
 You MUST call `cite` with at least one chunk ID before producing your final answer **when your answer is grounded on retrieved evidence**. Skip `cite` in two cases: (a) you are refusing for lack of information, or (b) your answer is a corpus-level computation (count, aggregation, listing) that doesn't draw on specific chunks. In those cases do **not** fabricate citations.
 
+## When the evidence is missing
+
+If neither search nor code traversal surfaces evidence for the question, say: "I cannot find enough information in the knowledge base to answer this question." Do not guess at a number, date, name, or category. Do not infer from tangentially related content. Do not extrapolate from a partial match. A clear refusal is more useful than a fabricated answer.
+
+A computed result counts as evidence only when the code actually located the items the question asks about. If a count, sum, or list comes out empty because the relevant items were not in the document, that is a refusal case, not a "zero" answer.
+
+In a refusal case do **not** call `cite` — there is nothing to cite.
+
 ## Important
 
 - Variables persist between `execute_code` calls — you can search in one call and process results in the next