diff --git a/CHANGELOG.md b/CHANGELOG.md index 19aa9590..0e930418 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ ### Fixed +- MTRAG retrieval queries have their `|speaker|: ` tag stripped. It reached the embedder, the BM25 query and the reranker's query, costing about 3.6pp recall on the reranker and nothing on the fusion path, measured paired over 777 queries. Affects `mtrag_clapnq`, `mtrag_clapnq_rewrite`, `mtrag_federated` and `mtrag_pooled` retrieval scores; the QA path never carried it. - Batched evaluation ingest converts inline content as text instead of letting `HaikuRAG.convert` disambiguate it, so a passage beginning with a URL is stored rather than fetched over HTTP. 187 MTRAG cloud and fiqa passages start with one; no clapnq passage does, so no existing dataset's numbers change. - `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results. - FTS and hybrid search on a database whose FTS index covers no rows. Chunk diff --git a/evaluations/evaluations/datasets/mtrag.py b/evaluations/evaluations/datasets/mtrag.py index 61cc29b0..d528acf5 100644 --- a/evaluations/evaluations/datasets/mtrag.py +++ b/evaluations/evaluations/datasets/mtrag.py @@ -1,4 +1,5 @@ import json +import re import zipfile from collections.abc import Iterable, Mapping from functools import partial @@ -85,6 +86,20 @@ def _validate_qrels_resolve( ) +_SPEAKER_TAG = re.compile(r"^\|[^|]+\|:\s*") + + +def strip_speaker_markup(text: str) -> str: + """Drop the leading `|speaker|: ` tag MTRAG's query files encode. + + Anchored, so only a leading tag is markup: a pipe later in the question is + content, and a second tag survives. The tag reaches the embedder, the BM25 + query and the reranker's query, and on the reranker it costs about 3.6pp + recall, measured paired over 777 queries. + """ + return _SPEAKER_TAG.sub("", text) + + def _join_queries_qrels( queries: Iterable[Mapping[str, Any]], qrels: Mapping[str, list[str]] ) -> list[dict[str, Any]]: @@ -97,7 +112,7 @@ def _join_queries_qrels( records.append( { "query_id": query_id, - "question": query["text"], + "question": strip_speaker_markup(query["text"]), "expected_uris": expected, } ) diff --git a/evaluations/evaluations/datasets/mtrag_federated.py b/evaluations/evaluations/datasets/mtrag_federated.py index 81a9db5b..bd362724 100644 --- a/evaluations/evaluations/datasets/mtrag_federated.py +++ b/evaluations/evaluations/datasets/mtrag_federated.py @@ -19,6 +19,7 @@ from evaluations.datasets.mtrag import ( load_clapnq_retrieval, map_mtrag_document, map_mtrag_retrieval, + strip_speaker_markup, ) from evaluations.evaluators import ( CitationMAPEvaluator, @@ -284,7 +285,7 @@ def load_pooled_queries(variant: str = "lastturn") -> list[dict[str, Any]]: out.append( { "query_id": f"{domain}/{query['_id']}", - "question": query["text"], + "question": strip_speaker_markup(query["text"]), "expected_uris": expected, "domain": domain, } diff --git a/evaluations/tests/test_mtrag.py b/evaluations/tests/test_mtrag.py index 90a7cc7b..70f6a21a 100644 --- a/evaluations/tests/test_mtrag.py +++ b/evaluations/tests/test_mtrag.py @@ -13,8 +13,10 @@ from evaluations.datasets.mtrag import ( _validate_qrels_resolve, build_mtrag_case, build_mtrag_live_case, + load_clapnq_retrieval, map_mtrag_document, map_mtrag_retrieval, + strip_speaker_markup, ) from evaluations.evaluators import ( CitationMAPEvaluator, @@ -276,3 +278,40 @@ class TestLiveConversations: "mtrag_mode": "live_session", "compaction": False, } + + +class TestSpeakerMarkup: + """MTRAG's retrieval query files encode the speaker into the query text. + It reaches the embedder, the BM25 query and the reranker's query; on the + reranker it costs about 3.6pp recall, measured paired over 777 queries. + """ + + def test_strips_a_leading_speaker_tag(self) -> None: + assert ( + strip_speaker_markup("|user|: How many teams are in the NFL?") + == "How many teams are in the NFL?" + ) + + def test_strips_any_speaker_not_just_user(self) -> None: + assert strip_speaker_markup("|agent|: Twelve of them.") == "Twelve of them." + + def test_leaves_an_unmarked_question_alone(self) -> None: + assert ( + strip_speaker_markup("How many teams are in the NFL?") + == "How many teams are in the NFL?" + ) + + def test_leaves_a_pipe_mid_sentence_alone(self) -> None: + """Only a leading tag is markup; a pipe in the question is content.""" + text = "What does the |> operator do?" + assert strip_speaker_markup(text) == text + + def test_does_not_strip_a_second_tag(self) -> None: + """One tag is the encoding; a second is content and must survive.""" + assert strip_speaker_markup("|user|: |agent|: nested") == "|agent|: nested" + + def test_retrieval_samples_arrive_clean(self) -> None: + rows = list(load_clapnq_retrieval("lastturn")) + assert rows, "no retrieval rows" + marked = [r for r in rows if r["question"].startswith("|")] + assert not marked, f"{len(marked)} of {len(rows)} still carry markup"