From a60d0f45d30c38933e76b039c42a6e11ee2bf592 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 1 Sep 2026 08:17:35 +0300 Subject: [PATCH] Strip the speaker tag from MTRAG retrieval queries The query files encode the speaker into the text, so every retrieval query arrived as "|user|: How many teams are in the NFL?". That reaches the embedder, the BM25 query and the reranker's query. Measured paired over 777 queries on four domains: stripping is worth +3.60pp recall@5 with a reranker (94 queries better, 33 worse, 650 tied) and nothing without one (40 better, 40 worse). A cross-encoder scores query against document directly, so junk tokens on the query side hurt it where a bag-of-words branch and a pooled embedding absorb them. Confined to the retrieval query files: 208 of 208 in both lastturn and rewrite carry it, while QA turn texts, answers and live questions carry none. Changes retrieval scores for mtrag_clapnq, mtrag_clapnq_rewrite, mtrag_federated and mtrag_pooled. The single-database direction is small and signed: hybrid -0.36pp, vector -1.83pp, FTS +1.25pp, the branches moving oppositely and nearly cancelling. Claude-Session: https://claude.ai/code/session_01WhudUtZm6qqiuv8Y1sbwSc --- CHANGELOG.md | 1 + evaluations/evaluations/datasets/mtrag.py | 17 +++++++- .../evaluations/datasets/mtrag_federated.py | 3 +- evaluations/tests/test_mtrag.py | 39 +++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 19aa9590..0e930418 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ ### Fixed +- MTRAG retrieval queries have their `|speaker|: ` tag stripped. It reached the embedder, the BM25 query and the reranker's query, costing about 3.6pp recall on the reranker and nothing on the fusion path, measured paired over 777 queries. Affects `mtrag_clapnq`, `mtrag_clapnq_rewrite`, `mtrag_federated` and `mtrag_pooled` retrieval scores; the QA path never carried it. - Batched evaluation ingest converts inline content as text instead of letting `HaikuRAG.convert` disambiguate it, so a passage beginning with a URL is stored rather than fetched over HTTP. 187 MTRAG cloud and fiqa passages start with one; no clapnq passage does, so no existing dataset's numbers change. - `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results. - FTS and hybrid search on a database whose FTS index covers no rows. Chunk diff --git a/evaluations/evaluations/datasets/mtrag.py b/evaluations/evaluations/datasets/mtrag.py index 61cc29b0..d528acf5 100644 --- a/evaluations/evaluations/datasets/mtrag.py +++ b/evaluations/evaluations/datasets/mtrag.py @@ -1,4 +1,5 @@ import json +import re import zipfile from collections.abc import Iterable, Mapping from functools import partial @@ -85,6 +86,20 @@ def _validate_qrels_resolve( ) +_SPEAKER_TAG = re.compile(r"^\|[^|]+\|:\s*") + + +def strip_speaker_markup(text: str) -> str: + """Drop the leading `|speaker|: ` tag MTRAG's query files encode. + + Anchored, so only a leading tag is markup: a pipe later in the question is + content, and a second tag survives. The tag reaches the embedder, the BM25 + query and the reranker's query, and on the reranker it costs about 3.6pp + recall, measured paired over 777 queries. + """ + return _SPEAKER_TAG.sub("", text) + + def _join_queries_qrels( queries: Iterable[Mapping[str, Any]], qrels: Mapping[str, list[str]] ) -> list[dict[str, Any]]: @@ -97,7 +112,7 @@ def _join_queries_qrels( records.append( { "query_id": query_id, - "question": query["text"], + "question": strip_speaker_markup(query["text"]), "expected_uris": expected, } ) diff --git a/evaluations/evaluations/datasets/mtrag_federated.py b/evaluations/evaluations/datasets/mtrag_federated.py index 81a9db5b..bd362724 100644 --- a/evaluations/evaluations/datasets/mtrag_federated.py +++ b/evaluations/evaluations/datasets/mtrag_federated.py @@ -19,6 +19,7 @@ from evaluations.datasets.mtrag import ( load_clapnq_retrieval, map_mtrag_document, map_mtrag_retrieval, + strip_speaker_markup, ) from evaluations.evaluators import ( CitationMAPEvaluator, @@ -284,7 +285,7 @@ def load_pooled_queries(variant: str = "lastturn") -> list[dict[str, Any]]: out.append( { "query_id": f"{domain}/{query['_id']}", - "question": query["text"], + "question": strip_speaker_markup(query["text"]), "expected_uris": expected, "domain": domain, } diff --git a/evaluations/tests/test_mtrag.py b/evaluations/tests/test_mtrag.py index 90a7cc7b..70f6a21a 100644 --- a/evaluations/tests/test_mtrag.py +++ b/evaluations/tests/test_mtrag.py @@ -13,8 +13,10 @@ from evaluations.datasets.mtrag import ( _validate_qrels_resolve, build_mtrag_case, build_mtrag_live_case, + load_clapnq_retrieval, map_mtrag_document, map_mtrag_retrieval, + strip_speaker_markup, ) from evaluations.evaluators import ( CitationMAPEvaluator, @@ -276,3 +278,40 @@ class TestLiveConversations: "mtrag_mode": "live_session", "compaction": False, } + + +class TestSpeakerMarkup: + """MTRAG's retrieval query files encode the speaker into the query text. + It reaches the embedder, the BM25 query and the reranker's query; on the + reranker it costs about 3.6pp recall, measured paired over 777 queries. + """ + + def test_strips_a_leading_speaker_tag(self) -> None: + assert ( + strip_speaker_markup("|user|: How many teams are in the NFL?") + == "How many teams are in the NFL?" + ) + + def test_strips_any_speaker_not_just_user(self) -> None: + assert strip_speaker_markup("|agent|: Twelve of them.") == "Twelve of them." + + def test_leaves_an_unmarked_question_alone(self) -> None: + assert ( + strip_speaker_markup("How many teams are in the NFL?") + == "How many teams are in the NFL?" + ) + + def test_leaves_a_pipe_mid_sentence_alone(self) -> None: + """Only a leading tag is markup; a pipe in the question is content.""" + text = "What does the |> operator do?" + assert strip_speaker_markup(text) == text + + def test_does_not_strip_a_second_tag(self) -> None: + """One tag is the encoding; a second is content and must survive.""" + assert strip_speaker_markup("|user|: |agent|: nested") == "|agent|: nested" + + def test_retrieval_samples_arrive_clean(self) -> None: + rows = list(load_clapnq_retrieval("lastturn")) + assert rows, "no retrieval rows" + marked = [r for r in rows if r["question"].startswith("|")] + assert not marked, f"{len(marked)} of {len(rows)} still carry markup"