From db2b8fb883655f157e058eb9100409ab941b6970 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 13 Aug 2026 22:52:36 +0300 Subject: [PATCH] Add compaction arms and grounding status to MTRAG live runs --- CHANGELOG.md | 2 +- docs/benchmarks.md | 6 +- evaluations/evaluations/benchmark.py | 3 + evaluations/evaluations/capability_runner.py | 30 ++++- evaluations/evaluations/config.py | 1 + evaluations/evaluations/datasets/__init__.py | 2 + evaluations/evaluations/datasets/mtrag.py | 29 ++-- evaluations/tests/test_benchmark.py | 50 +++++++ evaluations/tests/test_capability_runner.py | 133 ++++++++++++++++++- evaluations/tests/test_mtrag.py | 15 ++- 10 files changed, 251 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d370a087..5bcc7be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added - `evaluations run --filter/-f CLAUSE`: SQL `WHERE` clause over document columns, applied to the retrieval benchmark's searches and to every capability search during QA. Recorded as `document_filter` in experiment metadata. -- `mtrag_clapnq` / `mtrag_clapnq_rewrite` / `mtrag_clapnq_live` evaluation datasets: multi-turn QA with gold-prefix and live-session conversation replay, Recall@k/nDCG@k retrieval metrics, eligibility-aware citation scoring, refusal precision/recall, and per-turn tool-traffic attributes. +- `mtrag_clapnq` / `mtrag_clapnq_rewrite` / `mtrag_clapnq_live` / `mtrag_clapnq_live_uncompacted` evaluation datasets: multi-turn QA with gold-prefix and live-session conversation replay, live arms with and without `EvidenceCompactionCapability`, Recall@k/nDCG@k retrieval metrics, eligibility-aware citation scoring, refusal precision/recall, per-turn tool-traffic attributes, and `citation_status` / `turn_citation_status` eval attributes. ### Changed diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 419e6d7f..dd73d320 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -37,7 +37,7 @@ Active datasets: | `orb_multimodal_nemotron` — OpenRAG Bench, multimodal embedder (`nvidia/llama-nemotron-embed-vl-1b-v2`), the embedder behind the published headline results | ~16 GB | | `t2_finqa` — T²-RAGBench (FinQA) financial QA, text embedder (`qwen3-embedding:4b`); scored by exact numeric match, run with `--target analysis-capability` | ~2 GB | | `hotpotqa` — HotpotQA multi-hop QA over Wikipedia paragraphs, text embedder (`qwen3-embedding:4b`) | ~1.5 GB | -| `mtrag_clapnq` — MTRAG multi-turn RAG, ClapNQ (Wikipedia) passages, text embedder (`qwen3-embedding:4b`); also serves the `mtrag_clapnq_rewrite` and `mtrag_clapnq_live` keys | ~2.8 GB | +| `mtrag_clapnq` — MTRAG multi-turn RAG, ClapNQ (Wikipedia) passages, text embedder (`qwen3-embedding:4b`); also serves the `mtrag_clapnq_rewrite`, `mtrag_clapnq_live` and `mtrag_clapnq_live_uncompacted` keys | ~2.8 GB | After downloading, run benchmarks with `--skip-db`. Each database is built with a specific embedder, so pass its reference config from `evaluations/configs/` (a database only opens against a config whose embedder matches): @@ -231,7 +231,7 @@ The reranker's contribution is larger here than on the single-doc datasets: hybr [MTRAG](https://github.com/IBM/mt-rag-benchmark) is IBM's multi-turn RAG benchmark (TACL 2025, SemEval-2026 Task 8): human-authored conversations with per-turn answerability labels and binary relevance judgments. We evaluate the ClapNQ (Wikipedia) domain: 183,408 passages, 29 conversations, 224 turns, 208 retrieval queries. -Three dataset keys share one database. `mtrag_clapnq` retrieves with the raw last user turn and runs QA by replaying each task's reference conversation prefix as message history. `mtrag_clapnq_rewrite` retrieves with the human standalone rewrites. `mtrag_clapnq_live` replays whole conversations through a single capability session, carrying the model's own answers and tool history across turns. +Four dataset keys share one database. `mtrag_clapnq` retrieves with the raw last user turn and runs QA by replaying each task's reference conversation prefix as message history. `mtrag_clapnq_rewrite` retrieves with the human standalone rewrites. `mtrag_clapnq_live` replays whole conversations through a single capability session, carrying the model's own answers, tool history and capability state across turns, with `EvidenceCompactionCapability` registered. `mtrag_clapnq_live_uncompacted` is the same replay without compaction, isolating what compaction contributes. This is the only multi-turn evaluation, so it is the only one where compaction acts at all. ##### Retrieval (Recall@k / nDCG@k) @@ -251,4 +251,4 @@ Directly comparable with [IBM's published results](https://github.com/IBM/mt-rag | Gold-prefix (`mtrag_clapnq`) | `vllm:Gemma-4-26B-A4B-NVFP4` | 223 | 0.68 | 0.35 | | Live (`mtrag_clapnq_live`) | `vllm:Gemma-4-26B-A4B-NVFP4` | 195/224 scored | 0.72 micro / 0.73 macro | 0.35 | -*Measured on haiku.rag v0.67.1 with `qwen3-embedding:4b` (vLLM, dim 2560) and `Qwen3-Reranker-4B`, stock capability instructions, judged by `vllm:Qwen3.6-35B-A3B-NVFP4` at temperature 0. The judge sampling has since been re-pinned repo-wide (0.6 with thinking), so future runs re-baseline. QA numbers are internal (our judge and rubric) and not comparable with IBM's published generation metrics. Live mode additionally reports refusal precision/recall against the per-turn answerability labels and per-turn pass rates; pass rate declines with conversation depth (93% at turn 1 to 38% at turn 9). The dataset is text-only (ClapNQ passages), so it exercises no multimodal paths.* +*Measured on haiku.rag v0.67.1 with `qwen3-embedding:4b` (vLLM, dim 2560) and `Qwen3-Reranker-4B`, stock capability instructions, judged by `vllm:Qwen3.6-35B-A3B-NVFP4` at temperature 0. Two changes since re-baseline these numbers: the judge sampling was re-pinned repo-wide (0.6 with thinking), and 0.74.0 rewrote the citation instructions (refusals now declare an empty citation list instead of being exempt). Retrieval is unaffected by both and remains the control. QA numbers are internal (our judge and rubric) and not comparable with IBM's published generation metrics. Live mode additionally reports refusal precision/recall against the per-turn answerability labels and per-turn pass rates; pass rate declines with conversation depth (93% at turn 1 to 38% at turn 9). The dataset is text-only (ClapNQ passages), so it exercises no multimodal paths.* diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index f254b67b..9b18f962 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -630,6 +630,7 @@ async def run_qa_benchmark( set_eval_attribute("n_failed_tools", result.n_failed_tools) set_eval_attribute("n_executions", result.n_executions) set_eval_attribute("n_requests", result.n_requests) + set_eval_attribute("citation_status", result.citation_status) return result.answer report = await _evaluate(answer_question) @@ -778,6 +779,7 @@ async def run_live_qa_benchmark( config=config, questions=list(questions), capability_model=resolved_capability_model, + compaction=spec.compaction, ) set_eval_attribute("turn_cited_uris", [r.cited_uris for r in results]) set_eval_attribute("turn_n_search_calls", [r.n_search_calls for r in results]) @@ -786,6 +788,7 @@ async def run_live_qa_benchmark( ) set_eval_attribute("turn_n_failed_tools", [r.n_failed_tools for r in results]) set_eval_attribute("turn_n_requests", [r.n_requests for r in results]) + set_eval_attribute("turn_citation_status", [r.citation_status for r in results]) return [r.answer for r in results] report = await evaluation_dataset.evaluate( diff --git a/evaluations/evaluations/capability_runner.py b/evaluations/evaluations/capability_runner.py index 04ff75d1..89e0e334 100644 --- a/evaluations/evaluations/capability_runner.py +++ b/evaluations/evaluations/capability_runner.py @@ -16,8 +16,12 @@ from pydantic_ai.messages import ( ) from pydantic_ai.models import Model +from pydantic_ai.capabilities import AbstractCapability + from evaluations.config import Turn from haiku.rag.capabilities import RAGCapabilityBase +from haiku.rag.capabilities.compaction import create_capability as create_compaction +from haiku.rag.capabilities.ledger import CapabilityEvidenceRecord, citation_status from haiku.rag.config.models import AppConfig from haiku.rag.store.models.chunk import SearchResult from haiku.rag.store.models.citation import Citation @@ -40,6 +44,7 @@ class _RagLikeState(Protocol): document_filter: str | None citation_index: dict[str, Citation] citations: list[str] + evidence: CapabilityEvidenceRecord searches: dict[str, list[SearchResult]] @@ -55,6 +60,7 @@ class CapabilityRunResult: n_rejected_searches: int = 0 n_failed_tools: int = 0 n_requests: int = 0 + citation_status: str | None = None class ToolTraffic(NamedTuple): @@ -129,6 +135,7 @@ def _prepare_agent( capability_model: str | Model, document_filter: str | None, request_limit: int | None, + compaction: bool = False, ) -> tuple[RAGCapabilityBase[Any], _EvalDeps, Agent[_EvalDeps, str]]: capability = capability_factory( db_path=db_path, @@ -142,11 +149,14 @@ def _prepare_agent( if document_filter is not None: typed.document_filter = document_filter + capabilities: list[AbstractCapability] = [capability] + if compaction: + capabilities.append(create_compaction()) deps = _EvalDeps(state={capability.state_namespace: state.model_dump(mode="json")}) agent = Agent( capability_model, deps_type=_EvalDeps, - capabilities=[capability], + capabilities=capabilities, ) return capability, deps, agent @@ -203,13 +213,16 @@ async def run_capability_conversation( capability_model: str | Model, document_filter: str | None = None, request_limit: int | None = None, + compaction: bool = False, ) -> list[CapabilityRunResult]: """Run a conversation's user turns sequentially through one capability. Each turn runs with the previous turn's full ``all_messages()`` as history - (tool calls and returns included), so prior-turn compaction operates on - real evidence. Per-invocation state (citations, searches) is cleared by the - capability on every run, so each returned result reflects only its turn. + (tool calls and returns included) and the same state dict, which is what + lets ``EvidenceCompactionCapability`` (registered when ``compaction`` is + True) replace earlier questions' evidence on the request. Per-invocation + state (citations, searches) is cleared by the capability on every run, so + each returned result reflects only its turn. """ capability, deps, agent = _prepare_agent( capability_factory, @@ -218,6 +231,7 @@ async def run_capability_conversation( capability_model, document_filter, request_limit, + compaction=compaction, ) history: list[ModelMessage] | None = None results: list[CapabilityRunResult] = [] @@ -263,6 +277,13 @@ def _result_from_run( executions = getattr(typed, "executions", None) n_executions = len(executions) if executions is not None else 0 + record = typed.evidence + status = ( + citation_status([record], question=record.question) + if record.question is not None + else None + ) + return CapabilityRunResult( answer=answer, cited_uris=cited_uris, @@ -278,4 +299,5 @@ def _result_from_run( n_rejected_searches=traffic.n_rejected_searches, n_failed_tools=traffic.n_failed_tools, n_requests=traffic.n_requests, + citation_status=status, ) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index d8b0ebe0..23faef87 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -82,6 +82,7 @@ class DatasetSpec: ingest_batch_size: int | None = None evaluate_refusal: bool = False live: bool = False + compaction: bool = False experiment_metadata: dict[str, Any] | None = None def db_path(self, override_path: Path | None = None) -> Path: diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index e3532304..45c29adb 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -3,6 +3,7 @@ from evaluations.config import DatasetSpec from .hotpotqa import HOTPOTQA_SPEC from .mtrag import ( MTRAG_CLAPNQ_LIVE_SPEC, + MTRAG_CLAPNQ_LIVE_UNCOMPACTED_SPEC, MTRAG_CLAPNQ_REWRITE_SPEC, MTRAG_CLAPNQ_SPEC, ) @@ -20,6 +21,7 @@ DATASETS: dict[str, DatasetSpec] = { MTRAG_CLAPNQ_SPEC, MTRAG_CLAPNQ_REWRITE_SPEC, MTRAG_CLAPNQ_LIVE_SPEC, + MTRAG_CLAPNQ_LIVE_UNCOMPACTED_SPEC, ORB_TEXT_SPEC, ORB_MULTIMODAL_SPEC, ORB_MULTIMODAL_NEMOTRON_SPEC, diff --git a/evaluations/evaluations/datasets/mtrag.py b/evaluations/evaluations/datasets/mtrag.py index 205bef4b..420913fd 100644 --- a/evaluations/evaluations/datasets/mtrag.py +++ b/evaluations/evaluations/datasets/mtrag.py @@ -285,14 +285,23 @@ def _mtrag_spec(key: str, variant: str) -> DatasetSpec: MTRAG_CLAPNQ_SPEC = _mtrag_spec("mtrag_clapnq", "lastturn") MTRAG_CLAPNQ_REWRITE_SPEC = _mtrag_spec("mtrag_clapnq_rewrite", "rewrite") -MTRAG_CLAPNQ_LIVE_SPEC = DatasetSpec( - key="mtrag_clapnq_live", - db_filename="mtrag_clapnq.lancedb", - document_loader=load_clapnq_corpus, - document_mapper=map_mtrag_document, - qa_loader=load_clapnq_conversations, - qa_case_builder=build_mtrag_live_case, - ingest_batch_size=512, - live=True, - experiment_metadata={"mtrag_mode": "live_session"}, + +def _mtrag_live_spec(key: str, compaction: bool) -> DatasetSpec: + return DatasetSpec( + key=key, + db_filename="mtrag_clapnq.lancedb", + document_loader=load_clapnq_corpus, + document_mapper=map_mtrag_document, + qa_loader=load_clapnq_conversations, + qa_case_builder=build_mtrag_live_case, + ingest_batch_size=512, + live=True, + compaction=compaction, + experiment_metadata={"mtrag_mode": "live_session", "compaction": compaction}, + ) + + +MTRAG_CLAPNQ_LIVE_SPEC = _mtrag_live_spec("mtrag_clapnq_live", compaction=True) +MTRAG_CLAPNQ_LIVE_UNCOMPACTED_SPEC = _mtrag_live_spec( + "mtrag_clapnq_live_uncompacted", compaction=False ) diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 1edad597..a9aa6454 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -188,6 +188,52 @@ class TestConversationInputDispatch: assert history[0].parts[0].content == "q1" assert history[1].parts[0].content == "a1" + @pytest.mark.asyncio + async def test_records_citation_status_attribute(self, tmp_path: Path) -> None: + from dataclasses import dataclass + + from pydantic_evals import Case + from pydantic_evals.evaluators import Evaluator, EvaluatorContext + + from evaluations.capability_runner import CapabilityRunResult + + @dataclass + class AlwaysOne(Evaluator): + def evaluate(self, ctx: EvaluatorContext) -> float: + return 1.0 + + def build_case(idx: int, doc) -> Case: + return Case(name="c1", inputs="q1", expected_output="ref") + + spec = DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [{"id": "t1"}], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=build_case, + qa_evaluator=AlwaysOne(), + ) + + recorded: dict[str, object] = {} + with ( + patch("evaluations.benchmark.get_model", return_value="fake-model"), + patch( + "evaluations.benchmark.set_eval_attribute", + side_effect=lambda key, value: recorded.__setitem__(key, value), + ), + patch( + "evaluations.benchmark.run_capability_question", + new_callable=AsyncMock, + return_value=CapabilityRunResult( + answer="answer", citation_status="ungrounded" + ), + ), + ): + await run_qa_benchmark(spec, AppConfig(), db_path=tmp_path / "test.lancedb") + + assert recorded["citation_status"] == "ungrounded" + class TestRefusalMetrics: def _case(self, label: str | None, refused: bool | None) -> MagicMock: @@ -371,6 +417,7 @@ class TestLiveConversationDispatch: qa_loader=lambda: [{"id": "conv1"}], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] qa_case_builder=build_case, live=True, + compaction=True, ) turn_results = [ @@ -401,6 +448,7 @@ class TestLiveConversationDispatch: assert run_conversation.await_args is not None assert run_conversation.await_args.kwargs["questions"] == ["q1", "q2"] + assert run_conversation.await_args.kwargs["compaction"] is True @pytest.mark.asyncio async def test_live_records_per_turn_traffic_arrays(self, tmp_path: Path) -> None: @@ -442,6 +490,7 @@ class TestLiveConversationDispatch: n_rejected_searches=1, n_failed_tools=1, n_requests=4, + citation_status="grounded", ), CapabilityRunResult(answer="a2"), ] @@ -477,6 +526,7 @@ class TestLiveConversationDispatch: assert recorded["turn_n_rejected_searches"] == [1, 0] assert recorded["turn_n_failed_tools"] == [1, 0] assert recorded["turn_n_requests"] == [4, 0] + assert recorded["turn_citation_status"] == ["grounded", None] questions = 2 for key, value in recorded.items(): if key.startswith("turn_"): diff --git a/evaluations/tests/test_capability_runner.py b/evaluations/tests/test_capability_runner.py index 058ac734..f6e0fd67 100644 --- a/evaluations/tests/test_capability_runner.py +++ b/evaluations/tests/test_capability_runner.py @@ -13,7 +13,11 @@ from pydantic_ai.messages import ( ) from pydantic_ai.models.test import TestModel -from evaluations.capability_runner import _count_tool_traffic, run_capability_question +from evaluations.capability_runner import ( + CapabilityRunResult, + _count_tool_traffic, + run_capability_question, +) from haiku.rag.capabilities.analysis import create_capability as create_analysis from haiku.rag.capabilities.rag import create_capability as create_rag from haiku.rag.config.models import AppConfig @@ -118,6 +122,7 @@ async def test_runs_rag_capability_without_legacy_capability_layer(tmp_path): assert result.answer == "success (no tool calls)" assert result.cited_uris == [] assert result.n_searches == 0 + assert result.citation_status == "missing" async def test_runs_analysis_capability_without_legacy_capability_layer(tmp_path): @@ -161,6 +166,62 @@ async def test_analysis_capability_applies_request_limit(tmp_path, override, exp assert "usage_limits" not in run.call_args.kwargs +class TestCitationStatusDerivation: + """`citation_status` distinguishes an answer that declared nothing + (`missing`) from one that declared ungrounded (`ungrounded`) — refusals + now cite an empty list.""" + + def _result(self, record) -> CapabilityRunResult: + from evaluations.capability_runner import ToolTraffic, _result_from_run + from haiku.rag.capabilities.rag import RAGState + + state = RAGState(evidence=record) + return _result_from_run("answer", state, ToolTraffic(0, 0, 0, 1)) + + def test_grounded(self) -> None: + from haiku.rag.capabilities.ledger import ( + CapabilityEvidenceRecord, + CitationDeclaration, + EvidenceRef, + ) + + record = CapabilityEvidenceRecord( + question=2, + latest_evidence_epoch=3, + declaration=CitationDeclaration( + question=2, + epoch=5, + refs=[EvidenceRef(capability="rag", chunk_id="c1")], + ), + ) + assert self._result(record).citation_status == "grounded" + + def test_ungrounded(self) -> None: + from haiku.rag.capabilities.ledger import ( + CapabilityEvidenceRecord, + CitationDeclaration, + ) + + record = CapabilityEvidenceRecord( + question=2, + latest_evidence_epoch=3, + declaration=CitationDeclaration(question=2, epoch=5, refs=[]), + ) + assert self._result(record).citation_status == "ungrounded" + + def test_missing(self) -> None: + from haiku.rag.capabilities.ledger import CapabilityEvidenceRecord + + record = CapabilityEvidenceRecord(question=2, latest_evidence_epoch=3) + assert self._result(record).citation_status == "missing" + + def test_none_without_a_question(self) -> None: + """A record no run ever stamped (mocked runs) derives no status.""" + from haiku.rag.capabilities.ledger import CapabilityEvidenceRecord + + assert self._result(CapabilityEvidenceRecord()).citation_status is None + + class TestPrefixToMessages: def test_maps_turns_to_model_messages(self) -> None: from pydantic_ai.messages import ( @@ -258,6 +319,76 @@ async def test_conversation_threads_own_messages_across_turns(tmp_path): assert histories == [None, ["history after q1"], ["history after q2"]] +async def test_conversation_carries_one_state_dict_across_turns(tmp_path): + """Capabilities read and write state through the deps dict; carrying the + same dict across turns is what lets compaction see earlier questions' + records instead of refusing.""" + from evaluations.capability_runner import run_capability_conversation + + deps_seen: list[object] = [] + + async def _run(question, deps=None, message_history=None): + deps_seen.append(deps) + return SimpleNamespace( + output="a", all_messages=lambda: [], new_messages=lambda: [] + ) + + with patch("evaluations.capability_runner.Agent.run", side_effect=_run): + await run_capability_conversation( + create_rag, + tmp_path / "rag.lancedb", + AppConfig(), + ["q1", "q2", "q3"], + TestModel(call_tools=[]), + ) + + assert deps_seen[0] is deps_seen[1] is deps_seen[2] + + +@pytest.mark.parametrize(("compaction", "expected"), [(False, 0), (True, 1)]) +async def test_conversation_compaction_registration(tmp_path, compaction, expected): + from haiku.rag.capabilities.compaction import EvidenceCompactionCapability + + from evaluations.capability_runner import run_capability_conversation + + with patch("evaluations.capability_runner.Agent") as agent_cls: + agent_cls.return_value.run = AsyncMock( + return_value=SimpleNamespace( + output="a", all_messages=lambda: [], new_messages=lambda: [] + ) + ) + await run_capability_conversation( + create_rag, + tmp_path / "rag.lancedb", + AppConfig(), + ["q1"], + TestModel(call_tools=[]), + compaction=compaction, + ) + capabilities = agent_cls.call_args.kwargs["capabilities"] + + compactors = [ + c for c in capabilities if isinstance(c, EvidenceCompactionCapability) + ] + assert len(compactors) == expected + assert len(capabilities) == 1 + expected + + +async def test_conversation_end_to_end_with_compaction(tmp_path): + from evaluations.capability_runner import run_capability_conversation + + result = await run_capability_conversation( + create_rag, + tmp_path / "rag.lancedb", + AppConfig(), + ["first question", "follow-up"], + TestModel(call_tools=[]), + compaction=True, + ) + + assert [turn.answer for turn in result] == ["success (no tool calls)"] * 2 + + async def test_conversation_end_to_end_with_test_model(tmp_path): from evaluations.capability_runner import run_capability_conversation diff --git a/evaluations/tests/test_mtrag.py b/evaluations/tests/test_mtrag.py index 94a9cde4..0304a2fa 100644 --- a/evaluations/tests/test_mtrag.py +++ b/evaluations/tests/test_mtrag.py @@ -263,6 +263,19 @@ class TestLiveConversations: assert MTRAG_CLAPNQ_LIVE_SPEC.live is True assert MTRAG_CLAPNQ_LIVE_SPEC.retrieval_loader is None assert MTRAG_CLAPNQ_LIVE_SPEC.experiment_metadata == { - "mtrag_mode": "live_session" + "mtrag_mode": "live_session", + "compaction": True, } assert MTRAG_CLAPNQ_SPEC.experiment_metadata == {"mtrag_mode": "gold_prefix"} + + def test_live_compaction_arms(self) -> None: + assert MTRAG_CLAPNQ_LIVE_SPEC.compaction is True + uncompacted = DATASETS["mtrag_clapnq_live_uncompacted"] + assert uncompacted.compaction is False + assert uncompacted.live is True + assert uncompacted.db_filename == MTRAG_CLAPNQ_LIVE_SPEC.db_filename + assert uncompacted.qa_case_builder is MTRAG_CLAPNQ_LIVE_SPEC.qa_case_builder + assert uncompacted.experiment_metadata == { + "mtrag_mode": "live_session", + "compaction": False, + }