Thread the document filter through the live QA runner

evaluations/evaluations/benchmark.py

@@ -815,6 +815,7 @@ async def run_live_qa_benchmark(
             config=config,
             questions=list(questions),
             capability_model=run.capability_model,
+            document_filter=document_filter,
             compaction=spec.compaction,
         )
         set_eval_attribute("turn_cited_uris", [r.cited_uris for r in results])

evaluations/evaluations/capability_runner.py

@@ -211,6 +211,7 @@ async def run_capability_conversation(
     config: AppConfig,
     questions: list[str],
     capability_model: str | Model,
+    document_filter: str | None = None,
     compaction: bool = False,
 ) -> list[CapabilityRunResult]:
     """Run a conversation's user turns sequentially through one capability.
@@ -227,7 +228,7 @@ async def run_capability_conversation(
         db_path,
         config,
         capability_model,
-        document_filter=None,
+        document_filter=document_filter,
         request_limit=None,
         compaction=compaction,
     )

evaluations/tests/test_benchmark.py

@@ -443,12 +443,19 @@ class TestLiveConversationDispatch:
             ),
         ):
             await run_live_qa_benchmark(
-                spec, AppConfig(), db_path=tmp_path / "test.lancedb"
+                spec,
+                AppConfig(),
+                db_path=tmp_path / "test.lancedb",
+                document_filter="uri = 'manual.pdf'",
             )
 
         assert run_conversation.await_args is not None
         assert run_conversation.await_args.kwargs["questions"] == ["q1", "q2"]
         assert run_conversation.await_args.kwargs["compaction"] is True
+        assert (
+            run_conversation.await_args.kwargs["document_filter"]
+            == "uri = 'manual.pdf'"
+        )
 
     @pytest.mark.asyncio
     async def test_live_records_per_turn_traffic_arrays(self, tmp_path: Path) -> None:
@@ -1143,7 +1150,7 @@ class TestDocumentFilterThreading:
             retrieval_mapper=lambda d: RetrievalSample(
                 question=d["q"], expected_uris=d["uris"]
             ),
-            retrieval_evaluator=MAPEvaluator(),
+            retrieval_evaluators=[MAPEvaluator()],
         )
 
         with patch("evaluations.benchmark.HaikuRAG") as mock_haiku:

evaluations/tests/test_capability_runner.py

@@ -319,6 +319,32 @@ async def test_conversation_threads_own_messages_across_turns(tmp_path):
     assert histories == [None, ["history after q1"], ["history after q2"]]
 
 
+async def test_conversation_applies_document_filter(tmp_path):
+    """The filter must reach the capability state so every search in the
+    conversation is restricted, same as the single-question runner."""
+    from evaluations.capability_runner import run_capability_conversation
+
+    deps_seen = []
+
+    async def _run(question, deps=None, message_history=None):
+        deps_seen.append(deps)
+        return SimpleNamespace(
+            output="a", all_messages=lambda: [], new_messages=lambda: []
+        )
+
+    with patch("evaluations.capability_runner.Agent.run", side_effect=_run):
+        await run_capability_conversation(
+            create_rag,
+            tmp_path / "rag.lancedb",
+            AppConfig(),
+            ["q1"],
+            TestModel(call_tools=[]),
+            document_filter="uri = 'manual.pdf'",
+        )
+
+    assert deps_seen[0].state["rag"]["document_filter"] == "uri = 'manual.pdf'"
+
+
 async def test_conversation_carries_one_state_dict_across_turns(tmp_path):
     """Capabilities read and write state through the deps dict; carrying the
     same dict across turns is what lets compaction see earlier questions'