fix: resolve incorrect dataset and column namings

docs/benchmarks.md

@@ -89,11 +89,17 @@ evaluations:
 
 ### Restricting the corpus
 
-When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`, `db_source`):
+When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`):
 
 ```bash
-evaluations run mqf --skip-db --config haiku.rag.s3.yaml \
-  --filter "db_source in ('dataset')"
+evaluations run orb_text --skip-db --config haiku.rag.s3.yaml \
+  --filter "uri LIKE '%arxiv%'"
+```
+
+If the corpora are distinguished by a tag rather than by URI, attach it at ingest time as document metadata and match it with `LIKE`. `metadata` is stored as a `json.dumps` string, so there is no JSON subfield access — match the serialized key/value, including the space after the colon:
+
+```bash
+evaluations run orb_text --skip-db --filter "metadata LIKE '%\"corpus\": \"orb_text\"%'"
 ```
 
 The clause applies to both benchmark phases — the retrieval benchmark's searches and every search the capability runs during QA — so the two score the same subset. It is recorded as `search_filter` in the run's experiment metadata, so a filtered run is never mistaken for an unfiltered one when comparing results.
@@ -101,10 +107,10 @@ The clause applies to both benchmark phases — the retrieval benchmark's search
 A dataset can declare its own default in its `DatasetSpec`, so runs need no flag:
 
 ```python
-MQF_SPEC = DatasetSpec(
-    key="mqf",
+ORB_TEXT_SPEC = DatasetSpec(
+    key="orb_text",
     ...
-    search_filter="db_source in ('dataset')",
+    search_filter="uri LIKE '%arxiv%'",
 )
 ```
 

evaluations/evaluations/benchmark.py

@@ -57,6 +57,7 @@ configure_telemetry(service_name="evals", scrubbing=False)
 configure_cli_logging()
 console = Console()
 
+
 def resolve_search_filter(spec: DatasetSpec, override: str | None) -> str | None:
     """Pick the document filter for a run: `--filter` wins over the dataset's.
 
@@ -683,9 +684,10 @@ def run(
         "--filter",
         "-f",
         help=(
-            "SQL WHERE clause over document columns (id, uri, title, metadata, "
-            "db_source, ...) restricting every benchmark search, e.g. "
-            "\"db_source in ('dataset')\". Overrides the dataset's own "
+            "SQL WHERE clause over document columns (id, uri, title, "
+            "created_at, updated_at, metadata) restricting every benchmark "
+            "search, e.g. \"uri LIKE '%arxiv%'\". metadata is stored as a "
+            "string, so match it with LIKE. Overrides the dataset's own "
             "filter; pass an empty string to search the whole database."
         ),
     ),

evaluations/tests/test_benchmark.py

@@ -585,20 +585,20 @@ class TestResolveSearchFilter:
     def test_dataset_filter_used_when_no_override(self) -> None:
         from evaluations.benchmark import resolve_search_filter
 
-        spec = _stub_spec(search_filter="db_source in ('dataset')")
-        assert resolve_search_filter(spec, None) == "db_source in ('dataset')"
+        spec = _stub_spec(search_filter="uri LIKE '%arxiv%'")
+        assert resolve_search_filter(spec, None) == "uri LIKE '%arxiv%'"
 
     def test_override_wins(self) -> None:
         from evaluations.benchmark import resolve_search_filter
 
-        spec = _stub_spec(search_filter="db_source in ('dataset')")
+        spec = _stub_spec(search_filter="uri LIKE '%arxiv%'")
         assert resolve_search_filter(spec, "uri LIKE '%.pdf'") == "uri LIKE '%.pdf'"
 
     def test_empty_override_clears_dataset_filter(self) -> None:
         """`--filter ""` runs a filtered dataset against the whole database."""
         from evaluations.benchmark import resolve_search_filter
 
-        spec = _stub_spec(search_filter="db_source in ('dataset')")
+        spec = _stub_spec(search_filter="uri LIKE '%arxiv%'")
         assert resolve_search_filter(spec, "") is None
 
     def test_none_when_neither_is_set(self) -> None:
@@ -616,9 +616,9 @@ class TestSearchFilterThreading:
             dataset_key="test",
             test_cases=1,
             config=AppConfig(),
-            search_filter="db_source in ('dataset')",
+            search_filter="uri LIKE '%arxiv%'",
         )
-        assert result["search_filter"] == "db_source in ('dataset')"
+        assert result["search_filter"] == "uri LIKE '%arxiv%'"
 
     def test_metadata_filter_is_none_when_unset(self) -> None:
         result = build_experiment_metadata(
@@ -655,10 +655,10 @@ class TestSearchFilterThreading:
                 spec,
                 AppConfig(),
                 db_path=tmp_path / "test.lancedb",
-                search_filter="db_source in ('dataset')",
+                search_filter="uri LIKE '%arxiv%'",
             )
 
-        assert searches[0]["filter"] == "db_source in ('dataset')"
+        assert searches[0]["filter"] == "uri LIKE '%arxiv%'"
 
     @pytest.mark.asyncio
     async def test_qa_capability_run_receives_filter(self, tmp_path: Path) -> None:
@@ -691,16 +691,16 @@ class TestSearchFilterThreading:
                 spec,
                 AppConfig(),
                 db_path=tmp_path / "test.lancedb",
-                search_filter="db_source in ('dataset')",
+                search_filter="uri LIKE '%arxiv%'",
             )
 
         mock_run.assert_awaited_once()
-        assert mock_run.await_args[1]["document_filter"] == "db_source in ('dataset')"
+        assert mock_run.await_args[1]["document_filter"] == "uri LIKE '%arxiv%'"
 
     @pytest.mark.asyncio
     async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None:
         """The dataset's own filter reaches retrieval and QA without a flag."""
-        spec = _stub_spec(search_filter="db_source in ('dataset','other')")
+        spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""")
 
         with (
             patch(
@@ -721,13 +721,13 @@ class TestSearchFilterThreading:
                 db_path=None,
             )
 
-        expected = "db_source in ('dataset','other')"
+        expected = """metadata LIKE '%"corpus": "orb_text"%'"""
         assert mock_retrieval.call_args[1]["search_filter"] == expected
         assert mock_qa.call_args[1]["search_filter"] == expected
 
     @pytest.mark.asyncio
     async def test_evaluate_dataset_override_reaches_both_phases(self) -> None:
-        spec = _stub_spec(search_filter="db_source in ('dataset','other')")
+        spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""")
 
         with (
             patch(
@@ -746,11 +746,11 @@ class TestSearchFilterThreading:
                 limit=None,
                 name=None,
                 db_path=None,
-                search_filter="db_source in ('other')",
+                search_filter="title LIKE '%paper%'",
             )
 
-        assert mock_retrieval.call_args[1]["search_filter"] == "db_source in ('other')"
-        assert mock_qa.call_args[1]["search_filter"] == "db_source in ('other')"
+        assert mock_retrieval.call_args[1]["search_filter"] == "title LIKE '%paper%'"
+        assert mock_qa.call_args[1]["search_filter"] == "title LIKE '%paper%'"
 
 
 class TestEvaluateDatasetCaseIds: