From a68cb23b6ec6889a6b0492738158add93b7a1c26 Mon Sep 17 00:00:00 2001 From: cwiesen Date: Fri, 14 Aug 2026 16:27:04 -0500 Subject: [PATCH] fix: resolve incorrect dataset and column namings --- docs/benchmarks.md | 18 ++++++++++------ evaluations/evaluations/benchmark.py | 8 ++++--- evaluations/tests/test_benchmark.py | 32 ++++++++++++++-------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 6b9dff32..6a515cc1 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -89,11 +89,17 @@ evaluations: ### Restricting the corpus -When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`, `db_source`): +When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`): ```bash -evaluations run mqf --skip-db --config haiku.rag.s3.yaml \ - --filter "db_source in ('dataset')" +evaluations run orb_text --skip-db --config haiku.rag.s3.yaml \ + --filter "uri LIKE '%arxiv%'" +``` + +If the corpora are distinguished by a tag rather than by URI, attach it at ingest time as document metadata and match it with `LIKE`. `metadata` is stored as a `json.dumps` string, so there is no JSON subfield access — match the serialized key/value, including the space after the colon: + +```bash +evaluations run orb_text --skip-db --filter "metadata LIKE '%\"corpus\": \"orb_text\"%'" ``` The clause applies to both benchmark phases — the retrieval benchmark's searches and every search the capability runs during QA — so the two score the same subset. It is recorded as `search_filter` in the run's experiment metadata, so a filtered run is never mistaken for an unfiltered one when comparing results. @@ -101,10 +107,10 @@ The clause applies to both benchmark phases — the retrieval benchmark's search A dataset can declare its own default in its `DatasetSpec`, so runs need no flag: ```python -MQF_SPEC = DatasetSpec( - key="mqf", +ORB_TEXT_SPEC = DatasetSpec( + key="orb_text", ... - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) ``` diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index fd421688..267902d9 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -57,6 +57,7 @@ configure_telemetry(service_name="evals", scrubbing=False) configure_cli_logging() console = Console() + def resolve_search_filter(spec: DatasetSpec, override: str | None) -> str | None: """Pick the document filter for a run: `--filter` wins over the dataset's. @@ -683,9 +684,10 @@ def run( "--filter", "-f", help=( - "SQL WHERE clause over document columns (id, uri, title, metadata, " - "db_source, ...) restricting every benchmark search, e.g. " - "\"db_source in ('dataset')\". Overrides the dataset's own " + "SQL WHERE clause over document columns (id, uri, title, " + "created_at, updated_at, metadata) restricting every benchmark " + "search, e.g. \"uri LIKE '%arxiv%'\". metadata is stored as a " + "string, so match it with LIKE. Overrides the dataset's own " "filter; pass an empty string to search the whole database." ), ), diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index dca0b1ce..4cb75d33 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -585,20 +585,20 @@ class TestResolveSearchFilter: def test_dataset_filter_used_when_no_override(self) -> None: from evaluations.benchmark import resolve_search_filter - spec = _stub_spec(search_filter="db_source in ('dataset')") - assert resolve_search_filter(spec, None) == "db_source in ('dataset')" + spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") + assert resolve_search_filter(spec, None) == "uri LIKE '%arxiv%'" def test_override_wins(self) -> None: from evaluations.benchmark import resolve_search_filter - spec = _stub_spec(search_filter="db_source in ('dataset')") + spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") assert resolve_search_filter(spec, "uri LIKE '%.pdf'") == "uri LIKE '%.pdf'" def test_empty_override_clears_dataset_filter(self) -> None: """`--filter ""` runs a filtered dataset against the whole database.""" from evaluations.benchmark import resolve_search_filter - spec = _stub_spec(search_filter="db_source in ('dataset')") + spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") assert resolve_search_filter(spec, "") is None def test_none_when_neither_is_set(self) -> None: @@ -616,9 +616,9 @@ class TestSearchFilterThreading: dataset_key="test", test_cases=1, config=AppConfig(), - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) - assert result["search_filter"] == "db_source in ('dataset')" + assert result["search_filter"] == "uri LIKE '%arxiv%'" def test_metadata_filter_is_none_when_unset(self) -> None: result = build_experiment_metadata( @@ -655,10 +655,10 @@ class TestSearchFilterThreading: spec, AppConfig(), db_path=tmp_path / "test.lancedb", - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) - assert searches[0]["filter"] == "db_source in ('dataset')" + assert searches[0]["filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio async def test_qa_capability_run_receives_filter(self, tmp_path: Path) -> None: @@ -691,16 +691,16 @@ class TestSearchFilterThreading: spec, AppConfig(), db_path=tmp_path / "test.lancedb", - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) mock_run.assert_awaited_once() - assert mock_run.await_args[1]["document_filter"] == "db_source in ('dataset')" + assert mock_run.await_args[1]["document_filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None: """The dataset's own filter reaches retrieval and QA without a flag.""" - spec = _stub_spec(search_filter="db_source in ('dataset','other')") + spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""") with ( patch( @@ -721,13 +721,13 @@ class TestSearchFilterThreading: db_path=None, ) - expected = "db_source in ('dataset','other')" + expected = """metadata LIKE '%"corpus": "orb_text"%'""" assert mock_retrieval.call_args[1]["search_filter"] == expected assert mock_qa.call_args[1]["search_filter"] == expected @pytest.mark.asyncio async def test_evaluate_dataset_override_reaches_both_phases(self) -> None: - spec = _stub_spec(search_filter="db_source in ('dataset','other')") + spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""") with ( patch( @@ -746,11 +746,11 @@ class TestSearchFilterThreading: limit=None, name=None, db_path=None, - search_filter="db_source in ('other')", + search_filter="title LIKE '%paper%'", ) - assert mock_retrieval.call_args[1]["search_filter"] == "db_source in ('other')" - assert mock_qa.call_args[1]["search_filter"] == "db_source in ('other')" + assert mock_retrieval.call_args[1]["search_filter"] == "title LIKE '%paper%'" + assert mock_qa.call_args[1]["search_filter"] == "title LIKE '%paper%'" class TestEvaluateDatasetCaseIds: