From 9b2ae347d2a736df5fe8fd8e709623126bbc3b82 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 17 Aug 2026 10:19:51 +0300 Subject: [PATCH] Use filters that match the datasets they document The `--filter` examples used `uri LIKE '%arxiv%'`, which matches no `orb_text` document: its URIs are bare arXiv ids such as `2407.01528v3`. A clause that matches nothing scores MAP 0 rather than erroring, so the example failed silently. `await_args` is typed `_Call | None`, so subscripting it fails `ty check`; `call_args` carries the same call for an AsyncMock. --- docs/benchmarks.md | 6 +++--- evaluations/tests/test_benchmark.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9e1a94a6..dc7f4f8f 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -89,11 +89,11 @@ evaluations: ### Restricting the corpus -When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`): +When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`). Each dataset writes its own URIs: `orb_text` uses bare arXiv ids such as `2407.01528v3`, `hotpotqa` uses page titles. ```bash evaluations run orb_text --skip-db --config haiku.rag.s3.yaml \ - --filter "uri LIKE '%arxiv%'" + --filter "uri LIKE '2407%'" ``` If the corpora are distinguished by a tag rather than by URI, attach it at ingest time as document metadata and match it with `LIKE`. `metadata` is stored as a `json.dumps` string, so there is no JSON subfield access — match the serialized key/value, including the space after the colon: @@ -110,7 +110,7 @@ A dataset can declare its own default in its `DatasetSpec`, so runs need no flag ORB_TEXT_SPEC = DatasetSpec( key="orb_text", ... - search_filter="uri LIKE '%arxiv%'", + search_filter="metadata LIKE '%\"corpus\": \"orb_text\"%'", ) ``` diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 4cb75d33..a24a0ca0 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -695,7 +695,7 @@ class TestSearchFilterThreading: ) mock_run.assert_awaited_once() - assert mock_run.await_args[1]["document_filter"] == "uri LIKE '%arxiv%'" + assert mock_run.call_args[1]["document_filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None: