From 93c21272d156738b0f56875b52311ea30fa5e777 Mon Sep 17 00:00:00 2001 From: cwiesen Date: Thu, 13 Aug 2026 17:05:04 -0500 Subject: [PATCH 1/5] feat: add search_filter to evaluations --- docs/benchmarks.md | 36 +++++- evaluations/evaluations/benchmark.py | 41 ++++++- evaluations/evaluations/config.py | 1 + evaluations/tests/test_benchmark.py | 172 +++++++++++++++++++++++++++ 4 files changed, 244 insertions(+), 6 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 774159da..6b9dff32 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -65,6 +65,7 @@ evaluations run hotpotqa --config /path/to/haiku.rag.yaml --db /path/to/custom.l - `--name NAME` - Override the evaluation name - `--target {rag-capability,analysis-capability}` - Choose which [capability](capabilities/index.md) to benchmark end-to-end (default: `rag-capability`). The target names remain stable dataset identifiers. - `--capability-model PROVIDER:NAME` - Override the capability model independently from the judge (default: `config.qa.model`, or `config.analysis.model` when set for `--target analysis-capability`). +- `--filter CLAUSE` / `-f CLAUSE` - Restrict every benchmark search to a subset of the database (see [Restricting the corpus](#restricting-the-corpus)). If no config file is specified, the script searches standard locations: `./haiku.rag.yaml`, user config directory, then falls back to defaults. @@ -86,6 +87,31 @@ evaluations: enable_thinking: true ``` +### Restricting the corpus + +When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`, `db_source`): + +```bash +evaluations run mqf --skip-db --config haiku.rag.s3.yaml \ + --filter "db_source in ('dataset')" +``` + +The clause applies to both benchmark phases — the retrieval benchmark's searches and every search the capability runs during QA — so the two score the same subset. It is recorded as `search_filter` in the run's experiment metadata, so a filtered run is never mistaken for an unfiltered one when comparing results. + +A dataset can declare its own default in its `DatasetSpec`, so runs need no flag: + +```python +MQF_SPEC = DatasetSpec( + key="mqf", + ... + search_filter="db_source in ('dataset')", +) +``` + +`--filter` overrides that default; passing an empty string (`--filter ""`) clears it and searches the whole database. + +Filtering affects searches only — a run without `--skip-db` still populates the database with the dataset's full corpus. + ## Methodology ### Retrieval Metrics @@ -132,21 +158,21 @@ Two approaches are benchmarked separately: | Embedding Model | Reranker | Cases | MAP | |------------------------------------------|------------------------------------------------------|------:|-------:| | `Qwen/Qwen3-VL-Embedding-8B` | none | 3045 | 0.9774 | -| `nvidia/llama-nemotron-embed-vl-1b-v2` | none | 3045 | 0.9798 | +| `nvidia/llama-nemotron-embed-vl-1b-v2` | none | 3045 | 0.9709 | | `nvidia/llama-nemotron-embed-vl-1b-v2` | `nvidia/llama-nemotron-rerank-vl-1b-v2` (multimodal) | 3045 | 0.9913 | -*The nemotron row without a reranker is measured on this release. The reranked row uses `reranking.multimodal: true`: picture chunks reach the vision reranker as images alongside their description text, measured on haiku.rag main post-v0.67.3.* +*The reranked row uses `reranking.multimodal: true`: picture chunks reach the vision reranker as images alongside their description text. Measured on haiku.rag main post-v0.67.3 (multimodal reranking ships in the next release).* ##### QA accuracy + citation retrieval | Embedding Model | Target | Capability model | Cases | QA accuracy | Mean `cited_map` | |------------------------------------------|-----------------|-----------------------------------|------:|-------------|------------------| | `Qwen/Qwen3-VL-Embedding-8B` | `rag-capability` | `vllm:Gemma-4-26B-A4B-NVFP4` | 1409 | 0.89 | — | -| `nvidia/llama-nemotron-embed-vl-1b-v2` | `rag-capability` | `vllm:Gemma-4-26B-A4B-NVFP4` | 3039 | 0.9263 | 0.9761 | -| `nvidia/llama-nemotron-embed-vl-1b-v2` | `analysis-capability`| `vllm:Gemma-4-26B-A4B-NVFP4` | 3040 | 0.9362 | 0.9343 | +| `nvidia/llama-nemotron-embed-vl-1b-v2` | `rag-capability` | `vllm:Gemma-4-26B-A4B-NVFP4` | 3045 | 0.92 | 0.93 | +| `nvidia/llama-nemotron-embed-vl-1b-v2` | `analysis-capability`| `vllm:Gemma-4-26B-A4B-NVFP4` | 3045 | 0.94 | 0.78 | | `nvidia/llama-nemotron-embed-vl-1b-v2` | `analysis-capability`| `vllm:Qwen3.6-35B-A3B-NVFP4` | 3045 | 0.95 | 0.93 | -*Both nemotron `Gemma-4` rows are measured on this release, no reranker, judged by `vllm:Qwen3.6-35B-A3B-NVFP4` with thinking on, and exclude the cases that errored (6 of 3045 for `rag-capability`, 5 for `analysis-capability`). The `rag-capability` row cites at 99.64% with a mean of 1.08 citations per case, at a median 4.7s per case against 5.0s for `analysis-capability`. Citation coverage is what moved on this release: 4.9% of analysis cases register no citation, against 26.3% before, at unchanged searches and code executions per case. The remaining rows are from haiku.rag v0.52.0, where Qwen3-VL covered 1409 / 3045 cases.* +*Measured on haiku.rag v0.52.0, no reranker, judged by `vllm:Qwen3.6-35B-A3B-NVFP4`. Qwen3-VL covered 1409 / 3045 cases.* #### Text embedder + VLM picture descriptions diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 2a953d2b..fd421688 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -57,6 +57,16 @@ configure_telemetry(service_name="evals", scrubbing=False) configure_cli_logging() console = Console() +def resolve_search_filter(spec: DatasetSpec, override: str | None) -> str | None: + """Pick the document filter for a run: `--filter` wins over the dataset's. + + An empty `--filter ""` is honoured as "no filter", so a dataset that + declares one can still be run against the whole database. + """ + if override is None: + return spec.search_filter + return override or None + def build_experiment_metadata( dataset_key: str, @@ -65,6 +75,7 @@ def build_experiment_metadata( judge_config: ModelConfig | None = None, target: Target = "rag-capability", capability_config: ModelConfig | None = None, + search_filter: str | None = None, ) -> dict[str, Any]: """Build experiment metadata for Logfire tracking.""" metadata: dict[str, Any] = { @@ -88,6 +99,7 @@ def build_experiment_metadata( "qa_enable_thinking": config.qa.model.enable_thinking, "qa_extra_body": config.qa.model.extra_body, "qa_max_searches": config.qa.max_searches, + "search_filter": search_filter, } if judge_config is not None: metadata.update( @@ -191,6 +203,7 @@ async def run_retrieval_benchmark( name: str | None = None, db_path: Path | None = None, multimodal_only: bool = False, + search_filter: str | None = None, ) -> dict[str, float] | None: if spec.retrieval_loader is None or spec.retrieval_mapper is None: console.print("Skipping retrieval benchmark; no retrieval config.") @@ -246,7 +259,9 @@ async def run_retrieval_benchmark( async with HaikuRAG(db, config=config, read_only=True) as rag: async def retrieval_target(question: str) -> list[str]: - chunks = await rag.search(query=question, limit=5, include_images=False) + chunks = await rag.search( + query=question, limit=5, include_images=False, filter=search_filter + ) seen = set() identifiers = [] @@ -264,6 +279,7 @@ async def run_retrieval_benchmark( dataset_key=spec.key, test_cases=len(cases), config=config, + search_filter=search_filter, ) report = await dataset.evaluate( @@ -364,6 +380,7 @@ async def run_qa_benchmark( target: Target = "rag-capability", capability_model: ModelConfig | None = None, case_ids: set[str] | None = None, + search_filter: str | None = None, ) -> ReportCaseFailure[str, str, dict[str, str]] | None: corpus = spec.qa_loader() corpus = _filter_qa_corpus(corpus, case_ids) @@ -419,6 +436,7 @@ async def run_qa_benchmark( judge_config=judge_config, target=target, capability_config=capability_config, + search_filter=search_filter, ) async def _evaluate(answer_fn: Callable[[str], Awaitable[str]]): @@ -440,6 +458,7 @@ async def run_qa_benchmark( config=config, question=question, capability_model=resolved_capability_model, + document_filter=search_filter, ) set_eval_attribute("cited_uris", result.cited_uris) set_eval_attribute("cited_chunk_ids", result.cited_chunk_ids) @@ -530,7 +549,13 @@ async def evaluate_dataset( target: Target = "rag-capability", capability_model: ModelConfig | None = None, case_ids: set[str] | None = None, + search_filter: str | None = None, ) -> None: + # Resolved once so both phases score the same subset of the database. + resolved_filter = resolve_search_filter(spec, search_filter) + if resolved_filter is not None: + console.print(f"Document filter: {resolved_filter}", style="dim") + if not skip_db: console.print(f"Using dataset: {spec.key}", style="bold magenta") await populate_db( @@ -546,6 +571,7 @@ async def evaluate_dataset( name=name, db_path=db_path, multimodal_only=multimodal_only, + search_filter=resolved_filter, ) if not skip_qa: @@ -562,6 +588,7 @@ async def evaluate_dataset( target=target, capability_model=capability_model, case_ids=case_ids, + search_filter=resolved_filter, ) @@ -651,6 +678,17 @@ def run( "analysis.model when --target is analysis-capability) from the config." ), ), + search_filter: str | None = typer.Option( + None, + "--filter", + "-f", + help=( + "SQL WHERE clause over document columns (id, uri, title, metadata, " + "db_source, ...) restricting every benchmark search, e.g. " + "\"db_source in ('dataset')\". Overrides the dataset's own " + "filter; pass an empty string to search the whole database." + ), + ), filter_ids: Path | None = typer.Option( None, "--filter-ids", @@ -688,6 +726,7 @@ def run( target=target_value, capability_model=capability_model_config, case_ids=_load_case_ids(filter_ids), + search_filter=search_filter, ) ) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index c5ecfd17..a1025cde 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -46,6 +46,7 @@ class DatasetSpec: retrieval_evaluator: Evaluator | None = None qa_evaluator: Evaluator | None = None document_limit: int | None = None + search_filter: str | None = None def db_path(self, override_path: Path | None = None) -> Path: """Get the database path. diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 54d09ac1..dca0b1ce 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -581,6 +581,178 @@ class TestRetrievalTarget: assert result["map"] == 0.5 +class TestResolveSearchFilter: + def test_dataset_filter_used_when_no_override(self) -> None: + from evaluations.benchmark import resolve_search_filter + + spec = _stub_spec(search_filter="db_source in ('dataset')") + assert resolve_search_filter(spec, None) == "db_source in ('dataset')" + + def test_override_wins(self) -> None: + from evaluations.benchmark import resolve_search_filter + + spec = _stub_spec(search_filter="db_source in ('dataset')") + assert resolve_search_filter(spec, "uri LIKE '%.pdf'") == "uri LIKE '%.pdf'" + + def test_empty_override_clears_dataset_filter(self) -> None: + """`--filter ""` runs a filtered dataset against the whole database.""" + from evaluations.benchmark import resolve_search_filter + + spec = _stub_spec(search_filter="db_source in ('dataset')") + assert resolve_search_filter(spec, "") is None + + def test_none_when_neither_is_set(self) -> None: + from evaluations.benchmark import resolve_search_filter + + assert resolve_search_filter(_stub_spec(), None) is None + + +class TestSearchFilterThreading: + """The resolved filter must reach both benchmark phases, so retrieval and + QA score the same subset of the database.""" + + def test_metadata_records_filter(self) -> None: + result = build_experiment_metadata( + dataset_key="test", + test_cases=1, + config=AppConfig(), + search_filter="db_source in ('dataset')", + ) + assert result["search_filter"] == "db_source in ('dataset')" + + def test_metadata_filter_is_none_when_unset(self) -> None: + result = build_experiment_metadata( + dataset_key="test", test_cases=1, config=AppConfig() + ) + assert result["search_filter"] is None + + @pytest.mark.asyncio + async def test_retrieval_search_receives_filter(self, tmp_path: Path) -> None: + from haiku.rag.store.models.chunk import SearchResult + + from evaluations.benchmark import run_retrieval_benchmark + from evaluations.config import RetrievalSample + from evaluations.evaluators import MAPEvaluator + + searches: list[dict] = [] + + class FakeRag: + async def search(self, **kwargs) -> list[SearchResult]: + searches.append(kwargs) + return [SearchResult(content="x", score=1.0, document_uri="uri-x")] + + spec = _stub_spec( + retrieval_loader=lambda: [{"q": "What is X?", "uris": ("uri-x",)}], + retrieval_mapper=lambda d: RetrievalSample( + question=d["q"], expected_uris=d["uris"] + ), + retrieval_evaluator=MAPEvaluator(), + ) + + with patch("evaluations.benchmark.HaikuRAG") as mock_haiku: + mock_haiku.return_value.__aenter__.return_value = FakeRag() + await run_retrieval_benchmark( + spec, + AppConfig(), + db_path=tmp_path / "test.lancedb", + search_filter="db_source in ('dataset')", + ) + + assert searches[0]["filter"] == "db_source in ('dataset')" + + @pytest.mark.asyncio + async def test_qa_capability_run_receives_filter(self, tmp_path: Path) -> None: + from pydantic_evals import Case + + from evaluations.capability_runner import CapabilityRunResult + from evaluations.evaluators import NumberMatchEvaluator + + # A deterministic evaluator, so no judge model is constructed. + spec = DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [{"question": "What is X?", "answer": "42"}], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=lambda idx, doc: Case( + name=f"case-{idx}", + inputs=doc["question"], + expected_output=doc["answer"], + ), + qa_evaluator=NumberMatchEvaluator(), + ) + + with patch( + "evaluations.benchmark.run_capability_question", + new_callable=AsyncMock, + return_value=CapabilityRunResult(answer="ANSWER: 42"), + ) as mock_run: + await run_qa_benchmark( + spec, + AppConfig(), + db_path=tmp_path / "test.lancedb", + search_filter="db_source in ('dataset')", + ) + + mock_run.assert_awaited_once() + assert mock_run.await_args[1]["document_filter"] == "db_source in ('dataset')" + + @pytest.mark.asyncio + async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None: + """The dataset's own filter reaches retrieval and QA without a flag.""" + spec = _stub_spec(search_filter="db_source in ('dataset','other')") + + with ( + patch( + "evaluations.benchmark.run_retrieval_benchmark", new_callable=AsyncMock + ) as mock_retrieval, + patch( + "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock + ) as mock_qa, + ): + await evaluate_dataset( + spec=spec, + config=AppConfig(), + skip_db=True, + skip_retrieval=False, + skip_qa=False, + limit=None, + name=None, + db_path=None, + ) + + expected = "db_source in ('dataset','other')" + assert mock_retrieval.call_args[1]["search_filter"] == expected + assert mock_qa.call_args[1]["search_filter"] == expected + + @pytest.mark.asyncio + async def test_evaluate_dataset_override_reaches_both_phases(self) -> None: + spec = _stub_spec(search_filter="db_source in ('dataset','other')") + + with ( + patch( + "evaluations.benchmark.run_retrieval_benchmark", new_callable=AsyncMock + ) as mock_retrieval, + patch( + "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock + ) as mock_qa, + ): + await evaluate_dataset( + spec=spec, + config=AppConfig(), + skip_db=True, + skip_retrieval=False, + skip_qa=False, + limit=None, + name=None, + db_path=None, + search_filter="db_source in ('other')", + ) + + assert mock_retrieval.call_args[1]["search_filter"] == "db_source in ('other')" + assert mock_qa.call_args[1]["search_filter"] == "db_source in ('other')" + + class TestEvaluateDatasetCaseIds: def _spec(self) -> DatasetSpec: return _stub_spec() From a68cb23b6ec6889a6b0492738158add93b7a1c26 Mon Sep 17 00:00:00 2001 From: cwiesen Date: Fri, 14 Aug 2026 16:27:04 -0500 Subject: [PATCH 2/5] fix: resolve incorrect dataset and column namings --- docs/benchmarks.md | 18 ++++++++++------ evaluations/evaluations/benchmark.py | 8 ++++--- evaluations/tests/test_benchmark.py | 32 ++++++++++++++-------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 6b9dff32..6a515cc1 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -89,11 +89,17 @@ evaluations: ### Restricting the corpus -When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`, `db_source`): +When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`): ```bash -evaluations run mqf --skip-db --config haiku.rag.s3.yaml \ - --filter "db_source in ('dataset')" +evaluations run orb_text --skip-db --config haiku.rag.s3.yaml \ + --filter "uri LIKE '%arxiv%'" +``` + +If the corpora are distinguished by a tag rather than by URI, attach it at ingest time as document metadata and match it with `LIKE`. `metadata` is stored as a `json.dumps` string, so there is no JSON subfield access — match the serialized key/value, including the space after the colon: + +```bash +evaluations run orb_text --skip-db --filter "metadata LIKE '%\"corpus\": \"orb_text\"%'" ``` The clause applies to both benchmark phases — the retrieval benchmark's searches and every search the capability runs during QA — so the two score the same subset. It is recorded as `search_filter` in the run's experiment metadata, so a filtered run is never mistaken for an unfiltered one when comparing results. @@ -101,10 +107,10 @@ The clause applies to both benchmark phases — the retrieval benchmark's search A dataset can declare its own default in its `DatasetSpec`, so runs need no flag: ```python -MQF_SPEC = DatasetSpec( - key="mqf", +ORB_TEXT_SPEC = DatasetSpec( + key="orb_text", ... - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) ``` diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index fd421688..267902d9 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -57,6 +57,7 @@ configure_telemetry(service_name="evals", scrubbing=False) configure_cli_logging() console = Console() + def resolve_search_filter(spec: DatasetSpec, override: str | None) -> str | None: """Pick the document filter for a run: `--filter` wins over the dataset's. @@ -683,9 +684,10 @@ def run( "--filter", "-f", help=( - "SQL WHERE clause over document columns (id, uri, title, metadata, " - "db_source, ...) restricting every benchmark search, e.g. " - "\"db_source in ('dataset')\". Overrides the dataset's own " + "SQL WHERE clause over document columns (id, uri, title, " + "created_at, updated_at, metadata) restricting every benchmark " + "search, e.g. \"uri LIKE '%arxiv%'\". metadata is stored as a " + "string, so match it with LIKE. Overrides the dataset's own " "filter; pass an empty string to search the whole database." ), ), diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index dca0b1ce..4cb75d33 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -585,20 +585,20 @@ class TestResolveSearchFilter: def test_dataset_filter_used_when_no_override(self) -> None: from evaluations.benchmark import resolve_search_filter - spec = _stub_spec(search_filter="db_source in ('dataset')") - assert resolve_search_filter(spec, None) == "db_source in ('dataset')" + spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") + assert resolve_search_filter(spec, None) == "uri LIKE '%arxiv%'" def test_override_wins(self) -> None: from evaluations.benchmark import resolve_search_filter - spec = _stub_spec(search_filter="db_source in ('dataset')") + spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") assert resolve_search_filter(spec, "uri LIKE '%.pdf'") == "uri LIKE '%.pdf'" def test_empty_override_clears_dataset_filter(self) -> None: """`--filter ""` runs a filtered dataset against the whole database.""" from evaluations.benchmark import resolve_search_filter - spec = _stub_spec(search_filter="db_source in ('dataset')") + spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") assert resolve_search_filter(spec, "") is None def test_none_when_neither_is_set(self) -> None: @@ -616,9 +616,9 @@ class TestSearchFilterThreading: dataset_key="test", test_cases=1, config=AppConfig(), - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) - assert result["search_filter"] == "db_source in ('dataset')" + assert result["search_filter"] == "uri LIKE '%arxiv%'" def test_metadata_filter_is_none_when_unset(self) -> None: result = build_experiment_metadata( @@ -655,10 +655,10 @@ class TestSearchFilterThreading: spec, AppConfig(), db_path=tmp_path / "test.lancedb", - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) - assert searches[0]["filter"] == "db_source in ('dataset')" + assert searches[0]["filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio async def test_qa_capability_run_receives_filter(self, tmp_path: Path) -> None: @@ -691,16 +691,16 @@ class TestSearchFilterThreading: spec, AppConfig(), db_path=tmp_path / "test.lancedb", - search_filter="db_source in ('dataset')", + search_filter="uri LIKE '%arxiv%'", ) mock_run.assert_awaited_once() - assert mock_run.await_args[1]["document_filter"] == "db_source in ('dataset')" + assert mock_run.await_args[1]["document_filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None: """The dataset's own filter reaches retrieval and QA without a flag.""" - spec = _stub_spec(search_filter="db_source in ('dataset','other')") + spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""") with ( patch( @@ -721,13 +721,13 @@ class TestSearchFilterThreading: db_path=None, ) - expected = "db_source in ('dataset','other')" + expected = """metadata LIKE '%"corpus": "orb_text"%'""" assert mock_retrieval.call_args[1]["search_filter"] == expected assert mock_qa.call_args[1]["search_filter"] == expected @pytest.mark.asyncio async def test_evaluate_dataset_override_reaches_both_phases(self) -> None: - spec = _stub_spec(search_filter="db_source in ('dataset','other')") + spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""") with ( patch( @@ -746,11 +746,11 @@ class TestSearchFilterThreading: limit=None, name=None, db_path=None, - search_filter="db_source in ('other')", + search_filter="title LIKE '%paper%'", ) - assert mock_retrieval.call_args[1]["search_filter"] == "db_source in ('other')" - assert mock_qa.call_args[1]["search_filter"] == "db_source in ('other')" + assert mock_retrieval.call_args[1]["search_filter"] == "title LIKE '%paper%'" + assert mock_qa.call_args[1]["search_filter"] == "title LIKE '%paper%'" class TestEvaluateDatasetCaseIds: From db6996b629b49862fe917fd3d5bc162efb1cff28 Mon Sep 17 00:00:00 2001 From: cwiesen Date: Fri, 14 Aug 2026 16:36:51 -0500 Subject: [PATCH 3/5] fix: revert changes to benchmarks.md file --- docs/benchmarks.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 6a515cc1..9e1a94a6 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -164,21 +164,21 @@ Two approaches are benchmarked separately: | Embedding Model | Reranker | Cases | MAP | |------------------------------------------|------------------------------------------------------|------:|-------:| | `Qwen/Qwen3-VL-Embedding-8B` | none | 3045 | 0.9774 | -| `nvidia/llama-nemotron-embed-vl-1b-v2` | none | 3045 | 0.9709 | +| `nvidia/llama-nemotron-embed-vl-1b-v2` | none | 3045 | 0.9798 | | `nvidia/llama-nemotron-embed-vl-1b-v2` | `nvidia/llama-nemotron-rerank-vl-1b-v2` (multimodal) | 3045 | 0.9913 | -*The reranked row uses `reranking.multimodal: true`: picture chunks reach the vision reranker as images alongside their description text. Measured on haiku.rag main post-v0.67.3 (multimodal reranking ships in the next release).* +*The nemotron row without a reranker is measured on this release. The reranked row uses `reranking.multimodal: true`: picture chunks reach the vision reranker as images alongside their description text, measured on haiku.rag main post-v0.67.3.* ##### QA accuracy + citation retrieval | Embedding Model | Target | Capability model | Cases | QA accuracy | Mean `cited_map` | |------------------------------------------|-----------------|-----------------------------------|------:|-------------|------------------| | `Qwen/Qwen3-VL-Embedding-8B` | `rag-capability` | `vllm:Gemma-4-26B-A4B-NVFP4` | 1409 | 0.89 | — | -| `nvidia/llama-nemotron-embed-vl-1b-v2` | `rag-capability` | `vllm:Gemma-4-26B-A4B-NVFP4` | 3045 | 0.92 | 0.93 | -| `nvidia/llama-nemotron-embed-vl-1b-v2` | `analysis-capability`| `vllm:Gemma-4-26B-A4B-NVFP4` | 3045 | 0.94 | 0.78 | +| `nvidia/llama-nemotron-embed-vl-1b-v2` | `rag-capability` | `vllm:Gemma-4-26B-A4B-NVFP4` | 3039 | 0.9263 | 0.9761 | +| `nvidia/llama-nemotron-embed-vl-1b-v2` | `analysis-capability`| `vllm:Gemma-4-26B-A4B-NVFP4` | 3040 | 0.9362 | 0.9343 | | `nvidia/llama-nemotron-embed-vl-1b-v2` | `analysis-capability`| `vllm:Qwen3.6-35B-A3B-NVFP4` | 3045 | 0.95 | 0.93 | -*Measured on haiku.rag v0.52.0, no reranker, judged by `vllm:Qwen3.6-35B-A3B-NVFP4`. Qwen3-VL covered 1409 / 3045 cases.* +*Both nemotron `Gemma-4` rows are measured on this release, no reranker, judged by `vllm:Qwen3.6-35B-A3B-NVFP4` with thinking on, and exclude the cases that errored (6 of 3045 for `rag-capability`, 5 for `analysis-capability`). The `rag-capability` row cites at 99.64% with a mean of 1.08 citations per case, at a median 4.7s per case against 5.0s for `analysis-capability`. Citation coverage is what moved on this release: 4.9% of analysis cases register no citation, against 26.3% before, at unchanged searches and code executions per case. The remaining rows are from haiku.rag v0.52.0, where Qwen3-VL covered 1409 / 3045 cases.* #### Text embedder + VLM picture descriptions From 9b2ae347d2a736df5fe8fd8e709623126bbc3b82 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 17 Aug 2026 10:19:51 +0300 Subject: [PATCH 4/5] Use filters that match the datasets they document The `--filter` examples used `uri LIKE '%arxiv%'`, which matches no `orb_text` document: its URIs are bare arXiv ids such as `2407.01528v3`. A clause that matches nothing scores MAP 0 rather than erroring, so the example failed silently. `await_args` is typed `_Call | None`, so subscripting it fails `ty check`; `call_args` carries the same call for an AsyncMock. --- docs/benchmarks.md | 6 +++--- evaluations/tests/test_benchmark.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9e1a94a6..dc7f4f8f 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -89,11 +89,11 @@ evaluations: ### Restricting the corpus -When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`): +When a database holds documents from several corpora — only some of which a dataset's questions are drawn from — `--filter` restricts every benchmark search to a subset. It takes the same SQL `WHERE` clause as `haiku-rag search --filter`, over document columns (`id`, `uri`, `title`, `created_at`, `updated_at`, `metadata`). Each dataset writes its own URIs: `orb_text` uses bare arXiv ids such as `2407.01528v3`, `hotpotqa` uses page titles. ```bash evaluations run orb_text --skip-db --config haiku.rag.s3.yaml \ - --filter "uri LIKE '%arxiv%'" + --filter "uri LIKE '2407%'" ``` If the corpora are distinguished by a tag rather than by URI, attach it at ingest time as document metadata and match it with `LIKE`. `metadata` is stored as a `json.dumps` string, so there is no JSON subfield access — match the serialized key/value, including the space after the colon: @@ -110,7 +110,7 @@ A dataset can declare its own default in its `DatasetSpec`, so runs need no flag ORB_TEXT_SPEC = DatasetSpec( key="orb_text", ... - search_filter="uri LIKE '%arxiv%'", + search_filter="metadata LIKE '%\"corpus\": \"orb_text\"%'", ) ``` diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 4cb75d33..a24a0ca0 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -695,7 +695,7 @@ class TestSearchFilterThreading: ) mock_run.assert_awaited_once() - assert mock_run.await_args[1]["document_filter"] == "uri LIKE '%arxiv%'" + assert mock_run.call_args[1]["document_filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None: From a75d89122e4e57c9bc83b6eb825c6197f188d683 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 17 Aug 2026 10:29:28 +0300 Subject: [PATCH 5/5] Drop the unused per-dataset filter default No `DatasetSpec` declared `search_filter`, so `resolve_search_filter` and the `--filter ""` clearing rule reconciled the flag against a default that never existed. The flag alone covers the case. An empty clause reaches `ChunkRepository.search`, which already treats it as unfiltered. Rename to `document_filter` throughout, matching `run_capability_question`'s parameter and the metadata key that lands in Logfire. `_stub_spec` merges its overrides, so a test can override a loader instead of rebuilding the whole spec. --- CHANGELOG.md | 4 + docs/benchmarks.md | 14 +--- evaluations/evaluations/benchmark.py | 46 ++++-------- evaluations/evaluations/config.py | 1 - evaluations/tests/test_benchmark.py | 108 +++++++-------------------- 5 files changed, 46 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0eefce29..bda57732 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Added + +- `evaluations run --filter/-f CLAUSE`: SQL `WHERE` clause over document columns, applied to the retrieval benchmark's searches and to every capability search during QA. Recorded as `document_filter` in experiment metadata. + ### Removed - `wix` evaluation dataset and its reference config `evaluations/configs/wix.yaml`. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index dc7f4f8f..c8fe795e 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -102,19 +102,7 @@ If the corpora are distinguished by a tag rather than by URI, attach it at inges evaluations run orb_text --skip-db --filter "metadata LIKE '%\"corpus\": \"orb_text\"%'" ``` -The clause applies to both benchmark phases — the retrieval benchmark's searches and every search the capability runs during QA — so the two score the same subset. It is recorded as `search_filter` in the run's experiment metadata, so a filtered run is never mistaken for an unfiltered one when comparing results. - -A dataset can declare its own default in its `DatasetSpec`, so runs need no flag: - -```python -ORB_TEXT_SPEC = DatasetSpec( - key="orb_text", - ... - search_filter="metadata LIKE '%\"corpus\": \"orb_text\"%'", -) -``` - -`--filter` overrides that default; passing an empty string (`--filter ""`) clears it and searches the whole database. +The clause applies to both benchmark phases — the retrieval benchmark's searches and every search the capability runs during QA — so the two score the same subset. It is recorded as `document_filter` in the run's experiment metadata, so a filtered run is never mistaken for an unfiltered one when comparing results. Filtering affects searches only — a run without `--skip-db` still populates the database with the dataset's full corpus. diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 267902d9..36aaa70b 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -58,17 +58,6 @@ configure_cli_logging() console = Console() -def resolve_search_filter(spec: DatasetSpec, override: str | None) -> str | None: - """Pick the document filter for a run: `--filter` wins over the dataset's. - - An empty `--filter ""` is honoured as "no filter", so a dataset that - declares one can still be run against the whole database. - """ - if override is None: - return spec.search_filter - return override or None - - def build_experiment_metadata( dataset_key: str, test_cases: int, @@ -76,7 +65,7 @@ def build_experiment_metadata( judge_config: ModelConfig | None = None, target: Target = "rag-capability", capability_config: ModelConfig | None = None, - search_filter: str | None = None, + document_filter: str | None = None, ) -> dict[str, Any]: """Build experiment metadata for Logfire tracking.""" metadata: dict[str, Any] = { @@ -100,7 +89,7 @@ def build_experiment_metadata( "qa_enable_thinking": config.qa.model.enable_thinking, "qa_extra_body": config.qa.model.extra_body, "qa_max_searches": config.qa.max_searches, - "search_filter": search_filter, + "document_filter": document_filter, } if judge_config is not None: metadata.update( @@ -204,7 +193,7 @@ async def run_retrieval_benchmark( name: str | None = None, db_path: Path | None = None, multimodal_only: bool = False, - search_filter: str | None = None, + document_filter: str | None = None, ) -> dict[str, float] | None: if spec.retrieval_loader is None or spec.retrieval_mapper is None: console.print("Skipping retrieval benchmark; no retrieval config.") @@ -261,7 +250,7 @@ async def run_retrieval_benchmark( async def retrieval_target(question: str) -> list[str]: chunks = await rag.search( - query=question, limit=5, include_images=False, filter=search_filter + query=question, limit=5, include_images=False, filter=document_filter ) seen = set() @@ -280,7 +269,7 @@ async def run_retrieval_benchmark( dataset_key=spec.key, test_cases=len(cases), config=config, - search_filter=search_filter, + document_filter=document_filter, ) report = await dataset.evaluate( @@ -381,7 +370,7 @@ async def run_qa_benchmark( target: Target = "rag-capability", capability_model: ModelConfig | None = None, case_ids: set[str] | None = None, - search_filter: str | None = None, + document_filter: str | None = None, ) -> ReportCaseFailure[str, str, dict[str, str]] | None: corpus = spec.qa_loader() corpus = _filter_qa_corpus(corpus, case_ids) @@ -437,7 +426,7 @@ async def run_qa_benchmark( judge_config=judge_config, target=target, capability_config=capability_config, - search_filter=search_filter, + document_filter=document_filter, ) async def _evaluate(answer_fn: Callable[[str], Awaitable[str]]): @@ -459,7 +448,7 @@ async def run_qa_benchmark( config=config, question=question, capability_model=resolved_capability_model, - document_filter=search_filter, + document_filter=document_filter, ) set_eval_attribute("cited_uris", result.cited_uris) set_eval_attribute("cited_chunk_ids", result.cited_chunk_ids) @@ -550,12 +539,10 @@ async def evaluate_dataset( target: Target = "rag-capability", capability_model: ModelConfig | None = None, case_ids: set[str] | None = None, - search_filter: str | None = None, + document_filter: str | None = None, ) -> None: - # Resolved once so both phases score the same subset of the database. - resolved_filter = resolve_search_filter(spec, search_filter) - if resolved_filter is not None: - console.print(f"Document filter: {resolved_filter}", style="dim") + if document_filter is not None: + console.print(f"Document filter: {document_filter}", style="dim") if not skip_db: console.print(f"Using dataset: {spec.key}", style="bold magenta") @@ -572,7 +559,7 @@ async def evaluate_dataset( name=name, db_path=db_path, multimodal_only=multimodal_only, - search_filter=resolved_filter, + document_filter=document_filter, ) if not skip_qa: @@ -589,7 +576,7 @@ async def evaluate_dataset( target=target, capability_model=capability_model, case_ids=case_ids, - search_filter=resolved_filter, + document_filter=document_filter, ) @@ -679,7 +666,7 @@ def run( "analysis.model when --target is analysis-capability) from the config." ), ), - search_filter: str | None = typer.Option( + document_filter: str | None = typer.Option( None, "--filter", "-f", @@ -687,8 +674,7 @@ def run( "SQL WHERE clause over document columns (id, uri, title, " "created_at, updated_at, metadata) restricting every benchmark " "search, e.g. \"uri LIKE '%arxiv%'\". metadata is stored as a " - "string, so match it with LIKE. Overrides the dataset's own " - "filter; pass an empty string to search the whole database." + "string, so match it with LIKE." ), ), filter_ids: Path | None = typer.Option( @@ -728,7 +714,7 @@ def run( target=target_value, capability_model=capability_model_config, case_ids=_load_case_ids(filter_ids), - search_filter=search_filter, + document_filter=document_filter, ) ) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index a1025cde..c5ecfd17 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -46,7 +46,6 @@ class DatasetSpec: retrieval_evaluator: Evaluator | None = None qa_evaluator: Evaluator | None = None document_limit: int | None = None - search_filter: str | None = None def db_path(self, override_path: Path | None = None) -> Path: """Get the database path. diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index a24a0ca0..97ff0cc8 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -17,16 +17,16 @@ from haiku.rag.config.models import AppConfig, ModelConfig def _stub_spec(**overrides) -> DatasetSpec: """A DatasetSpec whose loaders/mappers are inert, for tests that only - exercise the surrounding plumbing.""" - return DatasetSpec( - key="test", - db_filename="test.lancedb", - document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - document_mapper=lambda doc: None, - qa_loader=lambda: [], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - **overrides, - ) + exercise the surrounding plumbing. Any field can be overridden.""" + fields: dict = { + "key": "test", + "db_filename": "test.lancedb", + "document_loader": lambda: None, + "document_mapper": lambda doc: None, + "qa_loader": lambda: [], + "qa_case_builder": lambda idx, doc: None, + } + return DatasetSpec(**{**fields, **overrides}) class TestBuildExperimentMetadata: @@ -581,50 +581,24 @@ class TestRetrievalTarget: assert result["map"] == 0.5 -class TestResolveSearchFilter: - def test_dataset_filter_used_when_no_override(self) -> None: - from evaluations.benchmark import resolve_search_filter - - spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") - assert resolve_search_filter(spec, None) == "uri LIKE '%arxiv%'" - - def test_override_wins(self) -> None: - from evaluations.benchmark import resolve_search_filter - - spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") - assert resolve_search_filter(spec, "uri LIKE '%.pdf'") == "uri LIKE '%.pdf'" - - def test_empty_override_clears_dataset_filter(self) -> None: - """`--filter ""` runs a filtered dataset against the whole database.""" - from evaluations.benchmark import resolve_search_filter - - spec = _stub_spec(search_filter="uri LIKE '%arxiv%'") - assert resolve_search_filter(spec, "") is None - - def test_none_when_neither_is_set(self) -> None: - from evaluations.benchmark import resolve_search_filter - - assert resolve_search_filter(_stub_spec(), None) is None - - -class TestSearchFilterThreading: - """The resolved filter must reach both benchmark phases, so retrieval and - QA score the same subset of the database.""" +class TestDocumentFilterThreading: + """The filter must reach both benchmark phases, so retrieval and QA score + the same subset of the database.""" def test_metadata_records_filter(self) -> None: result = build_experiment_metadata( dataset_key="test", test_cases=1, config=AppConfig(), - search_filter="uri LIKE '%arxiv%'", + document_filter="uri LIKE '%arxiv%'", ) - assert result["search_filter"] == "uri LIKE '%arxiv%'" + assert result["document_filter"] == "uri LIKE '%arxiv%'" def test_metadata_filter_is_none_when_unset(self) -> None: result = build_experiment_metadata( dataset_key="test", test_cases=1, config=AppConfig() ) - assert result["search_filter"] is None + assert result["document_filter"] is None @pytest.mark.asyncio async def test_retrieval_search_receives_filter(self, tmp_path: Path) -> None: @@ -655,7 +629,7 @@ class TestSearchFilterThreading: spec, AppConfig(), db_path=tmp_path / "test.lancedb", - search_filter="uri LIKE '%arxiv%'", + document_filter="uri LIKE '%arxiv%'", ) assert searches[0]["filter"] == "uri LIKE '%arxiv%'" @@ -668,12 +642,8 @@ class TestSearchFilterThreading: from evaluations.evaluators import NumberMatchEvaluator # A deterministic evaluator, so no judge model is constructed. - spec = DatasetSpec( - key="test", - db_filename="test.lancedb", - document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - document_mapper=lambda doc: None, - qa_loader=lambda: [{"question": "What is X?", "answer": "42"}], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + spec = _stub_spec( + qa_loader=lambda: [{"question": "What is X?", "answer": "42"}], qa_case_builder=lambda idx, doc: Case( name=f"case-{idx}", inputs=doc["question"], @@ -691,43 +661,15 @@ class TestSearchFilterThreading: spec, AppConfig(), db_path=tmp_path / "test.lancedb", - search_filter="uri LIKE '%arxiv%'", + document_filter="uri LIKE '%arxiv%'", ) mock_run.assert_awaited_once() assert mock_run.call_args[1]["document_filter"] == "uri LIKE '%arxiv%'" @pytest.mark.asyncio - async def test_evaluate_dataset_resolves_once_for_both_phases(self) -> None: - """The dataset's own filter reaches retrieval and QA without a flag.""" - spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""") - - with ( - patch( - "evaluations.benchmark.run_retrieval_benchmark", new_callable=AsyncMock - ) as mock_retrieval, - patch( - "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock - ) as mock_qa, - ): - await evaluate_dataset( - spec=spec, - config=AppConfig(), - skip_db=True, - skip_retrieval=False, - skip_qa=False, - limit=None, - name=None, - db_path=None, - ) - + async def test_evaluate_dataset_passes_filter_to_both_phases(self) -> None: expected = """metadata LIKE '%"corpus": "orb_text"%'""" - assert mock_retrieval.call_args[1]["search_filter"] == expected - assert mock_qa.call_args[1]["search_filter"] == expected - - @pytest.mark.asyncio - async def test_evaluate_dataset_override_reaches_both_phases(self) -> None: - spec = _stub_spec(search_filter="""metadata LIKE '%"corpus": "orb_text"%'""") with ( patch( @@ -738,7 +680,7 @@ class TestSearchFilterThreading: ) as mock_qa, ): await evaluate_dataset( - spec=spec, + spec=_stub_spec(), config=AppConfig(), skip_db=True, skip_retrieval=False, @@ -746,11 +688,11 @@ class TestSearchFilterThreading: limit=None, name=None, db_path=None, - search_filter="title LIKE '%paper%'", + document_filter=expected, ) - assert mock_retrieval.call_args[1]["search_filter"] == "title LIKE '%paper%'" - assert mock_qa.call_args[1]["search_filter"] == "title LIKE '%paper%'" + assert mock_retrieval.call_args[1]["document_filter"] == expected + assert mock_qa.call_args[1]["document_filter"] == expected class TestEvaluateDatasetCaseIds: