diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c7d7cbf..2c87b9ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and rerun `create-index`. ### Fixed +- Batched evaluation ingest converts inline content as text instead of letting `HaikuRAG.convert` disambiguate it, so a passage beginning with a URL is stored rather than fetched over HTTP. 187 MTRAG cloud and fiqa passages start with one; no clapnq passage does, so no existing dataset's numbers change. - `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results. ### Added diff --git a/evaluations/evaluations/population.py b/evaluations/evaluations/population.py index a4d28b4d..d57f349d 100644 --- a/evaluations/evaluations/population.py +++ b/evaluations/evaluations/population.py @@ -39,6 +39,9 @@ async def _ingest_batched( row["uri"]: row["id"] for row in uri_rows if row["id"] not in chunked_ids } + from haiku.rag.converters import get_converter + + converter = get_converter(rag._config) batch: list[DocumentImport] = [] for doc in corpus: payload = spec.document_mapper(cast(Mapping[str, Any], doc)) @@ -48,7 +51,13 @@ async def _ingest_batched( if payload.uri in chunkless: await rag.delete_document(chunkless[payload.uri]) assert payload.content is not None, "batched ingest requires inline content" - docling_document = await rag.convert(payload.content, format=payload.format) + # Convert as text explicitly. `rag.convert` disambiguates a str by + # parsing it, and a passage beginning with a URL (187 of them across + # MTRAG's cloud and fiqa corpora) is then fetched over HTTP instead of + # stored. Batched ingest has already asserted the content is inline. + docling_document = await converter.convert_text( + payload.content, format=payload.format + ) chunks = await rag.chunk(docling_document) batch.append( DocumentImport( diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index b471ac0f..52a90ee2 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -888,7 +888,11 @@ class TestBatchedIngest: rag.store.chunks_table = _table( [{"document_id": f"id-{uri}"} for uri in complete_uris] ) - rag.convert = AsyncMock(side_effect=lambda content, **kw: f"docling:{content}") + # Batched ingest converts text through the configured converter, the + # same path create_document uses, so the double needs a real config. + from haiku.rag.config.models import AppConfig + + rag._config = AppConfig() rag.chunk = AsyncMock(return_value=[]) rag.import_documents = AsyncMock() rag.delete_document = AsyncMock() @@ -933,8 +937,11 @@ class TestBatchedIngest: await _ingest_batched(rag, self._spec(), corpus, batch_size=10) (batch,), _ = rag.import_documents.call_args + # Conversion goes through the configured converter now, not rag.convert, + # so the batch contents are the assertion: exactly the incomplete uris, + # which is stricter than counting conversions. assert [imp.uri for imp in batch] == ["u1", "u3"] - assert rag.convert.await_count == 2 + assert len(batch) == 2 rag.delete_document.assert_not_awaited() @pytest.mark.asyncio