From 6042493c6715b1cdd75aca8ca89caa2d2c798998 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Mon, 31 Aug 2026 17:35:49 +0300
Subject: [PATCH] Convert batched ingest content as text, not as a locator

`_ingest_batched` asserts its payload is inline content and then passed it to
`HaikuRAG.convert`, which disambiguates a str by parsing it: anything whose
scheme reads as http or https is fetched over the network instead of stored.
`urlparse` strips leading whitespace, so a passage beginning with a newline
and a URL qualifies.

187 passages across MTRAG's cloud and fiqa corpora start that way, which
crashed the pooled build. No clapnq passage does, so mtrag_clapnq and every
other existing dataset is unaffected.

Now converts through the configured converter's text path, which is what
create_document already does.

Claude-Session: https://claude.ai/code/session_01WhudUtZm6qqiuv8Y1sbwSc
---
 CHANGELOG.md                          |  1 +
 evaluations/evaluations/population.py | 11 ++++++++++-
 evaluations/tests/test_benchmark.py   | 11 +++++++++--
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1e5cb7f2..1f0a69a3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 ### Fixed
 
+- Batched evaluation ingest converts inline content as text instead of letting `HaikuRAG.convert` disambiguate it, so a passage beginning with a URL is stored rather than fetched over HTTP. 187 MTRAG cloud and fiqa passages start with one; no clapnq passage does, so no existing dataset's numbers change.
 - `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results.
 
 ### Added
diff --git a/evaluations/evaluations/population.py b/evaluations/evaluations/population.py
index a4d28b4d..d57f349d 100644
--- a/evaluations/evaluations/population.py
+++ b/evaluations/evaluations/population.py
@@ -39,6 +39,9 @@ async def _ingest_batched(
         row["uri"]: row["id"] for row in uri_rows if row["id"] not in chunked_ids
     }
 
+    from haiku.rag.converters import get_converter
+
+    converter = get_converter(rag._config)
     batch: list[DocumentImport] = []
     for doc in corpus:
         payload = spec.document_mapper(cast(Mapping[str, Any], doc))
@@ -48,7 +51,13 @@ async def _ingest_batched(
         if payload.uri in chunkless:
             await rag.delete_document(chunkless[payload.uri])
         assert payload.content is not None, "batched ingest requires inline content"
-        docling_document = await rag.convert(payload.content, format=payload.format)
+        # Convert as text explicitly. `rag.convert` disambiguates a str by
+        # parsing it, and a passage beginning with a URL (187 of them across
+        # MTRAG's cloud and fiqa corpora) is then fetched over HTTP instead of
+        # stored. Batched ingest has already asserted the content is inline.
+        docling_document = await converter.convert_text(
+            payload.content, format=payload.format
+        )
         chunks = await rag.chunk(docling_document)
         batch.append(
             DocumentImport(
diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py
index b471ac0f..52a90ee2 100644
--- a/evaluations/tests/test_benchmark.py
+++ b/evaluations/tests/test_benchmark.py
@@ -888,7 +888,11 @@ class TestBatchedIngest:
         rag.store.chunks_table = _table(
             [{"document_id": f"id-{uri}"} for uri in complete_uris]
         )
-        rag.convert = AsyncMock(side_effect=lambda content, **kw: f"docling:{content}")
+        # Batched ingest converts text through the configured converter, the
+        # same path create_document uses, so the double needs a real config.
+        from haiku.rag.config.models import AppConfig
+
+        rag._config = AppConfig()
         rag.chunk = AsyncMock(return_value=[])
         rag.import_documents = AsyncMock()
         rag.delete_document = AsyncMock()
@@ -933,8 +937,11 @@ class TestBatchedIngest:
         await _ingest_batched(rag, self._spec(), corpus, batch_size=10)
 
         (batch,), _ = rag.import_documents.call_args
+        # Conversion goes through the configured converter now, not rag.convert,
+        # so the batch contents are the assertion: exactly the incomplete uris,
+        # which is stricter than counting conversions.
         assert [imp.uri for imp in batch] == ["u1", "u3"]
-        assert rag.convert.await_count == 2
+        assert len(batch) == 2
         rag.delete_document.assert_not_awaited()
 
     @pytest.mark.asyncio