From 9067b89d2f68312d081c72c3dc066769e0218e74 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 22 Apr 2026 14:47:23 +0300 Subject: [PATCH] fix convert() misreading text content that starts with a URL --- CHANGELOG.md | 4 ++ haiku_rag_slim/haiku/rag/client.py | 17 +++++-- tests/test_client.py | 75 ++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index acfaccac..5811cba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Fixed + +- **`create_document`, `update_document`, and rebuild (`RECHUNK` / full fallback) no longer misread URL-prefixed text as a URL to fetch.** These paths passed known-text content through `HaikuRAG.convert()`, which dispatches on `urlparse(source).scheme`; text whose first line was `https://...` (common for clipped web pages and notes) got handed to `httpx.get` and crashed with `httpx.InvalidURL` on embedded whitespace. Fixed by calling `converter.convert_text(...)` directly at those sites; `convert()` itself is unchanged for `create_document_from_source`. + ### Changed - **Skills share a single `HaikuRAG` client per invocation** via the new `haiku.skills>=0.15.0` `lifespan` hook. The skill's sub-agent opens one read-only client on entry, all tool calls reuse it, and it closes on exit — replacing the old pattern of open/close around every `search` / `list_documents` / `get_document` call. diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index 17b906a3..5287e05b 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -495,7 +495,8 @@ class HaikuRAG: from haiku.rag.embeddings import embed_chunks # Convert → Chunk → Embed using primitives - docling_document = await self.convert(content, format=format) + converter = get_converter(self._config) + docling_document = await converter.convert_text(content, format=format) chunks = await self.chunk(docling_document) embedded_chunks = await embed_chunks(chunks, self._config) @@ -1000,7 +1001,10 @@ class HaikuRAG: # Content provided without chunks - convert, chunk, and embed using primitives assert content is not None existing_doc.content = content - converted_docling = await self.convert(existing_doc.content) + converter = get_converter(self._config) + converted_docling = await converter.convert_text( + existing_doc.content, format="md" + ) existing_doc.set_docling(converted_docling) new_chunks = await self.chunk(converted_docling) @@ -1558,11 +1562,13 @@ class HaikuRAG: pending_docs: list[Document] = [] pending_doc_ids: list[str] = [] + converter = get_converter(self._config) + for doc in documents: assert doc.id is not None - # Convert content to DoclingDocument - docling_document = await self.convert(doc.content) + # Convert stored markdown to DoclingDocument + docling_document = await converter.convert_text(doc.content, format="md") # Chunk and embed chunks = await self.chunk(docling_document) @@ -1605,6 +1611,7 @@ class HaikuRAG: pending_chunks: list[Chunk] = [] pending_docs: list[Document] = [] pending_doc_ids: list[str] = [] + converter = get_converter(self._config) for doc in documents: assert doc.id is not None @@ -1643,7 +1650,7 @@ class HaikuRAG: "Source missing for %s, re-embedding from content", doc.uri ) - docling_document = await self.convert(doc.content) + docling_document = await converter.convert_text(doc.content, format="md") chunks = await self.chunk(docling_document) embedded_chunks = await embed_chunks(chunks, self._config) diff --git a/tests/test_client.py b/tests/test_client.py index 7c61a9b8..dbb79983 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1544,3 +1544,78 @@ async def test_sql_injection_is_blocked_with_escaping(temp_db_path): filter=f"title = '{injection_payload}'" ) assert len(docs_unescaped) == 2 # SQL injection succeeds without escaping + + +# ============================================================================= +# URL-prefixed content regression tests +# ============================================================================= + + +def _patch_embed_chunks(monkeypatch): + async def fake_embed_chunks(chunks, config): + for chunk in chunks: + chunk.embedding = [0.0] * 2560 + return chunks + + monkeypatch.setattr("haiku.rag.embeddings.embed_chunks", fake_embed_chunks) + + +async def test_create_document_with_url_prefixed_content(temp_db_path, monkeypatch): + """Text whose first line is a URL must be stored as text, not fetched.""" + _patch_embed_chunks(monkeypatch) + + async with HaikuRAG(temp_db_path, create=True) as client: + content = "https://example.com/foo\n\n# Heading\n\nBody text here." + doc = await client.create_document(content=content, uri="test://url-prefixed") + + assert doc.id is not None + assert "example.com" in doc.content + assert "Heading" in doc.content + + +async def test_update_document_with_url_prefixed_content(temp_db_path, monkeypatch): + """update_document(content=...) with URL-prefixed text must not fetch it.""" + _patch_embed_chunks(monkeypatch) + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document( + content="initial body", uri="test://update-url" + ) + assert doc.id is not None + + url_prefixed = "https://example.com/bar\n\n# New heading\n\nReplacement body." + updated = await client.update_document(doc.id, content=url_prefixed) + + assert "example.com" in updated.content + assert "New heading" in updated.content + + +async def test_rebuild_rechunk_with_url_prefixed_stored_content( + temp_db_path, monkeypatch +): + """RECHUNK rebuild must handle stored markdown whose first line is a URL.""" + from haiku.rag.client import RebuildMode + + _patch_embed_chunks(monkeypatch) + + async with HaikuRAG(temp_db_path, create=True) as client: + doc = await client.create_document( + content="plain seed content", uri="file:///nonexistent/path.txt" + ) + assert doc.id is not None + + # Overwrite stored content to simulate markdown that starts with a URL, + # bypassing the (also-affected) create_document path so this test + # specifically exercises the rebuild path. + doc.content = "https://example.com/baz\n\n# Stored\n\nStored body text." + await client.document_repository.update(doc) + + processed_ids = [ + doc_id async for doc_id in client.rebuild_database(mode=RebuildMode.RECHUNK) + ] + assert doc.id in processed_ids + + doc_after = await client.document_repository.get_by_id(doc.id) + assert doc_after is not None + assert "example.com" in doc_after.content + assert "Stored" in doc_after.content