fix convert() misreading text content that starts with a URL

This commit is contained in:
Yiorgis Gozadinos 2026-04-22 14:47:23 +03:00
parent f0016ebcd2
commit 9067b89d2f
No known key found for this signature in database
3 changed files with 91 additions and 5 deletions

View file

@ -1,6 +1,10 @@
# Changelog
## [Unreleased]
### Fixed
- **`create_document`, `update_document`, and rebuild (`RECHUNK` / full fallback) no longer misread URL-prefixed text as a URL to fetch.** These paths passed known-text content through `HaikuRAG.convert()`, which dispatches on `urlparse(source).scheme`; text whose first line was `https://...` (common for clipped web pages and notes) got handed to `httpx.get` and crashed with `httpx.InvalidURL` on embedded whitespace. Fixed by calling `converter.convert_text(...)` directly at those sites; `convert()` itself is unchanged for `create_document_from_source`.
### Changed
- **Skills share a single `HaikuRAG` client per invocation** via the new `haiku.skills>=0.15.0` `lifespan` hook. The skill's sub-agent opens one read-only client on entry, all tool calls reuse it, and it closes on exit — replacing the old pattern of open/close around every `search` / `list_documents` / `get_document` call.

View file

@ -495,7 +495,8 @@ class HaikuRAG:
from haiku.rag.embeddings import embed_chunks
# Convert → Chunk → Embed using primitives
docling_document = await self.convert(content, format=format)
converter = get_converter(self._config)
docling_document = await converter.convert_text(content, format=format)
chunks = await self.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, self._config)
@ -1000,7 +1001,10 @@ class HaikuRAG:
# Content provided without chunks - convert, chunk, and embed using primitives
assert content is not None
existing_doc.content = content
converted_docling = await self.convert(existing_doc.content)
converter = get_converter(self._config)
converted_docling = await converter.convert_text(
existing_doc.content, format="md"
)
existing_doc.set_docling(converted_docling)
new_chunks = await self.chunk(converted_docling)
@ -1558,11 +1562,13 @@ class HaikuRAG:
pending_docs: list[Document] = []
pending_doc_ids: list[str] = []
converter = get_converter(self._config)
for doc in documents:
assert doc.id is not None
# Convert content to DoclingDocument
docling_document = await self.convert(doc.content)
# Convert stored markdown to DoclingDocument
docling_document = await converter.convert_text(doc.content, format="md")
# Chunk and embed
chunks = await self.chunk(docling_document)
@ -1605,6 +1611,7 @@ class HaikuRAG:
pending_chunks: list[Chunk] = []
pending_docs: list[Document] = []
pending_doc_ids: list[str] = []
converter = get_converter(self._config)
for doc in documents:
assert doc.id is not None
@ -1643,7 +1650,7 @@ class HaikuRAG:
"Source missing for %s, re-embedding from content", doc.uri
)
docling_document = await self.convert(doc.content)
docling_document = await converter.convert_text(doc.content, format="md")
chunks = await self.chunk(docling_document)
embedded_chunks = await embed_chunks(chunks, self._config)

View file

@ -1544,3 +1544,78 @@ async def test_sql_injection_is_blocked_with_escaping(temp_db_path):
filter=f"title = '{injection_payload}'"
)
assert len(docs_unescaped) == 2 # SQL injection succeeds without escaping
# =============================================================================
# URL-prefixed content regression tests
# =============================================================================
def _patch_embed_chunks(monkeypatch):
async def fake_embed_chunks(chunks, config):
for chunk in chunks:
chunk.embedding = [0.0] * 2560
return chunks
monkeypatch.setattr("haiku.rag.embeddings.embed_chunks", fake_embed_chunks)
async def test_create_document_with_url_prefixed_content(temp_db_path, monkeypatch):
"""Text whose first line is a URL must be stored as text, not fetched."""
_patch_embed_chunks(monkeypatch)
async with HaikuRAG(temp_db_path, create=True) as client:
content = "https://example.com/foo\n\n# Heading\n\nBody text here."
doc = await client.create_document(content=content, uri="test://url-prefixed")
assert doc.id is not None
assert "example.com" in doc.content
assert "Heading" in doc.content
async def test_update_document_with_url_prefixed_content(temp_db_path, monkeypatch):
"""update_document(content=...) with URL-prefixed text must not fetch it."""
_patch_embed_chunks(monkeypatch)
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document(
content="initial body", uri="test://update-url"
)
assert doc.id is not None
url_prefixed = "https://example.com/bar\n\n# New heading\n\nReplacement body."
updated = await client.update_document(doc.id, content=url_prefixed)
assert "example.com" in updated.content
assert "New heading" in updated.content
async def test_rebuild_rechunk_with_url_prefixed_stored_content(
temp_db_path, monkeypatch
):
"""RECHUNK rebuild must handle stored markdown whose first line is a URL."""
from haiku.rag.client import RebuildMode
_patch_embed_chunks(monkeypatch)
async with HaikuRAG(temp_db_path, create=True) as client:
doc = await client.create_document(
content="plain seed content", uri="file:///nonexistent/path.txt"
)
assert doc.id is not None
# Overwrite stored content to simulate markdown that starts with a URL,
# bypassing the (also-affected) create_document path so this test
# specifically exercises the rebuild path.
doc.content = "https://example.com/baz\n\n# Stored\n\nStored body text."
await client.document_repository.update(doc)
processed_ids = [
doc_id async for doc_id in client.rebuild_database(mode=RebuildMode.RECHUNK)
]
assert doc.id in processed_ids
doc_after = await client.document_repository.get_by_id(doc.id)
assert doc_after is not None
assert "example.com" in doc_after.content
assert "Stored" in doc_after.content