fix convert() misreading text content that starts with a URL
This commit is contained in:
parent
f0016ebcd2
commit
9067b89d2f
3 changed files with 91 additions and 5 deletions
|
|
@ -1,6 +1,10 @@
|
|||
# Changelog
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`create_document`, `update_document`, and rebuild (`RECHUNK` / full fallback) no longer misread URL-prefixed text as a URL to fetch.** These paths passed known-text content through `HaikuRAG.convert()`, which dispatches on `urlparse(source).scheme`; text whose first line was `https://...` (common for clipped web pages and notes) got handed to `httpx.get` and crashed with `httpx.InvalidURL` on embedded whitespace. Fixed by calling `converter.convert_text(...)` directly at those sites; `convert()` itself is unchanged for `create_document_from_source`.
|
||||
|
||||
### Changed
|
||||
|
||||
- **Skills share a single `HaikuRAG` client per invocation** via the new `haiku.skills>=0.15.0` `lifespan` hook. The skill's sub-agent opens one read-only client on entry, all tool calls reuse it, and it closes on exit — replacing the old pattern of open/close around every `search` / `list_documents` / `get_document` call.
|
||||
|
|
|
|||
|
|
@ -495,7 +495,8 @@ class HaikuRAG:
|
|||
from haiku.rag.embeddings import embed_chunks
|
||||
|
||||
# Convert → Chunk → Embed using primitives
|
||||
docling_document = await self.convert(content, format=format)
|
||||
converter = get_converter(self._config)
|
||||
docling_document = await converter.convert_text(content, format=format)
|
||||
chunks = await self.chunk(docling_document)
|
||||
embedded_chunks = await embed_chunks(chunks, self._config)
|
||||
|
||||
|
|
@ -1000,7 +1001,10 @@ class HaikuRAG:
|
|||
# Content provided without chunks - convert, chunk, and embed using primitives
|
||||
assert content is not None
|
||||
existing_doc.content = content
|
||||
converted_docling = await self.convert(existing_doc.content)
|
||||
converter = get_converter(self._config)
|
||||
converted_docling = await converter.convert_text(
|
||||
existing_doc.content, format="md"
|
||||
)
|
||||
existing_doc.set_docling(converted_docling)
|
||||
|
||||
new_chunks = await self.chunk(converted_docling)
|
||||
|
|
@ -1558,11 +1562,13 @@ class HaikuRAG:
|
|||
pending_docs: list[Document] = []
|
||||
pending_doc_ids: list[str] = []
|
||||
|
||||
converter = get_converter(self._config)
|
||||
|
||||
for doc in documents:
|
||||
assert doc.id is not None
|
||||
|
||||
# Convert content to DoclingDocument
|
||||
docling_document = await self.convert(doc.content)
|
||||
# Convert stored markdown to DoclingDocument
|
||||
docling_document = await converter.convert_text(doc.content, format="md")
|
||||
|
||||
# Chunk and embed
|
||||
chunks = await self.chunk(docling_document)
|
||||
|
|
@ -1605,6 +1611,7 @@ class HaikuRAG:
|
|||
pending_chunks: list[Chunk] = []
|
||||
pending_docs: list[Document] = []
|
||||
pending_doc_ids: list[str] = []
|
||||
converter = get_converter(self._config)
|
||||
|
||||
for doc in documents:
|
||||
assert doc.id is not None
|
||||
|
|
@ -1643,7 +1650,7 @@ class HaikuRAG:
|
|||
"Source missing for %s, re-embedding from content", doc.uri
|
||||
)
|
||||
|
||||
docling_document = await self.convert(doc.content)
|
||||
docling_document = await converter.convert_text(doc.content, format="md")
|
||||
chunks = await self.chunk(docling_document)
|
||||
embedded_chunks = await embed_chunks(chunks, self._config)
|
||||
|
||||
|
|
|
|||
|
|
@ -1544,3 +1544,78 @@ async def test_sql_injection_is_blocked_with_escaping(temp_db_path):
|
|||
filter=f"title = '{injection_payload}'"
|
||||
)
|
||||
assert len(docs_unescaped) == 2 # SQL injection succeeds without escaping
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# URL-prefixed content regression tests
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _patch_embed_chunks(monkeypatch):
|
||||
async def fake_embed_chunks(chunks, config):
|
||||
for chunk in chunks:
|
||||
chunk.embedding = [0.0] * 2560
|
||||
return chunks
|
||||
|
||||
monkeypatch.setattr("haiku.rag.embeddings.embed_chunks", fake_embed_chunks)
|
||||
|
||||
|
||||
async def test_create_document_with_url_prefixed_content(temp_db_path, monkeypatch):
|
||||
"""Text whose first line is a URL must be stored as text, not fetched."""
|
||||
_patch_embed_chunks(monkeypatch)
|
||||
|
||||
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||
content = "https://example.com/foo\n\n# Heading\n\nBody text here."
|
||||
doc = await client.create_document(content=content, uri="test://url-prefixed")
|
||||
|
||||
assert doc.id is not None
|
||||
assert "example.com" in doc.content
|
||||
assert "Heading" in doc.content
|
||||
|
||||
|
||||
async def test_update_document_with_url_prefixed_content(temp_db_path, monkeypatch):
|
||||
"""update_document(content=...) with URL-prefixed text must not fetch it."""
|
||||
_patch_embed_chunks(monkeypatch)
|
||||
|
||||
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||
doc = await client.create_document(
|
||||
content="initial body", uri="test://update-url"
|
||||
)
|
||||
assert doc.id is not None
|
||||
|
||||
url_prefixed = "https://example.com/bar\n\n# New heading\n\nReplacement body."
|
||||
updated = await client.update_document(doc.id, content=url_prefixed)
|
||||
|
||||
assert "example.com" in updated.content
|
||||
assert "New heading" in updated.content
|
||||
|
||||
|
||||
async def test_rebuild_rechunk_with_url_prefixed_stored_content(
|
||||
temp_db_path, monkeypatch
|
||||
):
|
||||
"""RECHUNK rebuild must handle stored markdown whose first line is a URL."""
|
||||
from haiku.rag.client import RebuildMode
|
||||
|
||||
_patch_embed_chunks(monkeypatch)
|
||||
|
||||
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||
doc = await client.create_document(
|
||||
content="plain seed content", uri="file:///nonexistent/path.txt"
|
||||
)
|
||||
assert doc.id is not None
|
||||
|
||||
# Overwrite stored content to simulate markdown that starts with a URL,
|
||||
# bypassing the (also-affected) create_document path so this test
|
||||
# specifically exercises the rebuild path.
|
||||
doc.content = "https://example.com/baz\n\n# Stored\n\nStored body text."
|
||||
await client.document_repository.update(doc)
|
||||
|
||||
processed_ids = [
|
||||
doc_id async for doc_id in client.rebuild_database(mode=RebuildMode.RECHUNK)
|
||||
]
|
||||
assert doc.id in processed_ids
|
||||
|
||||
doc_after = await client.document_repository.get_by_id(doc.id)
|
||||
assert doc_after is not None
|
||||
assert "example.com" in doc_after.content
|
||||
assert "Stored" in doc_after.content
|
||||
|
|
|
|||
Loading…
Reference in a new issue