fix convert() misreading text content that starts with a URL
This commit is contained in:
parent
f0016ebcd2
commit
9067b89d2f
3 changed files with 91 additions and 5 deletions
|
|
@ -1,6 +1,10 @@
|
||||||
# Changelog
|
# Changelog
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- **`create_document`, `update_document`, and rebuild (`RECHUNK` / full fallback) no longer misread URL-prefixed text as a URL to fetch.** These paths passed known-text content through `HaikuRAG.convert()`, which dispatches on `urlparse(source).scheme`; text whose first line was `https://...` (common for clipped web pages and notes) got handed to `httpx.get` and crashed with `httpx.InvalidURL` on embedded whitespace. Fixed by calling `converter.convert_text(...)` directly at those sites; `convert()` itself is unchanged for `create_document_from_source`.
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- **Skills share a single `HaikuRAG` client per invocation** via the new `haiku.skills>=0.15.0` `lifespan` hook. The skill's sub-agent opens one read-only client on entry, all tool calls reuse it, and it closes on exit — replacing the old pattern of open/close around every `search` / `list_documents` / `get_document` call.
|
- **Skills share a single `HaikuRAG` client per invocation** via the new `haiku.skills>=0.15.0` `lifespan` hook. The skill's sub-agent opens one read-only client on entry, all tool calls reuse it, and it closes on exit — replacing the old pattern of open/close around every `search` / `list_documents` / `get_document` call.
|
||||||
|
|
|
||||||
|
|
@ -495,7 +495,8 @@ class HaikuRAG:
|
||||||
from haiku.rag.embeddings import embed_chunks
|
from haiku.rag.embeddings import embed_chunks
|
||||||
|
|
||||||
# Convert → Chunk → Embed using primitives
|
# Convert → Chunk → Embed using primitives
|
||||||
docling_document = await self.convert(content, format=format)
|
converter = get_converter(self._config)
|
||||||
|
docling_document = await converter.convert_text(content, format=format)
|
||||||
chunks = await self.chunk(docling_document)
|
chunks = await self.chunk(docling_document)
|
||||||
embedded_chunks = await embed_chunks(chunks, self._config)
|
embedded_chunks = await embed_chunks(chunks, self._config)
|
||||||
|
|
||||||
|
|
@ -1000,7 +1001,10 @@ class HaikuRAG:
|
||||||
# Content provided without chunks - convert, chunk, and embed using primitives
|
# Content provided without chunks - convert, chunk, and embed using primitives
|
||||||
assert content is not None
|
assert content is not None
|
||||||
existing_doc.content = content
|
existing_doc.content = content
|
||||||
converted_docling = await self.convert(existing_doc.content)
|
converter = get_converter(self._config)
|
||||||
|
converted_docling = await converter.convert_text(
|
||||||
|
existing_doc.content, format="md"
|
||||||
|
)
|
||||||
existing_doc.set_docling(converted_docling)
|
existing_doc.set_docling(converted_docling)
|
||||||
|
|
||||||
new_chunks = await self.chunk(converted_docling)
|
new_chunks = await self.chunk(converted_docling)
|
||||||
|
|
@ -1558,11 +1562,13 @@ class HaikuRAG:
|
||||||
pending_docs: list[Document] = []
|
pending_docs: list[Document] = []
|
||||||
pending_doc_ids: list[str] = []
|
pending_doc_ids: list[str] = []
|
||||||
|
|
||||||
|
converter = get_converter(self._config)
|
||||||
|
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
assert doc.id is not None
|
assert doc.id is not None
|
||||||
|
|
||||||
# Convert content to DoclingDocument
|
# Convert stored markdown to DoclingDocument
|
||||||
docling_document = await self.convert(doc.content)
|
docling_document = await converter.convert_text(doc.content, format="md")
|
||||||
|
|
||||||
# Chunk and embed
|
# Chunk and embed
|
||||||
chunks = await self.chunk(docling_document)
|
chunks = await self.chunk(docling_document)
|
||||||
|
|
@ -1605,6 +1611,7 @@ class HaikuRAG:
|
||||||
pending_chunks: list[Chunk] = []
|
pending_chunks: list[Chunk] = []
|
||||||
pending_docs: list[Document] = []
|
pending_docs: list[Document] = []
|
||||||
pending_doc_ids: list[str] = []
|
pending_doc_ids: list[str] = []
|
||||||
|
converter = get_converter(self._config)
|
||||||
|
|
||||||
for doc in documents:
|
for doc in documents:
|
||||||
assert doc.id is not None
|
assert doc.id is not None
|
||||||
|
|
@ -1643,7 +1650,7 @@ class HaikuRAG:
|
||||||
"Source missing for %s, re-embedding from content", doc.uri
|
"Source missing for %s, re-embedding from content", doc.uri
|
||||||
)
|
)
|
||||||
|
|
||||||
docling_document = await self.convert(doc.content)
|
docling_document = await converter.convert_text(doc.content, format="md")
|
||||||
chunks = await self.chunk(docling_document)
|
chunks = await self.chunk(docling_document)
|
||||||
embedded_chunks = await embed_chunks(chunks, self._config)
|
embedded_chunks = await embed_chunks(chunks, self._config)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1544,3 +1544,78 @@ async def test_sql_injection_is_blocked_with_escaping(temp_db_path):
|
||||||
filter=f"title = '{injection_payload}'"
|
filter=f"title = '{injection_payload}'"
|
||||||
)
|
)
|
||||||
assert len(docs_unescaped) == 2 # SQL injection succeeds without escaping
|
assert len(docs_unescaped) == 2 # SQL injection succeeds without escaping
|
||||||
|
|
||||||
|
|
||||||
|
# =============================================================================
|
||||||
|
# URL-prefixed content regression tests
|
||||||
|
# =============================================================================
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_embed_chunks(monkeypatch):
|
||||||
|
async def fake_embed_chunks(chunks, config):
|
||||||
|
for chunk in chunks:
|
||||||
|
chunk.embedding = [0.0] * 2560
|
||||||
|
return chunks
|
||||||
|
|
||||||
|
monkeypatch.setattr("haiku.rag.embeddings.embed_chunks", fake_embed_chunks)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_create_document_with_url_prefixed_content(temp_db_path, monkeypatch):
|
||||||
|
"""Text whose first line is a URL must be stored as text, not fetched."""
|
||||||
|
_patch_embed_chunks(monkeypatch)
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
content = "https://example.com/foo\n\n# Heading\n\nBody text here."
|
||||||
|
doc = await client.create_document(content=content, uri="test://url-prefixed")
|
||||||
|
|
||||||
|
assert doc.id is not None
|
||||||
|
assert "example.com" in doc.content
|
||||||
|
assert "Heading" in doc.content
|
||||||
|
|
||||||
|
|
||||||
|
async def test_update_document_with_url_prefixed_content(temp_db_path, monkeypatch):
|
||||||
|
"""update_document(content=...) with URL-prefixed text must not fetch it."""
|
||||||
|
_patch_embed_chunks(monkeypatch)
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.create_document(
|
||||||
|
content="initial body", uri="test://update-url"
|
||||||
|
)
|
||||||
|
assert doc.id is not None
|
||||||
|
|
||||||
|
url_prefixed = "https://example.com/bar\n\n# New heading\n\nReplacement body."
|
||||||
|
updated = await client.update_document(doc.id, content=url_prefixed)
|
||||||
|
|
||||||
|
assert "example.com" in updated.content
|
||||||
|
assert "New heading" in updated.content
|
||||||
|
|
||||||
|
|
||||||
|
async def test_rebuild_rechunk_with_url_prefixed_stored_content(
|
||||||
|
temp_db_path, monkeypatch
|
||||||
|
):
|
||||||
|
"""RECHUNK rebuild must handle stored markdown whose first line is a URL."""
|
||||||
|
from haiku.rag.client import RebuildMode
|
||||||
|
|
||||||
|
_patch_embed_chunks(monkeypatch)
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.create_document(
|
||||||
|
content="plain seed content", uri="file:///nonexistent/path.txt"
|
||||||
|
)
|
||||||
|
assert doc.id is not None
|
||||||
|
|
||||||
|
# Overwrite stored content to simulate markdown that starts with a URL,
|
||||||
|
# bypassing the (also-affected) create_document path so this test
|
||||||
|
# specifically exercises the rebuild path.
|
||||||
|
doc.content = "https://example.com/baz\n\n# Stored\n\nStored body text."
|
||||||
|
await client.document_repository.update(doc)
|
||||||
|
|
||||||
|
processed_ids = [
|
||||||
|
doc_id async for doc_id in client.rebuild_database(mode=RebuildMode.RECHUNK)
|
||||||
|
]
|
||||||
|
assert doc.id in processed_ids
|
||||||
|
|
||||||
|
doc_after = await client.document_repository.get_by_id(doc.id)
|
||||||
|
assert doc_after is not None
|
||||||
|
assert "example.com" in doc_after.content
|
||||||
|
assert "Stored" in doc_after.content
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue