Add DocumentRepository.get_docling_data() for lazy loading docling in expand_context
This commit is contained in:
parent
52712dbdb2
commit
6fdb15e3b2
3 changed files with 82 additions and 11 deletions
|
|
@ -1114,19 +1114,16 @@ class HaikuRAG:
|
|||
expanded_results.extend(doc_results)
|
||||
continue
|
||||
|
||||
# Fetch the document to get DoclingDocument
|
||||
doc = await self.get_document_by_id(doc_id)
|
||||
if doc is None:
|
||||
expanded_results.extend(doc_results)
|
||||
continue
|
||||
|
||||
docling_doc = doc.get_docling_document()
|
||||
|
||||
# Check if we can use DoclingDocument-based expansion
|
||||
has_docling = docling_doc is not None
|
||||
has_refs = any(r.doc_item_refs for r in doc_results)
|
||||
docling_doc = None
|
||||
|
||||
if has_docling and has_refs:
|
||||
if has_refs:
|
||||
# Only load docling data when refs exist (skips content blob)
|
||||
doc = await self.document_repository.get_docling_data(doc_id)
|
||||
if doc is not None:
|
||||
docling_doc = doc.get_docling_document()
|
||||
|
||||
if docling_doc is not None and has_refs:
|
||||
# Use DoclingDocument-based expansion
|
||||
expanded = await self._expand_with_docling(
|
||||
doc_results,
|
||||
|
|
|
|||
|
|
@ -90,6 +90,30 @@ class DocumentRepository:
|
|||
|
||||
return self._record_to_document(results[0])
|
||||
|
||||
_DOCLING_COLUMNS = ["id", "docling_document", "docling_version"]
|
||||
|
||||
async def get_docling_data(self, entity_id: str) -> Document | None:
|
||||
"""Get a document with only docling data loaded (skips content blob)."""
|
||||
safe_id = _escape_sql_string(entity_id)
|
||||
results = list(
|
||||
self.store.documents_table.search()
|
||||
.select(self._DOCLING_COLUMNS)
|
||||
.where(f"id = '{safe_id}'")
|
||||
.limit(1)
|
||||
.to_list()
|
||||
)
|
||||
|
||||
if not results:
|
||||
return None
|
||||
|
||||
row = results[0]
|
||||
return Document(
|
||||
id=row["id"],
|
||||
content="",
|
||||
docling_document=row.get("docling_document"),
|
||||
docling_version=row.get("docling_version"),
|
||||
)
|
||||
|
||||
async def update(self, entity: Document) -> Document:
|
||||
"""Update an existing document."""
|
||||
self.store._assert_writable()
|
||||
|
|
|
|||
|
|
@ -229,6 +229,56 @@ def test_document_get_docling_document_no_id_no_cache():
|
|||
assert doc1 is not doc2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_docling_data_loads_only_docling_columns(
|
||||
qa_corpus: Dataset, temp_db_path
|
||||
):
|
||||
"""get_docling_data returns docling blob without loading content."""
|
||||
import json
|
||||
|
||||
from haiku.rag.store.compression import compress_json
|
||||
|
||||
store = Store(temp_db_path, create=True)
|
||||
doc_repo = DocumentRepository(store)
|
||||
|
||||
doc_json = {
|
||||
"name": "test_doc",
|
||||
"texts": [],
|
||||
"tables": [],
|
||||
"pictures": [],
|
||||
"groups": [],
|
||||
"body": {"self_ref": "#/body", "children": []},
|
||||
"furniture": {"self_ref": "#/furniture", "children": []},
|
||||
}
|
||||
compressed = compress_json(json.dumps(doc_json))
|
||||
|
||||
doc = Document(
|
||||
content=qa_corpus[0]["document_extracted"],
|
||||
uri="https://example.com/doc.txt",
|
||||
docling_document=compressed,
|
||||
docling_version="2.1.0",
|
||||
)
|
||||
created = await doc_repo.create(doc)
|
||||
assert created.id is not None
|
||||
|
||||
result = await doc_repo.get_docling_data(created.id)
|
||||
assert result is not None
|
||||
assert result.id == created.id
|
||||
assert result.content == ""
|
||||
assert result.docling_document == compressed
|
||||
assert result.docling_version == "2.1.0"
|
||||
|
||||
# Verify docling document can be parsed
|
||||
docling_doc = result.get_docling_document()
|
||||
assert docling_doc is not None
|
||||
assert docling_doc.name == "test_doc"
|
||||
|
||||
# Non-existent ID returns None
|
||||
assert await doc_repo.get_docling_data("nonexistent-id") is None
|
||||
|
||||
store.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_document_get_by_uri_with_special_characters(
|
||||
qa_corpus: Dataset, temp_db_path
|
||||
|
|
|
|||
Loading…
Reference in a new issue