detect per-document failure status in docling-serve chunker
This commit is contained in:
parent
2e0564d7ea
commit
a3a1ad9811
2 changed files with 64 additions and 0 deletions
|
|
@ -114,6 +114,14 @@ class DoclingServeChunker(DocumentChunker):
|
||||||
name="document",
|
name="document",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Task-level polling status can be "success" while individual documents
|
||||||
|
# report "failure" (e.g. schema version mismatch), returning 0 chunks silently.
|
||||||
|
documents = result.get("documents", [])
|
||||||
|
for doc_result in documents:
|
||||||
|
if doc_result.get("status") not in ("success", "partial_success", None):
|
||||||
|
errors = doc_result.get("errors", [])
|
||||||
|
raise ValueError(f"Chunking failed: {errors}")
|
||||||
|
|
||||||
return result.get("chunks", [])
|
return result.get("chunks", [])
|
||||||
|
|
||||||
async def chunk(self, document: "DoclingDocument") -> list[Chunk]:
|
async def chunk(self, document: "DoclingDocument") -> list[Chunk]:
|
||||||
|
|
|
||||||
|
|
@ -464,6 +464,62 @@ class TestDoclingServeChunker:
|
||||||
with pytest.raises(ValueError, match="Authentication failed"):
|
with pytest.raises(ValueError, match="Authentication failed"):
|
||||||
await chunker.chunk(doc)
|
await chunker.chunk(doc)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("haiku.rag.providers.docling_serve.httpx.AsyncClient")
|
||||||
|
async def test_chunk_document_failure_status(self, mock_client_class, chunker):
|
||||||
|
"""Test that document-level failure status raises ValueError."""
|
||||||
|
result_data = {
|
||||||
|
"chunks": [],
|
||||||
|
"documents": [
|
||||||
|
{
|
||||||
|
"kind": "ExportResult",
|
||||||
|
"status": "failure",
|
||||||
|
"errors": ["Schema version mismatch"],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(result_data)
|
||||||
|
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||||
|
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||||
|
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||||
|
|
||||||
|
converter = get_converter(Config)
|
||||||
|
doc = await converter.convert_text("# Test", name="test.md")
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="Chunking failed"):
|
||||||
|
await chunker.chunk(doc)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@patch("haiku.rag.providers.docling_serve.httpx.AsyncClient")
|
||||||
|
async def test_chunk_document_success_empty_chunks(
|
||||||
|
self, mock_client_class, chunker
|
||||||
|
):
|
||||||
|
"""Test that successful status with empty chunks returns empty list."""
|
||||||
|
result_data = {
|
||||||
|
"chunks": [],
|
||||||
|
"documents": [
|
||||||
|
{
|
||||||
|
"kind": "ExportResult",
|
||||||
|
"status": "success",
|
||||||
|
"errors": [],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
submit_resp, poll_resp, result_resp = create_async_workflow_mocks(result_data)
|
||||||
|
|
||||||
|
mock_client = AsyncMock()
|
||||||
|
mock_client.post = AsyncMock(return_value=submit_resp)
|
||||||
|
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||||
|
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||||
|
|
||||||
|
converter = get_converter(Config)
|
||||||
|
doc = await converter.convert_text("# Test", name="test.md")
|
||||||
|
|
||||||
|
chunks = await chunker.chunk(doc)
|
||||||
|
assert chunks == []
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@patch("haiku.rag.providers.docling_serve.httpx.AsyncClient")
|
@patch("haiku.rag.providers.docling_serve.httpx.AsyncClient")
|
||||||
async def test_chunk_metadata_extraction(self, mock_client_class, chunker):
|
async def test_chunk_metadata_extraction(self, mock_client_class, chunker):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue