Widen chunk() type to accept None, add test for None guard
This commit is contained in:
parent
21a8f52893
commit
633517bf04
4 changed files with 10 additions and 3 deletions
|
|
@ -15,7 +15,7 @@ class DocumentChunker(ABC):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def chunk(self, document: "DoclingDocument") -> list["Chunk"]:
|
async def chunk(self, document: "DoclingDocument | None") -> list["Chunk"]:
|
||||||
"""Split a document into chunks with metadata.
|
"""Split a document into chunks with metadata.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
|
|
|
||||||
|
|
@ -162,7 +162,7 @@ class DoclingLocalChunker(DocumentChunker):
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
async def chunk(self, document: "DoclingDocument") -> list[Chunk]:
|
async def chunk(self, document: "DoclingDocument | None") -> list[Chunk]:
|
||||||
"""Split the document into chunks with metadata.
|
"""Split the document into chunks with metadata.
|
||||||
|
|
||||||
Extracts structured metadata from each DocChunk including:
|
Extracts structured metadata from each DocChunk including:
|
||||||
|
|
|
||||||
|
|
@ -129,7 +129,7 @@ class DoclingServeChunker(DocumentChunker):
|
||||||
|
|
||||||
return result.get("chunks", [])
|
return result.get("chunks", [])
|
||||||
|
|
||||||
async def chunk(self, document: "DoclingDocument") -> list[Chunk]:
|
async def chunk(self, document: "DoclingDocument | None") -> list[Chunk]:
|
||||||
"""Split the document into chunks with metadata via docling-serve.
|
"""Split the document into chunks with metadata via docling-serve.
|
||||||
|
|
||||||
Extracts structured metadata from the API response including:
|
Extracts structured metadata from the API response including:
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,13 @@ async def test_local_chunker(qa_corpus: list[dict[str, str]]):
|
||||||
assert abs(total_tokens - original_tokens) <= original_tokens * 0.1
|
assert abs(total_tokens - original_tokens) <= original_tokens * 0.1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_local_chunker_none_document():
|
||||||
|
"""Test DoclingLocalChunker returns empty list for None document."""
|
||||||
|
chunker = DoclingLocalChunker()
|
||||||
|
assert await chunker.chunk(None) == []
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_local_chunker_runs_off_event_loop_thread():
|
async def test_local_chunker_runs_off_event_loop_thread():
|
||||||
"""Chunking is CPU-bound; verify it runs in a worker thread."""
|
"""Chunking is CPU-bound; verify it runs in a worker thread."""
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue