Restore None guard, add thread-safety test for _chunk_sync

This commit is contained in:
Chris McDonough 2026-06-22 14:05:56 -04:00
parent 25b0a155b0
commit 99a200c9d2
2 changed files with 31 additions and 0 deletions

View file

@ -177,4 +177,7 @@ class DoclingLocalChunker(DocumentChunker):
Returns:
List of Chunk containing content and structured metadata.
"""
if document is None:
return []
return await asyncio.to_thread(self._chunk_sync, document)

View file

@ -55,6 +55,34 @@ async def test_local_chunker(qa_corpus: list[dict[str, str]]):
assert abs(total_tokens - original_tokens) <= original_tokens * 0.1
@pytest.mark.asyncio
async def test_local_chunker_runs_off_event_loop_thread():
"""Chunking is CPU-bound; verify it runs in a worker thread."""
import threading
chunker = DoclingLocalChunker()
event_loop_thread = threading.current_thread()
called_from: list[threading.Thread] = []
original = chunker._chunk_sync
def recording_chunk_sync(document):
called_from.append(threading.current_thread())
return original(document)
chunker._chunk_sync = recording_chunk_sync
converter = get_converter(Config)
doc = await converter.convert_text("# Hello\n\nWorld", name="test.md")
await chunker.chunk(doc)
assert called_from, "_chunk_sync was never called"
assert called_from[0] is not event_loop_thread, (
"_chunk_sync ran on the event-loop thread; "
"it must be dispatched via asyncio.to_thread"
)
@pytest.mark.asyncio
async def test_local_chunker_custom_config():
"""Test DoclingLocalChunker with custom configuration."""