Restore None guard, add thread-safety test for _chunk_sync
This commit is contained in:
parent
25b0a155b0
commit
99a200c9d2
2 changed files with 31 additions and 0 deletions
|
|
@ -177,4 +177,7 @@ class DoclingLocalChunker(DocumentChunker):
|
|||
Returns:
|
||||
List of Chunk containing content and structured metadata.
|
||||
"""
|
||||
if document is None:
|
||||
return []
|
||||
|
||||
return await asyncio.to_thread(self._chunk_sync, document)
|
||||
|
|
|
|||
|
|
@ -55,6 +55,34 @@ async def test_local_chunker(qa_corpus: list[dict[str, str]]):
|
|||
assert abs(total_tokens - original_tokens) <= original_tokens * 0.1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_chunker_runs_off_event_loop_thread():
|
||||
"""Chunking is CPU-bound; verify it runs in a worker thread."""
|
||||
import threading
|
||||
|
||||
chunker = DoclingLocalChunker()
|
||||
event_loop_thread = threading.current_thread()
|
||||
called_from: list[threading.Thread] = []
|
||||
|
||||
original = chunker._chunk_sync
|
||||
|
||||
def recording_chunk_sync(document):
|
||||
called_from.append(threading.current_thread())
|
||||
return original(document)
|
||||
|
||||
chunker._chunk_sync = recording_chunk_sync
|
||||
|
||||
converter = get_converter(Config)
|
||||
doc = await converter.convert_text("# Hello\n\nWorld", name="test.md")
|
||||
await chunker.chunk(doc)
|
||||
|
||||
assert called_from, "_chunk_sync was never called"
|
||||
assert called_from[0] is not event_loop_thread, (
|
||||
"_chunk_sync ran on the event-loop thread; "
|
||||
"it must be dispatched via asyncio.to_thread"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_chunker_custom_config():
|
||||
"""Test DoclingLocalChunker with custom configuration."""
|
||||
|
|
|
|||
Loading…
Reference in a new issue