Restore None guard, add thread-safety test for _chunk_sync
This commit is contained in:
parent
25b0a155b0
commit
99a200c9d2
2 changed files with 31 additions and 0 deletions
|
|
@ -177,4 +177,7 @@ class DoclingLocalChunker(DocumentChunker):
|
||||||
Returns:
|
Returns:
|
||||||
List of Chunk containing content and structured metadata.
|
List of Chunk containing content and structured metadata.
|
||||||
"""
|
"""
|
||||||
|
if document is None:
|
||||||
|
return []
|
||||||
|
|
||||||
return await asyncio.to_thread(self._chunk_sync, document)
|
return await asyncio.to_thread(self._chunk_sync, document)
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,34 @@ async def test_local_chunker(qa_corpus: list[dict[str, str]]):
|
||||||
assert abs(total_tokens - original_tokens) <= original_tokens * 0.1
|
assert abs(total_tokens - original_tokens) <= original_tokens * 0.1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_local_chunker_runs_off_event_loop_thread():
|
||||||
|
"""Chunking is CPU-bound; verify it runs in a worker thread."""
|
||||||
|
import threading
|
||||||
|
|
||||||
|
chunker = DoclingLocalChunker()
|
||||||
|
event_loop_thread = threading.current_thread()
|
||||||
|
called_from: list[threading.Thread] = []
|
||||||
|
|
||||||
|
original = chunker._chunk_sync
|
||||||
|
|
||||||
|
def recording_chunk_sync(document):
|
||||||
|
called_from.append(threading.current_thread())
|
||||||
|
return original(document)
|
||||||
|
|
||||||
|
chunker._chunk_sync = recording_chunk_sync
|
||||||
|
|
||||||
|
converter = get_converter(Config)
|
||||||
|
doc = await converter.convert_text("# Hello\n\nWorld", name="test.md")
|
||||||
|
await chunker.chunk(doc)
|
||||||
|
|
||||||
|
assert called_from, "_chunk_sync was never called"
|
||||||
|
assert called_from[0] is not event_loop_thread, (
|
||||||
|
"_chunk_sync ran on the event-loop thread; "
|
||||||
|
"it must be dispatched via asyncio.to_thread"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_local_chunker_custom_config():
|
async def test_local_chunker_custom_config():
|
||||||
"""Test DoclingLocalChunker with custom configuration."""
|
"""Test DoclingLocalChunker with custom configuration."""
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue