compare off-loop work against actual event-loop thread, fix ty
This commit is contained in:
parent
fe954a090e
commit
c6514c9df4
5 changed files with 17 additions and 10 deletions
|
|
@ -307,12 +307,13 @@ async def test_fs_source_fetch_reads_off_event_loop_thread(fs_root: Path):
|
|||
"""The file read and md5 are both proportional to file size and must run
|
||||
off the event-loop thread, or a large file would freeze every other
|
||||
worker's coroutine for the duration of the read. Capture the thread the
|
||||
read+hash runs on and assert it is not the main thread."""
|
||||
read+hash runs on and assert it is not the event-loop thread."""
|
||||
import threading
|
||||
|
||||
src = FSSource(root=fs_root)
|
||||
target = fs_root / "a.md"
|
||||
|
||||
event_loop_thread = threading.current_thread()
|
||||
called_from: list[threading.Thread] = []
|
||||
original = src._read_body
|
||||
|
||||
|
|
@ -320,12 +321,12 @@ async def test_fs_source_fetch_reads_off_event_loop_thread(fs_root: Path):
|
|||
called_from.append(threading.current_thread())
|
||||
return original(path, uri)
|
||||
|
||||
src._read_body = spy # type: ignore[method-assign]
|
||||
src._read_body = spy # type: ignore[method-assign] # ty: ignore[invalid-assignment]
|
||||
|
||||
result = await src.fetch(target.as_uri())
|
||||
assert result.body == b"alpha"
|
||||
assert called_from, "_read_body was never called"
|
||||
assert called_from[0] is not threading.main_thread(), (
|
||||
assert called_from[0] is not event_loop_thread, (
|
||||
"FSSource._read_body ran on the event-loop thread; the read+hash must "
|
||||
"be dispatched via asyncio.to_thread"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -631,6 +631,7 @@ This is content.
|
|||
mock_client.get = AsyncMock(side_effect=[poll_resp, result_resp])
|
||||
mock_client_class.return_value.__aenter__.return_value = mock_client
|
||||
|
||||
event_loop_thread = threading.current_thread()
|
||||
called_from: list[threading.Thread] = []
|
||||
|
||||
class FakeDoc:
|
||||
|
|
@ -642,7 +643,7 @@ This is content.
|
|||
|
||||
assert len(chunks) == 1
|
||||
assert called_from, "model_dump_json was never called"
|
||||
assert called_from[0] is not threading.main_thread(), (
|
||||
assert called_from[0] is not event_loop_thread, (
|
||||
"DoclingDocument.model_dump_json ran on the event-loop thread; it "
|
||||
"must be dispatched via asyncio.to_thread"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -108,21 +108,24 @@ async def test_parse_zip_runs_off_event_loop_thread():
|
|||
converter = get_converter(config)
|
||||
assert isinstance(converter, DoclingServeConverter)
|
||||
|
||||
converter.client.submit_and_poll_zip = AsyncMock(return_value=b"zip-bytes")
|
||||
converter.client.submit_and_poll_zip = AsyncMock( # ty: ignore[invalid-assignment]
|
||||
return_value=b"zip-bytes"
|
||||
)
|
||||
|
||||
event_loop_thread = threading.current_thread()
|
||||
called_from: list[threading.Thread] = []
|
||||
|
||||
def spy(zip_bytes, name):
|
||||
called_from.append(threading.current_thread())
|
||||
return Mock()
|
||||
|
||||
converter._parse_zip_to_docling = spy # type: ignore[method-assign]
|
||||
converter._parse_zip_to_docling = spy # type: ignore[method-assign] # ty: ignore[invalid-assignment]
|
||||
|
||||
files = {"files": ("doc.pdf", b"pdf", "application/octet-stream")}
|
||||
await converter._make_request(files, "doc.pdf")
|
||||
|
||||
assert called_from, "_parse_zip_to_docling was never called"
|
||||
assert called_from[0] is not threading.main_thread(), (
|
||||
assert called_from[0] is not event_loop_thread, (
|
||||
"_parse_zip_to_docling ran on the event-loop thread; it must be "
|
||||
"dispatched via asyncio.to_thread"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -494,7 +494,8 @@ async def test_extract_pdf_attachments_called_off_event_loop_thread(
|
|||
duration of pdfium I/O, stalling every other concurrent worker.
|
||||
|
||||
We verify this by capturing the thread identity inside a spy wrapper: if
|
||||
asyncio.to_thread is used correctly the spy runs on a non-main thread."""
|
||||
asyncio.to_thread is used correctly the spy runs off the event-loop thread."""
|
||||
event_loop_thread = threading.current_thread()
|
||||
called_from: list[threading.Thread] = []
|
||||
|
||||
def spy(body, uri, *, depth):
|
||||
|
|
@ -514,7 +515,7 @@ async def test_extract_pdf_attachments_called_off_event_loop_thread(
|
|||
await _reconcile_pdf_attachments(client, parent, pdf_bytes, depth=0)
|
||||
|
||||
assert called_from, "_extract_pdf_attachments was never called"
|
||||
assert called_from[0] is not threading.main_thread(), (
|
||||
assert called_from[0] is not event_loop_thread, (
|
||||
"_extract_pdf_attachments ran on the event-loop thread; "
|
||||
"it must be dispatched via asyncio.to_thread to avoid blocking the loop"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -174,6 +174,7 @@ async def test_concatenate_runs_off_event_loop_thread(tmp_path, monkeypatch):
|
|||
async def convert_file(self, path: Path, *, source_uri):
|
||||
return DoclingDocument(name="slice")
|
||||
|
||||
event_loop_thread = threading.current_thread()
|
||||
called_from: list[threading.Thread] = []
|
||||
|
||||
def spy(docs):
|
||||
|
|
@ -193,7 +194,7 @@ async def test_concatenate_runs_off_event_loop_thread(tmp_path, monkeypatch):
|
|||
)
|
||||
|
||||
assert called_from, "concatenate was never called"
|
||||
assert called_from[0] is not threading.main_thread(), (
|
||||
assert called_from[0] is not event_loop_thread, (
|
||||
"DoclingDocument.concatenate ran on the event-loop thread; it must be "
|
||||
"dispatched via asyncio.to_thread"
|
||||
)
|
||||
|
|
|
|||
Loading…
Reference in a new issue