From d9cc9a4eea3f6e85d30621a64608c735b9e6abba Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 23 Jul 2026 12:01:28 +0300 Subject: [PATCH] Narrow AutoTokenizer type for transformers 5 --- tests/test_chunker.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index c1f1a833..b795bdea 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -33,6 +33,7 @@ async def test_local_chunker(qa_corpus: list[dict[str, str]]): # Load tokenizer for verification tokenizer = AutoTokenizer.from_pretrained(chunker.tokenizer_name) + assert tokenizer is not None # Ensure that chunks are reasonably sized (allowing more flexibility for structure-aware chunking) total_tokens = 0 @@ -74,7 +75,7 @@ async def test_local_chunker_runs_off_event_loop_thread(): original = chunker._chunk_sync - def recording_chunk_sync(self, document): + def recording_chunk_sync(_self, document): called_from.append(threading.current_thread()) return original(document)