danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Narrow AutoTokenizer type for transformers 5

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-07-23 12:01:28 +03:00
parent c9466af986
commit d9cc9a4eea
No known key found for this signature in database
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
tests/test_chunker.py
View file

@ -33,6 +33,7 @@ async def test_local_chunker(qa_corpus: list[dict[str, str]]):
# Load tokenizer for verification # Load tokenizer for verification
tokenizer = AutoTokenizer.from_pretrained(chunker.tokenizer_name) tokenizer = AutoTokenizer.from_pretrained(chunker.tokenizer_name)
assert tokenizer is not None
# Ensure that chunks are reasonably sized (allowing more flexibility for structure-aware chunking) # Ensure that chunks are reasonably sized (allowing more flexibility for structure-aware chunking)
total_tokens = 0 total_tokens = 0
@ -74,7 +75,7 @@ async def test_local_chunker_runs_off_event_loop_thread():
original = chunker._chunk_sync original = chunker._chunk_sync
def recording_chunk_sync(self, document): def recording_chunk_sync(_self, document):
called_from.append(threading.current_thread()) called_from.append(threading.current_thread())
return original(document) return original(document)