danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Keep using the openai tokenizer

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-08-01 16:36:24 +02:00
parent 91f6122ef7
commit a453853211
No known key found for this signature in database
1 changed files with 6 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/haiku/rag/chunker.py
View file

@ -4,6 +4,7 @@ from typing import ClassVar
import tiktoken
from docling.chunking import HybridChunker # type: ignore
from docling.document_converter import DocumentConverter
from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
from docling_core.types.io import DocumentStream
from haiku.rag.config import Config
@ -26,7 +27,11 @@ class Chunker:
chunk_size: int = Config.CHUNK_SIZE,
):
self.chunk_size = chunk_size
self.docling_chunker = HybridChunker(max_tokens=chunk_size) # type: ignore
tokenizer = OpenAITokenizer(
tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size
)
self.docling_chunker = HybridChunker(tokenizer=tokenizer) # type: ignore
async def chunk(self, text: str) -> list[str]:
"""Split the text into chunks using docling's structure-aware chunking.