danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Contextualize chunks

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-08-01 16:54:51 +02:00
parent b428cb4c5d
commit ec1bc21a91
No known key found for this signature in database
1 changed files with 3 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/haiku/rag/chunker.py
View file

@ -31,7 +31,7 @@ class Chunker:
tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size
) )
self.docling_chunker = HybridChunker(tokenizer=tokenizer) # type: ignore self.chunker = HybridChunker(tokenizer=tokenizer) # type: ignore
async def chunk(self, text: str) -> list[str]: async def chunk(self, text: str) -> list[str]:
"""Split the text into chunks using docling's structure-aware chunking. """Split the text into chunks using docling's structure-aware chunking.
@ -53,8 +53,8 @@ class Chunker:
doc = result.document doc = result.document
# Chunk using docling's hybrid chunker # Chunk using docling's hybrid chunker
docling_chunks = list(self.docling_chunker.chunk(doc)) chunks = list(self.chunker.chunk(doc))
return [chunk.text for chunk in docling_chunks] return [self.chunker.contextualize(chunk) for chunk in chunks]
chunker = Chunker() chunker = Chunker()