From ec1bc21a91d3718b50ca847d550779ebd535e6d5 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 1 Aug 2025 16:54:51 +0200 Subject: [PATCH] Contextualize chunks --- src/haiku/rag/chunker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/haiku/rag/chunker.py b/src/haiku/rag/chunker.py index c7157763..ab29f749 100644 --- a/src/haiku/rag/chunker.py +++ b/src/haiku/rag/chunker.py @@ -31,7 +31,7 @@ class Chunker: tokenizer=tiktoken.encoding_for_model("gpt-4o"), max_tokens=chunk_size ) - self.docling_chunker = HybridChunker(tokenizer=tokenizer) # type: ignore + self.chunker = HybridChunker(tokenizer=tokenizer) # type: ignore async def chunk(self, text: str) -> list[str]: """Split the text into chunks using docling's structure-aware chunking. @@ -53,8 +53,8 @@ class Chunker: doc = result.document # Chunk using docling's hybrid chunker - docling_chunks = list(self.docling_chunker.chunk(doc)) - return [chunk.text for chunk in docling_chunks] + chunks = list(self.chunker.chunk(doc)) + return [self.chunker.contextualize(chunk) for chunk in chunks] chunker = Chunker()