From ac439b6c6966fd1167749824afd2d2441a9099ef Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 1 Dec 2025 14:21:12 +0200 Subject: [PATCH] Store raw text (do not contextualize) in chunk.text. Embed with contextualized text. --- CHANGELOG.md | 9 +++++++-- haiku_rag_slim/haiku/rag/chunkers/docling_local.py | 4 +++- haiku_rag_slim/haiku/rag/store/repositories/chunk.py | 12 ++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a9e373..19f6f1f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,9 +32,14 @@ - **BREAKING: Chunker Interface**: `DocumentChunker.chunk()` now returns `list[ChunkWithMetadata]` instead of `list[str]` - `ChunkWithMetadata` combines chunk text with `ChunkMetadata` (refs, labels, headings, page_numbers) - All chunker implementations updated: `DoclingLocalChunker`, `DoclingServeChunker` -- **Page Image Generation**: `generate_page_images=True` is now the default for local docling converter +- **Page Image Generation**: `generate_page_images=True` is now always enabled for local docling converter - Required for visual grounding features - - docling-serve already generates page images by default + - Removed `generate_page_images` config option (docling-serve already generates page images by default) +- **Chunk Text Storage**: Chunks now store raw text without heading contextualization + - Section headings are prepended only at embedding time for better semantic search + - Stored chunk content stays clean without duplicate heading prefixes + - Headings remain available in `ChunkMetadata` for display and citations + - Local and serve chunkers now produce identical output - **QA Prompts**: Updated to use page numbers and section headings in citations when available ### Migration diff --git a/haiku_rag_slim/haiku/rag/chunkers/docling_local.py b/haiku_rag_slim/haiku/rag/chunkers/docling_local.py index 31d05d74..7e40e9a3 100644 --- a/haiku_rag_slim/haiku/rag/chunkers/docling_local.py +++ b/haiku_rag_slim/haiku/rag/chunkers/docling_local.py @@ -117,7 +117,9 @@ class DoclingLocalChunker(DocumentChunker): result: list[ChunkWithMetadata] = [] for chunk in raw_chunks: - text = self.chunker.contextualize(chunk) + # Use raw chunk text - headings are stored separately in metadata + # and prepended at embedding time for better semantic search + text = chunk.text # Extract metadata from DocChunk.meta (cast to DocMeta for type safety) doc_item_refs: list[str] = [] diff --git a/haiku_rag_slim/haiku/rag/store/repositories/chunk.py b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py index 11055b5a..1e79f3bb 100644 --- a/haiku_rag_slim/haiku/rag/store/repositories/chunk.py +++ b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py @@ -227,8 +227,16 @@ class ChunkRepository: chunks_with_metadata = await chunker.chunk(processed_document) - chunk_texts = [c.text for c in chunks_with_metadata] - embeddings = await self.embedder.embed(chunk_texts) + # Build embedding texts with headings prepended for better semantic search + # The stored content stays raw, but embeddings capture section context + embedding_texts = [] + for c in chunks_with_metadata: + if c.metadata.headings: + embedding_text = "\n".join(c.metadata.headings) + "\n" + c.text + else: + embedding_text = c.text + embedding_texts.append(embedding_text) + embeddings = await self.embedder.embed(embedding_texts) # Prepare all chunk records for batch insertion chunk_records = []