diff --git a/CHANGELOG.md b/CHANGELOG.md index bb0a1ba6..2defdf9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ ### Changed +- **Chunker Sets Order**: Chunkers now set `chunk.order` directly - **Evaluations Vacuum Strategy**: `populate_db` now uses periodic vacuum to prevent disk exhaustion with large datasets - Disables auto_vacuum during population, vacuums every N documents with retention=0 - New `--vacuum-interval` CLI option (default: 100) to control vacuum frequency diff --git a/haiku_rag_slim/haiku/rag/chunkers/docling_local.py b/haiku_rag_slim/haiku/rag/chunkers/docling_local.py index 74f5c19f..1bebbe07 100644 --- a/haiku_rag_slim/haiku/rag/chunkers/docling_local.py +++ b/haiku_rag_slim/haiku/rag/chunkers/docling_local.py @@ -155,6 +155,12 @@ class DoclingLocalChunker(DocumentChunker): labels=labels, page_numbers=sorted(page_numbers), ) - result.append(Chunk(content=text, metadata=chunk_metadata.model_dump())) + result.append( + Chunk( + content=text, + metadata=chunk_metadata.model_dump(), + order=len(result), + ) + ) return result diff --git a/haiku_rag_slim/haiku/rag/chunkers/docling_serve.py b/haiku_rag_slim/haiku/rag/chunkers/docling_serve.py index efb2323f..10a26d95 100644 --- a/haiku_rag_slim/haiku/rag/chunkers/docling_serve.py +++ b/haiku_rag_slim/haiku/rag/chunkers/docling_serve.py @@ -192,6 +192,12 @@ class DoclingServeChunker(DocumentChunker): labels=labels, page_numbers=sorted(page_numbers) if page_numbers else [], ) - result.append(Chunk(content=text, metadata=chunk_metadata.model_dump())) + result.append( + Chunk( + content=text, + metadata=chunk_metadata.model_dump(), + order=len(result), + ) + ) return result diff --git a/haiku_rag_slim/haiku/rag/client.py b/haiku_rag_slim/haiku/rag/client.py index ee533c3e..1e57195a 100644 --- a/haiku_rag_slim/haiku/rag/client.py +++ b/haiku_rag_slim/haiku/rag/client.py @@ -194,13 +194,7 @@ class HaikuRAG: from haiku.rag.chunkers import get_chunker chunker = get_chunker(self._config) - chunks = await chunker.chunk(docling_document) - - # Set order for each chunk - for i, chunk in enumerate(chunks): - chunk.order = i - - return chunks + return await chunker.chunk(docling_document) async def _ensure_chunks_embedded(self, chunks: list[Chunk]) -> list[Chunk]: """Ensure all chunks have embeddings, embedding any that don't. diff --git a/tests/test_chunker.py b/tests/test_chunker.py index 104382c8..9576420c 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -144,6 +144,33 @@ async def test_local_chunker_markdown_tables(): assert "," in table_content and "|" not in table_content +@pytest.mark.asyncio +async def test_local_chunker_sets_order(): + """Test that DoclingLocalChunker sets sequential order on chunks.""" + sample_md = """# Introduction + +First paragraph with some content. + +## Section One + +Second paragraph. + +## Section Two + +Third paragraph. +""" + converter = get_converter(Config) + doc = await converter.convert_text(sample_md, name="test.md") + + chunker = DoclingLocalChunker() + chunks = await chunker.chunk(doc) + + assert len(chunks) > 0 + # Verify order is set sequentially starting from 0 + for i, chunk in enumerate(chunks): + assert chunk.order == i, f"Chunk {i} has order {chunk.order}, expected {i}" + + @pytest.mark.asyncio async def test_local_chunker_metadata_extraction(): """Test that DoclingLocalChunker extracts metadata correctly."""