From 09ff1013497d7e07015e0d976283363861a39df5 Mon Sep 17 00:00:00 2001 From: Lawrence Akka Date: Sat, 15 Aug 2026 15:21:09 +0200 Subject: [PATCH 1/6] Expose all chunk metadata on search results and citations through `SearchResult.chunk_meta` and `Citation.chunk_meta` --- CHANGELOG.md | 1 + docs/custom-pipelines.md | 6 +++ docs/python.md | 2 +- haiku_rag_slim/haiku/rag/context.py | 1 + .../haiku/rag/store/models/chunk.py | 6 +++ .../haiku/rag/store/models/citation.py | 7 ++++ tests/store/test_citation.py | 8 ++++ tests/test_chunk.py | 36 +++++++++++++++++ tests/test_context.py | 40 +++++++++++++++++++ 9 files changed, 106 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bcc7be9..c063d37e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed - `import_documents` embeds chunks across the whole batch in one pass instead of per document. +- Raw chunk metadata is now exposed to search and citation results, through `SearchResult.chunk_meta` and `Citation.chunk_meta`. For context-expanded results, the metadata is that of the anchor chunk. ### Removed diff --git a/docs/custom-pipelines.md b/docs/custom-pipelines.md index a523b4af..aa003a85 100644 --- a/docs/custom-pipelines.md +++ b/docs/custom-pipelines.md @@ -105,6 +105,8 @@ for chunk in chunks: print(f"Headings: {meta.headings}") print(f"Page numbers: {meta.page_numbers}") print(f"Labels: {meta.labels}") + # Access raw metadata (including headings, page_numbers and labels) + print(f"Raw metadata: {chunk.metadata}") ``` Chunks are returned with: @@ -115,6 +117,10 @@ Chunks are returned with: - `embedding` - `None` (not yet embedded) - `document_id` - `None` (not yet stored) +A custom `DocumentChunker` can provide other keys and values in `metadata`. They will +be stored with the chunk and are accessible when it is returned in a search result or citation, within +`SearchResult.chunk_meta` / `Citation.chunk_meta`. + ## Embed `embed_chunks()` generates embeddings for chunks using the client's embedder. It automatically contextualizes chunks (prepends section headings) before embedding for better semantic search, without modifying the stored content: diff --git a/docs/python.md b/docs/python.md index 48075426..e261d6cd 100644 --- a/docs/python.md +++ b/docs/python.md @@ -348,7 +348,7 @@ answer, citations = await client.ask( Images are passed to the model alongside the question. Retrieval stays text-based. The QA model must have `vision: true` in its configuration. -`client.ask` runs the [RAG capability](capabilities/rag.md) and returns `(answer_text, list[Citation])`. Citations include page numbers, section headings, document references, and the document's metadata (`document_meta`), so UIs can render metadata keys such as a public source URL alongside the citation. +`client.ask` runs the [RAG capability](capabilities/rag.md) and returns `(answer_text, list[Citation])`. Citations include page numbers, section headings, document references, the document's metadata (`document_meta`), and the cited chunk's raw, unparsed metadata (`chunk_meta`), so UIs can render metadata keys such as a public source URL alongside the citation. The QA provider and model are configured in `haiku.rag.yaml` or can be passed directly to the client (see [Configuration](configuration/index.md)). diff --git a/haiku_rag_slim/haiku/rag/context.py b/haiku_rag_slim/haiku/rag/context.py index 51e8375d..42073423 100644 --- a/haiku_rag_slim/haiku/rag/context.py +++ b/haiku_rag_slim/haiku/rag/context.py @@ -408,6 +408,7 @@ def _build_result( score=max(r.score for r in original_results), chunk_id=first.chunk_id, chunk_ids=chunk_ids, + chunk_meta=first.chunk_meta, document_id=first.document_id, document_uri=first.document_uri, document_title=first.document_title, diff --git a/haiku_rag_slim/haiku/rag/store/models/chunk.py b/haiku_rag_slim/haiku/rag/store/models/chunk.py index 95666a39..9a6cbadb 100644 --- a/haiku_rag_slim/haiku/rag/store/models/chunk.py +++ b/haiku_rag_slim/haiku/rag/store/models/chunk.py @@ -132,12 +132,17 @@ class SearchResult(BaseModel): ``document_meta`` carries the parent document's metadata for citation consumers (UIs). Never part of ``format_for_agent`` output. + + ``chunk_meta``is the anchor chunk's unparsed ``Chunk.metadata`` and does not + include the metadata of any other chunks merged with it. Never part of + ``format_for_agent`` output. """ content: str score: float chunk_id: str | None = None chunk_ids: list[str] = [] + chunk_meta: dict = {} document_id: str | None = None document_uri: str | None = None document_title: str | None = None @@ -172,6 +177,7 @@ class SearchResult(BaseModel): page_numbers=meta.page_numbers, headings=meta.headings, labels=meta.labels, + chunk_meta=chunk.metadata, image_data=image_data, ) diff --git a/haiku_rag_slim/haiku/rag/store/models/citation.py b/haiku_rag_slim/haiku/rag/store/models/citation.py index b7b38eba..161d8575 100644 --- a/haiku_rag_slim/haiku/rag/store/models/citation.py +++ b/haiku_rag_slim/haiku/rag/store/models/citation.py @@ -28,12 +28,18 @@ class Citation(BaseModel): ``picture_refs`` is the picture-labeled subset. ``document_meta`` carries the cited document's metadata for UIs. + + ``chunk_meta`` is the cited chunk's raw, unparsed ``Chunk.metadata`` + dict — lossless and independent of the typed fields above, so a + third-party chunker's own fields survive here even as this schema + evolves. """ index: int | None = None document_id: str chunk_id: str chunk_ids: list[str] = Field(default_factory=list) + chunk_meta: dict = Field(default_factory=dict) document_uri: str document_title: str | None = None document_meta: dict = Field(default_factory=dict) @@ -65,6 +71,7 @@ def resolve_citations( document_id=r.document_id or "", chunk_id=chunk_id, chunk_ids=r.chunk_ids or [chunk_id], + chunk_meta=r.chunk_meta, document_uri=r.document_uri or "", document_title=r.document_title, document_meta=r.document_meta, diff --git a/tests/store/test_citation.py b/tests/store/test_citation.py index 2bf8ce38..2f765112 100644 --- a/tests/store/test_citation.py +++ b/tests/store/test_citation.py @@ -6,6 +6,7 @@ def _result( chunk_id: str, chunk_ids: list[str] | None = None, document_meta: dict | None = None, + chunk_meta: dict | None = None, ) -> SearchResult: return SearchResult( content="content", @@ -15,6 +16,7 @@ def _result( document_id="doc-1", document_uri="test://doc", document_meta=document_meta or {}, + chunk_meta=chunk_meta or {}, ) @@ -54,3 +56,9 @@ def test_resolve_citations_copies_document_meta(): assert citations[0].document_meta == { "source_url": "https://example.org/report/view" } + + +def test_resolve_citations_copies_chunk_meta(): + result = _result("c1", chunk_meta={"para_no": "12", "speaker": "MR SMITH"}) + citations = resolve_citations(["c1"], [result]) + assert citations[0].chunk_meta == {"para_no": "12", "speaker": "MR SMITH"} diff --git a/tests/test_chunk.py b/tests/test_chunk.py index 5fe774f3..d052299f 100644 --- a/tests/test_chunk.py +++ b/tests/test_chunk.py @@ -222,6 +222,42 @@ def test_search_result_from_chunk_preserves_document_meta(): assert result.document_meta == {"source_url": "https://example.org/report/view"} +def test_search_result_from_chunk_preserves_chunk_meta(): + """Test flow through of unparsed chunk metadata from Chunk to SearchResult""" + chunk = Chunk( + id="chunk-1", + document_id="doc-1", + content="Some content.", + metadata={ + "headings": ["Chapter 1"], + "para_no": "12", + "speaker": "MR SMITH", + }, + ) + + result = SearchResult.from_chunk(chunk, score=0.9) + + assert result.chunk_meta == { + "headings": ["Chapter 1"], + "para_no": "12", + "speaker": "MR SMITH", + } + + +def test_search_result_format_for_agent_omits_chunk_meta(): + """Test that chunk_meta is never shown to the model""" + result = SearchResult( + content="Some content.", + score=0.9, + chunk_id="chunk-1", + chunk_meta={"para_no": "12"}, + ) + + formatted = result.format_for_agent(rank=1, total=1) + + assert "para_no" not in formatted + + def test_search_result_format_for_agent_omits_document_meta(): """Document metadata is UI plumbing, never shown to the model.""" result = SearchResult( diff --git a/tests/test_context.py b/tests/test_context.py index d99a70a9..4d307f60 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -799,6 +799,46 @@ class TestExpandWithItems: "source_url": "https://example.org/report/view" } + async def test_expanded_result_carries_anchor_chunk_meta(self, temp_db_path): + """chunk_meta belongs only to the anchor chunk (ie whichever constituent chunk earned the result its rank)""" + from haiku.rag.client import HaikuRAG + + async with HaikuRAG(temp_db_path, create=True) as rag: + items = [ + DocumentItem( + document_id="doc-1", + position=i, + self_ref=f"#/texts/{i}", + label="text", + text=f"Paragraph {i}. " * 10, + ) + for i in range(5) + ] + await rag.document_item_repository.create_items("doc-1", items) + + r1 = SearchResult( + content="Paragraph 1.", + score=0.9, + chunk_id="c1", + document_id="doc-1", + doc_item_refs=["#/texts/1"], + chunk_meta={"para_no": "12"}, + ) + r2 = SearchResult( + content="Paragraph 3.", + score=0.85, + chunk_id="c2", + document_id="doc-1", + doc_item_refs=["#/texts/3"], + chunk_meta={"para_no": "14"}, + ) + expanded = await expand_with_items( + rag.document_item_repository, "doc-1", [r1, r2], 5000 + ) + assert len(expanded) == 1 + assert expanded[0].chunk_id == "c1" + assert expanded[0].chunk_meta == {"para_no": "12"} + async def test_merged_anchor_is_highest_scoring_constituent(self, temp_db_path): """A merged result's chunk_id anchors on the best-scoring constituent, not whichever chunk sits earliest in the document.""" From 3a9ded2218052137d110a73d9f356d638c21faab Mon Sep 17 00:00:00 2001 From: Lawrence Akka Date: Mon, 17 Aug 2026 20:17:17 +0200 Subject: [PATCH 2/6] Display all chunk metadata in inspector --- .../rag/inspector/widgets/detail_view.py | 10 ++++ tests/test_inspector.py | 57 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py b/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py index 5b9c4e14..15026c8c 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py @@ -93,6 +93,10 @@ class DetailView(VerticalScroll): chunk_meta = chunk.get_chunk_metadata() content_parts.extend(self._format_provenance(chunk_meta)) + if chunk.metadata: + metadata_str = "\n".join(f" - {k}: {v}" for k, v in chunk.metadata.items()) + content_parts.append(f"**Metadata:**\n{metadata_str}") + if chunk.embedding: content_parts.append(f"**Embedding:** {len(chunk.embedding)} dimensions") @@ -121,6 +125,12 @@ class DetailView(VerticalScroll): content_parts.extend(self._format_provenance(search_result)) + if search_result.chunk_meta: + metadata_str = "\n".join( + f" - {k}: {v}" for k, v in search_result.chunk_meta.items() + ) + content_parts.append(f"**Metadata:**\n{metadata_str}") + if chunk.embedding: content_parts.append(f"**Embedding:** {len(chunk.embedding)} dimensions") diff --git a/tests/test_inspector.py b/tests/test_inspector.py index 9ee6d476..7d432821 100644 --- a/tests/test_inspector.py +++ b/tests/test_inspector.py @@ -165,6 +165,63 @@ async def test_document_list_tracks_has_more(): assert doc_list.has_more is False +@pytest.mark.asyncio +async def test_detail_view_shows_chunk_metadata(): + """show_chunk renders the raw chunk.metadata dict verbatim, not just the + typed provenance fields _format_provenance already covers.""" + from textual.app import App + + from haiku.rag.inspector.widgets.detail_view import DetailView + + chunk = Chunk( + id="chunk-1", + document_id="doc-1", + content="raw chunk text", + metadata={"headings": ["Chapter 1"], "para_no": "12"}, + ) + + class TestApp(App): + def compose(self): + yield DetailView(id="detail") + + app = TestApp() + async with app.run_test(): + detail_view = app.query_one(DetailView) + await detail_view.show_chunk(chunk) + source = detail_view.content_widget.source + assert "**Metadata:**" in source + assert "para_no: 12" in source + + +@pytest.mark.asyncio +async def test_detail_view_shows_search_result_chunk_meta(): + """show_search_result renders SearchResult.chunk_meta, the anchor + chunk's raw metadata carried through search/expansion.""" + from textual.app import App + + from haiku.rag.inspector.widgets.detail_view import DetailView + + chunk = Chunk(id="chunk-1", document_id="doc-1", content="raw chunk text") + search_result = SearchResult( + content="raw chunk text", + score=0.5, + chunk_id="chunk-1", + chunk_meta={"para_no": "12"}, + ) + + class TestApp(App): + def compose(self): + yield DetailView(id="detail") + + app = TestApp() + async with app.run_test(): + detail_view = app.query_one(DetailView) + await detail_view.show_search_result(chunk, search_result) + source = detail_view.content_widget.source + assert "**Metadata:**" in source + assert "para_no: 12" in source + + @pytest.mark.asyncio async def test_context_modal_renders_pictures_when_vision_enabled(): """ContextModal must mount one TextualImage per attached picture From 47feeb32ee675fe455d2e914b650d71346917375 Mon Sep 17 00:00:00 2001 From: Lawrence Akka Date: Tue, 18 Aug 2026 12:04:25 +0200 Subject: [PATCH 3/6] Linting, doc edits --- CHANGELOG.md | 3 ++- docs/custom-pipelines.md | 4 +--- docs/python.md | 2 +- haiku_rag_slim/haiku/rag/store/models/chunk.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c063d37e..9bfc79a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,11 +5,12 @@ - `evaluations run --filter/-f CLAUSE`: SQL `WHERE` clause over document columns, applied to the retrieval benchmark's searches and to every capability search during QA. Recorded as `document_filter` in experiment metadata. - `mtrag_clapnq` / `mtrag_clapnq_rewrite` / `mtrag_clapnq_live` / `mtrag_clapnq_live_uncompacted` evaluation datasets: multi-turn QA with gold-prefix and live-session conversation replay, live arms with and without `EvidenceCompactionCapability`, Recall@k/nDCG@k retrieval metrics, eligibility-aware citation scoring, refusal precision/recall, per-turn tool-traffic attributes, and `citation_status` / `turn_citation_status` eval attributes. +- Raw chunk metadata is now exposed to search and citation results, through `SearchResult.chunk_meta` and `Citation.chunk_meta`. For context-expanded results, the metadata is that of the anchor chunk. + ### Changed - `import_documents` embeds chunks across the whole batch in one pass instead of per document. -- Raw chunk metadata is now exposed to search and citation results, through `SearchResult.chunk_meta` and `Citation.chunk_meta`. For context-expanded results, the metadata is that of the anchor chunk. ### Removed diff --git a/docs/custom-pipelines.md b/docs/custom-pipelines.md index aa003a85..e4ad3202 100644 --- a/docs/custom-pipelines.md +++ b/docs/custom-pipelines.md @@ -117,9 +117,7 @@ Chunks are returned with: - `embedding` - `None` (not yet embedded) - `document_id` - `None` (not yet stored) -A custom `DocumentChunker` can provide other keys and values in `metadata`. They will -be stored with the chunk and are accessible when it is returned in a search result or citation, within -`SearchResult.chunk_meta` / `Citation.chunk_meta`. +A custom `DocumentChunker` can provide other keys and values in `metadata`. They will be stored with the chunk and are accessible when it is returned in a search result or citation, within `SearchResult.chunk_meta` / `Citation.chunk_meta`. ## Embed diff --git a/docs/python.md b/docs/python.md index e261d6cd..f7ed4854 100644 --- a/docs/python.md +++ b/docs/python.md @@ -196,7 +196,7 @@ for result in results: print(f"Document ID: {result.document_id}") ``` -Each result carries the parent document's metadata in `result.document_meta`. It is not shown to the model during QA. +Each result carries the parent document's metadata in `result.document_meta` and the relevant chunk's verbatim metadata in `result.chunk_meta`. Neither is shown to the model during QA. Search with different search types: ```python diff --git a/haiku_rag_slim/haiku/rag/store/models/chunk.py b/haiku_rag_slim/haiku/rag/store/models/chunk.py index 9a6cbadb..2a75fd4a 100644 --- a/haiku_rag_slim/haiku/rag/store/models/chunk.py +++ b/haiku_rag_slim/haiku/rag/store/models/chunk.py @@ -133,7 +133,7 @@ class SearchResult(BaseModel): ``document_meta`` carries the parent document's metadata for citation consumers (UIs). Never part of ``format_for_agent`` output. - ``chunk_meta``is the anchor chunk's unparsed ``Chunk.metadata`` and does not + ``chunk_meta`` is the anchor chunk's unparsed ``Chunk.metadata`` and does not include the metadata of any other chunks merged with it. Never part of ``format_for_agent`` output. """ From b221b4ac0364f8262051916a2c3aa2bbe47ca2f9 Mon Sep 17 00:00:00 2001 From: Lawrence Akka Date: Tue, 18 Aug 2026 12:20:56 +0200 Subject: [PATCH 4/6] Do not duplicate metadata items in inspector --- .../rag/inspector/widgets/detail_view.py | 29 +++++++----- tests/test_inspector.py | 47 +++++++++++++++++-- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py b/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py index 15026c8c..690db227 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py @@ -4,7 +4,7 @@ from textual.app import ComposeResult from textual.containers import VerticalScroll from textual.widgets import Markdown, Static -from haiku.rag.store.models import Chunk, Document, SearchResult +from haiku.rag.store.models import Chunk, ChunkMetadata, Document, SearchResult class ProvenanceData(Protocol): @@ -50,6 +50,21 @@ class DetailView(VerticalScroll): parts.append(f"**DocItem Refs:** `{refs_str}`") return parts + def _format_extra_metadata(self, metadata: dict) -> list[str]: + """Format raw metadata keys not already shown by `_format_provenance`. + + `ChunkMetadata`'s own fields (page_numbers, headings, labels, + doc_item_refs) are excluded here since `_format_provenance` already + renders them, with its own truncation for long ref lists. + """ + extra = { + k: v for k, v in metadata.items() if k not in ChunkMetadata.model_fields + } + if not extra: + return [] + metadata_str = "\n".join(f" - {k}: {v}" for k, v in extra.items()) + return [f"**Metadata:**\n{metadata_str}"] + async def show_document(self, document: Document) -> None: """Display document details.""" title = document.title or document.uri or "Untitled Document" @@ -92,10 +107,7 @@ class DetailView(VerticalScroll): chunk_meta = chunk.get_chunk_metadata() content_parts.extend(self._format_provenance(chunk_meta)) - - if chunk.metadata: - metadata_str = "\n".join(f" - {k}: {v}" for k, v in chunk.metadata.items()) - content_parts.append(f"**Metadata:**\n{metadata_str}") + content_parts.extend(self._format_extra_metadata(chunk.metadata)) if chunk.embedding: content_parts.append(f"**Embedding:** {len(chunk.embedding)} dimensions") @@ -124,12 +136,7 @@ class DetailView(VerticalScroll): content_parts.append(f"**Score:** {search_result.score:.4f}") content_parts.extend(self._format_provenance(search_result)) - - if search_result.chunk_meta: - metadata_str = "\n".join( - f" - {k}: {v}" for k, v in search_result.chunk_meta.items() - ) - content_parts.append(f"**Metadata:**\n{metadata_str}") + content_parts.extend(self._format_extra_metadata(search_result.chunk_meta)) if chunk.embedding: content_parts.append(f"**Embedding:** {len(chunk.embedding)} dimensions") diff --git a/tests/test_inspector.py b/tests/test_inspector.py index 7d432821..52c69c8d 100644 --- a/tests/test_inspector.py +++ b/tests/test_inspector.py @@ -167,8 +167,9 @@ async def test_document_list_tracks_has_more(): @pytest.mark.asyncio async def test_detail_view_shows_chunk_metadata(): - """show_chunk renders the raw chunk.metadata dict verbatim, not just the - typed provenance fields _format_provenance already covers.""" + """show_chunk renders metadata keys _format_provenance doesn't already + cover, but not a duplicate of the standard fields it does (headings, + page_numbers, labels, doc_item_refs).""" from textual.app import App from haiku.rag.inspector.widgets.detail_view import DetailView @@ -191,12 +192,40 @@ async def test_detail_view_shows_chunk_metadata(): source = detail_view.content_widget.source assert "**Metadata:**" in source assert "para_no: 12" in source + assert "headings:" not in source # already shown as **Section:** + + +@pytest.mark.asyncio +async def test_detail_view_omits_metadata_block_when_only_standard_fields(): + """No **Metadata:** block at all when chunk.metadata holds nothing + beyond what _format_provenance already renders.""" + from textual.app import App + + from haiku.rag.inspector.widgets.detail_view import DetailView + + chunk = Chunk( + id="chunk-1", + document_id="doc-1", + content="raw chunk text", + metadata={"headings": ["Chapter 1"], "page_numbers": [1]}, + ) + + class TestApp(App): + def compose(self): + yield DetailView(id="detail") + + app = TestApp() + async with app.run_test(): + detail_view = app.query_one(DetailView) + await detail_view.show_chunk(chunk) + source = detail_view.content_widget.source + assert "**Metadata:**" not in source @pytest.mark.asyncio async def test_detail_view_shows_search_result_chunk_meta(): - """show_search_result renders SearchResult.chunk_meta, the anchor - chunk's raw metadata carried through search/expansion.""" + """show_search_result renders SearchResult.chunk_meta's non-standard + keys, the anchor chunk's own metadata carried through search/expansion.""" from textual.app import App from haiku.rag.inspector.widgets.detail_view import DetailView @@ -206,7 +235,11 @@ async def test_detail_view_shows_search_result_chunk_meta(): content="raw chunk text", score=0.5, chunk_id="chunk-1", - chunk_meta={"para_no": "12"}, + doc_item_refs=[f"#/texts/{i}" for i in range(7)], + chunk_meta={ + "para_no": "12", + "doc_item_refs": [f"#/texts/{i}" for i in range(7)], + }, ) class TestApp(App): @@ -220,6 +253,10 @@ async def test_detail_view_shows_search_result_chunk_meta(): source = detail_view.content_widget.source assert "**Metadata:**" in source assert "para_no: 12" in source + # _format_provenance's own 5-item truncation for doc_item_refs is + # untouched by the filtered-out duplicate in chunk_meta. + assert "+2 more" in source + assert "doc_item_refs:" not in source @pytest.mark.asyncio From 2839a4434a45bed9593a80fb4c6486aa7bdd44a6 Mon Sep 17 00:00:00 2001 From: Lawrence Akka Date: Tue, 18 Aug 2026 12:36:46 +0200 Subject: [PATCH 5/6] Test that chunk metadata survivies FastMCP's wire serialization --- tests/test_mcp.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index f0c36474..04093e09 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -2,7 +2,7 @@ import pytest from haiku.rag.client import HaikuRAG from haiku.rag.mcp import create_mcp_server -from haiku.rag.store.models import Document, SearchResult +from haiku.rag.store.models import Chunk, Document, SearchResult from haiku.rag.tools.document import DocumentInfo @@ -69,6 +69,40 @@ class TestMCPReadTools: results = await search(query="artificial intelligence", limit=1) assert len(results) == 1 + @pytest.mark.asyncio + async def test_search_documents_preserves_chunk_meta_through_serialization( + self, mcp_db + ): + """Chunk_meta must survive FastMCP's actual wire serialization. + + Calling the tool function directly bypasses that serialization step entirely.""" + from fastmcp import Client + + async with HaikuRAG(mcp_db, create=True) as rag: + doc = await rag.get_document_by_uri("test://ai-overview") + embedding = (await rag.embedder.embed_documents(["x"]))[0] + await rag.chunk_repository.create( + Chunk( + document_id=doc.id, + content="Artificial intelligence is transforming industries worldwide.", + metadata={"fake-metadata-for-testing": "42"}, + embedding=embedding, + ) + ) + await rag.chunk_repository._ensure_fts_index() + + mcp = create_mcp_server(mcp_db, read_only=True) + async with Client(mcp) as client: + result = await client.call_tool( + "search_documents", {"query": "artificial intelligence"} + ) + + results = result.structured_content["result"] + assert results + assert any( + r["chunk_meta"] == {"fake-metadata-for-testing": "42"} for r in results + ) + @pytest.mark.asyncio async def test_get_document(self, mcp_db): mcp = create_mcp_server(mcp_db, read_only=True) From 4eb72d6a49ac48cab9f2c3e86ceb5e939cec4ae1 Mon Sep 17 00:00:00 2001 From: Lawrence Akka Date: Tue, 18 Aug 2026 14:08:08 +0200 Subject: [PATCH 6/6] Ignore warning caused by logfire See https://github.com/ggozad/haiku.rag/pull/551#issuecomment-5327750999 --- tests/test_mcp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 04093e09..9c3b4553 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -70,6 +70,7 @@ class TestMCPReadTools: assert len(results) == 1 @pytest.mark.asyncio + @pytest.mark.filterwarnings("ignore:Found propagated trace context:RuntimeWarning") async def test_search_documents_preserves_chunk_meta_through_serialization( self, mcp_db ):