From 7fac35d2af98983788cccbe538c464c4bc03b1a4 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 5 May 2026 16:15:47 +0300 Subject: [PATCH] rename embed_image_query to embed_image; run description check on text path --- haiku_rag_slim/haiku/rag/client/processing.py | 9 ++-- haiku_rag_slim/haiku/rag/client/search.py | 2 +- .../haiku/rag/embeddings/__init__.py | 6 +-- haiku_rag_slim/haiku/rag/embeddings/vllm.py | 2 +- tests/test_embedder.py | 8 +-- tests/test_mcp.py | 2 +- tests/test_picture_in_context.py | 6 +-- tests/test_processing.py | 49 ++++++++++++++++--- tests/test_search.py | 6 +-- 9 files changed, 64 insertions(+), 26 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/client/processing.py b/haiku_rag_slim/haiku/rag/client/processing.py index 9513733a..a56ff50f 100644 --- a/haiku_rag_slim/haiku/rag/client/processing.py +++ b/haiku_rag_slim/haiku/rag/client/processing.py @@ -132,8 +132,11 @@ async def convert( return doc else: - # Treat as text content - return await converter.convert_text(source, format=format) + # Raw text content — HTML and markdown can still embed pictures + # via /![](...) so the same description check applies. + doc = await converter.convert_text(source, format=format) + _warn_if_descriptions_missing(config, doc, "") + return doc async def chunk( @@ -205,7 +208,7 @@ def build_picture_chunks( its picture URIs stripped). Pictures with no available bytes are skipped. The bytes ride on ``Chunk._picture_data`` (a PrivateAttr — not serialized) - so ``embed_chunks`` can route them through ``embed_image_query``. The + so ``embed_chunks`` can route them through ``embed_image``. The ``order`` field is left at its default (0); the caller (``chunk()``) reassigns it after merging with text chunks in structural order. """ diff --git a/haiku_rag_slim/haiku/rag/client/search.py b/haiku_rag_slim/haiku/rag/client/search.py index 9009ec38..6d19ef4f 100644 --- a/haiku_rag_slim/haiku/rag/client/search.py +++ b/haiku_rag_slim/haiku/rag/client/search.py @@ -59,7 +59,7 @@ async def search( "Image queries require a multimodal embedder. Configure " "provider='vllm' (or another image-capable provider)." ) - query_vector = await embedder.embed_image_query(query) + query_vector = await embedder.embed_image(query) chunk_results = await client.chunk_repository.search( query="", limit=limit, diff --git a/haiku_rag_slim/haiku/rag/embeddings/__init__.py b/haiku_rag_slim/haiku/rag/embeddings/__init__.py index 5d00e1d3..3cd77a4f 100644 --- a/haiku_rag_slim/haiku/rag/embeddings/__init__.py +++ b/haiku_rag_slim/haiku/rag/embeddings/__init__.py @@ -47,7 +47,7 @@ class EmbedderWrapper: result = await self._embedder.embed_documents(texts) return [list(e) for e in result.embeddings] - async def embed_image_query(self, image: "Any") -> list[float]: + async def embed_image(self, image: "Any") -> list[float]: """Embed a single image into the same vector space as text. Multimodal providers override this. Picture embedding is single-image: @@ -125,9 +125,7 @@ async def embed_chunks( "provider='vllm', or omit picture chunks." ) for chunk in picture_chunks: - picture_embeddings.append( - await embedder.embed_image_query(chunk._picture_data) - ) + picture_embeddings.append(await embedder.embed_image(chunk._picture_data)) text_iter = iter(text_embeddings) picture_iter = iter(picture_embeddings) diff --git a/haiku_rag_slim/haiku/rag/embeddings/vllm.py b/haiku_rag_slim/haiku/rag/embeddings/vllm.py index c37c4f4f..6535f7ec 100644 --- a/haiku_rag_slim/haiku/rag/embeddings/vllm.py +++ b/haiku_rag_slim/haiku/rag/embeddings/vllm.py @@ -98,7 +98,7 @@ class VLLMMultimodalEmbedder(EmbedderWrapper): } ) - async def embed_image_query(self, image: "bytes | PILImage.Image") -> list[float]: + async def embed_image(self, image: "bytes | PILImage.Image") -> list[float]: rows = await self._post( { "model": self._model_name, diff --git a/tests/test_embedder.py b/tests/test_embedder.py index 2a5d62c0..43f788aa 100644 --- a/tests/test_embedder.py +++ b/tests/test_embedder.py @@ -162,7 +162,7 @@ async def test_embed_chunks_empty_list(): async def test_embed_chunks_picture_with_text_only_embedder_raises(): """A picture chunk fed through a text-only embedder must surface a - clear error, not silently drop the chunk or call ``embed_image_query`` + clear error, not silently drop the chunk or call ``embed_image`` on something that doesn't support it.""" chunk = Chunk(id="pic", content="x") chunk._picture_data = b"\x89PNG\r\n\x1a\nfake" @@ -254,7 +254,7 @@ async def test_text_only_embedder_does_not_support_images(): embedder = get_embedder(_ollama_text_only_config()) assert embedder.supports_images is False with pytest.raises(NotImplementedError, match="multimodal provider"): - await embedder.embed_image_query(b"\x89PNG\r\n\x1a\n") + await embedder.embed_image(b"\x89PNG\r\n\x1a\n") async def test_vllm_embed_text_request_shape(monkeypatch): @@ -344,7 +344,7 @@ async def test_vllm_embed_image_request_shape(monkeypatch): base_url="http://localhost:8000/v1", ) raw = b"\x89PNG\r\n\x1a\nfake" - vec = await embedder.embed_image_query(raw) + vec = await embedder.embed_image(raw) assert vec == [0.4, 0.4, 0.4, 0.4] content = captured["body"]["messages"][0]["content"] assert len(content) == 1 @@ -623,6 +623,6 @@ async def test_vllm_embed_text_and_image_end_to_end(): assert all(len(v) == vector_dim for v in text_batch) image = Image.new("RGB", (64, 64), color=(255, 0, 0)) - image_vec = await embedder.embed_image_query(image) + image_vec = await embedder.embed_image(image) assert len(image_vec) == vector_dim assert any(abs(x) > 1e-6 for x in image_vec), "image embedding is all zeros" diff --git a/tests/test_mcp.py b/tests/test_mcp.py index a88dd2b3..74bd866a 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -219,7 +219,7 @@ class TestMCPImageQuery: def __init__(self): super().__init__(embedder=None, vector_dim=2560) - async def embed_image_query(self, image): + async def embed_image(self, image): # Produce a deterministic-ish vector of the right dim. return [0.0] * 2560 diff --git a/tests/test_picture_in_context.py b/tests/test_picture_in_context.py index bc3675eb..c675225d 100644 --- a/tests/test_picture_in_context.py +++ b/tests/test_picture_in_context.py @@ -465,7 +465,7 @@ async def test_chunk_interleaves_picture_in_structural_order(monkeypatch): @pytest.mark.asyncio async def test_embed_chunks_dispatches_text_vs_picture(monkeypatch): """embed_chunks routes text chunks through embed_documents (batched) and - picture chunks through embed_image_query (one at a time), reassembling + picture chunks through embed_image (one at a time), reassembling in original order.""" from haiku.rag.embeddings import EmbedderWrapper, embed_chunks from haiku.rag.store.models.chunk import Chunk @@ -483,7 +483,7 @@ async def test_embed_chunks_dispatches_text_vs_picture(monkeypatch): text_calls.append(list(texts)) return [[0.1, 0.2, 0.3, 0.4] for _ in texts] - async def embed_image_query(self, image): + async def embed_image(self, image): image_calls.append(image) return [0.9, 0.8, 0.7, 0.6] @@ -554,7 +554,7 @@ async def test_ingest_emits_picture_chunks_with_multimodal_embedder( async def embed_documents(self, texts): return [[0.1] * 4 for _ in texts] - async def embed_image_query(self, image): + async def embed_image(self, image): return [0.9] * 4 monkeypatch.setattr( diff --git a/tests/test_processing.py b/tests/test_processing.py index fe3019ff..b02624fe 100644 --- a/tests/test_processing.py +++ b/tests/test_processing.py @@ -89,9 +89,10 @@ def test_no_warning_when_doc_has_no_pictures(caplog_warnings): def test_warns_when_pictures_present_but_no_descriptions(caplog_warnings): - """The silent-failure case: VLM was requested, the doc has pictures, - but the converter returned zero descriptions. Warn loudly so the user - can fix their VLM config before a long ingest.""" + """VLM was requested via ``picture_description.enabled = True``, the + doc has pictures, but the converter returned zero descriptions + (docling-serve swallows VLM errors). Warn loudly so the user can fix + their VLM config before a long ingest.""" config = AppConfig() config.processing.conversion_options.picture_description.enabled = True config.processing.conversion_options.picture_description.model.name = "qwen3.6" @@ -128,9 +129,10 @@ def test_no_warning_when_at_least_one_description_came_back(caplog_warnings): async def test_convert_emits_warning_via_chokepoint( monkeypatch, tmp_path, caplog_warnings ): - """End-to-end: ``convert(...)`` invokes the guard after the converter - returns, so a silent VLM failure surfaces as a warning to the user - regardless of which converter (local vs serve) actually ran.""" + """End-to-end: ``convert(...)`` runs the description-missing check + after the converter returns, so a VLM error swallowed inside + docling-serve still surfaces as a warning at the haiku.rag layer + regardless of which converter (local vs serve) ran.""" from haiku.rag.converters.base import DocumentConverter pdf = tmp_path / "fake.pdf" @@ -162,3 +164,38 @@ async def test_convert_emits_warning_via_chokepoint( "0 described" in r.getMessage() and "fake.pdf" in r.getMessage() for r in caplog_warnings ) + + +@pytest.mark.asyncio +async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings): + """Raw text input (HTML, markdown) can still produce pictures via + docling, so the description-missing check must run on the + convert_text branch too — otherwise an HTML-with-images source + would never trigger the warning even when picture_description is + enabled and the VLM didn't actually run.""" + from haiku.rag.converters.base import DocumentConverter + + class StubConverter(DocumentConverter): + @property + def supported_extensions(self) -> list[str]: + return [".html"] + + async def convert_file(self, path: Path): + return _doc_without_pictures() + + async def convert_text( + self, text: str, name: str = "content.md", format: str = "md" + ): + return _doc_with_pictures(with_descriptions=False) + + monkeypatch.setattr( + "haiku.rag.client.processing.get_converter", lambda config: StubConverter() + ) + + config = AppConfig() + config.processing.conversion_options.picture_description.enabled = True + + # No URL scheme, no Path → drops into the convert_text branch. + await convert(config, "") + + assert any("0 described" in r.getMessage() for r in caplog_warnings) diff --git a/tests/test_search.py b/tests/test_search.py index 8ab46e14..6ec7b827 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -336,7 +336,7 @@ def test_search_result_primary_label_prioritizes_structural_types(): async def test_search_with_bytes_query_uses_multimodal_embedder( temp_db_path, monkeypatch ): - """``client.search(bytes)`` embeds via ``embed_image_query`` and dispatches + """``client.search(bytes)`` embeds via ``embed_image`` and dispatches to vector-only chunk search (skipping FTS and reranker).""" from haiku.rag.embeddings import EmbedderWrapper from haiku.rag.store.models.chunk import Chunk @@ -349,7 +349,7 @@ async def test_search_with_bytes_query_uses_multimodal_embedder( def __init__(self): super().__init__(embedder=None, vector_dim=4) - async def embed_image_query(self, image): + async def embed_image(self, image): image_calls.append(image) return [0.5, 0.5, 0.5, 0.5] @@ -410,7 +410,7 @@ async def test_search_with_pil_image_works_like_bytes(temp_db_path, monkeypatch) def __init__(self): super().__init__(embedder=None, vector_dim=4) - async def embed_image_query(self, image): + async def embed_image(self, image): seen_types.append(type(image)) return [0.1] * 4