rename embed_image_query to embed_image; run description check on text path

This commit is contained in:
Yiorgis Gozadinos 2026-05-05 16:15:47 +03:00
parent 1ccb5b5fad
commit 7fac35d2af
No known key found for this signature in database
9 changed files with 64 additions and 26 deletions

View file

@ -132,8 +132,11 @@ async def convert(
return doc
else:
# Treat as text content
return await converter.convert_text(source, format=format)
# Raw text content — HTML and markdown can still embed pictures
# via <img>/![](...) so the same description check applies.
doc = await converter.convert_text(source, format=format)
_warn_if_descriptions_missing(config, doc, "<text input>")
return doc
async def chunk(
@ -205,7 +208,7 @@ def build_picture_chunks(
its picture URIs stripped). Pictures with no available bytes are skipped.
The bytes ride on ``Chunk._picture_data`` (a PrivateAttr not serialized)
so ``embed_chunks`` can route them through ``embed_image_query``. The
so ``embed_chunks`` can route them through ``embed_image``. The
``order`` field is left at its default (0); the caller (``chunk()``)
reassigns it after merging with text chunks in structural order.
"""

View file

@ -59,7 +59,7 @@ async def search(
"Image queries require a multimodal embedder. Configure "
"provider='vllm' (or another image-capable provider)."
)
query_vector = await embedder.embed_image_query(query)
query_vector = await embedder.embed_image(query)
chunk_results = await client.chunk_repository.search(
query="",
limit=limit,

View file

@ -47,7 +47,7 @@ class EmbedderWrapper:
result = await self._embedder.embed_documents(texts)
return [list(e) for e in result.embeddings]
async def embed_image_query(self, image: "Any") -> list[float]:
async def embed_image(self, image: "Any") -> list[float]:
"""Embed a single image into the same vector space as text.
Multimodal providers override this. Picture embedding is single-image:
@ -125,9 +125,7 @@ async def embed_chunks(
"provider='vllm', or omit picture chunks."
)
for chunk in picture_chunks:
picture_embeddings.append(
await embedder.embed_image_query(chunk._picture_data)
)
picture_embeddings.append(await embedder.embed_image(chunk._picture_data))
text_iter = iter(text_embeddings)
picture_iter = iter(picture_embeddings)

View file

@ -98,7 +98,7 @@ class VLLMMultimodalEmbedder(EmbedderWrapper):
}
)
async def embed_image_query(self, image: "bytes | PILImage.Image") -> list[float]:
async def embed_image(self, image: "bytes | PILImage.Image") -> list[float]:
rows = await self._post(
{
"model": self._model_name,

View file

@ -162,7 +162,7 @@ async def test_embed_chunks_empty_list():
async def test_embed_chunks_picture_with_text_only_embedder_raises():
"""A picture chunk fed through a text-only embedder must surface a
clear error, not silently drop the chunk or call ``embed_image_query``
clear error, not silently drop the chunk or call ``embed_image``
on something that doesn't support it."""
chunk = Chunk(id="pic", content="x")
chunk._picture_data = b"\x89PNG\r\n\x1a\nfake"
@ -254,7 +254,7 @@ async def test_text_only_embedder_does_not_support_images():
embedder = get_embedder(_ollama_text_only_config())
assert embedder.supports_images is False
with pytest.raises(NotImplementedError, match="multimodal provider"):
await embedder.embed_image_query(b"\x89PNG\r\n\x1a\n")
await embedder.embed_image(b"\x89PNG\r\n\x1a\n")
async def test_vllm_embed_text_request_shape(monkeypatch):
@ -344,7 +344,7 @@ async def test_vllm_embed_image_request_shape(monkeypatch):
base_url="http://localhost:8000/v1",
)
raw = b"\x89PNG\r\n\x1a\nfake"
vec = await embedder.embed_image_query(raw)
vec = await embedder.embed_image(raw)
assert vec == [0.4, 0.4, 0.4, 0.4]
content = captured["body"]["messages"][0]["content"]
assert len(content) == 1
@ -623,6 +623,6 @@ async def test_vllm_embed_text_and_image_end_to_end():
assert all(len(v) == vector_dim for v in text_batch)
image = Image.new("RGB", (64, 64), color=(255, 0, 0))
image_vec = await embedder.embed_image_query(image)
image_vec = await embedder.embed_image(image)
assert len(image_vec) == vector_dim
assert any(abs(x) > 1e-6 for x in image_vec), "image embedding is all zeros"

View file

@ -219,7 +219,7 @@ class TestMCPImageQuery:
def __init__(self):
super().__init__(embedder=None, vector_dim=2560)
async def embed_image_query(self, image):
async def embed_image(self, image):
# Produce a deterministic-ish vector of the right dim.
return [0.0] * 2560

View file

@ -465,7 +465,7 @@ async def test_chunk_interleaves_picture_in_structural_order(monkeypatch):
@pytest.mark.asyncio
async def test_embed_chunks_dispatches_text_vs_picture(monkeypatch):
"""embed_chunks routes text chunks through embed_documents (batched) and
picture chunks through embed_image_query (one at a time), reassembling
picture chunks through embed_image (one at a time), reassembling
in original order."""
from haiku.rag.embeddings import EmbedderWrapper, embed_chunks
from haiku.rag.store.models.chunk import Chunk
@ -483,7 +483,7 @@ async def test_embed_chunks_dispatches_text_vs_picture(monkeypatch):
text_calls.append(list(texts))
return [[0.1, 0.2, 0.3, 0.4] for _ in texts]
async def embed_image_query(self, image):
async def embed_image(self, image):
image_calls.append(image)
return [0.9, 0.8, 0.7, 0.6]
@ -554,7 +554,7 @@ async def test_ingest_emits_picture_chunks_with_multimodal_embedder(
async def embed_documents(self, texts):
return [[0.1] * 4 for _ in texts]
async def embed_image_query(self, image):
async def embed_image(self, image):
return [0.9] * 4
monkeypatch.setattr(

View file

@ -89,9 +89,10 @@ def test_no_warning_when_doc_has_no_pictures(caplog_warnings):
def test_warns_when_pictures_present_but_no_descriptions(caplog_warnings):
"""The silent-failure case: VLM was requested, the doc has pictures,
but the converter returned zero descriptions. Warn loudly so the user
can fix their VLM config before a long ingest."""
"""VLM was requested via ``picture_description.enabled = True``, the
doc has pictures, but the converter returned zero descriptions
(docling-serve swallows VLM errors). Warn loudly so the user can fix
their VLM config before a long ingest."""
config = AppConfig()
config.processing.conversion_options.picture_description.enabled = True
config.processing.conversion_options.picture_description.model.name = "qwen3.6"
@ -128,9 +129,10 @@ def test_no_warning_when_at_least_one_description_came_back(caplog_warnings):
async def test_convert_emits_warning_via_chokepoint(
monkeypatch, tmp_path, caplog_warnings
):
"""End-to-end: ``convert(...)`` invokes the guard after the converter
returns, so a silent VLM failure surfaces as a warning to the user
regardless of which converter (local vs serve) actually ran."""
"""End-to-end: ``convert(...)`` runs the description-missing check
after the converter returns, so a VLM error swallowed inside
docling-serve still surfaces as a warning at the haiku.rag layer
regardless of which converter (local vs serve) ran."""
from haiku.rag.converters.base import DocumentConverter
pdf = tmp_path / "fake.pdf"
@ -162,3 +164,38 @@ async def test_convert_emits_warning_via_chokepoint(
"0 described" in r.getMessage() and "fake.pdf" in r.getMessage()
for r in caplog_warnings
)
@pytest.mark.asyncio
async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings):
"""Raw text input (HTML, markdown) can still produce pictures via
docling, so the description-missing check must run on the
convert_text branch too otherwise an HTML-with-images source
would never trigger the warning even when picture_description is
enabled and the VLM didn't actually run."""
from haiku.rag.converters.base import DocumentConverter
class StubConverter(DocumentConverter):
@property
def supported_extensions(self) -> list[str]:
return [".html"]
async def convert_file(self, path: Path):
return _doc_without_pictures()
async def convert_text(
self, text: str, name: str = "content.md", format: str = "md"
):
return _doc_with_pictures(with_descriptions=False)
monkeypatch.setattr(
"haiku.rag.client.processing.get_converter", lambda config: StubConverter()
)
config = AppConfig()
config.processing.conversion_options.picture_description.enabled = True
# No URL scheme, no Path → drops into the convert_text branch.
await convert(config, "<html><img src='...'/></html>")
assert any("0 described" in r.getMessage() for r in caplog_warnings)

View file

@ -336,7 +336,7 @@ def test_search_result_primary_label_prioritizes_structural_types():
async def test_search_with_bytes_query_uses_multimodal_embedder(
temp_db_path, monkeypatch
):
"""``client.search(bytes)`` embeds via ``embed_image_query`` and dispatches
"""``client.search(bytes)`` embeds via ``embed_image`` and dispatches
to vector-only chunk search (skipping FTS and reranker)."""
from haiku.rag.embeddings import EmbedderWrapper
from haiku.rag.store.models.chunk import Chunk
@ -349,7 +349,7 @@ async def test_search_with_bytes_query_uses_multimodal_embedder(
def __init__(self):
super().__init__(embedder=None, vector_dim=4)
async def embed_image_query(self, image):
async def embed_image(self, image):
image_calls.append(image)
return [0.5, 0.5, 0.5, 0.5]
@ -410,7 +410,7 @@ async def test_search_with_pil_image_works_like_bytes(temp_db_path, monkeypatch)
def __init__(self):
super().__init__(embedder=None, vector_dim=4)
async def embed_image_query(self, image):
async def embed_image(self, image):
seen_types.append(type(image))
return [0.1] * 4