rename embed_image_query to embed_image; run description check on text path
This commit is contained in:
parent
1ccb5b5fad
commit
7fac35d2af
9 changed files with 64 additions and 26 deletions
|
|
@ -132,8 +132,11 @@ async def convert(
|
|||
return doc
|
||||
|
||||
else:
|
||||
# Treat as text content
|
||||
return await converter.convert_text(source, format=format)
|
||||
# Raw text content — HTML and markdown can still embed pictures
|
||||
# via <img>/ so the same description check applies.
|
||||
doc = await converter.convert_text(source, format=format)
|
||||
_warn_if_descriptions_missing(config, doc, "<text input>")
|
||||
return doc
|
||||
|
||||
|
||||
async def chunk(
|
||||
|
|
@ -205,7 +208,7 @@ def build_picture_chunks(
|
|||
its picture URIs stripped). Pictures with no available bytes are skipped.
|
||||
|
||||
The bytes ride on ``Chunk._picture_data`` (a PrivateAttr — not serialized)
|
||||
so ``embed_chunks`` can route them through ``embed_image_query``. The
|
||||
so ``embed_chunks`` can route them through ``embed_image``. The
|
||||
``order`` field is left at its default (0); the caller (``chunk()``)
|
||||
reassigns it after merging with text chunks in structural order.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ async def search(
|
|||
"Image queries require a multimodal embedder. Configure "
|
||||
"provider='vllm' (or another image-capable provider)."
|
||||
)
|
||||
query_vector = await embedder.embed_image_query(query)
|
||||
query_vector = await embedder.embed_image(query)
|
||||
chunk_results = await client.chunk_repository.search(
|
||||
query="",
|
||||
limit=limit,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class EmbedderWrapper:
|
|||
result = await self._embedder.embed_documents(texts)
|
||||
return [list(e) for e in result.embeddings]
|
||||
|
||||
async def embed_image_query(self, image: "Any") -> list[float]:
|
||||
async def embed_image(self, image: "Any") -> list[float]:
|
||||
"""Embed a single image into the same vector space as text.
|
||||
|
||||
Multimodal providers override this. Picture embedding is single-image:
|
||||
|
|
@ -125,9 +125,7 @@ async def embed_chunks(
|
|||
"provider='vllm', or omit picture chunks."
|
||||
)
|
||||
for chunk in picture_chunks:
|
||||
picture_embeddings.append(
|
||||
await embedder.embed_image_query(chunk._picture_data)
|
||||
)
|
||||
picture_embeddings.append(await embedder.embed_image(chunk._picture_data))
|
||||
|
||||
text_iter = iter(text_embeddings)
|
||||
picture_iter = iter(picture_embeddings)
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class VLLMMultimodalEmbedder(EmbedderWrapper):
|
|||
}
|
||||
)
|
||||
|
||||
async def embed_image_query(self, image: "bytes | PILImage.Image") -> list[float]:
|
||||
async def embed_image(self, image: "bytes | PILImage.Image") -> list[float]:
|
||||
rows = await self._post(
|
||||
{
|
||||
"model": self._model_name,
|
||||
|
|
|
|||
|
|
@ -162,7 +162,7 @@ async def test_embed_chunks_empty_list():
|
|||
|
||||
async def test_embed_chunks_picture_with_text_only_embedder_raises():
|
||||
"""A picture chunk fed through a text-only embedder must surface a
|
||||
clear error, not silently drop the chunk or call ``embed_image_query``
|
||||
clear error, not silently drop the chunk or call ``embed_image``
|
||||
on something that doesn't support it."""
|
||||
chunk = Chunk(id="pic", content="x")
|
||||
chunk._picture_data = b"\x89PNG\r\n\x1a\nfake"
|
||||
|
|
@ -254,7 +254,7 @@ async def test_text_only_embedder_does_not_support_images():
|
|||
embedder = get_embedder(_ollama_text_only_config())
|
||||
assert embedder.supports_images is False
|
||||
with pytest.raises(NotImplementedError, match="multimodal provider"):
|
||||
await embedder.embed_image_query(b"\x89PNG\r\n\x1a\n")
|
||||
await embedder.embed_image(b"\x89PNG\r\n\x1a\n")
|
||||
|
||||
|
||||
async def test_vllm_embed_text_request_shape(monkeypatch):
|
||||
|
|
@ -344,7 +344,7 @@ async def test_vllm_embed_image_request_shape(monkeypatch):
|
|||
base_url="http://localhost:8000/v1",
|
||||
)
|
||||
raw = b"\x89PNG\r\n\x1a\nfake"
|
||||
vec = await embedder.embed_image_query(raw)
|
||||
vec = await embedder.embed_image(raw)
|
||||
assert vec == [0.4, 0.4, 0.4, 0.4]
|
||||
content = captured["body"]["messages"][0]["content"]
|
||||
assert len(content) == 1
|
||||
|
|
@ -623,6 +623,6 @@ async def test_vllm_embed_text_and_image_end_to_end():
|
|||
assert all(len(v) == vector_dim for v in text_batch)
|
||||
|
||||
image = Image.new("RGB", (64, 64), color=(255, 0, 0))
|
||||
image_vec = await embedder.embed_image_query(image)
|
||||
image_vec = await embedder.embed_image(image)
|
||||
assert len(image_vec) == vector_dim
|
||||
assert any(abs(x) > 1e-6 for x in image_vec), "image embedding is all zeros"
|
||||
|
|
|
|||
|
|
@ -219,7 +219,7 @@ class TestMCPImageQuery:
|
|||
def __init__(self):
|
||||
super().__init__(embedder=None, vector_dim=2560)
|
||||
|
||||
async def embed_image_query(self, image):
|
||||
async def embed_image(self, image):
|
||||
# Produce a deterministic-ish vector of the right dim.
|
||||
return [0.0] * 2560
|
||||
|
||||
|
|
|
|||
|
|
@ -465,7 +465,7 @@ async def test_chunk_interleaves_picture_in_structural_order(monkeypatch):
|
|||
@pytest.mark.asyncio
|
||||
async def test_embed_chunks_dispatches_text_vs_picture(monkeypatch):
|
||||
"""embed_chunks routes text chunks through embed_documents (batched) and
|
||||
picture chunks through embed_image_query (one at a time), reassembling
|
||||
picture chunks through embed_image (one at a time), reassembling
|
||||
in original order."""
|
||||
from haiku.rag.embeddings import EmbedderWrapper, embed_chunks
|
||||
from haiku.rag.store.models.chunk import Chunk
|
||||
|
|
@ -483,7 +483,7 @@ async def test_embed_chunks_dispatches_text_vs_picture(monkeypatch):
|
|||
text_calls.append(list(texts))
|
||||
return [[0.1, 0.2, 0.3, 0.4] for _ in texts]
|
||||
|
||||
async def embed_image_query(self, image):
|
||||
async def embed_image(self, image):
|
||||
image_calls.append(image)
|
||||
return [0.9, 0.8, 0.7, 0.6]
|
||||
|
||||
|
|
@ -554,7 +554,7 @@ async def test_ingest_emits_picture_chunks_with_multimodal_embedder(
|
|||
async def embed_documents(self, texts):
|
||||
return [[0.1] * 4 for _ in texts]
|
||||
|
||||
async def embed_image_query(self, image):
|
||||
async def embed_image(self, image):
|
||||
return [0.9] * 4
|
||||
|
||||
monkeypatch.setattr(
|
||||
|
|
|
|||
|
|
@ -89,9 +89,10 @@ def test_no_warning_when_doc_has_no_pictures(caplog_warnings):
|
|||
|
||||
|
||||
def test_warns_when_pictures_present_but_no_descriptions(caplog_warnings):
|
||||
"""The silent-failure case: VLM was requested, the doc has pictures,
|
||||
but the converter returned zero descriptions. Warn loudly so the user
|
||||
can fix their VLM config before a long ingest."""
|
||||
"""VLM was requested via ``picture_description.enabled = True``, the
|
||||
doc has pictures, but the converter returned zero descriptions
|
||||
(docling-serve swallows VLM errors). Warn loudly so the user can fix
|
||||
their VLM config before a long ingest."""
|
||||
config = AppConfig()
|
||||
config.processing.conversion_options.picture_description.enabled = True
|
||||
config.processing.conversion_options.picture_description.model.name = "qwen3.6"
|
||||
|
|
@ -128,9 +129,10 @@ def test_no_warning_when_at_least_one_description_came_back(caplog_warnings):
|
|||
async def test_convert_emits_warning_via_chokepoint(
|
||||
monkeypatch, tmp_path, caplog_warnings
|
||||
):
|
||||
"""End-to-end: ``convert(...)`` invokes the guard after the converter
|
||||
returns, so a silent VLM failure surfaces as a warning to the user
|
||||
regardless of which converter (local vs serve) actually ran."""
|
||||
"""End-to-end: ``convert(...)`` runs the description-missing check
|
||||
after the converter returns, so a VLM error swallowed inside
|
||||
docling-serve still surfaces as a warning at the haiku.rag layer
|
||||
regardless of which converter (local vs serve) ran."""
|
||||
from haiku.rag.converters.base import DocumentConverter
|
||||
|
||||
pdf = tmp_path / "fake.pdf"
|
||||
|
|
@ -162,3 +164,38 @@ async def test_convert_emits_warning_via_chokepoint(
|
|||
"0 described" in r.getMessage() and "fake.pdf" in r.getMessage()
|
||||
for r in caplog_warnings
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_convert_text_path_also_warns(monkeypatch, caplog_warnings):
|
||||
"""Raw text input (HTML, markdown) can still produce pictures via
|
||||
docling, so the description-missing check must run on the
|
||||
convert_text branch too — otherwise an HTML-with-images source
|
||||
would never trigger the warning even when picture_description is
|
||||
enabled and the VLM didn't actually run."""
|
||||
from haiku.rag.converters.base import DocumentConverter
|
||||
|
||||
class StubConverter(DocumentConverter):
|
||||
@property
|
||||
def supported_extensions(self) -> list[str]:
|
||||
return [".html"]
|
||||
|
||||
async def convert_file(self, path: Path):
|
||||
return _doc_without_pictures()
|
||||
|
||||
async def convert_text(
|
||||
self, text: str, name: str = "content.md", format: str = "md"
|
||||
):
|
||||
return _doc_with_pictures(with_descriptions=False)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"haiku.rag.client.processing.get_converter", lambda config: StubConverter()
|
||||
)
|
||||
|
||||
config = AppConfig()
|
||||
config.processing.conversion_options.picture_description.enabled = True
|
||||
|
||||
# No URL scheme, no Path → drops into the convert_text branch.
|
||||
await convert(config, "<html><img src='...'/></html>")
|
||||
|
||||
assert any("0 described" in r.getMessage() for r in caplog_warnings)
|
||||
|
|
|
|||
|
|
@ -336,7 +336,7 @@ def test_search_result_primary_label_prioritizes_structural_types():
|
|||
async def test_search_with_bytes_query_uses_multimodal_embedder(
|
||||
temp_db_path, monkeypatch
|
||||
):
|
||||
"""``client.search(bytes)`` embeds via ``embed_image_query`` and dispatches
|
||||
"""``client.search(bytes)`` embeds via ``embed_image`` and dispatches
|
||||
to vector-only chunk search (skipping FTS and reranker)."""
|
||||
from haiku.rag.embeddings import EmbedderWrapper
|
||||
from haiku.rag.store.models.chunk import Chunk
|
||||
|
|
@ -349,7 +349,7 @@ async def test_search_with_bytes_query_uses_multimodal_embedder(
|
|||
def __init__(self):
|
||||
super().__init__(embedder=None, vector_dim=4)
|
||||
|
||||
async def embed_image_query(self, image):
|
||||
async def embed_image(self, image):
|
||||
image_calls.append(image)
|
||||
return [0.5, 0.5, 0.5, 0.5]
|
||||
|
||||
|
|
@ -410,7 +410,7 @@ async def test_search_with_pil_image_works_like_bytes(temp_db_path, monkeypatch)
|
|||
def __init__(self):
|
||||
super().__init__(embedder=None, vector_dim=4)
|
||||
|
||||
async def embed_image_query(self, image):
|
||||
async def embed_image(self, image):
|
||||
seen_types.append(type(image))
|
||||
return [0.1] * 4
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue