Add VoyageAI multimodal embedder

CHANGELOG.md

@@ -5,6 +5,7 @@
 
 - `haiku-rag doctor` checks a database for consistency (orphaned chunks/items, chunk-less documents classified by content and embedder modality, dangling `doc_item_refs`, vector-dimension mismatch, unembedded chunks, missing picture data, settings/embedding drift, pending migrations, vector-index coverage, provider API keys) and probes configured provider endpoints (Ollama `/api/tags` with model presence, docling-serve `/health`, OpenAI-compatible/vLLM `/models`); exits 1 when any check fails.
 - `embeddings.model.multimodal` (bool, default false) gates image embedding; `supports_images` derives from it instead of the provider name.
+- VoyageAI multimodal embedder (`provider: voyageai`, `multimodal: true`, e.g. `voyage-multimodal-3`) embedding text and pictures into a shared vector space.
 
 ### Changed
 

haiku_rag_slim/haiku/rag/client/search.py

@@ -58,8 +58,9 @@ async def search(
         embedder = client.embedder
         if not embedder.supports_images:
             raise ValueError(
-                "Image queries require a multimodal embedder. Configure "
-                "provider='vllm' (or another image-capable provider)."
+                "Image queries require a multimodal embedder. Set "
+                "embeddings.model.multimodal: true on a vllm, voyageai, or cohere "
+                "model."
             )
         query_vector = await embedder.embed_image(query)
         chunk_results = await client.chunk_repository.search(

haiku_rag_slim/haiku/rag/embeddings/__init__.py

@@ -20,8 +20,9 @@ ImageInput = "bytes | PILImage.Image"
 class EmbedderWrapper:
     """Wrapper around pydantic-ai Embedder with explicit query/document methods.
 
-    Subclasses pass ``supports_images=True`` and override the image methods when
-    the underlying model can encode pictures into the same vector space.
+    Subclasses that can encode pictures into the same vector space as text either
+    set the ``supports_images`` class attribute or pass ``supports_images=True``,
+    and override the image methods.
     """
 
     supports_images: bool = False
@@ -30,11 +31,12 @@ class EmbedderWrapper:
         self,
         embedder: Embedder | None,
         vector_dim: int,
-        supports_images: bool = False,
+        supports_images: bool | None = None,
     ):
         self._embedder = embedder
         self._vector_dim = vector_dim
-        self.supports_images = supports_images
+        if supports_images is not None:
+            self.supports_images = supports_images
 
     @property
     def vector_dim(self) -> int:
@@ -62,8 +64,8 @@ class EmbedderWrapper:
         ``messages`` superset. Callers loop when they need many.
         """
         raise NotImplementedError(
-            f"{type(self).__name__} does not support image embedding. "
-            "Configure a multimodal provider (e.g. provider='vllm')."
+            f"{type(self).__name__} does not support image embedding. Set "
+            "embeddings.model.multimodal: true on a vllm, voyageai, or cohere model."
         )
 
 
@@ -124,8 +126,9 @@ async def embed_chunks(
     if picture_chunks:
         if not embedder.supports_images:
             raise ValueError(
-                "Picture chunks require a multimodal embedder. Configure "
-                "provider='vllm', or omit picture chunks."
+                "Picture chunks require a multimodal embedder. Set "
+                "embeddings.model.multimodal: true on a vllm, voyageai, or cohere "
+                "model, or omit picture chunks."
             )
         for chunk in picture_chunks:
             picture_embeddings.append(await embedder.embed_image(chunk._picture_data))
@@ -238,4 +241,12 @@ def _get_multimodal_embedder(
             model_name, vector_dim, base_url=base_url, supports_images=True
         )
 
-    raise ValueError(f"Provider '{provider}' does not support multimodal embedding.")
+    if provider == "voyageai":
+        from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+        return VoyageMultimodalEmbedder(model_name, vector_dim)
+
+    raise ValueError(
+        f"Provider '{provider}' does not support multimodal embedding. Set "
+        "embeddings.model.multimodal: true on a vllm, voyageai, or cohere model."
+    )

haiku_rag_slim/haiku/rag/embeddings/vllm.py

@@ -100,6 +100,11 @@ class VLLMMultimodalEmbedder(EmbedderWrapper):
         )
 
     async def embed_image(self, image: "bytes | PILImage.Image") -> list[float]:
+        if not self.supports_images:
+            raise NotImplementedError(
+                "This vLLM embedder is text-only. Set "
+                "embeddings.model.multimodal: true to embed images."
+            )
         rows = await self._post(
             {
                 "model": self._model_name,

haiku_rag_slim/haiku/rag/embeddings/voyageai.py

@@ -0,0 +1,68 @@
+"""Multimodal embedder backed by VoyageAI's ``multimodal_embed`` endpoint.
+
+``voyage-multimodal-3`` maps text and images into a shared vector space. Text is
+embedded as single-element content lists; images are passed as ``PIL.Image``
+objects (the SDK accepts them directly). The API key is read from the
+environment (``VOYAGE_API_KEY``) like the text-only Voyage path.
+"""
+
+import io
+from typing import TYPE_CHECKING
+
+from haiku.rag.embeddings import EmbedderWrapper
+
+if TYPE_CHECKING:
+    from PIL import Image as PILImage
+
+
+class VoyageMultimodalEmbedder(EmbedderWrapper):
+    def __init__(
+        self,
+        model_name: str,
+        vector_dim: int,
+        api_key: str | None = None,
+    ):
+        super().__init__(embedder=None, vector_dim=vector_dim, supports_images=True)
+        import voyageai
+
+        self._model_name = model_name
+        self._client = voyageai.AsyncClient(api_key=api_key)
+
+    async def embed_query(self, text: str) -> list[float]:
+        result = await self._client.multimodal_embed(
+            inputs=[[text]],
+            model=self._model_name,
+            input_type="query",
+            output_dimension=self._vector_dim,
+        )
+        return list(result.embeddings[0])
+
+    async def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        result = await self._client.multimodal_embed(
+            inputs=[[text] for text in texts],
+            model=self._model_name,
+            input_type="document",
+            output_dimension=self._vector_dim,
+        )
+        return [list(e) for e in result.embeddings]
+
+    async def embed_image(self, image: "bytes | PILImage.Image") -> list[float]:
+        result = await self._client.multimodal_embed(
+            inputs=[[_to_pil(image)]],
+            model=self._model_name,
+            input_type="document",
+            output_dimension=self._vector_dim,
+        )
+        return list(result.embeddings[0])
+
+
+def _to_pil(image: "bytes | PILImage.Image") -> "PILImage.Image":
+    from PIL import Image as PILImageModule
+
+    if isinstance(image, bytes):
+        return PILImageModule.open(io.BytesIO(image))
+    if isinstance(image, PILImageModule.Image):
+        return image
+    raise TypeError(f"Unsupported image type: {type(image)!r}")

tests/test_embedder.py

@@ -308,7 +308,7 @@ def _ollama_text_only_config():
 async def test_text_only_embedder_does_not_support_images():
     embedder = get_embedder(_ollama_text_only_config())
     assert embedder.supports_images is False
-    with pytest.raises(NotImplementedError, match="multimodal provider"):
+    with pytest.raises(NotImplementedError, match="multimodal"):
         await embedder.embed_image(b"\x89PNG\r\n\x1a\n")
 
 
@@ -417,6 +417,20 @@ async def test_vllm_supports_images_flag():
     assert embedder.supports_images is True
 
 
+async def test_vllm_text_only_embed_image_raises():
+    from haiku.rag.embeddings.vllm import VLLMMultimodalEmbedder
+
+    embedder = VLLMMultimodalEmbedder(
+        model_name="x",
+        vector_dim=2,
+        base_url="http://localhost:8000/v1",
+        supports_images=False,
+    )
+    assert embedder.supports_images is False
+    with pytest.raises(NotImplementedError, match="text-only"):
+        await embedder.embed_image(b"\x89PNG\r\n\x1a\n")
+
+
 async def test_vllm_connect_error_surfaces_helpful_message(monkeypatch):
     import httpx
 
@@ -707,3 +721,136 @@ async def test_vllm_embed_text_and_image_end_to_end():
     image_vec = await embedder.embed_image(image)
     assert len(image_vec) == 4096
     assert any(abs(x) > 1e-6 for x in image_vec), "image embedding is all zeros"
+
+
+class _FakeVoyageResult:
+    def __init__(self, embeddings):
+        self.embeddings = embeddings
+
+
+def _fake_voyage_client(captured, embeddings):
+    class FakeAsyncClient:
+        def __init__(self, *args, **kwargs):
+            captured["init"] = kwargs
+
+        async def multimodal_embed(self, **kwargs):
+            captured.update(kwargs)
+            return _FakeVoyageResult(embeddings)
+
+    return FakeAsyncClient
+
+
+async def test_voyage_embed_documents_request_shape(monkeypatch):
+    from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+    captured: dict = {}
+    monkeypatch.setattr(
+        "voyageai.AsyncClient",
+        _fake_voyage_client(captured, [[0.1, 0.2], [0.3, 0.4]]),
+    )
+
+    embedder = VoyageMultimodalEmbedder("voyage-multimodal-3", vector_dim=2)
+    vecs = await embedder.embed_documents(["a cat", "a dog"])
+
+    assert vecs == [[0.1, 0.2], [0.3, 0.4]]
+    assert captured["model"] == "voyage-multimodal-3"
+    assert captured["input_type"] == "document"
+    assert captured["inputs"] == [["a cat"], ["a dog"]]
+    assert captured["output_dimension"] == 2
+
+
+async def test_voyage_embed_query_request_shape(monkeypatch):
+    from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+    captured: dict = {}
+    monkeypatch.setattr(
+        "voyageai.AsyncClient", _fake_voyage_client(captured, [[0.5, 0.6]])
+    )
+
+    embedder = VoyageMultimodalEmbedder("voyage-multimodal-3", vector_dim=2)
+    vec = await embedder.embed_query("find the cat")
+
+    assert vec == [0.5, 0.6]
+    assert captured["model"] == "voyage-multimodal-3"
+    assert captured["input_type"] == "query"
+    assert captured["inputs"] == [["find the cat"]]
+
+
+async def test_voyage_embed_image_passes_pil(monkeypatch):
+    from PIL import Image
+
+    from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+    captured: dict = {}
+    monkeypatch.setattr(
+        "voyageai.AsyncClient", _fake_voyage_client(captured, [[0.7, 0.8]])
+    )
+
+    embedder = VoyageMultimodalEmbedder("voyage-multimodal-3", vector_dim=2)
+    import io
+
+    buf = io.BytesIO()
+    Image.new("RGB", (4, 4), "red").save(buf, format="PNG")
+    vec = await embedder.embed_image(buf.getvalue())
+
+    assert vec == [0.7, 0.8]
+    assert captured["model"] == "voyage-multimodal-3"
+    inputs = captured["inputs"]
+    assert len(inputs) == 1 and len(inputs[0]) == 1
+    assert isinstance(inputs[0][0], Image.Image)
+    assert captured["input_type"] == "document"
+
+
+async def test_voyage_embed_documents_empty_list_skips_request(monkeypatch):
+    from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+    captured: dict = {}
+    monkeypatch.setattr("voyageai.AsyncClient", _fake_voyage_client(captured, []))
+
+    embedder = VoyageMultimodalEmbedder("voyage-multimodal-3", vector_dim=2)
+    assert await embedder.embed_documents([]) == []
+    assert "inputs" not in captured
+
+
+async def test_voyage_get_embedder_routes_to_multimodal(monkeypatch):
+    monkeypatch.setattr("voyageai.AsyncClient", _fake_voyage_client({}, []))
+    from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+    config = AppConfig(
+        embeddings=EmbeddingsConfig(
+            model=EmbeddingModelConfig(
+                provider="voyageai",
+                name="voyage-multimodal-3",
+                vector_dim=1024,
+                multimodal=True,
+            )
+        )
+    )
+    embedder = get_embedder(config)
+    assert isinstance(embedder, VoyageMultimodalEmbedder)
+    assert embedder.supports_images is True
+
+
+@pytest.mark.vcr()
+async def test_voyage_embed_text_and_image_end_to_end():
+    """End-to-end against the real VoyageAI ``multimodal_embed`` API: text and
+    image inputs return embeddings of the configured dimension in a shared
+    vector space. To re-record, set ``VOYAGE_API_KEY`` and run with
+    ``--record-mode=rewrite``."""
+    from PIL import Image
+
+    from haiku.rag.embeddings.voyageai import VoyageMultimodalEmbedder
+
+    embedder = VoyageMultimodalEmbedder("voyage-multimodal-3", vector_dim=1024)
+
+    text_vec = await embedder.embed_query("a photo of a red square")
+    assert len(text_vec) == 1024
+    assert any(abs(x) > 1e-6 for x in text_vec), "text embedding is all zeros"
+
+    text_batch = await embedder.embed_documents(["hello world", "another doc"])
+    assert len(text_batch) == 2
+    assert all(len(v) == 1024 for v in text_batch)
+
+    image_vec = await embedder.embed_image(Image.new("RGB", (64, 64), (255, 0, 0)))
+    assert len(image_vec) == 1024
+    assert any(abs(x) > 1e-6 for x in image_vec), "image embedding is all zeros"