From 0ea251219ccea34c42cc53d12c47be5d0b03c3f9 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 23 Jun 2026 11:51:48 +0300 Subject: [PATCH] Add explicit multimodal embedding flag, decoupled from provider name --- CHANGELOG.md | 5 ++ haiku_rag_slim/haiku/rag/config/models.py | 6 ++- haiku_rag_slim/haiku/rag/doctor.py | 5 +- .../haiku/rag/embeddings/__init__.py | 54 ++++++++++++++++--- haiku_rag_slim/haiku/rag/embeddings/vllm.py | 7 +-- tests/test_doctor.py | 14 +++-- tests/test_embedder.py | 35 ++++++++++++ 7 files changed, 111 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 477a083c..33bcdd70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ ### Added - `haiku-rag doctor` checks a database for consistency (orphaned chunks/items, chunk-less documents classified by content and embedder modality, dangling `doc_item_refs`, vector-dimension mismatch, unembedded chunks, missing picture data, settings/embedding drift, pending migrations, vector-index coverage, provider API keys) and probes configured provider endpoints (Ollama `/api/tags` with model presence, docling-serve `/health`, OpenAI-compatible/vLLM `/models`); exits 1 when any check fails. +- `embeddings.model.multimodal` (bool, default false) gates image embedding; `supports_images` derives from it instead of the provider name. + +### Changed + +- `provider: vllm` is text-only unless `embeddings.model.multimodal: true` is set. Existing multimodal vLLM configs must add the flag. ## [0.60.0] - 2026-06-22 diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index f6caa659..08622db7 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -39,16 +39,20 @@ class EmbeddingModelConfig(BaseModel): """Configuration for an embedding model. Attributes: - provider: Model provider (ollama, openai, voyageai, cohere, sentence-transformers) + provider: Model provider (ollama, openai, voyageai, cohere, sentence-transformers, vllm) name: Model name/identifier vector_dim: Vector dimensions produced by the model base_url: Optional base URL for OpenAI-compatible servers (vLLM, LM Studio, etc.) + multimodal: Whether the model embeds images into the same vector space as + text. Supported on the vllm, voyageai, and cohere providers; other + providers raise when this is set. """ provider: str = "ollama" name: str = "qwen3-embedding:4b" vector_dim: int = 2560 base_url: str | None = None + multimodal: bool = False class StorageConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/doctor.py b/haiku_rag_slim/haiku/rag/doctor.py index 275b167e..71d1c1d1 100644 --- a/haiku_rag_slim/haiku/rag/doctor.py +++ b/haiku_rag_slim/haiku/rag/doctor.py @@ -209,7 +209,10 @@ def _classify_unchunked( f"{len(picture_docs)} image-only document(s) have no chunks; " "a text-only embedder cannot index images." ), - remediation="Configure a multimodal embedder and rebuild to index images.", + remediation=( + "Set embeddings.model.multimodal: true on a vllm, voyageai, or " + "cohere model and rebuild to index images." + ), details=_sample(sorted(picture_docs)), ) ) diff --git a/haiku_rag_slim/haiku/rag/embeddings/__init__.py b/haiku_rag_slim/haiku/rag/embeddings/__init__.py index ce3e9e4a..21115575 100644 --- a/haiku_rag_slim/haiku/rag/embeddings/__init__.py +++ b/haiku_rag_slim/haiku/rag/embeddings/__init__.py @@ -10,6 +10,7 @@ from haiku.rag.config import AppConfig, Config if TYPE_CHECKING: from PIL import Image as PILImage + from haiku.rag.config.models import EmbeddingModelConfig from haiku.rag.store.models.chunk import Chunk @@ -19,15 +20,21 @@ ImageInput = "bytes | PILImage.Image" class EmbedderWrapper: """Wrapper around pydantic-ai Embedder with explicit query/document methods. - Subclasses set ``supports_images = True`` and override the image methods - when the underlying model can encode pictures into the same vector space. + Subclasses pass ``supports_images=True`` and override the image methods when + the underlying model can encode pictures into the same vector space. """ supports_images: bool = False - def __init__(self, embedder: Embedder | None, vector_dim: int): + def __init__( + self, + embedder: Embedder | None, + vector_dim: int, + supports_images: bool = False, + ): self._embedder = embedder self._vector_dim = vector_dim + self.supports_images = supports_images @property def vector_dim(self) -> int: @@ -159,6 +166,9 @@ def get_embedder(config: AppConfig = Config) -> EmbedderWrapper: model_name = embedding_model.name vector_dim = embedding_model.vector_dim + if embedding_model.multimodal: + return _get_multimodal_embedder(embedding_model) + if provider == "ollama": # Use model-level base_url if set, otherwise fall back to providers config base_url = embedding_model.base_url or config.providers.ollama.base_url @@ -193,9 +203,39 @@ def get_embedder(config: AppConfig = Config) -> EmbedderWrapper: if provider == "vllm": from haiku.rag.embeddings.vllm import VLLMMultimodalEmbedder - base_url = embedding_model.base_url or "http://localhost:8000/v1" - if not base_url.rstrip("/").endswith("/v1"): - base_url = base_url.rstrip("/") + "/v1" - return VLLMMultimodalEmbedder(model_name, vector_dim, base_url=base_url) + base_url = _vllm_base_url(embedding_model.base_url) + return VLLMMultimodalEmbedder( + model_name, vector_dim, base_url=base_url, supports_images=False + ) raise ValueError(f"Unsupported embedding provider: {provider}") + + +def _vllm_base_url(base_url: str | None) -> str: + base_url = base_url or "http://localhost:8000/v1" + if not base_url.rstrip("/").endswith("/v1"): + base_url = base_url.rstrip("/") + "/v1" + return base_url + + +def _get_multimodal_embedder( + embedding_model: "EmbeddingModelConfig", +) -> EmbedderWrapper: + """Build an image-capable embedder for providers that support multimodal. + + Each provider passes images in its own wire format, so the capability lives + in a per-provider embedder rather than a generic flag. + """ + provider = embedding_model.provider + model_name = embedding_model.name + vector_dim = embedding_model.vector_dim + + if provider == "vllm": + from haiku.rag.embeddings.vllm import VLLMMultimodalEmbedder + + base_url = _vllm_base_url(embedding_model.base_url) + return VLLMMultimodalEmbedder( + model_name, vector_dim, base_url=base_url, supports_images=True + ) + + raise ValueError(f"Provider '{provider}' does not support multimodal embedding.") diff --git a/haiku_rag_slim/haiku/rag/embeddings/vllm.py b/haiku_rag_slim/haiku/rag/embeddings/vllm.py index 6535f7ec..00023bed 100644 --- a/haiku_rag_slim/haiku/rag/embeddings/vllm.py +++ b/haiku_rag_slim/haiku/rag/embeddings/vllm.py @@ -24,8 +24,6 @@ if TYPE_CHECKING: class VLLMMultimodalEmbedder(EmbedderWrapper): - supports_images = True - def __init__( self, model_name: str, @@ -33,8 +31,11 @@ class VLLMMultimodalEmbedder(EmbedderWrapper): base_url: str, api_key: str | None = None, timeout: float = 60.0, + supports_images: bool = True, ): - super().__init__(embedder=None, vector_dim=vector_dim) + super().__init__( + embedder=None, vector_dim=vector_dim, supports_images=supports_images + ) self._model_name = model_name self._base_url = base_url.rstrip("/") self._api_key = api_key diff --git a/tests/test_doctor.py b/tests/test_doctor.py index c1760f03..d9a6fa62 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -49,11 +49,19 @@ VECTOR_DIM = 4 ChunkRecord = create_chunk_model(VECTOR_DIM) -def _config(provider: str = "ollama", name: str = "test", vector_dim: int = VECTOR_DIM): +def _config( + provider: str = "ollama", + name: str = "test", + vector_dim: int = VECTOR_DIM, + multimodal: bool = False, +): return AppConfig( embeddings=EmbeddingsConfig( model=EmbeddingModelConfig( - provider=provider, name=name, vector_dim=vector_dim + provider=provider, + name=name, + vector_dim=vector_dim, + multimodal=multimodal, ) ) ) @@ -312,7 +320,7 @@ async def test_image_only_document_multimodal_embedder_warns(temp_db_path): ], ) report = await run_doctor( - _config(provider="vllm", name="qwen-vl"), temp_db_path, {} + _config(provider="vllm", name="qwen-vl", multimodal=True), temp_db_path, {} ) result = _result(report, "documents_pictures_no_chunks") assert result.severity is Severity.WARN diff --git a/tests/test_embedder.py b/tests/test_embedder.py index 2f20db9c..3276db7c 100644 --- a/tests/test_embedder.py +++ b/tests/test_embedder.py @@ -630,6 +630,7 @@ async def test_vllm_get_embedder_routes_to_multimodal(): name="Qwen/Qwen3-VL-Embedding-2B", vector_dim=2048, base_url="http://my-vllm:8000/v1", + multimodal=True, ) ) ) @@ -638,6 +639,40 @@ async def test_vllm_get_embedder_routes_to_multimodal(): assert embedder._base_url == "http://my-vllm:8000/v1" # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] +def test_multimodal_defaults_to_false(): + model = EmbeddingModelConfig(provider="vllm", name="x", vector_dim=2) + assert model.multimodal is False + assert "multimodal" in model.model_dump() + + +async def test_vllm_text_only_when_multimodal_unset(): + config = AppConfig( + embeddings=EmbeddingsConfig( + model=EmbeddingModelConfig( + provider="vllm", + name="qwen3-embedding:4b", + vector_dim=2560, + base_url="http://my-vllm:8000/v1", + ) + ) + ) + embedder = get_embedder(config) + assert embedder.supports_images is False + + +@pytest.mark.parametrize("provider", ["ollama", "openai", "sentence-transformers"]) +async def test_multimodal_unsupported_provider_raises(provider): + config = AppConfig( + embeddings=EmbeddingsConfig( + model=EmbeddingModelConfig( + provider=provider, name="x", vector_dim=2, multimodal=True + ) + ) + ) + with pytest.raises(ValueError, match="does not support multimodal"): + get_embedder(config) + + @pytest.mark.vcr() async def test_vllm_embed_text_and_image_end_to_end(): """End-to-end against a real vLLM ``/v1/embeddings`` server: confirm