From 21f12cd77054437fcb78e1d91d0d8616e962dac5 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 4 May 2026 10:47:21 +0300 Subject: [PATCH] add vision: bool flag on ModelConfig (default False). Gate the agent search tool's BinaryContent attachment on qa.model.vision so picture bytes are only sent to vision-capable QA models. --- CHANGELOG.md | 1 + docs/configuration/processing.md | 47 +++++++++++++++++++++++ haiku_rag_slim/haiku/rag/config/models.py | 2 + haiku_rag_slim/haiku/rag/tools/search.py | 3 ++ tests/test_picture_in_context.py | 47 ++++++++++++++++++++++- 5 files changed, 98 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 832c1aba..df0d51bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Added +- **`vision: bool` flag on `ModelConfig`.** Tracks whether a configured language model can interpret images. Default `False`. The agent's `search` tool only attaches picture bytes (as `BinaryContent`) to the `ToolReturn` when `qa.model.vision = True`. Without the gate, sending image content to a text-only model behaves inconsistently across providers — Ollama silently accepts and the model hallucinates a confident wrong answer; OpenAI returns 400; others vary. Capability detection from a probe or a model-name whitelist is unreliable, so `vision` is an explicit user-set capability declaration. Set it to `True` for vision-capable QA models (`qwen2.5vl`, `qwen3.6`, `gpt-4o`, `claude-sonnet`, ...). - **Synthetic picture chunks at ingest under multimodal embedders.** `build_picture_chunks` (in `client/processing.py`) walks a `DoclingDocument`'s `pictures` and emits one synthetic `Chunk` per `PictureItem` with available bytes. Bytes ride on a `Chunk._picture_data` PrivateAttr (not serialized) so `embed_chunks` can route them through `embed_images` while text chunks keep going through `embed_documents`. Wired into the three ingest paths (`create_document`, `_create_document_from_file`, `_create_or_update_document_from_url`, `_update_document_with_chunks`, and `_rebuild_rechunk`) — guarded by `embedder.supports_images` so text-only configurations are unaffected. Snapshot/merge with `existing_picture_data` keeps picture chunks alive across rebuild round-trips. Picture chunks land in the same `chunks` table with the same vector dim as text chunks, so cross-modal search reuses the existing hybrid+RRF pipeline. - **Multimodal embedder support (`provider="vllm"`).** `EmbedderWrapper` gains `supports_images: bool` and `embed_image_query`. The `vllm` provider talks HTTP to a vLLM server's OpenAI-compatible `/v1/embeddings` endpoint — text inputs use the standard `input` field for true server-side batching; image inputs use vLLM's `messages` superset with `image_url` content parts carrying base64 data URIs. Works with `Qwen/Qwen3-VL-Embedding-8B` and `jinaai/jina-embeddings-v4`. No Python ML deps added — uses `httpx`. Text-only providers (`ollama`, `openai`, `cohere`, `sentence-transformers`) report `supports_images=False` and raise a clear error if image methods are called. - **Picture-handling mode enum.** `processing.pictures: "none" | "description" | "image"` (default `"none"`) replaces the legacy `generate_picture_images` + `picture_description.enabled` pair. diff --git a/docs/configuration/processing.md b/docs/configuration/processing.md index fe006026..b45e6a12 100644 --- a/docs/configuration/processing.md +++ b/docs/configuration/processing.md @@ -137,6 +137,53 @@ processing: **Switching modes on an existing database.** No reingest is required if you only need to change between `description` and `image` — the bytes are already there. Run `haiku-rag rebuild --rechunk` after the config change so the chunk-text composition reflects the new mode. Switching *down* to `none` clears `picture_data` on rebuild, reclaiming storage. +#### Pictures × embedder × QA model: how the pieces compose + +Three orthogonal settings drive what gets stored, what gets retrieved, and what reaches the QA model. Each setting answers one question: + +| Setting | Question it answers | Values | +|---|---|---| +| `processing.pictures` | What gets captured at ingest? | `none` / `description` / `image` | +| `embeddings.model.provider` | Can the embedder index image content? | text-only providers (`ollama`, `openai`, `cohere`, `sentence-transformers`) vs `vllm` (multimodal) | +| `qa.model.vision` | Can the QA model interpret images? | `false` (default) / `true` | + +**What gets stored** for each `pictures` × embedder combination: + +| `pictures` | Embedder | Text chunks contain… | `document_items.picture_data` | Synthetic picture chunks | +|---|---|---|---|---| +| `none` | text-only or multimodal | regular text only | empty | none | +| `description` | text-only | text + VLM descriptions | populated (kept for later use) | none | +| `description` | multimodal | text + VLM descriptions | populated | one per picture, content = description, vector = image embedding | +| `image` | text-only | text only (caption/surrounding) | populated (kept for later use) | none | +| `image` | multimodal | text only | populated | one per picture, content = caption/empty, vector = image embedding | + +**What QA receives** at search time, given stored state and `qa.model.vision`: + +| `pictures` at ingest | `qa.model.vision` | QA receives | +|---|---|---| +| `none` | either | text chunks only | +| `description` | `false` | text chunks (descriptions in chunk text answer figure questions in prose) | +| `description` | `true` | text chunks + raw picture bytes; vision model uses both signals | +| `image` | `false` | text chunks only (caption + surrounding text); model has no figure content to draw on | +| `image` | `true` | text chunks + raw picture bytes; vision model reads the figures directly | + +A few invariants worth knowing: + +- **`qa.model.vision` is independent of ingestion.** It only controls whether the agent's `search` tool attaches picture bytes to its `ToolReturn`. A text-only QA model with `vision: true` won't suddenly understand images — it will silently accept the bytes and confabulate. Default `false` is the safe choice. +- **`description` mode preserves the bytes**, so flipping the QA strategy later (text-only → vision, or vice versa) doesn't require reingesting. Just change `qa.model.vision` and optionally `qa.model` itself. +- **Cross-modal search** (text query → picture-chunk hits) requires a multimodal embedder. With a text-only embedder, picture-chunk vectors aren't generated; figures only surface via section-bounded expansion off matching text chunks. +- **`image` mode + text-only embedder** is rarely the right choice — pictures are stored but neither searchable as image vectors nor described in text. Picture bytes are then only reachable via expand-context's section bounds when an adjacent text chunk matches. + +**Recommended combinations** by use case: + +| Use case | `pictures` | Embedder | `qa.model.vision` | +|---|---|---|---| +| Pure text RAG, no figures | `none` | text-only | `false` | +| Text RAG, figures answered through descriptions | `description` | text-only | `false` | +| Vision QA on figure-rich docs (no cross-modal search) | `description` | text-only | `true` | +| Cross-modal search + vision QA (the full multimodal stack) | `description` or `image` | multimodal | `true` | +| Cross-modal search, text QA only | `description` | multimodal | `false` | + **`picture_description.model` configuration** (used only under `pictures: description`): - **model**: Standard model configuration diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 56750ebc..34404473 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -16,6 +16,7 @@ class ModelConfig(BaseModel): enable_thinking: Control reasoning behavior (true/false/None for default) temperature: Sampling temperature (0.0 to 1.0+) max_tokens: Maximum tokens to generate + vision: True if the model can interpret images. Default False. """ provider: str = "ollama" @@ -25,6 +26,7 @@ class ModelConfig(BaseModel): enable_thinking: bool | None = None temperature: float | None = None max_tokens: int | None = None + vision: bool = False class EmbeddingModelConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/tools/search.py b/haiku_rag_slim/haiku/rag/tools/search.py index 1c76a3fd..4b4e756c 100644 --- a/haiku_rag_slim/haiku/rag/tools/search.py +++ b/haiku_rag_slim/haiku/rag/tools/search.py @@ -90,6 +90,9 @@ def create_search_toolset( ] text = "\n\n".join(formatted) + if not config.qa.model.vision: + return text + binary_parts: list[BinaryContent] = [] seen: set[str] = set() for result in results_list: diff --git a/tests/test_picture_in_context.py b/tests/test_picture_in_context.py index 18b0b77a..7123545a 100644 --- a/tests/test_picture_in_context.py +++ b/tests/test_picture_in_context.py @@ -284,7 +284,9 @@ class _Deps: @pytest.mark.asyncio async def test_search_tool_returns_multimodal_when_picture_present(): """The agent-facing search tool must wrap text + BinaryContent in ToolReturn - whenever a result carries picture image_data.""" + whenever a result carries picture image_data AND the QA model is vision-capable.""" + from haiku.rag.config import AppConfig + picture_result = SearchResult( content="A diagram of the layout", score=1.0, @@ -299,7 +301,9 @@ async def test_search_tool_returns_multimodal_when_picture_present(): fake_client.search = AsyncMock(return_value=[picture_result]) fake_client.expand_context = AsyncMock(return_value=[picture_result]) - toolset = create_search_toolset(Config, expand_context=False) + config = AppConfig() + config.qa.model.vision = True + toolset = create_search_toolset(config, expand_context=False) func = toolset.tools["search"].function ctx = RunContext( @@ -568,6 +572,45 @@ async def test_ingest_emits_picture_chunks_with_multimodal_embedder( ) +@pytest.mark.asyncio +async def test_search_tool_skips_binary_content_when_qa_model_is_text_only(): + """The agent search tool must NOT attach picture bytes when the QA model + is text-only (``qa.model.vision = False``, the default). Sending image + parts to a text-only model would cause it to hallucinate confidently — + Ollama silently accepts the bytes and the model guesses.""" + from haiku.rag.config import AppConfig + + picture_result = SearchResult( + content="A diagram of the layout", + score=1.0, + chunk_id="chunk-1", + document_id="doc-1", + doc_item_refs=["#/pictures/0"], + labels=["picture"], + image_data={"#/pictures/0": PICTURE_B64}, + ) + + fake_client = AsyncMock() + fake_client.search = AsyncMock(return_value=[picture_result]) + fake_client.expand_context = AsyncMock(return_value=[picture_result]) + + config = AppConfig() + # vision defaults to False; assert anyway so the test reads explicitly. + assert config.qa.model.vision is False + toolset = create_search_toolset(config, expand_context=False) + func = toolset.tools["search"].function + + ctx = RunContext( + deps=_Deps(client=fake_client), # type: ignore[arg-type] + model=TestModel(), + usage=RunUsage(), + run_id="run-1", + ) + result = await func(ctx, "anything") + + assert isinstance(result, str) + + @pytest.mark.asyncio async def test_search_tool_returns_plain_string_when_no_pictures(): """When no result carries image_data the tool returns a plain str (no