From 248eed8d7be40c7be9d27121a0e2551334f7ad18 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 7 Jan 2026 10:53:45 +0200 Subject: [PATCH] Update docs --- CHANGELOG.md | 9 +++++++ docs/configuration/processing.md | 10 ++++++- docs/configuration/prompts.md | 27 +++++++++++++++++++ docs/remote-processing.md | 3 +-- .../ag-ui-research/haiku.rag.yaml.example | 1 - examples/docker/haiku.rag.yaml.example | 1 - 6 files changed, 46 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cb040d6..cd583aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,15 @@ # Changelog ## [Unreleased] +### Added + +- **VLM Picture Description**: Describe embedded images using Vision Language Models during document conversion + - Images are sent to a VLM for automatic description via OpenAI-compatible API + - Descriptions become searchable text, improving RAG retrieval for visual content + - Configure via `processing.conversion_options.picture_description` with `enabled`, `model`, `timeout`, `max_tokens` + - Default prompt customizable via `prompts.picture_description` + - Requires OpenAI-compatible `/v1/chat/completions` endpoint (Ollama, OpenAI, vLLM, LM Studio) + ## [0.23.2] - 2026-01-05 ### Fixed diff --git a/docs/configuration/processing.md b/docs/configuration/processing.md index 9d88f5e0..cd1a8321 100644 --- a/docs/configuration/processing.md +++ b/docs/configuration/processing.md @@ -113,6 +113,8 @@ conversion_options: - **timeout**: Request timeout in seconds - **max_tokens**: Maximum tokens in the VLM response +**Note:** Requires an OpenAI-compatible `/v1/chat/completions` endpoint. Providers with different API formats (e.g., Anthropic Claude) are not supported. + **Default prompt** (configured in `prompts.picture_description`): ``` @@ -165,6 +167,13 @@ conversion_options: 3. Descriptions are added as annotations on the image 4. When exported to markdown, descriptions appear as searchable text +**Using with docling-serve:** + +When using `converter: docling-serve`, the VLM calls are made by the docling-serve instance, not by haiku.rag. You must: + +1. Set `DOCLING_SERVE_ENABLE_REMOTE_SERVICES=true` when running docling-serve +2. Ensure the VLM endpoint is accessible from where docling-serve is running + ### Local vs Remote Processing **Local processing** (default): @@ -190,7 +199,6 @@ providers: docling_serve: base_url: http://localhost:5001 api_key: "your-api-key" # Optional - timeout: 300 # Request timeout in seconds ``` Conversion options work identically for both local and remote processing. diff --git a/docs/configuration/prompts.md b/docs/configuration/prompts.md index 30574c3c..14454d7c 100644 --- a/docs/configuration/prompts.md +++ b/docs/configuration/prompts.md @@ -16,6 +16,9 @@ prompts: # Full replacement for research synthesis prompt (optional) synthesis: null + + # VLM prompt for image description during conversion (optional) + picture_description: null # Uses default prompt ``` ## Domain Preamble @@ -92,6 +95,29 @@ prompts: - Avoid meta-commentary like "This report covers..." ``` +## Picture Description Prompt + +Customize the prompt used when generating VLM descriptions for embedded images during document conversion. This prompt is sent to the configured Vision Language Model for each image. + +**Default prompt:** + +``` +Describe this image for a blind user. State the image type (screenshot, chart, photo, etc.), +what it depicts, any visible text, and key visual details. Be concise and accurate. +``` + +**Custom example:** + +```yaml +prompts: + picture_description: | + Describe this image for a document search system. + Focus on: image type, main content, any text, key visual elements. + Be concise and factual. +``` + +The prompt is used when `processing.conversion_options.picture_description.enabled` is `true`. See [Picture Description (VLM)](processing.md#picture-description-vlm) for full configuration. + ## Programmatic Configuration ```python @@ -103,6 +129,7 @@ config = AppConfig( domain_preamble="You are answering questions about our product documentation.", qa=None, # Use default QA prompt synthesis=None, # Use default synthesis prompt + picture_description="Describe this image for search indexing.", ) ) ``` diff --git a/docs/remote-processing.md b/docs/remote-processing.md index 9b92beae..c841c394 100644 --- a/docs/remote-processing.md +++ b/docs/remote-processing.md @@ -55,8 +55,7 @@ processing: providers: docling_serve: base_url: http://localhost:5001 - api_key: "" # Optional API key for authentication - timeout: 300 # Request timeout in seconds + api_key: "" # Optional API key for authentication ``` ## Features diff --git a/examples/ag-ui-research/haiku.rag.yaml.example b/examples/ag-ui-research/haiku.rag.yaml.example index 461b2e7c..7ce0f926 100644 --- a/examples/ag-ui-research/haiku.rag.yaml.example +++ b/examples/ag-ui-research/haiku.rag.yaml.example @@ -12,7 +12,6 @@ providers: docling_serve: base_url: http://docling-serve:5001 api_key: "" - timeout: 300 ollama: base_url: http://host.docker.internal:11434 diff --git a/examples/docker/haiku.rag.yaml.example b/examples/docker/haiku.rag.yaml.example index 616e3ada..64376588 100644 --- a/examples/docker/haiku.rag.yaml.example +++ b/examples/docker/haiku.rag.yaml.example @@ -24,7 +24,6 @@ providers: docling_serve: base_url: http://docling-serve:5001 api_key: "" - timeout: 300 ollama: base_url: http://host.docker.internal:11434