Update picture description prompt, move its config under prompts

This commit is contained in:
Yiorgis Gozadinos 2026-01-05 14:33:58 +02:00
parent 3c4116397d
commit 769e0d4746
No known key found for this signature in database
4 changed files with 26 additions and 8 deletions

View file

@ -99,7 +99,6 @@ conversion_options:
model:
provider: ollama # ollama, openai, or custom
name: ministral-3 # VLM model name
prompt: "Describe this image in detail. Be precise and concise."
timeout: 90 # Request timeout in seconds
max_tokens: 200 # Maximum tokens in response
```
@ -111,10 +110,24 @@ conversion_options:
- `provider`: `ollama` (default), `openai`, or use `base_url` for custom endpoints
- `name`: Model name (e.g., `ministral-3`, `granite3.2-vision`, `gpt-4-vision`)
- `base_url`: Optional custom API endpoint for vLLM, LM Studio, etc.
- **prompt**: Instruction for the VLM when describing images
- **timeout**: Request timeout in seconds
- **max_tokens**: Maximum tokens in the VLM response
**Default prompt** (configured in `prompts.picture_description`):
```
Describe this image for a blind user. State the image type
(screenshot, chart, photo, etc.), what it depicts, any visible text,
and key visual details. Be concise and accurate.
```
To customize the prompt globally:
```yaml
prompts:
picture_description: "Your custom prompt here..."
```
**Using with Ollama:**
```yaml

View file

@ -106,7 +106,6 @@ class PictureDescriptionConfig(BaseModel):
name="ministral-3",
)
)
prompt: str = "Describe this image in detail. Be precise and concise."
timeout: int = 90
max_tokens: int = 200
@ -186,6 +185,12 @@ class PromptsConfig(BaseModel):
domain_preamble: str = ""
qa: str | None = None
synthesis: str | None = None
picture_description: str = (
"Describe this image for a blind user. "
"State the image type (screenshot, chart, photo, etc.), "
"what it depicts, any visible text, and key visual details. "
"Be concise and accurate."
)
class AppConfig(BaseModel):

View file

@ -117,6 +117,8 @@ class DoclingLocalConverter(DocumentConverter):
if pic_desc.enabled:
from pydantic import AnyUrl
prompt = self.config.prompts.picture_description
pipeline_options.enable_remote_services = True
pipeline_options.picture_description_options = PictureDescriptionApiOptions(
url=AnyUrl(self._get_vlm_api_url(pic_desc.model)),
@ -124,7 +126,7 @@ class DoclingLocalConverter(DocumentConverter):
model=pic_desc.model.name,
max_completion_tokens=pic_desc.max_tokens,
),
prompt=pic_desc.prompt,
prompt=prompt,
timeout=pic_desc.timeout,
)

View file

@ -361,19 +361,17 @@ class TestDoclingLocalConverter:
assert (
config.processing.conversion_options.picture_description.max_tokens == 200
)
# Default prompt is in PromptsConfig
assert "blind user" in config.prompts.picture_description
def test_picture_description_config_applied(self, config):
"""Test that picture description config is applied to converter."""
config.processing.conversion_options.picture_description.enabled = True
config.processing.conversion_options.picture_description.prompt = (
"Custom prompt for testing."
)
config.processing.conversion_options.picture_description.timeout = 120
converter = DoclingLocalConverter(config)
pic_desc = converter.config.processing.conversion_options.picture_description
assert pic_desc.enabled is True
assert pic_desc.prompt == "Custom prompt for testing."
assert pic_desc.timeout == 120