Update picture description prompt, move its config under prompts
This commit is contained in:
parent
3c4116397d
commit
769e0d4746
4 changed files with 26 additions and 8 deletions
|
|
@ -99,7 +99,6 @@ conversion_options:
|
|||
model:
|
||||
provider: ollama # ollama, openai, or custom
|
||||
name: ministral-3 # VLM model name
|
||||
prompt: "Describe this image in detail. Be precise and concise."
|
||||
timeout: 90 # Request timeout in seconds
|
||||
max_tokens: 200 # Maximum tokens in response
|
||||
```
|
||||
|
|
@ -111,10 +110,24 @@ conversion_options:
|
|||
- `provider`: `ollama` (default), `openai`, or use `base_url` for custom endpoints
|
||||
- `name`: Model name (e.g., `ministral-3`, `granite3.2-vision`, `gpt-4-vision`)
|
||||
- `base_url`: Optional custom API endpoint for vLLM, LM Studio, etc.
|
||||
- **prompt**: Instruction for the VLM when describing images
|
||||
- **timeout**: Request timeout in seconds
|
||||
- **max_tokens**: Maximum tokens in the VLM response
|
||||
|
||||
**Default prompt** (configured in `prompts.picture_description`):
|
||||
|
||||
```
|
||||
Describe this image for a blind user. State the image type
|
||||
(screenshot, chart, photo, etc.), what it depicts, any visible text,
|
||||
and key visual details. Be concise and accurate.
|
||||
```
|
||||
|
||||
To customize the prompt globally:
|
||||
|
||||
```yaml
|
||||
prompts:
|
||||
picture_description: "Your custom prompt here..."
|
||||
```
|
||||
|
||||
**Using with Ollama:**
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
|
|
@ -106,7 +106,6 @@ class PictureDescriptionConfig(BaseModel):
|
|||
name="ministral-3",
|
||||
)
|
||||
)
|
||||
prompt: str = "Describe this image in detail. Be precise and concise."
|
||||
timeout: int = 90
|
||||
max_tokens: int = 200
|
||||
|
||||
|
|
@ -186,6 +185,12 @@ class PromptsConfig(BaseModel):
|
|||
domain_preamble: str = ""
|
||||
qa: str | None = None
|
||||
synthesis: str | None = None
|
||||
picture_description: str = (
|
||||
"Describe this image for a blind user. "
|
||||
"State the image type (screenshot, chart, photo, etc.), "
|
||||
"what it depicts, any visible text, and key visual details. "
|
||||
"Be concise and accurate."
|
||||
)
|
||||
|
||||
|
||||
class AppConfig(BaseModel):
|
||||
|
|
|
|||
|
|
@ -117,6 +117,8 @@ class DoclingLocalConverter(DocumentConverter):
|
|||
if pic_desc.enabled:
|
||||
from pydantic import AnyUrl
|
||||
|
||||
prompt = self.config.prompts.picture_description
|
||||
|
||||
pipeline_options.enable_remote_services = True
|
||||
pipeline_options.picture_description_options = PictureDescriptionApiOptions(
|
||||
url=AnyUrl(self._get_vlm_api_url(pic_desc.model)),
|
||||
|
|
@ -124,7 +126,7 @@ class DoclingLocalConverter(DocumentConverter):
|
|||
model=pic_desc.model.name,
|
||||
max_completion_tokens=pic_desc.max_tokens,
|
||||
),
|
||||
prompt=pic_desc.prompt,
|
||||
prompt=prompt,
|
||||
timeout=pic_desc.timeout,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -361,19 +361,17 @@ class TestDoclingLocalConverter:
|
|||
assert (
|
||||
config.processing.conversion_options.picture_description.max_tokens == 200
|
||||
)
|
||||
# Default prompt is in PromptsConfig
|
||||
assert "blind user" in config.prompts.picture_description
|
||||
|
||||
def test_picture_description_config_applied(self, config):
|
||||
"""Test that picture description config is applied to converter."""
|
||||
config.processing.conversion_options.picture_description.enabled = True
|
||||
config.processing.conversion_options.picture_description.prompt = (
|
||||
"Custom prompt for testing."
|
||||
)
|
||||
config.processing.conversion_options.picture_description.timeout = 120
|
||||
converter = DoclingLocalConverter(config)
|
||||
|
||||
pic_desc = converter.config.processing.conversion_options.picture_description
|
||||
assert pic_desc.enabled is True
|
||||
assert pic_desc.prompt == "Custom prompt for testing."
|
||||
assert pic_desc.timeout == 120
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue