Update picture description prompt, move its config under prompts
This commit is contained in:
parent
3c4116397d
commit
769e0d4746
4 changed files with 26 additions and 8 deletions
|
|
@ -99,7 +99,6 @@ conversion_options:
|
||||||
model:
|
model:
|
||||||
provider: ollama # ollama, openai, or custom
|
provider: ollama # ollama, openai, or custom
|
||||||
name: ministral-3 # VLM model name
|
name: ministral-3 # VLM model name
|
||||||
prompt: "Describe this image in detail. Be precise and concise."
|
|
||||||
timeout: 90 # Request timeout in seconds
|
timeout: 90 # Request timeout in seconds
|
||||||
max_tokens: 200 # Maximum tokens in response
|
max_tokens: 200 # Maximum tokens in response
|
||||||
```
|
```
|
||||||
|
|
@ -111,10 +110,24 @@ conversion_options:
|
||||||
- `provider`: `ollama` (default), `openai`, or use `base_url` for custom endpoints
|
- `provider`: `ollama` (default), `openai`, or use `base_url` for custom endpoints
|
||||||
- `name`: Model name (e.g., `ministral-3`, `granite3.2-vision`, `gpt-4-vision`)
|
- `name`: Model name (e.g., `ministral-3`, `granite3.2-vision`, `gpt-4-vision`)
|
||||||
- `base_url`: Optional custom API endpoint for vLLM, LM Studio, etc.
|
- `base_url`: Optional custom API endpoint for vLLM, LM Studio, etc.
|
||||||
- **prompt**: Instruction for the VLM when describing images
|
|
||||||
- **timeout**: Request timeout in seconds
|
- **timeout**: Request timeout in seconds
|
||||||
- **max_tokens**: Maximum tokens in the VLM response
|
- **max_tokens**: Maximum tokens in the VLM response
|
||||||
|
|
||||||
|
**Default prompt** (configured in `prompts.picture_description`):
|
||||||
|
|
||||||
|
```
|
||||||
|
Describe this image for a blind user. State the image type
|
||||||
|
(screenshot, chart, photo, etc.), what it depicts, any visible text,
|
||||||
|
and key visual details. Be concise and accurate.
|
||||||
|
```
|
||||||
|
|
||||||
|
To customize the prompt globally:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
prompts:
|
||||||
|
picture_description: "Your custom prompt here..."
|
||||||
|
```
|
||||||
|
|
||||||
**Using with Ollama:**
|
**Using with Ollama:**
|
||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
|
|
|
||||||
|
|
@ -106,7 +106,6 @@ class PictureDescriptionConfig(BaseModel):
|
||||||
name="ministral-3",
|
name="ministral-3",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
prompt: str = "Describe this image in detail. Be precise and concise."
|
|
||||||
timeout: int = 90
|
timeout: int = 90
|
||||||
max_tokens: int = 200
|
max_tokens: int = 200
|
||||||
|
|
||||||
|
|
@ -186,6 +185,12 @@ class PromptsConfig(BaseModel):
|
||||||
domain_preamble: str = ""
|
domain_preamble: str = ""
|
||||||
qa: str | None = None
|
qa: str | None = None
|
||||||
synthesis: str | None = None
|
synthesis: str | None = None
|
||||||
|
picture_description: str = (
|
||||||
|
"Describe this image for a blind user. "
|
||||||
|
"State the image type (screenshot, chart, photo, etc.), "
|
||||||
|
"what it depicts, any visible text, and key visual details. "
|
||||||
|
"Be concise and accurate."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class AppConfig(BaseModel):
|
class AppConfig(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -117,6 +117,8 @@ class DoclingLocalConverter(DocumentConverter):
|
||||||
if pic_desc.enabled:
|
if pic_desc.enabled:
|
||||||
from pydantic import AnyUrl
|
from pydantic import AnyUrl
|
||||||
|
|
||||||
|
prompt = self.config.prompts.picture_description
|
||||||
|
|
||||||
pipeline_options.enable_remote_services = True
|
pipeline_options.enable_remote_services = True
|
||||||
pipeline_options.picture_description_options = PictureDescriptionApiOptions(
|
pipeline_options.picture_description_options = PictureDescriptionApiOptions(
|
||||||
url=AnyUrl(self._get_vlm_api_url(pic_desc.model)),
|
url=AnyUrl(self._get_vlm_api_url(pic_desc.model)),
|
||||||
|
|
@ -124,7 +126,7 @@ class DoclingLocalConverter(DocumentConverter):
|
||||||
model=pic_desc.model.name,
|
model=pic_desc.model.name,
|
||||||
max_completion_tokens=pic_desc.max_tokens,
|
max_completion_tokens=pic_desc.max_tokens,
|
||||||
),
|
),
|
||||||
prompt=pic_desc.prompt,
|
prompt=prompt,
|
||||||
timeout=pic_desc.timeout,
|
timeout=pic_desc.timeout,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -361,19 +361,17 @@ class TestDoclingLocalConverter:
|
||||||
assert (
|
assert (
|
||||||
config.processing.conversion_options.picture_description.max_tokens == 200
|
config.processing.conversion_options.picture_description.max_tokens == 200
|
||||||
)
|
)
|
||||||
|
# Default prompt is in PromptsConfig
|
||||||
|
assert "blind user" in config.prompts.picture_description
|
||||||
|
|
||||||
def test_picture_description_config_applied(self, config):
|
def test_picture_description_config_applied(self, config):
|
||||||
"""Test that picture description config is applied to converter."""
|
"""Test that picture description config is applied to converter."""
|
||||||
config.processing.conversion_options.picture_description.enabled = True
|
config.processing.conversion_options.picture_description.enabled = True
|
||||||
config.processing.conversion_options.picture_description.prompt = (
|
|
||||||
"Custom prompt for testing."
|
|
||||||
)
|
|
||||||
config.processing.conversion_options.picture_description.timeout = 120
|
config.processing.conversion_options.picture_description.timeout = 120
|
||||||
converter = DoclingLocalConverter(config)
|
converter = DoclingLocalConverter(config)
|
||||||
|
|
||||||
pic_desc = converter.config.processing.conversion_options.picture_description
|
pic_desc = converter.config.processing.conversion_options.picture_description
|
||||||
assert pic_desc.enabled is True
|
assert pic_desc.enabled is True
|
||||||
assert pic_desc.prompt == "Custom prompt for testing."
|
|
||||||
assert pic_desc.timeout == 120
|
assert pic_desc.timeout == 120
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue