From c6cd8472996a69aafac089eeef50b1af0d6383a2 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 13 May 2026 13:43:52 +0300 Subject: [PATCH] Add ModelConfig.extra_body for raw provider pass-through --- CHANGELOG.md | 1 + docs/configuration/providers.md | 22 ++++++++++++ haiku_rag_slim/haiku/rag/config/models.py | 6 ++++ haiku_rag_slim/haiku/rag/utils.py | 9 ++++- tests/test_utils.py | 41 +++++++++++++++++++++++ 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e5bebf2..44f3a420 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - **S3 / object-storage monitoring.** `monitor.s3: list[S3MonitorEntry]` adds a polling watcher per bucket prefix alongside the existing local-directory watcher. Each entry has its own `poll_interval`, `include_patterns`, `ignore_patterns`, `delete_orphans`, and `storage_options`. The same `serve --monitor` flag enables both. Orphan deletion is per-entry (scoped via `uri LIKE 's3://bucket/prefix/%'`); other buckets and prefixes are never touched. - **`[s3]` optional extra** (`obstore>=0.9`). Required for `s3://` sources and the S3 watcher. Uses obstore — the Python binding to the same Rust `object_store` crate that LanceDB uses internally — so `monitor.s3[*].storage_options` accepts the same dict shape as `lancedb.storage_options`. Empty/missing options fall back to the AWS default credential chain. - **`scripts/run-integration-tests.sh`** — wraps `docker compose up --wait`, `pytest -m integration`, and tear-down so the SeaweedFS-backed integration suite is a one-liner. +- **`ModelConfig.extra_body`**. Optional dict forwarded verbatim to `ModelSettings.extra_body`, the raw pass-through pydantic-ai exposes for openai/ollama/anthropic/groq. Lets configs reach provider-specific keys without haiku.rag modelling them — e.g. `extra_body: {chat_template_kwargs: {enable_thinking: false}}` to disable Qwen3 thinking on a vLLM endpoint, where the high-level `enable_thinking` flag is a no-op. ### Changed diff --git a/docs/configuration/providers.md b/docs/configuration/providers.md index 15169d44..0e17e05e 100644 --- a/docs/configuration/providers.md +++ b/docs/configuration/providers.md @@ -29,6 +29,7 @@ qa: - **max_tokens**: Maximum tokens in response. Default: unset (provider default), except title generation (100). - **enable_thinking**: Control reasoning behavior (see below) - **base_url**: Custom endpoint for OpenAI-compatible servers (vLLM, LM Studio, etc.) +- **extra_body**: Raw dict forwarded to the model SDK (see [Raw Provider Pass-through](#raw-provider-pass-through)) ### Thinking Control @@ -66,6 +67,27 @@ See the [Pydantic AI thinking documentation](https://ai.pydantic.dev/thinking/) - Enable for QA, research, complex reasoning, and mathematical problems - Disable for speed-critical applications, title generation, and simple tasks +### Raw Provider Pass-through + +The `extra_body` setting takes a dict that haiku.rag forwards verbatim to the underlying model SDK as `ModelSettings.extra_body`. Use it to reach provider-specific keys that haiku.rag does not model with a dedicated field. + +**Example — disable Qwen3 thinking on vLLM:** + +```yaml +qa: + model: + provider: openai + name: qwen3.6-35b + base_url: http://localhost:11430/v1 + extra_body: + chat_template_kwargs: + enable_thinking: false +``` + +vLLM serves Qwen3 chat templates that read their thinking switch from `chat_template_kwargs.enable_thinking`. The high-level `enable_thinking` setting on the openai provider maps to vLLM's `reasoning_effort` parameter, which Qwen3 templates ignore, so the field is a no-op for this combination. `extra_body` reaches the chat template directly and disables thinking. With it off, Qwen3 returns the answer in `content` immediately instead of emitting a hidden reasoning trace first. + +**Provider support:** honored by openai, ollama, anthropic, and groq via pydantic-ai's `ModelSettings.extra_body`. Silently ignored by gemini and bedrock. + ## Embedding Providers Embedding models require three settings: `provider`, `name`, and `vector_dim`. Optionally, use `base_url` for OpenAI-compatible servers. diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index c9b6adf6..5a12f384 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -17,6 +17,11 @@ class ModelConfig(BaseModel): temperature: Sampling temperature (0.0 to 1.0+) max_tokens: Maximum tokens to generate vision: True if the model can interpret images. Default False. + extra_body: Raw dict forwarded verbatim to the model SDK as + `ModelSettings.extra_body`. Provider-side escape hatch for + keys haiku.rag doesn't model explicitly (e.g. vLLM's + `chat_template_kwargs.enable_thinking: false` for Qwen3). + Honored by openai/ollama/anthropic/groq; ignored by gemini/bedrock. """ provider: str = "ollama" @@ -27,6 +32,7 @@ class ModelConfig(BaseModel): temperature: float | None = None max_tokens: int | None = None vision: bool = False + extra_body: dict | None = None class EmbeddingModelConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py index ef7a41e2..d6d104d3 100644 --- a/haiku_rag_slim/haiku/rag/utils.py +++ b/haiku_rag_slim/haiku/rag/utils.py @@ -101,7 +101,11 @@ def apply_common_settings( Returns: Updated settings instance or None if no settings to apply """ - if model_config.temperature is None and model_config.max_tokens is None: + if ( + model_config.temperature is None + and model_config.max_tokens is None + and model_config.extra_body is None + ): return settings if settings is None: @@ -115,6 +119,9 @@ def apply_common_settings( if model_config.max_tokens is not None: settings_dict["max_tokens"] = model_config.max_tokens + if model_config.extra_body is not None: + settings_dict["extra_body"] = model_config.extra_body + return settings_dict diff --git a/tests/test_utils.py b/tests/test_utils.py index 6aa35244..59324264 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -208,6 +208,47 @@ def test_get_model_openai_non_reasoning_model_ignores_thinking(): assert result._settings is None +def test_get_model_openai_extra_body_forwarded(): + """`extra_body` on ModelConfig is forwarded to ModelSettings.extra_body. + + pydantic-ai's OpenAI model branch reads `model_settings["extra_body"]` + and passes it verbatim to the OpenAI SDK. Enables vLLM-specific keys + like `chat_template_kwargs.enable_thinking` without coupling them to + the high-level `enable_thinking` flag. + """ + extra = {"chat_template_kwargs": {"enable_thinking": False}} + model_config = ModelConfig( + provider="openai", + name="qwen3.6-35b", + base_url="http://localhost:11430/v1", + extra_body=extra, + ) + result = get_model(model_config) + assert isinstance(result, OpenAIChatModel) + assert result._settings is not None + assert result._settings.get("extra_body") == extra + + +def test_get_model_ollama_extra_body_forwarded(): + """`extra_body` is forwarded through the Ollama (openai-compatible) branch too.""" + extra = {"chat_template_kwargs": {"enable_thinking": False}} + model_config = ModelConfig(provider="ollama", name="qwen3", extra_body=extra) + result = get_model(model_config) + assert isinstance(result, OpenAIChatModel) + assert result._settings is not None + assert result._settings.get("extra_body") == extra + + +def test_get_model_extra_body_absent_when_unset(): + """No `extra_body` key appears on the settings when the config omits it.""" + model_config = ModelConfig(provider="openai", name="gpt-4o-mini", temperature=0.3) + result = get_model(model_config) + assert isinstance(result, OpenAIChatModel) + # temperature triggers settings construction; extra_body should not be there. + assert result._settings is not None + assert "extra_body" not in result._settings + + @pytest.mark.skipif(not HAS_ANTHROPIC, reason="Anthropic not installed") def test_get_model_anthropic(): """Test get_model returns AnthropicModel for Anthropic."""