From 84d88f911c801a28484046dbd0387dd803aae51e Mon Sep 17 00:00:00 2001 From: LuoXi Date: Tue, 7 Apr 2026 16:19:40 +0900 Subject: [PATCH] perf: add reasoning_effort passthrough to avoid thinking-mode latency in local models (#1221) * perf: add reasoning_effort passthrough to avoid thinking-mode latency Reasoning models (Gemma 4, Qwen 3, etc.) default to thinking mode, adding 10-40x latency for simple transcript cleanup. Pass through the OpenAI-compatible reasoning_effort parameter so users can disable thinking when speed matters more than deep reasoning. - Add optional reasoning_effort field to ChatCompletionRequest - Thread through send_chat_completion / send_chat_completion_with_schema - New set_post_process_reasoning_effort Tauri command for persistence - Dropdown in post-processing settings, Custom provider only - i18n for en and ja * refactor: simplify reasoning_effort to on/off toggle Addresses review feedback on the reasoning_effort passthrough: post-processing rarely benefits from reasoning, so a 5-way dropdown (default/none/low/medium/high) is overkill. Collapse to a single boolean toggle "Disable reasoning" that defaults to ON for Custom providers (sends reasoning_effort: "none"). Cloud providers continue to receive no reasoning_effort parameter. - Replace post_process_reasoning_effort: Option with post_process_disable_reasoning: bool (default true) in settings - Map bool to Option at the request site (true -> "none", false -> omit) so the API contract is unchanged - Rename Tauri command to set_post_process_disable_reasoning(disable) so parameter naming matches the setting and avoids inversion bugs - Swap the Dropdown for a ToggleSwitch in the Custom provider section - Collapse five i18n strings to one label + description, and add the translation to all 18 additional locales so non-en/ja users don't fall back to English for this control * refactor: remove disable-reasoning toggle, always disable for custom provider Per PR #1221 discussion with @cjpais: since this is an alpha feature, simplify the UI by always sending reasoning_effort: "none" for custom provider post-processing. If this causes issues with specific provider setups, we can revisit. * disable for openrouter too --------- Co-authored-by: CJ Pais --- src-tauri/src/actions.rs | 29 +++++++++++++++++++++++++++-- src-tauri/src/llm_client.rs | 32 +++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs index 3219caf..bf65667 100644 --- a/src-tauri/src/actions.rs +++ b/src-tauri/src/actions.rs @@ -125,6 +125,22 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str) .cloned() .unwrap_or_default(); + // Disable reasoning for providers where post-processing rarely benefits from it. + // - custom: top-level reasoning_effort (works for local OpenAI-compat servers) + // - openrouter: nested reasoning object; exclude:true also keeps reasoning text + // out of the response so it can't pollute structured-output JSON parsing + let (reasoning_effort, reasoning) = match provider.id.as_str() { + "custom" => (Some("none".to_string()), None), + "openrouter" => ( + None, + Some(crate::llm_client::ReasoningConfig { + effort: Some("none".to_string()), + exclude: Some(true), + }), + ), + _ => (None, None), + }; + if provider.supports_structured_output { debug!("Using structured outputs for provider '{}'", provider.id); @@ -195,6 +211,8 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str) user_content, Some(system_prompt), Some(json_schema), + reasoning_effort.clone(), + reasoning.clone(), ) .await { @@ -244,8 +262,15 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str) let processed_prompt = prompt.replace("${output}", transcription); debug!("Processed prompt length: {} chars", processed_prompt.len()); - match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt) - .await + match crate::llm_client::send_chat_completion( + &provider, + api_key, + &model, + processed_prompt, + reasoning_effort, + reasoning, + ) + .await { Ok(Some(content)) => { let content = strip_invisible_chars(&content); diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rs index 2c8e17a..08102c6 100644 --- a/src-tauri/src/llm_client.rs +++ b/src-tauri/src/llm_client.rs @@ -24,12 +24,24 @@ struct ResponseFormat { json_schema: JsonSchema, } +#[derive(Debug, Serialize, Clone, Default)] +pub struct ReasoningConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub exclude: Option, +} + #[derive(Debug, Serialize)] struct ChatCompletionRequest { model: String, messages: Vec, #[serde(skip_serializing_if = "Option::is_none")] response_format: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reasoning_effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reasoning: Option, } #[derive(Debug, Deserialize)] @@ -101,13 +113,27 @@ pub async fn send_chat_completion( api_key: String, model: &str, prompt: String, + reasoning_effort: Option, + reasoning: Option, ) -> Result, String> { - send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await + send_chat_completion_with_schema( + provider, + api_key, + model, + prompt, + None, + None, + reasoning_effort, + reasoning, + ) + .await } /// Send a chat completion request with structured output support /// When json_schema is provided, uses structured outputs mode /// system_prompt is used as the system message when provided +/// reasoning_effort sets the OpenAI-style top-level field (e.g., "none", "low", "medium", "high") +/// reasoning sets the OpenRouter-style nested object (effort + exclude) pub async fn send_chat_completion_with_schema( provider: &PostProcessProvider, api_key: String, @@ -115,6 +141,8 @@ pub async fn send_chat_completion_with_schema( user_content: String, system_prompt: Option, json_schema: Option, + reasoning_effort: Option, + reasoning: Option, ) -> Result, String> { let base_url = provider.base_url.trim_end_matches('/'); let url = format!("{}/chat/completions", base_url); @@ -154,6 +182,8 @@ pub async fn send_chat_completion_with_schema( model: model.to_string(), messages, response_format, + reasoning_effort, + reasoning, }; let response = client