perf: add reasoning_effort passthrough to avoid thinking-mode latency in local models (#1221)
* perf: add reasoning_effort passthrough to avoid thinking-mode latency Reasoning models (Gemma 4, Qwen 3, etc.) default to thinking mode, adding 10-40x latency for simple transcript cleanup. Pass through the OpenAI-compatible reasoning_effort parameter so users can disable thinking when speed matters more than deep reasoning. - Add optional reasoning_effort field to ChatCompletionRequest - Thread through send_chat_completion / send_chat_completion_with_schema - New set_post_process_reasoning_effort Tauri command for persistence - Dropdown in post-processing settings, Custom provider only - i18n for en and ja * refactor: simplify reasoning_effort to on/off toggle Addresses review feedback on the reasoning_effort passthrough: post-processing rarely benefits from reasoning, so a 5-way dropdown (default/none/low/medium/high) is overkill. Collapse to a single boolean toggle "Disable reasoning" that defaults to ON for Custom providers (sends reasoning_effort: "none"). Cloud providers continue to receive no reasoning_effort parameter. - Replace post_process_reasoning_effort: Option<String> with post_process_disable_reasoning: bool (default true) in settings - Map bool to Option<String> at the request site (true -> "none", false -> omit) so the API contract is unchanged - Rename Tauri command to set_post_process_disable_reasoning(disable) so parameter naming matches the setting and avoids inversion bugs - Swap the Dropdown for a ToggleSwitch in the Custom provider section - Collapse five i18n strings to one label + description, and add the translation to all 18 additional locales so non-en/ja users don't fall back to English for this control * refactor: remove disable-reasoning toggle, always disable for custom provider Per PR #1221 discussion with @cjpais: since this is an alpha feature, simplify the UI by always sending reasoning_effort: "none" for custom provider post-processing. If this causes issues with specific provider setups, we can revisit. * disable for openrouter too --------- Co-authored-by: CJ Pais <cj@cjpais.com>
This commit is contained in:
parent
c1697b2a95
commit
84d88f911c
2 changed files with 58 additions and 3 deletions
|
|
@ -125,6 +125,22 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
|
|||
.cloned()
|
||||
.unwrap_or_default();
|
||||
|
||||
// Disable reasoning for providers where post-processing rarely benefits from it.
|
||||
// - custom: top-level reasoning_effort (works for local OpenAI-compat servers)
|
||||
// - openrouter: nested reasoning object; exclude:true also keeps reasoning text
|
||||
// out of the response so it can't pollute structured-output JSON parsing
|
||||
let (reasoning_effort, reasoning) = match provider.id.as_str() {
|
||||
"custom" => (Some("none".to_string()), None),
|
||||
"openrouter" => (
|
||||
None,
|
||||
Some(crate::llm_client::ReasoningConfig {
|
||||
effort: Some("none".to_string()),
|
||||
exclude: Some(true),
|
||||
}),
|
||||
),
|
||||
_ => (None, None),
|
||||
};
|
||||
|
||||
if provider.supports_structured_output {
|
||||
debug!("Using structured outputs for provider '{}'", provider.id);
|
||||
|
||||
|
|
@ -195,6 +211,8 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
|
|||
user_content,
|
||||
Some(system_prompt),
|
||||
Some(json_schema),
|
||||
reasoning_effort.clone(),
|
||||
reasoning.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
|
@ -244,8 +262,15 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
|
|||
let processed_prompt = prompt.replace("${output}", transcription);
|
||||
debug!("Processed prompt length: {} chars", processed_prompt.len());
|
||||
|
||||
match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt)
|
||||
.await
|
||||
match crate::llm_client::send_chat_completion(
|
||||
&provider,
|
||||
api_key,
|
||||
&model,
|
||||
processed_prompt,
|
||||
reasoning_effort,
|
||||
reasoning,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Some(content)) => {
|
||||
let content = strip_invisible_chars(&content);
|
||||
|
|
|
|||
|
|
@ -24,12 +24,24 @@ struct ResponseFormat {
|
|||
json_schema: JsonSchema,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Clone, Default)]
|
||||
pub struct ReasoningConfig {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub effort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub exclude: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct ChatCompletionRequest {
|
||||
model: String,
|
||||
messages: Vec<ChatMessage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
response_format: Option<ResponseFormat>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning_effort: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning: Option<ReasoningConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
|
|
@ -101,13 +113,27 @@ pub async fn send_chat_completion(
|
|||
api_key: String,
|
||||
model: &str,
|
||||
prompt: String,
|
||||
reasoning_effort: Option<String>,
|
||||
reasoning: Option<ReasoningConfig>,
|
||||
) -> Result<Option<String>, String> {
|
||||
send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await
|
||||
send_chat_completion_with_schema(
|
||||
provider,
|
||||
api_key,
|
||||
model,
|
||||
prompt,
|
||||
None,
|
||||
None,
|
||||
reasoning_effort,
|
||||
reasoning,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Send a chat completion request with structured output support
|
||||
/// When json_schema is provided, uses structured outputs mode
|
||||
/// system_prompt is used as the system message when provided
|
||||
/// reasoning_effort sets the OpenAI-style top-level field (e.g., "none", "low", "medium", "high")
|
||||
/// reasoning sets the OpenRouter-style nested object (effort + exclude)
|
||||
pub async fn send_chat_completion_with_schema(
|
||||
provider: &PostProcessProvider,
|
||||
api_key: String,
|
||||
|
|
@ -115,6 +141,8 @@ pub async fn send_chat_completion_with_schema(
|
|||
user_content: String,
|
||||
system_prompt: Option<String>,
|
||||
json_schema: Option<Value>,
|
||||
reasoning_effort: Option<String>,
|
||||
reasoning: Option<ReasoningConfig>,
|
||||
) -> Result<Option<String>, String> {
|
||||
let base_url = provider.base_url.trim_end_matches('/');
|
||||
let url = format!("{}/chat/completions", base_url);
|
||||
|
|
@ -154,6 +182,8 @@ pub async fn send_chat_completion_with_schema(
|
|||
model: model.to_string(),
|
||||
messages,
|
||||
response_format,
|
||||
reasoning_effort,
|
||||
reasoning,
|
||||
};
|
||||
|
||||
let response = client
|
||||
|
|
|
|||
Loading…
Reference in a new issue