From 0cb8ab21624cce2c3f95824494029aabbe6a110b Mon Sep 17 00:00:00 2001 From: Chirag Agggarwal <96696271+Schreezer@users.noreply.github.com> Date: Tue, 17 Feb 2026 09:44:00 +0530 Subject: [PATCH] feat: implement structured outputs for post-processing providers (#706) * feat: implement structured outputs for Cerebras, OpenRouter, OpenAI, and Apple Intelligence - Add structured output support with JSON schema in llm_client.rs - Update actions.rs to use system prompt + user content approach - Remove legacy ${output} variable substitution for supported providers - Update Apple Intelligence Swift code to accept system prompts - All providers now use consistent structured output format - Remove duplicate check_apple_intelligence_availability function * wip changes * fix(structured-outputs): address PR #706 review comments - Add settings migration to sync supports_structured_output field for existing providers - Fix fallback behavior: structured output failures now fall through to legacy mode - Clone api_key to prevent ownership issues in fallback path - Clean up build_system_prompt(): remove placeholder entirely (instead of replacing with 'the user's message' which reads awkwardly) - Add warn import from log crate * refactor(structured-outputs): apply best practice improvements - Optimize settings migration: use single match instead of double iteration - Add TRANSCRIPTION_FIELD constant to replace magic strings - Keep Apple Intelligence behavior unchanged (no API fallback for privacy) Addresses code review feedback on PR #706: 1. More efficient provider lookup in ensure_post_process_defaults() 2. Eliminates hardcoded 'transcription' string in JSON parsing 3. Maintains privacy-first approach for Apple Intelligence * fix groq output --------- Co-authored-by: CJ Pais --- src-tauri/src/actions.rs | 183 +++++++++++++----- src-tauri/src/apple_intelligence.rs | 27 ++- src-tauri/src/llm_client.rs | 64 +++++- src-tauri/src/settings.rs | 36 +++- src-tauri/swift/apple_intelligence.swift | 19 +- src-tauri/swift/apple_intelligence_bridge.h | 4 +- src-tauri/swift/apple_intelligence_stub.swift | 7 +- src/bindings.ts | 2 +- 8 files changed, 265 insertions(+), 77 deletions(-) diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs index 5339388..f19ce58 100644 --- a/src-tauri/src/actions.rs +++ b/src-tauri/src/actions.rs @@ -12,7 +12,7 @@ use crate::utils::{ }; use crate::TranscriptionCoordinator; use ferrous_opencc::{config::BuiltinConfig, OpenCC}; -use log::{debug, error}; +use log::{debug, error, warn}; use once_cell::sync::Lazy; use std::collections::HashMap; use std::sync::Arc; @@ -42,6 +42,20 @@ struct TranscribeAction { post_process: bool, } +/// Field name for structured output JSON schema +const TRANSCRIPTION_FIELD: &str = "transcription"; + +/// Strip invisible Unicode characters that some LLMs may insert +fn strip_invisible_chars(s: &str) -> String { + s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "") +} + +/// Build a system prompt from the user's prompt template. +/// Removes `${output}` placeholder since the transcription is sent as the user message. +fn build_system_prompt(prompt_template: &str) -> String { + prompt_template.replace("${output}", "").trim().to_string() +} + async fn post_process_transcription(settings: &AppSettings, transcription: &str) -> Option { let provider = match settings.active_post_process_provider().cloned() { Some(provider) => provider, @@ -98,63 +112,136 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str) provider.id, model ); - // Replace ${output} variable in the prompt with the actual text - let processed_prompt = prompt.replace("${output}", transcription); - debug!("Processed prompt length: {} chars", processed_prompt.len()); - - if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID { - #[cfg(all(target_os = "macos", target_arch = "aarch64"))] - { - if !apple_intelligence::check_apple_intelligence_availability() { - debug!("Apple Intelligence selected but not currently available on this device"); - return None; - } - - let token_limit = model.trim().parse::().unwrap_or(0); - return match apple_intelligence::process_text(&processed_prompt, token_limit) { - Ok(result) => { - if result.trim().is_empty() { - debug!("Apple Intelligence returned an empty response"); - None - } else { - debug!( - "Apple Intelligence post-processing succeeded. Output length: {} chars", - result.len() - ); - Some(result) - } - } - Err(err) => { - error!("Apple Intelligence post-processing failed: {}", err); - None - } - }; - } - - #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] - { - debug!("Apple Intelligence provider selected on unsupported platform"); - return None; - } - } - let api_key = settings .post_process_api_keys .get(&provider.id) .cloned() .unwrap_or_default(); - // Send the chat completion request + if provider.supports_structured_output { + debug!("Using structured outputs for provider '{}'", provider.id); + + let system_prompt = build_system_prompt(&prompt); + let user_content = transcription.to_string(); + + // Handle Apple Intelligence separately since it uses native Swift APIs + if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID { + #[cfg(all(target_os = "macos", target_arch = "aarch64"))] + { + if !apple_intelligence::check_apple_intelligence_availability() { + debug!( + "Apple Intelligence selected but not currently available on this device" + ); + return None; + } + + let token_limit = model.trim().parse::().unwrap_or(0); + return match apple_intelligence::process_text_with_system_prompt( + &system_prompt, + &user_content, + token_limit, + ) { + Ok(result) => { + if result.trim().is_empty() { + debug!("Apple Intelligence returned an empty response"); + None + } else { + let result = strip_invisible_chars(&result); + debug!( + "Apple Intelligence post-processing succeeded. Output length: {} chars", + result.len() + ); + Some(result) + } + } + Err(err) => { + error!("Apple Intelligence post-processing failed: {}", err); + None + } + }; + } + + #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] + { + debug!("Apple Intelligence provider selected on unsupported platform"); + return None; + } + } + + // Define JSON schema for transcription output + let json_schema = serde_json::json!({ + "type": "object", + "properties": { + (TRANSCRIPTION_FIELD): { + "type": "string", + "description": "The cleaned and processed transcription text" + } + }, + "required": [TRANSCRIPTION_FIELD], + "additionalProperties": false + }); + + match crate::llm_client::send_chat_completion_with_schema( + &provider, + api_key.clone(), + &model, + user_content, + Some(system_prompt), + Some(json_schema), + ) + .await + { + Ok(Some(content)) => { + // Parse the JSON response to extract the transcription field + match serde_json::from_str::(&content) { + Ok(json) => { + if let Some(transcription_value) = + json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str()) + { + let result = strip_invisible_chars(transcription_value); + debug!( + "Structured output post-processing succeeded for provider '{}'. Output length: {} chars", + provider.id, + result.len() + ); + return Some(result); + } else { + error!("Structured output response missing 'transcription' field"); + return Some(strip_invisible_chars(&content)); + } + } + Err(e) => { + error!( + "Failed to parse structured output JSON: {}. Returning raw content.", + e + ); + return Some(strip_invisible_chars(&content)); + } + } + } + Ok(None) => { + error!("LLM API response has no content"); + return None; + } + Err(e) => { + warn!( + "Structured output failed for provider '{}': {}. Falling back to legacy mode.", + provider.id, e + ); + // Fall through to legacy mode below + } + } + } + + // Legacy mode: Replace ${output} variable in the prompt with the actual text + let processed_prompt = prompt.replace("${output}", transcription); + debug!("Processed prompt length: {} chars", processed_prompt.len()); + match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt) .await { Ok(Some(content)) => { - // Strip invisible Unicode characters that some LLMs (e.g., Qwen) may insert - let content = content - .replace('\u{200B}', "") // Zero-Width Space - .replace('\u{200C}', "") // Zero-Width Non-Joiner - .replace('\u{200D}', "") // Zero-Width Joiner - .replace('\u{FEFF}', ""); // Byte Order Mark / Zero-Width No-Break Space + let content = strip_invisible_chars(&content); debug!( "LLM post-processing succeeded for provider '{}'. Output length: {} chars", provider.id, diff --git a/src-tauri/src/apple_intelligence.rs b/src-tauri/src/apple_intelligence.rs index e34be15..753ddce 100644 --- a/src-tauri/src/apple_intelligence.rs +++ b/src-tauri/src/apple_intelligence.rs @@ -12,10 +12,6 @@ pub struct AppleLLMResponse { // Link to the Swift functions extern "C" { pub fn is_apple_intelligence_available() -> c_int; - pub fn process_text_with_apple_llm( - prompt: *const c_char, - max_tokens: i32, - ) -> *mut AppleLLMResponse; pub fn free_apple_llm_response(response: *mut AppleLLMResponse); } @@ -24,10 +20,27 @@ pub fn check_apple_intelligence_availability() -> bool { unsafe { is_apple_intelligence_available() == 1 } } -pub fn process_text(prompt: &str, max_tokens: i32) -> Result { - let prompt_cstr = CString::new(prompt).map_err(|e| e.to_string())?; +// Link to the Swift function for system prompt support +extern "C" { + pub fn process_text_with_system_prompt_apple( + system_prompt: *const c_char, + user_content: *const c_char, + max_tokens: i32, + ) -> *mut AppleLLMResponse; +} - let response_ptr = unsafe { process_text_with_apple_llm(prompt_cstr.as_ptr(), max_tokens) }; +/// Process text with Apple Intelligence using separate system prompt and user content +pub fn process_text_with_system_prompt( + system_prompt: &str, + user_content: &str, + max_tokens: i32, +) -> Result { + let system_cstr = CString::new(system_prompt).map_err(|e| e.to_string())?; + let user_cstr = CString::new(user_content).map_err(|e| e.to_string())?; + + let response_ptr = unsafe { + process_text_with_system_prompt_apple(system_cstr.as_ptr(), user_cstr.as_ptr(), max_tokens) + }; if response_ptr.is_null() { return Err("Null response from Apple LLM".to_string()); diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rs index 57fe67e..2c8e17a 100644 --- a/src-tauri/src/llm_client.rs +++ b/src-tauri/src/llm_client.rs @@ -2,6 +2,7 @@ use crate::settings::PostProcessProvider; use log::debug; use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT}; use serde::{Deserialize, Serialize}; +use serde_json::Value; #[derive(Debug, Serialize)] struct ChatMessage { @@ -9,10 +10,26 @@ struct ChatMessage { content: String, } +#[derive(Debug, Serialize)] +struct JsonSchema { + name: String, + strict: bool, + schema: Value, +} + +#[derive(Debug, Serialize)] +struct ResponseFormat { + #[serde(rename = "type")] + format_type: String, + json_schema: JsonSchema, +} + #[derive(Debug, Serialize)] struct ChatCompletionRequest { model: String, messages: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + response_format: Option, } #[derive(Debug, Deserialize)] @@ -84,6 +101,20 @@ pub async fn send_chat_completion( api_key: String, model: &str, prompt: String, +) -> Result, String> { + send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await +} + +/// Send a chat completion request with structured output support +/// When json_schema is provided, uses structured outputs mode +/// system_prompt is used as the system message when provided +pub async fn send_chat_completion_with_schema( + provider: &PostProcessProvider, + api_key: String, + model: &str, + user_content: String, + system_prompt: Option, + json_schema: Option, ) -> Result, String> { let base_url = provider.base_url.trim_end_matches('/'); let url = format!("{}/chat/completions", base_url); @@ -92,12 +123,37 @@ pub async fn send_chat_completion( let client = create_client(provider, &api_key)?; + // Build messages vector + let mut messages = Vec::new(); + + // Add system prompt if provided + if let Some(system) = system_prompt { + messages.push(ChatMessage { + role: "system".to_string(), + content: system, + }); + } + + // Add user message + messages.push(ChatMessage { + role: "user".to_string(), + content: user_content, + }); + + // Build response_format if schema is provided + let response_format = json_schema.map(|schema| ResponseFormat { + format_type: "json_schema".to_string(), + json_schema: JsonSchema { + name: "transcription_output".to_string(), + strict: true, + schema, + }, + }); + let request_body = ChatCompletionRequest { model: model.to_string(), - messages: vec![ChatMessage { - role: "user".to_string(), - content: prompt, - }], + messages, + response_format, }; let response = client diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs index b59a4d7..9468a84 100644 --- a/src-tauri/src/settings.rs +++ b/src-tauri/src/settings.rs @@ -101,6 +101,8 @@ pub struct PostProcessProvider { pub allow_base_url_edit: bool, #[serde(default)] pub models_endpoint: Option, + #[serde(default)] + pub supports_structured_output: bool, } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Type)] @@ -455,6 +457,7 @@ fn default_post_process_providers() -> Vec { base_url: "https://api.openai.com/v1".to_string(), allow_base_url_edit: false, models_endpoint: Some("/models".to_string()), + supports_structured_output: true, }, PostProcessProvider { id: "openrouter".to_string(), @@ -462,6 +465,7 @@ fn default_post_process_providers() -> Vec { base_url: "https://openrouter.ai/api/v1".to_string(), allow_base_url_edit: false, models_endpoint: Some("/models".to_string()), + supports_structured_output: true, }, PostProcessProvider { id: "anthropic".to_string(), @@ -469,6 +473,7 @@ fn default_post_process_providers() -> Vec { base_url: "https://api.anthropic.com/v1".to_string(), allow_base_url_edit: false, models_endpoint: Some("/models".to_string()), + supports_structured_output: false, }, PostProcessProvider { id: "groq".to_string(), @@ -476,6 +481,7 @@ fn default_post_process_providers() -> Vec { base_url: "https://api.groq.com/openai/v1".to_string(), allow_base_url_edit: false, models_endpoint: Some("/models".to_string()), + supports_structured_output: false, }, PostProcessProvider { id: "cerebras".to_string(), @@ -483,6 +489,7 @@ fn default_post_process_providers() -> Vec { base_url: "https://api.cerebras.ai/v1".to_string(), allow_base_url_edit: false, models_endpoint: Some("/models".to_string()), + supports_structured_output: true, }, ]; @@ -498,6 +505,7 @@ fn default_post_process_providers() -> Vec { base_url: "apple-intelligence://local".to_string(), allow_base_url_edit: false, models_endpoint: None, + supports_structured_output: true, }); } @@ -508,6 +516,7 @@ fn default_post_process_providers() -> Vec { base_url: "http://localhost:11434/v1".to_string(), allow_base_url_edit: true, models_endpoint: Some("/models".to_string()), + supports_structured_output: false, }); providers @@ -554,13 +563,30 @@ fn default_typing_tool() -> TypingTool { fn ensure_post_process_defaults(settings: &mut AppSettings) -> bool { let mut changed = false; for provider in default_post_process_providers() { - if settings + // Use match to do a single lookup - either sync existing or add new + match settings .post_process_providers - .iter() - .all(|existing| existing.id != provider.id) + .iter_mut() + .find(|p| p.id == provider.id) { - settings.post_process_providers.push(provider.clone()); - changed = true; + Some(existing) => { + // Sync supports_structured_output field for existing providers (migration) + if existing.supports_structured_output != provider.supports_structured_output { + debug!( + "Updating supports_structured_output for provider '{}' from {} to {}", + provider.id, + existing.supports_structured_output, + provider.supports_structured_output + ); + existing.supports_structured_output = provider.supports_structured_output; + changed = true; + } + } + None => { + // Provider doesn't exist, add it + settings.post_process_providers.push(provider.clone()); + changed = true; + } } if !settings.post_process_api_keys.contains_key(&provider.id) { diff --git a/src-tauri/swift/apple_intelligence.swift b/src-tauri/swift/apple_intelligence.swift index 9830137..df0dfda 100644 --- a/src-tauri/swift/apple_intelligence.swift +++ b/src-tauri/swift/apple_intelligence.swift @@ -50,12 +50,14 @@ public func isAppleIntelligenceAvailable() -> Int32 { } } -@_cdecl("process_text_with_apple_llm") -public func processTextWithAppleLLM( - _ prompt: UnsafePointer, +@_cdecl("process_text_with_system_prompt_apple") +public func processTextWithSystemPrompt( + _ systemPrompt: UnsafePointer, + _ userContent: UnsafePointer, maxTokens: Int32 ) -> UnsafeMutablePointer { - let swiftPrompt = String(cString: prompt) + let swiftSystemPrompt = String(cString: systemPrompt) + let swiftUserContent = String(cString: userContent) let responsePtr = ResponsePointer.allocate(capacity: 1) responsePtr.initialize(to: AppleLLMResponse(response: nil, success: 0, error_message: nil)) @@ -87,17 +89,20 @@ public func processTextWithAppleLLM( Task.detached(priority: .userInitiated) { defer { semaphore.signal() } do { - let session = LanguageModelSession(model: model) + let session = LanguageModelSession( + model: model, + instructions: swiftSystemPrompt + ) var output: String do { let structured = try await session.respond( - to: swiftPrompt, + to: swiftUserContent, generating: CleanedTranscript.self ) output = structured.content.cleanedText } catch { - let fallbackGeneration = try await session.respond(to: swiftPrompt) + let fallbackGeneration = try await session.respond(to: swiftUserContent) output = fallbackGeneration.content } diff --git a/src-tauri/swift/apple_intelligence_bridge.h b/src-tauri/swift/apple_intelligence_bridge.h index a5c1ab0..3adffce 100644 --- a/src-tauri/swift/apple_intelligence_bridge.h +++ b/src-tauri/swift/apple_intelligence_bridge.h @@ -16,8 +16,8 @@ typedef struct { // Check if Apple Intelligence is available on the device int is_apple_intelligence_available(void); -// Process text using Apple's on-device LLM -AppleLLMResponse* process_text_with_apple_llm(const char* prompt, int max_tokens); +// Process text using Apple's on-device LLM with separate system prompt and user content +AppleLLMResponse* process_text_with_system_prompt_apple(const char* system_prompt, const char* user_content, int max_tokens); // Free memory allocated by the Apple LLM response void free_apple_llm_response(AppleLLMResponse* response); diff --git a/src-tauri/swift/apple_intelligence_stub.swift b/src-tauri/swift/apple_intelligence_stub.swift index 205a83e..4ae0818 100644 --- a/src-tauri/swift/apple_intelligence_stub.swift +++ b/src-tauri/swift/apple_intelligence_stub.swift @@ -11,9 +11,10 @@ public func isAppleIntelligenceAvailable() -> Int32 { return 0 } -@_cdecl("process_text_with_apple_llm") -public func processTextWithAppleLLM( - _ prompt: UnsafePointer, +@_cdecl("process_text_with_system_prompt_apple") +public func processTextWithSystemPrompt( + _ systemPrompt: UnsafePointer, + _ userContent: UnsafePointer, maxTokens: Int32 ) -> UnsafeMutablePointer { let responsePtr = ResponsePointer.allocate(capacity: 1) diff --git a/src/bindings.ts b/src/bindings.ts index d85b915..82c56cc 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -754,7 +754,7 @@ export type ModelLoadStatus = { is_loaded: boolean; current_model: string | null export type ModelUnloadTimeout = "never" | "immediately" | "min_2" | "min_5" | "min_10" | "min_15" | "hour_1" | "sec_5" export type OverlayPosition = "none" | "top" | "bottom" export type PasteMethod = "ctrl_v" | "direct" | "none" | "shift_insert" | "ctrl_shift_v" -export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null } +export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null; supports_structured_output?: boolean } export type RecordingRetentionPeriod = "never" | "preserve_limit" | "days_3" | "weeks_2" | "months_3" export type ShortcutBinding = { id: string; name: string; description: string; default_binding: string; current_binding: string } export type SoundTheme = "marimba" | "pop" | "custom"