danvics/Handy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: implement structured outputs for post-processing providers (#706)

Browse source 
* feat: implement structured outputs for Cerebras, OpenRouter, OpenAI, and Apple Intelligence

- Add structured output support with JSON schema in llm_client.rs
- Update actions.rs to use system prompt + user content approach
- Remove legacy ${output} variable substitution for supported providers
- Update Apple Intelligence Swift code to accept system prompts
- All providers now use consistent structured output format
- Remove duplicate check_apple_intelligence_availability function

* wip changes

* fix(structured-outputs): address PR #706 review comments

- Add settings migration to sync supports_structured_output field for existing providers
- Fix fallback behavior: structured output failures now fall through to legacy mode
- Clone api_key to prevent ownership issues in fallback path
- Clean up build_system_prompt(): remove  placeholder entirely
  (instead of replacing with 'the user's message' which reads awkwardly)
- Add warn import from log crate

* refactor(structured-outputs): apply best practice improvements

- Optimize settings migration: use single match instead of double iteration
- Add TRANSCRIPTION_FIELD constant to replace magic strings
- Keep Apple Intelligence behavior unchanged (no API fallback for privacy)

Addresses code review feedback on PR #706:
1. More efficient provider lookup in ensure_post_process_defaults()
2. Eliminates hardcoded 'transcription' string in JSON parsing
3. Maintains privacy-first approach for Apple Intelligence

* fix groq output

---------

Co-authored-by: CJ Pais <cj@cjpais.com>
This commit is contained in:
Chirag Agggarwal 2026-02-17 09:44:00 +05:30 committed by GitHub
parent 83e6f5c492
commit 0cb8ab2162
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 265 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

183
src-tauri/src/actions.rs
View file

@ -12,7 +12,7 @@ use crate::utils::{
};
use crate::TranscriptionCoordinator;
use ferrous_opencc::{config::BuiltinConfig, OpenCC};
use log::{debug, error};
use log::{debug, error, warn};
use once_cell::sync::Lazy;
use std::collections::HashMap;
use std::sync::Arc;
@ -42,6 +42,20 @@ struct TranscribeAction {
post_process: bool,
}
/// Field name for structured output JSON schema
const TRANSCRIPTION_FIELD: &str = "transcription";
/// Strip invisible Unicode characters that some LLMs may insert
fn strip_invisible_chars(s: &str) -> String {
s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "")
}
/// Build a system prompt from the user's prompt template.
/// Removes `${output}` placeholder since the transcription is sent as the user message.
fn build_system_prompt(prompt_template: &str) -> String {
prompt_template.replace("${output}", "").trim().to_string()
}
async fn post_process_transcription(settings: &AppSettings, transcription: &str) -> Option<String> {
let provider = match settings.active_post_process_provider().cloned() {
Some(provider) => provider,
@ -98,63 +112,136 @@ async fn post_process_transcription(settings: &AppSettings, transcription: &str)
provider.id, model
);
// Replace ${output} variable in the prompt with the actual text
let processed_prompt = prompt.replace("${output}", transcription);
debug!("Processed prompt length: {} chars", processed_prompt.len());
if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
{
if !apple_intelligence::check_apple_intelligence_availability() {
debug!("Apple Intelligence selected but not currently available on this device");
return None;
}
let token_limit = model.trim().parse::<i32>().unwrap_or(0);
return match apple_intelligence::process_text(&processed_prompt, token_limit) {
Ok(result) => {
if result.trim().is_empty() {
debug!("Apple Intelligence returned an empty response");
None
} else {
debug!(
"Apple Intelligence post-processing succeeded. Output length: {} chars",
result.len()
);
Some(result)
}
}
Err(err) => {
error!("Apple Intelligence post-processing failed: {}", err);
None
}
};
}
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
{
debug!("Apple Intelligence provider selected on unsupported platform");
return None;
}
}
let api_key = settings
.post_process_api_keys
.get(&provider.id)
.cloned()
.unwrap_or_default();
// Send the chat completion request
if provider.supports_structured_output {
debug!("Using structured outputs for provider '{}'", provider.id);
let system_prompt = build_system_prompt(&prompt);
let user_content = transcription.to_string();
// Handle Apple Intelligence separately since it uses native Swift APIs
if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID {
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
{
if !apple_intelligence::check_apple_intelligence_availability() {
debug!(
"Apple Intelligence selected but not currently available on this device"
);
return None;
}
let token_limit = model.trim().parse::<i32>().unwrap_or(0);
return match apple_intelligence::process_text_with_system_prompt(
&system_prompt,
&user_content,
token_limit,
) {
Ok(result) => {
if result.trim().is_empty() {
debug!("Apple Intelligence returned an empty response");
None
} else {
let result = strip_invisible_chars(&result);
debug!(
"Apple Intelligence post-processing succeeded. Output length: {} chars",
result.len()
);
Some(result)
}
}
Err(err) => {
error!("Apple Intelligence post-processing failed: {}", err);
None
}
};
}
#[cfg(not(all(target_os = "macos", target_arch = "aarch64")))]
{
debug!("Apple Intelligence provider selected on unsupported platform");
return None;
}
}
// Define JSON schema for transcription output
let json_schema = serde_json::json!({
"type": "object",
"properties": {
(TRANSCRIPTION_FIELD): {
"type": "string",
"description": "The cleaned and processed transcription text"
}
},
"required": [TRANSCRIPTION_FIELD],
"additionalProperties": false
});
match crate::llm_client::send_chat_completion_with_schema(
&provider,
api_key.clone(),
&model,
user_content,
Some(system_prompt),
Some(json_schema),
)
.await
{
Ok(Some(content)) => {
// Parse the JSON response to extract the transcription field
match serde_json::from_str::<serde_json::Value>(&content) {
Ok(json) => {
if let Some(transcription_value) =
json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str())
{
let result = strip_invisible_chars(transcription_value);
debug!(
"Structured output post-processing succeeded for provider '{}'. Output length: {} chars",
provider.id,
result.len()
);
return Some(result);
} else {
error!("Structured output response missing 'transcription' field");
return Some(strip_invisible_chars(&content));
}
}
Err(e) => {
error!(
"Failed to parse structured output JSON: {}. Returning raw content.",
e
);
return Some(strip_invisible_chars(&content));
}
}
}
Ok(None) => {
error!("LLM API response has no content");
return None;
}
Err(e) => {
warn!(
"Structured output failed for provider '{}': {}. Falling back to legacy mode.",
provider.id, e
);
// Fall through to legacy mode below
}
}
}
// Legacy mode: Replace ${output} variable in the prompt with the actual text
let processed_prompt = prompt.replace("${output}", transcription);
debug!("Processed prompt length: {} chars", processed_prompt.len());
match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt)
.await
{
Ok(Some(content)) => {
// Strip invisible Unicode characters that some LLMs (e.g., Qwen) may insert
let content = content
.replace('\u{200B}', "") // Zero-Width Space
.replace('\u{200C}', "") // Zero-Width Non-Joiner
.replace('\u{200D}', "") // Zero-Width Joiner
.replace('\u{FEFF}', ""); // Byte Order Mark / Zero-Width No-Break Space
let content = strip_invisible_chars(&content);
debug!(
"LLM post-processing succeeded for provider '{}'. Output length: {} chars",
provider.id,

27
src-tauri/src/apple_intelligence.rs
View file

@ -12,10 +12,6 @@ pub struct AppleLLMResponse {
// Link to the Swift functions
extern "C" {
pub fn is_apple_intelligence_available() -> c_int;
pub fn process_text_with_apple_llm(
prompt: *const c_char,
max_tokens: i32,
) -> *mut AppleLLMResponse;
pub fn free_apple_llm_response(response: *mut AppleLLMResponse);
}
@ -24,10 +20,27 @@ pub fn check_apple_intelligence_availability() -> bool {
unsafe { is_apple_intelligence_available() == 1 }
}
pub fn process_text(prompt: &str, max_tokens: i32) -> Result<String, String> {
let prompt_cstr = CString::new(prompt).map_err(|e| e.to_string())?;
// Link to the Swift function for system prompt support
extern "C" {
pub fn process_text_with_system_prompt_apple(
system_prompt: *const c_char,
user_content: *const c_char,
max_tokens: i32,
) -> *mut AppleLLMResponse;
}
let response_ptr = unsafe { process_text_with_apple_llm(prompt_cstr.as_ptr(), max_tokens) };
/// Process text with Apple Intelligence using separate system prompt and user content
pub fn process_text_with_system_prompt(
system_prompt: &str,
user_content: &str,
max_tokens: i32,
) -> Result<String, String> {
let system_cstr = CString::new(system_prompt).map_err(|e| e.to_string())?;
let user_cstr = CString::new(user_content).map_err(|e| e.to_string())?;
let response_ptr = unsafe {
process_text_with_system_prompt_apple(system_cstr.as_ptr(), user_cstr.as_ptr(), max_tokens)
};
if response_ptr.is_null() {
return Err("Null response from Apple LLM".to_string());

64
src-tauri/src/llm_client.rs
View file

@ -2,6 +2,7 @@ use crate::settings::PostProcessProvider;
use log::debug;
use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT};
use serde::{Deserialize, Serialize};
use serde_json::Value;
#[derive(Debug, Serialize)]
struct ChatMessage {
@ -9,10 +10,26 @@ struct ChatMessage {
content: String,
}
#[derive(Debug, Serialize)]
struct JsonSchema {
name: String,
strict: bool,
schema: Value,
}
#[derive(Debug, Serialize)]
struct ResponseFormat {
#[serde(rename = "type")]
format_type: String,
json_schema: JsonSchema,
}
#[derive(Debug, Serialize)]
struct ChatCompletionRequest {
model: String,
messages: Vec<ChatMessage>,
#[serde(skip_serializing_if = "Option::is_none")]
response_format: Option<ResponseFormat>,
}
#[derive(Debug, Deserialize)]
@ -84,6 +101,20 @@ pub async fn send_chat_completion(
api_key: String,
model: &str,
prompt: String,
) -> Result<Option<String>, String> {
send_chat_completion_with_schema(provider, api_key, model, prompt, None, None).await
}
/// Send a chat completion request with structured output support
/// When json_schema is provided, uses structured outputs mode
/// system_prompt is used as the system message when provided
pub async fn send_chat_completion_with_schema(
provider: &PostProcessProvider,
api_key: String,
model: &str,
user_content: String,
system_prompt: Option<String>,
json_schema: Option<Value>,
) -> Result<Option<String>, String> {
let base_url = provider.base_url.trim_end_matches('/');
let url = format!("{}/chat/completions", base_url);
@ -92,12 +123,37 @@ pub async fn send_chat_completion(
let client = create_client(provider, &api_key)?;
// Build messages vector
let mut messages = Vec::new();
// Add system prompt if provided
if let Some(system) = system_prompt {
messages.push(ChatMessage {
role: "system".to_string(),
content: system,
});
}
// Add user message
messages.push(ChatMessage {
role: "user".to_string(),
content: user_content,
});
// Build response_format if schema is provided
let response_format = json_schema.map(|schema| ResponseFormat {
format_type: "json_schema".to_string(),
json_schema: JsonSchema {
name: "transcription_output".to_string(),
strict: true,
schema,
},
});
let request_body = ChatCompletionRequest {
model: model.to_string(),
messages: vec![ChatMessage {
role: "user".to_string(),
content: prompt,
}],
messages,
response_format,
};
let response = client

36
src-tauri/src/settings.rs
View file

@ -101,6 +101,8 @@ pub struct PostProcessProvider {
pub allow_base_url_edit: bool,
#[serde(default)]
pub models_endpoint: Option<String>,
#[serde(default)]
pub supports_structured_output: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Type)]
@ -455,6 +457,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.openai.com/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
supports_structured_output: true,
},
PostProcessProvider {
id: "openrouter".to_string(),
@ -462,6 +465,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://openrouter.ai/api/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
supports_structured_output: true,
},
PostProcessProvider {
id: "anthropic".to_string(),
@ -469,6 +473,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.anthropic.com/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
supports_structured_output: false,
},
PostProcessProvider {
id: "groq".to_string(),
@ -476,6 +481,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.groq.com/openai/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
supports_structured_output: false,
},
PostProcessProvider {
id: "cerebras".to_string(),
@ -483,6 +489,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "https://api.cerebras.ai/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
supports_structured_output: true,
},
];
@ -498,6 +505,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "apple-intelligence://local".to_string(),
allow_base_url_edit: false,
models_endpoint: None,
supports_structured_output: true,
});
}
@ -508,6 +516,7 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
base_url: "http://localhost:11434/v1".to_string(),
allow_base_url_edit: true,
models_endpoint: Some("/models".to_string()),
supports_structured_output: false,
});
providers
@ -554,13 +563,30 @@ fn default_typing_tool() -> TypingTool {
fn ensure_post_process_defaults(settings: &mut AppSettings) -> bool {
let mut changed = false;
for provider in default_post_process_providers() {
if settings
// Use match to do a single lookup - either sync existing or add new
match settings
.post_process_providers
.iter()
.all(|existing| existing.id != provider.id)
.iter_mut()
.find(|p| p.id == provider.id)
{
settings.post_process_providers.push(provider.clone());
changed = true;
Some(existing) => {
// Sync supports_structured_output field for existing providers (migration)
if existing.supports_structured_output != provider.supports_structured_output {
debug!(
"Updating supports_structured_output for provider '{}' from {} to {}",
provider.id,
existing.supports_structured_output,
provider.supports_structured_output
);
existing.supports_structured_output = provider.supports_structured_output;
changed = true;
}
}
None => {
// Provider doesn't exist, add it
settings.post_process_providers.push(provider.clone());
changed = true;
}
}
if !settings.post_process_api_keys.contains_key(&provider.id) {

19
src-tauri/swift/apple_intelligence.swift
View file

@ -50,12 +50,14 @@ public func isAppleIntelligenceAvailable() -> Int32 {
}
}
@_cdecl("process_text_with_apple_llm")
public func processTextWithAppleLLM(
_ prompt: UnsafePointer<CChar>,
@_cdecl("process_text_with_system_prompt_apple")
public func processTextWithSystemPrompt(
_ systemPrompt: UnsafePointer<CChar>,
_ userContent: UnsafePointer<CChar>,
maxTokens: Int32
) -> UnsafeMutablePointer<AppleLLMResponse> {
let swiftPrompt = String(cString: prompt)
let swiftSystemPrompt = String(cString: systemPrompt)
let swiftUserContent = String(cString: userContent)
let responsePtr = ResponsePointer.allocate(capacity: 1)
responsePtr.initialize(to: AppleLLMResponse(response: nil, success: 0, error_message: nil))
@ -87,17 +89,20 @@ public func processTextWithAppleLLM(
Task.detached(priority: .userInitiated) {
defer { semaphore.signal() }
do {
let session = LanguageModelSession(model: model)
let session = LanguageModelSession(
model: model,
instructions: swiftSystemPrompt
)
var output: String
do {
let structured = try await session.respond(
to: swiftPrompt,
to: swiftUserContent,
generating: CleanedTranscript.self
)
output = structured.content.cleanedText
} catch {
let fallbackGeneration = try await session.respond(to: swiftPrompt)
let fallbackGeneration = try await session.respond(to: swiftUserContent)
output = fallbackGeneration.content
}

4
src-tauri/swift/apple_intelligence_bridge.h
View file

@ -16,8 +16,8 @@ typedef struct {
// Check if Apple Intelligence is available on the device
int is_apple_intelligence_available(void);
// Process text using Apple's on-device LLM
AppleLLMResponse* process_text_with_apple_llm(const char* prompt, int max_tokens);
// Process text using Apple's on-device LLM with separate system prompt and user content
AppleLLMResponse* process_text_with_system_prompt_apple(const char* system_prompt, const char* user_content, int max_tokens);
// Free memory allocated by the Apple LLM response
void free_apple_llm_response(AppleLLMResponse* response);

7
src-tauri/swift/apple_intelligence_stub.swift
View file

@ -11,9 +11,10 @@ public func isAppleIntelligenceAvailable() -> Int32 {
return 0
}
@_cdecl("process_text_with_apple_llm")
public func processTextWithAppleLLM(
_ prompt: UnsafePointer<CChar>,
@_cdecl("process_text_with_system_prompt_apple")
public func processTextWithSystemPrompt(
_ systemPrompt: UnsafePointer<CChar>,
_ userContent: UnsafePointer<CChar>,
maxTokens: Int32
) -> UnsafeMutablePointer<AppleLLMResponse> {
let responsePtr = ResponsePointer.allocate(capacity: 1)

2
src/bindings.ts
View file

@ -754,7 +754,7 @@ export type ModelLoadStatus = { is_loaded: boolean; current_model: string | null
export type ModelUnloadTimeout = "never" | "immediately" | "min_2" | "min_5" | "min_10" | "min_15" | "hour_1" | "sec_5"
export type OverlayPosition = "none" | "top" | "bottom"
export type PasteMethod = "ctrl_v" | "direct" | "none" | "shift_insert" | "ctrl_shift_v"
export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null }
export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null; supports_structured_output?: boolean }
export type RecordingRetentionPeriod = "never" | "preserve_limit" | "days_3" | "weeks_2" | "months_3"
export type ShortcutBinding = { id: string; name: string; description: string; default_binding: string; current_binding: string }
export type SoundTheme = "marimba" | "pop" | "custom"