#[cfg(all(target_os = "macos", target_arch = "aarch64"))] use crate::apple_intelligence; use crate::audio_feedback::{play_feedback_sound, play_feedback_sound_blocking, SoundType}; use crate::audio_toolkit::{is_microphone_access_denied, is_no_input_device_error}; use crate::managers::audio::AudioRecordingManager; use crate::managers::history::HistoryManager; use crate::managers::transcription::TranscriptionManager; use crate::settings::{get_settings, AppSettings, APPLE_INTELLIGENCE_PROVIDER_ID}; use crate::shortcut; use crate::tray::{change_tray_icon, TrayIconState}; use crate::utils::{ self, show_processing_overlay, show_recording_overlay, show_transcribing_overlay, }; use crate::TranscriptionCoordinator; use ferrous_opencc::{config::BuiltinConfig, OpenCC}; use log::{debug, error, warn}; use once_cell::sync::Lazy; use std::collections::HashMap; use std::sync::Arc; use std::time::Instant; use tauri::Manager; use tauri::{AppHandle, Emitter}; #[derive(Clone, serde::Serialize)] struct RecordingErrorEvent { error_type: String, detail: Option, } /// Drop guard that notifies the [`TranscriptionCoordinator`] when the /// transcription pipeline finishes — whether it completes normally or panics. struct FinishGuard(AppHandle); impl Drop for FinishGuard { fn drop(&mut self) { if let Some(c) = self.0.try_state::() { c.notify_processing_finished(); } } } // Shortcut Action Trait pub trait ShortcutAction: Send + Sync { fn start(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str); fn stop(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str); } // Transcribe Action struct TranscribeAction { post_process: bool, } /// Field name for structured output JSON schema const TRANSCRIPTION_FIELD: &str = "transcription"; /// Strip invisible Unicode characters that some LLMs may insert fn strip_invisible_chars(s: &str) -> String { s.replace(['\u{200B}', '\u{200C}', '\u{200D}', '\u{FEFF}'], "") } /// Build a system prompt from the user's prompt template. /// Removes `${output}` placeholder since the transcription is sent as the user message. fn build_system_prompt(prompt_template: &str) -> String { prompt_template.replace("${output}", "").trim().to_string() } async fn post_process_transcription(settings: &AppSettings, transcription: &str) -> Option { let provider = match settings.active_post_process_provider().cloned() { Some(provider) => provider, None => { debug!("Post-processing enabled but no provider is selected"); return None; } }; let model = settings .post_process_models .get(&provider.id) .cloned() .unwrap_or_default(); if model.trim().is_empty() { debug!( "Post-processing skipped because provider '{}' has no model configured", provider.id ); return None; } let selected_prompt_id = match &settings.post_process_selected_prompt_id { Some(id) => id.clone(), None => { debug!("Post-processing skipped because no prompt is selected"); return None; } }; let prompt = match settings .post_process_prompts .iter() .find(|prompt| prompt.id == selected_prompt_id) { Some(prompt) => prompt.prompt.clone(), None => { debug!( "Post-processing skipped because prompt '{}' was not found", selected_prompt_id ); return None; } }; if prompt.trim().is_empty() { debug!("Post-processing skipped because the selected prompt is empty"); return None; } debug!( "Starting LLM post-processing with provider '{}' (model: {})", provider.id, model ); let api_key = settings .post_process_api_keys .get(&provider.id) .cloned() .unwrap_or_default(); if provider.supports_structured_output { debug!("Using structured outputs for provider '{}'", provider.id); let system_prompt = build_system_prompt(&prompt); let user_content = transcription.to_string(); // Handle Apple Intelligence separately since it uses native Swift APIs if provider.id == APPLE_INTELLIGENCE_PROVIDER_ID { #[cfg(all(target_os = "macos", target_arch = "aarch64"))] { if !apple_intelligence::check_apple_intelligence_availability() { debug!( "Apple Intelligence selected but not currently available on this device" ); return None; } let token_limit = model.trim().parse::().unwrap_or(0); return match apple_intelligence::process_text_with_system_prompt( &system_prompt, &user_content, token_limit, ) { Ok(result) => { if result.trim().is_empty() { debug!("Apple Intelligence returned an empty response"); None } else { let result = strip_invisible_chars(&result); debug!( "Apple Intelligence post-processing succeeded. Output length: {} chars", result.len() ); Some(result) } } Err(err) => { error!("Apple Intelligence post-processing failed: {}", err); None } }; } #[cfg(not(all(target_os = "macos", target_arch = "aarch64")))] { debug!("Apple Intelligence provider selected on unsupported platform"); return None; } } // Define JSON schema for transcription output let json_schema = serde_json::json!({ "type": "object", "properties": { (TRANSCRIPTION_FIELD): { "type": "string", "description": "The cleaned and processed transcription text" } }, "required": [TRANSCRIPTION_FIELD], "additionalProperties": false }); match crate::llm_client::send_chat_completion_with_schema( &provider, api_key.clone(), &model, user_content, Some(system_prompt), Some(json_schema), ) .await { Ok(Some(content)) => { // Parse the JSON response to extract the transcription field match serde_json::from_str::(&content) { Ok(json) => { if let Some(transcription_value) = json.get(TRANSCRIPTION_FIELD).and_then(|t| t.as_str()) { let result = strip_invisible_chars(transcription_value); debug!( "Structured output post-processing succeeded for provider '{}'. Output length: {} chars", provider.id, result.len() ); return Some(result); } else { error!("Structured output response missing 'transcription' field"); return Some(strip_invisible_chars(&content)); } } Err(e) => { error!( "Failed to parse structured output JSON: {}. Returning raw content.", e ); return Some(strip_invisible_chars(&content)); } } } Ok(None) => { error!("LLM API response has no content"); return None; } Err(e) => { warn!( "Structured output failed for provider '{}': {}. Falling back to legacy mode.", provider.id, e ); // Fall through to legacy mode below } } } // Legacy mode: Replace ${output} variable in the prompt with the actual text let processed_prompt = prompt.replace("${output}", transcription); debug!("Processed prompt length: {} chars", processed_prompt.len()); match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt) .await { Ok(Some(content)) => { let content = strip_invisible_chars(&content); debug!( "LLM post-processing succeeded for provider '{}'. Output length: {} chars", provider.id, content.len() ); Some(content) } Ok(None) => { error!("LLM API response has no content"); None } Err(e) => { error!( "LLM post-processing failed for provider '{}': {}. Falling back to original transcription.", provider.id, e ); None } } } async fn maybe_convert_chinese_variant( settings: &AppSettings, transcription: &str, ) -> Option { // Check if language is set to Simplified or Traditional Chinese let is_simplified = settings.selected_language == "zh-Hans"; let is_traditional = settings.selected_language == "zh-Hant"; if !is_simplified && !is_traditional { debug!("selected_language is not Simplified or Traditional Chinese; skipping translation"); return None; } debug!( "Starting Chinese translation using OpenCC for language: {}", settings.selected_language ); // Use OpenCC to convert based on selected language let config = if is_simplified { // Convert Traditional Chinese to Simplified Chinese BuiltinConfig::Tw2sp } else { // Convert Simplified Chinese to Traditional Chinese BuiltinConfig::S2twp }; match OpenCC::from_config(config) { Ok(converter) => { let converted = converter.convert(transcription); debug!( "OpenCC translation completed. Input length: {}, Output length: {}", transcription.len(), converted.len() ); Some(converted) } Err(e) => { error!("Failed to initialize OpenCC converter: {}. Falling back to original transcription.", e); None } } } pub(crate) struct ProcessedTranscription { pub final_text: String, pub post_processed_text: Option, pub post_process_prompt: Option, } pub(crate) async fn process_transcription_output( app: &AppHandle, transcription: &str, post_process: bool, ) -> ProcessedTranscription { let settings = get_settings(app); let mut final_text = transcription.to_string(); let mut post_processed_text: Option = None; let mut post_process_prompt: Option = None; if let Some(converted_text) = maybe_convert_chinese_variant(&settings, transcription).await { final_text = converted_text; } if post_process { if let Some(processed_text) = post_process_transcription(&settings, &final_text).await { post_processed_text = Some(processed_text.clone()); final_text = processed_text; if let Some(prompt_id) = &settings.post_process_selected_prompt_id { if let Some(prompt) = settings .post_process_prompts .iter() .find(|prompt| &prompt.id == prompt_id) { post_process_prompt = Some(prompt.prompt.clone()); } } } } else if final_text != transcription { post_processed_text = Some(final_text.clone()); } ProcessedTranscription { final_text, post_processed_text, post_process_prompt, } } impl ShortcutAction for TranscribeAction { fn start(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) { let start_time = Instant::now(); debug!("TranscribeAction::start called for binding: {}", binding_id); // Load model in the background let tm = app.state::>(); tm.initiate_model_load(); let binding_id = binding_id.to_string(); change_tray_icon(app, TrayIconState::Recording); show_recording_overlay(app); let rm = app.state::>(); // Get the microphone mode to determine audio feedback timing let settings = get_settings(app); let is_always_on = settings.always_on_microphone; debug!("Microphone mode - always_on: {}", is_always_on); let mut recording_error: Option = None; if is_always_on { // Always-on mode: Play audio feedback immediately, then apply mute after sound finishes debug!("Always-on mode: Playing audio feedback immediately"); let rm_clone = Arc::clone(&rm); let app_clone = app.clone(); // The blocking helper exits immediately if audio feedback is disabled, // so we can always reuse this thread to ensure mute happens right after playback. std::thread::spawn(move || { play_feedback_sound_blocking(&app_clone, SoundType::Start); rm_clone.apply_mute(); }); if let Err(e) = rm.try_start_recording(&binding_id) { debug!("Recording failed: {}", e); recording_error = Some(e); } } else { // On-demand mode: Start recording first, then play audio feedback, then apply mute // This allows the microphone to be activated before playing the sound debug!("On-demand mode: Starting recording first, then audio feedback"); let recording_start_time = Instant::now(); match rm.try_start_recording(&binding_id) { Ok(()) => { debug!("Recording started in {:?}", recording_start_time.elapsed()); // Small delay to ensure microphone stream is active let app_clone = app.clone(); let rm_clone = Arc::clone(&rm); std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(100)); debug!("Handling delayed audio feedback/mute sequence"); // Helper handles disabled audio feedback by returning early, so we reuse it // to keep mute sequencing consistent in every mode. play_feedback_sound_blocking(&app_clone, SoundType::Start); rm_clone.apply_mute(); }); } Err(e) => { debug!("Failed to start recording: {}", e); recording_error = Some(e); } } } if recording_error.is_none() { // Dynamically register the cancel shortcut in a separate task to avoid deadlock shortcut::register_cancel_shortcut(app); } else { // Starting failed (for example due to blocked microphone permissions). // Revert UI state so we don't stay stuck in the recording overlay. utils::hide_recording_overlay(app); change_tray_icon(app, TrayIconState::Idle); if let Some(err) = recording_error { let error_type = if is_microphone_access_denied(&err) { "microphone_permission_denied" } else if is_no_input_device_error(&err) { "no_input_device" } else { "unknown" }; let _ = app.emit( "recording-error", RecordingErrorEvent { error_type: error_type.to_string(), detail: Some(err), }, ); } } debug!( "TranscribeAction::start completed in {:?}", start_time.elapsed() ); } fn stop(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) { // Unregister the cancel shortcut when transcription stops shortcut::unregister_cancel_shortcut(app); let stop_time = Instant::now(); debug!("TranscribeAction::stop called for binding: {}", binding_id); let ah = app.clone(); let rm = Arc::clone(&app.state::>()); let tm = Arc::clone(&app.state::>()); let hm = Arc::clone(&app.state::>()); change_tray_icon(app, TrayIconState::Transcribing); show_transcribing_overlay(app); // Unmute before playing audio feedback so the stop sound is audible rm.remove_mute(); // Play audio feedback for recording stop play_feedback_sound(app, SoundType::Stop); let binding_id = binding_id.to_string(); // Clone binding_id for the async task let post_process = self.post_process; tauri::async_runtime::spawn(async move { let _guard = FinishGuard(ah.clone()); debug!( "Starting async transcription task for binding: {}", binding_id ); let stop_recording_time = Instant::now(); if let Some(samples) = rm.stop_recording(&binding_id) { debug!( "Recording stopped and samples retrieved in {:?}, sample count: {}", stop_recording_time.elapsed(), samples.len() ); if samples.is_empty() { debug!("Recording produced no audio samples; skipping persistence"); utils::hide_recording_overlay(&ah); change_tray_icon(&ah, TrayIconState::Idle); } else { // Save WAV concurrently with transcription let sample_count = samples.len(); let file_name = format!("handy-{}.wav", chrono::Utc::now().timestamp()); let wav_path = hm.recordings_dir().join(&file_name); let wav_path_for_verify = wav_path.clone(); let samples_for_wav = samples.clone(); let wav_handle = tauri::async_runtime::spawn_blocking(move || { crate::audio_toolkit::save_wav_file(&wav_path, &samples_for_wav) }); // Transcribe concurrently with WAV save let transcription_time = Instant::now(); let transcription_result = tm.transcribe(samples); // Await WAV save and verify let wav_saved = match wav_handle.await { Ok(Ok(())) => { match crate::audio_toolkit::verify_wav_file( &wav_path_for_verify, sample_count, ) { Ok(()) => true, Err(e) => { error!("WAV verification failed: {}", e); false } } } Ok(Err(e)) => { error!("Failed to save WAV file: {}", e); false } Err(e) => { error!("WAV save task panicked: {}", e); false } }; match transcription_result { Ok(transcription) => { debug!( "Transcription completed in {:?}: '{}'", transcription_time.elapsed(), transcription ); if post_process { show_processing_overlay(&ah); } let processed = process_transcription_output(&ah, &transcription, post_process) .await; // Save to history if WAV was saved if wav_saved { if let Err(err) = hm.save_entry( file_name, transcription, post_process, processed.post_processed_text.clone(), processed.post_process_prompt.clone(), ) { error!("Failed to save history entry: {}", err); } } if processed.final_text.is_empty() { utils::hide_recording_overlay(&ah); change_tray_icon(&ah, TrayIconState::Idle); } else { let ah_clone = ah.clone(); let paste_time = Instant::now(); let final_text = processed.final_text; ah.run_on_main_thread(move || { match utils::paste(final_text, ah_clone.clone()) { Ok(()) => debug!( "Text pasted successfully in {:?}", paste_time.elapsed() ), Err(e) => error!("Failed to paste transcription: {}", e), } utils::hide_recording_overlay(&ah_clone); change_tray_icon(&ah_clone, TrayIconState::Idle); }) .unwrap_or_else(|e| { error!("Failed to run paste on main thread: {:?}", e); utils::hide_recording_overlay(&ah); change_tray_icon(&ah, TrayIconState::Idle); }); } } Err(err) => { debug!("Global Shortcut Transcription error: {}", err); // Save entry with empty text so user can retry if wav_saved { if let Err(save_err) = hm.save_entry( file_name, String::new(), post_process, None, None, ) { error!("Failed to save failed history entry: {}", save_err); } } utils::hide_recording_overlay(&ah); change_tray_icon(&ah, TrayIconState::Idle); } } } } else { debug!("No samples retrieved from recording stop"); utils::hide_recording_overlay(&ah); change_tray_icon(&ah, TrayIconState::Idle); } }); debug!( "TranscribeAction::stop completed in {:?}", stop_time.elapsed() ); } } // Cancel Action struct CancelAction; impl ShortcutAction for CancelAction { fn start(&self, app: &AppHandle, _binding_id: &str, _shortcut_str: &str) { utils::cancel_current_operation(app); } fn stop(&self, _app: &AppHandle, _binding_id: &str, _shortcut_str: &str) { // Nothing to do on stop for cancel } } // Test Action struct TestAction; impl ShortcutAction for TestAction { fn start(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str) { log::info!( "Shortcut ID '{}': Started - {} (App: {})", // Changed "Pressed" to "Started" for consistency binding_id, shortcut_str, app.package_info().name ); } fn stop(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str) { log::info!( "Shortcut ID '{}': Stopped - {} (App: {})", // Changed "Released" to "Stopped" for consistency binding_id, shortcut_str, app.package_info().name ); } } // Static Action Map pub static ACTION_MAP: Lazy>> = Lazy::new(|| { let mut map = HashMap::new(); map.insert( "transcribe".to_string(), Arc::new(TranscribeAction { post_process: false, }) as Arc, ); map.insert( "transcribe_with_post_process".to_string(), Arc::new(TranscribeAction { post_process: true }) as Arc, ); map.insert( "cancel".to_string(), Arc::new(CancelAction) as Arc, ); map.insert( "test".to_string(), Arc::new(TestAction) as Arc, ); map });