From e307da8544dd96718fa9156753a16a71f67caf13 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Thu, 15 May 2025 13:00:16 -0700 Subject: [PATCH 1/2] fix bugs on linux (mix two channels down to 1, build with vulkan) --- src-tauri/Cargo.toml | 6 +++++ src-tauri/src/managers/audio.rs | 29 +++++++++++++++++++++---- src-tauri/src/managers/transcription.rs | 7 ++++-- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 35d40d9..33871fc 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -42,6 +42,12 @@ tauri-plugin-store = "2" [target.'cfg(target_os = "macos")'.dependencies] whisper-rs = { version = "0.13.2", features = ["metal"] } +[target.'cfg(target_os = "windows")'.dependencies] +whisper-rs = { version = "0.13.2", features = ["vulkan"] } + +[target.'cfg(target_os = "linux")'.dependencies] +whisper-rs = { version = "0.13.2", features = ["openblas", "vulkan"] } + [target.'cfg(not(any(target_os = "android", target_os = "ios")))'.dependencies] tauri-plugin-autostart = "2" tauri-plugin-global-shortcut = "2" diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs index e27d00f..aec8dcc 100644 --- a/src-tauri/src/managers/audio.rs +++ b/src-tauri/src/managers/audio.rs @@ -12,6 +12,7 @@ pub enum RecordingState { Recording { binding_id: String }, } +#[derive(Clone)] pub struct AudioRecordingManager { state: Arc>, buffer: Arc>>, @@ -84,6 +85,7 @@ impl AudioRecordingManager { // Generic function to process audio data fn process_audio( data: &[T], + channels: usize, state_clone: Arc>, temp_buffer_clone: Arc>>, resampler_clone: Arc>>, @@ -95,9 +97,20 @@ impl AudioRecordingManager { if let RecordingState::Recording { .. } = *state_guard { let mut temp_buffer = temp_buffer_clone.lock().unwrap(); - // Convert incoming data to f32 - let f32_data: Vec = data.iter().map(|sample| sample.to_f32()).collect(); - temp_buffer.extend_from_slice(&f32_data); + // Handle multichannel audio by mixing down to mono + if channels > 1 { + // Process chunks of `channels` samples at a time (each chunk is one audio frame) + for chunk in data.chunks(channels) { + // Average the channels to create a mono sample + let mono_sample: f32 = + chunk.iter().map(|s| s.to_f32()).sum::() / channels as f32; + temp_buffer.push(mono_sample); + } + } else { + // Single channel processing + let f32_data: Vec = data.iter().map(|sample| sample.to_f32()).collect(); + temp_buffer.extend_from_slice(&f32_data); + } // Process when we have enough samples while temp_buffer.len() >= 1024 { @@ -142,12 +155,17 @@ impl AudioRecordingManager { let err_fn = |err| eprintln!("Error in stream: {}", err); // Build the appropriate stream based on the sample format + // Store the number of channels for use in the closure + let channels = config.channels() as usize; + println!("Using {} channel(s) for recording", channels); + let stream = match config.sample_format() { SampleFormat::I8 => device.build_input_stream( &config.into(), move |data: &[i8], _| { process_audio( data, + channels, Arc::clone(&state_clone), Arc::clone(&temp_buffer_clone), Arc::clone(&resampler_clone), @@ -164,6 +182,7 @@ impl AudioRecordingManager { move |data: &[i16], _| { process_audio( data, + channels, Arc::clone(&state_clone), Arc::clone(&temp_buffer_clone), Arc::clone(&resampler_clone), @@ -180,6 +199,7 @@ impl AudioRecordingManager { move |data: &[i32], _| { process_audio( data, + channels, Arc::clone(&state_clone), Arc::clone(&temp_buffer_clone), Arc::clone(&resampler_clone), @@ -196,6 +216,7 @@ impl AudioRecordingManager { move |data: &[f32], _| { process_audio( data, + channels, Arc::clone(&state_clone), Arc::clone(&temp_buffer_clone), Arc::clone(&resampler_clone), @@ -268,7 +289,7 @@ impl AudioRecordingManager { } else { // Pad to minimum 1000ms if needed if duration_ms < 1000.0 { - let target_samples = (16400.0 * (1000.0 / 1000.0)) as usize; // 16000 samples for 1 second + let target_samples = (16000.0 * (1000.0 / 1000.0)) as usize; // 16000 samples for 1 second let mut padded_audio = audio_data; padded_audio.resize(target_samples, 0.0); // Pad with silence (zeros) Some(padded_audio) diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs index 83555fe..f142ceb 100644 --- a/src-tauri/src/managers/transcription.rs +++ b/src-tauri/src/managers/transcription.rs @@ -1,5 +1,6 @@ use anyhow::Result; use std::sync::Mutex; +use std::thread; use tauri::{App, Manager}; use whisper_rs::install_whisper_log_trampoline; use whisper_rs::{ @@ -7,7 +8,6 @@ use whisper_rs::{ }; pub struct TranscriptionManager { - context: WhisperContext, state: Mutex, } @@ -30,7 +30,6 @@ impl TranscriptionManager { let state = context.create_state().expect("failed to create state"); Ok(Self { - context, state: Mutex::new(state), }) } @@ -47,6 +46,10 @@ impl TranscriptionManager { return Ok(result); } + let num_threads = thread::available_parallelism() + .map(|count| count.get() as i32) + .unwrap_or(1); + let mut state = self.state.lock().unwrap(); // Initialize parameters let mut params = FullParams::new(SamplingStrategy::default()); From 821e12bdf6fe55e794ca352538994a32409bcdf0 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Thu, 15 May 2025 13:20:54 -0700 Subject: [PATCH 2/2] handle app close uniformly xplatform --- src-tauri/src/lib.rs | 6 +++--- src-tauri/src/managers/transcription.rs | 5 ----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 3a57cdb..fd056ee 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -98,6 +98,8 @@ pub fn run() { }) .on_window_event(|window, event| match event { tauri::WindowEvent::CloseRequested { api, .. } => { + api.prevent_close(); + let _res = window.hide(); #[cfg(target_os = "macos")] { let res = window @@ -107,10 +109,8 @@ pub fn run() { println!("Failed to set activation policy: {}", e); } - api.prevent_close(); - // TODO may be different on windows, this works for macos - tauri::AppHandle::hide(window.app_handle()).unwrap(); + // tauri::AppHandle::hide(window.app_handle()).unwrap(); } } _ => {} diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs index f142ceb..9757c66 100644 --- a/src-tauri/src/managers/transcription.rs +++ b/src-tauri/src/managers/transcription.rs @@ -1,6 +1,5 @@ use anyhow::Result; use std::sync::Mutex; -use std::thread; use tauri::{App, Manager}; use whisper_rs::install_whisper_log_trampoline; use whisper_rs::{ @@ -46,10 +45,6 @@ impl TranscriptionManager { return Ok(result); } - let num_threads = thread::available_parallelism() - .map(|count| count.get() as i32) - .unwrap_or(1); - let mut state = self.state.lock().unwrap(); // Initialize parameters let mut params = FullParams::new(SamplingStrategy::default());