Merge pull request #1 from cjpais/resample

Resample

src-tauri/Cargo.lock

@@ -5529,6 +5529,7 @@ version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "40b6fc553156b521663bfa8e713e7ad58c7ca262d46de9998cd7f2e4de5ba0d9"
 dependencies = [
+ "log",
  "whisper-rs-sys",
 ]
 

src-tauri/Cargo.toml

@@ -26,7 +26,7 @@ tauri-plugin-clipboard-manager = "2"
 tauri-plugin-macos-permissions = "2.0.4"
 rdev = { git = "https://github.com/rustdesk-org/rdev" }
 cpal = "0.15.3"
-whisper-rs = { version = "0.13.2" }
+whisper-rs = { version = "0.13.2", features = ["whisper-cpp-log"] }
 anyhow = "1.0.95"
 rubato = "0.16.1"
 samplerate = "0.2.4"

src-tauri/src/managers/audio.rs

@@ -1,9 +1,8 @@
 use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
+use rubato::{FftFixedIn, Resampler};
 use std::sync::{Arc, Mutex};
 use std::vec::Vec;
 
-use samplerate::{convert, ConverterType};
-
 #[derive(Clone, Debug)]
 pub enum RecordingState {
     Initializing,
@@ -16,6 +15,7 @@ pub struct AudioRecordingManager {
     buffer: Arc<Mutex<Vec<f32>>>,
     channels: u16,
     sample_rate: u32,
+    resampler: Arc<Mutex<FftFixedIn<f32>>>,
 }
 
 impl AudioRecordingManager {
@@ -27,22 +27,30 @@ impl AudioRecordingManager {
 
         let config = device.default_input_config()?;
 
-        // Log the audio configuration details
-        println!("Sample Format: {:?}", config.sample_format());
-        println!("Channel Count: {}", config.channels());
-        println!("Sample Rate: {} Hz", config.sample_rate().0);
-        println!("Buffer Size: {:?}", config.buffer_size());
-
         let channels = config.channels();
         let sample_rate = config.sample_rate().0;
 
+        // Configure the resampler with more flexible parameters
+        let resampler = FftFixedIn::new(
+            sample_rate as usize,
+            16000,
+            1024,
+            2,
+            1, // Match input channels
+        )?;
+
         let state = Arc::new(Mutex::new(RecordingState::Idle));
         let buffer = Arc::new(Mutex::new(Vec::new()));
+        let resampler = Arc::new(Mutex::new(resampler));
 
         let state_clone = Arc::clone(&state);
         let buffer_clone = Arc::clone(&buffer);
+        let resampler_clone = Arc::clone(&resampler);
+
+        // Create a temporary buffer to accumulate samples
+        let temp_buffer = Arc::new(Mutex::new(Vec::new()));
+        let temp_buffer_clone = Arc::clone(&temp_buffer);
 
-        // Create and start stream in a separate thread
         std::thread::spawn(move || {
             let stream = match config.sample_format() {
                 cpal::SampleFormat::F32 => device.build_input_stream(
@@ -50,8 +58,29 @@ impl AudioRecordingManager {
                     move |data: &[f32], _: &cpal::InputCallbackInfo| {
                         let state_guard = state_clone.lock().unwrap();
                         if let RecordingState::Recording { .. } = *state_guard {
-                            let mut buffer = buffer_clone.lock().unwrap();
+                            let mut temp_buffer = temp_buffer_clone.lock().unwrap();
-                            buffer.extend_from_slice(data);
+                            temp_buffer.extend_from_slice(data);
+
+                            // Process when we have enough samples
+                            if temp_buffer.len() >= 1024 {
+                                // Convert input data to the format expected by Rubato
+                                let mut chunks = Vec::new();
+                                for chunk in temp_buffer.chunks(1024) {
+                                    chunks.push(chunk.to_vec());
+                                }
+                                let input_frames = vec![chunks.remove(0)]; // Take the first complete chunk
+
+                                // Process the audio chunk
+                                let mut resampler = resampler_clone.lock().unwrap();
+                                if let Ok(resampled) = resampler.process(&input_frames, None) {
+                                    // Store the resampled audio
+                                    let mut buffer = buffer_clone.lock().unwrap();
+                                    buffer.extend_from_slice(&resampled[0]);
+                                }
+
+                                // Keep remaining samples
+                                *temp_buffer = temp_buffer.split_off(1024);
+                            }
                         }
                     },
                     |err| eprintln!("Error in stream: {}", err),
@@ -62,8 +91,6 @@ impl AudioRecordingManager {
             .unwrap();
 
             stream.play().unwrap();
-
-            // Keep the stream alive
             std::thread::park();
         });
 
@@ -72,6 +99,7 @@ impl AudioRecordingManager {
             buffer,
             channels,
             sample_rate,
+            resampler,
         })
     }
 
@@ -113,21 +141,7 @@ impl AudioRecordingManager {
                 println!("Stopped recording for binding {}", binding_id);
 
                 let mut buffer = self.buffer.lock().unwrap();
-                let mut toResample: Vec<f32> = buffer.drain(..).collect();
+                Some(buffer.drain(..).collect())
-
-                let start = std::time::Instant::now();
-                let resampled = convert(
-                    self.sample_rate,
-                    16000,
-                    1,
-                    ConverterType::SincBestQuality,
-                    &toResample,
-                )
-                .unwrap();
-                let duration = start.elapsed();
-                println!("Resampling took: {:?}", duration);
-
-                Some(resampled)
             }
             _ => {
                 println!("Cannot stop recording: not recording or wrong binding");

src-tauri/src/managers/transcription.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
 use std::sync::Mutex;
+use whisper_rs::install_whisper_log_trampoline;
 use whisper_rs::{
     FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters, WhisperState,
 };
@@ -11,6 +12,7 @@ pub struct TranscriptionManager {
 
 impl TranscriptionManager {
     pub fn new() -> Result<Self> {
+        install_whisper_log_trampoline();
         // Load the model
         let context = WhisperContext::new_with_params(
             "resources/ggml-small.bin",
@@ -33,6 +35,12 @@ impl TranscriptionManager {
         let mut result = String::new();
         println!("Audio vector length: {}", audio.len());
 
+        if audio.len() == 0 {
+            println!("Empty audio vector");
+            // TODO error
+            return Ok(result);
+        }
+
         let mut state = self.state.lock().unwrap();
         // Initialize parameters
         let mut params = FullParams::new(SamplingStrategy::default());
@@ -40,6 +48,7 @@ impl TranscriptionManager {
         params.set_print_progress(false);
         params.set_print_realtime(false);
         params.set_print_timestamps(false);
+        params.set_suppress_non_speech_tokens(true);
 
         state
             .full(params, &audio)