From 8cd40c391d6a80226338f2688225b429ac633721 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Tue, 25 Feb 2025 19:09:53 -0800 Subject: [PATCH] Add VAD. --- src-tauri/Cargo.lock | 208 ++++++++++++++++++++++++++++++++ src-tauri/Cargo.toml | 1 + src-tauri/src/managers/audio.rs | 62 +++++++--- 3 files changed, 254 insertions(+), 17 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 0cca731..927dcc0 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -969,6 +969,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + [[package]] name = "crypto-common" version = "0.1.6" @@ -1051,6 +1057,15 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "dasp_frame" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6" +dependencies = [ + "dasp_sample", +] + [[package]] name = "dasp_sample" version = "0.11.0" @@ -1228,6 +1243,18 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +[[package]] +name = "ebur128" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e227cc62d64d6fe01abbef48134b9c1f17d470cef1e7a56337ad05b1f81df7f9" +dependencies = [ + "bitflags 1.3.2", + "dasp_frame", + "dasp_sample", + "smallvec", +] + [[package]] name = "either" version = "1.13.0" @@ -1396,6 +1423,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1421,6 +1458,18 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" +dependencies = [ + "cfg-if", + "libc", + "libredox", + "windows-sys 0.59.0", +] + [[package]] name = "flate2" version = "1.0.35" @@ -1960,6 +2009,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "handy-app" version = "0.1.0" @@ -1983,6 +2042,7 @@ dependencies = [ "tauri-plugin-opener", "tauri-plugin-single-instance", "tokio", + "vad-rs", "whisper-rs", ] @@ -2403,6 +2463,12 @@ dependencies = [ "tiff", ] +[[package]] +name = "indenter" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683" + [[package]] name = "indexmap" version = "1.9.3" @@ -2701,6 +2767,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.8.0", "libc", + "redox_syscall", ] [[package]] @@ -2794,6 +2861,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matrixmultiply" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2891,6 +2968,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "ndk" version = "0.8.0" @@ -3390,6 +3482,31 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ort" +version = "2.0.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52afb44b6b0cffa9bf45e4d37e5a4935b0334a51570658e279e9e3e6cf324aa5" +dependencies = [ + "half", + "ndarray", + "ort-sys", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41d7757331aef2d04b9cb09b45583a59217628beaf91895b7e76187b6e8c088" +dependencies = [ + "flate2", + "pkg-config", + "sha2", + "tar", + "ureq", +] + [[package]] name = "pango" version = "0.18.3" @@ -3660,6 +3777,21 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3924,6 +4056,12 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539" +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rdev" version = "0.5.0-2" @@ -4138,6 +4276,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ringbuffer" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53" + [[package]] name = "rubato" version = "0.16.1" @@ -4211,6 +4355,7 @@ version = "0.23.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -4598,6 +4743,17 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "softbuffer" version = "0.4.6" @@ -4869,6 +5025,17 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -5624,6 +5791,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "log", + "once_cell", + "rustls", + "rustls-pki-types", + "socks", + "url", + "webpki-roots", +] + [[package]] name = "url" version = "2.5.4" @@ -5682,6 +5865,20 @@ dependencies = [ "serde", ] +[[package]] +name = "vad-rs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c53326a6c46b158e5376c26d5745c15b499159fb7d800e5d28ac4b6c2f25443b" +dependencies = [ + "ebur128", + "eyre", + "ndarray", + "ort", + "ringbuffer", + "samplerate", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -6602,6 +6799,17 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d" +[[package]] +name = "xattr" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 3e4ad4f..b2e15dc 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -34,6 +34,7 @@ rig-core = "0.8.0" env_logger = "0.11.6" log = "0.4.25" tokio = "1.43.0" +vad-rs = "0.1.5" [target.'cfg(target_os = "macos")'.dependencies] whisper-rs = { version = "0.13.2", features = ["metal"] } diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs index 54de375..1dce626 100644 --- a/src-tauri/src/managers/audio.rs +++ b/src-tauri/src/managers/audio.rs @@ -2,6 +2,7 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use rubato::{FftFixedIn, Resampler}; use std::sync::{Arc, Mutex}; use std::vec::Vec; +use vad_rs::{Vad, VadStatus}; #[derive(Clone, Debug)] pub enum RecordingState { @@ -22,13 +23,16 @@ impl AudioRecordingManager { .ok_or_else(|| anyhow::Error::msg("No input device available"))?; let config = device.default_input_config()?; - - // let channels = config.channels(); let sample_rate = config.sample_rate().0; - // Configure the resampler with more flexible parameters + // Configure the resampler - keeping 1024 as input size for FFT efficiency let resampler = FftFixedIn::new(sample_rate as usize, 16000, 1024, 2, 1)?; + let vad = Arc::new(Mutex::new( + Vad::new("resources/silero_vad_v4.onnx", 16000).unwrap(), + )); + let vad_clone = Arc::clone(&vad); + let state = Arc::new(Mutex::new(RecordingState::Idle)); let buffer = Arc::new(Mutex::new(Vec::new())); let resampler = Arc::new(Mutex::new(resampler)); @@ -41,6 +45,10 @@ impl AudioRecordingManager { let temp_buffer = Arc::new(Mutex::new(Vec::new())); let temp_buffer_clone = Arc::clone(&temp_buffer); + // Create a buffer for resampled chunks waiting for VAD processing + let vad_buffer = Arc::new(Mutex::new(Vec::new())); + let vad_buffer_clone = Arc::clone(&vad_buffer); + std::thread::spawn(move || { let stream = match config.sample_format() { cpal::SampleFormat::F32 => device.build_input_stream( @@ -52,24 +60,44 @@ impl AudioRecordingManager { temp_buffer.extend_from_slice(data); // Process when we have enough samples - if temp_buffer.len() >= 1024 { - // Convert input data to the format expected by Rubato - let mut chunks = Vec::new(); - for chunk in temp_buffer.chunks(1024) { - chunks.push(chunk.to_vec()); - } - let input_frames = vec![chunks.remove(0)]; // Take the first complete chunk + while temp_buffer.len() >= 1024 { + // Take the first 1024 samples for processing + let chunk: Vec = temp_buffer.drain(..1024).collect(); - // Process the audio chunk + // Convert input data to the format expected by Rubato + let input_frames = vec![chunk]; + + // Process the audio chunk through the resampler let mut resampler = resampler_clone.lock().unwrap(); if let Ok(resampled) = resampler.process(&input_frames, None) { - // Store the resampled audio - let mut buffer = buffer_clone.lock().unwrap(); - buffer.extend_from_slice(&resampled[0]); - } + // Add resampled data to VAD buffer + let mut vad_buffer = vad_buffer_clone.lock().unwrap(); + vad_buffer.extend_from_slice(&resampled[0]); - // Keep remaining samples - *temp_buffer = temp_buffer.split_off(1024); + // Process 30ms chunks (480 samples) for VAD + while vad_buffer.len() >= 480 { + let chunk = vad_buffer.drain(..480).collect::>(); + + // Use VAD to detect speech + if let Ok(mut vad) = vad_clone.lock() { + // println!("VAD lock acquired"); + match vad.compute(&chunk) { + Ok(mut result) => match result.status() { + VadStatus::Speech => { + let mut buffer = + buffer_clone.lock().unwrap(); + buffer.extend_from_slice(&chunk); + } + VadStatus::Silence => {} + _ => {} + }, + Err(error) => { + eprintln!("Error computing VAD: {:?}", error) + } + } + } + } + } } } },