danvics/Handy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add VAD.

Browse source
This commit is contained in:
CJ Pais 2025-02-25 19:09:53 -08:00
parent 9ba4becaee
commit 8cd40c391d
3 changed files with 254 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

208
src-tauri/Cargo.lock generated
View file

@ -969,6 +969,12 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -1051,6 +1057,15 @@ dependencies = [
"syn 2.0.96",
]
[[package]]
name = "dasp_frame"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a3937f5fe2135702897535c8d4a5553f8b116f76c1529088797f2eee7c5cd6"
dependencies = [
"dasp_sample",
]
[[package]]
name = "dasp_sample"
version = "0.11.0"
@ -1228,6 +1243,18 @@ version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125"
[[package]]
name = "ebur128"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e227cc62d64d6fe01abbef48134b9c1f17d470cef1e7a56337ad05b1f81df7f9"
dependencies = [
"bitflags 1.3.2",
"dasp_frame",
"dasp_sample",
"smallvec",
]
[[package]]
name = "either"
version = "1.13.0"
@ -1396,6 +1423,16 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "eyre"
version = "0.6.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec"
dependencies = [
"indenter",
"once_cell",
]
[[package]]
name = "fastrand"
version = "2.3.0"
@ -1421,6 +1458,18 @@ dependencies = [
"rustc_version",
]
[[package]]
name = "filetime"
version = "0.2.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586"
dependencies = [
"cfg-if",
"libc",
"libredox",
"windows-sys 0.59.0",
]
[[package]]
name = "flate2"
version = "1.0.35"
@ -1960,6 +2009,16 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]]
name = "handy-app"
version = "0.1.0"
@ -1983,6 +2042,7 @@ dependencies = [
"tauri-plugin-opener",
"tauri-plugin-single-instance",
"tokio",
"vad-rs",
"whisper-rs",
]
@ -2403,6 +2463,12 @@ dependencies = [
"tiff",
]
[[package]]
name = "indenter"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
version = "1.9.3"
@ -2701,6 +2767,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.8.0",
"libc",
"redox_syscall",
]
[[package]]
@ -2794,6 +2861,16 @@ version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
[[package]]
name = "matrixmultiply"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9380b911e3e96d10c1f415da0876389aaf1b56759054eeb0de7df940c456ba1a"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "memchr"
version = "2.7.4"
@ -2891,6 +2968,21 @@ dependencies = [
"tempfile",
]
[[package]]
name = "ndarray"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
dependencies = [
"matrixmultiply",
"num-complex",
"num-integer",
"num-traits",
"portable-atomic",
"portable-atomic-util",
"rawpointer",
]
[[package]]
name = "ndk"
version = "0.8.0"
@ -3390,6 +3482,31 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "ort"
version = "2.0.0-rc.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52afb44b6b0cffa9bf45e4d37e5a4935b0334a51570658e279e9e3e6cf324aa5"
dependencies = [
"half",
"ndarray",
"ort-sys",
"tracing",
]
[[package]]
name = "ort-sys"
version = "2.0.0-rc.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41d7757331aef2d04b9cb09b45583a59217628beaf91895b7e76187b6e8c088"
dependencies = [
"flate2",
"pkg-config",
"sha2",
"tar",
"ureq",
]
[[package]]
name = "pango"
version = "0.18.3"
@ -3660,6 +3777,21 @@ dependencies = [
"windows-sys 0.59.0",
]
[[package]]
name = "portable-atomic"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e"
[[package]]
name = "portable-atomic-util"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
dependencies = [
"portable-atomic",
]
[[package]]
name = "powerfmt"
version = "0.2.0"
@ -3924,6 +4056,12 @@ version = "0.6.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20675572f6f24e9e76ef639bc5552774ed45f1c30e2951e1e99c59888861c539"
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "rdev"
version = "0.5.0-2"
@ -4138,6 +4276,12 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "ringbuffer"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3df6368f71f205ff9c33c076d170dd56ebf68e8161c733c0caa07a7a5509ed53"
[[package]]
name = "rubato"
version = "0.16.1"
@ -4211,6 +4355,7 @@ version = "0.23.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7"
dependencies = [
"log",
"once_cell",
"ring",
"rustls-pki-types",
@ -4598,6 +4743,17 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "socks"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
dependencies = [
"byteorder",
"libc",
"winapi",
]
[[package]]
name = "softbuffer"
version = "0.4.6"
@ -4869,6 +5025,17 @@ dependencies = [
"syn 2.0.96",
]
[[package]]
name = "tar"
version = "0.4.44"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a"
dependencies = [
"filetime",
"libc",
"xattr",
]
[[package]]
name = "target-lexicon"
version = "0.12.16"
@ -5624,6 +5791,22 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "2.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
dependencies = [
"base64 0.22.1",
"log",
"once_cell",
"rustls",
"rustls-pki-types",
"socks",
"url",
"webpki-roots",
]
[[package]]
name = "url"
version = "2.5.4"
@ -5682,6 +5865,20 @@ dependencies = [
"serde",
]
[[package]]
name = "vad-rs"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c53326a6c46b158e5376c26d5745c15b499159fb7d800e5d28ac4b6c2f25443b"
dependencies = [
"ebur128",
"eyre",
"ndarray",
"ort",
"ringbuffer",
"samplerate",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
@ -6602,6 +6799,17 @@ version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec107c4503ea0b4a98ef47356329af139c0a4f7750e621cf2973cd3385ebcb3d"
[[package]]
name = "xattr"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909"
dependencies = [
"libc",
"linux-raw-sys",
"rustix",
]
[[package]]
name = "xdg-home"
version = "1.3.0"

1
src-tauri/Cargo.toml
View file

@ -34,6 +34,7 @@ rig-core = "0.8.0"
env_logger = "0.11.6"
log = "0.4.25"
tokio = "1.43.0"
vad-rs = "0.1.5"
[target.'cfg(target_os = "macos")'.dependencies]
whisper-rs = { version = "0.13.2", features = ["metal"] }

62
src-tauri/src/managers/audio.rs
View file

@ -2,6 +2,7 @@ use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use rubato::{FftFixedIn, Resampler};
use std::sync::{Arc, Mutex};
use std::vec::Vec;
use vad_rs::{Vad, VadStatus};
#[derive(Clone, Debug)]
pub enum RecordingState {
@ -22,13 +23,16 @@ impl AudioRecordingManager {
.ok_or_else(|| anyhow::Error::msg("No input device available"))?;
let config = device.default_input_config()?;
// let channels = config.channels();
let sample_rate = config.sample_rate().0;
// Configure the resampler with more flexible parameters
// Configure the resampler - keeping 1024 as input size for FFT efficiency
let resampler = FftFixedIn::new(sample_rate as usize, 16000, 1024, 2, 1)?;
let vad = Arc::new(Mutex::new(
Vad::new("resources/silero_vad_v4.onnx", 16000).unwrap(),
));
let vad_clone = Arc::clone(&vad);
let state = Arc::new(Mutex::new(RecordingState::Idle));
let buffer = Arc::new(Mutex::new(Vec::new()));
let resampler = Arc::new(Mutex::new(resampler));
@ -41,6 +45,10 @@ impl AudioRecordingManager {
let temp_buffer = Arc::new(Mutex::new(Vec::new()));
let temp_buffer_clone = Arc::clone(&temp_buffer);
// Create a buffer for resampled chunks waiting for VAD processing
let vad_buffer = Arc::new(Mutex::new(Vec::new()));
let vad_buffer_clone = Arc::clone(&vad_buffer);
std::thread::spawn(move || {
let stream = match config.sample_format() {
cpal::SampleFormat::F32 => device.build_input_stream(
@ -52,24 +60,44 @@ impl AudioRecordingManager {
temp_buffer.extend_from_slice(data);
// Process when we have enough samples
if temp_buffer.len() >= 1024 {
// Convert input data to the format expected by Rubato
let mut chunks = Vec::new();
for chunk in temp_buffer.chunks(1024) {
chunks.push(chunk.to_vec());
}
let input_frames = vec![chunks.remove(0)]; // Take the first complete chunk
while temp_buffer.len() >= 1024 {
// Take the first 1024 samples for processing
let chunk: Vec<f32> = temp_buffer.drain(..1024).collect();
// Process the audio chunk
// Convert input data to the format expected by Rubato
let input_frames = vec![chunk];
// Process the audio chunk through the resampler
let mut resampler = resampler_clone.lock().unwrap();
if let Ok(resampled) = resampler.process(&input_frames, None) {
// Store the resampled audio
let mut buffer = buffer_clone.lock().unwrap();
buffer.extend_from_slice(&resampled[0]);
}
// Add resampled data to VAD buffer
let mut vad_buffer = vad_buffer_clone.lock().unwrap();
vad_buffer.extend_from_slice(&resampled[0]);
// Keep remaining samples
*temp_buffer = temp_buffer.split_off(1024);
// Process 30ms chunks (480 samples) for VAD
while vad_buffer.len() >= 480 {
let chunk = vad_buffer.drain(..480).collect::<Vec<f32>>();
// Use VAD to detect speech
if let Ok(mut vad) = vad_clone.lock() {
// println!("VAD lock acquired");
match vad.compute(&chunk) {
Ok(mut result) => match result.status() {
VadStatus::Speech => {
let mut buffer =
buffer_clone.lock().unwrap();
buffer.extend_from_slice(&chunk);
}
VadStatus::Silence => {}
_ => {}
},
Err(error) => {
eprintln!("Error computing VAD: {:?}", error)
}
}
}
}
}
}
}
},