danvics/Handy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

theoretically have different sample formats working as expected.

Browse source
This commit is contained in:
CJ Pais 2025-05-13 13:31:51 -07:00
parent 4f6d5853b5
commit 113edbab5c
1 changed files with 148 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

191
src-tauri/src/managers/audio.rs
View file

@ -1,4 +1,5 @@
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::SampleFormat;
use rubato::{FftFixedIn, Resampler}; use rubato::{FftFixedIn, Resampler};
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::vec::Vec; use std::vec::Vec;
@ -16,6 +17,34 @@ pub struct AudioRecordingManager {
buffer: Arc<Mutex<Vec<f32>>>, buffer: Arc<Mutex<Vec<f32>>>,
} }
trait SampleToF32 {
fn to_f32(&self) -> f32;
}
impl SampleToF32 for i8 {
fn to_f32(&self) -> f32 {
*self as f32 / 128.0
}
}
impl SampleToF32 for i16 {
fn to_f32(&self) -> f32 {
*self as f32 / 32768.0
}
}
impl SampleToF32 for i32 {
fn to_f32(&self) -> f32 {
*self as f32 / 2147483648.0
}
}
impl SampleToF32 for f32 {
fn to_f32(&self) -> f32 {
*self
}
}
impl AudioRecordingManager { impl AudioRecordingManager {
pub fn new(app: &App) -> Result<Self, anyhow::Error> { pub fn new(app: &App) -> Result<Self, anyhow::Error> {
let vad_path = app.path().resolve( let vad_path = app.path().resolve(
@ -52,64 +81,140 @@ impl AudioRecordingManager {
let vad_buffer = Arc::new(Mutex::new(Vec::new())); let vad_buffer = Arc::new(Mutex::new(Vec::new()));
let vad_buffer_clone = Arc::clone(&vad_buffer); let vad_buffer_clone = Arc::clone(&vad_buffer);
std::thread::spawn(move || { // Generic function to process audio data
let stream = match config.sample_format() { fn process_audio<T: SampleToF32 + Send + 'static>(
cpal::SampleFormat::F32 => device.build_input_stream( data: &[T],
&config.into(), state_clone: Arc<Mutex<RecordingState>>,
move |data: &[f32], _: &cpal::InputCallbackInfo| { temp_buffer_clone: Arc<Mutex<Vec<f32>>>,
let state_guard = state_clone.lock().unwrap(); resampler_clone: Arc<Mutex<FftFixedIn<f32>>>,
if let RecordingState::Recording { .. } = *state_guard { vad_buffer_clone: Arc<Mutex<Vec<f32>>>,
let mut temp_buffer = temp_buffer_clone.lock().unwrap(); buffer_clone: Arc<Mutex<Vec<f32>>>,
temp_buffer.extend_from_slice(data); vad_clone: Arc<Mutex<Vad>>,
) {
let state_guard = state_clone.lock().unwrap();
if let RecordingState::Recording { .. } = *state_guard {
let mut temp_buffer = temp_buffer_clone.lock().unwrap();
// Process when we have enough samples // Convert incoming data to f32
while temp_buffer.len() >= 1024 { let f32_data: Vec<f32> = data.iter().map(|sample| sample.to_f32()).collect();
// Take the first 1024 samples for processing temp_buffer.extend_from_slice(&f32_data);
let chunk: Vec<f32> = temp_buffer.drain(..1024).collect();
// Convert input data to the format expected by Rubato // Process when we have enough samples
let input_frames = vec![chunk]; while temp_buffer.len() >= 1024 {
// Take the first 1024 samples for processing
let chunk: Vec<f32> = temp_buffer.drain(..1024).collect();
// Process the audio chunk through the resampler // Convert input data to the format expected by Rubato
let mut resampler = resampler_clone.lock().unwrap(); let input_frames = vec![chunk];
if let Ok(resampled) = resampler.process(&input_frames, None) {
// Add resampled data to VAD buffer
let mut vad_buffer = vad_buffer_clone.lock().unwrap();
vad_buffer.extend_from_slice(&resampled[0]);
// Process 30ms chunks (480 samples) for VAD // Process the audio chunk through the resampler
while vad_buffer.len() >= 480 { let mut resampler = resampler_clone.lock().unwrap();
let chunk = vad_buffer.drain(..480).collect::<Vec<f32>>(); if let Ok(resampled) = resampler.process(&input_frames, None) {
// Add resampled data to VAD buffer
let mut vad_buffer = vad_buffer_clone.lock().unwrap();
vad_buffer.extend_from_slice(&resampled[0]);
// Use VAD to detect speech // Process 30ms chunks (480 samples) for VAD
if let Ok(mut vad) = vad_clone.lock() { while vad_buffer.len() >= 480 {
// println!("VAD lock acquired"); let chunk = vad_buffer.drain(..480).collect::<Vec<f32>>();
match vad.compute(&chunk) {
Ok(result) => { // Use VAD to detect speech
if result.prob > 0.15 { if let Ok(mut vad) = vad_clone.lock() {
let mut buffer = match vad.compute(&chunk) {
buffer_clone.lock().unwrap(); Ok(result) => {
buffer.extend_from_slice(&chunk); if result.prob > 0.15 {
} let mut buffer = buffer_clone.lock().unwrap();
} buffer.extend_from_slice(&chunk);
Err(error) => {
eprintln!("Error computing VAD: {:?}", error)
}
}
} }
} }
Err(error) => {
eprintln!("Error computing VAD: {:?}", error)
}
} }
} }
} }
}
}
}
}
std::thread::spawn(move || {
let err_fn = |err| eprintln!("Error in stream: {}", err);
// Build the appropriate stream based on the sample format
let stream = match config.sample_format() {
SampleFormat::I8 => device.build_input_stream(
&config.into(),
move |data: &[i8], _| {
process_audio(
data,
Arc::clone(&state_clone),
Arc::clone(&temp_buffer_clone),
Arc::clone(&resampler_clone),
Arc::clone(&vad_buffer_clone),
Arc::clone(&buffer_clone),
Arc::clone(&vad_clone),
)
}, },
|err| eprintln!("Error in stream: {}", err), err_fn,
None, None,
), ),
sample_format => panic!("Unsupported sample format: {:?}", sample_format), SampleFormat::I16 => device.build_input_stream(
&config.into(),
move |data: &[i16], _| {
process_audio(
data,
Arc::clone(&state_clone),
Arc::clone(&temp_buffer_clone),
Arc::clone(&resampler_clone),
Arc::clone(&vad_buffer_clone),
Arc::clone(&buffer_clone),
Arc::clone(&vad_clone),
)
},
err_fn,
None,
),
SampleFormat::I32 => device.build_input_stream(
&config.into(),
move |data: &[i32], _| {
process_audio(
data,
Arc::clone(&state_clone),
Arc::clone(&temp_buffer_clone),
Arc::clone(&resampler_clone),
Arc::clone(&vad_buffer_clone),
Arc::clone(&buffer_clone),
Arc::clone(&vad_clone),
)
},
err_fn,
None,
),
SampleFormat::F32 => device.build_input_stream(
&config.into(),
move |data: &[f32], _| {
process_audio(
data,
Arc::clone(&state_clone),
Arc::clone(&temp_buffer_clone),
Arc::clone(&resampler_clone),
Arc::clone(&vad_buffer_clone),
Arc::clone(&buffer_clone),
Arc::clone(&vad_clone),
)
},
err_fn,
None,
),
sample_format => {
// Use anyhow to return a proper error instead of panicking
panic!("Unsupported sample format: {:?}", sample_format);
}
} }
.unwrap(); .expect("Failed to build input stream");
stream.play().unwrap(); stream.play().expect("Failed to play stream");
std::thread::park(); std::thread::park();
}); });