Add hysteresis to VAD.
This commit is contained in:
parent
c23befd98a
commit
dc90edadb1
2 changed files with 33 additions and 14 deletions
|
|
@ -6,9 +6,11 @@ pub struct SmoothedVad {
|
||||||
inner_vad: Box<dyn VoiceActivityDetector>,
|
inner_vad: Box<dyn VoiceActivityDetector>,
|
||||||
prefill_frames: usize,
|
prefill_frames: usize,
|
||||||
hangover_frames: usize,
|
hangover_frames: usize,
|
||||||
|
onset_frames: usize,
|
||||||
|
|
||||||
frame_buffer: VecDeque<Vec<f32>>,
|
frame_buffer: VecDeque<Vec<f32>>,
|
||||||
hangover_counter: usize,
|
hangover_counter: usize,
|
||||||
|
onset_counter: usize,
|
||||||
in_speech: bool,
|
in_speech: bool,
|
||||||
|
|
||||||
temp_out: Vec<f32>,
|
temp_out: Vec<f32>,
|
||||||
|
|
@ -19,13 +21,16 @@ impl SmoothedVad {
|
||||||
inner_vad: Box<dyn VoiceActivityDetector>,
|
inner_vad: Box<dyn VoiceActivityDetector>,
|
||||||
prefill_frames: usize,
|
prefill_frames: usize,
|
||||||
hangover_frames: usize,
|
hangover_frames: usize,
|
||||||
|
onset_frames: usize,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
inner_vad,
|
inner_vad,
|
||||||
prefill_frames,
|
prefill_frames,
|
||||||
hangover_frames,
|
hangover_frames,
|
||||||
|
onset_frames,
|
||||||
frame_buffer: VecDeque::new(),
|
frame_buffer: VecDeque::new(),
|
||||||
hangover_counter: 0,
|
hangover_counter: 0,
|
||||||
|
onset_counter: 0,
|
||||||
in_speech: false,
|
in_speech: false,
|
||||||
temp_out: Vec::new(),
|
temp_out: Vec::new(),
|
||||||
}
|
}
|
||||||
|
|
@ -42,19 +47,28 @@ impl VoiceActivityDetector for SmoothedVad {
|
||||||
|
|
||||||
// 2. Delegate to the wrapped boolean VAD
|
// 2. Delegate to the wrapped boolean VAD
|
||||||
let is_voice = self.inner_vad.is_voice(frame)?;
|
let is_voice = self.inner_vad.is_voice(frame)?;
|
||||||
|
// println!("Is Voice: {}", is_voice);
|
||||||
|
|
||||||
match (self.in_speech, is_voice) {
|
match (self.in_speech, is_voice) {
|
||||||
// Start of Speech
|
// Potential start of speech - need to accumulate onset frames
|
||||||
(false, true) => {
|
(false, true) => {
|
||||||
self.in_speech = true;
|
self.onset_counter += 1;
|
||||||
self.hangover_counter = self.hangover_frames;
|
if self.onset_counter >= self.onset_frames {
|
||||||
|
// We have enough consecutive voice frames to trigger speech
|
||||||
|
self.in_speech = true;
|
||||||
|
self.hangover_counter = self.hangover_frames;
|
||||||
|
self.onset_counter = 0; // Reset for next time
|
||||||
|
|
||||||
// Collect prefill + current frame
|
// Collect prefill + current frame
|
||||||
self.temp_out.clear();
|
self.temp_out.clear();
|
||||||
for buf in &self.frame_buffer {
|
for buf in &self.frame_buffer {
|
||||||
self.temp_out.extend(buf);
|
self.temp_out.extend(buf);
|
||||||
|
}
|
||||||
|
Ok(VadFrame::Speech(&self.temp_out))
|
||||||
|
} else {
|
||||||
|
// Not enough frames yet, still silence
|
||||||
|
Ok(VadFrame::Noise)
|
||||||
}
|
}
|
||||||
Ok(VadFrame::Speech(&self.temp_out))
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ongoing Speech
|
// Ongoing Speech
|
||||||
|
|
@ -63,7 +77,7 @@ impl VoiceActivityDetector for SmoothedVad {
|
||||||
Ok(VadFrame::Speech(frame))
|
Ok(VadFrame::Speech(frame))
|
||||||
}
|
}
|
||||||
|
|
||||||
// End of Speech
|
// End of Speech or interruption during onset phase
|
||||||
(true, false) => {
|
(true, false) => {
|
||||||
if self.hangover_counter > 0 {
|
if self.hangover_counter > 0 {
|
||||||
self.hangover_counter -= 1;
|
self.hangover_counter -= 1;
|
||||||
|
|
@ -74,14 +88,18 @@ impl VoiceActivityDetector for SmoothedVad {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Silence
|
// Silence or broken onset sequence
|
||||||
(false, false) => Ok(VadFrame::Noise),
|
(false, false) => {
|
||||||
|
self.onset_counter = 0; // Reset onset counter on silence
|
||||||
|
Ok(VadFrame::Noise)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn reset(&mut self) {
|
fn reset(&mut self) {
|
||||||
self.frame_buffer.clear();
|
self.frame_buffer.clear();
|
||||||
self.hangover_counter = 0;
|
self.hangover_counter = 0;
|
||||||
|
self.onset_counter = 0;
|
||||||
self.in_speech = false;
|
self.in_speech = false;
|
||||||
self.temp_out.clear();
|
self.temp_out.clear();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -24,9 +24,9 @@ pub enum MicrophoneMode {
|
||||||
/* ──────────────────────────────────────────────────────────────── */
|
/* ──────────────────────────────────────────────────────────────── */
|
||||||
|
|
||||||
fn create_audio_recorder(vad_path: &str) -> Result<AudioRecorder, anyhow::Error> {
|
fn create_audio_recorder(vad_path: &str) -> Result<AudioRecorder, anyhow::Error> {
|
||||||
let silero = SileroVad::new(vad_path, 0.5)
|
let silero = SileroVad::new(vad_path, 0.3)
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to create SileroVad: {}", e))?;
|
.map_err(|e| anyhow::anyhow!("Failed to create SileroVad: {}", e))?;
|
||||||
let smoothed_vad = SmoothedVad::new(Box::new(silero), 15, 15);
|
let smoothed_vad = SmoothedVad::new(Box::new(silero), 15, 15, 2);
|
||||||
let recorder = AudioRecorder::new()
|
let recorder = AudioRecorder::new()
|
||||||
.map_err(|e| anyhow::anyhow!("Failed to create AudioRecorder: {}", e))?
|
.map_err(|e| anyhow::anyhow!("Failed to create AudioRecorder: {}", e))?
|
||||||
.with_vad(Box::new(smoothed_vad));
|
.with_vad(Box::new(smoothed_vad));
|
||||||
|
|
@ -246,7 +246,8 @@ impl AudioRecordingManager {
|
||||||
|
|
||||||
// Pad if very short
|
// Pad if very short
|
||||||
let s_len = samples.len();
|
let s_len = samples.len();
|
||||||
if s_len < WHISPER_SAMPLE_RATE && s_len > 1000 {
|
// println!("Got {} samples", { s_len });
|
||||||
|
if s_len < WHISPER_SAMPLE_RATE && s_len > 0 {
|
||||||
let mut padded = samples;
|
let mut padded = samples;
|
||||||
padded.resize(WHISPER_SAMPLE_RATE * 5 / 4, 0.0);
|
padded.resize(WHISPER_SAMPLE_RATE * 5 / 4, 0.0);
|
||||||
Some(padded)
|
Some(padded)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue