From ec2fefb9fefd51e71c1cffddf95eb5efcdefd4ca Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Fri, 1 Aug 2025 14:12:23 -0700 Subject: [PATCH] Audio visualizer generally working nicely. --- src-tauri/Cargo.lock | 6 +- src-tauri/Cargo.toml | 1 + src-tauri/src/audio_toolkit/audio/mod.rs | 2 + src-tauri/src/audio_toolkit/audio/recorder.rs | 37 ++++- .../src/audio_toolkit/audio/visualizer.rs | 156 ++++++++++++++++++ src-tauri/src/managers/audio.rs | 25 ++- src-tauri/src/utils.rs | 11 ++ src/overlay/RecordingOverlay.css | 22 +-- src/overlay/RecordingOverlay.tsx | 27 ++- 9 files changed, 258 insertions(+), 29 deletions(-) create mode 100644 src-tauri/src/audio_toolkit/audio/visualizer.rs diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index d268f1f..d9f0460 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -2018,6 +2018,7 @@ dependencies = [ "reqwest 0.11.27", "rodio", "rubato", + "rustfft", "serde", "serde_json", "tauri", @@ -4404,9 +4405,9 @@ dependencies = [ [[package]] name = "rustfft" -version = "6.2.0" +version = "6.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +checksum = "c6f140db74548f7c9d7cce60912c9ac414e74df5e718dc947d514b051b42f3f4" dependencies = [ "num-complex", "num-integer", @@ -4414,7 +4415,6 @@ dependencies = [ "primal-check", "strength_reduce", "transpose", - "version_check", ] [[package]] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 6407516..b86d834 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -52,6 +52,7 @@ rodio = { git = "https://github.com/cjpais/rodio.git" } reqwest = { version = "0.11", features = ["json", "stream"] } futures-util = "0.3" tauri-plugin-fs = "2" +rustfft = "6.4.0" [dependencies.ort-sys] version = "=2.0.0-rc.9" diff --git a/src-tauri/src/audio_toolkit/audio/mod.rs b/src-tauri/src/audio_toolkit/audio/mod.rs index e837b8e..83e307c 100644 --- a/src-tauri/src/audio_toolkit/audio/mod.rs +++ b/src-tauri/src/audio_toolkit/audio/mod.rs @@ -2,7 +2,9 @@ mod device; mod recorder; mod resampler; +mod visualizer; pub use device::{list_input_devices, list_output_devices, CpalDeviceInfo}; pub use recorder::AudioRecorder; pub use resampler::FrameResampler; +pub use visualizer::AudioVisualiser; diff --git a/src-tauri/src/audio_toolkit/audio/recorder.rs b/src-tauri/src/audio_toolkit/audio/recorder.rs index 3ec466d..a3fb159 100644 --- a/src-tauri/src/audio_toolkit/audio/recorder.rs +++ b/src-tauri/src/audio_toolkit/audio/recorder.rs @@ -10,7 +10,7 @@ use cpal::{ }; use crate::audio_toolkit::{ - audio::FrameResampler, + audio::{AudioVisualiser, FrameResampler}, constants, vad::{self, VadFrame}, VoiceActivityDetector, @@ -27,6 +27,7 @@ pub struct AudioRecorder { cmd_tx: Option>, worker_handle: Option>, vad: Option>>>, + level_cb: Option) + Send + Sync + 'static>>, } impl AudioRecorder { @@ -36,6 +37,7 @@ impl AudioRecorder { cmd_tx: None, worker_handle: None, vad: None, + level_cb: None, }) } @@ -44,6 +46,14 @@ impl AudioRecorder { self } + pub fn with_level_callback(mut self, cb: F) -> Self + where + F: Fn(Vec) + Send + Sync + 'static, + { + self.level_cb = Some(Arc::new(cb)); + self + } + pub fn open(&mut self, device: Option) -> Result<(), Box> { if self.worker_handle.is_some() { return Ok(()); // already open @@ -62,6 +72,8 @@ impl AudioRecorder { let thread_device = device.clone(); let vad = self.vad.clone(); + // Move the optional level callback into the worker thread + let level_cb = self.level_cb.clone(); let worker = std::thread::spawn(move || { let config = AudioRecorder::get_preferred_config(&thread_device) @@ -105,7 +117,7 @@ impl AudioRecorder { stream.play().expect("failed to start stream"); // keep the stream alive while we process samples - run_consumer(sample_rate, vad, sample_rx, cmd_rx); + run_consumer(sample_rate, vad, sample_rx, cmd_rx, level_cb); // stream is dropped here, after run_consumer returns }); @@ -215,6 +227,7 @@ fn run_consumer( vad: Option>>>, sample_rx: mpsc::Receiver>, cmd_rx: mpsc::Receiver, + level_cb: Option) + Send + Sync + 'static>>, ) { let mut frame_resampler = FrameResampler::new( in_sample_rate as usize, @@ -225,6 +238,17 @@ fn run_consumer( let mut processed_samples = Vec::::new(); let mut recording = false; + // ---------- spectrum visualisation setup ---------------------------- // + const BUCKETS: usize = 16; + const WINDOW_SIZE: usize = 512; + let mut visualizer = AudioVisualiser::new( + in_sample_rate, + WINDOW_SIZE, + BUCKETS, + 80.0, // vocal_min_hz + 4000.0, // vocal_max_hz + ); + fn handle_frame( samples: &[f32], recording: bool, @@ -252,6 +276,14 @@ fn run_consumer( Err(_) => break, // stream closed }; + // ---------- spectrum processing ---------------------------------- // + if let Some(buckets) = visualizer.feed(&raw) { + if let Some(cb) = &level_cb { + cb(buckets); + } + } + + // ---------- existing pipeline ------------------------------------ // frame_resampler.push(&raw, &mut |frame: &[f32]| { handle_frame(frame, recording, &vad, &mut processed_samples) }); @@ -262,6 +294,7 @@ fn run_consumer( Cmd::Start => { processed_samples.clear(); recording = true; + visualizer.reset(); // Reset visualization buffer if let Some(v) = &vad { v.lock().unwrap().reset(); } diff --git a/src-tauri/src/audio_toolkit/audio/visualizer.rs b/src-tauri/src/audio_toolkit/audio/visualizer.rs new file mode 100644 index 0000000..b0ef038 --- /dev/null +++ b/src-tauri/src/audio_toolkit/audio/visualizer.rs @@ -0,0 +1,156 @@ +use rustfft::{num_complex::Complex32, Fft, FftPlanner}; +use std::sync::Arc; + +const DB_MIN: f32 = -55.0; +const DB_MAX: f32 = -8.0; +const GAIN: f32 = 1.3; +const CURVE_POWER: f32 = 0.7; + +pub struct AudioVisualiser { + fft: Arc>, + window: Vec, + bucket_ranges: Vec<(usize, usize)>, + fft_input: Vec, + noise_floor: Vec, + buffer: Vec, + window_size: usize, + buckets: usize, +} + +impl AudioVisualiser { + pub fn new( + sample_rate: u32, + window_size: usize, + buckets: usize, + freq_min: f32, + freq_max: f32, + ) -> Self { + let mut planner = FftPlanner::::new(); + let fft = planner.plan_fft_forward(window_size); + + // Pre-compute Hann window + let window: Vec = (0..window_size) + .map(|i| { + 0.5 * (1.0 - (2.0 * std::f32::consts::PI * i as f32 / window_size as f32).cos()) + }) + .collect(); + + // Pre-compute bucket frequency ranges + let nyquist = sample_rate as f32 / 2.0; + let freq_min = freq_min.min(nyquist); + let freq_max = freq_max.min(nyquist); + + let mut bucket_ranges = Vec::with_capacity(buckets); + + for b in 0..buckets { + // Use logarithmic spacing for better perceptual representation + let log_start = (b as f32 / buckets as f32).powi(2); + let log_end = ((b + 1) as f32 / buckets as f32).powi(2); + + let start_hz = freq_min + (freq_max - freq_min) * log_start; + let end_hz = freq_min + (freq_max - freq_min) * log_end; + + let start_bin = ((start_hz * window_size as f32) / sample_rate as f32) as usize; + let mut end_bin = ((end_hz * window_size as f32) / sample_rate as f32) as usize; + + // Ensure each bucket has at least one bin + if end_bin <= start_bin { + end_bin = start_bin + 1; + } + + // Clamp to valid range + let start_bin = start_bin.min(window_size / 2); + let end_bin = end_bin.min(window_size / 2); + + bucket_ranges.push((start_bin, end_bin)); + } + + Self { + fft, + window, + bucket_ranges, + fft_input: vec![Complex32::new(0.0, 0.0); window_size], + noise_floor: vec![-40.0; buckets], // Initialize to reasonable noise floor + buffer: Vec::with_capacity(window_size * 2), + window_size, + buckets, + } + } + + pub fn feed(&mut self, samples: &[f32]) -> Option> { + // Add new samples to buffer + self.buffer.extend_from_slice(samples); + + // Only process if we have enough samples + if self.buffer.len() < self.window_size { + return None; + } + + // Take the required window of samples + let window_samples = &self.buffer[..self.window_size]; + + // Remove DC component + let mean = window_samples.iter().sum::() / self.window_size as f32; + + // Apply window function and prepare FFT input + for (i, &sample) in window_samples.iter().enumerate() { + let windowed_sample = (sample - mean) * self.window[i]; + self.fft_input[i] = Complex32::new(windowed_sample, 0.0); + } + + // Perform FFT + self.fft.process(&mut self.fft_input); + + // Compute power spectrum and bucket levels + let mut buckets = vec![0.0; self.buckets]; + + for (bucket_idx, &(start_bin, end_bin)) in self.bucket_ranges.iter().enumerate() { + if start_bin >= end_bin || end_bin > self.fft_input.len() / 2 { + continue; + } + + // Calculate average power in this frequency range + let mut power_sum = 0.0; + for bin_idx in start_bin..end_bin { + let magnitude = self.fft_input[bin_idx].norm(); + power_sum += magnitude * magnitude; + } + + let avg_power = power_sum / (end_bin - start_bin) as f32; + + // Convert to dB with proper scaling + let db = if avg_power > 1e-12 { + 20.0 * (avg_power.sqrt() / self.window_size as f32).log10() + } else { + -80.0 // Very low floor for zero power + }; + + // Only update noise floor when signal is quiet (below current floor + 10dB) + if db < self.noise_floor[bucket_idx] + 10.0 { + const NOISE_ALPHA: f32 = 0.001; // Very slow adaptation + self.noise_floor[bucket_idx] = + NOISE_ALPHA * db + (1.0 - NOISE_ALPHA) * self.noise_floor[bucket_idx]; + } + + // Map configurable dB range to 0-1 with gain and curve shaping + let normalized = ((db - DB_MIN) / (DB_MAX - DB_MIN)).clamp(0.0, 1.0); + buckets[bucket_idx] = (normalized * GAIN).powf(CURVE_POWER).clamp(0.0, 1.0); + } + + // Apply light smoothing to reduce jitter + for i in 1..buckets.len() - 1 { + buckets[i] = buckets[i] * 0.7 + buckets[i - 1] * 0.15 + buckets[i + 1] * 0.15; + } + + // Clear processed samples from buffer + self.buffer.clear(); + + Some(buckets) + } + + pub fn reset(&mut self) { + self.buffer.clear(); + // Reset noise floor to initial values + self.noise_floor.fill(-40.0); + } +} diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs index e5c8994..9bc1ab1 100644 --- a/src-tauri/src/managers/audio.rs +++ b/src-tauri/src/managers/audio.rs @@ -1,9 +1,10 @@ use crate::audio_toolkit::{list_input_devices, vad::SmoothedVad, AudioRecorder, SileroVad}; use crate::settings::get_settings; +use crate::utils; use log::{debug, info}; use std::sync::{Arc, Mutex}; use std::time::Instant; -use tauri::{App, Manager}; +use tauri::{App, Emitter, Manager}; const WHISPER_SAMPLE_RATE: usize = 16000; @@ -23,13 +24,26 @@ pub enum MicrophoneMode { /* ──────────────────────────────────────────────────────────────── */ -fn create_audio_recorder(vad_path: &str) -> Result { +fn create_audio_recorder( + vad_path: &str, + app_handle: &tauri::AppHandle, +) -> Result { let silero = SileroVad::new(vad_path, 0.3) .map_err(|e| anyhow::anyhow!("Failed to create SileroVad: {}", e))?; let smoothed_vad = SmoothedVad::new(Box::new(silero), 15, 15, 2); + + // Recorder with VAD plus a spectrum-level callback that forwards updates to + // the frontend. let recorder = AudioRecorder::new() .map_err(|e| anyhow::anyhow!("Failed to create AudioRecorder: {}", e))? - .with_vad(Box::new(smoothed_vad)); + .with_vad(Box::new(smoothed_vad)) + .with_level_callback({ + let app_handle = app_handle.clone(); + move |levels| { + utils::emit_levels(&app_handle, &levels); + } + }); + Ok(recorder) } @@ -97,7 +111,10 @@ impl AudioRecordingManager { let mut recorder_opt = self.recorder.lock().unwrap(); if recorder_opt.is_none() { - *recorder_opt = Some(create_audio_recorder(vad_path.to_str().unwrap())?); + *recorder_opt = Some(create_audio_recorder( + vad_path.to_str().unwrap(), + &self.app_handle, + )?); } // Get the selected device from settings diff --git a/src-tauri/src/utils.rs b/src-tauri/src/utils.rs index 0d3d421..8597465 100644 --- a/src-tauri/src/utils.rs +++ b/src-tauri/src/utils.rs @@ -274,3 +274,14 @@ pub fn hide_recording_overlay(app_handle: &AppHandle) { }); } } + +pub fn emit_levels(app_handle: &AppHandle, levels: &Vec) { + println!("levels {:#?}", levels); + // emit levels to main app + let _ = app_handle.emit("mic-level", levels); + + // also emit to the recording overlay if it's open + if let Some(overlay_window) = app_handle.get_webview_window("recording_overlay") { + let _ = overlay_window.emit("mic-level", levels); + } +} diff --git a/src/overlay/RecordingOverlay.css b/src/overlay/RecordingOverlay.css index 24f93dc..80f7e35 100644 --- a/src/overlay/RecordingOverlay.css +++ b/src/overlay/RecordingOverlay.css @@ -18,29 +18,19 @@ align-items: end; justify-content: center; gap: 3px; - height: 30px; + padding-bottom: 0px; + height: 28px; } .bar { - width: 4px; + width: 6px; background: linear-gradient(to top, #faa2ca, #f28cbb); - border-radius: 2px; - animation: pulse infinite ease-in-out; + max-height: 28px; + border-radius: 4px; + transition: height 80ms linear; min-height: 4px; } -@keyframes pulse { - 0%, - 100% { - height: 4px; - opacity: 0.6; - } - 50% { - height: 28px; - opacity: 1; - } -} - .recording-overlay.fade-in { opacity: 1; } diff --git a/src/overlay/RecordingOverlay.tsx b/src/overlay/RecordingOverlay.tsx index 2b6fa02..8297120 100644 --- a/src/overlay/RecordingOverlay.tsx +++ b/src/overlay/RecordingOverlay.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useState } from "react"; +import React, { useEffect, useState, useRef } from "react"; import "./RecordingOverlay.css"; import { listen } from "@tauri-apps/api/event"; @@ -7,6 +7,8 @@ type OverlayState = "recording" | "transcribing"; const RecordingOverlay: React.FC = () => { const [isVisible, setIsVisible] = useState(false); const [state, setState] = useState("recording"); + const [levels, setLevels] = useState(Array(16).fill(0)); + const smoothedLevelsRef = useRef(Array(16).fill(0)); useEffect(() => { const setupEventListeners = async () => { @@ -22,10 +24,26 @@ const RecordingOverlay: React.FC = () => { setIsVisible(false); }); + // Listen for mic-level updates + const unlistenLevel = await listen("mic-level", (event) => { + const newLevels = event.payload as number[]; + + // Apply smoothing to reduce jitter + const smoothed = smoothedLevelsRef.current.map((prev, i) => { + const target = newLevels[i] || 0; + return prev * 0.7 + target * 0.3; // Smooth transition + }); + + smoothedLevelsRef.current = smoothed; + console.log(smoothed.length); + setLevels(smoothed.slice(0, 10)); + }); + // Cleanup function return () => { unlistenShow(); unlistenHide(); + unlistenLevel(); }; }; @@ -53,13 +71,14 @@ const RecordingOverlay: React.FC = () => { /> {state === "recording" && (
- {Array.from({ length: 12 }, (_, i) => ( + {levels.map((v, i) => (
))}