* Add LLM post-processing feature with settings management and UI integration * Refactor LLM post-processing settings and prompt selection logic * Rename CLAUDE.md to more standard AGENTS.md * Replace Open Router with OpenAI Compatible endpoints for post-processing * Fix post processing settings api styles * refactor or something * simplify * cleanup a bit? * remove useless code * minor improvements * fix laggy textarea * Store post-processing results in db * fix text color * new default prompt --------- Co-authored-by: CJ Pais <cj@cjpais.com>
376 lines
14 KiB
Rust
376 lines
14 KiB
Rust
use crate::audio_feedback::{play_feedback_sound, SoundType};
|
|
use crate::managers::audio::AudioRecordingManager;
|
|
use crate::managers::history::HistoryManager;
|
|
use crate::managers::transcription::TranscriptionManager;
|
|
use crate::overlay::{show_recording_overlay, show_transcribing_overlay};
|
|
use crate::settings::{get_settings, AppSettings};
|
|
use crate::tray::{change_tray_icon, TrayIconState};
|
|
use crate::utils;
|
|
use async_openai::types::{
|
|
ChatCompletionRequestMessage, ChatCompletionRequestUserMessageArgs,
|
|
CreateChatCompletionRequestArgs,
|
|
};
|
|
use log::{debug, error};
|
|
use once_cell::sync::Lazy;
|
|
use std::collections::HashMap;
|
|
use std::sync::Arc;
|
|
use std::time::Instant;
|
|
use tauri::AppHandle;
|
|
use tauri::Manager;
|
|
|
|
// Shortcut Action Trait
|
|
pub trait ShortcutAction: Send + Sync {
|
|
fn start(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str);
|
|
fn stop(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str);
|
|
}
|
|
|
|
// Transcribe Action
|
|
struct TranscribeAction;
|
|
|
|
async fn maybe_post_process_transcription(
|
|
settings: &AppSettings,
|
|
transcription: &str,
|
|
) -> Option<String> {
|
|
if !settings.post_process_enabled {
|
|
return None;
|
|
}
|
|
|
|
let provider = match settings.active_post_process_provider().cloned() {
|
|
Some(provider) => provider,
|
|
None => {
|
|
debug!("Post-processing enabled but no provider is selected");
|
|
return None;
|
|
}
|
|
};
|
|
|
|
let model = settings
|
|
.post_process_models
|
|
.get(&provider.id)
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
|
|
if model.trim().is_empty() {
|
|
debug!(
|
|
"Post-processing skipped because provider '{}' has no model configured",
|
|
provider.id
|
|
);
|
|
return None;
|
|
}
|
|
|
|
let selected_prompt_id = match &settings.post_process_selected_prompt_id {
|
|
Some(id) => id.clone(),
|
|
None => {
|
|
debug!("Post-processing skipped because no prompt is selected");
|
|
return None;
|
|
}
|
|
};
|
|
|
|
let prompt = match settings
|
|
.post_process_prompts
|
|
.iter()
|
|
.find(|prompt| prompt.id == selected_prompt_id)
|
|
{
|
|
Some(prompt) => prompt.prompt.clone(),
|
|
None => {
|
|
debug!(
|
|
"Post-processing skipped because prompt '{}' was not found",
|
|
selected_prompt_id
|
|
);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
if prompt.trim().is_empty() {
|
|
debug!("Post-processing skipped because the selected prompt is empty");
|
|
return None;
|
|
}
|
|
|
|
let api_key = settings
|
|
.post_process_api_keys
|
|
.get(&provider.id)
|
|
.cloned()
|
|
.unwrap_or_default();
|
|
|
|
debug!(
|
|
"Starting LLM post-processing with provider '{}' (model: {})",
|
|
provider.id, model
|
|
);
|
|
|
|
// Replace ${output} variable in the prompt with the actual text
|
|
let processed_prompt = prompt.replace("${output}", transcription);
|
|
debug!("Processed prompt length: {} chars", processed_prompt.len());
|
|
|
|
// Create OpenAI-compatible client
|
|
let client = match crate::llm_client::create_client(&provider, api_key) {
|
|
Ok(client) => client,
|
|
Err(e) => {
|
|
error!("Failed to create LLM client: {}", e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
// Build the chat completion request
|
|
let message = match ChatCompletionRequestUserMessageArgs::default()
|
|
.content(processed_prompt)
|
|
.build()
|
|
{
|
|
Ok(msg) => ChatCompletionRequestMessage::User(msg),
|
|
Err(e) => {
|
|
error!("Failed to build chat message: {}", e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
let request = match CreateChatCompletionRequestArgs::default()
|
|
.model(&model)
|
|
.messages(vec![message])
|
|
.build()
|
|
{
|
|
Ok(req) => req,
|
|
Err(e) => {
|
|
error!("Failed to build chat completion request: {}", e);
|
|
return None;
|
|
}
|
|
};
|
|
|
|
// Send the request
|
|
match client.chat().create(request).await {
|
|
Ok(response) => {
|
|
if let Some(choice) = response.choices.first() {
|
|
if let Some(content) = &choice.message.content {
|
|
debug!(
|
|
"LLM post-processing succeeded for provider '{}'. Output length: {} chars",
|
|
provider.id,
|
|
content.len()
|
|
);
|
|
return Some(content.clone());
|
|
}
|
|
}
|
|
error!("LLM API response has no content");
|
|
None
|
|
}
|
|
Err(e) => {
|
|
error!(
|
|
"LLM post-processing failed for provider '{}': {}. Falling back to original transcription.",
|
|
provider.id,
|
|
e
|
|
);
|
|
None
|
|
}
|
|
}
|
|
}
|
|
|
|
impl ShortcutAction for TranscribeAction {
|
|
fn start(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) {
|
|
let start_time = Instant::now();
|
|
debug!("TranscribeAction::start called for binding: {}", binding_id);
|
|
|
|
// Load model in the background
|
|
let tm = app.state::<Arc<TranscriptionManager>>();
|
|
tm.initiate_model_load();
|
|
|
|
let binding_id = binding_id.to_string();
|
|
change_tray_icon(app, TrayIconState::Recording);
|
|
show_recording_overlay(app);
|
|
|
|
let rm = app.state::<Arc<AudioRecordingManager>>();
|
|
|
|
// Get the microphone mode to determine audio feedback timing
|
|
let settings = get_settings(app);
|
|
let is_always_on = settings.always_on_microphone;
|
|
debug!("Microphone mode - always_on: {}", is_always_on);
|
|
|
|
if is_always_on {
|
|
// Always-on mode: Play audio feedback immediately
|
|
debug!("Always-on mode: Playing audio feedback immediately");
|
|
play_feedback_sound(app, SoundType::Start);
|
|
let recording_started = rm.try_start_recording(&binding_id);
|
|
debug!("Recording started: {}", recording_started);
|
|
} else {
|
|
// On-demand mode: Start recording first, then play audio feedback
|
|
// This allows the microphone to be activated before playing the sound
|
|
debug!("On-demand mode: Starting recording first, then audio feedback");
|
|
let recording_start_time = Instant::now();
|
|
if rm.try_start_recording(&binding_id) {
|
|
debug!("Recording started in {:?}", recording_start_time.elapsed());
|
|
// Small delay to ensure microphone stream is active
|
|
let app_clone = app.clone();
|
|
std::thread::spawn(move || {
|
|
std::thread::sleep(std::time::Duration::from_millis(100));
|
|
debug!("Playing delayed audio feedback");
|
|
play_feedback_sound(&app_clone, SoundType::Start);
|
|
});
|
|
} else {
|
|
debug!("Failed to start recording");
|
|
}
|
|
}
|
|
|
|
debug!(
|
|
"TranscribeAction::start completed in {:?}",
|
|
start_time.elapsed()
|
|
);
|
|
}
|
|
|
|
fn stop(&self, app: &AppHandle, binding_id: &str, _shortcut_str: &str) {
|
|
let stop_time = Instant::now();
|
|
debug!("TranscribeAction::stop called for binding: {}", binding_id);
|
|
|
|
let ah = app.clone();
|
|
let rm = Arc::clone(&app.state::<Arc<AudioRecordingManager>>());
|
|
let tm = Arc::clone(&app.state::<Arc<TranscriptionManager>>());
|
|
let hm = Arc::clone(&app.state::<Arc<HistoryManager>>());
|
|
|
|
change_tray_icon(app, TrayIconState::Transcribing);
|
|
show_transcribing_overlay(app);
|
|
|
|
// Play audio feedback for recording stop
|
|
play_feedback_sound(app, SoundType::Stop);
|
|
|
|
let binding_id = binding_id.to_string(); // Clone binding_id for the async task
|
|
|
|
tauri::async_runtime::spawn(async move {
|
|
let binding_id = binding_id.clone(); // Clone for the inner async task
|
|
debug!(
|
|
"Starting async transcription task for binding: {}",
|
|
binding_id
|
|
);
|
|
|
|
let stop_recording_time = Instant::now();
|
|
if let Some(samples) = rm.stop_recording(&binding_id) {
|
|
debug!(
|
|
"Recording stopped and samples retrieved in {:?}, sample count: {}",
|
|
stop_recording_time.elapsed(),
|
|
samples.len()
|
|
);
|
|
|
|
let transcription_time = Instant::now();
|
|
let samples_clone = samples.clone(); // Clone for history saving
|
|
match tm.transcribe(samples) {
|
|
Ok(transcription) => {
|
|
debug!(
|
|
"Transcription completed in {:?}: '{}'",
|
|
transcription_time.elapsed(),
|
|
transcription
|
|
);
|
|
if !transcription.is_empty() {
|
|
let settings = get_settings(&ah);
|
|
let mut final_text = transcription.clone();
|
|
let mut post_processed_text: Option<String> = None;
|
|
let mut post_process_prompt: Option<String> = None;
|
|
|
|
if let Some(processed_text) =
|
|
maybe_post_process_transcription(&settings, &transcription).await
|
|
{
|
|
final_text = processed_text.clone();
|
|
post_processed_text = Some(processed_text);
|
|
|
|
// Get the prompt that was used
|
|
if let Some(prompt_id) = &settings.post_process_selected_prompt_id {
|
|
if let Some(prompt) = settings
|
|
.post_process_prompts
|
|
.iter()
|
|
.find(|p| &p.id == prompt_id)
|
|
{
|
|
post_process_prompt = Some(prompt.prompt.clone());
|
|
}
|
|
}
|
|
}
|
|
|
|
// Save to history with post-processed text and prompt
|
|
let hm_clone = Arc::clone(&hm);
|
|
let transcription_for_history = transcription.clone();
|
|
tauri::async_runtime::spawn(async move {
|
|
if let Err(e) = hm_clone
|
|
.save_transcription(
|
|
samples_clone,
|
|
transcription_for_history,
|
|
post_processed_text,
|
|
post_process_prompt,
|
|
)
|
|
.await
|
|
{
|
|
error!("Failed to save transcription to history: {}", e);
|
|
}
|
|
});
|
|
|
|
// Paste the final text (either processed or original)
|
|
let ah_clone = ah.clone();
|
|
let paste_time = Instant::now();
|
|
ah.run_on_main_thread(move || {
|
|
match utils::paste(final_text, ah_clone.clone()) {
|
|
Ok(()) => debug!(
|
|
"Text pasted successfully in {:?}",
|
|
paste_time.elapsed()
|
|
),
|
|
Err(e) => eprintln!("Failed to paste transcription: {}", e),
|
|
}
|
|
// Hide the overlay after transcription is complete
|
|
utils::hide_recording_overlay(&ah_clone);
|
|
change_tray_icon(&ah_clone, TrayIconState::Idle);
|
|
})
|
|
.unwrap_or_else(|e| {
|
|
eprintln!("Failed to run paste on main thread: {:?}", e);
|
|
utils::hide_recording_overlay(&ah);
|
|
change_tray_icon(&ah, TrayIconState::Idle);
|
|
});
|
|
} else {
|
|
utils::hide_recording_overlay(&ah);
|
|
change_tray_icon(&ah, TrayIconState::Idle);
|
|
}
|
|
}
|
|
Err(err) => {
|
|
debug!("Global Shortcut Transcription error: {}", err);
|
|
utils::hide_recording_overlay(&ah);
|
|
change_tray_icon(&ah, TrayIconState::Idle);
|
|
}
|
|
}
|
|
} else {
|
|
debug!("No samples retrieved from recording stop");
|
|
utils::hide_recording_overlay(&ah);
|
|
change_tray_icon(&ah, TrayIconState::Idle);
|
|
}
|
|
});
|
|
|
|
debug!(
|
|
"TranscribeAction::stop completed in {:?}",
|
|
stop_time.elapsed()
|
|
);
|
|
}
|
|
}
|
|
|
|
// Test Action
|
|
struct TestAction;
|
|
|
|
impl ShortcutAction for TestAction {
|
|
fn start(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str) {
|
|
println!(
|
|
"Shortcut ID '{}': Started - {} (App: {})", // Changed "Pressed" to "Started" for consistency
|
|
binding_id,
|
|
shortcut_str,
|
|
app.package_info().name
|
|
);
|
|
}
|
|
|
|
fn stop(&self, app: &AppHandle, binding_id: &str, shortcut_str: &str) {
|
|
println!(
|
|
"Shortcut ID '{}': Stopped - {} (App: {})", // Changed "Released" to "Stopped" for consistency
|
|
binding_id,
|
|
shortcut_str,
|
|
app.package_info().name
|
|
);
|
|
}
|
|
}
|
|
|
|
// Static Action Map
|
|
pub static ACTION_MAP: Lazy<HashMap<String, Arc<dyn ShortcutAction>>> = Lazy::new(|| {
|
|
let mut map = HashMap::new();
|
|
map.insert(
|
|
"transcribe".to_string(),
|
|
Arc::new(TranscribeAction) as Arc<dyn ShortcutAction>,
|
|
);
|
|
map.insert(
|
|
"test".to_string(),
|
|
Arc::new(TestAction) as Arc<dyn ShortcutAction>,
|
|
);
|
|
map
|
|
});
|