From 6b088deaf551c6b38b267d840814f436148a18b1 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 17 Feb 2025 09:08:48 -0800 Subject: [PATCH] Very basic version of the clipboard implemented. --- src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 1 + src-tauri/src/lib.rs | 284 +++++++++++++++++---------- src-tauri/src/managers/keybinding.rs | 3 + 4 files changed, 188 insertions(+), 101 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 615b273..0cca731 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1982,6 +1982,7 @@ dependencies = [ "tauri-plugin-macos-permissions", "tauri-plugin-opener", "tauri-plugin-single-instance", + "tokio", "whisper-rs", ] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 0e4720b..3e4ad4f 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -33,6 +33,7 @@ samplerate = "0.2.4" rig-core = "0.8.0" env_logger = "0.11.6" log = "0.4.25" +tokio = "1.43.0" [target.'cfg(target_os = "macos")'.dependencies] whisper-rs = { version = "0.13.2", features = ["metal"] } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7b480a2..213b898 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -4,10 +4,166 @@ use log::{info, Level}; use managers::keybinding::KeyBindingManager; use managers::transcription::TranscriptionManager; use managers::{audio::AudioRecordingManager, transcription}; -use rdev::Key; +use rdev::{simulate, EventType, Key, SimulateError}; use rig::{completion::Prompt, providers::anthropic}; use std::sync::{Arc, Mutex}; +use std::{thread, time}; use tauri_plugin_autostart::MacosLauncher; +use tauri_plugin_clipboard_manager::ClipboardExt; + +fn try_send_event(event: &EventType) { + if let Err(SimulateError) = simulate(event) { + println!("We could not send {:?}", event); + } +} + +fn send(event: EventType) { + try_send_event(&event); + thread::sleep(time::Duration::from_millis(20)); +} + +fn send_multiple(events: &[EventType]) { + events.iter().for_each(try_send_event); + thread::sleep(time::Duration::from_millis(20)); +} + +fn send_paste() { + // Determine the modifier key based on the OS + #[cfg(target_os = "macos")] + let modifier_key = Key::MetaLeft; // Command key on macOS + #[cfg(not(target_os = "macos"))] + let modifier_key = Key::ControlLeft; // Control key on other systems + + // Press both keys + send(EventType::KeyPress(modifier_key)); + send(EventType::KeyPress(Key::KeyV)); + + // Release both keys simultaneously + send_multiple(&[ + EventType::KeyRelease(Key::KeyV), + EventType::KeyRelease(modifier_key), + ]); + + // Additional delay after the complete combination + // thread::sleep(time::Duration::from_millis(50)); +} + +fn paste(text: String, app_handle: tauri::AppHandle) { + let clipboard = app_handle.clipboard(); + + // get the current clipboard content + let clipboard_content = clipboard.read_text().unwrap_or_default(); + + clipboard.write_text(&text).unwrap(); + send_paste(); + + // restore the clipboard + clipboard.write_text(&clipboard_content).unwrap(); +} + +fn get_highlighted_text(app_handle: tauri::AppHandle) -> String { + let clipboard = app_handle.clipboard(); + + // save the clipboard content + let clipboard_content = clipboard.read_text().unwrap_or_default(); + + // empty the clipboard + clipboard.write_text("").unwrap(); + + // get the highlighted text + let highlighted_text = clipboard.read_text().unwrap_or_default(); + + // restore the clipboard content + clipboard.write_text(&clipboard_content).unwrap(); + + highlighted_text +} + +fn register_bindings(manager: &mut KeyBindingManager) { + manager.register( + "ctrl-meta".to_string(), + vec![Key::ControlRight, Key::MetaRight], + |ctx| { + info!("Ctrl+Meta pressed!"); + ctx.recording_manager.try_start_recording("ctrl-meta"); + None + }, + |ctx| { + info!("release being called from ctrl-meta"); + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { + match ctx.transcription_manager.transcribe(samples) { + Ok(transcription) => { + println!("Transcription: {}", transcription); + paste(transcription, ctx.app_handle.clone()); + } + Err(err) => println!("Transcription error: {}", err), + } + } + })) + }, + ); + + // Register LLM Call after Transcription + manager.register( + "shift-alt".to_string(), + vec![Key::ShiftLeft, Key::Alt], + |ctx| { + info!("Shift+Alt pressed!"); + ctx.recording_manager.try_start_recording("shift-alt"); + None + }, + |ctx| { + info!("release being called from shift-alt"); + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { + if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) { + println!("Transcription: {}", transcription); + match ctx.sonnet.prompt(transcription).await { + Ok(response) => { + let highlighted_text = get_highlighted_text(ctx.app_handle.clone()); + println!("Highlighted Text: {}", highlighted_text); + println!("Sonnet response: {}", response); + paste(response, ctx.app_handle.clone()); + } + Err(err) => println!("Sonnet error: {}", err), + } + } + } + })) + }, + ); + + manager.register( + "ctrl-alt-meta".to_string(), + vec![Key::ControlLeft, Key::Alt, Key::MetaLeft], + |ctx| { + info!("Ctrl+Alt+Meta pressed!"); + ctx.recording_manager.try_start_recording("ctrl-alt-meta"); + None + }, + |ctx| { + info!("release being called from ctrl-alt-meta"); + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") { + let samples: Vec = samples; // explicit type annotation + match ctx.transcription_manager.transcribe(samples) { + Ok(transcription) => { + println!("Transcription: {}", transcription); + // Call LLM for code + } + Err(err) => println!("Transcription error: {}", err), + } + } else { + println!("No samples recorded"); + } + })) + }, + ); +} #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { @@ -27,106 +183,6 @@ pub fn run() { .build(), ); - let manager = Arc::new(Mutex::new(KeyBindingManager::new( - recording_manager.clone(), - transcription_manager.clone(), - sonnet.clone(), - ))); - - // Register your key bindings - { - let mut manager = manager.lock().unwrap(); - - // Register Basic Transcription - manager.register( - "ctrl-meta".to_string(), - vec![Key::ControlRight, Key::MetaRight], - |ctx| { - info!("Ctrl+Meta pressed!"); - ctx.recording_manager.try_start_recording("ctrl-meta"); - None - }, - |ctx| { - info!("release being called from ctrl-meta"); - let ctx = ctx.clone(); - Some(tauri::async_runtime::spawn(async move { - if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { - match ctx.transcription_manager.transcribe(samples) { - Ok(transcription) => println!("Transcription: {}", transcription), - Err(err) => println!("Transcription error: {}", err), - } - } - })) - }, - ); - - // Register LLM Call after Transcription - manager.register( - "shift-alt".to_string(), - vec![Key::ShiftLeft, Key::Alt], - |ctx| { - info!("Shift+Alt pressed!"); - ctx.recording_manager.try_start_recording("shift-alt"); - None - }, - |ctx| { - info!("release being called from shift-alt"); - let ctx = ctx.clone(); - Some(tauri::async_runtime::spawn(async move { - if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { - if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) { - println!("Transcription: {}", transcription); - match ctx.sonnet.prompt(transcription).await { - Ok(response) => println!("Sonnet response: {}", response), - Err(err) => println!("Sonnet error: {}", err), - } - } - } - })) - }, - ); - - manager.register( - "ctrl-alt-meta".to_string(), - vec![Key::ControlLeft, Key::Alt, Key::MetaLeft], - |ctx| { - info!("Ctrl+Alt+Meta pressed!"); - ctx.recording_manager.try_start_recording("ctrl-alt-meta"); - None - }, - |ctx| { - info!("release being called from ctrl-alt-meta"); - let ctx = ctx.clone(); - Some(tauri::async_runtime::spawn(async move { - if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") { - let samples: Vec = samples; // explicit type annotation - match ctx.transcription_manager.transcribe(samples) { - Ok(transcription) => { - println!("Transcription: {}", transcription); - // Call LLM for code - } - Err(err) => println!("Transcription error: {}", err), - } - } else { - println!("No samples recorded"); - } - })) - }, - ); - } - - tauri::async_runtime::spawn({ - let manager = manager.clone(); - async move { - rdev::listen(move |event| { - if let Ok(manager) = manager.lock() { - manager.handle_event(&event); - } - }) - .unwrap(); - } - }); - tauri::Builder::default() .plugin(tauri_plugin_global_shortcut::Builder::new().build()) .plugin(tauri_plugin_clipboard_manager::init()) @@ -136,6 +192,32 @@ pub fn run() { )) .plugin(tauri_plugin_macos_permissions::init()) .plugin(tauri_plugin_opener::init()) + .setup(move |app| { + let app_handle = app.handle().clone(); + + let manager = Arc::new(Mutex::new(KeyBindingManager::new( + recording_manager.clone(), + transcription_manager.clone(), + sonnet.clone(), + app_handle.clone(), + ))); + + { + let mut manager = manager.lock().unwrap(); + register_bindings(&mut manager); + } + + let manager_clone = manager.clone(); + tauri::async_runtime::spawn(async move { + rdev::listen(move |event| { + if let Ok(manager) = manager_clone.lock() { + manager.handle_event(&event); + } + }) + .unwrap(); + }); + Ok(()) + }) .invoke_handler(tauri::generate_handler![]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/managers/keybinding.rs b/src-tauri/src/managers/keybinding.rs index a2c4072..b424a6f 100644 --- a/src-tauri/src/managers/keybinding.rs +++ b/src-tauri/src/managers/keybinding.rs @@ -13,6 +13,7 @@ pub struct BindingContext { pub recording_manager: Arc, pub transcription_manager: Arc, pub sonnet: Arc>, + pub app_handle: tauri::AppHandle, } pub struct KeyBinding { @@ -92,6 +93,7 @@ impl KeyBindingManager { recording_manager: Arc, transcription_manager: Arc, sonnet: Arc>, + app_handle: tauri::AppHandle, ) -> Self { Self { bindings: Vec::new(), @@ -99,6 +101,7 @@ impl KeyBindingManager { recording_manager, transcription_manager, sonnet, + app_handle, }, } }