From 661c46eae8aaf9d9e6c9e3e74c21b518ab81b7bb Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 16 Feb 2025 19:17:44 -0800 Subject: [PATCH 1/4] most basic example LLM calling --- src-tauri/Cargo.lock | 402 +++++++++++++++++++++++++-- src-tauri/Cargo.toml | 1 + src-tauri/src/lib.rs | 83 +++++- src-tauri/src/managers/keybinding.rs | 40 +-- 4 files changed, 475 insertions(+), 51 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 12473f4..6e4a957 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -228,6 +228,28 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "async-task" version = "4.7.1" @@ -1176,6 +1198,15 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endi" version = "1.1.0" @@ -1394,6 +1425,21 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -1401,6 +1447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -1468,6 +1515,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -1814,6 +1862,25 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "h2" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.7.1", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "handy-app" version = "0.1.0" @@ -1821,6 +1888,7 @@ dependencies = [ "anyhow", "cpal", "rdev", + "rig-core", "rubato", "samplerate", "serde", @@ -1895,6 +1963,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa 1.0.14", +] + [[package]] name = "http" version = "1.2.0" @@ -1906,6 +1985,17 @@ dependencies = [ "itoa 1.0.14", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1913,7 +2003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.2.0", ] [[package]] @@ -1924,8 +2014,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1935,6 +2025,36 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa 1.0.14", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.6.0" @@ -1944,8 +2064,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "httparse", "itoa 1.0.14", "pin-project-lite", @@ -1961,8 +2081,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.2.0", + "hyper 1.6.0", "hyper-util", "rustls", "rustls-pki-types", @@ -1972,6 +2092,19 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper 0.14.32", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "hyper-util" version = "0.1.10" @@ -1981,9 +2114,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.2.0", + "http-body 1.0.1", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", @@ -2647,6 +2780,23 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "native-tls" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dab59f8e050d5df8e4dd87d9206fb6f65a483e20ac9fda365ade4fab353196c" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndk" version = "0.8.0" @@ -3077,12 +3227,65 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openssl" +version = "0.10.71" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e14130c6a98cd258fdcb0fb6d744152343ff729cbfcb28c656a9d12b999fbcd" +dependencies = [ + "bitflags 2.8.0", + "cfg-if", + "foreign-types 0.3.2", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb61ea9811cc39e3c2069f40b8b8e2e70d8569b361f879786cc7ed48b777cdd" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-stream" version = "0.2.0" @@ -3720,6 +3923,48 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "reqwest" +version = "0.11.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +dependencies = [ + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-tls", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg 0.50.0", +] + [[package]] name = "reqwest" version = "0.12.12" @@ -3730,10 +3975,10 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", - "http-body", + "http 1.2.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.6.0", "hyper-rustls", "hyper-util", "ipnet", @@ -3745,12 +3990,12 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-pemfile", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tokio-rustls", "tokio-util", @@ -3765,6 +4010,25 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "rig-core" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aaf53f8631db9cc12287b5595b14964ae36a213f0d247055efcd26714f2da2b" +dependencies = [ + "async-stream", + "bytes", + "futures", + "glob", + "ordered-float", + "reqwest 0.11.27", + "schemars", + "serde", + "serde_json", + "thiserror 1.0.69", + "tracing", +] + [[package]] name = "ring" version = "0.17.8" @@ -3861,6 +4125,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -3920,6 +4193,15 @@ dependencies = [ "libsamplerate-sys", ] +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "schemars" version = "0.8.21" @@ -3953,6 +4235,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.8.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "selectors" version = "0.22.0" @@ -4361,6 +4666,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4381,6 +4692,27 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "system-deps" version = "6.2.2" @@ -4465,7 +4797,7 @@ dependencies = [ "glob", "gtk", "heck 0.5.0", - "http", + "http 1.2.0", "jni", "libc", "log", @@ -4477,7 +4809,7 @@ dependencies = [ "percent-encoding", "plist", "raw-window-handle", - "reqwest", + "reqwest 0.12.12", "serde", "serde_json", "serde_repr", @@ -4681,7 +5013,7 @@ checksum = "2274ef891ccc0a8d318deffa9d70053f947664d12d58b9c0d1ae5e89237e01f7" dependencies = [ "dpi", "gtk", - "http", + "http 1.2.0", "jni", "raw-window-handle", "serde", @@ -4699,7 +5031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3707b40711d3b9f6519150869e358ffbde7c57567fb9b5a8b51150606939b2a0" dependencies = [ "gtk", - "http", + "http 1.2.0", "jni", "log", "objc2", @@ -4730,7 +5062,7 @@ dependencies = [ "dunce", "glob", "html5ever", - "http", + "http 1.2.0", "infer", "json-patch", "kuchikiki", @@ -4918,6 +5250,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.1" @@ -5020,7 +5362,7 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tower-layer", "tower-service", @@ -5240,6 +5582,12 @@ dependencies = [ "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version-compare" version = "0.2.0" @@ -6032,6 +6380,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "winreg" version = "0.52.0" @@ -6078,7 +6436,7 @@ dependencies = [ "gdkx11", "gtk", "html5ever", - "http", + "http 1.2.0", "javascriptcore-rs", "jni", "kuchikiki", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 51709b0..0307ce4 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -30,6 +30,7 @@ whisper-rs = { version = "0.13.2", features = ["whisper-cpp-log"] } anyhow = "1.0.95" rubato = "0.16.1" samplerate = "0.2.4" +rig-core = "0.8.0" [target.'cfg(target_os = "macos")'.dependencies] whisper-rs = { version = "0.13.2", features = ["metal"] } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index c891d25..ec22091 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -4,6 +4,7 @@ use managers::keybinding::KeyBindingManager; use managers::transcription::TranscriptionManager; use managers::{audio::AudioRecordingManager, transcription}; use rdev::Key; +use rig::{completion::Prompt, providers::anthropic}; use std::sync::{Arc, Mutex}; use tauri_plugin_autostart::MacosLauncher; @@ -14,34 +15,96 @@ pub fn run() { let transcription_manager = Arc::new(TranscriptionManager::new().expect("Failed to initialize transcription manager")); // let transcription_manager = Arc::new(TranscriptionManager::new()); + let claude_client = anthropic::Client::from_env(); + let sonnet: Arc> = Arc::new( + claude_client + .agent(anthropic::CLAUDE_3_5_SONNET) + .preamble("Be precise and concise.") + .temperature(0.5) + .build(), + ); + let manager = Arc::new(Mutex::new(KeyBindingManager::new( recording_manager.clone(), transcription_manager.clone(), + sonnet.clone(), ))); // Register your key bindings { let mut manager = manager.lock().unwrap(); - // Example: Register Ctrl+Shift combination + // Register Basic Transcription manager.register( "ctrl-meta".to_string(), vec![Key::ControlRight, Key::MetaRight], |ctx| { println!("Ctrl+Meta pressed!"); ctx.recording_manager.try_start_recording("ctrl-meta"); + None }, |ctx| { - println!("Ctrl+Meta released!"); - if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { - let samples: Vec = samples; // explicit type annotation - match ctx.transcription_manager.transcribe(samples) { - Ok(transcription) => println!("Transcription: {}", transcription), - Err(err) => println!("Transcription error: {}", err), + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { + match ctx.transcription_manager.transcribe(samples) { + Ok(transcription) => println!("Transcription: {}", transcription), + Err(err) => println!("Transcription error: {}", err), + } } - } else { - println!("No samples recorded"); - } + })) + }, + ); + + // Register LLM Call after Transcription + manager.register( + "shift-alt".to_string(), + vec![Key::ShiftLeft, Key::Alt], + |ctx| { + println!("Shift+Alt pressed!"); + ctx.recording_manager.try_start_recording("shift-alt"); + None + }, + |ctx| { + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { + if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) { + println!("Transcription: {}", transcription); + match ctx.sonnet.prompt(transcription).await { + Ok(response) => println!("Sonnet response: {}", response), + Err(err) => println!("Sonnet error: {}", err), + } + } + } + })) + }, + ); + + manager.register( + "ctrl-alt-meta".to_string(), + vec![Key::ControlLeft, Key::Alt, Key::MetaLeft], + |ctx| { + println!("Ctrl+Alt+Meta pressed!"); + ctx.recording_manager.try_start_recording("ctrl-alt-meta"); + None + }, + |ctx| { + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") { + let samples: Vec = samples; // explicit type annotation + match ctx.transcription_manager.transcribe(samples) { + Ok(transcription) => { + println!("Transcription: {}", transcription); + // Call LLM for code + } + Err(err) => println!("Transcription error: {}", err), + } + } else { + println!("No samples recorded"); + } + })) }, ); } diff --git a/src-tauri/src/managers/keybinding.rs b/src-tauri/src/managers/keybinding.rs index ea11eca..be7ba7d 100644 --- a/src-tauri/src/managers/keybinding.rs +++ b/src-tauri/src/managers/keybinding.rs @@ -1,35 +1,37 @@ -// First, let's define some types for our audio recording functionality use super::audio::AudioRecordingManager; use super::transcription::TranscriptionManager; use rdev::EventType; +use rig::agent::Agent; +use rig::providers::anthropic; use std::collections::HashSet; use std::sync::{Arc, Mutex}; type KeySet = HashSet; -// Now let's modify the KeyBinding structure to work with our new system #[derive(Clone)] pub struct BindingContext { pub recording_manager: Arc, pub transcription_manager: Arc, - // Add other managers here as needed - // pub transcription_manager: Arc, - // pub llm_manager: Arc, + pub sonnet: Arc>, } pub struct KeyBinding { id: String, keys: KeySet, - on_press: Box, - on_release: Box, + on_press: Box< + dyn Fn(&BindingContext) -> Option> + Send + 'static, + >, + on_release: Box< + dyn Fn(&BindingContext) -> Option> + Send + 'static, + >, currently_pressed: Arc>, } impl KeyBinding { fn new(id: String, keys: Vec, on_press: F, on_release: G) -> Self where - F: Fn(&BindingContext) + Send + 'static, - G: Fn(&BindingContext) + Send + 'static, + F: Fn(&BindingContext) -> Option> + Send + 'static, + G: Fn(&BindingContext) -> Option> + Send + 'static, { Self { id, @@ -41,21 +43,19 @@ impl KeyBinding { } fn handle_event(&self, key: rdev::Key, is_press: bool, context: &BindingContext) -> bool { - if !self.keys.contains(&key) { - return false; - } - let mut pressed = self.currently_pressed.lock().unwrap(); if is_press { + if pressed.contains(&key) { + return false; + } pressed.insert(key); if pressed.len() == self.keys.len() && pressed.is_subset(&self.keys) { - (self.on_press)(context); + let _ = (self.on_press)(context); return true; } } else { - pressed.remove(&key); - if pressed.len() == self.keys.len() - 1 { - (self.on_release)(context); + if pressed.remove(&key) && pressed.len() == self.keys.len() - 1 { + let _ = (self.on_release)(context); return true; } } @@ -72,20 +72,22 @@ impl KeyBindingManager { pub fn new( recording_manager: Arc, transcription_manager: Arc, + sonnet: Arc>, ) -> Self { Self { bindings: Vec::new(), context: BindingContext { recording_manager, transcription_manager, + sonnet, }, } } pub fn register(&mut self, id: String, keys: Vec, on_press: F, on_release: G) where - F: Fn(&BindingContext) + Send + 'static, - G: Fn(&BindingContext) + Send + 'static, + F: Fn(&BindingContext) -> Option> + Send + 'static, + G: Fn(&BindingContext) -> Option> + Send + 'static, { self.bindings .push(KeyBinding::new(id, keys, on_press, on_release)); From 3d774b8367b8a9f641340dc6215bb1c22a8131d9 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Sun, 16 Feb 2025 19:37:32 -0800 Subject: [PATCH 2/4] Fix some weird keybinding issues --- src-tauri/Cargo.lock | 99 ++++++++++++++++++++++++++++ src-tauri/Cargo.toml | 2 + src-tauri/src/lib.rs | 12 +++- src-tauri/src/managers/audio.rs | 3 +- src-tauri/src/managers/keybinding.rs | 31 +++++++-- 5 files changed, 139 insertions(+), 8 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 6e4a957..615b273 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -78,6 +78,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +dependencies = [ + "anstyle", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.95" @@ -715,6 +765,12 @@ dependencies = [ "objc", ] +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "combine" version = "4.6.7" @@ -1254,6 +1310,29 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "epoll" version = "4.3.3" @@ -1887,6 +1966,8 @@ version = "0.1.0" dependencies = [ "anyhow", "cpal", + "env_logger", + "log", "rdev", "rig-core", "rubato", @@ -2031,6 +2112,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.32" @@ -2391,6 +2478,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -5572,6 +5665,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.12.1" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 0307ce4..0e4720b 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -31,6 +31,8 @@ anyhow = "1.0.95" rubato = "0.16.1" samplerate = "0.2.4" rig-core = "0.8.0" +env_logger = "0.11.6" +log = "0.4.25" [target.'cfg(target_os = "macos")'.dependencies] whisper-rs = { version = "0.13.2", features = ["metal"] } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index ec22091..7b480a2 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,5 +1,6 @@ mod managers; +use log::{info, Level}; use managers::keybinding::KeyBindingManager; use managers::transcription::TranscriptionManager; use managers::{audio::AudioRecordingManager, transcription}; @@ -10,6 +11,8 @@ use tauri_plugin_autostart::MacosLauncher; #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { + env_logger::init(); + let recording_manager = Arc::new(AudioRecordingManager::new().expect("Failed to initialize recording manager")); let transcription_manager = @@ -39,11 +42,12 @@ pub fn run() { "ctrl-meta".to_string(), vec![Key::ControlRight, Key::MetaRight], |ctx| { - println!("Ctrl+Meta pressed!"); + info!("Ctrl+Meta pressed!"); ctx.recording_manager.try_start_recording("ctrl-meta"); None }, |ctx| { + info!("release being called from ctrl-meta"); let ctx = ctx.clone(); Some(tauri::async_runtime::spawn(async move { if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { @@ -61,11 +65,12 @@ pub fn run() { "shift-alt".to_string(), vec![Key::ShiftLeft, Key::Alt], |ctx| { - println!("Shift+Alt pressed!"); + info!("Shift+Alt pressed!"); ctx.recording_manager.try_start_recording("shift-alt"); None }, |ctx| { + info!("release being called from shift-alt"); let ctx = ctx.clone(); Some(tauri::async_runtime::spawn(async move { if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { @@ -85,11 +90,12 @@ pub fn run() { "ctrl-alt-meta".to_string(), vec![Key::ControlLeft, Key::Alt, Key::MetaLeft], |ctx| { - println!("Ctrl+Alt+Meta pressed!"); + info!("Ctrl+Alt+Meta pressed!"); ctx.recording_manager.try_start_recording("ctrl-alt-meta"); None }, |ctx| { + info!("release being called from ctrl-alt-meta"); let ctx = ctx.clone(); Some(tauri::async_runtime::spawn(async move { if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") { diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs index 90a109d..222a046 100644 --- a/src-tauri/src/managers/audio.rs +++ b/src-tauri/src/managers/audio.rs @@ -133,6 +133,7 @@ impl AudioRecordingManager { pub fn stop_recording(&self, binding_id: &str) -> Option> { let mut state = self.state.lock().unwrap(); + println!("Stop recording called from binding {}", binding_id); match *state { RecordingState::Recording { binding_id: ref active_id, @@ -144,7 +145,7 @@ impl AudioRecordingManager { Some(buffer.drain(..).collect()) } _ => { - println!("Cannot stop recording: not recording or wrong binding"); + // println!("Cannot stop recording: not recording or wrong binding"); None } } diff --git a/src-tauri/src/managers/keybinding.rs b/src-tauri/src/managers/keybinding.rs index be7ba7d..a2c4072 100644 --- a/src-tauri/src/managers/keybinding.rs +++ b/src-tauri/src/managers/keybinding.rs @@ -44,19 +44,38 @@ impl KeyBinding { fn handle_event(&self, key: rdev::Key, is_press: bool, context: &BindingContext) -> bool { let mut pressed = self.currently_pressed.lock().unwrap(); + if is_press { + if !self.keys.contains(&key) { + return false; // Ignore keys that aren't part of this binding + } + if pressed.contains(&key) { return false; } + pressed.insert(key); if pressed.len() == self.keys.len() && pressed.is_subset(&self.keys) { let _ = (self.on_press)(context); return true; } } else { - if pressed.remove(&key) && pressed.len() == self.keys.len() - 1 { - let _ = (self.on_release)(context); - return true; + // Release event + if !self.keys.contains(&key) { + return false; + } + + if pressed.remove(&key) { + if pressed.len() == self.keys.len() - 1 { + let _ = (self.on_release)(context); + + // Clear the currently_pressed set if all keys are released + if pressed.is_empty() { + pressed.clear(); + } + + return true; + } } } false @@ -96,8 +115,12 @@ impl KeyBindingManager { pub fn handle_event(&self, event: &rdev::Event) { if let EventType::KeyPress(key) | EventType::KeyRelease(key) = event.event_type { let is_press = matches!(event.event_type, EventType::KeyPress(_)); + + // Only process the first binding that successfully handles the event for binding in &self.bindings { - binding.handle_event(key, is_press, &self.context); + if binding.handle_event(key, is_press, &self.context) { + break; // Exit after first successful handling + } } } } From 6b088deaf551c6b38b267d840814f436148a18b1 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 17 Feb 2025 09:08:48 -0800 Subject: [PATCH 3/4] Very basic version of the clipboard implemented. --- src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 1 + src-tauri/src/lib.rs | 284 +++++++++++++++++---------- src-tauri/src/managers/keybinding.rs | 3 + 4 files changed, 188 insertions(+), 101 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 615b273..0cca731 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -1982,6 +1982,7 @@ dependencies = [ "tauri-plugin-macos-permissions", "tauri-plugin-opener", "tauri-plugin-single-instance", + "tokio", "whisper-rs", ] diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 0e4720b..3e4ad4f 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -33,6 +33,7 @@ samplerate = "0.2.4" rig-core = "0.8.0" env_logger = "0.11.6" log = "0.4.25" +tokio = "1.43.0" [target.'cfg(target_os = "macos")'.dependencies] whisper-rs = { version = "0.13.2", features = ["metal"] } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 7b480a2..213b898 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -4,10 +4,166 @@ use log::{info, Level}; use managers::keybinding::KeyBindingManager; use managers::transcription::TranscriptionManager; use managers::{audio::AudioRecordingManager, transcription}; -use rdev::Key; +use rdev::{simulate, EventType, Key, SimulateError}; use rig::{completion::Prompt, providers::anthropic}; use std::sync::{Arc, Mutex}; +use std::{thread, time}; use tauri_plugin_autostart::MacosLauncher; +use tauri_plugin_clipboard_manager::ClipboardExt; + +fn try_send_event(event: &EventType) { + if let Err(SimulateError) = simulate(event) { + println!("We could not send {:?}", event); + } +} + +fn send(event: EventType) { + try_send_event(&event); + thread::sleep(time::Duration::from_millis(20)); +} + +fn send_multiple(events: &[EventType]) { + events.iter().for_each(try_send_event); + thread::sleep(time::Duration::from_millis(20)); +} + +fn send_paste() { + // Determine the modifier key based on the OS + #[cfg(target_os = "macos")] + let modifier_key = Key::MetaLeft; // Command key on macOS + #[cfg(not(target_os = "macos"))] + let modifier_key = Key::ControlLeft; // Control key on other systems + + // Press both keys + send(EventType::KeyPress(modifier_key)); + send(EventType::KeyPress(Key::KeyV)); + + // Release both keys simultaneously + send_multiple(&[ + EventType::KeyRelease(Key::KeyV), + EventType::KeyRelease(modifier_key), + ]); + + // Additional delay after the complete combination + // thread::sleep(time::Duration::from_millis(50)); +} + +fn paste(text: String, app_handle: tauri::AppHandle) { + let clipboard = app_handle.clipboard(); + + // get the current clipboard content + let clipboard_content = clipboard.read_text().unwrap_or_default(); + + clipboard.write_text(&text).unwrap(); + send_paste(); + + // restore the clipboard + clipboard.write_text(&clipboard_content).unwrap(); +} + +fn get_highlighted_text(app_handle: tauri::AppHandle) -> String { + let clipboard = app_handle.clipboard(); + + // save the clipboard content + let clipboard_content = clipboard.read_text().unwrap_or_default(); + + // empty the clipboard + clipboard.write_text("").unwrap(); + + // get the highlighted text + let highlighted_text = clipboard.read_text().unwrap_or_default(); + + // restore the clipboard content + clipboard.write_text(&clipboard_content).unwrap(); + + highlighted_text +} + +fn register_bindings(manager: &mut KeyBindingManager) { + manager.register( + "ctrl-meta".to_string(), + vec![Key::ControlRight, Key::MetaRight], + |ctx| { + info!("Ctrl+Meta pressed!"); + ctx.recording_manager.try_start_recording("ctrl-meta"); + None + }, + |ctx| { + info!("release being called from ctrl-meta"); + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { + match ctx.transcription_manager.transcribe(samples) { + Ok(transcription) => { + println!("Transcription: {}", transcription); + paste(transcription, ctx.app_handle.clone()); + } + Err(err) => println!("Transcription error: {}", err), + } + } + })) + }, + ); + + // Register LLM Call after Transcription + manager.register( + "shift-alt".to_string(), + vec![Key::ShiftLeft, Key::Alt], + |ctx| { + info!("Shift+Alt pressed!"); + ctx.recording_manager.try_start_recording("shift-alt"); + None + }, + |ctx| { + info!("release being called from shift-alt"); + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { + if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) { + println!("Transcription: {}", transcription); + match ctx.sonnet.prompt(transcription).await { + Ok(response) => { + let highlighted_text = get_highlighted_text(ctx.app_handle.clone()); + println!("Highlighted Text: {}", highlighted_text); + println!("Sonnet response: {}", response); + paste(response, ctx.app_handle.clone()); + } + Err(err) => println!("Sonnet error: {}", err), + } + } + } + })) + }, + ); + + manager.register( + "ctrl-alt-meta".to_string(), + vec![Key::ControlLeft, Key::Alt, Key::MetaLeft], + |ctx| { + info!("Ctrl+Alt+Meta pressed!"); + ctx.recording_manager.try_start_recording("ctrl-alt-meta"); + None + }, + |ctx| { + info!("release being called from ctrl-alt-meta"); + let ctx = ctx.clone(); + Some(tauri::async_runtime::spawn(async move { + if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") { + let samples: Vec = samples; // explicit type annotation + match ctx.transcription_manager.transcribe(samples) { + Ok(transcription) => { + println!("Transcription: {}", transcription); + // Call LLM for code + } + Err(err) => println!("Transcription error: {}", err), + } + } else { + println!("No samples recorded"); + } + })) + }, + ); +} #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { @@ -27,106 +183,6 @@ pub fn run() { .build(), ); - let manager = Arc::new(Mutex::new(KeyBindingManager::new( - recording_manager.clone(), - transcription_manager.clone(), - sonnet.clone(), - ))); - - // Register your key bindings - { - let mut manager = manager.lock().unwrap(); - - // Register Basic Transcription - manager.register( - "ctrl-meta".to_string(), - vec![Key::ControlRight, Key::MetaRight], - |ctx| { - info!("Ctrl+Meta pressed!"); - ctx.recording_manager.try_start_recording("ctrl-meta"); - None - }, - |ctx| { - info!("release being called from ctrl-meta"); - let ctx = ctx.clone(); - Some(tauri::async_runtime::spawn(async move { - if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-meta") { - match ctx.transcription_manager.transcribe(samples) { - Ok(transcription) => println!("Transcription: {}", transcription), - Err(err) => println!("Transcription error: {}", err), - } - } - })) - }, - ); - - // Register LLM Call after Transcription - manager.register( - "shift-alt".to_string(), - vec![Key::ShiftLeft, Key::Alt], - |ctx| { - info!("Shift+Alt pressed!"); - ctx.recording_manager.try_start_recording("shift-alt"); - None - }, - |ctx| { - info!("release being called from shift-alt"); - let ctx = ctx.clone(); - Some(tauri::async_runtime::spawn(async move { - if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { - if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) { - println!("Transcription: {}", transcription); - match ctx.sonnet.prompt(transcription).await { - Ok(response) => println!("Sonnet response: {}", response), - Err(err) => println!("Sonnet error: {}", err), - } - } - } - })) - }, - ); - - manager.register( - "ctrl-alt-meta".to_string(), - vec![Key::ControlLeft, Key::Alt, Key::MetaLeft], - |ctx| { - info!("Ctrl+Alt+Meta pressed!"); - ctx.recording_manager.try_start_recording("ctrl-alt-meta"); - None - }, - |ctx| { - info!("release being called from ctrl-alt-meta"); - let ctx = ctx.clone(); - Some(tauri::async_runtime::spawn(async move { - if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") { - let samples: Vec = samples; // explicit type annotation - match ctx.transcription_manager.transcribe(samples) { - Ok(transcription) => { - println!("Transcription: {}", transcription); - // Call LLM for code - } - Err(err) => println!("Transcription error: {}", err), - } - } else { - println!("No samples recorded"); - } - })) - }, - ); - } - - tauri::async_runtime::spawn({ - let manager = manager.clone(); - async move { - rdev::listen(move |event| { - if let Ok(manager) = manager.lock() { - manager.handle_event(&event); - } - }) - .unwrap(); - } - }); - tauri::Builder::default() .plugin(tauri_plugin_global_shortcut::Builder::new().build()) .plugin(tauri_plugin_clipboard_manager::init()) @@ -136,6 +192,32 @@ pub fn run() { )) .plugin(tauri_plugin_macos_permissions::init()) .plugin(tauri_plugin_opener::init()) + .setup(move |app| { + let app_handle = app.handle().clone(); + + let manager = Arc::new(Mutex::new(KeyBindingManager::new( + recording_manager.clone(), + transcription_manager.clone(), + sonnet.clone(), + app_handle.clone(), + ))); + + { + let mut manager = manager.lock().unwrap(); + register_bindings(&mut manager); + } + + let manager_clone = manager.clone(); + tauri::async_runtime::spawn(async move { + rdev::listen(move |event| { + if let Ok(manager) = manager_clone.lock() { + manager.handle_event(&event); + } + }) + .unwrap(); + }); + Ok(()) + }) .invoke_handler(tauri::generate_handler![]) .run(tauri::generate_context!()) .expect("error while running tauri application"); diff --git a/src-tauri/src/managers/keybinding.rs b/src-tauri/src/managers/keybinding.rs index a2c4072..b424a6f 100644 --- a/src-tauri/src/managers/keybinding.rs +++ b/src-tauri/src/managers/keybinding.rs @@ -13,6 +13,7 @@ pub struct BindingContext { pub recording_manager: Arc, pub transcription_manager: Arc, pub sonnet: Arc>, + pub app_handle: tauri::AppHandle, } pub struct KeyBinding { @@ -92,6 +93,7 @@ impl KeyBindingManager { recording_manager: Arc, transcription_manager: Arc, sonnet: Arc>, + app_handle: tauri::AppHandle, ) -> Self { Self { bindings: Vec::new(), @@ -99,6 +101,7 @@ impl KeyBindingManager { recording_manager, transcription_manager, sonnet, + app_handle, }, } } From 14b2789e4bc93bd094337a2b3cbd99d67602b774 Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Mon, 17 Feb 2025 10:06:01 -0800 Subject: [PATCH 4/4] Most basic working LLM end-to-end feature parity with handy. --- src-tauri/src/lib.rs | 93 +++++++++++++++++++++++----- src-tauri/src/managers/audio.rs | 21 ++++++- src-tauri/src/managers/keybinding.rs | 6 +- 3 files changed, 102 insertions(+), 18 deletions(-) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 213b898..c9089dd 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -5,6 +5,7 @@ use managers::keybinding::KeyBindingManager; use managers::transcription::TranscriptionManager; use managers::{audio::AudioRecordingManager, transcription}; use rdev::{simulate, EventType, Key, SimulateError}; +use rig::streaming::StreamingChoice; use rig::{completion::Prompt, providers::anthropic}; use std::sync::{Arc, Mutex}; use std::{thread, time}; @@ -45,7 +46,25 @@ fn send_paste() { ]); // Additional delay after the complete combination - // thread::sleep(time::Duration::from_millis(50)); + thread::sleep(time::Duration::from_millis(20)); +} + +fn send_copy() { + // Determine the modifier key based on the OS + #[cfg(target_os = "macos")] + let modifier_key = Key::MetaLeft; // Command key on macOS + #[cfg(not(target_os = "macos"))] + let modifier_key = Key::ControlLeft; // Control key on other systems + + // Press both keys + send(EventType::KeyPress(modifier_key)); + send(EventType::KeyPress(Key::KeyC)); + + // Release both keys simultaneously + send_multiple(&[ + EventType::KeyRelease(Key::KeyC), + EventType::KeyRelease(modifier_key), + ]); } fn paste(text: String, app_handle: tauri::AppHandle) { @@ -70,6 +89,9 @@ fn get_highlighted_text(app_handle: tauri::AppHandle) -> String { // empty the clipboard clipboard.write_text("").unwrap(); + // issue 'copy' + send_copy(); + // get the highlighted text let highlighted_text = clipboard.read_text().unwrap_or_default(); @@ -79,6 +101,28 @@ fn get_highlighted_text(app_handle: tauri::AppHandle) -> String { highlighted_text } +const INSTRUCT_SYS: &str = r#" +You are a helpful assistant. You will receive voice transcriptions +from a user that may include both a command/question and some +minimal context to help you respond appropriately. + +For example, the user might say: +- A direct question with no context: "What is the capital of France?" +- A command with context: "get commit message I fixed the bug in the login system" +"#; + +const CODE_SYS: &str = r#" +You are a code-only assistant. I will provide you with selected text or clipboard content along with instructions. If I request code, output only the exact code implementation. If I request a terminal command, provide only the valid command syntax. Never use markdown, explanations, or additional text. + + When I share selected text or clipboard content, use that as context for generating your response. The output should be ready to copy and paste directly, with no formatting or commentary. For terminal commands, ensure they are valid for the specified environment. Note for terminal commands, I typically use lowercase instead of uppercase. You may also be given them directly, but need to translate them into a way that can actually be executed in the terminal because the transcription you are given might be poor. + + Output only: + - Raw code implementation when code is requested + - Terminal command syntax when a command is requested + - No markdown, no backticks, no explanations + - No additional text or descriptions +"#; + fn register_bindings(manager: &mut KeyBindingManager) { manager.register( "ctrl-meta".to_string(), @@ -121,10 +165,20 @@ fn register_bindings(manager: &mut KeyBindingManager) { if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") { if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) { println!("Transcription: {}", transcription); - match ctx.sonnet.prompt(transcription).await { + + let instruct = ctx + .anthropic + .agent(anthropic::CLAUDE_3_5_SONNET) + .preamble(INSTRUCT_SYS) + .temperature(0.5) + .build(); + + let highlighted_text = get_highlighted_text(ctx.app_handle.clone()); + println!("Highlighted Text: {}", highlighted_text); + let prompt = format!("{}\n\ncontext:{}\n", transcription, highlighted_text); + + match instruct.prompt(prompt).await { Ok(response) => { - let highlighted_text = get_highlighted_text(ctx.app_handle.clone()); - println!("Highlighted Text: {}", highlighted_text); println!("Sonnet response: {}", response); paste(response, ctx.app_handle.clone()); } @@ -153,7 +207,25 @@ fn register_bindings(manager: &mut KeyBindingManager) { match ctx.transcription_manager.transcribe(samples) { Ok(transcription) => { println!("Transcription: {}", transcription); - // Call LLM for code + + let code = ctx + .anthropic + .agent(anthropic::CLAUDE_3_5_SONNET) + .preamble(CODE_SYS) + .temperature(0.5) + .build(); + + let highlighted_text = get_highlighted_text(ctx.app_handle.clone()); + let prompt = + format!("{}\n\ncontext:{}\n", transcription, highlighted_text); + + match code.prompt(prompt).await { + Ok(response) => { + println!("Sonnet response: {}", response); + paste(response, ctx.app_handle.clone()); + } + Err(err) => println!("Sonnet error: {}", err), + } } Err(err) => println!("Transcription error: {}", err), } @@ -174,14 +246,7 @@ pub fn run() { let transcription_manager = Arc::new(TranscriptionManager::new().expect("Failed to initialize transcription manager")); // let transcription_manager = Arc::new(TranscriptionManager::new()); - let claude_client = anthropic::Client::from_env(); - let sonnet: Arc> = Arc::new( - claude_client - .agent(anthropic::CLAUDE_3_5_SONNET) - .preamble("Be precise and concise.") - .temperature(0.5) - .build(), - ); + let claude_client = Arc::new(anthropic::Client::from_env()); tauri::Builder::default() .plugin(tauri_plugin_global_shortcut::Builder::new().build()) @@ -198,7 +263,7 @@ pub fn run() { let manager = Arc::new(Mutex::new(KeyBindingManager::new( recording_manager.clone(), transcription_manager.clone(), - sonnet.clone(), + claude_client.clone(), app_handle.clone(), ))); diff --git a/src-tauri/src/managers/audio.rs b/src-tauri/src/managers/audio.rs index 222a046..75bfcf3 100644 --- a/src-tauri/src/managers/audio.rs +++ b/src-tauri/src/managers/audio.rs @@ -142,7 +142,26 @@ impl AudioRecordingManager { println!("Stopped recording for binding {}", binding_id); let mut buffer = self.buffer.lock().unwrap(); - Some(buffer.drain(..).collect()) + let audio_data: Vec = buffer.drain(..).collect(); + + // Calculate duration in milliseconds + // 16000 is our target sample rate after resampling + let duration_ms = (audio_data.len() as f32 / 16000.0) * 1000.0; + + if duration_ms < 300.0 { + // Discard the audio if it's too short + Some(Vec::new()) + } else { + // Pad to minimum 1000ms if needed + if duration_ms < 1000.0 { + let target_samples = (16000.0 * (1000.0 / 1000.0)) as usize; // 16000 samples for 1 second + let mut padded_audio = audio_data; + padded_audio.resize(target_samples, 0.0); // Pad with silence (zeros) + Some(padded_audio) + } else { + Some(audio_data) + } + } } _ => { // println!("Cannot stop recording: not recording or wrong binding"); diff --git a/src-tauri/src/managers/keybinding.rs b/src-tauri/src/managers/keybinding.rs index b424a6f..b61b667 100644 --- a/src-tauri/src/managers/keybinding.rs +++ b/src-tauri/src/managers/keybinding.rs @@ -12,7 +12,7 @@ type KeySet = HashSet; pub struct BindingContext { pub recording_manager: Arc, pub transcription_manager: Arc, - pub sonnet: Arc>, + pub anthropic: Arc, pub app_handle: tauri::AppHandle, } @@ -92,7 +92,7 @@ impl KeyBindingManager { pub fn new( recording_manager: Arc, transcription_manager: Arc, - sonnet: Arc>, + anthropic: Arc, app_handle: tauri::AppHandle, ) -> Self { Self { @@ -100,7 +100,7 @@ impl KeyBindingManager { context: BindingContext { recording_manager, transcription_manager, - sonnet, + anthropic, app_handle, }, }