danvics/Handy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

working? ability to package/run on macos

Browse source
This commit is contained in:
CJ Pais 2025-05-05 12:13:27 -07:00
parent 8cd40c391d
commit 6b545d3acd
18 changed files with 1142 additions and 146 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

BIN
bun.lockb
View file

Binary file not shown.

2
package.json
View file

@ -16,6 +16,8 @@
"@tauri-apps/plugin-clipboard-manager": "~2",
"@tauri-apps/plugin-global-shortcut": "~2",
"@tauri-apps/plugin-opener": "^2",
"@tauri-apps/plugin-stronghold": "~2",
"@tauri-apps/plugin-upload": "~2",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"tailwindcss": "^4.0.2",

967
src-tauri/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

2
src-tauri/Cargo.toml
View file

@ -35,6 +35,8 @@ env_logger = "0.11.6"
log = "0.4.25"
tokio = "1.43.0"
vad-rs = "0.1.5"
tauri-plugin-upload = "2"
tauri-plugin-stronghold = "2"
[target.'cfg(target_os = "macos")'.dependencies]
whisper-rs = { version = "0.13.2", features = ["metal"] }

10
src-tauri/Entitlements.plist Normal file
View file

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.device.microphone</key>
<true/>
<key>com.apple.security.device.audio-input</key>
<true/>
</dict>
</plist>

8
src-tauri/Info.plist Normal file
View file

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSMicrophoneUsageDescription</key>
<string>Request microphone access to transcribe audio locally</string>
</dict>
</plist>

17
src-tauri/capabilities/desktop.json
View file

@ -1,15 +1,6 @@
{
"identifier": "desktop-capability",
"platforms": [
"macOS",
"windows",
"linux"
],
"windows": [
"main"
],
"permissions": [
"autostart:default",
"global-shortcut:default"
]
}
"platforms": ["macOS", "windows", "linux"],
"windows": ["main"],
"permissions": ["autostart:default", "global-shortcut:default"]
}

BIN
src-tauri/icons/128x128.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 3.4 KiB

After

Width:  |  Height:  |  Size: 6.3 KiB

BIN
src-tauri/icons/32x32.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 974 B

After

Width:  |  Height:  |  Size: 1.5 KiB

BIN
src-tauri/icons/icon.icns
View file

Binary file not shown.

BIN
src-tauri/icons/icon.png
View file

Binary file not shown.
Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 28 KiB

193
src-tauri/src/lib.rs
View file

@ -8,6 +8,7 @@ use rdev::{simulate, EventType, Key, SimulateError};
use rig::{completion::Prompt, providers::anthropic};
use std::sync::{Arc, Mutex};
use std::{thread, time};
use tauri::Manager;
use tauri_plugin_autostart::MacosLauncher;
use tauri_plugin_clipboard_manager::ClipboardExt;
@ -19,7 +20,7 @@ fn try_send_event(event: &EventType) {
fn send(event: EventType) {
try_send_event(&event);
thread::sleep(time::Duration::from_millis(40));
thread::sleep(time::Duration::from_millis(60));
}
fn send_paste() {
@ -137,105 +138,100 @@ fn register_bindings(manager: &mut KeyBindingManager) {
);
// Register LLM Call after Transcription
manager.register(
"shift-alt".to_string(),
vec![Key::ShiftLeft, Key::Alt],
|ctx| {
info!("Shift+Alt pressed!");
ctx.recording_manager.try_start_recording("shift-alt");
None
},
|ctx| {
info!("release being called from shift-alt");
let ctx = ctx.clone();
Some(tauri::async_runtime::spawn(async move {
if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") {
if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) {
println!("Transcription: {}", transcription);
// manager.register(
// "shift-alt".to_string(),
// vec![Key::ShiftLeft, Key::Alt],
// |ctx| {
// info!("Shift+Alt pressed!");
// ctx.recording_manager.try_start_recording("shift-alt");
// None
// },
// |ctx| {
// info!("release being called from shift-alt");
// let ctx = ctx.clone();
// Some(tauri::async_runtime::spawn(async move {
// if let Some(samples) = ctx.recording_manager.stop_recording("shift-alt") {
// if let Ok(transcription) = ctx.transcription_manager.transcribe(samples) {
// println!("Transcription: {}", transcription);
let instruct = ctx
.anthropic
.agent(anthropic::CLAUDE_3_5_SONNET)
.preamble(INSTRUCT_SYS)
.temperature(0.5)
.build();
// let instruct = ctx
// .anthropic
// .agent(anthropic::CLAUDE_3_5_SONNET)
// .preamble(INSTRUCT_SYS)
// .temperature(0.5)
// .build();
let highlighted_text = get_highlighted_text(ctx.app_handle.clone());
println!("Highlighted Text: {}", highlighted_text);
let prompt = format!("{}\n\ncontext:{}\n", transcription, highlighted_text);
// let highlighted_text = get_highlighted_text(ctx.app_handle.clone());
// println!("Highlighted Text: {}", highlighted_text);
// let prompt = format!("{}\n\ncontext:{}\n", transcription, highlighted_text);
match instruct.prompt(prompt).await {
Ok(response) => {
println!("Sonnet response: {}", response);
paste(response, ctx.app_handle.clone());
}
Err(err) => println!("Sonnet error: {}", err),
}
}
}
}))
},
);
// match instruct.prompt(prompt).await {
// Ok(response) => {
// println!("Sonnet response: {}", response);
// paste(response, ctx.app_handle.clone());
// }
// Err(err) => println!("Sonnet error: {}", err),
// }
// }
// }
// }))
// },
// );
manager.register(
"ctrl-alt-meta".to_string(),
vec![Key::ControlLeft, Key::Alt, Key::MetaLeft],
|ctx| {
info!("Ctrl+Alt+Meta pressed!");
ctx.recording_manager.try_start_recording("ctrl-alt-meta");
None
},
|ctx| {
info!("release being called from ctrl-alt-meta");
let ctx = ctx.clone();
Some(tauri::async_runtime::spawn(async move {
if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") {
let samples: Vec<f32> = samples; // explicit type annotation
match ctx.transcription_manager.transcribe(samples) {
Ok(transcription) => {
println!("Transcription: {}", transcription);
// manager.register(
// "ctrl-alt-meta".to_string(),
// vec![Key::ControlLeft, Key::Alt, Key::MetaLeft],
// |ctx| {
// info!("Ctrl+Alt+Meta pressed!");
// ctx.recording_manager.try_start_recording("ctrl-alt-meta");
// None
// },
// |ctx| {
// info!("release being called from ctrl-alt-meta");
// let ctx = ctx.clone();
// Some(tauri::async_runtime::spawn(async move {
// if let Some(samples) = ctx.recording_manager.stop_recording("ctrl-alt-meta") {
// let samples: Vec<f32> = samples; // explicit type annotation
// match ctx.transcription_manager.transcribe(samples) {
// Ok(transcription) => {
// println!("Transcription: {}", transcription);
let code = ctx
.anthropic
.agent(anthropic::CLAUDE_3_5_SONNET)
.preamble(CODE_SYS)
.temperature(0.5)
.build();
// let code = ctx
// .anthropic
// .agent(anthropic::CLAUDE_3_5_SONNET)
// .preamble(CODE_SYS)
// .temperature(0.5)
// .build();
let highlighted_text = get_highlighted_text(ctx.app_handle.clone());
let prompt =
format!("{}\n\ncontext:{}\n", transcription, highlighted_text);
// let highlighted_text = get_highlighted_text(ctx.app_handle.clone());
// let prompt =
// format!("{}\n\ncontext:{}\n", transcription, highlighted_text);
match code.prompt(prompt).await {
Ok(response) => {
println!("Sonnet response: {}", response);
paste(response, ctx.app_handle.clone());
}
Err(err) => println!("Sonnet error: {}", err),
}
}
Err(err) => println!("Transcription error: {}", err),
}
} else {
println!("No samples recorded");
}
}))
},
);
// match code.prompt(prompt).await {
// Ok(response) => {
// println!("Sonnet response: {}", response);
// paste(response, ctx.app_handle.clone());
// }
// Err(err) => println!("Sonnet error: {}", err),
// }
// }
// Err(err) => println!("Transcription error: {}", err),
// }
// } else {
// println!("No samples recorded");
// }
// }))
// },
// );
}
#[cfg_attr(mobile, tauri::mobile_entry_point)]
pub fn run() {
env_logger::init();
let recording_manager =
Arc::new(AudioRecordingManager::new().expect("Failed to initialize recording manager"));
let transcription_manager =
Arc::new(TranscriptionManager::new().expect("Failed to initialize transcription manager"));
// let transcription_manager = Arc::new(TranscriptionManager::new());
let claude_client = Arc::new(anthropic::Client::from_env());
tauri::Builder::default()
.plugin(tauri_plugin_stronghold::Builder::new(|pass| todo!()).build())
.plugin(tauri_plugin_upload::init())
.plugin(tauri_plugin_global_shortcut::Builder::new().build())
.plugin(tauri_plugin_clipboard_manager::init())
.plugin(tauri_plugin_autostart::init(
@ -247,10 +243,33 @@ pub fn run() {
.setup(move |app| {
let app_handle = app.handle().clone();
let vad_path = app.path().resolve(
"resources/silero_vad_v4.onnx",
tauri::path::BaseDirectory::Resource,
)?;
let whisper_path = app.path().resolve(
"resources/ggml-small.bin",
tauri::path::BaseDirectory::Resource,
)?;
let recording_manager = Arc::new(
AudioRecordingManager::new(&vad_path)
.expect("Failed to initialize recording manager"),
);
let transcription_manager = Arc::new(
TranscriptionManager::new(
whisper_path
.to_str()
.expect("Path contains invalid UTF-8 Chars"),
)
.expect("Failed to initialize transcription manager"),
);
let manager = Arc::new(Mutex::new(KeyBindingManager::new(
recording_manager.clone(),
transcription_manager.clone(),
claude_client.clone(),
// claude_client.clone(),
app_handle.clone(),
)));

15
src-tauri/src/managers/audio.rs
View file

@ -1,5 +1,6 @@
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use rubato::{FftFixedIn, Resampler};
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::vec::Vec;
use vad_rs::{Vad, VadStatus};
@ -16,7 +17,7 @@ pub struct AudioRecordingManager {
}
impl AudioRecordingManager {
pub fn new() -> Result<Self, anyhow::Error> {
pub fn new(vad_path: &PathBuf) -> Result<Self, anyhow::Error> {
let host = cpal::default_host();
let device = host
.default_input_device()
@ -28,9 +29,7 @@ impl AudioRecordingManager {
// Configure the resampler - keeping 1024 as input size for FFT efficiency
let resampler = FftFixedIn::new(sample_rate as usize, 16000, 1024, 2, 1)?;
let vad = Arc::new(Mutex::new(
Vad::new("resources/silero_vad_v4.onnx", 16000).unwrap(),
));
let vad = Arc::new(Mutex::new(Vad::new(vad_path, 16000).unwrap()));
let vad_clone = Arc::clone(&vad);
let state = Arc::new(Mutex::new(RecordingState::Idle));
@ -82,15 +81,13 @@ impl AudioRecordingManager {
if let Ok(mut vad) = vad_clone.lock() {
// println!("VAD lock acquired");
match vad.compute(&chunk) {
Ok(mut result) => match result.status() {
VadStatus::Speech => {
Ok(mut result) => {
if result.prob > 0.15 {
let mut buffer =
buffer_clone.lock().unwrap();
buffer.extend_from_slice(&chunk);
}
VadStatus::Silence => {}
_ => {}
},
}
Err(error) => {
eprintln!("Error computing VAD: {:?}", error)
}

6
src-tauri/src/managers/keybinding.rs
View file

@ -11,7 +11,7 @@ type KeySet = HashSet<Key>;
pub struct BindingContext {
pub recording_manager: Arc<AudioRecordingManager>,
pub transcription_manager: Arc<TranscriptionManager>,
pub anthropic: Arc<anthropic::Client>,
// pub anthropic: Arc<anthropic::Client>,
pub app_handle: tauri::AppHandle,
}
@ -52,7 +52,7 @@ impl KeyBindingManager {
pub fn new(
recording_manager: Arc<AudioRecordingManager>,
transcription_manager: Arc<TranscriptionManager>,
anthropic: Arc<anthropic::Client>,
// anthropic: Arc<anthropic::Client>,
app_handle: tauri::AppHandle,
) -> Self {
Self {
@ -62,7 +62,7 @@ impl KeyBindingManager {
context: BindingContext {
recording_manager,
transcription_manager,
anthropic,
// anthropic,
app_handle,
},
}

10
src-tauri/src/managers/transcription.rs
View file

@ -11,14 +11,12 @@ pub struct TranscriptionManager {
}
impl TranscriptionManager {
pub fn new() -> Result<Self> {
pub fn new(whisper_path: &str) -> Result<Self> {
install_whisper_log_trampoline();
// Load the model
let context = WhisperContext::new_with_params(
"resources/ggml-small.bin",
WhisperContextParameters::default(),
)
.map_err(|e| anyhow::anyhow!("Failed to load whisper model: {}", e))?;
let context =
WhisperContext::new_with_params(whisper_path, WhisperContextParameters::default())
.map_err(|e| anyhow::anyhow!("Failed to load whisper model: {}", e))?;
// Create state
let state = context.create_state().expect("failed to create state");

14
src-tauri/tauri.conf.json
View file

@ -1,6 +1,6 @@
{
"$schema": "https://schema.tauri.app/config/2",
"productName": "handy-app",
"productName": "Handy",
"version": "0.1.0",
"identifier": "com.handy-app.app",
"build": {
@ -12,7 +12,7 @@
"app": {
"windows": [
{
"title": "handy-app",
"title": "Handy",
"width": 800,
"height": 600
}
@ -24,12 +24,20 @@
"bundle": {
"active": true,
"targets": "all",
"resources": ["resources/*"],
"icon": [
"icons/32x32.png",
"icons/128x128.png",
"icons/128x128@2x.png",
"icons/icon.icns",
"icons/icon.ico"
]
],
"macOS": {
"files": {},
"hardenedRuntime": true,
"minimumSystemVersion": "10.13",
"signingIdentity": "-",
"entitlements": "Entitlements.plist"
}
}
}

2
src/App.tsx
View file

@ -39,4 +39,4 @@ function App() {
);
}
export default App;
export default App;

42
src/components/settings/KeyboardShortcuts.tsx
View file

@ -1,4 +1,4 @@
import React from 'react';
import React from "react";
interface Shortcut {
id: string;
@ -11,23 +11,23 @@ export const KeyboardShortcuts: React.FC = () => {
// These would normally come from the backend configuration
const shortcuts: Shortcut[] = [
{
id: 'transcribe',
name: 'Transcribe',
description: 'Convert speech to text',
keys: ['⌃', '⌘'],
},
{
id: 'instruct',
name: 'AI Chat',
description: 'Start a conversation with AI',
keys: ['⇧', '⌥'],
},
{
id: 'code',
name: 'Generate Code',
description: 'Generate code from voice input',
keys: ['⌃', '⌥', '⌘'],
id: "transcribe",
name: "Transcribe",
description: "Convert speech to text",
keys: ["⌃", "⌘"],
},
// {
// id: 'instruct',
// name: 'AI Chat',
// description: 'Start a conversation with AI',
// keys: ['⇧', '⌥'],
// },
// {
// id: 'code',
// name: 'Generate Code',
// description: 'Generate code from voice input',
// keys: ['⌃', '⌥', '⌘'],
// },
];
return (
@ -38,7 +38,9 @@ export const KeyboardShortcuts: React.FC = () => {
className="flex items-center justify-between p-4 rounded-lg border border-gray-200 hover:bg-gray-50"
>
<div>
<h3 className="text-sm font-medium text-gray-900">{shortcut.name}</h3>
<h3 className="text-sm font-medium text-gray-900">
{shortcut.name}
</h3>
<p className="text-sm text-gray-500">{shortcut.description}</p>
</div>
<div className="flex items-center space-x-1">
@ -60,7 +62,7 @@ export const KeyboardShortcuts: React.FC = () => {
<div className="flex">
<div className="ml-3">
<p className="text-sm text-yellow-700">
Note: Keyboard shortcuts are currently managed by the system.
Note: Keyboard shortcuts are currently managed by the system.
Custom configuration will be available in a future update.
</p>
</div>
@ -68,4 +70,4 @@ export const KeyboardShortcuts: React.FC = () => {
</div>
</div>
);
};
};