From 5c5e428e06a1cbb684f75c93abea93883d00d666 Mon Sep 17 00:00:00 2001 From: Dmitry Gordin Date: Sat, 27 Dec 2025 09:21:27 +0500 Subject: [PATCH] fix: replace async-openai library with post request (#480) * fix: replace async-openai library with post request to support custom providers * move models call to llm_client too * tweak post processing ui and add groq and cerebras --------- Co-authored-by: CJ Pais --- src-tauri/Cargo.lock | 29 +-- src-tauri/Cargo.toml | 1 - src-tauri/src/actions.rs | 59 +---- src-tauri/src/llm_client.rs | 202 ++++++++++++++++-- src-tauri/src/settings.rs | 24 +-- src-tauri/src/shortcut.rs | 111 +--------- src/bindings.ts | 2 +- .../usePostProcessProviderState.ts | 2 +- .../PostProcessingSettings.tsx | 43 ++-- 9 files changed, 227 insertions(+), 246 deletions(-) diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index 3a3acf1..5c2c2a2 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -282,32 +282,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "async-openai" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf39a15c8d613eb61892dc9a287c02277639ebead41ee611ad23aaa613f1a82" -dependencies = [ - "async-openai-macros", - "backoff", - "base64 0.22.1", - "bytes", - "derive_builder", - "eventsource-stream", - "futures", - "rand 0.9.2", - "reqwest", - "reqwest-eventsource", - "secrecy", - "serde", - "serde_json", - "thiserror 2.0.17", - "tokio", - "tokio-stream", - "tokio-util", - "tracing", -] - [[package]] name = "async-openai-macros" version = "0.1.0" @@ -2407,7 +2381,6 @@ name = "handy" version = "0.6.9" dependencies = [ "anyhow", - "async-openai 0.30.1", "chrono", "cpal", "enigo", @@ -6997,7 +6970,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "856646076d3d9739998ba693ebfff3afe95c0cdb71d4fead32e761b11496094f" dependencies = [ - "async-openai 0.29.6", + "async-openai", "async-trait", "derive_builder", "env_logger", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 44d768d..fcf2827 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -59,7 +59,6 @@ vad-rs = { git = "https://github.com/cjpais/vad-rs", default-features = false } enigo = "0.6.1" rodio = { git = "https://github.com/cjpais/rodio.git" } reqwest = { version = "0.12", features = ["json", "stream"] } -async-openai = "0.30.1" futures-util = "0.3" rustfft = "6.4.0" strsim = "0.11.0" diff --git a/src-tauri/src/actions.rs b/src-tauri/src/actions.rs index ca14c25..602de2e 100644 --- a/src-tauri/src/actions.rs +++ b/src-tauri/src/actions.rs @@ -8,10 +8,6 @@ use crate::settings::{get_settings, AppSettings, APPLE_INTELLIGENCE_PROVIDER_ID} use crate::shortcut; use crate::tray::{change_tray_icon, TrayIconState}; use crate::utils::{self, show_recording_overlay, show_transcribing_overlay}; -use async_openai::types::{ - ChatCompletionRequestMessage, ChatCompletionRequestUserMessageArgs, - CreateChatCompletionRequestArgs, -}; use ferrous_opencc::{config::BuiltinConfig, OpenCC}; use log::{debug, error}; use once_cell::sync::Lazy; @@ -139,52 +135,19 @@ async fn maybe_post_process_transcription( .cloned() .unwrap_or_default(); - // Create OpenAI-compatible client - let client = match crate::llm_client::create_client(&provider, api_key) { - Ok(client) => client, - Err(e) => { - error!("Failed to create LLM client: {}", e); - return None; - } - }; - - // Build the chat completion request - let message = match ChatCompletionRequestUserMessageArgs::default() - .content(processed_prompt) - .build() + // Send the chat completion request + match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt) + .await { - Ok(msg) => ChatCompletionRequestMessage::User(msg), - Err(e) => { - error!("Failed to build chat message: {}", e); - return None; + Ok(Some(content)) => { + debug!( + "LLM post-processing succeeded for provider '{}'. Output length: {} chars", + provider.id, + content.len() + ); + Some(content) } - }; - - let request = match CreateChatCompletionRequestArgs::default() - .model(&model) - .messages(vec![message]) - .build() - { - Ok(req) => req, - Err(e) => { - error!("Failed to build chat completion request: {}", e); - return None; - } - }; - - // Send the request - match client.chat().create(request).await { - Ok(response) => { - if let Some(choice) = response.choices.first() { - if let Some(content) = &choice.message.content { - debug!( - "LLM post-processing succeeded for provider '{}'. Output length: {} chars", - provider.id, - content.len() - ); - return Some(content.clone()); - } - } + Ok(None) => { error!("LLM API response has no content"); None } diff --git a/src-tauri/src/llm_client.rs b/src-tauri/src/llm_client.rs index 2177ea6..57fe67e 100644 --- a/src-tauri/src/llm_client.rs +++ b/src-tauri/src/llm_client.rs @@ -1,33 +1,191 @@ use crate::settings::PostProcessProvider; -use async_openai::{config::OpenAIConfig, Client}; +use log::debug; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT}; +use serde::{Deserialize, Serialize}; -/// Create an OpenAI-compatible client configured for the given provider -pub fn create_client( +#[derive(Debug, Serialize)] +struct ChatMessage { + role: String, + content: String, +} + +#[derive(Debug, Serialize)] +struct ChatCompletionRequest { + model: String, + messages: Vec, +} + +#[derive(Debug, Deserialize)] +struct ChatCompletionResponse { + choices: Vec, +} + +#[derive(Debug, Deserialize)] +struct ChatChoice { + message: ChatMessageResponse, +} + +#[derive(Debug, Deserialize)] +struct ChatMessageResponse { + content: Option, +} + +/// Build headers for API requests based on provider type +fn build_headers(provider: &PostProcessProvider, api_key: &str) -> Result { + let mut headers = HeaderMap::new(); + + // Common headers + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + headers.insert( + REFERER, + HeaderValue::from_static("https://github.com/cjpais/Handy"), + ); + headers.insert( + USER_AGENT, + HeaderValue::from_static("Handy/1.0 (+https://github.com/cjpais/Handy)"), + ); + headers.insert("X-Title", HeaderValue::from_static("Handy")); + + // Provider-specific auth headers + if !api_key.is_empty() { + if provider.id == "anthropic" { + headers.insert( + "x-api-key", + HeaderValue::from_str(api_key) + .map_err(|e| format!("Invalid API key header value: {}", e))?, + ); + headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01")); + } else { + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", api_key)) + .map_err(|e| format!("Invalid authorization header value: {}", e))?, + ); + } + } + + Ok(headers) +} + +/// Create an HTTP client with provider-specific headers +fn create_client(provider: &PostProcessProvider, api_key: &str) -> Result { + let headers = build_headers(provider, api_key)?; + reqwest::Client::builder() + .default_headers(headers) + .build() + .map_err(|e| format!("Failed to build HTTP client: {}", e)) +} + +/// Send a chat completion request to an OpenAI-compatible API +/// Returns Ok(Some(content)) on success, Ok(None) if response has no content, +/// or Err on actual errors (HTTP, parsing, etc.) +pub async fn send_chat_completion( provider: &PostProcessProvider, api_key: String, -) -> Result, String> { + model: &str, + prompt: String, +) -> Result, String> { let base_url = provider.base_url.trim_end_matches('/'); - let config = OpenAIConfig::new() - .with_api_base(base_url) - .with_api_key(api_key); + let url = format!("{}/chat/completions", base_url); - // Create client with Anthropic-specific header if needed - let client = if provider.id == "anthropic" { - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - "anthropic-version", - reqwest::header::HeaderValue::from_static("2023-06-01"), - ); + debug!("Sending chat completion request to: {}", url); - let http_client = reqwest::Client::builder() - .default_headers(headers) - .build() - .map_err(|e| format!("Failed to build HTTP client: {}", e))?; + let client = create_client(provider, &api_key)?; - Client::with_config(config).with_http_client(http_client) - } else { - Client::with_config(config) + let request_body = ChatCompletionRequest { + model: model.to_string(), + messages: vec![ChatMessage { + role: "user".to_string(), + content: prompt, + }], }; - Ok(client) + let response = client + .post(&url) + .json(&request_body) + .send() + .await + .map_err(|e| format!("HTTP request failed: {}", e))?; + + let status = response.status(); + if !status.is_success() { + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Failed to read error response".to_string()); + return Err(format!( + "API request failed with status {}: {}", + status, error_text + )); + } + + let completion: ChatCompletionResponse = response + .json() + .await + .map_err(|e| format!("Failed to parse API response: {}", e))?; + + Ok(completion + .choices + .first() + .and_then(|choice| choice.message.content.clone())) +} + +/// Fetch available models from an OpenAI-compatible API +/// Returns a list of model IDs +pub async fn fetch_models( + provider: &PostProcessProvider, + api_key: String, +) -> Result, String> { + let base_url = provider.base_url.trim_end_matches('/'); + let url = format!("{}/models", base_url); + + debug!("Fetching models from: {}", url); + + let client = create_client(provider, &api_key)?; + + let response = client + .get(&url) + .send() + .await + .map_err(|e| format!("Failed to fetch models: {}", e))?; + + let status = response.status(); + if !status.is_success() { + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Unknown error".to_string()); + return Err(format!( + "Model list request failed ({}): {}", + status, error_text + )); + } + + let parsed: serde_json::Value = response + .json() + .await + .map_err(|e| format!("Failed to parse response: {}", e))?; + + let mut models = Vec::new(); + + // Handle OpenAI format: { data: [ { id: "..." }, ... ] } + if let Some(data) = parsed.get("data").and_then(|d| d.as_array()) { + for entry in data { + if let Some(id) = entry.get("id").and_then(|i| i.as_str()) { + models.push(id.to_string()); + } else if let Some(name) = entry.get("name").and_then(|n| n.as_str()) { + models.push(name.to_string()); + } + } + } + // Handle array format: [ "model1", "model2", ... ] + else if let Some(array) = parsed.as_array() { + for entry in array { + if let Some(model) = entry.as_str() { + models.push(model.to_string()); + } + } + } + + Ok(models) } diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs index 4b2380e..8ec885d 100644 --- a/src-tauri/src/settings.rs +++ b/src-tauri/src/settings.rs @@ -97,10 +97,6 @@ pub struct PostProcessProvider { pub id: String, pub label: String, pub base_url: String, - #[serde(default)] - pub allow_base_url_edit: bool, - #[serde(default)] - pub models_endpoint: Option, } #[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Type)] @@ -378,29 +374,31 @@ fn default_post_process_providers() -> Vec { id: "openai".to_string(), label: "OpenAI".to_string(), base_url: "https://api.openai.com/v1".to_string(), - allow_base_url_edit: false, - models_endpoint: Some("/models".to_string()), }, PostProcessProvider { id: "openrouter".to_string(), label: "OpenRouter".to_string(), base_url: "https://openrouter.ai/api/v1".to_string(), - allow_base_url_edit: false, - models_endpoint: Some("/models".to_string()), }, PostProcessProvider { id: "anthropic".to_string(), label: "Anthropic".to_string(), base_url: "https://api.anthropic.com/v1".to_string(), - allow_base_url_edit: false, - models_endpoint: Some("/models".to_string()), + }, + PostProcessProvider { + id: "groq".to_string(), + label: "Groq".to_string(), + base_url: "https://api.groq.com/openai/v1".to_string(), + }, + PostProcessProvider { + id: "cerebras".to_string(), + label: "Cerebras".to_string(), + base_url: "https://api.cerebras.ai/v1".to_string(), }, PostProcessProvider { id: "custom".to_string(), label: "Custom".to_string(), base_url: "http://localhost:11434/v1".to_string(), - allow_base_url_edit: true, - models_endpoint: Some("/models".to_string()), }, ]; @@ -411,8 +409,6 @@ fn default_post_process_providers() -> Vec { id: APPLE_INTELLIGENCE_PROVIDER_ID.to_string(), label: "Apple Intelligence".to_string(), base_url: "apple-intelligence://local".to_string(), - allow_base_url_edit: false, - models_endpoint: None, }); } } diff --git a/src-tauri/src/shortcut.rs b/src-tauri/src/shortcut.rs index aaa99e0..f983b0b 100644 --- a/src-tauri/src/shortcut.rs +++ b/src-tauri/src/shortcut.rs @@ -390,7 +390,7 @@ pub fn change_post_process_base_url_setting( .post_process_provider_mut(&provider_id) .expect("Provider looked up above must exist"); - if !provider.allow_base_url_edit { + if provider.id != "custom" { return Err(format!( "Provider '{}' does not allow editing the base URL", label @@ -573,114 +573,7 @@ pub async fn fetch_post_process_models( )); } - // TODO: In the future, we can use async-openai's models API: - // let client = crate::llm_client::create_client(provider, api_key)?; - // let response = client.models().list().await?; - // return Ok(response.data.iter().map(|m| m.id.clone()).collect()); - - // For now, use manual HTTP request to have more control over the endpoint - fetch_models_manual(provider, api_key).await -} - -/// Fetch models using manual HTTP request -/// This gives us more control and avoids issues with non-standard endpoints -async fn fetch_models_manual( - provider: &crate::settings::PostProcessProvider, - api_key: String, -) -> Result, String> { - // Build the endpoint URL - let base_url = provider.base_url.trim_end_matches('/'); - let models_endpoint = provider - .models_endpoint - .as_ref() - .map(|s| s.trim_start_matches('/')) - .unwrap_or("models"); - let endpoint = format!("{}/{}", base_url, models_endpoint); - - // Create HTTP client with headers - let mut headers = reqwest::header::HeaderMap::new(); - headers.insert( - "HTTP-Referer", - reqwest::header::HeaderValue::from_static("https://github.com/cjpais/Handy"), - ); - headers.insert( - "X-Title", - reqwest::header::HeaderValue::from_static("Handy"), - ); - - // Add provider-specific headers - if provider.id == "anthropic" { - if !api_key.is_empty() { - headers.insert( - "x-api-key", - reqwest::header::HeaderValue::from_str(&api_key) - .map_err(|e| format!("Invalid API key: {}", e))?, - ); - } - headers.insert( - "anthropic-version", - reqwest::header::HeaderValue::from_static("2023-06-01"), - ); - } else if !api_key.is_empty() { - headers.insert( - "Authorization", - reqwest::header::HeaderValue::from_str(&format!("Bearer {}", api_key)) - .map_err(|e| format!("Invalid API key: {}", e))?, - ); - } - - let http_client = reqwest::Client::builder() - .default_headers(headers) - .build() - .map_err(|e| format!("Failed to build HTTP client: {}", e))?; - - // Make the request - let response = http_client - .get(&endpoint) - .send() - .await - .map_err(|e| format!("Failed to fetch models: {}", e))?; - - if !response.status().is_success() { - let status = response.status(); - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - return Err(format!( - "Model list request failed ({}): {}", - status, error_text - )); - } - - // Parse the response - let parsed: serde_json::Value = response - .json() - .await - .map_err(|e| format!("Failed to parse response: {}", e))?; - - let mut models = Vec::new(); - - // Handle OpenAI format: { data: [ { id: "..." }, ... ] } - if let Some(data) = parsed.get("data").and_then(|d| d.as_array()) { - for entry in data { - if let Some(id) = entry.get("id").and_then(|i| i.as_str()) { - models.push(id.to_string()); - } else if let Some(name) = entry.get("name").and_then(|n| n.as_str()) { - models.push(name.to_string()); - } - } - } - // Handle array format: [ "model1", "model2", ... ] - else if let Some(array) = parsed.as_array() { - for entry in array { - if let Some(model) = entry.as_str() { - models.push(model.to_string()); - } - } - } - - Ok(models) + crate::llm_client::fetch_models(provider, api_key).await } #[tauri::command] diff --git a/src/bindings.ts b/src/bindings.ts index 7e271a4..24aef75 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -635,7 +635,7 @@ export type ModelLoadStatus = { is_loaded: boolean; current_model: string | null export type ModelUnloadTimeout = "never" | "immediately" | "min_2" | "min_5" | "min_10" | "min_15" | "hour_1" | "sec_5" export type OverlayPosition = "none" | "top" | "bottom" export type PasteMethod = "ctrl_v" | "direct" | "none" | "shift_insert" | "ctrl_shift_v" -export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null } +export type PostProcessProvider = { id: string; label: string; base_url: string } export type RecordingRetentionPeriod = "never" | "preserve_limit" | "days_3" | "weeks_2" | "months_3" export type ShortcutBinding = { id: string; name: string; description: string; default_binding: string; current_binding: string } export type SoundTheme = "marimba" | "pop" | "custom" diff --git a/src/components/settings/PostProcessingSettingsApi/usePostProcessProviderState.ts b/src/components/settings/PostProcessingSettingsApi/usePostProcessProviderState.ts index 4cd04ee..c5c7364 100644 --- a/src/components/settings/PostProcessingSettingsApi/usePostProcessProviderState.ts +++ b/src/components/settings/PostProcessingSettingsApi/usePostProcessProviderState.ts @@ -84,7 +84,7 @@ export const usePostProcessProviderState = (): PostProcessProviderState => { const handleBaseUrlChange = useCallback( (value: string) => { - if (!selectedProvider || !selectedProvider.allow_base_url_edit) { + if (!selectedProvider || selectedProvider.id !== "custom") { return; } const trimmed = value.trim(); diff --git a/src/components/settings/post-processing/PostProcessingSettings.tsx b/src/components/settings/post-processing/PostProcessingSettings.tsx index 52ad96d..2db6188 100644 --- a/src/components/settings/post-processing/PostProcessingSettings.tsx +++ b/src/components/settings/post-processing/PostProcessingSettings.tsx @@ -72,28 +72,27 @@ const PostProcessingSettingsApiComponent: React.FC = () => { ) : ( <> - -
- -
- + {state.selectedProvider?.id === "custom" && ( + +
+ +
+ + )}