danvics/Handy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: replace async-openai library with post request (#480)

Browse source 
* fix: replace async-openai library with post request to support custom providers

* move models call to llm_client too

* tweak post processing ui and add groq and cerebras

---------

Co-authored-by: CJ Pais <cj@cjpais.com>
This commit is contained in:
Dmitry Gordin 2025-12-27 09:21:27 +05:00 committed by GitHub
parent b55c3fddde
commit 5c5e428e06
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 227 additions and 246 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src-tauri/Cargo.lock generated
View file

@ -282,32 +282,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "async-openai"
version = "0.30.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bf39a15c8d613eb61892dc9a287c02277639ebead41ee611ad23aaa613f1a82"
dependencies = [
"async-openai-macros",
"backoff",
"base64 0.22.1",
"bytes",
"derive_builder",
"eventsource-stream",
"futures",
"rand 0.9.2",
"reqwest",
"reqwest-eventsource",
"secrecy",
"serde",
"serde_json",
"thiserror 2.0.17",
"tokio",
"tokio-stream",
"tokio-util",
"tracing",
]
[[package]]
name = "async-openai-macros"
version = "0.1.0"
@ -2407,7 +2381,6 @@ name = "handy"
version = "0.6.9"
dependencies = [
"anyhow",
"async-openai 0.30.1",
"chrono",
"cpal",
"enigo",
@ -6997,7 +6970,7 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "856646076d3d9739998ba693ebfff3afe95c0cdb71d4fead32e761b11496094f"
dependencies = [
"async-openai 0.29.6",
"async-openai",
"async-trait",
"derive_builder",
"env_logger",

1
src-tauri/Cargo.toml
View file

@ -59,7 +59,6 @@ vad-rs = { git = "https://github.com/cjpais/vad-rs", default-features = false }
enigo = "0.6.1"
rodio = { git = "https://github.com/cjpais/rodio.git" }
reqwest = { version = "0.12", features = ["json", "stream"] }
async-openai = "0.30.1"
futures-util = "0.3"
rustfft = "6.4.0"
strsim = "0.11.0"

59
src-tauri/src/actions.rs
View file

@ -8,10 +8,6 @@ use crate::settings::{get_settings, AppSettings, APPLE_INTELLIGENCE_PROVIDER_ID}
use crate::shortcut;
use crate::tray::{change_tray_icon, TrayIconState};
use crate::utils::{self, show_recording_overlay, show_transcribing_overlay};
use async_openai::types::{
ChatCompletionRequestMessage, ChatCompletionRequestUserMessageArgs,
CreateChatCompletionRequestArgs,
};
use ferrous_opencc::{config::BuiltinConfig, OpenCC};
use log::{debug, error};
use once_cell::sync::Lazy;
@ -139,52 +135,19 @@ async fn maybe_post_process_transcription(
.cloned()
.unwrap_or_default();
// Create OpenAI-compatible client
let client = match crate::llm_client::create_client(&provider, api_key) {
Ok(client) => client,
Err(e) => {
error!("Failed to create LLM client: {}", e);
return None;
}
};
// Build the chat completion request
let message = match ChatCompletionRequestUserMessageArgs::default()
.content(processed_prompt)
.build()
// Send the chat completion request
match crate::llm_client::send_chat_completion(&provider, api_key, &model, processed_prompt)
.await
{
Ok(msg) => ChatCompletionRequestMessage::User(msg),
Err(e) => {
error!("Failed to build chat message: {}", e);
return None;
Ok(Some(content)) => {
debug!(
"LLM post-processing succeeded for provider '{}'. Output length: {} chars",
provider.id,
content.len()
);
Some(content)
}
};
let request = match CreateChatCompletionRequestArgs::default()
.model(&model)
.messages(vec![message])
.build()
{
Ok(req) => req,
Err(e) => {
error!("Failed to build chat completion request: {}", e);
return None;
}
};
// Send the request
match client.chat().create(request).await {
Ok(response) => {
if let Some(choice) = response.choices.first() {
if let Some(content) = &choice.message.content {
debug!(
"LLM post-processing succeeded for provider '{}'. Output length: {} chars",
provider.id,
content.len()
);
return Some(content.clone());
}
}
Ok(None) => {
error!("LLM API response has no content");
None
}

202
src-tauri/src/llm_client.rs
View file

@ -1,33 +1,191 @@
use crate::settings::PostProcessProvider;
use async_openai::{config::OpenAIConfig, Client};
use log::debug;
use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION, CONTENT_TYPE, REFERER, USER_AGENT};
use serde::{Deserialize, Serialize};
/// Create an OpenAI-compatible client configured for the given provider
pub fn create_client(
#[derive(Debug, Serialize)]
struct ChatMessage {
role: String,
content: String,
}
#[derive(Debug, Serialize)]
struct ChatCompletionRequest {
model: String,
messages: Vec<ChatMessage>,
}
#[derive(Debug, Deserialize)]
struct ChatCompletionResponse {
choices: Vec<ChatChoice>,
}
#[derive(Debug, Deserialize)]
struct ChatChoice {
message: ChatMessageResponse,
}
#[derive(Debug, Deserialize)]
struct ChatMessageResponse {
content: Option<String>,
}
/// Build headers for API requests based on provider type
fn build_headers(provider: &PostProcessProvider, api_key: &str) -> Result<HeaderMap, String> {
let mut headers = HeaderMap::new();
// Common headers
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
headers.insert(
REFERER,
HeaderValue::from_static("https://github.com/cjpais/Handy"),
);
headers.insert(
USER_AGENT,
HeaderValue::from_static("Handy/1.0 (+https://github.com/cjpais/Handy)"),
);
headers.insert("X-Title", HeaderValue::from_static("Handy"));
// Provider-specific auth headers
if !api_key.is_empty() {
if provider.id == "anthropic" {
headers.insert(
"x-api-key",
HeaderValue::from_str(api_key)
.map_err(|e| format!("Invalid API key header value: {}", e))?,
);
headers.insert("anthropic-version", HeaderValue::from_static("2023-06-01"));
} else {
headers.insert(
AUTHORIZATION,
HeaderValue::from_str(&format!("Bearer {}", api_key))
.map_err(|e| format!("Invalid authorization header value: {}", e))?,
);
}
}
Ok(headers)
}
/// Create an HTTP client with provider-specific headers
fn create_client(provider: &PostProcessProvider, api_key: &str) -> Result<reqwest::Client, String> {
let headers = build_headers(provider, api_key)?;
reqwest::Client::builder()
.default_headers(headers)
.build()
.map_err(|e| format!("Failed to build HTTP client: {}", e))
}
/// Send a chat completion request to an OpenAI-compatible API
/// Returns Ok(Some(content)) on success, Ok(None) if response has no content,
/// or Err on actual errors (HTTP, parsing, etc.)
pub async fn send_chat_completion(
provider: &PostProcessProvider,
api_key: String,
) -> Result<Client<OpenAIConfig>, String> {
model: &str,
prompt: String,
) -> Result<Option<String>, String> {
let base_url = provider.base_url.trim_end_matches('/');
let config = OpenAIConfig::new()
.with_api_base(base_url)
.with_api_key(api_key);
let url = format!("{}/chat/completions", base_url);
// Create client with Anthropic-specific header if needed
let client = if provider.id == "anthropic" {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
"anthropic-version",
reqwest::header::HeaderValue::from_static("2023-06-01"),
);
debug!("Sending chat completion request to: {}", url);
let http_client = reqwest::Client::builder()
.default_headers(headers)
.build()
.map_err(|e| format!("Failed to build HTTP client: {}", e))?;
let client = create_client(provider, &api_key)?;
Client::with_config(config).with_http_client(http_client)
} else {
Client::with_config(config)
let request_body = ChatCompletionRequest {
model: model.to_string(),
messages: vec![ChatMessage {
role: "user".to_string(),
content: prompt,
}],
};
Ok(client)
let response = client
.post(&url)
.json(&request_body)
.send()
.await
.map_err(|e| format!("HTTP request failed: {}", e))?;
let status = response.status();
if !status.is_success() {
let error_text = response
.text()
.await
.unwrap_or_else(|_| "Failed to read error response".to_string());
return Err(format!(
"API request failed with status {}: {}",
status, error_text
));
}
let completion: ChatCompletionResponse = response
.json()
.await
.map_err(|e| format!("Failed to parse API response: {}", e))?;
Ok(completion
.choices
.first()
.and_then(|choice| choice.message.content.clone()))
}
/// Fetch available models from an OpenAI-compatible API
/// Returns a list of model IDs
pub async fn fetch_models(
provider: &PostProcessProvider,
api_key: String,
) -> Result<Vec<String>, String> {
let base_url = provider.base_url.trim_end_matches('/');
let url = format!("{}/models", base_url);
debug!("Fetching models from: {}", url);
let client = create_client(provider, &api_key)?;
let response = client
.get(&url)
.send()
.await
.map_err(|e| format!("Failed to fetch models: {}", e))?;
let status = response.status();
if !status.is_success() {
let error_text = response
.text()
.await
.unwrap_or_else(|_| "Unknown error".to_string());
return Err(format!(
"Model list request failed ({}): {}",
status, error_text
));
}
let parsed: serde_json::Value = response
.json()
.await
.map_err(|e| format!("Failed to parse response: {}", e))?;
let mut models = Vec::new();
// Handle OpenAI format: { data: [ { id: "..." }, ... ] }
if let Some(data) = parsed.get("data").and_then(|d| d.as_array()) {
for entry in data {
if let Some(id) = entry.get("id").and_then(|i| i.as_str()) {
models.push(id.to_string());
} else if let Some(name) = entry.get("name").and_then(|n| n.as_str()) {
models.push(name.to_string());
}
}
}
// Handle array format: [ "model1", "model2", ... ]
else if let Some(array) = parsed.as_array() {
for entry in array {
if let Some(model) = entry.as_str() {
models.push(model.to_string());
}
}
}
Ok(models)
}

24
src-tauri/src/settings.rs
View file

@ -97,10 +97,6 @@ pub struct PostProcessProvider {
pub id: String,
pub label: String,
pub base_url: String,
#[serde(default)]
pub allow_base_url_edit: bool,
#[serde(default)]
pub models_endpoint: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Type)]
@ -378,29 +374,31 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
id: "openai".to_string(),
label: "OpenAI".to_string(),
base_url: "https://api.openai.com/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
},
PostProcessProvider {
id: "openrouter".to_string(),
label: "OpenRouter".to_string(),
base_url: "https://openrouter.ai/api/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
},
PostProcessProvider {
id: "anthropic".to_string(),
label: "Anthropic".to_string(),
base_url: "https://api.anthropic.com/v1".to_string(),
allow_base_url_edit: false,
models_endpoint: Some("/models".to_string()),
},
PostProcessProvider {
id: "groq".to_string(),
label: "Groq".to_string(),
base_url: "https://api.groq.com/openai/v1".to_string(),
},
PostProcessProvider {
id: "cerebras".to_string(),
label: "Cerebras".to_string(),
base_url: "https://api.cerebras.ai/v1".to_string(),
},
PostProcessProvider {
id: "custom".to_string(),
label: "Custom".to_string(),
base_url: "http://localhost:11434/v1".to_string(),
allow_base_url_edit: true,
models_endpoint: Some("/models".to_string()),
},
];
@ -411,8 +409,6 @@ fn default_post_process_providers() -> Vec<PostProcessProvider> {
id: APPLE_INTELLIGENCE_PROVIDER_ID.to_string(),
label: "Apple Intelligence".to_string(),
base_url: "apple-intelligence://local".to_string(),
allow_base_url_edit: false,
models_endpoint: None,
});
}
}

111
src-tauri/src/shortcut.rs
View file

@ -390,7 +390,7 @@ pub fn change_post_process_base_url_setting(
.post_process_provider_mut(&provider_id)
.expect("Provider looked up above must exist");
if !provider.allow_base_url_edit {
if provider.id != "custom" {
return Err(format!(
"Provider '{}' does not allow editing the base URL",
label
@ -573,114 +573,7 @@ pub async fn fetch_post_process_models(
));
}
// TODO: In the future, we can use async-openai's models API:
// let client = crate::llm_client::create_client(provider, api_key)?;
// let response = client.models().list().await?;
// return Ok(response.data.iter().map(|m| m.id.clone()).collect());
// For now, use manual HTTP request to have more control over the endpoint
fetch_models_manual(provider, api_key).await
}
/// Fetch models using manual HTTP request
/// This gives us more control and avoids issues with non-standard endpoints
async fn fetch_models_manual(
provider: &crate::settings::PostProcessProvider,
api_key: String,
) -> Result<Vec<String>, String> {
// Build the endpoint URL
let base_url = provider.base_url.trim_end_matches('/');
let models_endpoint = provider
.models_endpoint
.as_ref()
.map(|s| s.trim_start_matches('/'))
.unwrap_or("models");
let endpoint = format!("{}/{}", base_url, models_endpoint);
// Create HTTP client with headers
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
"HTTP-Referer",
reqwest::header::HeaderValue::from_static("https://github.com/cjpais/Handy"),
);
headers.insert(
"X-Title",
reqwest::header::HeaderValue::from_static("Handy"),
);
// Add provider-specific headers
if provider.id == "anthropic" {
if !api_key.is_empty() {
headers.insert(
"x-api-key",
reqwest::header::HeaderValue::from_str(&api_key)
.map_err(|e| format!("Invalid API key: {}", e))?,
);
}
headers.insert(
"anthropic-version",
reqwest::header::HeaderValue::from_static("2023-06-01"),
);
} else if !api_key.is_empty() {
headers.insert(
"Authorization",
reqwest::header::HeaderValue::from_str(&format!("Bearer {}", api_key))
.map_err(|e| format!("Invalid API key: {}", e))?,
);
}
let http_client = reqwest::Client::builder()
.default_headers(headers)
.build()
.map_err(|e| format!("Failed to build HTTP client: {}", e))?;
// Make the request
let response = http_client
.get(&endpoint)
.send()
.await
.map_err(|e| format!("Failed to fetch models: {}", e))?;
if !response.status().is_success() {
let status = response.status();
let error_text = response
.text()
.await
.unwrap_or_else(|_| "Unknown error".to_string());
return Err(format!(
"Model list request failed ({}): {}",
status, error_text
));
}
// Parse the response
let parsed: serde_json::Value = response
.json()
.await
.map_err(|e| format!("Failed to parse response: {}", e))?;
let mut models = Vec::new();
// Handle OpenAI format: { data: [ { id: "..." }, ... ] }
if let Some(data) = parsed.get("data").and_then(|d| d.as_array()) {
for entry in data {
if let Some(id) = entry.get("id").and_then(|i| i.as_str()) {
models.push(id.to_string());
} else if let Some(name) = entry.get("name").and_then(|n| n.as_str()) {
models.push(name.to_string());
}
}
}
// Handle array format: [ "model1", "model2", ... ]
else if let Some(array) = parsed.as_array() {
for entry in array {
if let Some(model) = entry.as_str() {
models.push(model.to_string());
}
}
}
Ok(models)
crate::llm_client::fetch_models(provider, api_key).await
}
#[tauri::command]

2
src/bindings.ts
View file

@ -635,7 +635,7 @@ export type ModelLoadStatus = { is_loaded: boolean; current_model: string | null
export type ModelUnloadTimeout = "never" | "immediately" | "min_2" | "min_5" | "min_10" | "min_15" | "hour_1" | "sec_5"
export type OverlayPosition = "none" | "top" | "bottom"
export type PasteMethod = "ctrl_v" | "direct" | "none" | "shift_insert" | "ctrl_shift_v"
export type PostProcessProvider = { id: string; label: string; base_url: string; allow_base_url_edit?: boolean; models_endpoint?: string | null }
export type PostProcessProvider = { id: string; label: string; base_url: string }
export type RecordingRetentionPeriod = "never" | "preserve_limit" | "days_3" | "weeks_2" | "months_3"
export type ShortcutBinding = { id: string; name: string; description: string; default_binding: string; current_binding: string }
export type SoundTheme = "marimba" | "pop" | "custom"

2
src/components/settings/PostProcessingSettingsApi/usePostProcessProviderState.ts
View file

@ -84,7 +84,7 @@ export const usePostProcessProviderState = (): PostProcessProviderState => {
const handleBaseUrlChange = useCallback(
(value: string) => {
if (!selectedProvider || !selectedProvider.allow_base_url_edit) {
if (!selectedProvider || selectedProvider.id !== "custom") {
return;
}
const trimmed = value.trim();

43
src/components/settings/post-processing/PostProcessingSettings.tsx
View file

@ -72,28 +72,27 @@ const PostProcessingSettingsApiComponent: React.FC = () => {
</SettingContainer>
) : (
<>
<SettingContainer
title={t("settings.postProcessing.api.baseUrl.title")}
description={t("settings.postProcessing.api.baseUrl.description")}
descriptionMode="tooltip"
layout="horizontal"
grouped={true}
>
<div className="flex items-center gap-2">
<BaseUrlField
value={state.baseUrl}
onBlur={state.handleBaseUrlChange}
placeholder={t(
"settings.postProcessing.api.baseUrl.placeholder",
)}
disabled={
!state.selectedProvider?.allow_base_url_edit ||
state.isBaseUrlUpdating
}
className="min-w-[380px]"
/>
</div>
</SettingContainer>
{state.selectedProvider?.id === "custom" && (
<SettingContainer
title={t("settings.postProcessing.api.baseUrl.title")}
description={t("settings.postProcessing.api.baseUrl.description")}
descriptionMode="tooltip"
layout="horizontal"
grouped={true}
>
<div className="flex items-center gap-2">
<BaseUrlField
value={state.baseUrl}
onBlur={state.handleBaseUrlChange}
placeholder={t(
"settings.postProcessing.api.baseUrl.placeholder",
)}
disabled={state.isBaseUrlUpdating}
className="min-w-[380px]"
/>
</div>
</SettingContainer>
)}
<SettingContainer
title={t("settings.postProcessing.api.apiKey.title")}