diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index c7c5280..db36fed 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -7086,10 +7086,11 @@ dependencies = [ [[package]] name = "transcribe-rs" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a15e55d3bd4f74a474b7ff2c87ab416130223db490102c5ad32c6e2227686b0" +checksum = "2039bf80d6497655ecdef8ebfa10583862fad3cc017077e105bc1c18b7502ff8" dependencies = [ + "base64 0.22.1", "derive_builder", "env_logger", "hound", @@ -7098,6 +7099,7 @@ dependencies = [ "once_cell", "ort", "regex", + "rustfft", "serde", "serde_json", "thiserror 2.0.17", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index db7f747..1074245 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -68,7 +68,7 @@ chrono = "0.4" rusqlite = { version = "0.37", features = ["bundled"] } tar = "0.4.44" flate2 = "1.0" -transcribe-rs = { version = "0.2.2", features = ["whisper", "parakeet", "moonshine"] } +transcribe-rs = { version = "0.2.3", features = ["whisper", "parakeet", "moonshine", "sense_voice"] } handy-keys = "0.1.4" ferrous-opencc = "0.2.3" specta = "=2.0.0-rc.22" diff --git a/src-tauri/src/managers/model.rs b/src-tauri/src/managers/model.rs index 28fbcfa..73fbaee 100644 --- a/src-tauri/src/managers/model.rs +++ b/src-tauri/src/managers/model.rs @@ -21,6 +21,7 @@ pub enum EngineType { Whisper, Parakeet, Moonshine, + SenseVoice, } #[derive(Debug, Clone, Serialize, Deserialize, Type)] @@ -289,6 +290,37 @@ impl ModelManager { }, ); + // SenseVoice supported languages + let sense_voice_languages: Vec = + vec!["zh", "zh-Hans", "zh-Hant", "en", "yue", "ja", "ko"] + .into_iter() + .map(String::from) + .collect(); + + available_models.insert( + "sense-voice-int8".to_string(), + ModelInfo { + id: "sense-voice-int8".to_string(), + name: "SenseVoice".to_string(), + description: "Very fast. Chinese, English, Japanese, Korean, Cantonese." + .to_string(), + filename: "sense-voice-int8".to_string(), + url: Some("https://blob.handy.computer/sense-voice-int8.tar.gz".to_string()), + size_mb: 160, + is_downloaded: false, + is_downloading: false, + partial_size: 0, + is_directory: true, + engine_type: EngineType::SenseVoice, + accuracy_score: 0.65, + speed_score: 0.95, + supports_translation: false, + is_recommended: false, + supported_languages: sense_voice_languages, + is_custom: false, + }, + ); + // Auto-discover custom Whisper models (.bin files) in the models directory if let Err(e) = Self::discover_custom_whisper_models(&models_dir, &mut available_models) { warn!("Failed to discover custom models: {}", e); diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs index 4287533..0703e1a 100644 --- a/src-tauri/src/managers/transcription.rs +++ b/src-tauri/src/managers/transcription.rs @@ -15,6 +15,10 @@ use transcribe_rs::{ parakeet::{ ParakeetEngine, ParakeetInferenceParams, ParakeetModelParams, TimestampGranularity, }, + sense_voice::{ + Language as SenseVoiceLanguage, SenseVoiceEngine, SenseVoiceInferenceParams, + SenseVoiceModelParams, + }, whisper::{WhisperEngine, WhisperInferenceParams}, }, TranscriptionEngine, @@ -32,6 +36,7 @@ enum LoadedEngine { Whisper(WhisperEngine), Parakeet(ParakeetEngine), Moonshine(MoonshineEngine), + SenseVoice(SenseVoiceEngine), } #[derive(Clone)] @@ -145,6 +150,7 @@ impl TranscriptionManager { LoadedEngine::Whisper(ref mut e) => e.unload_model(), LoadedEngine::Parakeet(ref mut e) => e.unload_model(), LoadedEngine::Moonshine(ref mut e) => e.unload_model(), + LoadedEngine::SenseVoice(ref mut e) => e.unload_model(), } } *engine = None; // Drop the engine to free memory @@ -284,6 +290,26 @@ impl TranscriptionManager { })?; LoadedEngine::Moonshine(engine) } + EngineType::SenseVoice => { + let mut engine = SenseVoiceEngine::new(); + engine + .load_model_with_params(&model_path, SenseVoiceModelParams::int8()) + .map_err(|e| { + let error_msg = + format!("Failed to load SenseVoice model {}: {}", model_id, e); + let _ = self.app_handle.emit( + "model-state-changed", + ModelStateEvent { + event_type: "loading_failed".to_string(), + model_id: Some(model_id.to_string()), + model_name: Some(model_info.name.clone()), + error: Some(error_msg.clone()), + }, + ); + anyhow::anyhow!(error_msg) + })?; + LoadedEngine::SenseVoice(engine) + } }; // Update the current engine and model ID @@ -426,6 +452,23 @@ impl TranscriptionManager { LoadedEngine::Moonshine(moonshine_engine) => moonshine_engine .transcribe_samples(audio, None) .map_err(|e| anyhow::anyhow!("Moonshine transcription failed: {}", e))?, + LoadedEngine::SenseVoice(sense_voice_engine) => { + let language = match settings.selected_language.as_str() { + "zh" | "zh-Hans" | "zh-Hant" => SenseVoiceLanguage::Chinese, + "en" => SenseVoiceLanguage::English, + "ja" => SenseVoiceLanguage::Japanese, + "ko" => SenseVoiceLanguage::Korean, + "yue" => SenseVoiceLanguage::Cantonese, + _ => SenseVoiceLanguage::Auto, + }; + let params = SenseVoiceInferenceParams { + language, + use_itn: true, + }; + sense_voice_engine + .transcribe_samples(audio, Some(params)) + .map_err(|e| anyhow::anyhow!("SenseVoice transcription failed: {}", e))? + } } }; diff --git a/src/bindings.ts b/src/bindings.ts index cc6a336..0cbe439 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -700,7 +700,7 @@ export type AudioDevice = { index: string; name: string; is_default: boolean } export type BindingResponse = { success: boolean; binding: ShortcutBinding | null; error: string | null } export type ClipboardHandling = "dont_modify" | "copy_to_clipboard" export type CustomSounds = { start: boolean; stop: boolean } -export type EngineType = "Whisper" | "Parakeet" | "Moonshine" +export type EngineType = "Whisper" | "Parakeet" | "Moonshine" | "SenseVoice" export type HistoryEntry = { id: number; file_name: string; timestamp: number; saved: boolean; title: string; transcription_text: string; post_processed_text: string | null; post_process_prompt: string | null } /** * Result of changing keyboard implementation diff --git a/src/components/settings/LanguageSelector.tsx b/src/components/settings/LanguageSelector.tsx index 893d98f..30cba20 100644 --- a/src/components/settings/LanguageSelector.tsx +++ b/src/components/settings/LanguageSelector.tsx @@ -8,11 +8,13 @@ import { LANGUAGES } from "../../lib/constants/languages"; interface LanguageSelectorProps { descriptionMode?: "inline" | "tooltip"; grouped?: boolean; + supportedLanguages?: string[]; } export const LanguageSelector: React.FC = ({ descriptionMode = "tooltip", grouped = false, + supportedLanguages, }) => { const { t } = useTranslation(); const { getSetting, updateSetting, resetSetting, isUpdating } = useSettings(); @@ -46,12 +48,21 @@ export const LanguageSelector: React.FC = ({ } }, [isOpen]); + const availableLanguages = useMemo(() => { + if (!supportedLanguages || supportedLanguages.length === 0) + return LANGUAGES; + return LANGUAGES.filter( + (lang) => + lang.value === "auto" || supportedLanguages.includes(lang.value), + ); + }, [supportedLanguages]); + const filteredLanguages = useMemo( () => - LANGUAGES.filter((language) => + availableLanguages.filter((language) => language.label.toLowerCase().includes(searchQuery.toLowerCase()), ), - [searchQuery], + [searchQuery, availableLanguages], ); const selectedLanguageName = diff --git a/src/components/settings/general/ModelSettingsCard.tsx b/src/components/settings/general/ModelSettingsCard.tsx index f91258c..39b8df4 100644 --- a/src/components/settings/general/ModelSettingsCard.tsx +++ b/src/components/settings/general/ModelSettingsCard.tsx @@ -12,8 +12,9 @@ export const ModelSettingsCard: React.FC = () => { const currentModelInfo = models.find((m: ModelInfo) => m.id === currentModel); - // Only Whisper models support manual language selection - const supportsLanguageSelection = currentModelInfo?.engine_type === "Whisper"; + const supportsLanguageSelection = + currentModelInfo?.engine_type === "Whisper" || + currentModelInfo?.engine_type === "SenseVoice"; const supportsTranslation = currentModelInfo?.supports_translation ?? false; const hasAnySettings = supportsLanguageSelection || supportsTranslation; @@ -29,7 +30,11 @@ export const ModelSettingsCard: React.FC = () => { })} > {supportsLanguageSelection && ( - + )} {supportsTranslation && ( diff --git a/src/lib/constants/languages.ts b/src/lib/constants/languages.ts index 38920ba..f4db56b 100644 --- a/src/lib/constants/languages.ts +++ b/src/lib/constants/languages.ts @@ -8,6 +8,7 @@ export const LANGUAGES: Language[] = [ { value: "en", label: "English" }, { value: "zh-Hans", label: "Simplified Chinese" }, { value: "zh-Hant", label: "Traditional Chinese" }, + { value: "yue", label: "Cantonese" }, { value: "de", label: "German" }, { value: "es", label: "Spanish" }, { value: "ru", label: "Russian" },