From f8c74e9449f3883fb7e40b837f2ca25351e7872d Mon Sep 17 00:00:00 2001 From: Giovanni Donisi <82442402+gdonisi@users.noreply.github.com> Date: Fri, 6 Mar 2026 13:10:41 +0100 Subject: [PATCH 1/2] Fix Italian translation (#973) --- src/i18n/locales/it/translation.json | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/i18n/locales/it/translation.json b/src/i18n/locales/it/translation.json index 229d126..26b5737 100644 --- a/src/i18n/locales/it/translation.json +++ b/src/i18n/locales/it/translation.json @@ -19,7 +19,7 @@ "onboarding": { "subtitle": "Per cominciare, scegli un modello di riconoscimento vocale", "recommended": "Raccomandato", - "download": "Scaricare", + "download": "Scarica", "downloading": "Download in corso...", "customModelDescription": "Non ufficialmente supportato", "downloadFailed": "Download fallito. Riprova.", @@ -156,7 +156,7 @@ }, "noModelsMatch": "Nessun modello corrisponde a questo filtro.", "yourModels": "Modelli scaricati", - "availableModels": "Disponibili per il download" + "availableModels": "Disponibili per il Download" }, "general": { "title": "Generale", @@ -303,7 +303,7 @@ }, "modelUnload": { "title": "Disattiva Model", - "description": "Libera automaticamente la memoria della GPU/CPU quando il modello non viene utilizzato per un certo periodo.", + "description": "Libera automaticamente la memoria della GPU/CPU quando il modello non viene utilizzato per un certo periodo", "options": { "never": "Mai", "immediately": "Immediatamente", @@ -353,9 +353,9 @@ }, "model": { "title": "Modello", - "descriptionApple": "Fornisci un limite numerico facoltativo per i token o mantenere l'impostazione predefinita sul dispositivo.", + "descriptionApple": "Fornisci un limite numerico facoltativo per i token o mantieni l'impostazione predefinita sul dispositivo.", "descriptionCustom": "Fornisci l'identificatore del modello previsto dal tuo endpoint personalizzato.", - "descriptionDefault": "Scegli un modello reso disponibile dal provider personalizzato.", + "descriptionDefault": "Scegli un modello reso disponibile dal provider selezionato.", "placeholderApple": "Apple Intelligence", "placeholderWithOptions": "Cerca o scegli un modello", "placeholderNoOptions": "Digita il nome di un modello", @@ -368,7 +368,7 @@ "title": "Prompt Selezionato", "description": "Seleziona un template per migliorare le trascrizioni o creane uno nuovo. Usa ${output} nel prompt per fare riferimento alla trascrizione." }, - "noPrompts": "Nessun prompt disponibile.", + "noPrompts": "Nessun prompt disponibile", "selectPrompt": "Scegli un prompt", "createNew": "Crea un nuovo prompt", "promptLabel": "Etichetta del Prompt", @@ -393,7 +393,7 @@ "save": "Salva la trascrizione", "unsave": "Rimuovi dai salvataggi", "delete": "Elimina elemento", - "deleteError": "Errore nell'eliminazione dell'elemento. Per favore, prova di nuovo." + "deleteError": "Errore nell'eliminazione dell'elemento. Riprova." }, "debug": { "title": "Debug", @@ -423,14 +423,14 @@ "entries": "elementi" }, "recordingRetention": { - "title": "Salvataggio delle Registrazioni", - "description": "Per quanto conservare le registrazioni audio", + "title": "Eliminazione Automatica Registrazioni", + "description": "Elimina automaticamente le registrazioni vecchie per liberare spazio", "never": "Mai", - "preserveLimit": "Conserva {{count}} Registrazioni", - "days3": "Dopo 3 Giorni", - "weeks2": "Dopo 2 Settimane", - "months3": "Dopo 3 Mesi", - "placeholder": "Seleziona periodo di salvataggio..." + "preserveLimit": "Conserva le ultime {{count}}", + "days3": "Dopo 3 giorni", + "weeks2": "Dopo 2 settimane", + "months3": "Dopo 3 mesi", + "placeholder": "Seleziona periodo di conservazione..." }, "alwaysOnMicrophone": { "label": "Microfono Sempre Attivo", From 615b3c93bb272cbf3d9d8fa3b403cffe279e0b7a Mon Sep 17 00:00:00 2001 From: CJ Pais Date: Fri, 6 Mar 2026 20:24:29 +0800 Subject: [PATCH 2/2] feat: language-aware filler word removal (#971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove filler words that conflict with European languages Removed three filler words from the transcription filter that are actual words in European languages: - 'um' - Portuguese/Spanish indefinite article meaning 'a/an' (masculine) Example: Portuguese 'Isto é um teste' (This is a test) Removing this was breaking Portuguese transcriptions - 'ha' - Spanish/Italian/Norwegian/Swedish auxiliary verb meaning 'has/have' Example: Spanish 'Él ha comido' (He has eaten) This is a very common verb form in Romance and Scandinavian languages - 'eh' - Italian interjection and Canadian English discourse marker While less critical, this can appear in legitimate Italian speech The remaining filler words are primarily English vocalized hesitations that don't conflict with common words in other European languages. Updated tests to use 'uhm' instead of 'um' where needed. Fixes #941 * customize list --------- Co-authored-by: Alexander Yastrebov --- src-tauri/src/audio_toolkit/text.rs | 168 +++++++++++++++++++----- src-tauri/src/managers/transcription.rs | 6 +- src-tauri/src/settings.rs | 3 + src/bindings.ts | 2 +- 4 files changed, 143 insertions(+), 36 deletions(-) diff --git a/src-tauri/src/audio_toolkit/text.rs b/src-tauri/src/audio_toolkit/text.rs index 0f780bc..29f43ff 100644 --- a/src-tauri/src/audio_toolkit/text.rs +++ b/src-tauri/src/audio_toolkit/text.rs @@ -194,11 +194,40 @@ fn extract_punctuation(word: &str) -> (&str, &str) { (prefix, suffix) } -/// Filler words to remove from transcriptions -const FILLER_WORDS: &[&str] = &[ - "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha", - "ehh", -]; +/// Returns filler words appropriate for the given language code. +/// +/// Some words like "um" and "ha" are real words in certain languages +/// (e.g., Portuguese "um" = "a/an", Spanish "ha" = "has"), so we only +/// include them as fillers for languages where they are truly fillers. +fn get_filler_words_for_language(lang: &str) -> &'static [&'static str] { + let base_lang = lang.split(&['-', '_'][..]).next().unwrap_or(lang); + + match base_lang { + "en" => &[ + "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "hmm", "hm", "mmm", "mm", "mh", "eh", + "ehh", "ha", + ], + "es" => &["ehm", "mmm", "hmm", "hm"], + "pt" => &["ahm", "hmm", "mmm", "hm"], + "fr" => &["euh", "hmm", "hm", "mmm"], + "de" => &["äh", "ähm", "hmm", "hm", "mmm"], + "it" => &["ehm", "hmm", "mmm", "hm"], + "cs" => &["ehm", "hmm", "mmm", "hm"], + "pl" => &["hmm", "mmm", "hm"], + "tr" => &["hmm", "mmm", "hm"], + "ru" => &["хм", "ммм", "hmm", "mmm"], + "uk" => &["хм", "ммм", "hmm", "mmm"], + "ar" => &["hmm", "mmm"], + "ja" => &["hmm", "mmm"], + "ko" => &["hmm", "mmm"], + "vi" => &["hmm", "mmm", "hm"], + "zh" => &["hmm", "mmm"], + // Conservative universal fallback (no "um", "eh", "ha") + _ => &[ + "uh", "uhm", "umm", "uhh", "uhhh", "ah", "hmm", "hm", "mmm", "mm", "mh", "ehh", + ], + } +} static MULTI_SPACE_PATTERN: Lazy = Lazy::new(|| Regex::new(r"\s{2,}").unwrap()); @@ -242,34 +271,42 @@ fn collapse_stutters(text: &str) -> String { result.join(" ") } -/// Pre-compiled filler word patterns (built lazily) -static FILLER_PATTERNS: Lazy> = Lazy::new(|| { - FILLER_WORDS - .iter() - .map(|word| { - // Match filler word with word boundaries, optionally followed by comma or period - Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).unwrap() - }) - .collect() -}); - /// Filters transcription output by removing filler words and stutter artifacts. /// /// This function cleans up raw transcription text by: -/// 1. Removing filler words (uh, um, hmm, etc.) +/// 1. Removing filler words based on the app language (or custom list) /// 2. Collapsing repeated 1-2 letter stutters (e.g., "wh wh wh" -> "wh") /// 3. Cleaning up excess whitespace /// /// # Arguments /// * `text` - The raw transcription text to filter +/// * `lang` - The app language code (e.g., "en", "pt-BR") used to select filler words +/// * `custom_filler_words` - Optional user-provided filler word list. `Some(vec)` overrides +/// language defaults; `Some(empty vec)` disables filtering; `None` uses language defaults. /// /// # Returns /// The filtered text with filler words and stutters removed -pub fn filter_transcription_output(text: &str) -> String { +pub fn filter_transcription_output( + text: &str, + lang: &str, + custom_filler_words: &Option>, +) -> String { let mut filtered = text.to_string(); + // Build filler patterns from custom list or language defaults + let patterns: Vec = match custom_filler_words { + Some(words) => words + .iter() + .filter_map(|word| Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).ok()) + .collect(), + None => get_filler_words_for_language(lang) + .iter() + .map(|word| Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).unwrap()) + .collect(), + }; + // Remove filler words - for pattern in FILLER_PATTERNS.iter() { + for pattern in &patterns { filtered = pattern.replace_all(&filtered, "").to_string(); } @@ -327,81 +364,144 @@ mod tests { #[test] fn test_filter_filler_words() { - let text = "So um I was thinking uh about this"; - let result = filter_transcription_output(text); + let text = "So uhm I was thinking uh about this"; + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "So I was thinking about this"); } #[test] fn test_filter_filler_words_case_insensitive() { - let text = "UM this is UH a test"; - let result = filter_transcription_output(text); + let text = "UHM this is UH a test"; + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "this is a test"); } #[test] fn test_filter_filler_words_with_punctuation() { - let text = "Well, um, I think, uh. that's right"; - let result = filter_transcription_output(text); + let text = "Well, uhm, I think, uh. that's right"; + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "Well, I think, that's right"); } #[test] fn test_filter_cleans_whitespace() { let text = "Hello world test"; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "Hello world test"); } #[test] fn test_filter_trims() { let text = " Hello world "; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "Hello world"); } #[test] fn test_filter_combined() { - let text = " Um, so I was, uh, thinking about this "; - let result = filter_transcription_output(text); + let text = " Uhm, so I was, uh, thinking about this "; + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "so I was, thinking about this"); } #[test] fn test_filter_preserves_valid_text() { let text = "This is a completely normal sentence."; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "This is a completely normal sentence."); } #[test] fn test_filter_stutter_collapse() { let text = "w wh wh wh wh wh wh wh wh wh why"; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "w wh why"); } #[test] fn test_filter_stutter_short_words() { let text = "I I I I think so so so so"; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "I think so"); } #[test] fn test_filter_stutter_mixed_case() { let text = "No NO no NO no"; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "No"); } #[test] fn test_filter_stutter_preserves_two_repetitions() { let text = "no no is fine"; - let result = filter_transcription_output(text); + let result = filter_transcription_output(text, "en", &None); assert_eq!(result, "no no is fine"); } + #[test] + fn test_filter_english_removes_um() { + let text = "um I think um this is good"; + let result = filter_transcription_output(text, "en", &None); + assert_eq!(result, "I think this is good"); + } + + #[test] + fn test_filter_portuguese_preserves_um() { + // "um" means "a/an" in Portuguese + let text = "um gato bonito"; + let result = filter_transcription_output(text, "pt", &None); + assert_eq!(result, "um gato bonito"); + } + + #[test] + fn test_filter_spanish_preserves_ha() { + // "ha" means "has" in Spanish + let text = "ha sido un buen día"; + let result = filter_transcription_output(text, "es", &None); + assert_eq!(result, "ha sido un buen día"); + } + + #[test] + fn test_filter_language_code_with_region() { + // "pt-BR" should normalize to "pt" + let text = "um gato bonito"; + let result = filter_transcription_output(text, "pt-BR", &None); + assert_eq!(result, "um gato bonito"); + } + + #[test] + fn test_filter_custom_filler_words_override() { + let custom = Some(vec!["okay".to_string(), "right".to_string()]); + let text = "okay so I think right this works"; + let result = filter_transcription_output(text, "en", &custom); + assert_eq!(result, "so I think this works"); + } + + #[test] + fn test_filter_custom_filler_words_empty_disables() { + let custom = Some(vec![]); + let text = "So uhm I was thinking uh about this"; + let result = filter_transcription_output(text, "en", &custom); + // No filler words removed since custom list is empty + assert_eq!(result, "So uhm I was thinking uh about this"); + } + + #[test] + fn test_filter_unknown_language_uses_fallback() { + let text = "uh I think uhm this works"; + let result = filter_transcription_output(text, "xx", &None); + assert_eq!(result, "I think this works"); + } + + #[test] + fn test_filter_fallback_does_not_remove_um() { + // Fallback (unknown language) should not remove "um" since it's a real word in some languages + let text = "um I think this works"; + let result = filter_transcription_output(text, "xx", &None); + assert_eq!(result, "um I think this works"); + } + #[test] fn test_apply_custom_words_ngram_two_words() { let text = "il cui nome è Charge B, che permette"; diff --git a/src-tauri/src/managers/transcription.rs b/src-tauri/src/managers/transcription.rs index e9a3311..e9f5484 100644 --- a/src-tauri/src/managers/transcription.rs +++ b/src-tauri/src/managers/transcription.rs @@ -614,7 +614,11 @@ impl TranscriptionManager { }; // Filter out filler words and hallucinations - let filtered_result = filter_transcription_output(&corrected_result); + let filtered_result = filter_transcription_output( + &corrected_result, + &settings.app_language, + &settings.custom_filler_words, + ); let et = std::time::Instant::now(); let translation_note = if settings.translate_to_english { diff --git a/src-tauri/src/settings.rs b/src-tauri/src/settings.rs index 430ec76..7e242eb 100644 --- a/src-tauri/src/settings.rs +++ b/src-tauri/src/settings.rs @@ -360,6 +360,8 @@ pub struct AppSettings { #[serde(default = "default_typing_tool")] pub typing_tool: TypingTool, pub external_script_path: Option, + #[serde(default)] + pub custom_filler_words: Option>, } fn default_model() -> String { @@ -724,6 +726,7 @@ pub fn get_default_settings() -> AppSettings { paste_delay_ms: default_paste_delay_ms(), typing_tool: default_typing_tool(), external_script_path: None, + custom_filler_words: None, } } diff --git a/src/bindings.ts b/src/bindings.ts index 0a9edce..b4d1bbc 100644 --- a/src/bindings.ts +++ b/src/bindings.ts @@ -738,7 +738,7 @@ async isLaptop() : Promise> { /** user-defined types **/ -export type AppSettings = { bindings: Partial<{ [key in string]: ShortcutBinding }>; push_to_talk: boolean; audio_feedback: boolean; audio_feedback_volume?: number; sound_theme?: SoundTheme; start_hidden?: boolean; autostart_enabled?: boolean; update_checks_enabled?: boolean; selected_model?: string; always_on_microphone?: boolean; selected_microphone?: string | null; clamshell_microphone?: string | null; selected_output_device?: string | null; translate_to_english?: boolean; selected_language?: string; overlay_position?: OverlayPosition; debug_mode?: boolean; log_level?: LogLevel; custom_words?: string[]; model_unload_timeout?: ModelUnloadTimeout; word_correction_threshold?: number; history_limit?: number; recording_retention_period?: RecordingRetentionPeriod; paste_method?: PasteMethod; clipboard_handling?: ClipboardHandling; auto_submit?: boolean; auto_submit_key?: AutoSubmitKey; post_process_enabled?: boolean; post_process_provider_id?: string; post_process_providers?: PostProcessProvider[]; post_process_api_keys?: Partial<{ [key in string]: string }>; post_process_models?: Partial<{ [key in string]: string }>; post_process_prompts?: LLMPrompt[]; post_process_selected_prompt_id?: string | null; mute_while_recording?: boolean; append_trailing_space?: boolean; app_language?: string; experimental_enabled?: boolean; keyboard_implementation?: KeyboardImplementation; show_tray_icon?: boolean; paste_delay_ms?: number; typing_tool?: TypingTool; external_script_path: string | null } +export type AppSettings = { bindings: Partial<{ [key in string]: ShortcutBinding }>; push_to_talk: boolean; audio_feedback: boolean; audio_feedback_volume?: number; sound_theme?: SoundTheme; start_hidden?: boolean; autostart_enabled?: boolean; update_checks_enabled?: boolean; selected_model?: string; always_on_microphone?: boolean; selected_microphone?: string | null; clamshell_microphone?: string | null; selected_output_device?: string | null; translate_to_english?: boolean; selected_language?: string; overlay_position?: OverlayPosition; debug_mode?: boolean; log_level?: LogLevel; custom_words?: string[]; model_unload_timeout?: ModelUnloadTimeout; word_correction_threshold?: number; history_limit?: number; recording_retention_period?: RecordingRetentionPeriod; paste_method?: PasteMethod; clipboard_handling?: ClipboardHandling; auto_submit?: boolean; auto_submit_key?: AutoSubmitKey; post_process_enabled?: boolean; post_process_provider_id?: string; post_process_providers?: PostProcessProvider[]; post_process_api_keys?: Partial<{ [key in string]: string }>; post_process_models?: Partial<{ [key in string]: string }>; post_process_prompts?: LLMPrompt[]; post_process_selected_prompt_id?: string | null; mute_while_recording?: boolean; append_trailing_space?: boolean; app_language?: string; experimental_enabled?: boolean; keyboard_implementation?: KeyboardImplementation; show_tray_icon?: boolean; paste_delay_ms?: number; typing_tool?: TypingTool; external_script_path: string | null; custom_filler_words?: string[] | null } export type AudioDevice = { index: string; name: string; is_default: boolean } export type AutoSubmitKey = "enter" | "ctrl_enter" | "cmd_enter" export type BindingResponse = { success: boolean; binding: ShortcutBinding | null; error: string | null }