feat: language-aware filler word removal (#971)
* Remove filler words that conflict with European languages Removed three filler words from the transcription filter that are actual words in European languages: - 'um' - Portuguese/Spanish indefinite article meaning 'a/an' (masculine) Example: Portuguese 'Isto é um teste' (This is a test) Removing this was breaking Portuguese transcriptions - 'ha' - Spanish/Italian/Norwegian/Swedish auxiliary verb meaning 'has/have' Example: Spanish 'Él ha comido' (He has eaten) This is a very common verb form in Romance and Scandinavian languages - 'eh' - Italian interjection and Canadian English discourse marker While less critical, this can appear in legitimate Italian speech The remaining filler words are primarily English vocalized hesitations that don't conflict with common words in other European languages. Updated tests to use 'uhm' instead of 'um' where needed. Fixes #941 * customize list --------- Co-authored-by: Alexander Yastrebov <yastrebov.alex@gmail.com>
This commit is contained in:
CJ Pais
GitHub
parent
f8c74e9449
commit
615b3c93bb
4 changed files with 143 additions and 36 deletions
|
|
@ -194,11 +194,40 @@ fn extract_punctuation(word: &str) -> (&str, &str) {
|
|||
(prefix, suffix)
|
||||
}
|
||||
|
||||
/// Filler words to remove from transcriptions
|
||||
const FILLER_WORDS: &[&str] = &[
|
||||
"uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha",
|
||||
"ehh",
|
||||
];
|
||||
/// Returns filler words appropriate for the given language code.
|
||||
///
|
||||
/// Some words like "um" and "ha" are real words in certain languages
|
||||
/// (e.g., Portuguese "um" = "a/an", Spanish "ha" = "has"), so we only
|
||||
/// include them as fillers for languages where they are truly fillers.
|
||||
fn get_filler_words_for_language(lang: &str) -> &'static [&'static str] {
|
||||
let base_lang = lang.split(&['-', '_'][..]).next().unwrap_or(lang);
|
||||
|
||||
match base_lang {
|
||||
"en" => &[
|
||||
"uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "hmm", "hm", "mmm", "mm", "mh", "eh",
|
||||
"ehh", "ha",
|
||||
],
|
||||
"es" => &["ehm", "mmm", "hmm", "hm"],
|
||||
"pt" => &["ahm", "hmm", "mmm", "hm"],
|
||||
"fr" => &["euh", "hmm", "hm", "mmm"],
|
||||
"de" => &["äh", "ähm", "hmm", "hm", "mmm"],
|
||||
"it" => &["ehm", "hmm", "mmm", "hm"],
|
||||
"cs" => &["ehm", "hmm", "mmm", "hm"],
|
||||
"pl" => &["hmm", "mmm", "hm"],
|
||||
"tr" => &["hmm", "mmm", "hm"],
|
||||
"ru" => &["хм", "ммм", "hmm", "mmm"],
|
||||
"uk" => &["хм", "ммм", "hmm", "mmm"],
|
||||
"ar" => &["hmm", "mmm"],
|
||||
"ja" => &["hmm", "mmm"],
|
||||
"ko" => &["hmm", "mmm"],
|
||||
"vi" => &["hmm", "mmm", "hm"],
|
||||
"zh" => &["hmm", "mmm"],
|
||||
// Conservative universal fallback (no "um", "eh", "ha")
|
||||
_ => &[
|
||||
"uh", "uhm", "umm", "uhh", "uhhh", "ah", "hmm", "hm", "mmm", "mm", "mh", "ehh",
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
static MULTI_SPACE_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r"\s{2,}").unwrap());
|
||||
|
||||
|
|
@ -242,34 +271,42 @@ fn collapse_stutters(text: &str) -> String {
|
|||
result.join(" ")
|
||||
}
|
||||
|
||||
/// Pre-compiled filler word patterns (built lazily)
|
||||
static FILLER_PATTERNS: Lazy<Vec<Regex>> = Lazy::new(|| {
|
||||
FILLER_WORDS
|
||||
.iter()
|
||||
.map(|word| {
|
||||
// Match filler word with word boundaries, optionally followed by comma or period
|
||||
Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).unwrap()
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
/// Filters transcription output by removing filler words and stutter artifacts.
|
||||
///
|
||||
/// This function cleans up raw transcription text by:
|
||||
/// 1. Removing filler words (uh, um, hmm, etc.)
|
||||
/// 1. Removing filler words based on the app language (or custom list)
|
||||
/// 2. Collapsing repeated 1-2 letter stutters (e.g., "wh wh wh" -> "wh")
|
||||
/// 3. Cleaning up excess whitespace
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `text` - The raw transcription text to filter
|
||||
/// * `lang` - The app language code (e.g., "en", "pt-BR") used to select filler words
|
||||
/// * `custom_filler_words` - Optional user-provided filler word list. `Some(vec)` overrides
|
||||
/// language defaults; `Some(empty vec)` disables filtering; `None` uses language defaults.
|
||||
///
|
||||
/// # Returns
|
||||
/// The filtered text with filler words and stutters removed
|
||||
pub fn filter_transcription_output(text: &str) -> String {
|
||||
pub fn filter_transcription_output(
|
||||
text: &str,
|
||||
lang: &str,
|
||||
custom_filler_words: &Option<Vec<String>>,
|
||||
) -> String {
|
||||
let mut filtered = text.to_string();
|
||||
|
||||
// Build filler patterns from custom list or language defaults
|
||||
let patterns: Vec<Regex> = match custom_filler_words {
|
||||
Some(words) => words
|
||||
.iter()
|
||||
.filter_map(|word| Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).ok())
|
||||
.collect(),
|
||||
None => get_filler_words_for_language(lang)
|
||||
.iter()
|
||||
.map(|word| Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).unwrap())
|
||||
.collect(),
|
||||
};
|
||||
|
||||
// Remove filler words
|
||||
for pattern in FILLER_PATTERNS.iter() {
|
||||
for pattern in &patterns {
|
||||
filtered = pattern.replace_all(&filtered, "").to_string();
|
||||
}
|
||||
|
||||
|
|
@ -327,81 +364,144 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn test_filter_filler_words() {
|
||||
let text = "So um I was thinking uh about this";
|
||||
let result = filter_transcription_output(text);
|
||||
let text = "So uhm I was thinking uh about this";
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "So I was thinking about this");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_filler_words_case_insensitive() {
|
||||
let text = "UM this is UH a test";
|
||||
let result = filter_transcription_output(text);
|
||||
let text = "UHM this is UH a test";
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "this is a test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_filler_words_with_punctuation() {
|
||||
let text = "Well, um, I think, uh. that's right";
|
||||
let result = filter_transcription_output(text);
|
||||
let text = "Well, uhm, I think, uh. that's right";
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "Well, I think, that's right");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_cleans_whitespace() {
|
||||
let text = "Hello world test";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "Hello world test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_trims() {
|
||||
let text = " Hello world ";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "Hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_combined() {
|
||||
let text = " Um, so I was, uh, thinking about this ";
|
||||
let result = filter_transcription_output(text);
|
||||
let text = " Uhm, so I was, uh, thinking about this ";
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "so I was, thinking about this");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_preserves_valid_text() {
|
||||
let text = "This is a completely normal sentence.";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "This is a completely normal sentence.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_stutter_collapse() {
|
||||
let text = "w wh wh wh wh wh wh wh wh wh why";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "w wh why");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_stutter_short_words() {
|
||||
let text = "I I I I think so so so so";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "I think so");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_stutter_mixed_case() {
|
||||
let text = "No NO no NO no";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "No");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_stutter_preserves_two_repetitions() {
|
||||
let text = "no no is fine";
|
||||
let result = filter_transcription_output(text);
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "no no is fine");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_english_removes_um() {
|
||||
let text = "um I think um this is good";
|
||||
let result = filter_transcription_output(text, "en", &None);
|
||||
assert_eq!(result, "I think this is good");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_portuguese_preserves_um() {
|
||||
// "um" means "a/an" in Portuguese
|
||||
let text = "um gato bonito";
|
||||
let result = filter_transcription_output(text, "pt", &None);
|
||||
assert_eq!(result, "um gato bonito");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_spanish_preserves_ha() {
|
||||
// "ha" means "has" in Spanish
|
||||
let text = "ha sido un buen día";
|
||||
let result = filter_transcription_output(text, "es", &None);
|
||||
assert_eq!(result, "ha sido un buen día");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_language_code_with_region() {
|
||||
// "pt-BR" should normalize to "pt"
|
||||
let text = "um gato bonito";
|
||||
let result = filter_transcription_output(text, "pt-BR", &None);
|
||||
assert_eq!(result, "um gato bonito");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_custom_filler_words_override() {
|
||||
let custom = Some(vec!["okay".to_string(), "right".to_string()]);
|
||||
let text = "okay so I think right this works";
|
||||
let result = filter_transcription_output(text, "en", &custom);
|
||||
assert_eq!(result, "so I think this works");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_custom_filler_words_empty_disables() {
|
||||
let custom = Some(vec![]);
|
||||
let text = "So uhm I was thinking uh about this";
|
||||
let result = filter_transcription_output(text, "en", &custom);
|
||||
// No filler words removed since custom list is empty
|
||||
assert_eq!(result, "So uhm I was thinking uh about this");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_unknown_language_uses_fallback() {
|
||||
let text = "uh I think uhm this works";
|
||||
let result = filter_transcription_output(text, "xx", &None);
|
||||
assert_eq!(result, "I think this works");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_filter_fallback_does_not_remove_um() {
|
||||
// Fallback (unknown language) should not remove "um" since it's a real word in some languages
|
||||
let text = "um I think this works";
|
||||
let result = filter_transcription_output(text, "xx", &None);
|
||||
assert_eq!(result, "um I think this works");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_apply_custom_words_ngram_two_words() {
|
||||
let text = "il cui nome è Charge B, che permette";
|
||||
|
|
|
|||
|
|
@ -614,7 +614,11 @@ impl TranscriptionManager {
|
|||
};
|
||||
|
||||
// Filter out filler words and hallucinations
|
||||
let filtered_result = filter_transcription_output(&corrected_result);
|
||||
let filtered_result = filter_transcription_output(
|
||||
&corrected_result,
|
||||
&settings.app_language,
|
||||
&settings.custom_filler_words,
|
||||
);
|
||||
|
||||
let et = std::time::Instant::now();
|
||||
let translation_note = if settings.translate_to_english {
|
||||
|
|
|
|||
|
|
@ -360,6 +360,8 @@ pub struct AppSettings {
|
|||
#[serde(default = "default_typing_tool")]
|
||||
pub typing_tool: TypingTool,
|
||||
pub external_script_path: Option<String>,
|
||||
#[serde(default)]
|
||||
pub custom_filler_words: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn default_model() -> String {
|
||||
|
|
@ -724,6 +726,7 @@ pub fn get_default_settings() -> AppSettings {
|
|||
paste_delay_ms: default_paste_delay_ms(),
|
||||
typing_tool: default_typing_tool(),
|
||||
external_script_path: None,
|
||||
custom_filler_words: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -738,7 +738,7 @@ async isLaptop() : Promise<Result<boolean, string>> {
|
|||
|
||||
/** user-defined types **/
|
||||
|
||||
export type AppSettings = { bindings: Partial<{ [key in string]: ShortcutBinding }>; push_to_talk: boolean; audio_feedback: boolean; audio_feedback_volume?: number; sound_theme?: SoundTheme; start_hidden?: boolean; autostart_enabled?: boolean; update_checks_enabled?: boolean; selected_model?: string; always_on_microphone?: boolean; selected_microphone?: string | null; clamshell_microphone?: string | null; selected_output_device?: string | null; translate_to_english?: boolean; selected_language?: string; overlay_position?: OverlayPosition; debug_mode?: boolean; log_level?: LogLevel; custom_words?: string[]; model_unload_timeout?: ModelUnloadTimeout; word_correction_threshold?: number; history_limit?: number; recording_retention_period?: RecordingRetentionPeriod; paste_method?: PasteMethod; clipboard_handling?: ClipboardHandling; auto_submit?: boolean; auto_submit_key?: AutoSubmitKey; post_process_enabled?: boolean; post_process_provider_id?: string; post_process_providers?: PostProcessProvider[]; post_process_api_keys?: Partial<{ [key in string]: string }>; post_process_models?: Partial<{ [key in string]: string }>; post_process_prompts?: LLMPrompt[]; post_process_selected_prompt_id?: string | null; mute_while_recording?: boolean; append_trailing_space?: boolean; app_language?: string; experimental_enabled?: boolean; keyboard_implementation?: KeyboardImplementation; show_tray_icon?: boolean; paste_delay_ms?: number; typing_tool?: TypingTool; external_script_path: string | null }
|
||||
export type AppSettings = { bindings: Partial<{ [key in string]: ShortcutBinding }>; push_to_talk: boolean; audio_feedback: boolean; audio_feedback_volume?: number; sound_theme?: SoundTheme; start_hidden?: boolean; autostart_enabled?: boolean; update_checks_enabled?: boolean; selected_model?: string; always_on_microphone?: boolean; selected_microphone?: string | null; clamshell_microphone?: string | null; selected_output_device?: string | null; translate_to_english?: boolean; selected_language?: string; overlay_position?: OverlayPosition; debug_mode?: boolean; log_level?: LogLevel; custom_words?: string[]; model_unload_timeout?: ModelUnloadTimeout; word_correction_threshold?: number; history_limit?: number; recording_retention_period?: RecordingRetentionPeriod; paste_method?: PasteMethod; clipboard_handling?: ClipboardHandling; auto_submit?: boolean; auto_submit_key?: AutoSubmitKey; post_process_enabled?: boolean; post_process_provider_id?: string; post_process_providers?: PostProcessProvider[]; post_process_api_keys?: Partial<{ [key in string]: string }>; post_process_models?: Partial<{ [key in string]: string }>; post_process_prompts?: LLMPrompt[]; post_process_selected_prompt_id?: string | null; mute_while_recording?: boolean; append_trailing_space?: boolean; app_language?: string; experimental_enabled?: boolean; keyboard_implementation?: KeyboardImplementation; show_tray_icon?: boolean; paste_delay_ms?: number; typing_tool?: TypingTool; external_script_path: string | null; custom_filler_words?: string[] | null }
|
||||
export type AudioDevice = { index: string; name: string; is_default: boolean }
|
||||
export type AutoSubmitKey = "enter" | "ctrl_enter" | "cmd_enter"
|
||||
export type BindingResponse = { success: boolean; binding: ShortcutBinding | null; error: string | null }
|
||||
|
|
|
|||
Loading…
Reference in a new issue