use natural::phonetics::soundex; use once_cell::sync::Lazy; use regex::Regex; use strsim::levenshtein; /// Applies custom word corrections to transcribed text using fuzzy matching /// /// This function corrects words in the input text by finding the best matches /// from a list of custom words using a combination of: /// - Levenshtein distance for string similarity /// - Soundex phonetic matching for pronunciation similarity /// /// # Arguments /// * `text` - The input text to correct /// * `custom_words` - List of custom words to match against /// * `threshold` - Maximum similarity score to accept (0.0 = exact match, 1.0 = any match) /// /// # Returns /// The corrected text with custom words applied pub fn apply_custom_words(text: &str, custom_words: &[String], threshold: f64) -> String { if custom_words.is_empty() { return text.to_string(); } // Pre-compute lowercase versions to avoid repeated allocations let custom_words_lower: Vec = custom_words.iter().map(|w| w.to_lowercase()).collect(); let words: Vec<&str> = text.split_whitespace().collect(); let mut corrected_words = Vec::new(); for word in words { let cleaned_word = word .trim_matches(|c: char| !c.is_alphabetic()) .to_lowercase(); if cleaned_word.is_empty() { corrected_words.push(word.to_string()); continue; } // Skip extremely long words to avoid performance issues if cleaned_word.len() > 50 { corrected_words.push(word.to_string()); continue; } let mut best_match: Option<&String> = None; let mut best_score = f64::MAX; for (i, custom_word_lower) in custom_words_lower.iter().enumerate() { // Skip if lengths are too different (optimization) let len_diff = (cleaned_word.len() as i32 - custom_word_lower.len() as i32).abs(); if len_diff > 5 { continue; } // Calculate Levenshtein distance (normalized by length) let levenshtein_dist = levenshtein(&cleaned_word, custom_word_lower); let max_len = cleaned_word.len().max(custom_word_lower.len()) as f64; let levenshtein_score = if max_len > 0.0 { levenshtein_dist as f64 / max_len } else { 1.0 }; // Calculate phonetic similarity using Soundex let phonetic_match = soundex(&cleaned_word, custom_word_lower); // Combine scores: favor phonetic matches, but also consider string similarity let combined_score = if phonetic_match { levenshtein_score * 0.3 // Give significant boost to phonetic matches } else { levenshtein_score }; // Accept if the score is good enough (configurable threshold) if combined_score < threshold && combined_score < best_score { best_match = Some(&custom_words[i]); best_score = combined_score; } } if let Some(replacement) = best_match { // Preserve the original case pattern as much as possible let corrected = preserve_case_pattern(word, replacement); // Preserve punctuation from original word let (prefix, suffix) = extract_punctuation(word); corrected_words.push(format!("{}{}{}", prefix, corrected, suffix)); } else { corrected_words.push(word.to_string()); } } corrected_words.join(" ") } /// Preserves the case pattern of the original word when applying a replacement fn preserve_case_pattern(original: &str, replacement: &str) -> String { if original.chars().all(|c| c.is_uppercase()) { replacement.to_uppercase() } else if original.chars().next().map_or(false, |c| c.is_uppercase()) { let mut chars: Vec = replacement.chars().collect(); if let Some(first_char) = chars.get_mut(0) { *first_char = first_char.to_uppercase().next().unwrap_or(*first_char); } chars.into_iter().collect() } else { replacement.to_string() } } /// Extracts punctuation prefix and suffix from a word fn extract_punctuation(word: &str) -> (&str, &str) { let prefix_end = word.chars().take_while(|c| !c.is_alphabetic()).count(); let suffix_start = word .char_indices() .rev() .take_while(|(_, c)| !c.is_alphabetic()) .count(); let prefix = if prefix_end > 0 { &word[..prefix_end] } else { "" }; let suffix = if suffix_start > 0 { &word[word.len() - suffix_start..] } else { "" }; (prefix, suffix) } /// Filler words to remove from transcriptions const FILLER_WORDS: &[&str] = &[ "uh", "um", "uhm", "umm", "uhh", "uhhh", "ah", "eh", "hmm", "hm", "mmm", "mm", "mh", "ha", "ehh", ]; static MULTI_SPACE_PATTERN: Lazy = Lazy::new(|| Regex::new(r"\s{2,}").unwrap()); /// Collapses repeated 1-2 letter words (3+ repetitions) to a single instance. /// E.g., "wh wh wh wh" -> "wh", "I I I I" -> "I" fn collapse_stutters(text: &str) -> String { let words: Vec<&str> = text.split_whitespace().collect(); if words.is_empty() { return text.to_string(); } let mut result: Vec<&str> = Vec::new(); let mut i = 0; while i < words.len() { let word = words[i]; let word_lower = word.to_lowercase(); // Only process 1-2 letter words if word_lower.len() <= 2 && word_lower.chars().all(|c| c.is_alphabetic()) { // Count consecutive repetitions (case-insensitive) let mut count = 1; while i + count < words.len() && words[i + count].to_lowercase() == word_lower { count += 1; } // If 3+ repetitions, collapse to single instance if count >= 3 { result.push(word); i += count; } else { result.push(word); i += 1; } } else { result.push(word); i += 1; } } result.join(" ") } /// Pre-compiled filler word patterns (built lazily) static FILLER_PATTERNS: Lazy> = Lazy::new(|| { FILLER_WORDS .iter() .map(|word| { // Match filler word with word boundaries, optionally followed by comma or period Regex::new(&format!(r"(?i)\b{}\b[,.]?", regex::escape(word))).unwrap() }) .collect() }); /// Filters transcription output by removing filler words and stutter artifacts. /// /// This function cleans up raw transcription text by: /// 1. Removing filler words (uh, um, hmm, etc.) /// 2. Collapsing repeated 1-2 letter stutters (e.g., "wh wh wh" -> "wh") /// 3. Cleaning up excess whitespace /// /// # Arguments /// * `text` - The raw transcription text to filter /// /// # Returns /// The filtered text with filler words and stutters removed pub fn filter_transcription_output(text: &str) -> String { let mut filtered = text.to_string(); // Remove filler words for pattern in FILLER_PATTERNS.iter() { filtered = pattern.replace_all(&filtered, "").to_string(); } // Collapse repeated 1-2 letter words (stutter artifacts like "wh wh wh wh") filtered = collapse_stutters(&filtered); // Clean up multiple spaces to single space filtered = MULTI_SPACE_PATTERN.replace_all(&filtered, " ").to_string(); // Trim leading/trailing whitespace filtered.trim().to_string() } #[cfg(test)] mod tests { use super::*; #[test] fn test_apply_custom_words_exact_match() { let text = "hello world"; let custom_words = vec!["Hello".to_string(), "World".to_string()]; let result = apply_custom_words(text, &custom_words, 0.5); assert_eq!(result, "Hello World"); } #[test] fn test_apply_custom_words_fuzzy_match() { let text = "helo wrold"; let custom_words = vec!["hello".to_string(), "world".to_string()]; let result = apply_custom_words(text, &custom_words, 0.5); assert_eq!(result, "hello world"); } #[test] fn test_preserve_case_pattern() { assert_eq!(preserve_case_pattern("HELLO", "world"), "WORLD"); assert_eq!(preserve_case_pattern("Hello", "world"), "World"); assert_eq!(preserve_case_pattern("hello", "WORLD"), "WORLD"); } #[test] fn test_extract_punctuation() { assert_eq!(extract_punctuation("hello"), ("", "")); assert_eq!(extract_punctuation("!hello?"), ("!", "?")); assert_eq!(extract_punctuation("...hello..."), ("...", "...")); } #[test] fn test_empty_custom_words() { let text = "hello world"; let custom_words = vec![]; let result = apply_custom_words(text, &custom_words, 0.5); assert_eq!(result, "hello world"); } #[test] fn test_filter_filler_words() { let text = "So um I was thinking uh about this"; let result = filter_transcription_output(text); assert_eq!(result, "So I was thinking about this"); } #[test] fn test_filter_filler_words_case_insensitive() { let text = "UM this is UH a test"; let result = filter_transcription_output(text); assert_eq!(result, "this is a test"); } #[test] fn test_filter_filler_words_with_punctuation() { let text = "Well, um, I think, uh. that's right"; let result = filter_transcription_output(text); assert_eq!(result, "Well, I think, that's right"); } #[test] fn test_filter_cleans_whitespace() { let text = "Hello world test"; let result = filter_transcription_output(text); assert_eq!(result, "Hello world test"); } #[test] fn test_filter_trims() { let text = " Hello world "; let result = filter_transcription_output(text); assert_eq!(result, "Hello world"); } #[test] fn test_filter_combined() { let text = " Um, so I was, uh, thinking about this "; let result = filter_transcription_output(text); assert_eq!(result, "so I was, thinking about this"); } #[test] fn test_filter_preserves_valid_text() { let text = "This is a completely normal sentence."; let result = filter_transcription_output(text); assert_eq!(result, "This is a completely normal sentence."); } #[test] fn test_filter_stutter_collapse() { let text = "w wh wh wh wh wh wh wh wh wh why"; let result = filter_transcription_output(text); assert_eq!(result, "w wh why"); } #[test] fn test_filter_stutter_short_words() { let text = "I I I I think so so so so"; let result = filter_transcription_output(text); assert_eq!(result, "I think so"); } #[test] fn test_filter_stutter_mixed_case() { let text = "No NO no NO no"; let result = filter_transcription_output(text); assert_eq!(result, "No"); } #[test] fn test_filter_stutter_preserves_two_repetitions() { let text = "no no is fine"; let result = filter_transcription_output(text); assert_eq!(result, "no no is fine"); } }