danvics/Handy
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[bugfix] remove long repeating words (#976)

Browse source 
* audio_toolkit: remove long repeating words

I've noticed a couple of times that Paraket v3 may produce
repeats of words that are not folded by Handy so I propose
to lift up word length limitation.

* Update text.rs

---------

Co-authored-by: CJ Pais <cj@cjpais.com>
This commit is contained in:
Alexander Yastrebov 2026-03-28 11:40:47 +01:00 committed by GitHub
parent b710ff27c9
commit 075fb5f78d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
src-tauri/src/audio_toolkit/text.rs
View file

@ -231,7 +231,7 @@ fn get_filler_words_for_language(lang: &str) -> &'static [&'static str] {
static MULTI_SPACE_PATTERN: Lazy<Regex> = Lazy::new(|| Regex::new(r"\s{2,}").unwrap());
/// Collapses repeated 1-2 letter words (3+ repetitions) to a single instance.
/// Collapses repeated words (3+ repetitions) to a single instance.
/// E.g., "wh wh wh wh" -> "wh", "I I I I" -> "I"
fn collapse_stutters(text: &str) -> String {
let words: Vec<&str> = text.split_whitespace().collect();
@ -246,8 +246,7 @@ fn collapse_stutters(text: &str) -> String {
let word = words[i];
let word_lower = word.to_lowercase();
// Only process 1-2 letter words
if word_lower.len() <= 2 && word_lower.chars().all(|c| c.is_alphabetic()) {
if word_lower.chars().all(|c| c.is_alphabetic()) {
// Count consecutive repetitions (case-insensitive)
let mut count = 1;
while i + count < words.len() && words[i + count].to_lowercase() == word_lower {
@ -275,7 +274,7 @@ fn collapse_stutters(text: &str) -> String {
///
/// This function cleans up raw transcription text by:
/// 1. Removing filler words based on the app language (or custom list)
/// 2. Collapsing repeated 1-2 letter stutters (e.g., "wh wh wh" -> "wh")
/// 2. Collapsing repeated word stutters (e.g., "wh wh wh" -> "wh")
/// 3. Cleaning up excess whitespace
///
/// # Arguments
@ -425,6 +424,13 @@ mod tests {
assert_eq!(result, "I think so");
}
#[test]
fn test_filter_stutter_longer_words() {
let text = "Check data doc doc doc doc documentation.";
let result = filter_transcription_output(text, "en", &None);
assert_eq!(result, "Check data doc documentation.");
}
#[test]
fn test_filter_stutter_mixed_case() {
let text = "No NO no NO no";