diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs index 51db2ac72c..b4f6cdd0ff 100644 --- a/crates/lib/tts_common/src/clean_symbols.rs +++ b/crates/lib/tts_common/src/clean_symbols.rs @@ -149,7 +149,17 @@ pub fn clean_symbols(input_text: &str) -> String { }) .collect::>(); - segmented.join("") + let before_clean : String = segmented.join(""); + + before_clean.chars() + .filter(|segment| { + match segment { + '\x00'..='\x7f' => true, // Full ASCII range + //'\x01'...'\x08' | '\u{10FFFE}'...'\u{10FFFF}' => true, + _ => false, + } + }) + .collect::() } #[cfg(test)] @@ -167,6 +177,29 @@ mod tests { assert_eq!(clean_symbols("one\ntwo\r\nthree "), "one\ntwo\r\nthree ".to_string()); } + #[test] + fn assert_ascii_retained() { + assert_converted("This, sentence. It\nhas\nnewlines.", + "This, sentence. It\nhas\nnewlines."); + assert_converted("12:34", "12:34"); + assert_converted("Punctuation!?.", "Punctuation!?."); + } + + #[test] + fn assert_non_handled_emoji_removed() { + assert_converted("👹", ""); + assert_converted("👹👍✅", ""); + assert_converted("✨what✨", "what"); + } + + #[test] + fn assert_non_handled_languages_removed() { + assert_converted("これはテスト", ""); + assert_converted("これはテストdesu", "desu"); + assert_converted("你好", ""); + assert_converted("hello你好hello", "hellohello"); + } + #[test] fn filters_smart_quotes() { assert_eq!(clean_symbols("That’s ok"), "That's ok".to_string());