remove non-ASCII entirely

crates/lib/tts_common/src/clean_symbols.rs

@@ -149,7 +149,17 @@ pub fn clean_symbols(input_text: &str) -> String {
       })
       .collect::<Vec<&str>>();
 
-  segmented.join("")
+  let before_clean : String = segmented.join("");
+
+  before_clean.chars()
+      .filter(|segment| {
+        match segment {
+          '\x00'..='\x7f' => true, // Full ASCII range
+          //'\x01'...'\x08' | '\u{10FFFE}'...'\u{10FFFF}' => true,
+          _ => false,
+        }
+      })
+      .collect::<String>()
 }
 
 #[cfg(test)]
@@ -167,6 +177,29 @@ mod tests {
     assert_eq!(clean_symbols("one\ntwo\r\nthree    "), "one\ntwo\r\nthree    ".to_string());
   }
 
+  #[test]
+  fn assert_ascii_retained() {
+    assert_converted("This, sentence. It\nhas\nnewlines.",
+                     "This, sentence. It\nhas\nnewlines.");
+    assert_converted("12:34", "12:34");
+    assert_converted("Punctuation!?.", "Punctuation!?.");
+  }
+
+  #[test]
+  fn assert_non_handled_emoji_removed() {
+    assert_converted("👹", "");
+    assert_converted("👹👍✅", "");
+    assert_converted("✨what✨", "what");
+  }
+
+  #[test]
+  fn assert_non_handled_languages_removed() {
+    assert_converted("これはテスト", "");
+    assert_converted("これはテストdesu", "desu");
+    assert_converted("你好", "");
+    assert_converted("hello你好hello", "hellohello");
+  }
+
   #[test]
   fn filters_smart_quotes() {
     assert_eq!(clean_symbols("That’s ok"), "That's ok".to_string());