From 75461a49dff5793ee84e6ece01571e9036affb93 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Sat, 5 Feb 2022 22:40:04 -0500
Subject: [PATCH] new small crate to get rid of smart quotes, etc

---
 Cargo.lock                                 | 12 +++++++
 Cargo.toml                                 |  1 +
 crates/lib/tts_common/Cargo.toml           | 26 +++++++++++++++
 crates/lib/tts_common/src/clean_symbols.rs | 39 ++++++++++++++++++++++
 crates/lib/tts_common/src/lib.rs           |  1 +
 5 files changed, 79 insertions(+)
 create mode 100644 crates/lib/tts_common/Cargo.toml
 create mode 100644 crates/lib/tts_common/src/clean_symbols.rs
 create mode 100644 crates/lib/tts_common/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index 83b5ebb9d5..5dc9b98b01 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4357,6 +4357,18 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
+[[package]]
+name = "tts_common"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "container_common",
+ "log",
+ "once_cell",
+ "regex",
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "tungstenite"
 version = "0.14.0"
diff --git a/Cargo.toml b/Cargo.toml
index d25059b54f..590406f5ab 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,6 +7,7 @@ members = [
   "crates/lib/database_queries",
   "crates/lib/http_server_common",
   "crates/lib/redis_common",
+  "crates/lib/tts_common",
   "crates/lib/twitch_common",
 
   # Application 
diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml
new file mode 100644
index 0000000000..4d71f79846
--- /dev/null
+++ b/crates/lib/tts_common/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "tts_common"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "tts_common"
+path = "src/lib.rs"
+
+[dependencies]
+
+# Internal
+container_common = { path = "../container_common" }
+
+# External
+anyhow = "1.0.40"
+log = "0.4.14"
+once_cell = "1.9.0"
+regex = "1.5.4"
+unicode-segmentation = "1.7.1"
+
diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/tts_common/src/clean_symbols.rs
new file mode 100644
index 0000000000..bd261acee9
--- /dev/null
+++ b/crates/lib/tts_common/src/clean_symbols.rs
@@ -0,0 +1,39 @@
+use once_cell::sync::Lazy;
+use std::collections::HashMap;
+use log::info;
+use unicode_segmentation::UnicodeSegmentation;
+
+static REPLACEMENTS : Lazy<HashMap<String, String>> = Lazy::new(|| {
+  let mut map = HashMap::new();
+  map.insert("’".to_string(), "'".to_string());
+  map.insert("‘".to_string(), "'".to_string());
+  map.insert("“".to_string(), "\"".to_string());
+  map.insert("”".to_string(), "\"".to_string());
+  map
+});
+
+pub fn clean_symbols(input_text: &str) -> String {
+  let segmented= UnicodeSegmentation::graphemes(input_text, true)
+      .map(|segment| {
+        if let Some(replace) = REPLACEMENTS.get(segment) {
+          return replace.as_str();
+        }
+        return segment;
+      })
+      .collect::<Vec<&str>>();
+
+  segmented.join("")
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::clean_symbols::clean_symbols;
+
+  #[test]
+  fn filters_smart_quotes() {
+    assert_eq!(clean_symbols("That’s ok"), "That's ok".to_string());
+    assert_eq!(clean_symbols("That’s it’s new home"), "That's it's new home".to_string());
+    assert_eq!(clean_symbols("‘foo’"), "'foo'".to_string());
+    assert_eq!(clean_symbols("“as ‘shown’ here.”"), "\"as 'shown' here.\"".to_string());
+  }
+}
diff --git a/crates/lib/tts_common/src/lib.rs b/crates/lib/tts_common/src/lib.rs
new file mode 100644
index 0000000000..82df584c77
--- /dev/null
+++ b/crates/lib/tts_common/src/lib.rs
@@ -0,0 +1 @@
+pub mod clean_symbols;
\ No newline at end of file