Currency symbol tokenizing

TODO.md

@@ -1,24 +1,28 @@
 Remaining Tasks
 ===============
 
+Audio Samples
+  - MORE SAMPLES
+  - IMPROVED SAMPLES
+  - Remove extra space between words (using parsing to insert spacing)
+  - No more missing phonemes
+
 Tokenizing
-  - Hyphenated words
-  - Monetary units - just parse the leading `$`, etc.
-    and let the numbers be parsed as numbers!
   - Plurals
   - Times (9 p.m. ET)
 
 Parsing
   - (Everything in tokenizing)
+  - CamelCase
+  - hyphenated-words
   - Dates
+  - Numbers that are years (2016)!
   - Twitter hashtags, usernames
     - CamelCase parsing
-    - conjoinedword parsing
-  - Get rid of static with "Uhhh", "Ummm" of variable length.
+    - conjoinedword parsing (difficult)
 
-Audio
-  - Remove extra space between words (using parsing to insert spacing)
-  - No more missing phonemes
+Synthesis
+  - Get rid of static with "Uhhh", "Ummm" of variable length (option).
 
 Cleanup
   - Refactor into a very well-defined "SpeakRequest" with all

src/lang/parser.rs

@@ -33,14 +33,15 @@ impl Parser {
 
     for token in tokens {
       match token {
+        Token::CamelCaseString { value: _v } => {}, // Skip (for now)
+        Token::CurrencySymbol { value: _v } => {}, // Skip (for now)
         Token::Date { value: _v } => {}, // Skip (for now)
         Token::Emoji { value: _v } => {}, // Skip (for now)
         Token::Hashtag { value: _v } => {}, // Skip (for now)
         Token::HyphenatedString { value: _v } => {}, // Skip (for now)
-        Token::CamelCaseString { value: _v } => {}, // Skip (for now)
         Token::Mention { value: _v } => {}, // Skip (for now)
         Token::Punctuation { value: _v } => {}, // Skip (for now)
-        Token::Url { value: _v } => {}, // Skip
+        Token::Url { value: _v } => {}, // Skip (forever)
         Token::DictionaryWord { value : ref v } => {
           sentence.push(v.value.to_string());
         }

src/lang/token.rs

@@ -20,6 +20,16 @@ pub struct CamelCaseString {
   pub value: String,
 }
 
+#[derive(Clone, Debug, PartialEq)]
+pub enum CurrencySymbol {
+  Bitcoin,
+  Cent,
+  Dollar,
+  Euro,
+  Pound,
+  Yen,
+}
+
 #[derive(Clone, PartialEq)]
 pub struct Date {
   pub value: String,
@@ -111,6 +121,7 @@ pub struct Url {
 pub enum Token {
   Abbreviation { value: Abbreviation },
   CamelCaseString { value: CamelCaseString },
+  CurrencySymbol { value: CurrencySymbol },
   Date { value: Date },
   DictionaryWord { value: DictionaryWord }, // The primary type.
   Emoji { value: Emoji },
@@ -131,6 +142,7 @@ impl fmt::Debug for Token {
     let val = match *self {
       Token::Abbreviation { value : ref v } => format!("Abbreviation {}", v.value),
       Token::CamelCaseString { value : ref v } => format!("CamelCase {}", v.value),
+      Token::CurrencySymbol { value : ref v } => format!("Currency {:?}", v),
       Token::Date { value : ref v } => format!("Date {}", v.value),
       Token::DictionaryWord { value : ref v } => format!("Word {}", v.value),
       Token::Emoji { value : ref v } => format!("Emoji {}", v.value),
@@ -169,10 +181,18 @@ impl Token {
     Token::Symbol { value: Symbol::Ampersand }
   }
 
+  pub fn bitcoin() -> Token {
+    Token::CurrencySymbol { value: CurrencySymbol::Bitcoin }
+  }
+
   pub fn camel(value: String) -> Token {
     Token::CamelCaseString { value: CamelCaseString { value: value } }
   }
 
+  pub fn cent() -> Token {
+    Token::CurrencySymbol { value: CurrencySymbol::Cent }
+  }
+
   pub fn comma() -> Token {
     Token::Punctuation { value: Punctuation::Comma }
   }
@@ -189,6 +209,10 @@ impl Token {
     Token::DictionaryWord { value: DictionaryWord { value: value } }
   }
 
+  pub fn dollar() -> Token {
+    Token::CurrencySymbol { value: CurrencySymbol::Dollar }
+  }
+
   pub fn ellipsis() -> Token {
     Token::Punctuation { value: Punctuation::Ellipsis }
   }
@@ -197,6 +221,10 @@ impl Token {
     Token::Emoji { value: Emoji { value: value } }
   }
 
+  pub fn euro() -> Token {
+    Token::CurrencySymbol { value: CurrencySymbol::Euro }
+  }
+
   pub fn exclamation() -> Token {
     Token::Punctuation { value: Punctuation::Exclamation }
   }
@@ -245,6 +273,10 @@ impl Token {
     Token::Symbol { value: Symbol::Plus }
   }
 
+  pub fn pound() -> Token {
+    Token::CurrencySymbol { value: CurrencySymbol::Pound }
+  }
+
   pub fn question() -> Token {
     Token::Punctuation { value: Punctuation::Question }
   }
@@ -260,5 +292,9 @@ impl Token {
   pub fn url(value: String) -> Token {
     Token::Url { value: Url { value: value } }
   }
+
+  pub fn yen() -> Token {
+    Token::CurrencySymbol { value: CurrencySymbol::Yen }
+  }
 }
 

src/lang/tokenizer.rs

@@ -34,6 +34,7 @@ impl Tokenizer {
       // Note: Rust regexes do not support lookaround.
       // Designed not to match times (5:00) or URLs (http://)
       pub static ref RE_ALPHA_COLON: Regex = Regex::new(r"([A-Za-z]):([^/])").unwrap();
+      pub static ref RE_CURRENCY: Regex = Regex::new(r"([Ƀ¢$€£₤¥])").unwrap();
       // Designed to match percents that follow digits.
       // If they match in URL encoding, well, too bad... I don't care about that.
       pub static ref RE_DIGIT_PERCENT: Regex = Regex::new(r"(\d+)%").unwrap();
@@ -64,6 +65,7 @@ impl Tokenizer {
 
     // Regexes with match groups
     filtered = RE_ALPHA_COLON.replace_all(&filtered, "$1 "); // TODO TEST
+    filtered = RE_CURRENCY.replace_all(&filtered, "$1 ");
     filtered = RE_DIGIT_PERCENT.replace_all(&filtered, "$1 % ");
 
     let split_spaces = split_sentence(&filtered);
@@ -172,6 +174,28 @@ impl Tokenizer {
         continue;
       }
 
+      // Currency Symbols
+      // FIXME: Map lookup for speedup.
+      if unknown == "Ƀ" {
+        output.push_back(Token::bitcoin());
+        continue;
+      } else if unknown == "¢" {
+        output.push_back(Token::cent());
+        continue;
+      } else if unknown == "$" {
+        output.push_back(Token::dollar());
+        continue;
+      } else if unknown == "€" {
+        output.push_back(Token::euro());
+        continue;
+      } else if unknown == "£" || unknown == "₤" {
+        output.push_back(Token::pound());
+        continue;
+      } else if unknown == "¥" {
+        output.push_back(Token::yen());
+        continue;
+      }
+
       if URL.is_match(&unknown) {
         output.push_back(Token::url(unknown.to_string()));
         continue;
@@ -819,7 +843,7 @@ mod tests {
     let result = t.tokenize("& < > + % @");
     let expected = vec![
       Token::ampersand(),
-      Token::less_than(), 
+      Token::less_than(),
       Token::greater_than(),
       Token::plus(),
       Token::percent(),
@@ -833,6 +857,52 @@ mod tests {
     // assert_eq!(expected, result);
   }
 
+  #[test]
+  fn test_currency() {
+    let t = make_tokenizer();
+
+    let result = t.tokenize("$");
+    let expected = vec![Token::dollar()];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("€");
+    let expected = vec![Token::euro()];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("Ƀ ¢ $ € £ ₤ ¥");
+    let expected = vec![
+      Token::bitcoin(),
+      Token::cent(),
+      Token::dollar(),
+      Token::euro(),
+      Token::pound(),
+      Token::pound(),
+      Token::yen(),
+    ];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("Ƀ¢$€£₤¥");
+    let expected = vec![
+      Token::bitcoin(),
+      Token::cent(),
+      Token::dollar(),
+      Token::euro(),
+      Token::pound(),
+      Token::pound(),
+      Token::yen(),
+    ];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("$99");
+    let expected = vec![Token::dollar(), n("99")];
+    assert_eq!(expected, result);
+
+    // TODO: Make this work -
+    // let result = t.tokenize("99¢");
+    // let expected = vec![n("99"), Token::cent()];
+    // assert_eq!(expected, result);
+  }
+
   #[test]
   fn test_percent_symbol() {
     let t = make_tokenizer();