Handle percentages

src/lang/parser.rs

@@ -76,6 +76,7 @@ impl Parser {
           match *v {
             Symbol::Ampersand => sentence.push("and".to_string()),
             Symbol::AtSign => sentence.push("at".to_string()),
+            Symbol::Percent  => sentence.push("percent".to_string()),
             Symbol::GreaterThan => {
               sentence.push("greater".to_string());
               sentence.push("than".to_string());

src/lang/token.rs

@@ -33,6 +33,7 @@ pub enum Symbol {
   AtSign,
   GreaterThan,
   LessThan,
+  Percent,
 }
 
 #[derive(Clone, PartialEq)]
@@ -201,6 +202,10 @@ impl Token {
     Token::Symbol { value: Symbol::AtSign }
   }
 
+  pub fn percent() -> Token {
+    Token::Symbol { value: Symbol::Percent }
+  }
+
   pub fn less_than() -> Token {
     Token::Symbol { value: Symbol::LessThan }
   }

src/lang/tokenizer.rs

@@ -34,6 +34,9 @@ impl Tokenizer {
       // Note: Rust regexes do not support lookaround.
       // Designed not to match times (5:00) or URLs (http://)
       pub static ref RE_ALPHA_COLON: Regex = Regex::new(r"([A-Za-z]):([^/])").unwrap();
+      // Designed to match percents that follow digits.
+      // If they match in URL encoding, well, too bad... I don't care about that.
+      pub static ref RE_DIGIT_PERCENT: Regex = Regex::new(r"(\d+)%").unwrap();
       pub static ref RE_ANY_DOUBLE_QUOTE: Regex= Regex::new("[\"“”]").unwrap();
       pub static ref RE_ANY_SMART_SINGLE_QUOTE: Regex= Regex::new("[‘’]").unwrap();
       pub static ref RE_BEGIN_SINGLE_QUOTE: Regex = Regex::new("^'").unwrap();
@@ -59,10 +62,9 @@ impl Tokenizer {
     filtered = RE_SPACE_SINGLE_QUOTE.replace_all(&filtered, " ");
     filtered = RE_GARBAGE.replace_all(&filtered, " ");
 
-    // TODO: TEST
-    // Handle colons
-    // Designed not to match times (5:00) or URLs (http://)
-    filtered = RE_ALPHA_COLON.replace_all(&filtered, "$1 ");
+    // Regexes with match groups
+    filtered = RE_ALPHA_COLON.replace_all(&filtered, "$1 "); // TODO TEST
+    filtered = RE_DIGIT_PERCENT.replace_all(&filtered, "$1 % ");
 
     let split_spaces = split_sentence(&filtered);
     let mut tokenized = LinkedList::new();
@@ -150,6 +152,9 @@ impl Tokenizer {
       } else if unknown == "&" {
         output.push_back(Token::ampersand());
         continue;
+      } else if unknown == "%" {
+        output.push_back(Token::percent());
+        continue;
       } else if unknown == "<" {
         output.push_back(Token::less_than());
         continue;
@@ -398,6 +403,7 @@ mod tests {
         "quote",
         "sign",
         "stuff",
+        "sure",
         "testing",
         "that",
         "the",
@@ -673,6 +679,28 @@ mod tests {
     assert_eq!(expected, result);
   }
 
+  #[test]
+  fn test_percent_symbol() {
+    let t = make_tokenizer();
+
+    let result = t.tokenize("%");
+    let expected = vec![Token::percent()];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("8 %");
+    let expected = vec![n("8"), Token::percent()];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("75%");
+    let expected = vec![n("75"), Token::percent()];
+    assert_eq!(expected, result);
+
+    let result = t.tokenize("99.999% sure");
+    let expected = vec![n("99.999"), Token::percent(), w("sure")];
+    assert_eq!(expected, result);
+  }
+
+
   #[test]
   fn test_split_sentence() {
     fn sen(list: &[&str]) -> Vec<String> {
@@ -718,6 +746,12 @@ mod tests {
     Token::dictionary_word(value.to_string())
   }
 
+  // Helper function.
+  fn n(value: &str) -> Token {
+    Token::number(value.to_string())
+  }
+
+
   // Helper function.
   fn u(value: &str) -> Token {
     Token::unknown(value.to_string())