diff --git a/src/lang/parser.rs b/src/lang/parser.rs index a460187a7b..e37a76bbb4 100644 --- a/src/lang/parser.rs +++ b/src/lang/parser.rs @@ -76,6 +76,7 @@ impl Parser { match *v { Symbol::Ampersand => sentence.push("and".to_string()), Symbol::AtSign => sentence.push("at".to_string()), + Symbol::Percent => sentence.push("percent".to_string()), Symbol::GreaterThan => { sentence.push("greater".to_string()); sentence.push("than".to_string()); diff --git a/src/lang/token.rs b/src/lang/token.rs index 780e1a2720..4d8041e3f5 100644 --- a/src/lang/token.rs +++ b/src/lang/token.rs @@ -33,6 +33,7 @@ pub enum Symbol { AtSign, GreaterThan, LessThan, + Percent, } #[derive(Clone, PartialEq)] @@ -201,6 +202,10 @@ impl Token { Token::Symbol { value: Symbol::AtSign } } + pub fn percent() -> Token { + Token::Symbol { value: Symbol::Percent } + } + pub fn less_than() -> Token { Token::Symbol { value: Symbol::LessThan } } diff --git a/src/lang/tokenizer.rs b/src/lang/tokenizer.rs index e45ab2f037..63b3e5c416 100644 --- a/src/lang/tokenizer.rs +++ b/src/lang/tokenizer.rs @@ -34,6 +34,9 @@ impl Tokenizer { // Note: Rust regexes do not support lookaround. // Designed not to match times (5:00) or URLs (http://) pub static ref RE_ALPHA_COLON: Regex = Regex::new(r"([A-Za-z]):([^/])").unwrap(); + // Designed to match percents that follow digits. + // If they match in URL encoding, well, too bad... I don't care about that. + pub static ref RE_DIGIT_PERCENT: Regex = Regex::new(r"(\d+)%").unwrap(); pub static ref RE_ANY_DOUBLE_QUOTE: Regex= Regex::new("[\"“”]").unwrap(); pub static ref RE_ANY_SMART_SINGLE_QUOTE: Regex= Regex::new("[‘’]").unwrap(); pub static ref RE_BEGIN_SINGLE_QUOTE: Regex = Regex::new("^'").unwrap(); @@ -59,10 +62,9 @@ impl Tokenizer { filtered = RE_SPACE_SINGLE_QUOTE.replace_all(&filtered, " "); filtered = RE_GARBAGE.replace_all(&filtered, " "); - // TODO: TEST - // Handle colons - // Designed not to match times (5:00) or URLs (http://) - filtered = RE_ALPHA_COLON.replace_all(&filtered, "$1 "); + // Regexes with match groups + filtered = RE_ALPHA_COLON.replace_all(&filtered, "$1 "); // TODO TEST + filtered = RE_DIGIT_PERCENT.replace_all(&filtered, "$1 % "); let split_spaces = split_sentence(&filtered); let mut tokenized = LinkedList::new(); @@ -150,6 +152,9 @@ impl Tokenizer { } else if unknown == "&" { output.push_back(Token::ampersand()); continue; + } else if unknown == "%" { + output.push_back(Token::percent()); + continue; } else if unknown == "<" { output.push_back(Token::less_than()); continue; @@ -398,6 +403,7 @@ mod tests { "quote", "sign", "stuff", + "sure", "testing", "that", "the", @@ -673,6 +679,28 @@ mod tests { assert_eq!(expected, result); } + #[test] + fn test_percent_symbol() { + let t = make_tokenizer(); + + let result = t.tokenize("%"); + let expected = vec![Token::percent()]; + assert_eq!(expected, result); + + let result = t.tokenize("8 %"); + let expected = vec![n("8"), Token::percent()]; + assert_eq!(expected, result); + + let result = t.tokenize("75%"); + let expected = vec![n("75"), Token::percent()]; + assert_eq!(expected, result); + + let result = t.tokenize("99.999% sure"); + let expected = vec![n("99.999"), Token::percent(), w("sure")]; + assert_eq!(expected, result); + } + + #[test] fn test_split_sentence() { fn sen(list: &[&str]) -> Vec { @@ -718,6 +746,12 @@ mod tests { Token::dictionary_word(value.to_string()) } + // Helper function. + fn n(value: &str) -> Token { + Token::number(value.to_string()) + } + + // Helper function. fn u(value: &str) -> Token { Token::unknown(value.to_string())