diff --git a/dictionary/NEEDED.md b/dictionary/NEEDED.md index 8a53577032..aa70946656 100644 --- a/dictionary/NEEDED.md +++ b/dictionary/NEEDED.md @@ -26,3 +26,5 @@ waker waluigi wanker wario +wwii +✓ diff --git a/src/lang/tokenizer.rs b/src/lang/tokenizer.rs index b524d2e97f..e540934ba8 100644 --- a/src/lang/tokenizer.rs +++ b/src/lang/tokenizer.rs @@ -49,6 +49,9 @@ impl Tokenizer { .replace("“", "") .replace("”", ""); + // Replace smart quotes. + sentence = sentence.replace("’", "'"); + // Regex removals and replacements. sentence = RE_ALPHA_COLON.replace_all(&sentence, "$1 "); sentence = RE_BEGIN_SINGLE_QUOTE.replace_all(&sentence, ""); @@ -130,6 +133,7 @@ mod tests { // Drop smart quotes, periods. assert_eq!("it would be a bad idea", &t.convert(&s, "“It would be a bad idea.”")); + assert_eq!("a lot of people can't handle it", &t.convert(&s, "A lot of people can’t handle it.")); // Complex examples taken from real tweets. assert_eq!("will be in novi michigan this friday at 5:00pm join the movement tickets available at",