From 5fc4592369d3c7d80600700ea77f99b16dd01c13 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 3 Oct 2016 22:18:30 -0400 Subject: [PATCH] fix smart quote --- dictionary/NEEDED.md | 2 ++ src/lang/tokenizer.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/dictionary/NEEDED.md b/dictionary/NEEDED.md index 8a53577032..aa70946656 100644 --- a/dictionary/NEEDED.md +++ b/dictionary/NEEDED.md @@ -26,3 +26,5 @@ waker waluigi wanker wario +wwii +✓ diff --git a/src/lang/tokenizer.rs b/src/lang/tokenizer.rs index b524d2e97f..e540934ba8 100644 --- a/src/lang/tokenizer.rs +++ b/src/lang/tokenizer.rs @@ -49,6 +49,9 @@ impl Tokenizer { .replace("“", "") .replace("”", ""); + // Replace smart quotes. + sentence = sentence.replace("’", "'"); + // Regex removals and replacements. sentence = RE_ALPHA_COLON.replace_all(&sentence, "$1 "); sentence = RE_BEGIN_SINGLE_QUOTE.replace_all(&sentence, ""); @@ -130,6 +133,7 @@ mod tests { // Drop smart quotes, periods. assert_eq!("it would be a bad idea", &t.convert(&s, "“It would be a bad idea.”")); + assert_eq!("a lot of people can't handle it", &t.convert(&s, "A lot of people can’t handle it.")); // Complex examples taken from real tweets. assert_eq!("will be in novi michigan this friday at 5:00pm join the movement tickets available at",