diff --git a/TODO.md b/TODO.md index 1c1ab6fe92..7db984c00f 100644 --- a/TODO.md +++ b/TODO.md @@ -1,24 +1,28 @@ Remaining Tasks =============== +Audio Samples + - MORE SAMPLES + - IMPROVED SAMPLES + - Remove extra space between words (using parsing to insert spacing) + - No more missing phonemes + Tokenizing - - Hyphenated words - - Monetary units - just parse the leading `$`, etc. - and let the numbers be parsed as numbers! - Plurals - Times (9 p.m. ET) Parsing - (Everything in tokenizing) + - CamelCase + - hyphenated-words - Dates + - Numbers that are years (2016)! - Twitter hashtags, usernames - CamelCase parsing - - conjoinedword parsing - - Get rid of static with "Uhhh", "Ummm" of variable length. + - conjoinedword parsing (difficult) -Audio - - Remove extra space between words (using parsing to insert spacing) - - No more missing phonemes +Synthesis + - Get rid of static with "Uhhh", "Ummm" of variable length (option). Cleanup - Refactor into a very well-defined "SpeakRequest" with all diff --git a/src/lang/parser.rs b/src/lang/parser.rs index cfd4e2b7c1..7308c7e0da 100644 --- a/src/lang/parser.rs +++ b/src/lang/parser.rs @@ -33,14 +33,15 @@ impl Parser { for token in tokens { match token { + Token::CamelCaseString { value: _v } => {}, // Skip (for now) + Token::CurrencySymbol { value: _v } => {}, // Skip (for now) Token::Date { value: _v } => {}, // Skip (for now) Token::Emoji { value: _v } => {}, // Skip (for now) Token::Hashtag { value: _v } => {}, // Skip (for now) Token::HyphenatedString { value: _v } => {}, // Skip (for now) - Token::CamelCaseString { value: _v } => {}, // Skip (for now) Token::Mention { value: _v } => {}, // Skip (for now) Token::Punctuation { value: _v } => {}, // Skip (for now) - Token::Url { value: _v } => {}, // Skip + Token::Url { value: _v } => {}, // Skip (forever) Token::DictionaryWord { value : ref v } => { sentence.push(v.value.to_string()); } diff --git a/src/lang/token.rs b/src/lang/token.rs index 130af19b82..b558b3156d 100644 --- a/src/lang/token.rs +++ b/src/lang/token.rs @@ -20,6 +20,16 @@ pub struct CamelCaseString { pub value: String, } +#[derive(Clone, Debug, PartialEq)] +pub enum CurrencySymbol { + Bitcoin, + Cent, + Dollar, + Euro, + Pound, + Yen, +} + #[derive(Clone, PartialEq)] pub struct Date { pub value: String, @@ -111,6 +121,7 @@ pub struct Url { pub enum Token { Abbreviation { value: Abbreviation }, CamelCaseString { value: CamelCaseString }, + CurrencySymbol { value: CurrencySymbol }, Date { value: Date }, DictionaryWord { value: DictionaryWord }, // The primary type. Emoji { value: Emoji }, @@ -131,6 +142,7 @@ impl fmt::Debug for Token { let val = match *self { Token::Abbreviation { value : ref v } => format!("Abbreviation {}", v.value), Token::CamelCaseString { value : ref v } => format!("CamelCase {}", v.value), + Token::CurrencySymbol { value : ref v } => format!("Currency {:?}", v), Token::Date { value : ref v } => format!("Date {}", v.value), Token::DictionaryWord { value : ref v } => format!("Word {}", v.value), Token::Emoji { value : ref v } => format!("Emoji {}", v.value), @@ -169,10 +181,18 @@ impl Token { Token::Symbol { value: Symbol::Ampersand } } + pub fn bitcoin() -> Token { + Token::CurrencySymbol { value: CurrencySymbol::Bitcoin } + } + pub fn camel(value: String) -> Token { Token::CamelCaseString { value: CamelCaseString { value: value } } } + pub fn cent() -> Token { + Token::CurrencySymbol { value: CurrencySymbol::Cent } + } + pub fn comma() -> Token { Token::Punctuation { value: Punctuation::Comma } } @@ -189,6 +209,10 @@ impl Token { Token::DictionaryWord { value: DictionaryWord { value: value } } } + pub fn dollar() -> Token { + Token::CurrencySymbol { value: CurrencySymbol::Dollar } + } + pub fn ellipsis() -> Token { Token::Punctuation { value: Punctuation::Ellipsis } } @@ -197,6 +221,10 @@ impl Token { Token::Emoji { value: Emoji { value: value } } } + pub fn euro() -> Token { + Token::CurrencySymbol { value: CurrencySymbol::Euro } + } + pub fn exclamation() -> Token { Token::Punctuation { value: Punctuation::Exclamation } } @@ -245,6 +273,10 @@ impl Token { Token::Symbol { value: Symbol::Plus } } + pub fn pound() -> Token { + Token::CurrencySymbol { value: CurrencySymbol::Pound } + } + pub fn question() -> Token { Token::Punctuation { value: Punctuation::Question } } @@ -260,5 +292,9 @@ impl Token { pub fn url(value: String) -> Token { Token::Url { value: Url { value: value } } } + + pub fn yen() -> Token { + Token::CurrencySymbol { value: CurrencySymbol::Yen } + } } diff --git a/src/lang/tokenizer.rs b/src/lang/tokenizer.rs index be8abc408a..94ca55aedf 100644 --- a/src/lang/tokenizer.rs +++ b/src/lang/tokenizer.rs @@ -34,6 +34,7 @@ impl Tokenizer { // Note: Rust regexes do not support lookaround. // Designed not to match times (5:00) or URLs (http://) pub static ref RE_ALPHA_COLON: Regex = Regex::new(r"([A-Za-z]):([^/])").unwrap(); + pub static ref RE_CURRENCY: Regex = Regex::new(r"([Ƀ¢$€£₤¥])").unwrap(); // Designed to match percents that follow digits. // If they match in URL encoding, well, too bad... I don't care about that. pub static ref RE_DIGIT_PERCENT: Regex = Regex::new(r"(\d+)%").unwrap(); @@ -64,6 +65,7 @@ impl Tokenizer { // Regexes with match groups filtered = RE_ALPHA_COLON.replace_all(&filtered, "$1 "); // TODO TEST + filtered = RE_CURRENCY.replace_all(&filtered, "$1 "); filtered = RE_DIGIT_PERCENT.replace_all(&filtered, "$1 % "); let split_spaces = split_sentence(&filtered); @@ -172,6 +174,28 @@ impl Tokenizer { continue; } + // Currency Symbols + // FIXME: Map lookup for speedup. + if unknown == "Ƀ" { + output.push_back(Token::bitcoin()); + continue; + } else if unknown == "¢" { + output.push_back(Token::cent()); + continue; + } else if unknown == "$" { + output.push_back(Token::dollar()); + continue; + } else if unknown == "€" { + output.push_back(Token::euro()); + continue; + } else if unknown == "£" || unknown == "₤" { + output.push_back(Token::pound()); + continue; + } else if unknown == "¥" { + output.push_back(Token::yen()); + continue; + } + if URL.is_match(&unknown) { output.push_back(Token::url(unknown.to_string())); continue; @@ -819,7 +843,7 @@ mod tests { let result = t.tokenize("& < > + % @"); let expected = vec![ Token::ampersand(), - Token::less_than(), + Token::less_than(), Token::greater_than(), Token::plus(), Token::percent(), @@ -833,6 +857,52 @@ mod tests { // assert_eq!(expected, result); } + #[test] + fn test_currency() { + let t = make_tokenizer(); + + let result = t.tokenize("$"); + let expected = vec![Token::dollar()]; + assert_eq!(expected, result); + + let result = t.tokenize("€"); + let expected = vec![Token::euro()]; + assert_eq!(expected, result); + + let result = t.tokenize("Ƀ ¢ $ € £ ₤ ¥"); + let expected = vec![ + Token::bitcoin(), + Token::cent(), + Token::dollar(), + Token::euro(), + Token::pound(), + Token::pound(), + Token::yen(), + ]; + assert_eq!(expected, result); + + let result = t.tokenize("Ƀ¢$€£₤¥"); + let expected = vec![ + Token::bitcoin(), + Token::cent(), + Token::dollar(), + Token::euro(), + Token::pound(), + Token::pound(), + Token::yen(), + ]; + assert_eq!(expected, result); + + let result = t.tokenize("$99"); + let expected = vec![Token::dollar(), n("99")]; + assert_eq!(expected, result); + + // TODO: Make this work - + // let result = t.tokenize("99¢"); + // let expected = vec![n("99"), Token::cent()]; + // assert_eq!(expected, result); + } + #[test] fn test_percent_symbol() { let t = make_tokenizer();