diff --git a/TODO.md b/TODO.md index cff6e3cd87..17001948de 100644 --- a/TODO.md +++ b/TODO.md @@ -9,13 +9,14 @@ Audio Samples Tokenizing - Plurals - - Decimal numbers - Times (9 p.m. ET) Parsing - (Everything in tokenizing) + - Output a structured form of data with breath/pauses + (periods, commas, etc...) - CamelCase - - hyphenated-words + - Decimal numbers - Monetary units ($10 -> ten dollars, $10 million -> ten million dollars) - Dates - Numbers that are years (2016)! @@ -28,7 +29,7 @@ Synthesis Cleanup - Refactor into a very well-defined "SpeakRequest" with all - parameters, including new ones like "is_twitter", + parameters, including new ones like "is_twitter", "missing_word_mode = slience, record_pop, umm", etc. - Remove old query parameters - Set default volume diff --git a/src/lang/parser.rs b/src/lang/parser.rs index 7308c7e0da..7030727e69 100644 --- a/src/lang/parser.rs +++ b/src/lang/parser.rs @@ -4,6 +4,7 @@ // (The synthesizer only has knowledge of how to handle filtered words.) use lang::abbr::AbbreviationsMap; +use lang::dictionary::UniversalDictionary; use lang::numbers::number_to_words; use lang::ordinals::ordinal_to_words; use lang::token::*; @@ -13,15 +14,18 @@ use std::sync::Arc; pub struct Parser { tokenizer: Tokenizer, + dictionary: Arc, abbreviations: Arc, } impl Parser { /// CTOR. - pub fn new(tokenizer: Tokenizer, abbreviations: Arc) - -> Parser { + pub fn new(tokenizer: Tokenizer, + dictionary: Arc, + abbreviations: Arc) -> Parser { Parser { tokenizer: tokenizer, + dictionary: dictionary, abbreviations: abbreviations, } } @@ -38,7 +42,6 @@ impl Parser { Token::Date { value: _v } => {}, // Skip (for now) Token::Emoji { value: _v } => {}, // Skip (for now) Token::Hashtag { value: _v } => {}, // Skip (for now) - Token::HyphenatedString { value: _v } => {}, // Skip (for now) Token::Mention { value: _v } => {}, // Skip (for now) Token::Punctuation { value: _v } => {}, // Skip (for now) Token::Url { value: _v } => {}, // Skip (forever) @@ -71,9 +74,7 @@ impl Parser { match self.abbreviations.get_words(&v.value) { None => { continue; }, Some(words) => { - for word in words { - sentence.push(word.to_string()); - } + for word in words { sentence.push(word.to_string()); } } } }, @@ -84,6 +85,16 @@ impl Parser { sentence.push(letter.to_string()); } }, + Token::HyphenatedString { value: ref v } => { + match self.try_expand_hypenated(&v.value) { + None => { + sentence.push(v.value.to_string()); + }, + Some(words) => { + for word in words { sentence.push(word.to_string()); } + } + } + }, Token::Symbol { value : ref v } => { match *v { Symbol::Ampersand => sentence.push("and".to_string()), @@ -110,6 +121,21 @@ impl Parser { final_sentence } + + /// Attempt to expand a hyphenated non-dictionary word into a list + /// of dictionary words. If any of the words don't match, this fails. + fn try_expand_hypenated(&self, hyphenated: &str) -> Option> { + let split : Vec<&str> = hyphenated.split("-").collect(); + let mut words = Vec::new(); + for word in split { + let lower = word.to_lowercase(); + if !self.dictionary.contains(&lower) { + return None; + } + words.push(lower); + } + Some(words) + } } #[cfg(test)] @@ -138,6 +164,7 @@ mod tests { w.insert("be".to_string()); w.insert("can't".to_string()); w.insert("echelon".to_string()); + w.insert("ending".to_string()); w.insert("five".to_string()); w.insert("foo".to_string()); w.insert("four".to_string()); @@ -150,11 +177,15 @@ mod tests { w.insert("in".to_string()); w.insert("it".to_string()); w.insert("it's".to_string()); + w.insert("jack".to_string()); w.insert("join".to_string()); + w.insert("lantern".to_string()); w.insert("link".to_string()); w.insert("lot".to_string()); w.insert("me".to_string()); w.insert("movement".to_string()); + w.insert("never".to_string()); + w.insert("o".to_string()); w.insert("of".to_string()); w.insert("one".to_string()); w.insert("people".to_string()); @@ -217,6 +248,29 @@ mod tests { // TODO: WAAAY MORE TESTS. } + #[test] + fn test_hyphenated_expansion() { + let p = make_parser(); + let s = Speaker::new("speaker".to_string()); + + let result = &p.parse(&s, "never-ending"); + let expected = "never ending"; + assert_eq!(expected, result); + + let result = &p.parse(&s, "jack-o-lantern"); + let expected = "jack o lantern"; + assert_eq!(expected, result); + + // A partially unknown hyphenation + let result = &p.parse(&s, "jack-o-qwerty"); + let expected = "jack-o-qwerty"; + assert_eq!(expected, result); + + // An entirely unknown hyphenation + let result = &p.parse(&s, "qwerty-uiop"); + let expected = "qwerty-uiop"; + assert_eq!(expected, result); + } // Helper function. fn make_dictionary() -> UniversalDictionary { @@ -229,11 +283,11 @@ mod tests { // Helper function. fn make_parser() -> Parser { - let dictionary = make_dictionary(); + let dictionary = Arc::new(make_dictionary()); let abbreviations = Arc::new(AbbreviationsMap::empty()); - let tokenizer = Tokenizer::new(Arc::new(dictionary), + let tokenizer = Tokenizer::new(dictionary.clone(), abbreviations.clone()); - Parser::new(tokenizer, abbreviations.clone()) + Parser::new(tokenizer, dictionary.clone(), abbreviations.clone()) } } diff --git a/src/main.rs b/src/main.rs index 9cea4aa7a0..8e852d835c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -141,12 +141,14 @@ fn create_synthesizer(config: &Config) -> Synthesizer { let mut dictionary = UniversalDictionary::new(); dictionary.set_arpabet_dictionary(arpabet.to_dictionary()); + let dictionary = Arc::new(dictionary); + info!("Reading Abbreviations Map..."); let abbreviations = Arc::new(AbbreviationsMap::load_from_file( &config.abbreviation_file.clone().unwrap()).unwrap()); - let tokenizer = Tokenizer::new(Arc::new(dictionary), abbreviations.clone()); - let parser = Parser::new(tokenizer, abbreviations.clone()); + let tokenizer = Tokenizer::new(dictionary.clone(), abbreviations.clone()); + let parser = Parser::new(tokenizer, dictionary.clone(), abbreviations.clone()); info!("Building Synthesizer..."); let audiobank = Audiobank::new(&config.sound_path.clone().unwrap());