From 159eff2a1e9cab55073c46666ee3de40a25e1dfa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Sep 2016 00:20:10 -0400 Subject: [PATCH] Added phoneme padding params and pause synthesis --- src/effects/mod.rs | 1 + src/handlers/audio_synth_handler.rs | 104 +++++++++++++++++++++++++--- src/synthesizer.rs | 49 ++++++++++--- src/web/js/events.js | 8 ++- src/web/js/initialize.js | 10 ++- src/web/js/url.js | 64 ++++++++++++++++- web/index.html | 26 +++++-- 7 files changed, 231 insertions(+), 31 deletions(-) diff --git a/src/effects/mod.rs b/src/effects/mod.rs index 1c09e5cf5d..bfeb33078b 100644 --- a/src/effects/mod.rs +++ b/src/effects/mod.rs @@ -1,5 +1,6 @@ // Copyright (c) 2016 Brandon Thomas +pub mod pause; pub mod speed; pub mod volume; diff --git a/src/handlers/audio_synth_handler.rs b/src/handlers/audio_synth_handler.rs index 0f22ef59de..5348273c8c 100644 --- a/src/handlers/audio_synth_handler.rs +++ b/src/handlers/audio_synth_handler.rs @@ -37,9 +37,12 @@ const SPEED_PARAM : &'static str = "spd"; const USE_PHONEMES_PARAM: &'static str = "up"; const USE_WORDS_PARAM: &'static str = "uw"; const VOLUME_PARAM : &'static str = "vol"; +const MONOPHONE_PADDING_START_PARAM : &'static str = "mps"; +const MONOPHONE_PADDING_END_PARAM : &'static str = "mpe"; +const POLYPHONE_PADDING_END_PARAM : &'static str = "ppe"; /// Represents a request to this endpoint. -#[derive(Debug)] +#[derive(Clone, Debug)] struct SpeakRequest { /** The sentence to be spoken. */ pub sentence: String, @@ -58,6 +61,15 @@ struct SpeakRequest { /** Whether to use words. */ pub use_words: bool, + + /** Padding before a monophone. */ + pub monophone_padding_start: Option, + + /** Padding after a monophone. */ + pub monophone_padding_end: Option, + + /** Padding after a polyphone. */ + pub polyphone_padding_end: Option, } enum SpeakerRequestError { @@ -82,13 +94,14 @@ pub struct AudioSynthHandler { } impl SpeakRequest { - pub fn parse(http_request: &mut Request) -> Result { + pub fn parse(http_request: &mut Request) + -> Result { let sentence_error = Err(SpeakerRequestError::SentenceInvalid); let speaker_error = Err(SpeakerRequestError::SpeakerInvalid); // Get the request sentence and speaker. - // TODO: Cleanup + // TODO: OMFG WTF CLEANUP THIS GARBAGE. match http_request.get_ref::() { Err(_) => { return sentence_error; }, Ok(ref map) => { @@ -179,6 +192,69 @@ impl SpeakRequest { }, }; + let mps : Option = match map.get(MONOPHONE_PADDING_START_PARAM) { + None => { None }, + Some(list) => { + match list.get(0) { + None => { None }, + Some(s) => { + match s.trim().parse::() { + Err(_) => None, + Ok(i) => { + if i == 0 { + None + } else { + Some(i) + } + } + } + }, + } + }, + }; + + let mpe : Option = match map.get(MONOPHONE_PADDING_END_PARAM) { + None => { None }, + Some(list) => { + match list.get(0) { + None => { None }, + Some(s) => { + match s.trim().parse::() { + Err(_) => None, + Ok(i) => { + if i == 0 { + None + } else { + Some(i) + } + } + } + }, + } + }, + }; + + let ppe : Option = match map.get(POLYPHONE_PADDING_END_PARAM) { + None => { None }, + Some(list) => { + match list.get(0) { + None => { None }, + Some(s) => { + match s.trim().parse::() { + Err(_) => None, + Ok(i) => { + if i == 0 { + None + } else { + Some(i) + } + } + } + }, + } + }, + }; + Ok(SpeakRequest { sentence: sen, speaker: spk, @@ -186,6 +262,9 @@ impl SpeakRequest { speed: speed, use_phonemes: use_phonemes, use_words: use_words, + monophone_padding_start: mps, + monophone_padding_end: mpe, + polyphone_padding_end: ppe, }) }, } @@ -236,9 +315,7 @@ impl Handler for AudioSynthHandler { } } - let result = self.create_audio(&request.speaker, &request.sentence, - request.use_words, request.use_phonemes, - request.volume, request.speed); + let result = self.create_audio(request); let mime_type = "audio/wav".parse::().unwrap(); @@ -261,14 +338,19 @@ impl AudioSynthHandler { // TODO: Return errors. /// Create audio from the sentence. - fn create_audio(&self, speaker: &str, sentence: &str, use_words: bool, - use_phonemes: bool, volume: Option, - speed: Option) -> Vec { + fn create_audio(&self, request: SpeakRequest) -> Vec { match self.synthesizer.read() { Err(_) => Vec::new(), // TODO Actual error. Ok(synth) => { - let generated = synth.generate(sentence, speaker, use_words, - use_phonemes, volume, speed); + let generated = synth.generate(&request.sentence, + &request.speaker, + request.use_words, + request.use_phonemes, + request.volume, + request.speed, + request.monophone_padding_start, + request.monophone_padding_end, + request.polyphone_padding_end); match generated { Err(e) => { println!("Error synthesizing: {:?}", e); diff --git a/src/synthesizer.rs b/src/synthesizer.rs index f8715222cb..45c50194e1 100644 --- a/src/synthesizer.rs +++ b/src/synthesizer.rs @@ -2,6 +2,7 @@ use arpabet::ArpabetDictionary; use audiobank::Audiobank; +use effects::pause::generate_pause; use effects::speed::change_speed; use effects::volume::change_volume; use error::SynthError; @@ -41,10 +42,17 @@ impl Synthesizer { /** * Generate a spoken wav from text input. */ - pub fn generate(&self, sentence: &str, speaker: &str, - use_words: bool, use_phonemes: bool, - volume: Option, speed: Option) - -> Result { + pub fn generate(&self, + sentence: &str, + speaker: &str, + use_words: bool, + use_phonemes: bool, + volume: Option, + speed: Option, + monophone_padding_start: Option, + monophone_padding_end: Option, + polyphone_padding_end: Option) + -> Result { let mut words = split_sentence(sentence); @@ -71,7 +79,12 @@ impl Synthesizer { } if !word_added && use_phonemes { - word_added = self.concatenate_polyphone(&mut concatenated_waveform, speaker, word); + word_added = self.concatenate_polyphone(&mut concatenated_waveform, + speaker, + word, + monophone_padding_start, + monophone_padding_end, + polyphone_padding_end); } } @@ -115,9 +128,13 @@ impl Synthesizer { /// Concatenate a polyphone corresponding to the word to the /// waveform we're building. Returns whether or not the word was /// successfully found and concatenated. - fn concatenate_polyphone(&self, concatenated_waveform: &mut Vec, - speaker: &str, word: &str) -> bool { - + fn concatenate_polyphone(&self, + concatenated_waveform: &mut Vec, + speaker: &str, + word: &str, + monophone_padding_start: Option, + monophone_padding_end: Option, + polyphone_padding_end: Option) -> bool { let polyphone = match self.arpabet_dictionary.get_polyphone(word) { Some(p) => { p }, @@ -157,15 +174,25 @@ impl Synthesizer { continue; }, Some(waveform_data) => { + if monophone_padding_start.is_some() { + let pause = generate_pause(monophone_padding_start.unwrap()); + concatenated_waveform.extend(pause); + } + concatenated_waveform.extend(waveform_data); + + if monophone_padding_end.is_some() { + let pause = generate_pause(monophone_padding_end.unwrap()); + concatenated_waveform.extend(pause); + } }, } } // Insert space after polyphone. - match self.audiobank.get_misc("pause") { - None => {}, - Some(pause) => { concatenated_waveform.extend(pause); }, + if polyphone_padding_end.is_some() { + let pause = generate_pause(polyphone_padding_end.unwrap()); + concatenated_waveform.extend(pause); } true diff --git a/src/web/js/events.js b/src/web/js/events.js index fab57fe6f5..ea4a789c2a 100644 --- a/src/web/js/events.js +++ b/src/web/js/events.js @@ -40,17 +40,23 @@ speaker = $('select').val(), volume = $('input#vol').val(), speed = $('input#speed').val(), + mps = $('input#monophone_padding_start').val(), + mpe = $('input#monophone_padding_end').val(), + ppe = $('input#polyphone_padding_end').val(), query = encodeURIComponent(sentence), url = '/speak?v=' + speaker + '&s=' + query + '&vol=' + volume + '&spd=' + speed + + '&mps=' + mps + + '&mpe=' + mpe + + '&ppe=' + ppe + '&up=' + $('#use_phonemes').prop('checked') + '&uw=' + $('#use_words').prop('checked'); ev.preventDefault(); - Url.setState(speaker, sentence, volume, speed); + Url.setState(speaker, sentence, volume, speed, mps, mpe, ppe); Sound.play(url); return false; diff --git a/src/web/js/initialize.js b/src/web/js/initialize.js index 90ada043d7..37b450826f 100644 --- a/src/web/js/initialize.js +++ b/src/web/js/initialize.js @@ -17,6 +17,9 @@ speaker = Url.getSpeaker() || Library.getDefaultSpeaker(), volume = Url.getVolume() || Volume.DEFAULT, speed = Url.getSpeed() || 1.0, + mps = Url.getMonophonePaddingStart() || 0, + mpe = Url.getMonophonePaddingEnd() || 0, + ppe = Url.getPolyphonePaddingEnd() || 600, urlPreviousState = false; if (speaker) { @@ -28,12 +31,17 @@ urlPreviousState = true; } - $('input#vol').val(volume); + speed = parseFloat(speed).toFixed(2); $('input#speed').val(speed); + $('input#vol').val(volume); $('#use_phonemes').prop('use_phonemes', Url.getUsePhonemes()); $('#use_words').prop('use_words', Url.getUseWords()); + $('input#monophone_padding_start').val(mps); + $('input#monophone_padding_end').val(mpe); + $('input#polyphone_padding_end').val(ppe); + Library.load(function(library) { Ui.libraryLoadCallback(library); }); Sound.install(); diff --git a/src/web/js/url.js b/src/web/js/url.js index 4796bc4ab0..3a97861275 100644 --- a/src/web/js/url.js +++ b/src/web/js/url.js @@ -22,6 +22,10 @@ /** JSON key for the `use_words` param. */ _USE_WORDS_KEY: 'uw', + _MONOPHONE_PADDING_START_KEY: 'mps', + _MONOPHONE_PADDING_END_KEY: 'mpe', + _POLYPHONE_PADDING_END_KEY: 'ppe', + /** Get the sentence from the `window.location`. */ getSentence: function() { var state = this.parseState(window.location), @@ -84,16 +88,67 @@ } }, + /** Get the MPS value from the `window.location`. */ + getMonophonePaddingStart: function() { + var state = this.parseState(window.location); + if (!state || !(this._MONOPHONE_PADDING_START_KEY in state)) { + return null; + } else { + return state[this._MONOPHONE_PADDING_START_KEY]; + } + }, + + /** Get the MPE value from the `window.location`. */ + getMonophonePaddingEnd: function() { + var state = this.parseState(window.location); + if (!state || !(this._MONOPHONE_PADDING_END_KEY in state)) { + return null; + } else { + return state[this._MONOPHONE_PADDING_END_KEY]; + } + }, + + /** Get the PPE value from the `window.location`. */ + getPolyphonePaddingEnd: function() { + var state = this.parseState(window.location); + if (!state || !(this._POLYPHONE_PADDING_END_KEY in state)) { + return null; + } else { + return state[this._POLYPHONE_PADDING_END_KEY]; + } + }, + // TODO: Independent functions to set both speaker and sentence separately. /** Set the `window.history` speaker, sentence, phoneme use, and word use. */ - setState: function(speaker, rawSentence, volume, speed, usePhonemes, useWords) { - var urlHash = this.fromParams(speaker, rawSentence, volume, speed, usePhonemes, useWords); + setState: function(speaker, + rawSentence, + volume, + speed, + usePhonemes, + useWords, + monophonePaddingStart, + monophonePaddingEnd, + polyphonePaddingEnd) { + + var urlHash = this.fromParams(speaker, rawSentence, volume, speed, + usePhonemes, useWords, monophonePaddingStart, monophonePaddingEnd, + polyphonePaddingEnd); + console.log('set state, urlhash = ', urlHash); window.history.replaceState(null, null, urlHash); }, /** Encode a speaker and sentence in a URL hash. */ - fromParams: function(speaker, rawSentence, volume, speed, usePhonemes, useWords) { + fromParams: function(speaker, + rawSentence, + volume, + speed, + usePhonemes, + useWords, + monophonePaddingStart, + monophonePaddingEnd, + polyphonePaddingEnd) { + var cleanedSentence = SentenceHelper.cleanSentence(rawSentence), cleanedSpeaker = speaker, // TODO: Check against speakers. cleanedVolume = volume, // TODO: Filter invalid values. @@ -107,6 +162,9 @@ state[this._SPEED_KEY] = speed; // TODO: Filter invalid values. state[this._USE_PHONEMES_KEY] = !!usePhonemes; state[this._USE_WORDS_KEY] = !!useWords; + state[this._MONOPHONE_PADDING_START] = monophonePaddingStart; + state[this._MONOPHONE_PADDING_END] = monophonePaddingEnd; + state[this._POLYPHONE_PADDING_END] = polyphonePaddingEnd; json = JSON.stringify(state); diff --git a/web/index.html b/web/index.html index 164398b04d..ed59e823f4 100644 --- a/web/index.html +++ b/web/index.html @@ -26,14 +26,32 @@
- + - +
+
- - +
+ + +
+
+ + +
+
+ + +
+
+ + +