diff --git a/src/audiobank.rs b/src/audiobank.rs index 763b89ee5d..280aeb1ed0 100644 --- a/src/audiobank.rs +++ b/src/audiobank.rs @@ -22,10 +22,17 @@ pub type SampleBytes = Vec; * /1-phones * /_begin (starting syllable of word) * /IY0.wav + * ... * /_end (ending syllable of word) * /IY0.wav + * ... + * /_middle (syllables only in the middle) + * ... + * /_question (syllables that end in an interrogative tone) + * ... * /IY0.wav * /IY1.wav + * ... * /2-phones * /_begin * /F_R.wav @@ -41,6 +48,18 @@ pub struct Audiobank { audio_path: PathBuf, } +#[derive(Debug)] +pub enum SamplePreference { + /// Prefer a word-opening syllable. + Begin, + /// Prefer syllable sampled from the middle of a word. + Middle, + /// Prefer syllable taken from the end of a word. + End, + /// Prefer a syllable taken from the end of a question. + EndQuestion, +} + impl Audiobank { /// CTOR. pub fn new(audio_path: &str) -> Audiobank { @@ -76,8 +95,18 @@ impl Audiobank { Some(all_samples) } - /** Get an n-phone in the form "{1st}_{2nd}_{...}_{nth}.wav". */ - pub fn get_n_phone(&self, speaker: &str, n_phone: &[String]) + /** + * Get an n-phone in the form "{1st}_{2nd}_{...}_{nth}.wav". + * - speaker: the voice to use + * - n_phone: the phoneme slice, eg ['AH1', 'NG']. + * - sample_preference: which end of a word to prefer. + * - use_ends: use special "begin"/"end" etc. n-phones. + */ + pub fn get_n_phone(&self, + speaker: &str, + n_phone: &[String], + sample_preference: SamplePreference, + use_ends: bool) -> Option { let (directory, filename) = match n_phone.len() { @@ -117,29 +146,42 @@ impl Audiobank { return None; } + // Try to obtain our sample from the end we prefer first. + if use_ends { + let mut pref_path = self.audio_path.join(format!("{}/", speaker)) + .join(directory); + + pref_path = match sample_preference { + SamplePreference::Begin => { pref_path.join("_begin") }, + SamplePreference::Middle => { pref_path.join("_middle") }, + SamplePreference::End => { pref_path.join("_end") }, + SamplePreference::EndQuestion => { pref_path.join("_question") }, + }; + + pref_path = pref_path.join(&filename); + + match self.read_wave_file(&pref_path) { + Some(p) => { + info!("Used {:?} for {:?}", sample_preference, n_phone); + return Some(p) + }, + None => {}, + } + } + let path = self.audio_path.join(format!("{}/", speaker)) .join(directory) .join(filename); - let mut reader = match WavReader::open(path) { - Err(_) => { return None; }, - Ok(reader) => reader, - }; - - // TODO: Inefficient. - let mut all_samples = Vec::new(); - let samples = reader.samples::(); - for sample in samples { - all_samples.push(sample.unwrap()); - } - - Some(all_samples) + self.read_wave_file(&path) } /** - * Get the wav data for the (speaker,phoneme), or None if it does not exist. + * Get the wav data for the (speaker,phoneme), or None if it does not + * exist. */ - pub fn get_phoneme(&self, speaker: &str, phoneme: &str) -> Option> { + pub fn get_phoneme(&self, speaker: &str, phoneme: &str) + -> Option> { if check_path(speaker).is_err() || check_path(phoneme).is_err() { return None; } @@ -295,6 +337,23 @@ impl Audiobank { let reader = try!(WavReader::open(path)); Ok(reader.spec()) } + + /// Read a waveform. + fn read_wave_file(&self, path: &PathBuf) -> Option> { + let mut reader = match WavReader::open(path) { + Err(_) => { return None; }, + Ok(reader) => reader, + }; + + // TODO: Inefficient. + let mut all_samples = Vec::new(); + let samples = reader.samples::(); + for sample in samples { + all_samples.push(sample.unwrap()); + } + + Some(all_samples) + } } fn check_path(path: &str) -> Result<(), SynthError> { diff --git a/src/handlers/audio_synth_handler.rs b/src/handlers/audio_synth_handler.rs index 3badfad52e..a9dfd5e336 100644 --- a/src/handlers/audio_synth_handler.rs +++ b/src/handlers/audio_synth_handler.rs @@ -38,6 +38,7 @@ const USE_PHONEMES_PARAM: &'static str = "up"; const USE_DIPHONES_PARAM: &'static str = "ud"; const USE_N_PHONES_PARAM: &'static str = "un"; const USE_WORDS_PARAM: &'static str = "uw"; +const USE_ENDS_PARAM: &'static str = "ue"; const VOLUME_PARAM : &'static str = "vol"; const MONOPHONE_PADDING_START_PARAM : &'static str = "mps"; const MONOPHONE_PADDING_END_PARAM : &'static str = "mpe"; @@ -70,6 +71,9 @@ struct SpeakRequest { /** Whether to use words. */ pub use_words: bool, + /** Whether to use "ends": start, end, etc. */ + pub use_ends: bool, + /** Padding before a monophone. */ pub monophone_padding_start: Option, @@ -230,6 +234,21 @@ impl SpeakRequest { }, }; + let use_ends = match map.get(USE_ENDS_PARAM) { + None => { true }, + Some(list) => { + match list.get(0) { + None => { return speaker_error; }, + Some(s) => { + match FromStr::from_str(s) { + Ok(b) => b, + Err(_) => { return speaker_error; }, + } + }, + } + }, + }; + let mps : Option = match map.get(MONOPHONE_PADDING_START_PARAM) { None => { None }, Some(list) => { @@ -302,6 +321,7 @@ impl SpeakRequest { use_diphones: use_diphones, use_n_phones: use_n_phones, use_words: use_words, + use_ends: use_ends, monophone_padding_start: mps, monophone_padding_end: mpe, polyphone_padding_end: ppe, @@ -388,6 +408,7 @@ impl AudioSynthHandler { request.use_phonemes, request.use_diphones, request.use_n_phones, + request.use_ends, request.volume, request.speed, request.monophone_padding_start, @@ -424,9 +445,10 @@ impl AudioSynthHandler { } let mut use_byte = 0u8; - if request.use_phonemes { use_byte |= (1 << 1); } - if request.use_diphones { use_byte |= (1 << 2); } - if request.use_words { use_byte |= (1 << 3); } + if request.use_phonemes { use_byte |= 1 << 1; } + if request.use_diphones { use_byte |= 1 << 2; } + if request.use_words { use_byte |= 1 << 3; } + if request.use_ends { use_byte |= 1 << 4; } println!("Use byte: {}", use_byte); diff --git a/src/synthesizer.rs b/src/synthesizer.rs index ffcad22f90..05d9b384e7 100644 --- a/src/synthesizer.rs +++ b/src/synthesizer.rs @@ -3,6 +3,7 @@ use arpabet::ArpabetDictionary; use audiobank::Audiobank; use audiobank::SampleBytes; +use audiobank::SamplePreference; use effects::pause::generate_pause; use effects::speed::change_speed; use effects::volume::change_volume; @@ -51,6 +52,7 @@ impl Synthesizer { use_phonemes: bool, use_diphones: bool, use_n_phones: bool, + use_ends: bool, volume: Option, speed: Option, monophone_padding_start: Option, @@ -89,7 +91,8 @@ impl Synthesizer { word_added = self.concatenate_n_phone(&mut concatenated_waveform, speaker, word, - polyphone_padding_end); + polyphone_padding_end, + use_ends); } if !word_added && use_phonemes { // Old phoneme algorithm @@ -261,9 +264,12 @@ impl Synthesizer { concatenated_waveform: &mut Vec, speaker: &str, word: &str, - polyphone_padding_end: Option) -> bool { + polyphone_padding_end: Option, + use_ends: bool) -> bool { - let samples = match self.get_n_phone_samples(speaker, word) { + let pref = SamplePreference::Begin; + + let samples = match self.get_n_phone_samples(speaker, word, pref, use_ends) { Some(s) => s, None => { return false; @@ -287,7 +293,11 @@ impl Synthesizer { // TODO: Make this super easy to test (and write some tests). // TODO: Make this super efficient. - fn get_n_phone_samples(&self, speaker: &str, word: &str) + fn get_n_phone_samples(&self, + speaker: &str, + word: &str, + sample_preference: SamplePreference, + use_ends: bool) -> Option> { let polyphone = match self.arpabet_dictionary.get_polyphone(word) { @@ -302,7 +312,8 @@ impl Synthesizer { Vec::with_capacity(polyphone.len()); // Use this to debug the synthesis. - let mut debug : Vec> = Vec::with_capacity(polyphone.len()); + let mut debug : Vec> = + Vec::with_capacity(polyphone.len()); for i in 0..polyphone.len() { fulfilled.push(false); @@ -314,13 +325,25 @@ impl Synthesizer { if polyphone.len() >= 4 { let range = polyphone.len() - 3; for i in 0..range { - if fulfilled[i] || fulfilled[i+1] || fulfilled[i+2] || fulfilled[i+3] { + if fulfilled[i] + || fulfilled[i+1] + || fulfilled[i+2] + || fulfilled[i+3] { continue; } let candidate_n_phone = &polyphone[i..i+4]; - let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone); + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + let phone = self.audiobank.get_n_phone(speaker, + candidate_n_phone, + sample_preference, + use_ends); if phone.is_some() { chunks[i] = phone; @@ -345,7 +368,16 @@ impl Synthesizer { let candidate_n_phone = &polyphone[i..i+3]; - let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone); + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + let phone = self.audiobank.get_n_phone(speaker, + candidate_n_phone, + sample_preference, + use_ends); if phone.is_some() { chunks[i] = phone; @@ -363,14 +395,22 @@ impl Synthesizer { if polyphone.len() >= 2 { let range = polyphone.len() - 1; for i in 0..range { - //println!("i: {}, range: {}", i, range); if fulfilled[i] || fulfilled[i+1] { continue; } let candidate_n_phone = &polyphone[i..i+2]; - let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone); + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + let phone = self.audiobank.get_n_phone(speaker, + candidate_n_phone, + sample_preference, + use_ends); if phone.is_some() { chunks[i] = phone; @@ -390,7 +430,15 @@ impl Synthesizer { continue; } - let phone = self.audiobank.get_phoneme(speaker, &polyphone[i]); + // TODO - + /*let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == polyphone.len() - 1 => { SamplePreference::Middle }, + _ => { SamplePreference::End }, + };*/ + + let phone = self.audiobank.get_phoneme(speaker, + &polyphone[i]); if phone.is_some() { chunks[i] = phone; @@ -417,7 +465,9 @@ impl Synthesizer { .filter_map(|x| x) // Woo, already Option! .collect(); - info!("Results Length: {} of {} phones", results.len(), polyphone.len()); + info!("Results Length: {} of {} phones", + results.len(), + polyphone.len()); Some(results) } diff --git a/src/web/js/events.js b/src/web/js/events.js index 25d75a090c..64ce318fc2 100644 --- a/src/web/js/events.js +++ b/src/web/js/events.js @@ -59,6 +59,7 @@ ud = $('#use_diphones').prop('checked'), un = $('#use_n_phones').prop('checked'), uw = $('#use_words').prop('checked'), + ue = $('#use_ends').prop('checked'), mps = $('input#monophone_padding_start').val(), mpe = $('input#monophone_padding_end').val(), ppe = $('input#polyphone_padding_end').val(), @@ -73,7 +74,8 @@ + '&up=' + up + '&ud=' + ud + '&un=' + un - + '&uw=' + uw; + + '&uw=' + uw + + '&ue=' + ue; ev.preventDefault(); @@ -85,6 +87,7 @@ ud, un, uw, + ue, mps, mpe, ppe); diff --git a/src/web/js/initialize.js b/src/web/js/initialize.js index 3f309fc1d0..8635beb385 100644 --- a/src/web/js/initialize.js +++ b/src/web/js/initialize.js @@ -21,6 +21,7 @@ useDiphones = Url.getUseDiphones(), useNPhones = Url.getUseNPhones(), useWords = Url.getUseWords(), + useEnds = Url.getUseEnds(), mps = Url.getMonophonePaddingStart() || 0, mpe = Url.getMonophonePaddingEnd() || 0, ppe = Url.getPolyphonePaddingEnd() || 600, @@ -30,6 +31,7 @@ if (useDiphones === null) useDiphones = true; if (useNPhones === null) useNPhones = true; if (useWords === null) useWords = true; + if (useEnds === null) useEnds = true; if (speaker) { $('select').val(speaker); // TODO: whitelist @@ -48,6 +50,7 @@ $('#use_diphones').prop('checked', useDiphones); $('#use_n_phones').prop('checked', useNPhones); $('#use_words').prop('checked', useWords); + $('#use_ends').prop('checked', useEnds); $('input#monophone_padding_start').val(mps); $('input#monophone_padding_end').val(mpe); diff --git a/src/web/js/url.js b/src/web/js/url.js index b385309232..decab58f87 100644 --- a/src/web/js/url.js +++ b/src/web/js/url.js @@ -28,6 +28,9 @@ /** JSON key for the `use_words` param. */ _USE_WORDS_KEY: 'uw', + /** JSON key for the `use_ends` param. */ + _USE_ENDS_KEY: 'ue', + _MONOPHONE_PADDING_START_KEY: 'mps', _MONOPHONE_PADDING_END_KEY: 'mpe', _POLYPHONE_PADDING_END_KEY: 'ppe', @@ -114,6 +117,16 @@ } }, + /** Get the "use ends" from the `window.location`. */ + getUseEnds: function() { + var state = this.parseState(window.location); + if (!state || !(this._USE_ENDS_KEY in state)) { + return null; + } else { + return state[this._USE_ENDS_KEY]; + } + }, + /** Get the MPS value from the `window.location`. */ getMonophonePaddingStart: function() { var state = this.parseState(window.location); @@ -154,6 +167,7 @@ useDiphones, useNPhones, useWords, + useEnds, monophonePaddingStart, monophonePaddingEnd, polyphonePaddingEnd) { @@ -166,6 +180,7 @@ useDiphones, useNPhones, useWords, + useEnds, monophonePaddingStart, monophonePaddingEnd, polyphonePaddingEnd); @@ -183,6 +198,7 @@ useDiphones, useNPhones, useWords, + useEnds, monophonePaddingStart, monophonePaddingEnd, polyphonePaddingEnd) { @@ -202,6 +218,7 @@ state[this._USE_DIPHONES_KEY] = !!useDiphones; state[this._USE_N_PHONES_KEY] = !!useNPhones; state[this._USE_WORDS_KEY] = !!useWords; + state[this._USE_ENDS_KEY] = !!useEnds; state[this._MONOPHONE_PADDING_START_KEY] = monophonePaddingStart; state[this._MONOPHONE_PADDING_END_KEY] = monophonePaddingEnd; state[this._POLYPHONE_PADDING_END_KEY] = polyphonePaddingEnd; diff --git a/web/index.html b/web/index.html index ec0737e288..60223c15bf 100644 --- a/web/index.html +++ b/web/index.html @@ -48,6 +48,11 @@ checked="checked"> +
+ + +