From 0ea02db27fa9f14b4e00e0d7c681cb8a35df93b3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 29 Oct 2016 23:40:47 -0400 Subject: [PATCH] Refactor of synthesis-related code - refactor and modularize code into "sentence synth" and "word synth" - change synth params in code and UI - change error types, better error handling --- .idea/dictionaries/bt.xml | 2 + frontend/old_frontend/js/events.js | 24 +- frontend/old_frontend/js/initialize.js | 21 +- frontend/old_frontend/js/url.js | 75 ++-- src/error.rs | 1 - src/handlers/audio_synth_handler.rs | 79 ++-- src/main.rs | 3 +- src/synthesis/audio.rs | 7 + src/{ => synthesis}/audiobank.rs | 21 +- src/synthesis/effects/pause.rs | 4 +- src/synthesis/error.rs | 61 +++ src/synthesis/mod.rs | 4 + src/synthesis/synthesizer.rs | 518 ++++--------------------- src/synthesis/word_synthesizer.rs | 394 +++++++++++++++++++ web/old.html | 33 +- 15 files changed, 663 insertions(+), 584 deletions(-) create mode 100644 src/synthesis/audio.rs rename src/{ => synthesis}/audiobank.rs (94%) create mode 100644 src/synthesis/error.rs create mode 100644 src/synthesis/word_synthesizer.rs diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 0a1dc801e7..a0941f2443 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -2,7 +2,9 @@ arpabet + monophones polyphone + polyphones \ No newline at end of file diff --git a/frontend/old_frontend/js/events.js b/frontend/old_frontend/js/events.js index 276776e212..9f4843cd11 100644 --- a/frontend/old_frontend/js/events.js +++ b/frontend/old_frontend/js/events.js @@ -50,18 +50,16 @@ /** Handle form submission. */ handleFormSubmit: function(ev) { - var $audio = $('#sound'), - sentence = $('#text').val(), + var sentence = $('#text').val(), speaker = $('select').val(), volume = $('input#vol').val(), speed = $('input#speed').val(), - up = $('#use_phonemes').prop('checked'), - ud = $('#use_diphones').prop('checked'), + um = $('#use_monophones').prop('checked'), un = $('#use_n_phones').prop('checked'), uw = $('#use_words').prop('checked'), ue = $('#use_ends').prop('checked'), - mps = $('input#monophone_padding_start').val(), - mpe = $('input#monophone_padding_end').val(), + pbp = $('input#padding_between_phones').val(), + pps = $('input#polyphone_padding_start').val(), ppe = $('input#polyphone_padding_end').val(), wps = $('input#word_padding_start').val(), wpe = $('input#word_padding_end').val(), @@ -70,13 +68,12 @@ + '&s=' + query + '&vol=' + volume + '&spd=' + speed - + '&mps=' + mps - + '&mpe=' + mpe + + '&pbp=' + pbp + + '&pps=' + pps + '&ppe=' + ppe + '&wps=' + wps + '&wpe=' + wpe - + '&up=' + up - + '&ud=' + ud + + '&um=' + um + '&un=' + un + '&uw=' + uw + '&ue=' + ue; @@ -87,13 +84,12 @@ sentence, volume, speed, - up, - ud, + um, un, uw, ue, - mps, - mpe, + pbp, + pps, ppe, wps, wpe); diff --git a/frontend/old_frontend/js/initialize.js b/frontend/old_frontend/js/initialize.js index 79e8c30cb4..d69b789854 100644 --- a/frontend/old_frontend/js/initialize.js +++ b/frontend/old_frontend/js/initialize.js @@ -9,7 +9,7 @@ $('input#text').on('keyup', function(ev) { return Events.handleTyping(ev); }); $('body').on('keydown', function(ev) { return Events.handleBodyTyping(ev); }); $('select').on('change', function(ev) { return Events.handleVoiceSelect(ev); }); - } + }; /** Init the UI, possibly from a state passed in from the URL. */ var uiInitialize = function() { @@ -17,20 +17,18 @@ speaker = Url.getSpeaker() || Library.getDefaultSpeaker(), volume = Url.getVolume() || Volume.DEFAULT, speed = Url.getSpeed() || 1.0, - usePhonemes = Url.getUsePhonemes(), - useDiphones = Url.getUseDiphones(), + useMonophones = Url.getUseMonophones(), useNPhones = Url.getUseNPhones(), useWords = Url.getUseWords(), useEnds = Url.getUseEnds(), - mps = Url.getMonophonePaddingStart() || 0, - mpe = Url.getMonophonePaddingEnd() || 0, + pbp = Url.getPaddingBetweenPhones() || 0, + pps = Url.getPolyphonePaddingStart() || 600, ppe = Url.getPolyphonePaddingEnd() || 600, wps = Url.getWordPaddingStart() || 600, wpe = Url.getWordPaddingEnd() || 600, urlPreviousState = false; - if (usePhonemes === null) usePhonemes = true; - if (useDiphones === null) useDiphones = true; + if (useMonophones === null) useMonophones = true; if (useNPhones === null) useNPhones = true; if (useWords === null) useWords = true; if (useEnds === null) useEnds = true; @@ -48,14 +46,13 @@ $('input#speed').val(speed); $('input#vol').val(volume); - $('#use_phonemes').prop('checked', usePhonemes); - $('#use_diphones').prop('checked', useDiphones); + $('#use_monophones').prop('checked', useMonophones); $('#use_n_phones').prop('checked', useNPhones); $('#use_words').prop('checked', useWords); $('#use_ends').prop('checked', useEnds); - $('input#monophone_padding_start').val(mps); - $('input#monophone_padding_end').val(mpe); + $('input#padding_between_phones').val(pbp); + $('input#polyphone_padding_start').val(pps); $('input#polyphone_padding_end').val(ppe); $('input#word_padding_start').val(wps); $('input#word_padding_end').val(wpe); @@ -66,7 +63,7 @@ if (urlPreviousState) { $('form').submit(); } - } + }; $(function() { installEventHandlers(); diff --git a/frontend/old_frontend/js/url.js b/frontend/old_frontend/js/url.js index 081c602f67..799c4ae6c0 100644 --- a/frontend/old_frontend/js/url.js +++ b/frontend/old_frontend/js/url.js @@ -16,11 +16,8 @@ /** JSON key for the `speed` param. */ _SPEED_KEY: 'spd', - /** JSON key for the `use_phonemes` param. */ - _USE_PHONEMES_KEY: 'up', - - /** JSON key for the `use_diphones` param. */ - _USE_DIPHONES_KEY: 'ud', + /** JSON key for the `use_monophones` param. */ + _USE_MONOPHONES_KEY: 'um', /** JSON key for the `use_n_phones` param. */ _USE_N_PHONES_KEY: 'un', @@ -31,8 +28,8 @@ /** JSON key for the `use_ends` param. */ _USE_ENDS_KEY: 'ue', - _MONOPHONE_PADDING_START_KEY: 'mps', - _MONOPHONE_PADDING_END_KEY: 'mpe', + _PADDING_BETWEEN_PHONES: 'pbp', + _POLYPHONE_PADDING_START_KEY: 'pps', _POLYPHONE_PADDING_END_KEY: 'ppe', _WORD_PADDING_START_KEY: 'wps', _WORD_PADDING_END_KEY: 'wpe', @@ -79,23 +76,13 @@ } }, - /** Get the "use phonemes" from the `window.location`. */ - getUsePhonemes: function() { + /** Get the "use monophones" from the `window.location`. */ + getUseMonophones: function() { var state = this.parseState(window.location); - if (!state || !(this._USE_PHONEMES_KEY in state)) { + if (!state || !(this._USE_MONOPHONES_KEY in state)) { return null; } else { - return state[this._USE_PHONEMES_KEY]; - } - }, - - /** Get the "use diphones" from the `window.location`. */ - getUseDiphones: function() { - var state = this.parseState(window.location); - if (!state || !(this._USE_DIPHONES_KEY in state)) { - return null; - } else { - return state[this._USE_DIPHONES_KEY]; + return state[this._USE_MONOPHONES_KEY]; } }, @@ -129,23 +116,23 @@ } }, - /** Get the MPS value from the `window.location`. */ - getMonophonePaddingStart: function() { + /** Get the PBP value from the `window.location`. */ + getPaddingBetweenPhones: function() { var state = this.parseState(window.location); - if (!state || !(this._MONOPHONE_PADDING_START_KEY in state)) { + if (!state || !(this._PADDING_BETWEEN_PHONES in state)) { return null; } else { - return state[this._MONOPHONE_PADDING_START_KEY]; + return state[this._PADDING_BETWEEN_PHONES]; } }, - /** Get the MPE value from the `window.location`. */ - getMonophonePaddingEnd: function() { + /** Get the PPS value from the `window.location`. */ + getPolyphonePaddingStart: function() { var state = this.parseState(window.location); - if (!state || !(this._MONOPHONE_PADDING_END_KEY in state)) { + if (!state || !(this._POLYPHONE_PADDING_START_KEY in state)) { return null; } else { - return state[this._MONOPHONE_PADDING_END_KEY]; + return state[this._POLYPHONE_PADDING_START_KEY]; } }, @@ -184,13 +171,12 @@ rawSentence, volume, speed, - usePhonemes, - useDiphones, + useMonophones, useNPhones, useWords, useEnds, - monophonePaddingStart, - monophonePaddingEnd, + paddingBetweenPhones, + polyphonePaddingStart, polyphonePaddingEnd, wordPaddingStart, wordPaddingEnd) { @@ -199,13 +185,12 @@ rawSentence, volume, speed, - usePhonemes, - useDiphones, + useMonophones, useNPhones, useWords, useEnds, - monophonePaddingStart, - monophonePaddingEnd, + paddingBetweenPhones, + polyphonePaddingStart, polyphonePaddingEnd, wordPaddingStart, wordPaddingEnd); @@ -219,13 +204,12 @@ rawSentence, volume, speed, - usePhonemes, - useDiphones, + useMonophones, useNPhones, useWords, useEnds, - monophonePaddingStart, - monophonePaddingEnd, + paddingBetweenPhones, + polyphonePaddingStart, polyphonePaddingEnd, wordPaddingStart, wordPaddingEnd) { @@ -234,20 +218,19 @@ cleanedSpeaker = speaker, // TODO: Check against speakers. cleanedVolume = volume, // TODO: Filter invalid values. state = {}, - json = null, + json, uriEncoded = null; state[this._SPEAKER_KEY] = cleanedSpeaker; state[this._SENTENCE_KEY] = cleanedSentence; state[this._VOLUME_KEY] = cleanedVolume; state[this._SPEED_KEY] = speed; // TODO: Filter invalid values. - state[this._USE_PHONEMES_KEY] = !!usePhonemes; - state[this._USE_DIPHONES_KEY] = !!useDiphones; + state[this._USE_MONOPHONES_KEY] = !!useMonophones; state[this._USE_N_PHONES_KEY] = !!useNPhones; state[this._USE_WORDS_KEY] = !!useWords; state[this._USE_ENDS_KEY] = !!useEnds; - state[this._MONOPHONE_PADDING_START_KEY] = monophonePaddingStart; - state[this._MONOPHONE_PADDING_END_KEY] = monophonePaddingEnd; + state[this._PADDING_BETWEEN_PHONES] = paddingBetweenPhones; + state[this._POLYPHONE_PADDING_START_KEY] = polyphonePaddingStart; state[this._POLYPHONE_PADDING_END_KEY] = polyphonePaddingEnd; state[this._WORD_PADDING_START_KEY] = wordPaddingStart; state[this._WORD_PADDING_END_KEY] = wordPaddingEnd; diff --git a/src/error.rs b/src/error.rs index 48e47cbdb0..4d96464206 100644 --- a/src/error.rs +++ b/src/error.rs @@ -41,7 +41,6 @@ impl Error for SynthError { }*/ } - impl From for SynthError { fn from(error: io::Error) -> SynthError { SynthError::IoError { cause: error } diff --git a/src/handlers/audio_synth_handler.rs b/src/handlers/audio_synth_handler.rs index 81b549a094..4aa43bbbe3 100644 --- a/src/handlers/audio_synth_handler.rs +++ b/src/handlers/audio_synth_handler.rs @@ -3,7 +3,6 @@ use config::Config; use crypto::digest::Digest; use crypto::sha1::Sha1; -use error::SynthError; use iron::Handler; use iron::IronError; use iron::Plugin; @@ -23,6 +22,7 @@ use std::str::FromStr; use std::sync::Arc; use std::sync::RwLock; use super::error_filter::build_error; +use synthesis::error::SynthesisError; use synthesis::synthesizer::SynthesisParams; use synthesis::synthesizer::Synthesizer; use time::PreciseTime; @@ -31,14 +31,14 @@ use urlencoded::UrlEncodedQuery; const SENTENCE_PARAM : &'static str = "s"; const SPEAKER_PARAM : &'static str = "v"; const SPEED_PARAM : &'static str = "spd"; -const USE_PHONEMES_PARAM: &'static str = "up"; +const USE_MONOPHONES_PARAM: &'static str = "um"; const USE_SYLLABLES_PARAM: &'static str = "us"; const USE_N_PHONES_PARAM: &'static str = "un"; const USE_WORDS_PARAM: &'static str = "uw"; const USE_ENDS_PARAM: &'static str = "ue"; const VOLUME_PARAM : &'static str = "vol"; -const MONOPHONE_PADDING_START_PARAM : &'static str = "mps"; -const MONOPHONE_PADDING_END_PARAM : &'static str = "mpe"; +const PADDING_BETWEEN_PHONES_PARAM: &'static str = "pbp"; +const POLYPHONE_PADDING_START_PARAM : &'static str = "pps"; const POLYPHONE_PADDING_END_PARAM : &'static str = "ppe"; const WORD_PADDING_START_PARAM : &'static str = "wps"; const WORD_PADDING_END_PARAM : &'static str = "wpe"; @@ -60,26 +60,26 @@ struct SpeakRequest { /** An optional speed multiplier. */ pub speed: Option, - /** Whether to use phonemes. */ - pub use_phonemes: bool, + /** Whether to use words. */ + pub use_words: bool, /** Whether to use n-phones. */ pub use_n_phones: bool, + /** Whether to use bare monophones. */ + pub use_bare_monophones: bool, + /** Whether to use syllable boundaries. */ pub use_syllables: bool, - /** Whether to use words. */ - pub use_words: bool, - /** Whether to use "ends": start, end, etc. */ pub use_ends: bool, - /** Padding before a monophone. */ - pub monophone_padding_start: Option, + /** Padding between monophones or n-phone clusters. */ + pub padding_between_phones: Option, - /** Padding after a monophone. */ - pub monophone_padding_end: Option, + /** Padding before a polyphone. */ + pub polyphone_padding_start: Option, /** Padding after a polyphone. */ pub polyphone_padding_end: Option, @@ -148,14 +148,14 @@ impl SpeakRequest { } }); - let use_phonemes = get_bool(params, USE_PHONEMES_PARAM).unwrap_or(true); + let use_monophones = get_bool(params, USE_MONOPHONES_PARAM).unwrap_or(true); let use_n_phones = get_bool(params, USE_N_PHONES_PARAM).unwrap_or(true); let use_syllables = get_bool(params, USE_SYLLABLES_PARAM).unwrap_or(true); let use_words = get_bool(params, USE_WORDS_PARAM).unwrap_or(true); let use_ends = get_bool(params, USE_ENDS_PARAM).unwrap_or(true); - let mps = get_u16(params, MONOPHONE_PADDING_START_PARAM); - let mpe = get_u16(params, MONOPHONE_PADDING_END_PARAM); + let pbp = get_u16(params, PADDING_BETWEEN_PHONES_PARAM).or(None); + let pps = get_u16(params, POLYPHONE_PADDING_START_PARAM).or(Some(600)); let ppe = get_u16(params, POLYPHONE_PADDING_END_PARAM).or(Some(600)); let wps = get_u16(params, WORD_PADDING_START_PARAM).or(Some(600)); let wpe = get_u16(params, WORD_PADDING_END_PARAM).or(Some(600)); @@ -165,13 +165,13 @@ impl SpeakRequest { speaker: spk, volume: volume, speed: speed, - use_phonemes: use_phonemes, + use_bare_monophones: use_monophones, use_n_phones: use_n_phones, use_syllables: use_syllables, use_words: use_words, use_ends: use_ends, - monophone_padding_start: mps, - monophone_padding_end: mpe, + padding_between_phones: pbp, + polyphone_padding_start: pps, polyphone_padding_end: ppe, word_padding_start: wps, word_padding_end: wpe, @@ -200,7 +200,7 @@ impl SpeakRequest { } let mut use_byte = 0u8; - if self.use_phonemes { use_byte |= 1 << 1; } + if self.use_bare_monophones { use_byte |= 1 << 1; } if self.use_n_phones { use_byte |= 1 << 2; } if self.use_syllables { use_byte |= 1 << 3; } if self.use_words { use_byte |= 1 << 4; } @@ -275,22 +275,23 @@ impl AudioSynthHandler { } /// Create audio from the sentence. - fn create_audio(&self, request: SpeakRequest) -> Result, SynthError> { + fn create_audio(&self, request: SpeakRequest) + -> Result, SynthesisError> { let synth = match self.synthesizer.read() { - Err(_) => { return Err(SynthError::LockError); }, + Err(_) => { return Err(SynthesisError::LockError); }, Ok(synth) => synth, }; let params = SynthesisParams { use_words: request.use_words, - use_phonemes: request.use_phonemes, + use_bare_monophones: request.use_bare_monophones, use_n_phones: request.use_n_phones, use_syllables: request.use_syllables, use_ends: request.use_ends, volume: request.volume, speed: request.speed, - monophone_padding_start: request.monophone_padding_start, - monophone_padding_end: request.monophone_padding_end, + padding_between_phones: request.padding_between_phones, + polyphone_padding_start: request.polyphone_padding_start, polyphone_padding_end: request.polyphone_padding_end, word_padding_start: request.word_padding_start, word_padding_end: request.word_padding_end, @@ -329,8 +330,8 @@ fn get_f32(params: &QueryParams, param_name: &str) -> Option { .and_then(|res| res.ok()) } -impl From for IronError { - fn from(error: SynthError) -> IronError { +impl From for IronError { + fn from(error: SynthesisError) -> IronError { let mime = "text/plain".parse::().unwrap(); let response = (mime, status::InternalServerError, "Service Error"); IronError::new(error, response) @@ -349,13 +350,13 @@ mod tests { speaker: Speaker::new("speaker".to_string()), volume: Some(2.0), speed: Some(1.5), - use_phonemes: true, + use_bare_monophones: true, use_n_phones: false, use_syllables: true, use_words: false, use_ends: false, - monophone_padding_start: Some(300), - monophone_padding_end: None, + padding_between_phones: Some(300), + polyphone_padding_start: None, polyphone_padding_end: None, word_padding_start: None, word_padding_end: None, @@ -366,13 +367,13 @@ mod tests { speaker: Speaker::new("speaker".to_string()), volume: Some(2.0), speed: Some(1.5), - use_phonemes: true, + use_bare_monophones: true, use_n_phones: false, use_syllables: true, use_words: false, use_ends: false, - monophone_padding_start: Some(300), - monophone_padding_end: None, + padding_between_phones: Some(300), + polyphone_padding_start: None, polyphone_padding_end: None, word_padding_start: None, word_padding_end: None, @@ -385,13 +386,13 @@ mod tests { speaker: Speaker::new("speaker".to_string()), volume: None, speed: Some(1.5), - use_phonemes: false, + use_bare_monophones: false, use_n_phones: true, use_syllables: false, use_words: true, use_ends: false, - monophone_padding_start: Some(300), - monophone_padding_end: None, + padding_between_phones: Some(300), + polyphone_padding_start: None, polyphone_padding_end: None, word_padding_start: None, word_padding_end: Some(400), @@ -404,13 +405,13 @@ mod tests { speaker: Speaker::new("speaker".to_string()), volume: Some(2.0), speed: Some(1.5), - use_phonemes: true, + use_bare_monophones: true, use_n_phones: false, use_syllables: true, use_words: false, use_ends: false, - monophone_padding_start: Some(300), - monophone_padding_end: None, + padding_between_phones: Some(300), + polyphone_padding_start: None, polyphone_padding_end: None, word_padding_start: None, word_padding_end: None, diff --git a/src/main.rs b/src/main.rs index 32edd4fc88..154a158bf5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,6 @@ extern crate time; extern crate toml; extern crate urlencoded; -pub mod audiobank; pub mod config; pub mod error; pub mod handlers; @@ -28,7 +27,6 @@ pub mod old_dictionary; pub mod speaker; pub mod synthesis; -use audiobank::Audiobank; use clap::{App, Arg, ArgMatches}; use config::Config; use handlers::audio_synth_handler::AudioSynthHandler; @@ -50,6 +48,7 @@ use staticfile::Static; use std::path::Path; use std::sync::Arc; use std::sync::RwLock; +use synthesis::audiobank::Audiobank; use synthesis::synthesizer::Synthesizer; fn main() { diff --git a/src/synthesis/audio.rs b/src/synthesis/audio.rs new file mode 100644 index 0000000000..6f09182bef --- /dev/null +++ b/src/synthesis/audio.rs @@ -0,0 +1,7 @@ +// Copyright (c) 2016 Brandon Thomas + +/// Waveform data delivered to the client. +pub type WavBytes = Vec; + +/// Waveform data as used internally. +pub type SampleBytes = Vec; diff --git a/src/audiobank.rs b/src/synthesis/audiobank.rs similarity index 94% rename from src/audiobank.rs rename to src/synthesis/audiobank.rs index 655982fe9f..2466bc8f6e 100644 --- a/src/audiobank.rs +++ b/src/synthesis/audiobank.rs @@ -1,12 +1,12 @@ // Copyright (c) 2016 Brandon Thomas -use error::SynthError; use hound::WavReader; use hound::WavSpec; +use speaker::Speaker; use std::path::Path; use std::path::PathBuf; - -pub type SampleBytes = Vec; +use synthesis::audio::SampleBytes; +use synthesis::error::SynthesisError; /** * Fetch wav audio files from the audio bank. @@ -42,6 +42,7 @@ pub type SampleBytes = Vec; * TODO: Caching of wav files in memory. * TODO: Path generation is _insecure_. */ +#[derive(Clone)] pub struct Audiobank { /// Root path to the files. audio_path: PathBuf, @@ -70,12 +71,12 @@ impl Audiobank { /** * Get the wav data for the (speaker,word), or None if it does not exist. */ - pub fn get_word(&self, speaker: &str, word: &str) -> Option { - if check_path(speaker).is_err() || check_path(word).is_err() { + pub fn get_word(&self, speaker: &Speaker, word: &str) -> Option { + if check_path(speaker.as_str()).is_err() || check_path(word).is_err() { return None; } - let path = self.audio_path.join(format!("{}/", speaker)) + let path = self.audio_path.join(format!("{}/", speaker.as_str())) .join("words/") .join(format!("{}.wav", word)); @@ -292,7 +293,7 @@ impl Audiobank { // TODO: This should be removed. We should cache the wav headers. pub fn get_spec(&self, speaker: &str, word: &str) - -> Result { + -> Result { try!(check_path(speaker)); try!(check_path(word)); @@ -305,7 +306,7 @@ impl Audiobank { } // TODO: This should be removed. We should cache the wav headers. - pub fn get_misc_spec(&self, name: &str) -> Result { + pub fn get_misc_spec(&self, name: &str) -> Result { try!(check_path(name)); let path = self.audio_path.join("misc/") @@ -333,14 +334,14 @@ impl Audiobank { } } -fn check_path(path: &str) -> Result<(), SynthError> { +fn check_path(path: &str) -> Result<(), SynthesisError> { // FIXME: Hack so I can release soon. Rewrite this whole logic plz. if path.contains("..") || path.contains("/") || path.contains("$") || path.contains("~") { - Err(SynthError::BadInput { description: "invalid path" }) + Err(SynthesisError::BadInput { description: "invalid path" }) } else { Ok(()) } diff --git a/src/synthesis/effects/pause.rs b/src/synthesis/effects/pause.rs index 9c02005948..bff649a746 100644 --- a/src/synthesis/effects/pause.rs +++ b/src/synthesis/effects/pause.rs @@ -1,7 +1,9 @@ // Copyright (c) 2016 Brandon Thomas +use synthesis::audio::SampleBytes; + /// Create a waveform pause of the desired length. -pub fn generate_pause(length: u16) -> Vec { +pub fn generate_pause(length: u16) -> SampleBytes { let mut sound = Vec::new(); let len = length * 2; // number of channels diff --git a/src/synthesis/error.rs b/src/synthesis/error.rs new file mode 100644 index 0000000000..018d1d19c2 --- /dev/null +++ b/src/synthesis/error.rs @@ -0,0 +1,61 @@ +// Copyright (c) 2016 Brandon Thomas + +use hound::Error as HoundError; +use std::convert::From; +use std::error::Error; +use std::fmt; +use std::io; + +/// Error in synthesizing audio. +#[derive(Debug)] +pub enum SynthesisError { + ArpabetEntryDne, + + /// Bad user input. + BadInput { description: &'static str }, + + /// Cannot load an expected audio file, eg. "blank.wav". + CannotLoadAudioFile, + + CannotSynthesizeWord, + + /// Wraps a Hound audio library error. + HoundError { cause: HoundError }, + + /// Wraps an IoError. + IoError { cause: io::Error }, + + /// Could not obtain a lock. + LockError, + + MonophoneDne, + WordSampleDne, +} + +impl Error for SynthesisError { + fn description(&self) -> &str { + "Synthesis Error" // TODO + } + + /*fn cause(&self) -> Option<&Error> { + Some(&self) + }*/ +} + +impl fmt::Display for SynthesisError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "Synthesis Error") + } +} + +impl From for SynthesisError { + fn from(error: io::Error) -> SynthesisError { + SynthesisError::IoError { cause: error } + } +} + +impl From for SynthesisError { + fn from(error: HoundError) -> SynthesisError { + SynthesisError::HoundError { cause: error } + } +} diff --git a/src/synthesis/mod.rs b/src/synthesis/mod.rs index 0531f70717..8a439e2707 100644 --- a/src/synthesis/mod.rs +++ b/src/synthesis/mod.rs @@ -1,5 +1,9 @@ // Copyright (c) 2016 Brandon Thomas +pub mod audio; +pub mod audiobank; pub mod effects; +pub mod error; pub mod synthesizer; pub mod tokens; +pub mod word_synthesizer; diff --git a/src/synthesis/synthesizer.rs b/src/synthesis/synthesizer.rs index 884c41adf7..6c381a9188 100644 --- a/src/synthesis/synthesizer.rs +++ b/src/synthesis/synthesizer.rs @@ -1,33 +1,31 @@ // Copyright (c) 2016 Brandon Thomas -use audiobank::Audiobank; -use audiobank::SampleBytes; -use audiobank::SamplePreference; -use error::SynthError; use hound::WavSpec; use hound::WavWriter; use lang::arpabet::Arpabet; use speaker::Speaker; use std::io::BufWriter; use std::io::Cursor; +use synthesis::audio::WavBytes; +use synthesis::audiobank::Audiobank; use synthesis::effects::pause::generate_pause; use synthesis::effects::speed::change_speed; use synthesis::effects::volume::change_volume; +use synthesis::error::SynthesisError; use synthesis::tokens::*; - -pub type WavBytes = Vec; +use synthesis::word_synthesizer::WordSynthesizer; /// Parameters for synthesis unrelated to the body of text or the speaker. pub struct SynthesisParams { pub use_words: bool, - pub use_phonemes: bool, pub use_n_phones: bool, pub use_syllables: bool, + pub use_bare_monophones: bool, pub use_ends: bool, pub volume: Option, pub speed: Option, - pub monophone_padding_start: Option, - pub monophone_padding_end: Option, + pub padding_between_phones: Option, + pub polyphone_padding_start: Option, pub polyphone_padding_end: Option, pub word_padding_start: Option, pub word_padding_end: Option, @@ -37,19 +35,20 @@ pub struct SynthesisParams { * Generates wave files from text input. */ pub struct Synthesizer { - /// Arpabet, (Word) -> (PhonemeList) - arpabet_dictionary: Arpabet, - - /// WAV Audiobank for sound generation. audiobank: Audiobank, + word_synthesizer: WordSynthesizer, } impl Synthesizer { /// CTOR - pub fn new(arpabet_dictionary: Arpabet, audiobank: Audiobank) -> Synthesizer { + pub fn new(arpabet_dictionary: Arpabet, audiobank: Audiobank) -> Self { + let word_synthesizer = WordSynthesizer::new( + arpabet_dictionary, + audiobank.clone() // FIXME + ); Synthesizer { - arpabet_dictionary: arpabet_dictionary, audiobank: audiobank, + word_synthesizer: word_synthesizer, } } @@ -60,30 +59,27 @@ impl Synthesizer { tokens: Vec, speaker: &Speaker, params: SynthesisParams) - -> Result { + -> Result { if tokens.len() == 0 { - return Err(SynthError::BadInput { + return Err(SynthesisError::BadInput { description: "There were no words to synthesize." }); } let mut concatenated_waveform : Vec = Vec::new(); - if !self.concatenate_misc(&mut concatenated_waveform, "padding_ends") { - return Err(SynthError::BadInput { - description: "Cannot prefix with padding." - }); - } + try!(self.pad_audio(&mut concatenated_waveform)); for token in tokens { match token { SynthToken::Filler { value: _v } => { // TODO: Variable length filler content. - self.handle_word_token(&mut concatenated_waveform, + try!(self.synthesize_word( + &mut concatenated_waveform, &speaker, "um", - ¶ms); + ¶ms)); }, SynthToken::Pause { value: ref v } => { let pause_duration = match *v { @@ -95,17 +91,16 @@ impl Synthesizer { concatenated_waveform.extend(pause); }, SynthToken::Word { value: ref v } => { - self.handle_word_token(&mut concatenated_waveform, &speaker, &v.value, - ¶ms); + try!(self.synthesize_word( + &mut concatenated_waveform, + &speaker, + &v.value, + ¶ms)); } } } - if !self.concatenate_misc(&mut concatenated_waveform, "padding_ends") { - return Err(SynthError::BadInput { - description: "Cannot suffix with padding." - }); - } + try!(self.pad_audio(&mut concatenated_waveform)); // FIXME: Super inefficient pieces. if params.speed.is_some() { @@ -118,435 +113,78 @@ impl Synthesizer { params.volume.unwrap()); } + // This is the prototype we use for audio headers. // TODO: Cache waveform headers. + // TODO: Add custom headers with "author" etc. metadata. let spec = try!(self.audiobank.get_misc_spec("pause")); Ok(self.write_buffer(&spec, concatenated_waveform)) } - fn handle_word_token(&self, - concatenated_waveform: &mut Vec, - speaker: &Speaker, - word: &str, - params: &SynthesisParams) { - - let mut word_added = false; + /// Add a word to the output waveform. + fn synthesize_word(&self, + concatenated_waveform: &mut Vec, + speaker: &Speaker, + word: &str, + params: &SynthesisParams) + -> Result<(), SynthesisError> { + let mut added = false; if params.use_words { - word_added = self.concatenate_word(concatenated_waveform, + // TODO: Prefix buffer and suffix buffer. + added = self.word_synthesizer.append_word_sample( speaker, word, params.word_padding_start, - params.word_padding_end); + params.word_padding_end, + concatenated_waveform) + .is_ok(); // Dictionary lookup is allowed to fail. } - if !word_added && params.use_n_phones { - // New phoneme algorithm - word_added = self.concatenate_n_phone(concatenated_waveform, - speaker, - word, - params.polyphone_padding_end, - params.use_ends); + if !added { + if params.use_n_phones { + // TODO: Prefix buffer and suffix buffer. + // TODO: Param for monophone padding. + added = try!(self.word_synthesizer.append_n_phones( + speaker, + word, + params.use_ends, + params.polyphone_padding_start, + params.polyphone_padding_end, + params.padding_between_phones, + concatenated_waveform) + .map(|_| true)); + } else if params.use_bare_monophones { + // TODO: Prefix buffer and suffix buffer. + // TODO: Param for monophone padding. + added = try!(self.word_synthesizer.append_monophones( + speaker, + word, + params.polyphone_padding_start, + params.polyphone_padding_end, + params.padding_between_phones, + concatenated_waveform) + .map(|_| true)); + } } - if !word_added && params.use_phonemes { - // Old phoneme algorithm - self.concatenate_polyphone(concatenated_waveform, - speaker, - word, - params.use_syllables, - params.monophone_padding_start, - params.monophone_padding_end, - params.polyphone_padding_end); + + if added { + Ok(()) + } else { + Err(SynthesisError::CannotSynthesizeWord) } } - /// Concatenate a word to the waveform we're building. Returns - /// whether or not the word was successfully found and concatenated. - fn concatenate_word(&self, - concatenated_waveform: &mut Vec, - speaker: &Speaker, - word: &str, - word_padding_start: Option, - word_padding_end: Option) -> bool { - match self.audiobank.get_word(speaker.as_str(), word) { - None => { - // The word does not exist in the database. - false - }, - Some(waveform_data) => { - // The word exists in the database. - if word_padding_start.is_some() { - let pause = generate_pause(word_padding_start.unwrap()); - concatenated_waveform.extend(pause); - } - - concatenated_waveform.extend(waveform_data); - - if word_padding_end.is_some() { - let pause = generate_pause(word_padding_end.unwrap()); - concatenated_waveform.extend(pause); - } - - true - }, + /// Add "silence" padding to the waveform. + /// This is necessary for smooth playback in the frontend UI. + fn pad_audio(&self, buffer: &mut Vec) -> Result<(), SynthesisError> { + if self.concatenate_misc(buffer, "padding_ends") { + Ok(()) + } else { + Err(SynthesisError::CannotLoadAudioFile) } } - /// Concatenate a polyphone corresponding to the word to the - /// waveform we're building. Returns whether or not the word was - /// successfully found and concatenated. - fn concatenate_polyphone(&self, - concatenated_waveform: &mut Vec, - speaker: &Speaker, - word: &str, - _use_syllables: bool, - monophone_padding_start: Option, - monophone_padding_end: Option, - polyphone_padding_end: Option) -> bool { - - let polyphone = match self.arpabet_dictionary.get_polyphone(word) { - Some(p) => { p }, - None => { - info!(target: "synthesis", - "Word '{}' does not exist in polyphone database.", word); - // XXX: Adding static as a cue to denote that the given - // word->polyphone mapping doesn't exist in the database. - match self.audiobank.get_misc("record_static") { - None => {}, - Some(waveform_data) => { - concatenated_waveform.extend(waveform_data); - }, - } - return false; - }, - }; - - info!(target: "synthesis", - "Word '{}' maps to polyphone '{:?}'", word, polyphone); - - // Insert space before polyphone. - match self.audiobank.get_misc("pause") { - None => {}, - Some(pause) => { concatenated_waveform.extend(pause); }, - } - - let end = polyphone.len() - 1; - let mut skip_next = false; - - for i in 0..polyphone.len() { - if skip_next { - // We just read a diphone. - skip_next = false; - continue; - } - - let phoneme = &polyphone[i]; - - // Attempt to read a "begin" or "end" monophone. - let mut read_results = if i == 0 { - self.audiobank.get_begin_phoneme(speaker.as_str(), &phoneme) - } else if i == end { - self.audiobank.get_end_phoneme(speaker.as_str(), &phoneme) - } else { - None - }; - - // Read a regular monophone. - if read_results.is_none() { - read_results = self.audiobank.get_phoneme(speaker.as_str(), &phoneme); - } - - match read_results { - None => { - // XXX: Adding beep to denote that the given monophone - // doesn't exist in the database. - match self.audiobank.get_misc("beep") { - None => {}, - Some(waveform_data) => { - concatenated_waveform.extend(waveform_data); - }, - } - continue; - }, - Some(waveform_data) => { - if monophone_padding_start.is_some() { - let pause = generate_pause(monophone_padding_start.unwrap()); - concatenated_waveform.extend(pause); - } - - concatenated_waveform.extend(waveform_data); - - if monophone_padding_end.is_some() { - let pause = generate_pause(monophone_padding_end.unwrap()); - concatenated_waveform.extend(pause); - } - }, - } - } - - // Insert space after polyphone. - if polyphone_padding_end.is_some() { - let pause = generate_pause(polyphone_padding_end.unwrap()); - concatenated_waveform.extend(pause); - } - - true - } - - // TODO: Doc and test. - fn concatenate_n_phone(&self, - concatenated_waveform: &mut Vec, - speaker: &Speaker, - word: &str, - polyphone_padding_end: Option, - use_ends: bool) -> bool { - - let pref = SamplePreference::Begin; - - let samples = match self.get_n_phone_samples(speaker, word, pref, use_ends) { - Some(s) => s, - None => { - return false; - }, - }; - - - for sample in samples { - concatenated_waveform.extend(sample); - } - - // Insert space after polyphone. - if polyphone_padding_end.is_some() { - let pause = generate_pause(polyphone_padding_end.unwrap()); - concatenated_waveform.extend(pause); - } - - true - } - - - // TODO: Make this super easy to test (and write some tests). - // TODO: Make this super efficient. - fn get_n_phone_samples(&self, - speaker: &Speaker, - word: &str, - _sample_preference: SamplePreference, - use_ends: bool) - -> Option> { - - let polyphone = match self.arpabet_dictionary.get_polyphone(word) { - None => { - return None; - }, - Some(p) => p, - }; - - let mut fulfilled : Vec = Vec::with_capacity(polyphone.len()); - let mut chunks : Vec> = - Vec::with_capacity(polyphone.len()); - - // Use this to debug the synthesis. - let mut debug : Vec> = - Vec::with_capacity(polyphone.len()); - - for _ in 0..polyphone.len() { - fulfilled.push(false); - chunks.push(None); - debug.push(None); - } - - // 5-phone - if polyphone.len() >= 5 { - let range = polyphone.len() - 4; - for i in 0..range { - if fulfilled[i] - || fulfilled[i+1] - || fulfilled[i+2] - || fulfilled[i+3] - || fulfilled[i+4] { - continue; - } - - let candidate_n_phone = &polyphone[i..i+5]; - - let sample_preference = match i { - 0 => { SamplePreference::Begin }, - _ if i == range - 1 => { SamplePreference::End }, - _ => { SamplePreference::Middle }, - }; - - //println!("n-phone: {:?}", candidate_n_phone); - let phone = self.audiobank.get_n_phone(speaker.as_str(), - candidate_n_phone, - sample_preference, - use_ends); - - if phone.is_some() { - chunks[i] = phone; - debug[i] = Some(polyphone[i..i+5].join("_")); - - for j in i..i+5 { - fulfilled[j] = true; - } - } - } - } - - info!(target: "synthesis", "5-fulfilled: {:?}", fulfilled); - - // 4-phone - if polyphone.len() >= 4 { - let range = polyphone.len() - 3; - for i in 0..range { - if fulfilled[i] - || fulfilled[i+1] - || fulfilled[i+2] - || fulfilled[i+3] { - continue; - } - - let candidate_n_phone = &polyphone[i..i+4]; - - let sample_preference = match i { - 0 => { SamplePreference::Begin }, - _ if i == range - 1 => { SamplePreference::End }, - _ => { SamplePreference::Middle }, - }; - - //println!("n-phone: {:?}", candidate_n_phone); - let phone = self.audiobank.get_n_phone(speaker.as_str(), - candidate_n_phone, - sample_preference, - use_ends); - - if phone.is_some() { - chunks[i] = phone; - debug[i] = Some(polyphone[i..i+4].join("_")); - - for j in i..i+4 { - fulfilled[j] = true; - } - } - } - } - - info!(target: "synthesis", "4-fulfilled: {:?}", fulfilled); - - // 3-phone - if polyphone.len() >= 3 { - let range = polyphone.len() - 2; - for i in 0..range { - if fulfilled[i] || fulfilled[i+1] || fulfilled[i+2] { - continue; - } - - let candidate_n_phone = &polyphone[i..i+3]; - - let sample_preference = match i { - 0 => { SamplePreference::Begin }, - _ if i == range - 1 => { SamplePreference::End }, - _ => { SamplePreference::Middle }, - }; - - let phone = self.audiobank.get_n_phone(speaker.as_str(), - candidate_n_phone, - sample_preference, - use_ends); - - if phone.is_some() { - chunks[i] = phone; - debug[i] = Some(polyphone[i..i+3].join("_")); - for j in i..i+3 { - fulfilled[j] = true; - } - } - } - } - - info!(target: "synthesis", "3-fulfilled: {:?}", fulfilled); - - // 2-phone - if polyphone.len() >= 2 { - let range = polyphone.len() - 1; - for i in 0..range { - if fulfilled[i] || fulfilled[i+1] { - continue; - } - - let candidate_n_phone = &polyphone[i..i+2]; - - let sample_preference = match i { - 0 => { SamplePreference::Begin }, - _ if i == range - 1 => { SamplePreference::End }, - _ => { SamplePreference::Middle }, - }; - - let phone = self.audiobank.get_n_phone(speaker.as_str(), - candidate_n_phone, - sample_preference, - use_ends); - - if phone.is_some() { - chunks[i] = phone; - debug[i] = Some(polyphone[i..i+2].join("_")); - for j in i..i+2 { - fulfilled[j] = true; - } - } - } - } - - info!(target: "synthesis", "2-fulfilled: {:?}", fulfilled); - - // 1-phone - for i in 0..polyphone.len() { - if fulfilled[i] { - continue; - } - - let sample_preference = match i { - 0 => { SamplePreference::Begin }, - _ if i == polyphone.len() - 1 => { SamplePreference::End }, - _ => { SamplePreference::Middle }, - }; - - let phone = self.audiobank.get_n_phone(speaker.as_str(), - &polyphone[i..i+1], - sample_preference, - use_ends); - - if phone.is_some() { - chunks[i] = phone; - debug[i] = Some(polyphone[i].to_string()); - fulfilled[i] = true; - } - } - - info!(target: "synthesis", "1-fulfilled: {:?}", fulfilled); - - for x in fulfilled { - if !x { - return None; - } - } - - let debug_str : Vec = debug.into_iter() - .map(|x| if x.is_some() { x.unwrap() } else { "None".to_string() }) - .collect(); - - info!(target: "synthesis", "Comprised of: {:?}", debug_str); - - let results : Vec = chunks.into_iter() - .filter_map(|x| x) // Woo, already Option! - .collect(); - - info!(target: "synthesis", "Results Length: {} of {} phones", - results.len(), - polyphone.len()); - - Some(results) - } - /// Concatenate a sound effect to the waveform we're building. Returns /// whether or not the effect was successfully found and concatenated. fn concatenate_misc(&self, concatenated_waveform: &mut Vec, diff --git a/src/synthesis/word_synthesizer.rs b/src/synthesis/word_synthesizer.rs new file mode 100644 index 0000000000..3ce1a61c9a --- /dev/null +++ b/src/synthesis/word_synthesizer.rs @@ -0,0 +1,394 @@ +// Copyright (c) 2016 Brandon Thomas + +use lang::arpabet::Arpabet; +use speaker::Speaker; +use synthesis::audio::SampleBytes; +use synthesis::audiobank::Audiobank; +use synthesis::audiobank::SamplePreference; +use synthesis::effects::pause::generate_pause; +use synthesis::error::SynthesisError; + +/// The word synthesizer is responsible for generating "word" waveforms. +/// It does not control the strategy for how a word is synthesized. +pub struct WordSynthesizer { + arpabet_dictionary: Arpabet, + audiobank: Audiobank, +} + +impl WordSynthesizer { + /// CTOR + pub fn new(arpabet_dictionary: Arpabet, audiobank: Audiobank) -> Self { + WordSynthesizer { + arpabet_dictionary: arpabet_dictionary, + audiobank: audiobank, + } + } + + /// Append a word sample pulled from the whole-word sample database. + pub fn append_word_sample(&self, + speaker: &Speaker, + word: &str, + padding_before_word: Option, + padding_after_word: Option, + buffer: &mut SampleBytes) + -> Result<(), SynthesisError> { + let bytes = try!(self.audiobank.get_word(speaker, word) + .ok_or(SynthesisError::WordSampleDne)); + + if padding_before_word.is_some() { + let pause = generate_pause(padding_before_word.unwrap()); + buffer.extend(pause); + } + + buffer.extend(bytes); + + if padding_after_word.is_some() { + let pause = generate_pause(padding_after_word.unwrap()); + buffer.extend(pause); + } + + Ok(()) + } + + /// Append a word sample generated from Arpabet polyphones. + /// This uses the n-phone algorithm that builds samples from the + /// longest-contiguous n-phones that complete the polyphone. + pub fn append_n_phones(&self, + speaker: &Speaker, + word: &str, + use_ends: bool, + padding_before_polyphone: Option, + padding_after_polyphone: Option, + debug_padding_between_phones: Option, + buffer: &mut SampleBytes) + -> Result<(), SynthesisError> { + let samples = try!(self.get_n_phone_samples(speaker, word, use_ends)); + + if padding_before_polyphone.is_some() { + let pause = generate_pause(padding_before_polyphone.unwrap()); + buffer.extend(pause); + } + + if debug_padding_between_phones.is_some() { + let mut padding = None; + if debug_padding_between_phones.is_some() { + let pause = generate_pause(debug_padding_between_phones.unwrap()); + padding = Some(pause) + } + + let last = samples.len() - 1; + for (i, sample) in samples.into_iter().enumerate() { + buffer.extend(sample); + if i < last { + buffer.extend(padding.clone().unwrap()); + } + } + } else { + for sample in samples { + buffer.extend(sample); + } + } + + if padding_after_polyphone.is_some() { + let pause = generate_pause(padding_after_polyphone.unwrap()); + buffer.extend(pause); + } + + Ok(()) + } + + /// Similar to 'append_n_phones', but only uses single monophones. + pub fn append_monophones(&self, + speaker: &Speaker, + word: &str, + padding_before_polyphone: Option, + padding_after_polyphone: Option, + debug_padding_between_phones: Option, + buffer: &mut SampleBytes) + -> Result<(), SynthesisError> { + let samples = try!(self.get_monophone_samples(speaker, word, + debug_padding_between_phones)); + + if padding_before_polyphone.is_some() { + let pause = generate_pause(padding_before_polyphone.unwrap()); + buffer.extend(pause); + } + + buffer.extend(samples); + + if padding_after_polyphone.is_some() { + let pause = generate_pause(padding_after_polyphone.unwrap()); + buffer.extend(pause); + } + + Ok(()) + } + + /// TODO DOC + /// TODO DOC + // TODO: Make this super easy to test (and write some tests). + // TODO: Make this super efficient. + fn get_n_phone_samples(&self, + speaker: &Speaker, + word: &str, + use_ends: bool) + -> Result, SynthesisError> { + + let polyphone = try!(self.arpabet_dictionary.get_polyphone(word) + .ok_or(SynthesisError::ArpabetEntryDne)); + + let mut fulfilled : Vec = Vec::with_capacity(polyphone.len()); + let mut chunks : Vec> = + Vec::with_capacity(polyphone.len()); + + // Use this to debug the synthesis. + let mut debug : Vec> = Vec::with_capacity(polyphone.len()); + + for _ in 0..polyphone.len() { + fulfilled.push(false); + chunks.push(None); + debug.push(None); + } + + // 5-phone + if polyphone.len() >= 5 { + let range = polyphone.len() - 4; + for i in 0..range { + if fulfilled[i] + || fulfilled[i+1] + || fulfilled[i+2] + || fulfilled[i+3] + || fulfilled[i+4] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+5]; + + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + //println!("n-phone: {:?}", candidate_n_phone); + let phone = self.audiobank.get_n_phone(speaker.as_str(), + candidate_n_phone, + sample_preference, + use_ends); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+5].join("_")); + + for j in i..i+5 { + fulfilled[j] = true; + } + } + } + } + + info!(target: "synthesis", "5-fulfilled: {:?}", fulfilled); + + // 4-phone + if polyphone.len() >= 4 { + let range = polyphone.len() - 3; + for i in 0..range { + if fulfilled[i] + || fulfilled[i+1] + || fulfilled[i+2] + || fulfilled[i+3] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+4]; + + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + //println!("n-phone: {:?}", candidate_n_phone); + let phone = self.audiobank.get_n_phone(speaker.as_str(), + candidate_n_phone, + sample_preference, + use_ends); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+4].join("_")); + + for j in i..i+4 { + fulfilled[j] = true; + } + } + } + } + + info!(target: "synthesis", "4-fulfilled: {:?}", fulfilled); + + // 3-phone + if polyphone.len() >= 3 { + let range = polyphone.len() - 2; + for i in 0..range { + if fulfilled[i] || fulfilled[i+1] || fulfilled[i+2] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+3]; + + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + let phone = self.audiobank.get_n_phone(speaker.as_str(), + candidate_n_phone, + sample_preference, + use_ends); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+3].join("_")); + for j in i..i+3 { + fulfilled[j] = true; + } + } + } + } + + info!(target: "synthesis", "3-fulfilled: {:?}", fulfilled); + + // 2-phone + if polyphone.len() >= 2 { + let range = polyphone.len() - 1; + for i in 0..range { + if fulfilled[i] || fulfilled[i+1] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+2]; + + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == range - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + let phone = self.audiobank.get_n_phone(speaker.as_str(), + candidate_n_phone, + sample_preference, + use_ends); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+2].join("_")); + for j in i..i+2 { + fulfilled[j] = true; + } + } + } + } + + info!(target: "synthesis", "2-fulfilled: {:?}", fulfilled); + + // 1-phone + for i in 0..polyphone.len() { + if fulfilled[i] { + continue; + } + + let sample_preference = match i { + 0 => { SamplePreference::Begin }, + _ if i == polyphone.len() - 1 => { SamplePreference::End }, + _ => { SamplePreference::Middle }, + }; + + let phone = self.audiobank.get_n_phone(speaker.as_str(), + &polyphone[i..i+1], + sample_preference, + use_ends); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i].to_string()); + fulfilled[i] = true; + } + } + + info!(target: "synthesis", "1-fulfilled: {:?}", fulfilled); + + for x in fulfilled { + if !x { return Err(SynthesisError::MonophoneDne); } + } + + let debug_str : Vec = debug.into_iter() + .map(|x| if x.is_some() { x.unwrap() } else { "None".to_string() }) + .collect(); + + info!(target: "synthesis", "Comprised of: {:?}", debug_str); + + let results : Vec = chunks.into_iter() + .filter_map(|x| x) // Woo, already Option! + .collect(); + + info!(target: "synthesis", "Results Length: {} of {} phones", + results.len(), + polyphone.len()); + + Ok(results) + } + + /// TODO DOC + /// TODO DOC + fn get_monophone_samples(&self, + speaker: &Speaker, + word: &str, + debug_padding_between_phones: Option) + -> Result { + + let polyphone = try!(self.arpabet_dictionary.get_polyphone(word) + .ok_or(SynthesisError::ArpabetEntryDne)); + + let mut padding = None; + if debug_padding_between_phones.is_some() { + let pause = generate_pause(debug_padding_between_phones.unwrap()); + padding = Some(pause) + } + + info!(target: "synthesis", + "Word '{}' maps to polyphone '{:?}'", word, polyphone); + + let end = polyphone.len() - 1; + let mut output_waveform = Vec::new(); + + for i in 0..polyphone.len() { + let phoneme = &polyphone[i]; + + // Attempt to read a "begin" or "end" monophone. + let mut read_results = if i == 0 { + self.audiobank.get_begin_phoneme(speaker.as_str(), &phoneme) + } else if i == end { + self.audiobank.get_end_phoneme(speaker.as_str(), &phoneme) + } else { + None + }; + + // Read a regular monophone. + if read_results.is_none() { + read_results = self.audiobank.get_phoneme(speaker.as_str(), &phoneme); + } + + let waveform = try!(read_results.ok_or(SynthesisError::MonophoneDne)); + + if padding.is_some() { + output_waveform.extend(padding.clone().unwrap()); + } + + output_waveform.extend(waveform); + } + + Ok(output_waveform) + } +} diff --git a/web/old.html b/web/old.html index 96060a3ffa..60675bd89d 100644 --- a/web/old.html +++ b/web/old.html @@ -31,28 +31,23 @@
- -
-
- - -
-
- - +
- + +
+
+ +
+ Use _begin and _end n-phone samples
@@ -62,14 +57,14 @@
- - + +
- - + +