From 93f3a2d372576fa1f78dec2bb82c83faba1bf86d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 8 Sep 2016 00:42:05 -0400 Subject: [PATCH] Synthesize using n-phones! n-phones comprise monophones, diphones, 3-phones, and more. This algorithm allows for smoother sounding synthesis. Also updates the UI for a new n-phones option and adds/updates scripts related to the arpabet with respect to n-phones. --- script/arpabet_histogram.py | 17 ++- script/arpabet_progress.py | 2 +- src/audiobank.rs | 76 ++++++++++-- src/handlers/audio_synth_handler.rs | 21 ++++ src/synthesizer.rs | 180 +++++++++++++++++++++++++++- src/web/js/events.js | 3 + src/web/js/initialize.js | 3 + src/web/js/url.js | 17 +++ web/index.html | 7 +- 9 files changed, 312 insertions(+), 14 deletions(-) diff --git a/script/arpabet_histogram.py b/script/arpabet_histogram.py index 7559984901..484a2a68e5 100755 --- a/script/arpabet_histogram.py +++ b/script/arpabet_histogram.py @@ -35,14 +35,14 @@ def get_n_phones(phoneme_list, n): elif n == len(phoneme_list): return [phoneme_list[:]] else: - span = len(phoneme_list) - n + span = len(phoneme_list) - n + 1 n_phones = [] for i in range(span): n_phone = [phoneme_list[j] for j in range(i, i+n)] n_phones.append(n_phone) return n_phones -def print_histogram(n): +def print_histogram(n, print_length = 20): """ Build and print the histogram for the given size """ @@ -58,11 +58,20 @@ def print_histogram(n): ordered = sorted(n_phone_map.items(), key=lambda x: len(x[1]), reverse=True) - print("MOST FREQUENT {}-PHONES:\n".format(n)) + title = "MOST FREQUENT {}-PHONES:".format(n) + print("\n" + title) + print("="*len(title) + "\n") - for it in ordered[0:100]: + for it in ordered[0:print_length]: line = "{}, {} - eg.{}".format(it[0], len(it[1]), it[1][0]) print(line) +print_histogram(2) print_histogram(3) +print_histogram(4) + +# Beyond 4-phones, it becomes computationally expensive +# if we try to guarantee best fit (exponential-time). +#print_histogram(5) +#print_histogram(6) diff --git a/script/arpabet_progress.py b/script/arpabet_progress.py index b7d1eda0f8..e112667c80 100755 --- a/script/arpabet_progress.py +++ b/script/arpabet_progress.py @@ -6,7 +6,7 @@ import os import re DICTIONARY_FILE = '/home/bt/dev/trumpet/dictionary/cmudict-0.7b' -PHONE_DIR = '/home/bt/Dropbox/jungle.horse/sounds/trump/_phonemes' +PHONE_DIR = '/home/bt/Dropbox/jungle.horse/sounds/trump/1-phones' def current_phonemes(): """ diff --git a/src/audiobank.rs b/src/audiobank.rs index f6732bc997..763b89ee5d 100644 --- a/src/audiobank.rs +++ b/src/audiobank.rs @@ -7,6 +7,8 @@ use hound::WavWriter; use std::path::Path; use std::path::PathBuf; +pub type SampleBytes = Vec; + /** * Fetch wav audio files from the audio bank. * @@ -17,14 +19,14 @@ use std::path::PathBuf; * /_words * /all.wav * /words.wav - * /_phonemes + * /1-phones * /_begin (starting syllable of word) * /IY0.wav * /_end (ending syllable of word) * /IY0.wav * /IY0.wav * /IY1.wav - * /_diphones + * /2-phones * /_begin * /F_R.wav * /_end @@ -50,7 +52,7 @@ impl Audiobank { /** * Get the wav data for the (speaker,word), or None if it does not exist. */ - pub fn get_word(&self, speaker: &str, word: &str) -> Option> { + pub fn get_word(&self, speaker: &str, word: &str) -> Option { if check_path(speaker).is_err() || check_path(word).is_err() { return None; } @@ -74,6 +76,66 @@ impl Audiobank { Some(all_samples) } + /** Get an n-phone in the form "{1st}_{2nd}_{...}_{nth}.wav". */ + pub fn get_n_phone(&self, speaker: &str, n_phone: &[String]) + -> Option { + + let (directory, filename) = match n_phone.len() { + 1 => { + ( + "1-phones/", + format!("{}.wav", n_phone[0]), + ) + }, + 2 => { + ( + "2-phones/", + format!("{}_{}.wav", n_phone[0], n_phone[1]), + ) + }, + 3 => { + ( + "3-phones/", + format!("{}_{}_{}.wav", n_phone[0], n_phone[1], n_phone[2]), + ) + }, + 4 => { + ( + "4-phones/", + format!("{}_{}_{}_{}.wav", + n_phone[0], n_phone[1], n_phone[2], n_phone[3]), + ) + }, + _ => { + return None; // Don't support more than 4-phones. + } + }; + + // FIXME: This is a lame check. + if check_path(speaker).is_err() + || check_path(&filename).is_err() { + return None; + } + + let path = self.audio_path.join(format!("{}/", speaker)) + .join(directory) + .join(filename); + + let mut reader = match WavReader::open(path) { + Err(_) => { return None; }, + Ok(reader) => reader, + }; + + // TODO: Inefficient. + let mut all_samples = Vec::new(); + let samples = reader.samples::(); + for sample in samples { + all_samples.push(sample.unwrap()); + } + + Some(all_samples) + } + /** * Get the wav data for the (speaker,phoneme), or None if it does not exist. */ @@ -83,7 +145,7 @@ impl Audiobank { } let path = self.audio_path.join(format!("{}/", speaker)) - .join("_phonemes/") + .join("1-phones/") .join(format!("{}.wav", phoneme)); let mut reader = match WavReader::open(path) { @@ -109,7 +171,7 @@ impl Audiobank { } let path = self.audio_path.join(format!("{}/", speaker)) - .join("_phonemes/_begin") + .join("1-phones/_begin") .join(format!("{}.wav", phoneme)); let mut reader = match WavReader::open(path) { @@ -135,7 +197,7 @@ impl Audiobank { } let path = self.audio_path.join(format!("{}/", speaker)) - .join("_phonemes/_end") + .join("1-phones/_end") .join(format!("{}.wav", phoneme)); let mut reader = match WavReader::open(path) { @@ -164,7 +226,7 @@ impl Audiobank { } let path = self.audio_path.join(format!("{}/", speaker)) - .join("_diphones/") + .join("2-phones/") .join(format!("{}_{}.wav", first, second)); let mut reader = match WavReader::open(path) { diff --git a/src/handlers/audio_synth_handler.rs b/src/handlers/audio_synth_handler.rs index c796106a12..3badfad52e 100644 --- a/src/handlers/audio_synth_handler.rs +++ b/src/handlers/audio_synth_handler.rs @@ -36,6 +36,7 @@ const SPEAKER_PARAM : &'static str = "v"; const SPEED_PARAM : &'static str = "spd"; const USE_PHONEMES_PARAM: &'static str = "up"; const USE_DIPHONES_PARAM: &'static str = "ud"; +const USE_N_PHONES_PARAM: &'static str = "un"; const USE_WORDS_PARAM: &'static str = "uw"; const VOLUME_PARAM : &'static str = "vol"; const MONOPHONE_PADDING_START_PARAM : &'static str = "mps"; @@ -63,6 +64,9 @@ struct SpeakRequest { /** Whether to use diphones. */ pub use_diphones: bool, + /** Whether to use n-phones. */ + pub use_n_phones: bool, + /** Whether to use words. */ pub use_words: bool, @@ -196,6 +200,21 @@ impl SpeakRequest { }, }; + let use_n_phones = match map.get(USE_N_PHONES_PARAM) { + None => { true }, + Some(list) => { + match list.get(0) { + None => { return speaker_error; }, + Some(s) => { + match FromStr::from_str(s) { + Ok(b) => b, + Err(_) => { return speaker_error; }, + } + }, + } + }, + }; + let use_words = match map.get(USE_WORDS_PARAM) { None => { true }, Some(list) => { @@ -281,6 +300,7 @@ impl SpeakRequest { speed: speed, use_phonemes: use_phonemes, use_diphones: use_diphones, + use_n_phones: use_n_phones, use_words: use_words, monophone_padding_start: mps, monophone_padding_end: mpe, @@ -367,6 +387,7 @@ impl AudioSynthHandler { request.use_words, request.use_phonemes, request.use_diphones, + request.use_n_phones, request.volume, request.speed, request.monophone_padding_start, diff --git a/src/synthesizer.rs b/src/synthesizer.rs index 6b3aac2f14..25607efaa1 100644 --- a/src/synthesizer.rs +++ b/src/synthesizer.rs @@ -2,6 +2,7 @@ use arpabet::ArpabetDictionary; use audiobank::Audiobank; +use audiobank::SampleBytes; use effects::pause::generate_pause; use effects::speed::change_speed; use effects::volume::change_volume; @@ -49,6 +50,7 @@ impl Synthesizer { use_words: bool, use_phonemes: bool, use_diphones: bool, + use_n_phones: bool, volume: Option, speed: Option, monophone_padding_start: Option, @@ -77,10 +79,20 @@ impl Synthesizer { let mut word_added = false; if use_words { - word_added = self.concatenate_word(&mut concatenated_waveform, speaker, word); + word_added = self.concatenate_word(&mut concatenated_waveform, + speaker, + word); } + if !word_added && use_n_phones { + // New phoneme algorithm + word_added = self.concatenate_n_phone(&mut concatenated_waveform, + speaker, + word, + polyphone_padding_end); + } if !word_added && use_phonemes { + // Old phoneme algorithm word_added = self.concatenate_polyphone(&mut concatenated_waveform, speaker, word, @@ -242,6 +254,172 @@ impl Synthesizer { true } + // TODO: Doc and test. + fn concatenate_n_phone(&self, + concatenated_waveform: &mut Vec, + speaker: &str, + word: &str, + polyphone_padding_end: Option) -> bool { + + let samples = match self.get_n_phone_samples(speaker, word) { + Some(s) => s, + None => { + return false; + }, + }; + + + for sample in samples { + concatenated_waveform.extend(sample); + } + + // Insert space after polyphone. + if polyphone_padding_end.is_some() { + let pause = generate_pause(polyphone_padding_end.unwrap()); + concatenated_waveform.extend(pause); + } + + true + } + + + // TODO: Make this super easy to test (and write some tests). + // TODO: Make this super efficient. + fn get_n_phone_samples(&self, speaker: &str, word: &str) + -> Option> { + + let polyphone = match self.arpabet_dictionary.get_polyphone(word) { + None => { + return None; + }, + Some(p) => p, + }; + + let mut fulfilled : Vec = Vec::with_capacity(polyphone.len()); + let mut chunks : Vec> = + Vec::with_capacity(polyphone.len()); + + // Use this to debug the synthesis. + let mut debug : Vec> = Vec::with_capacity(polyphone.len()); + + for i in 0..polyphone.len() { + fulfilled.push(false); + chunks.push(None); + debug.push(None); + } + + // 4-phone + if polyphone.len() >= 4 { + let range = polyphone.len() - 3; + for i in 0..range { + if fulfilled[i] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+4]; + + let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+4].join("_")); + + for j in i..i+4 { + fulfilled[j] = true; + } + } + } + } + + info!("4-Fulfilled: {:?}", fulfilled); + + // 3-phone + if polyphone.len() >= 3 { + let range = polyphone.len() - 2; + for i in 0..range { + if fulfilled[i] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+3]; + + let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+3].join("_")); + for j in i..i+3 { + fulfilled[j] = true; + } + } + } + } + + info!("3-Fulfilled: {:?}", fulfilled); + + // 2-phone + if polyphone.len() >= 2 { + let range = polyphone.len() - 1; + for i in 0..range { + //println!("i: {}, range: {}", i, range); + if fulfilled[i] { + continue; + } + + let candidate_n_phone = &polyphone[i..i+2]; + + let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i..i+2].join("_")); + for j in i..i+2 { + fulfilled[j] = true; + } + } + } + } + + info!("2-Fulfilled: {:?}", fulfilled); + + // 1-phone + for i in 0..polyphone.len() { + if fulfilled[i] { + continue; + } + + let phone = self.audiobank.get_phoneme(speaker, &polyphone[i]); + + if phone.is_some() { + chunks[i] = phone; + debug[i] = Some(polyphone[i].to_string()); + fulfilled[i] = true; + } + } + + info!("1-Fulfilled: {:?}", fulfilled); + + for x in fulfilled { + if !x { + return None; + } + } + + let mut debug_str : Vec = debug.into_iter() + .map(|x| if x.is_some() { x.unwrap() } else { "None".to_string() }) + .collect(); + + info!("Comprised of: {:?}", debug_str); + + let results : Vec = chunks.into_iter() + .filter_map(|x| x) // Woo, already Option! + .collect(); + + info!("Results Length: {} of {} phones", results.len(), polyphone.len()); + + Some(results) + } + /// Concatenate a sound effect to the waveform we're building. Returns /// whether or not the effect was successfully found and concatenated. fn concatenate_misc(&self, concatenated_waveform: &mut Vec, diff --git a/src/web/js/events.js b/src/web/js/events.js index c01375de16..2043b7802d 100644 --- a/src/web/js/events.js +++ b/src/web/js/events.js @@ -42,6 +42,7 @@ speed = $('input#speed').val(), up = $('#use_phonemes').prop('checked'), ud = $('#use_diphones').prop('checked'), + un = $('#use_n_phones').prop('checked'), uw = $('#use_words').prop('checked'), mps = $('input#monophone_padding_start').val(), mpe = $('input#monophone_padding_end').val(), @@ -56,6 +57,7 @@ + '&ppe=' + ppe + '&up=' + up + '&ud=' + ud + + '&un=' + un + '&uw=' + uw; ev.preventDefault(); @@ -66,6 +68,7 @@ speed, up, ud, + un, uw, mps, mpe, diff --git a/src/web/js/initialize.js b/src/web/js/initialize.js index 12718d73cb..f9db8f4d77 100644 --- a/src/web/js/initialize.js +++ b/src/web/js/initialize.js @@ -19,6 +19,7 @@ speed = Url.getSpeed() || 1.0, usePhonemes = Url.getUsePhonemes(), useDiphones = Url.getUseDiphones(), + useNPhones = Url.getUseNPhones(), useWords = Url.getUseWords(), mps = Url.getMonophonePaddingStart() || 0, mpe = Url.getMonophonePaddingEnd() || 0, @@ -27,6 +28,7 @@ if (usePhonemes === null) usePhonemes = true; if (useDiphones === null) useDiphones = true; + if (useNPhones === null) useNPhones = true; if (useWords === null) useWords = true; if (speaker) { @@ -44,6 +46,7 @@ $('#use_phonemes').prop('checked', usePhonemes); $('#use_diphones').prop('checked', useDiphones); + $('#use_n_phones').prop('checked', useNPhones); $('#use_words').prop('checked', useWords); $('input#monophone_padding_start').val(mps); diff --git a/src/web/js/url.js b/src/web/js/url.js index f21dab600a..b385309232 100644 --- a/src/web/js/url.js +++ b/src/web/js/url.js @@ -22,6 +22,9 @@ /** JSON key for the `use_diphones` param. */ _USE_DIPHONES_KEY: 'ud', + /** JSON key for the `use_n_phones` param. */ + _USE_N_PHONES_KEY: 'un', + /** JSON key for the `use_words` param. */ _USE_WORDS_KEY: 'uw', @@ -91,6 +94,16 @@ } }, + /** Get the "use n-phones" from the `window.location`. */ + getUseNPhones: function() { + var state = this.parseState(window.location); + if (!state || !(this._USE_N_PHONES_KEY in state)) { + return null; + } else { + return state[this._USE_N_PHONES_KEY]; + } + }, + /** Get the "use words" from the `window.location`. */ getUseWords: function() { var state = this.parseState(window.location); @@ -139,6 +152,7 @@ speed, usePhonemes, useDiphones, + useNPhones, useWords, monophonePaddingStart, monophonePaddingEnd, @@ -150,6 +164,7 @@ speed, usePhonemes, useDiphones, + useNPhones, useWords, monophonePaddingStart, monophonePaddingEnd, @@ -166,6 +181,7 @@ speed, usePhonemes, useDiphones, + useNPhones, useWords, monophonePaddingStart, monophonePaddingEnd, @@ -184,6 +200,7 @@ state[this._SPEED_KEY] = speed; // TODO: Filter invalid values. state[this._USE_PHONEMES_KEY] = !!usePhonemes; state[this._USE_DIPHONES_KEY] = !!useDiphones; + state[this._USE_N_PHONES_KEY] = !!useNPhones; state[this._USE_WORDS_KEY] = !!useWords; state[this._MONOPHONE_PADDING_START_KEY] = monophonePaddingStart; state[this._MONOPHONE_PADDING_END_KEY] = monophonePaddingEnd; diff --git a/web/index.html b/web/index.html index 5586d187d6..05facaa390 100644 --- a/web/index.html +++ b/web/index.html @@ -32,13 +32,18 @@
- +
+
+ + +