Synthesize using n-phones!

n-phones comprise monophones, diphones, 3-phones, and more.
This algorithm allows for smoother sounding synthesis.

Also updates the UI for a new n-phones option and adds/updates
scripts related to the arpabet with respect to n-phones.

script/arpabet_histogram.py

@@ -35,14 +35,14 @@ def get_n_phones(phoneme_list, n):
     elif n == len(phoneme_list):
         return [phoneme_list[:]]
     else:
-        span = len(phoneme_list) - n
+        span = len(phoneme_list) - n + 1
         n_phones = []
         for i in range(span):
             n_phone = [phoneme_list[j] for j in range(i, i+n)]
             n_phones.append(n_phone)
         return n_phones
 
-def print_histogram(n):
+def print_histogram(n, print_length = 20):
     """
     Build and print the histogram for the given size
     """
@@ -58,11 +58,20 @@ def print_histogram(n):
 
     ordered = sorted(n_phone_map.items(), key=lambda x: len(x[1]), reverse=True)
 
-    print("MOST FREQUENT {}-PHONES:\n".format(n))
+    title = "MOST FREQUENT {}-PHONES:".format(n)
+    print("\n" + title)
+    print("="*len(title) + "\n")
 
-    for it in ordered[0:100]:
+    for it in ordered[0:print_length]:
         line = "{}, {} - eg.{}".format(it[0], len(it[1]), it[1][0])
         print(line)
 
+print_histogram(2)
 print_histogram(3)
+print_histogram(4)
+
+# Beyond 4-phones, it becomes computationally expensive
+# if we try to guarantee best fit (exponential-time).
+#print_histogram(5)
+#print_histogram(6)
 

script/arpabet_progress.py

@@ -6,7 +6,7 @@ import os
 import re
 
 DICTIONARY_FILE = '/home/bt/dev/trumpet/dictionary/cmudict-0.7b'
-PHONE_DIR = '/home/bt/Dropbox/jungle.horse/sounds/trump/_phonemes'
+PHONE_DIR = '/home/bt/Dropbox/jungle.horse/sounds/trump/1-phones'
 
 def current_phonemes():
     """

src/audiobank.rs

@@ -7,6 +7,8 @@ use hound::WavWriter;
 use std::path::Path;
 use std::path::PathBuf;
 
+pub type SampleBytes = Vec<i16>;
+
 /**
  * Fetch wav audio files from the audio bank.
  *
@@ -17,14 +19,14 @@ use std::path::PathBuf;
  *       /_words
  *         /all.wav
  *         /words.wav
- *       /_phonemes
+ *       /1-phones
  *         /_begin (starting syllable of word)
  *           /IY0.wav
  *         /_end (ending syllable of word)
  *           /IY0.wav
  *         /IY0.wav
  *         /IY1.wav
- *       /_diphones
+ *       /2-phones
  *         /_begin
  *           /F_R.wav
  *         /_end
@@ -50,7 +52,7 @@ impl Audiobank {
   /**
    * Get the wav data for the (speaker,word), or None if it does not exist.
    */
-  pub fn get_word(&self, speaker: &str, word: &str) -> Option<Vec<i16>> {
+  pub fn get_word(&self, speaker: &str, word: &str) -> Option<SampleBytes> {
     if check_path(speaker).is_err() || check_path(word).is_err() {
       return None;
     }
@@ -74,6 +76,66 @@ impl Audiobank {
     Some(all_samples)
   }
 
+  /** Get an n-phone in the form "{1st}_{2nd}_{...}_{nth}.wav". */
+  pub fn get_n_phone(&self, speaker: &str, n_phone: &[String])
+      -> Option<SampleBytes> {
+
+    let (directory, filename) = match n_phone.len() {
+      1 => {
+        (
+          "1-phones/",
+          format!("{}.wav", n_phone[0]),
+        )
+      },
+      2 => {
+        (
+          "2-phones/",
+          format!("{}_{}.wav", n_phone[0], n_phone[1]),
+        )
+      },
+      3 => {
+        (
+          "3-phones/",
+          format!("{}_{}_{}.wav", n_phone[0], n_phone[1], n_phone[2]),
+        )
+      },
+      4 => {
+        (
+          "4-phones/",
+          format!("{}_{}_{}_{}.wav",
+                  n_phone[0], n_phone[1], n_phone[2], n_phone[3]),
+        )
+      },
+      _ => {
+        return None; // Don't support more than 4-phones.
+      }
+    };
+
+    // FIXME: This is a lame check.
+    if check_path(speaker).is_err()
+        || check_path(&filename).is_err() {
+      return None;
+    }
+
+    let path = self.audio_path.join(format!("{}/", speaker))
+        .join(directory)
+        .join(filename);
+
+    let mut reader = match WavReader::open(path) {
+      Err(_) => { return None; },
+      Ok(reader) => reader,
+    };
+
+    // TODO: Inefficient.
+    let mut all_samples = Vec::new();
+    let samples = reader.samples::<i16>();
+    for sample in samples {
+      all_samples.push(sample.unwrap());
+    }
+
+    Some(all_samples)
+  }
+
   /**
    * Get the wav data for the (speaker,phoneme), or None if it does not exist.
    */
@@ -83,7 +145,7 @@ impl Audiobank {
     }
 
     let path = self.audio_path.join(format!("{}/", speaker))
-        .join("_phonemes/")
+        .join("1-phones/")
         .join(format!("{}.wav", phoneme));
 
     let mut reader = match WavReader::open(path) {
@@ -109,7 +171,7 @@ impl Audiobank {
     }
 
     let path = self.audio_path.join(format!("{}/", speaker))
-        .join("_phonemes/_begin")
+        .join("1-phones/_begin")
         .join(format!("{}.wav", phoneme));
 
     let mut reader = match WavReader::open(path) {
@@ -135,7 +197,7 @@ impl Audiobank {
     }
 
     let path = self.audio_path.join(format!("{}/", speaker))
-        .join("_phonemes/_end")
+        .join("1-phones/_end")
         .join(format!("{}.wav", phoneme));
 
     let mut reader = match WavReader::open(path) {
@@ -164,7 +226,7 @@ impl Audiobank {
     }
 
     let path = self.audio_path.join(format!("{}/", speaker))
-        .join("_diphones/")
+        .join("2-phones/")
         .join(format!("{}_{}.wav", first, second));
 
     let mut reader = match WavReader::open(path) {

src/handlers/audio_synth_handler.rs

@@ -36,6 +36,7 @@ const SPEAKER_PARAM : &'static str = "v";
 const SPEED_PARAM : &'static str = "spd";
 const USE_PHONEMES_PARAM: &'static str = "up";
 const USE_DIPHONES_PARAM: &'static str = "ud";
+const USE_N_PHONES_PARAM: &'static str = "un";
 const USE_WORDS_PARAM: &'static str = "uw";
 const VOLUME_PARAM : &'static str = "vol";
 const MONOPHONE_PADDING_START_PARAM : &'static str = "mps";
@@ -63,6 +64,9 @@ struct SpeakRequest {
   /** Whether to use diphones. */
   pub use_diphones: bool,
 
+  /** Whether to use n-phones. */
+  pub use_n_phones: bool,
+
   /** Whether to use words. */
   pub use_words: bool,
 
@@ -196,6 +200,21 @@ impl SpeakRequest {
           },
         };
 
+        let use_n_phones = match map.get(USE_N_PHONES_PARAM) {
+          None => { true },
+          Some(list) => {
+            match list.get(0) {
+              None => { return speaker_error; },
+              Some(s) => {
+                match FromStr::from_str(s) {
+                  Ok(b) => b,
+                  Err(_) => { return speaker_error; },
+                }
+              },
+            }
+          },
+        };
+
         let use_words = match map.get(USE_WORDS_PARAM) {
           None => { true },
           Some(list) => {
@@ -281,6 +300,7 @@ impl SpeakRequest {
           speed: speed,
           use_phonemes: use_phonemes,
           use_diphones: use_diphones,
+          use_n_phones: use_n_phones,
           use_words: use_words,
           monophone_padding_start: mps,
           monophone_padding_end: mpe,
@@ -367,6 +387,7 @@ impl AudioSynthHandler {
                                        request.use_words,
                                        request.use_phonemes,
                                        request.use_diphones,
+                                       request.use_n_phones,
                                        request.volume,
                                        request.speed,
                                        request.monophone_padding_start,

src/synthesizer.rs

@@ -2,6 +2,7 @@
 
 use arpabet::ArpabetDictionary;
 use audiobank::Audiobank;
+use audiobank::SampleBytes;
 use effects::pause::generate_pause;
 use effects::speed::change_speed;
 use effects::volume::change_volume;
@@ -49,6 +50,7 @@ impl Synthesizer {
                   use_words: bool,
                   use_phonemes: bool,
                   use_diphones: bool,
+                  use_n_phones: bool,
                   volume: Option<f32>,
                   speed: Option<f32>,
                   monophone_padding_start: Option<u16>,
@@ -77,10 +79,20 @@ impl Synthesizer {
       let mut word_added = false;
 
       if use_words {
-        word_added = self.concatenate_word(&mut concatenated_waveform, speaker, word);
+        word_added = self.concatenate_word(&mut concatenated_waveform,
+                                           speaker,
+                                           word);
       }
 
+      if !word_added && use_n_phones {
+        // New phoneme algorithm
+        word_added = self.concatenate_n_phone(&mut concatenated_waveform,
+                                              speaker,
+                                              word,
+                                              polyphone_padding_end);
+      }
       if !word_added && use_phonemes {
+        // Old phoneme algorithm
         word_added = self.concatenate_polyphone(&mut concatenated_waveform,
                                                 speaker,
                                                 word,
@@ -242,6 +254,172 @@ impl Synthesizer {
     true
   }
 
+  // TODO: Doc and test.
+  fn concatenate_n_phone(&self,
+                         concatenated_waveform: &mut Vec<i16>,
+                         speaker: &str,
+                         word: &str,
+                         polyphone_padding_end: Option<u16>) -> bool {
+
+    let samples = match self.get_n_phone_samples(speaker, word) {
+      Some(s) => s,
+      None => {
+        return false;
+      },
+    };
+
+
+    for sample in samples {
+      concatenated_waveform.extend(sample);
+    }
+
+    // Insert space after polyphone.
+    if polyphone_padding_end.is_some() {
+      let pause = generate_pause(polyphone_padding_end.unwrap());
+      concatenated_waveform.extend(pause);
+    }
+
+    true
+  }
+
+
+  // TODO: Make this super easy to test (and write some tests).
+  // TODO: Make this super efficient.
+  fn get_n_phone_samples(&self, speaker: &str, word: &str)
+      -> Option<Vec<SampleBytes>> {
+
+    let polyphone = match self.arpabet_dictionary.get_polyphone(word) {
+      None => {
+        return None;
+      },
+      Some(p) => p,
+    };
+
+    let mut fulfilled : Vec<bool> = Vec::with_capacity(polyphone.len());
+    let mut chunks : Vec<Option<SampleBytes>> =
+        Vec::with_capacity(polyphone.len());
+
+    // Use this to debug the synthesis.
+    let mut debug : Vec<Option<String>> = Vec::with_capacity(polyphone.len());
+
+    for i in 0..polyphone.len() {
+      fulfilled.push(false);
+      chunks.push(None);
+      debug.push(None);
+    }
+
+    // 4-phone
+    if polyphone.len() >= 4 {
+      let range = polyphone.len() - 3;
+      for i in 0..range {
+        if fulfilled[i] {
+          continue;
+        }
+
+        let candidate_n_phone = &polyphone[i..i+4];
+
+        let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone);
+
+        if phone.is_some() {
+          chunks[i] = phone;
+          debug[i] = Some(polyphone[i..i+4].join("_"));
+
+          for j in i..i+4 {
+            fulfilled[j] = true;
+          }
+        }
+      }
+    }
+
+    info!("4-Fulfilled: {:?}", fulfilled);
+
+    // 3-phone
+    if polyphone.len() >= 3 {
+      let range = polyphone.len() - 2;
+      for i in 0..range {
+        if fulfilled[i] {
+          continue;
+        }
+
+        let candidate_n_phone = &polyphone[i..i+3];
+
+        let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone);
+
+        if phone.is_some() {
+          chunks[i] = phone;
+          debug[i] = Some(polyphone[i..i+3].join("_"));
+          for j in i..i+3 {
+            fulfilled[j] = true;
+          }
+        }
+      }
+    }
+
+    info!("3-Fulfilled: {:?}", fulfilled);
+
+    // 2-phone
+    if polyphone.len() >= 2 {
+      let range = polyphone.len() - 1;
+      for i in 0..range {
+        //println!("i: {}, range: {}", i, range);
+        if fulfilled[i] {
+          continue;
+        }
+
+        let candidate_n_phone = &polyphone[i..i+2];
+
+        let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone);
+
+        if phone.is_some() {
+          chunks[i] = phone;
+          debug[i] = Some(polyphone[i..i+2].join("_"));
+          for j in i..i+2 {
+            fulfilled[j] = true;
+          }
+        }
+      }
+    }
+
+    info!("2-Fulfilled: {:?}", fulfilled);
+
+    // 1-phone
+    for i in 0..polyphone.len() {
+      if fulfilled[i] {
+        continue;
+      }
+
+      let phone = self.audiobank.get_phoneme(speaker, &polyphone[i]);
+
+      if phone.is_some() {
+        chunks[i] = phone;
+        debug[i] = Some(polyphone[i].to_string());
+        fulfilled[i] = true;
+      }
+    }
+
+    info!("1-Fulfilled: {:?}", fulfilled);
+
+    for x in fulfilled {
+      if !x {
+        return None;
+      }
+    }
+
+    let mut debug_str : Vec<String> = debug.into_iter()
+      .map(|x| if x.is_some() { x.unwrap() } else { "None".to_string() })
+      .collect();
+
+    info!("Comprised of: {:?}", debug_str);
+
+    let results : Vec<SampleBytes> = chunks.into_iter()
+      .filter_map(|x| x) // Woo, already Option<T>!
+      .collect();
+
+    info!("Results Length: {} of {} phones", results.len(), polyphone.len());
+
+    Some(results)
+  }
+
   /// Concatenate a sound effect to the waveform we're building. Returns
   /// whether or not the effect was successfully found and concatenated.
   fn concatenate_misc(&self, concatenated_waveform: &mut Vec<i16>,

src/web/js/events.js

@@ -42,6 +42,7 @@
           speed = $('input#speed').val(),
           up = $('#use_phonemes').prop('checked'),
           ud = $('#use_diphones').prop('checked'),
+          un = $('#use_n_phones').prop('checked'),
           uw = $('#use_words').prop('checked'),
           mps = $('input#monophone_padding_start').val(),
           mpe = $('input#monophone_padding_end').val(),
@@ -56,6 +57,7 @@
               + '&ppe=' + ppe
               + '&up=' + up
               + '&ud=' + ud
+              + '&un=' + un
               + '&uw=' + uw;
 
       ev.preventDefault();
@@ -66,6 +68,7 @@
                    speed,
                    up,
                    ud,
+                   un,
                    uw,
                    mps,
                    mpe,

src/web/js/initialize.js

@@ -19,6 +19,7 @@
         speed = Url.getSpeed() || 1.0,
         usePhonemes = Url.getUsePhonemes(),
         useDiphones = Url.getUseDiphones(),
+        useNPhones = Url.getUseNPhones(),
         useWords = Url.getUseWords(),
         mps = Url.getMonophonePaddingStart() || 0,
         mpe = Url.getMonophonePaddingEnd() || 0,
@@ -27,6 +28,7 @@
 
     if (usePhonemes === null) usePhonemes = true;
     if (useDiphones === null) useDiphones = true;
+    if (useNPhones === null) useNPhones = true;
     if (useWords === null) useWords = true;
 
     if (speaker) {
@@ -44,6 +46,7 @@
 
     $('#use_phonemes').prop('checked', usePhonemes);
     $('#use_diphones').prop('checked', useDiphones);
+    $('#use_n_phones').prop('checked', useNPhones);
     $('#use_words').prop('checked', useWords);
 
     $('input#monophone_padding_start').val(mps);

src/web/js/url.js

@@ -22,6 +22,9 @@
     /** JSON key for the `use_diphones` param. */
     _USE_DIPHONES_KEY: 'ud',
 
+    /** JSON key for the `use_n_phones` param. */
+    _USE_N_PHONES_KEY: 'un',
+
     /** JSON key for the `use_words` param. */
     _USE_WORDS_KEY: 'uw',
 
@@ -91,6 +94,16 @@
       }
     },
 
+    /** Get the "use n-phones" from the `window.location`. */
+    getUseNPhones: function() {
+      var state = this.parseState(window.location);
+      if (!state || !(this._USE_N_PHONES_KEY in state)) {
+        return null;
+      } else {
+        return state[this._USE_N_PHONES_KEY];
+      }
+    },
+
     /** Get the "use words" from the `window.location`. */
     getUseWords: function() {
       var state = this.parseState(window.location);
@@ -139,6 +152,7 @@
                        speed,
                        usePhonemes,
                        useDiphones,
+                       useNPhones,
                        useWords,
                        monophonePaddingStart,
                        monophonePaddingEnd,
@@ -150,6 +164,7 @@
                                     speed,
                                     usePhonemes,
                                     useDiphones,
+                                    useNPhones,
                                     useWords,
                                     monophonePaddingStart,
                                     monophonePaddingEnd,
@@ -166,6 +181,7 @@
                          speed,
                          usePhonemes,
                          useDiphones,
+                         useNPhones,
                          useWords,
                          monophonePaddingStart,
                          monophonePaddingEnd,
@@ -184,6 +200,7 @@
       state[this._SPEED_KEY] = speed; // TODO: Filter invalid values.
       state[this._USE_PHONEMES_KEY] = !!usePhonemes;
       state[this._USE_DIPHONES_KEY] = !!useDiphones;
+      state[this._USE_N_PHONES_KEY] = !!useNPhones;
       state[this._USE_WORDS_KEY] = !!useWords;
       state[this._MONOPHONE_PADDING_START_KEY] = monophonePaddingStart;
       state[this._MONOPHONE_PADDING_END_KEY] = monophonePaddingEnd;

web/index.html

@@ -32,13 +32,18 @@
             <div>
               <input type="checkbox" id="use_phonemes" name="use_phonemes"
                      checked="checked">
-              <label for="use_phonemes">Use Phonemes</label>
+              <label for="use_phonemes">Use Phonemes (old algorithm)</label>
             </div>
             <div>
               <input type="checkbox" id="use_diphones" name="use_diphones"
                      checked="checked">
               <label for="use_diphones">Use Diphones</label>
             </div>
+            <div>
+              <input type="checkbox" id="use_n_phones" name="use_n_phones"
+                     checked="checked">
+              <label for="use_n_phones">Use n-phones (new algorithm)</label>
+            </div>
           </div>
 
           <div id="more_options">