Add start/end concept to the n-phone-based synth

src/audiobank.rs

@@ -22,10 +22,17 @@ pub type SampleBytes = Vec<i16>;
  *       /1-phones
  *         /_begin (starting syllable of word)
  *           /IY0.wav
+ *           ...
  *         /_end (ending syllable of word)
  *           /IY0.wav
+ *           ...
+ *         /_middle (syllables only in the middle)
+ *           ...
+ *         /_question (syllables that end in an interrogative tone)
+ *           ...
  *         /IY0.wav
  *         /IY1.wav
+ *         ...
  *       /2-phones
  *         /_begin
  *           /F_R.wav
@@ -41,6 +48,18 @@ pub struct Audiobank {
   audio_path: PathBuf,
 }
 
+#[derive(Debug)]
+pub enum SamplePreference {
+  /// Prefer a word-opening syllable.
+  Begin,
+  /// Prefer syllable sampled from the middle of a word.
+  Middle,
+  /// Prefer syllable taken from the end of a word.
+  End,
+  /// Prefer a syllable taken from the end of a question.
+  EndQuestion,
+}
+
 impl Audiobank {
   /// CTOR.
   pub fn new(audio_path: &str) -> Audiobank {
@@ -76,8 +95,18 @@ impl Audiobank {
     Some(all_samples)
   }
 
-  /** Get an n-phone in the form "{1st}_{2nd}_{...}_{nth}.wav". */
-  pub fn get_n_phone(&self, speaker: &str, n_phone: &[String])
+  /**
+   * Get an n-phone in the form "{1st}_{2nd}_{...}_{nth}.wav".
+   *  - speaker: the voice to use
+   *  - n_phone: the phoneme slice, eg ['AH1', 'NG'].
+   *  - sample_preference: which end of a word to prefer.
+   *  - use_ends: use special "begin"/"end" etc. n-phones.
+   */
+  pub fn get_n_phone(&self,
+                     speaker: &str,
+                     n_phone: &[String],
+                     sample_preference: SamplePreference,
+                     use_ends: bool)
       -> Option<SampleBytes> {
 
     let (directory, filename) = match n_phone.len() {
@@ -117,29 +146,42 @@ impl Audiobank {
       return None;
     }
 
+    // Try to obtain our sample from the end we prefer first.
+    if use_ends {
+      let mut pref_path = self.audio_path.join(format!("{}/", speaker))
+          .join(directory);
+
+      pref_path = match sample_preference {
+        SamplePreference::Begin => { pref_path.join("_begin") },
+        SamplePreference::Middle => { pref_path.join("_middle") },
+        SamplePreference::End => { pref_path.join("_end") },
+        SamplePreference::EndQuestion => { pref_path.join("_question") },
+      };
+
+      pref_path = pref_path.join(&filename);
+
+      match self.read_wave_file(&pref_path) {
+        Some(p) => {
+          info!("Used {:?} for {:?}", sample_preference, n_phone);
+          return Some(p)
+        },
+        None => {},
+      }
+    }
+
     let path = self.audio_path.join(format!("{}/", speaker))
         .join(directory)
         .join(filename);
 
-    let mut reader = match WavReader::open(path) {
-      Err(_) => { return None; },
-      Ok(reader) => reader,
-    };
-
-    // TODO: Inefficient.
-    let mut all_samples = Vec::new();
-    let samples = reader.samples::<i16>();
-    for sample in samples {
-      all_samples.push(sample.unwrap());
-    }
-
-    Some(all_samples)
+    self.read_wave_file(&path)
   }
 
   /**
-   * Get the wav data for the (speaker,phoneme), or None if it does not exist.
+   * Get the wav data for the (speaker,phoneme), or None if it does not
+   * exist.
    */
-  pub fn get_phoneme(&self, speaker: &str, phoneme: &str) -> Option<Vec<i16>> {
+  pub fn get_phoneme(&self, speaker: &str, phoneme: &str)
+      -> Option<Vec<i16>> {
     if check_path(speaker).is_err() || check_path(phoneme).is_err() {
       return None;
     }
@@ -295,6 +337,23 @@ impl Audiobank {
     let reader = try!(WavReader::open(path));
     Ok(reader.spec())
   }
+
+  /// Read a waveform.
+  fn read_wave_file(&self, path: &PathBuf) -> Option<Vec<i16>> {
+    let mut reader = match WavReader::open(path) {
+      Err(_) => { return None; },
+      Ok(reader) => reader,
+    };
+
+    // TODO: Inefficient.
+    let mut all_samples = Vec::new();
+    let samples = reader.samples::<i16>();
+    for sample in samples {
+      all_samples.push(sample.unwrap());
+    }
+
+    Some(all_samples)
+  }
 }
 
 fn check_path(path: &str) -> Result<(), SynthError> {

src/handlers/audio_synth_handler.rs

@@ -38,6 +38,7 @@ const USE_PHONEMES_PARAM: &'static str = "up";
 const USE_DIPHONES_PARAM: &'static str = "ud";
 const USE_N_PHONES_PARAM: &'static str = "un";
 const USE_WORDS_PARAM: &'static str = "uw";
+const USE_ENDS_PARAM: &'static str = "ue";
 const VOLUME_PARAM : &'static str = "vol";
 const MONOPHONE_PADDING_START_PARAM : &'static str = "mps";
 const MONOPHONE_PADDING_END_PARAM : &'static str = "mpe";
@@ -70,6 +71,9 @@ struct SpeakRequest {
   /** Whether to use words. */
   pub use_words: bool,
 
+  /** Whether to use "ends": start, end, etc. */
+  pub use_ends: bool,
+
   /** Padding before a monophone. */
   pub monophone_padding_start: Option<u16>,
 
@@ -230,6 +234,21 @@ impl SpeakRequest {
           },
         };
 
+        let use_ends = match map.get(USE_ENDS_PARAM) {
+          None => { true },
+          Some(list) => {
+            match list.get(0) {
+              None => { return speaker_error; },
+              Some(s) => {
+                match FromStr::from_str(s) {
+                  Ok(b) => b,
+                  Err(_) => { return speaker_error; },
+                }
+              },
+            }
+          },
+        };
+
         let mps : Option<u16> = match map.get(MONOPHONE_PADDING_START_PARAM) {
           None => { None },
           Some(list) => {
@@ -302,6 +321,7 @@ impl SpeakRequest {
           use_diphones: use_diphones,
           use_n_phones: use_n_phones,
           use_words: use_words,
+          use_ends: use_ends,
           monophone_padding_start: mps,
           monophone_padding_end: mpe,
           polyphone_padding_end: ppe,
@@ -388,6 +408,7 @@ impl AudioSynthHandler {
                                        request.use_phonemes,
                                        request.use_diphones,
                                        request.use_n_phones,
+                                       request.use_ends,
                                        request.volume,
                                        request.speed,
                                        request.monophone_padding_start,
@@ -424,9 +445,10 @@ impl AudioSynthHandler {
     }
 
     let mut use_byte = 0u8;
-    if request.use_phonemes { use_byte |= (1 << 1); }
-    if request.use_diphones { use_byte |= (1 << 2); }
-    if request.use_words { use_byte |= (1 << 3); }
+    if request.use_phonemes { use_byte |= 1 << 1; }
+    if request.use_diphones { use_byte |= 1 << 2; }
+    if request.use_words { use_byte |= 1 << 3; }
+    if request.use_ends { use_byte |= 1 << 4; }
 
     println!("Use byte: {}", use_byte);
 

src/synthesizer.rs

@@ -3,6 +3,7 @@
 use arpabet::ArpabetDictionary;
 use audiobank::Audiobank;
 use audiobank::SampleBytes;
+use audiobank::SamplePreference;
 use effects::pause::generate_pause;
 use effects::speed::change_speed;
 use effects::volume::change_volume;
@@ -51,6 +52,7 @@ impl Synthesizer {
                   use_phonemes: bool,
                   use_diphones: bool,
                   use_n_phones: bool,
+                  use_ends: bool,
                   volume: Option<f32>,
                   speed: Option<f32>,
                   monophone_padding_start: Option<u16>,
@@ -89,7 +91,8 @@ impl Synthesizer {
         word_added = self.concatenate_n_phone(&mut concatenated_waveform,
                                               speaker,
                                               word,
-                                              polyphone_padding_end);
+                                              polyphone_padding_end,
+                                              use_ends);
       }
       if !word_added && use_phonemes {
         // Old phoneme algorithm
@@ -261,9 +264,12 @@ impl Synthesizer {
                          concatenated_waveform: &mut Vec<i16>,
                          speaker: &str,
                          word: &str,
-                         polyphone_padding_end: Option<u16>) -> bool {
+                         polyphone_padding_end: Option<u16>,
+                         use_ends: bool) -> bool {
 
-    let samples = match self.get_n_phone_samples(speaker, word) {
+    let pref = SamplePreference::Begin;
+
+    let samples = match self.get_n_phone_samples(speaker, word, pref, use_ends) {
       Some(s) => s,
       None => {
         return false;
@@ -287,7 +293,11 @@ impl Synthesizer {
 
   // TODO: Make this super easy to test (and write some tests).
   // TODO: Make this super efficient.
-  fn get_n_phone_samples(&self, speaker: &str, word: &str)
+  fn get_n_phone_samples(&self,
+                         speaker: &str,
+                         word: &str,
+                         sample_preference: SamplePreference,
+                         use_ends: bool)
       -> Option<Vec<SampleBytes>> {
 
     let polyphone = match self.arpabet_dictionary.get_polyphone(word) {
@@ -302,7 +312,8 @@ impl Synthesizer {
         Vec::with_capacity(polyphone.len());
 
     // Use this to debug the synthesis.
-    let mut debug : Vec<Option<String>> = Vec::with_capacity(polyphone.len());
+    let mut debug : Vec<Option<String>> =
+        Vec::with_capacity(polyphone.len());
 
     for i in 0..polyphone.len() {
       fulfilled.push(false);
@@ -314,13 +325,25 @@ impl Synthesizer {
     if polyphone.len() >= 4 {
       let range = polyphone.len() - 3;
       for i in 0..range {
-        if fulfilled[i] || fulfilled[i+1] || fulfilled[i+2] || fulfilled[i+3] {
+        if fulfilled[i]
+            || fulfilled[i+1]
+            || fulfilled[i+2]
+            || fulfilled[i+3] {
           continue;
         }
 
         let candidate_n_phone = &polyphone[i..i+4];
 
-        let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone);
+        let sample_preference = match i {
+          0 => { SamplePreference::Begin },
+          _ if i == range - 1 => { SamplePreference::End },
+          _ => { SamplePreference::Middle },
+        };
+
+        let phone = self.audiobank.get_n_phone(speaker,
+                                               candidate_n_phone,
+                                               sample_preference,
+                                               use_ends);
 
         if phone.is_some() {
           chunks[i] = phone;
@@ -345,7 +368,16 @@ impl Synthesizer {
 
         let candidate_n_phone = &polyphone[i..i+3];
 
-        let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone);
+        let sample_preference = match i {
+          0 => { SamplePreference::Begin },
+          _ if i == range - 1 => { SamplePreference::End },
+          _ => { SamplePreference::Middle },
+        };
+
+        let phone = self.audiobank.get_n_phone(speaker,
+                                               candidate_n_phone,
+                                               sample_preference,
+                                               use_ends);
 
         if phone.is_some() {
           chunks[i] = phone;
@@ -363,14 +395,22 @@ impl Synthesizer {
     if polyphone.len() >= 2 {
       let range = polyphone.len() - 1;
       for i in 0..range {
-        //println!("i: {}, range: {}", i, range);
         if fulfilled[i] || fulfilled[i+1] {
           continue;
         }
 
         let candidate_n_phone = &polyphone[i..i+2];
 
-        let phone = self.audiobank.get_n_phone(speaker, candidate_n_phone);
+        let sample_preference = match i {
+          0 => { SamplePreference::Begin },
+          _ if i == range - 1 => { SamplePreference::End },
+          _ => { SamplePreference::Middle },
+        };
+
+        let phone = self.audiobank.get_n_phone(speaker,
+                                               candidate_n_phone,
+                                               sample_preference,
+                                               use_ends);
 
         if phone.is_some() {
           chunks[i] = phone;
@@ -390,7 +430,15 @@ impl Synthesizer {
         continue;
       }
 
-      let phone = self.audiobank.get_phoneme(speaker, &polyphone[i]);
+      // TODO -
+      /*let sample_preference = match i {
+        0 => { SamplePreference::Begin },
+        _ if i == polyphone.len() - 1 => { SamplePreference::Middle },
+        _ => { SamplePreference::End },
+      };*/
+
+      let phone = self.audiobank.get_phoneme(speaker,
+                                             &polyphone[i]);
 
       if phone.is_some() {
         chunks[i] = phone;
@@ -417,7 +465,9 @@ impl Synthesizer {
       .filter_map(|x| x) // Woo, already Option<T>!
       .collect();
 
-    info!("Results Length: {} of {} phones", results.len(), polyphone.len());
+    info!("Results Length: {} of {} phones",
+          results.len(),
+          polyphone.len());
 
     Some(results)
   }

src/web/js/events.js

@@ -59,6 +59,7 @@
           ud = $('#use_diphones').prop('checked'),
           un = $('#use_n_phones').prop('checked'),
           uw = $('#use_words').prop('checked'),
+          ue = $('#use_ends').prop('checked'),
           mps = $('input#monophone_padding_start').val(),
           mpe = $('input#monophone_padding_end').val(),
           ppe = $('input#polyphone_padding_end').val(),
@@ -73,7 +74,8 @@
               + '&up=' + up
               + '&ud=' + ud
               + '&un=' + un
-              + '&uw=' + uw;
+              + '&uw=' + uw
+              + '&ue=' + ue;
 
       ev.preventDefault();
 
@@ -85,6 +87,7 @@
                    ud,
                    un,
                    uw,
+                   ue,
                    mps,
                    mpe,
                    ppe);

src/web/js/initialize.js

@@ -21,6 +21,7 @@
         useDiphones = Url.getUseDiphones(),
         useNPhones = Url.getUseNPhones(),
         useWords = Url.getUseWords(),
+        useEnds = Url.getUseEnds(),
         mps = Url.getMonophonePaddingStart() || 0,
         mpe = Url.getMonophonePaddingEnd() || 0,
         ppe = Url.getPolyphonePaddingEnd() || 600,
@@ -30,6 +31,7 @@
     if (useDiphones === null) useDiphones = true;
     if (useNPhones === null) useNPhones = true;
     if (useWords === null) useWords = true;
+    if (useEnds === null) useEnds = true;
 
     if (speaker) {
       $('select').val(speaker); // TODO: whitelist
@@ -48,6 +50,7 @@
     $('#use_diphones').prop('checked', useDiphones);
     $('#use_n_phones').prop('checked', useNPhones);
     $('#use_words').prop('checked', useWords);
+    $('#use_ends').prop('checked', useEnds);
 
     $('input#monophone_padding_start').val(mps);
     $('input#monophone_padding_end').val(mpe);

src/web/js/url.js

@@ -28,6 +28,9 @@
     /** JSON key for the `use_words` param. */
     _USE_WORDS_KEY: 'uw',
 
+    /** JSON key for the `use_ends` param. */
+    _USE_ENDS_KEY: 'ue',
+
     _MONOPHONE_PADDING_START_KEY: 'mps',
     _MONOPHONE_PADDING_END_KEY: 'mpe',
     _POLYPHONE_PADDING_END_KEY: 'ppe',
@@ -114,6 +117,16 @@
       }
     },
 
+    /** Get the "use ends" from the `window.location`. */
+    getUseEnds: function() {
+      var state = this.parseState(window.location);
+      if (!state || !(this._USE_ENDS_KEY in state)) {
+        return null;
+      } else {
+        return state[this._USE_ENDS_KEY];
+      }
+    },
+
     /** Get the MPS value from the `window.location`. */
     getMonophonePaddingStart: function() {
       var state = this.parseState(window.location);
@@ -154,6 +167,7 @@
                        useDiphones,
                        useNPhones,
                        useWords,
+                       useEnds,
                        monophonePaddingStart,
                        monophonePaddingEnd,
                        polyphonePaddingEnd) {
@@ -166,6 +180,7 @@
                                     useDiphones,
                                     useNPhones,
                                     useWords,
+                                    useEnds,
                                     monophonePaddingStart,
                                     monophonePaddingEnd,
                                     polyphonePaddingEnd);
@@ -183,6 +198,7 @@
                          useDiphones,
                          useNPhones,
                          useWords,
+                         useEnds,
                          monophonePaddingStart,
                          monophonePaddingEnd,
                          polyphonePaddingEnd) {
@@ -202,6 +218,7 @@
       state[this._USE_DIPHONES_KEY] = !!useDiphones;
       state[this._USE_N_PHONES_KEY] = !!useNPhones;
       state[this._USE_WORDS_KEY] = !!useWords;
+      state[this._USE_ENDS_KEY] = !!useEnds;
       state[this._MONOPHONE_PADDING_START_KEY] = monophonePaddingStart;
       state[this._MONOPHONE_PADDING_END_KEY] = monophonePaddingEnd;
       state[this._POLYPHONE_PADDING_END_KEY] = polyphonePaddingEnd;

web/index.html

@@ -48,6 +48,11 @@
                      checked="checked">
               <label for="use_n_phones">Use n-phones (new algorithm)</label>
             </div>
+            <div>
+              <input type="checkbox" id="use_ends" name="use_ends"
+                     checked="checked">
+              <label for="use_ends">Use ends of n-phones (new algorithm)</label>
+            </div>
           </div>
 
           <div id="more_options">