WIP: pipeline system, complex lifetime and box traits

tts_service/models.toml

@@ -132,6 +132,116 @@ glow_tts_multi_speaker = "glow_ljs_base.torchjit"
 melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
 speaker_id = 50
 
+[[speakers]]
+slug = "multi_dense1_g44"
+name = "LJS Dense Multi Base 1 (G44)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base_dense.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 1
+
+[[speakers]]
+slug = "multi_dense2_g44"
+name = "LJS Dense Multi Base 2 (G44)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base_dense.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 2
+
+[[speakers]]
+slug = "multi_dense1_g142"
+name = "LJS Dense Multi Base 1 (G142)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base_dense2.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 1
+
+[[speakers]]
+slug = "multi_dense2_g142"
+name = "LJS Dense Multi Base 2 (G142)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base_dense2.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 2
+
+[[speakers]]
+slug = "multi_dense1_g264"
+name = "LJS Dense Multi Base 1 (G264)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 1
+
+[[speakers]]
+slug = "multi_dense2_g264"
+name = "LJS Dense Multi Base 2 (G264)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+speaker_id = 2
+
+# -------
+
+[[speakers]]
+slug = "multi_speaker_G309_0"
+name = "LJS Dense Multi Speaker Donald Trump (G309)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
+#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7125.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+melgan = "melgan-vocodes.torchjit"
+speaker_id = 0
+
+[[speakers]]
+slug = "multi_speaker_G309_1"
+name = "LJS Dense Multi Speaker John Oliver (G309)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
+#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit"
+melgan = "melgan-vocodes.torchjit"
+speaker_id = 1
+
+[[speakers]]
+slug = "multi_speaker_G309_2"
+name = "LJS Dense Multi Speaker Bill Gates (G309)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
+#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit"
+melgan = "melgan-vocodes.torchjit"
+speaker_id = 2
+
+[[speakers]]
+slug = "multi_speaker_G309_3"
+name = "LJS Dense Multi Speaker Arnold (G309)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
+#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit"
+melgan = "melgan-vocodes.torchjit"
+speaker_id = 3
+
+[[speakers]]
+slug = "multi_speaker_G309_4"
+name = "LJS Dense Multi Speaker Reagan (G309)"
+model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan"
+#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit"
+glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit"
+#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit"
+#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
+#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit"
+melgan = "melgan-vocodes.torchjit"
+speaker_id = 4
 
 # --------------- Model configs ---------------
 
@@ -282,6 +392,22 @@ description = "Mutli-speaker garbage"
 file_path = "glow_ljs_base.torchjit"
 description = "Mutli-speaker base model (LambdaLabs expensive Tesla GPU)"
 
+[[glow_tts_multi_speaker]]
+file_path = "glow_ljs_base_dense.torchjit"
+description = "Denser multi-speaker model. 8 speakers (train on a Tesla GPU)"
+
+[[glow_tts_multi_speaker]]
+file_path = "glow_ljs_base_dense2.torchjit"
+description = "Denser multi-speaker model. 8 speakers (train on a Tesla GPU)"
+
+[[glow_tts_multi_speaker]]
+file_path = "glow_ljs_base_dense3.torchjit"
+description = "Denser multi-speaker model. 8 speakers (train on a Tesla GPU)"
+
+[[glow_tts_multi_speaker]]
+file_path = "glow_ljs_multispeaker_1.torchjit"
+description = "Multispeaker now..."
+
 # ---------------- Melgan ----------------
 
 # TODO: Get a high-quality LJS Melgan
@@ -325,3 +451,24 @@ description = "first multivoice (txlearned)"
 [[melgan]]
 file_path = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit"
 description = "first multivoice (txlearned)"
+
+[[melgan]]
+file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_6675.torchjit"
+description = "second multivoice; more non-trump samples (txlearned)"
+
+[[melgan]]
+file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_6900.torchjit"
+description = "second multivoice; more non-trump samples (txlearned)"
+
+[[melgan]]
+file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_7125.torchjit"
+description = "second multivoice; more non-trump samples (txlearned)"
+
+[[melgan]]
+file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_7400.torchjit"
+description = "second multivoice; more non-trump samples (txlearned)"
+
+[[melgan]]
+file_path = "melgan-vocodes.torchjit" # Stop having to create entries while training ongoing
+description = "second multivoice; more non-trump samples (txlearned)"
+

tts_service/src/inference/audio.rs

@@ -0,0 +1,5 @@
+
+#[derive(Serialize, Default)]
+pub struct Base64WaveAudio {
+  pub bytes_base64: String,
+}

tts_service/src/inference/inference.rs

@@ -0,0 +1,17 @@
+use anyhow::Result as AnyhowResult;
+use crate::inference::spectrogram::Base64MelSpectrogram;
+use crate::inference::audio::Base64WaveAudio;
+
+pub trait InferencePipelineStart {
+  fn infer_mel<'a>(&'a self, text: &str, speaker_id: i32) -> AnyhowResult<Box<dyn InferencePipelineMelDone + 'a>>;
+}
+
+pub trait InferencePipelineMelDone {
+  //fn infer_audio(self) -> AnyhowResult<Box<dyn InferencePipelineAudioDone + 'a>>;
+  //fn infer_audio(&'a self);
+}
+
+pub trait InferencePipelineAudioDone {
+  fn get_base64_mel_spectrogram(&self) -> AnyhowResult<Base64MelSpectrogram>;
+  fn get_base64_audio(&self) -> AnyhowResult<Base64WaveAudio>;
+}

tts_service/src/inference/mod.rs

@@ -0,0 +1,4 @@
+mod audio;
+mod spectrogram;
+mod inference;
+mod pipelines;

tts_service/src/inference/pipelines/glowtts_multispeaker_melgan.rs

@@ -0,0 +1,72 @@
+use anyhow::Result as AnyhowResult;
+use crate::model::melgan_model::MelganModel;
+use crate::model::arpabet_glow_tts_multi_speaker_model::ArpabetGlowTtsMultiSpeakerModel;
+use crate::inference::inference::{InferencePipelineStart, InferencePipelineMelDone, InferencePipelineAudioDone};
+use tch::Tensor;
+use crate::inference::audio::Base64WaveAudio;
+use crate::inference::spectrogram::Base64MelSpectrogram;
+
+struct GlowTtsMultiSpeakerMelganPipeline<'a> {
+  glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel,
+  melgan: &'a MelganModel,
+}
+
+struct GlowTtsMultiSpeakerMelganPipelineMelDone<'a> {
+  glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel,
+  melgan: &'a MelganModel,
+  mel: Option<Tensor>,
+}
+
+struct GlowTtsMultiSpeakerMelganPipelineAudioDone<'a> {
+  glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel,
+  melgan: &'a MelganModel,
+  mel: Option<Tensor>,
+  wave_audio: Option<Tensor>,
+}
+
+impl <'a> GlowTtsMultiSpeakerMelganPipeline <'a> {
+  pub fn new(glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel, melgan: &'a MelganModel) -> Self {
+    Self {
+      glow_tts,
+      melgan,
+    }
+  }
+}
+
+impl InferencePipelineStart for GlowTtsMultiSpeakerMelganPipeline <'_> {
+  fn infer_mel<'a>(&'a self, text: &str, speaker_id: i32) -> AnyhowResult<Box<dyn InferencePipelineMelDone + 'a>> {
+    let inner = GlowTtsMultiSpeakerMelganPipelineMelDone {
+      glow_tts: self.glow_tts,
+      melgan: self.melgan,
+      mel: None,
+    };
+
+    let outer : Box<dyn InferencePipelineMelDone> = Box::new(inner);
+
+    Ok(outer)
+  }
+}
+
+impl <'a> InferencePipelineMelDone for GlowTtsMultiSpeakerMelganPipelineMelDone <'a> {
+  /*fn infer_audio(&'a self) {
+    unimplemented!()
+  }*/
+  /*fn infer_audio(self) -> AnyhowResult<Box<dyn InferencePipelineAudioDone + 'a>> {
+    Ok(Box::new(GlowTtsMultiSpeakerMelganPipelineAudioDone {
+      glow_tts: self.glow_tts,
+      melgan: self.melgan,
+      mel: None,
+      wave_audio: None,
+    }))
+  }*/
+}
+
+/*impl <'a> InferencePipelineAudioDone for GlowTtsMultiSpeakerMelganPipelineAudioDone <'a> {
+  fn get_base64_mel_spectrogram(&self) -> AnyhowResult<Base64MelSpectrogram> {
+    unimplemented!()
+  }
+
+  fn get_base64_audio(&self) -> AnyhowResult<Base64WaveAudio> {
+    unimplemented!()
+  }
+}*/

tts_service/src/inference/pipelines/mod.rs

@@ -0,0 +1 @@
+mod glowtts_multispeaker_melgan;

tts_service/src/inference/spectrogram.rs

@@ -0,0 +1,7 @@
+
+#[derive(Serialize, Default)]
+pub struct Base64MelSpectrogram {
+  pub bytes_base64: String,
+  pub width: i64,
+  pub height: i64,
+}

tts_service/src/main.rs

@@ -1,3 +1,5 @@
+mod inference;
+
 #[macro_use] extern crate anyhow;
 #[macro_use] extern crate diesel;
 #[macro_use] extern crate log;