diff --git a/tts_service/models.toml b/tts_service/models.toml index 445a683669..260ad46dce 100644 --- a/tts_service/models.toml +++ b/tts_service/models.toml @@ -132,6 +132,116 @@ glow_tts_multi_speaker = "glow_ljs_base.torchjit" melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" speaker_id = 50 +[[speakers]] +slug = "multi_dense1_g44" +name = "LJS Dense Multi Base 1 (G44)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +glow_tts_multi_speaker = "glow_ljs_base_dense.torchjit" +melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +speaker_id = 1 + +[[speakers]] +slug = "multi_dense2_g44" +name = "LJS Dense Multi Base 2 (G44)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +glow_tts_multi_speaker = "glow_ljs_base_dense.torchjit" +melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +speaker_id = 2 + +[[speakers]] +slug = "multi_dense1_g142" +name = "LJS Dense Multi Base 1 (G142)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +glow_tts_multi_speaker = "glow_ljs_base_dense2.torchjit" +melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +speaker_id = 1 + +[[speakers]] +slug = "multi_dense2_g142" +name = "LJS Dense Multi Base 2 (G142)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +glow_tts_multi_speaker = "glow_ljs_base_dense2.torchjit" +melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +speaker_id = 2 + +[[speakers]] +slug = "multi_dense1_g264" +name = "LJS Dense Multi Base 1 (G264)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +speaker_id = 1 + +[[speakers]] +slug = "multi_dense2_g264" +name = "LJS Dense Multi Base 2 (G264)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +speaker_id = 2 + +# ------- + +[[speakers]] +slug = "multi_speaker_G309_0" +name = "LJS Dense Multi Speaker Donald Trump (G309)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit" +#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" +#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7125.torchjit" +#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +melgan = "melgan-vocodes.torchjit" +speaker_id = 0 + +[[speakers]] +slug = "multi_speaker_G309_1" +name = "LJS Dense Multi Speaker John Oliver (G309)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit" +#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" +#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit" +melgan = "melgan-vocodes.torchjit" +speaker_id = 1 + +[[speakers]] +slug = "multi_speaker_G309_2" +name = "LJS Dense Multi Speaker Bill Gates (G309)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit" +#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" +#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit" +melgan = "melgan-vocodes.torchjit" +speaker_id = 2 + +[[speakers]] +slug = "multi_speaker_G309_3" +name = "LJS Dense Multi Speaker Arnold (G309)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit" +#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" +#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit" +melgan = "melgan-vocodes.torchjit" +speaker_id = 3 + +[[speakers]] +slug = "multi_speaker_G309_4" +name = "LJS Dense Multi Speaker Reagan (G309)" +model_pipeline = "ArpabetGlowTtsMultiSpeakerMelgan" +#glow_tts_multi_speaker = "glow_ljs_base_dense3.torchjit" +glow_tts_multi_speaker = "glow_ljs_multispeaker_1.torchjit" +#melgan = "melgan_trump_txlearn_2020-05-30_steno_chkpt9800.torchjit" +#melgan = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" +#melgan = "melgan_vocodes1.0_multispeaker_2020_06_19_12d5838_7400.torchjit" +melgan = "melgan-vocodes.torchjit" +speaker_id = 4 # --------------- Model configs --------------- @@ -282,6 +392,22 @@ description = "Mutli-speaker garbage" file_path = "glow_ljs_base.torchjit" description = "Mutli-speaker base model (LambdaLabs expensive Tesla GPU)" +[[glow_tts_multi_speaker]] +file_path = "glow_ljs_base_dense.torchjit" +description = "Denser multi-speaker model. 8 speakers (train on a Tesla GPU)" + +[[glow_tts_multi_speaker]] +file_path = "glow_ljs_base_dense2.torchjit" +description = "Denser multi-speaker model. 8 speakers (train on a Tesla GPU)" + +[[glow_tts_multi_speaker]] +file_path = "glow_ljs_base_dense3.torchjit" +description = "Denser multi-speaker model. 8 speakers (train on a Tesla GPU)" + +[[glow_tts_multi_speaker]] +file_path = "glow_ljs_multispeaker_1.torchjit" +description = "Multispeaker now..." + # ---------------- Melgan ---------------- # TODO: Get a high-quality LJS Melgan @@ -325,3 +451,24 @@ description = "first multivoice (txlearned)" [[melgan]] file_path = "melgan_multivoice-1.0_2020.06.15_12d5838_6725.torchjit" description = "first multivoice (txlearned)" + +[[melgan]] +file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_6675.torchjit" +description = "second multivoice; more non-trump samples (txlearned)" + +[[melgan]] +file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_6900.torchjit" +description = "second multivoice; more non-trump samples (txlearned)" + +[[melgan]] +file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_7125.torchjit" +description = "second multivoice; more non-trump samples (txlearned)" + +[[melgan]] +file_path = "vocodes1.0_multispeaker_2020_05_19_12d5838_7400.torchjit" +description = "second multivoice; more non-trump samples (txlearned)" + +[[melgan]] +file_path = "melgan-vocodes.torchjit" # Stop having to create entries while training ongoing +description = "second multivoice; more non-trump samples (txlearned)" + diff --git a/tts_service/src/inference/audio.rs b/tts_service/src/inference/audio.rs new file mode 100644 index 0000000000..1ab31a33de --- /dev/null +++ b/tts_service/src/inference/audio.rs @@ -0,0 +1,5 @@ + +#[derive(Serialize, Default)] +pub struct Base64WaveAudio { + pub bytes_base64: String, +} diff --git a/tts_service/src/inference/inference.rs b/tts_service/src/inference/inference.rs new file mode 100644 index 0000000000..c093be895a --- /dev/null +++ b/tts_service/src/inference/inference.rs @@ -0,0 +1,17 @@ +use anyhow::Result as AnyhowResult; +use crate::inference::spectrogram::Base64MelSpectrogram; +use crate::inference::audio::Base64WaveAudio; + +pub trait InferencePipelineStart { + fn infer_mel<'a>(&'a self, text: &str, speaker_id: i32) -> AnyhowResult>; +} + +pub trait InferencePipelineMelDone { + //fn infer_audio(self) -> AnyhowResult>; + //fn infer_audio(&'a self); +} + +pub trait InferencePipelineAudioDone { + fn get_base64_mel_spectrogram(&self) -> AnyhowResult; + fn get_base64_audio(&self) -> AnyhowResult; +} diff --git a/tts_service/src/inference/mod.rs b/tts_service/src/inference/mod.rs new file mode 100644 index 0000000000..9b4925d98e --- /dev/null +++ b/tts_service/src/inference/mod.rs @@ -0,0 +1,4 @@ +mod audio; +mod spectrogram; +mod inference; +mod pipelines; \ No newline at end of file diff --git a/tts_service/src/inference/pipelines/glowtts_multispeaker_melgan.rs b/tts_service/src/inference/pipelines/glowtts_multispeaker_melgan.rs new file mode 100644 index 0000000000..eff446ca55 --- /dev/null +++ b/tts_service/src/inference/pipelines/glowtts_multispeaker_melgan.rs @@ -0,0 +1,72 @@ +use anyhow::Result as AnyhowResult; +use crate::model::melgan_model::MelganModel; +use crate::model::arpabet_glow_tts_multi_speaker_model::ArpabetGlowTtsMultiSpeakerModel; +use crate::inference::inference::{InferencePipelineStart, InferencePipelineMelDone, InferencePipelineAudioDone}; +use tch::Tensor; +use crate::inference::audio::Base64WaveAudio; +use crate::inference::spectrogram::Base64MelSpectrogram; + +struct GlowTtsMultiSpeakerMelganPipeline<'a> { + glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel, + melgan: &'a MelganModel, +} + +struct GlowTtsMultiSpeakerMelganPipelineMelDone<'a> { + glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel, + melgan: &'a MelganModel, + mel: Option, +} + +struct GlowTtsMultiSpeakerMelganPipelineAudioDone<'a> { + glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel, + melgan: &'a MelganModel, + mel: Option, + wave_audio: Option, +} + +impl <'a> GlowTtsMultiSpeakerMelganPipeline <'a> { + pub fn new(glow_tts: &'a ArpabetGlowTtsMultiSpeakerModel, melgan: &'a MelganModel) -> Self { + Self { + glow_tts, + melgan, + } + } +} + +impl InferencePipelineStart for GlowTtsMultiSpeakerMelganPipeline <'_> { + fn infer_mel<'a>(&'a self, text: &str, speaker_id: i32) -> AnyhowResult> { + let inner = GlowTtsMultiSpeakerMelganPipelineMelDone { + glow_tts: self.glow_tts, + melgan: self.melgan, + mel: None, + }; + + let outer : Box = Box::new(inner); + + Ok(outer) + } +} + +impl <'a> InferencePipelineMelDone for GlowTtsMultiSpeakerMelganPipelineMelDone <'a> { + /*fn infer_audio(&'a self) { + unimplemented!() + }*/ + /*fn infer_audio(self) -> AnyhowResult> { + Ok(Box::new(GlowTtsMultiSpeakerMelganPipelineAudioDone { + glow_tts: self.glow_tts, + melgan: self.melgan, + mel: None, + wave_audio: None, + })) + }*/ +} + +/*impl <'a> InferencePipelineAudioDone for GlowTtsMultiSpeakerMelganPipelineAudioDone <'a> { + fn get_base64_mel_spectrogram(&self) -> AnyhowResult { + unimplemented!() + } + + fn get_base64_audio(&self) -> AnyhowResult { + unimplemented!() + } +}*/ diff --git a/tts_service/src/inference/pipelines/mod.rs b/tts_service/src/inference/pipelines/mod.rs new file mode 100644 index 0000000000..00a5772539 --- /dev/null +++ b/tts_service/src/inference/pipelines/mod.rs @@ -0,0 +1 @@ +mod glowtts_multispeaker_melgan; \ No newline at end of file diff --git a/tts_service/src/inference/spectrogram.rs b/tts_service/src/inference/spectrogram.rs new file mode 100644 index 0000000000..a3dc858869 --- /dev/null +++ b/tts_service/src/inference/spectrogram.rs @@ -0,0 +1,7 @@ + +#[derive(Serialize, Default)] +pub struct Base64MelSpectrogram { + pub bytes_base64: String, + pub width: i64, + pub height: i64, +} diff --git a/tts_service/src/main.rs b/tts_service/src/main.rs index 0c1c21d410..73bdc6b0aa 100644 --- a/tts_service/src/main.rs +++ b/tts_service/src/main.rs @@ -1,3 +1,5 @@ +mod inference; + #[macro_use] extern crate anyhow; #[macro_use] extern crate diesel; #[macro_use] extern crate log;