From cc1d7f6de1fa2b5acd0e973db7310d575cdf88ad Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 26 Nov 2023 22:38:05 -0500 Subject: [PATCH] theoretically TTS HTTP works with inference-job --- .idea/dictionaries/bt.xml | 1 + .../config/inference-job.development.env | 2 +- .../tts/tacotron2_v2_early_fakeyou/mod.rs | 1 + .../tacotron2_v2_early_fakeyou/process_job.rs | 45 +++++++++++++------ .../tacotron2_dependencies.rs | 2 + .../tacotron2_inference_command.rs | 14 +----- .../tacotron2_inference_sidecar_client.rs | 45 +++++++++++-------- .../vocoder_option.rs | 13 ++++++ 8 files changed, 78 insertions(+), 45 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/vocoder_option.rs diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 5689e725cd..1cce46b73d 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -47,6 +47,7 @@ vocoder vocoders vocodes + waveglow \ No newline at end of file diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 8ded4961c5..0df22ea669 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -81,7 +81,7 @@ SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vit TT2_LEGACY_ROOT_DIRECTORY="/models/tts" TT2_LEGACY_MAYBE_PYTHON_INTERPRETER="python3.6" -TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="9f1945388110" +TT2_LEGACY_MAYBE_DOCKER_IMAGE_SHA="0d29da13e2df" # ================================= # # # diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs index e96b66620b..d8c0717997 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/mod.rs @@ -6,3 +6,4 @@ pub mod tacotron2_dependencies; pub mod tacotron2_inference_command; pub mod tacotron2_inference_sidecar_client; pub mod tacotron2_sidecar_health_check_client; +pub mod vocoder_option; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index 451a223700..61657b7218 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -25,7 +25,8 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::download_static_dependencies::download_static_dependencies; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::seconds_to_decoder_steps::seconds_to_decoder_steps; -use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor, VocoderForInferenceOption}; +use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::{InferenceArgs, MelMultiplyFactor}; +use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption; use crate::job_dependencies::JobDependencies; use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; @@ -198,7 +199,6 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result) -> Result, } -#[derive(Clone)] -pub enum VocoderForInferenceOption> { - Waveglow { - waveglow_vocoder_checkpoint_path: P, - }, - HifiganSuperres { - hifigan_vocoder_checkpoint_path: P, - hifigan_superres_vocoder_checkpoint_path: P, - } -} - pub enum MelMultiplyFactor { // NB: Default is typically "1.4" DefaultMultiplyFactor, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs index 52ea364e5d..613a47bfb0 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_inference_sidecar_client.rs @@ -6,8 +6,11 @@ use hyper::client::Client; use log::info; use container_common::anyhow_result::AnyhowResult; +use filesys::path_to_string::path_to_string; use mysql_queries::column_types::vocoder_type::VocoderType; +use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::vocoder_option::VocoderForInferenceOption; + /// This is used to talk to the TT2 Python HTTP sidecar (rather than shelling out over bash). /// This keeps models loaded in memory and results in faster inference time. @@ -67,11 +70,8 @@ impl Tacotron2InferenceSidecarClient { raw_text: &str, max_decoder_steps: u32, synthesizer_checkpoint_path: P, - vocoder_type: VocoderType, text_pipeline_type: &str, - hifigan_vocoder_checkpoint_path: P, - hifigan_superres_vocoder_checkpoint_path: P, - waveglow_vocoder_checkpoint_path: P, + vocoder_option: VocoderForInferenceOption

, output_audio_filename: P, output_spectrogram_filename: P, output_metadata_filename: P, @@ -80,23 +80,30 @@ impl Tacotron2InferenceSidecarClient { maybe_custom_mel_multiply_factor: Option, ) -> AnyhowResult<()> { - let waveglow_vocoder_checkpoint_path = waveglow_vocoder_checkpoint_path - .as_ref() - .to_str() - .map(|s| s.to_string()) - .ok_or(anyhow!("bad waveglow vocoder path"))?; + // TODO(bt,2023-11-26): Clean this mutability / multi-configuration up + let vocoder_type; - let hifigan_vocoder_checkpoint_path = hifigan_vocoder_checkpoint_path - .as_ref() - .to_str() - .map(|s| s.to_string()) - .ok_or(anyhow!("bad hifigan vocoder path"))?; + let waveglow_vocoder_checkpoint_path; + let hifigan_vocoder_checkpoint_path; + let hifigan_superres_vocoder_checkpoint_path; - let hifigan_superres_vocoder_checkpoint_path = hifigan_superres_vocoder_checkpoint_path - .as_ref() - .to_str() - .map(|s| s.to_string()) - .ok_or(anyhow!("bad hifigan super resolution vocoder path"))?; + match vocoder_option { + VocoderForInferenceOption::Waveglow { waveglow_vocoder_checkpoint_path : path } => { + vocoder_type = VocoderType::WaveGlow; + waveglow_vocoder_checkpoint_path = path_to_string(path.as_ref()); + hifigan_vocoder_checkpoint_path = "BOGUS_NOT_USED".to_string(); + hifigan_superres_vocoder_checkpoint_path = "BOGUS_NOT_USED".to_string(); + } + VocoderForInferenceOption::HifiganSuperres { + hifigan_vocoder_checkpoint_path: path, + hifigan_superres_vocoder_checkpoint_path : super_res_path, + } => { + vocoder_type = VocoderType::HifiGanSuperResolution; + waveglow_vocoder_checkpoint_path = "BOGUS_NOT_USED".to_string(); + hifigan_vocoder_checkpoint_path = path_to_string(path.as_ref()); + hifigan_superres_vocoder_checkpoint_path = path_to_string(super_res_path.as_ref()); + } + } let synthesizer_checkpoint_path = synthesizer_checkpoint_path .as_ref() diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/vocoder_option.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/vocoder_option.rs new file mode 100644 index 0000000000..1a2e4669f7 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/vocoder_option.rs @@ -0,0 +1,13 @@ +use std::path::Path; + +#[derive(Clone)] +pub enum VocoderForInferenceOption> { + Waveglow { + waveglow_vocoder_checkpoint_path: P, + }, + HifiganSuperres { + hifigan_vocoder_checkpoint_path: P, + hifigan_superres_vocoder_checkpoint_path: P, + } +} +