diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 554d092fde..05f1227d14 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -9,6 +9,18 @@ HTTP_BIND_ADDRESS=0.0.0.0:11223 # NB: Disable if not testing Redis REDIS_FOR_KEEPALIVE_URL="redis://localhost/0" +# ================================= # +# # +# Vall-E-X # +# # +# ================================= # + +# should rename my folder +# abs path to the folder that contains the inference code need to get that container in the repo need basepath? relative to everyones env +VALL_E_X_INFERENCE_ROOT_DIRECTORY = "/home/tensor/code/Storyteller/storyteller-ml/tts/VALL-E-X" +VALL_E_X_INFERENCE_COMMAND = "python3 main.py" +VALL_E_X_INFERENCE_MAYBE_VENV_COMMAND = "source .env/bin/activate" + # ================================= # # # # RVC (v2) # diff --git a/crates/service/job/inference_job/src/job/job_types/tts/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/mod.rs index 729c17f0ed..26d581e703 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/mod.rs @@ -1,4 +1,4 @@ pub mod process_single_tts_job; pub mod tacotron2_v2_early_fakeyou; pub mod vits; -pub mod vallEX; +pub mod vall_e_x; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs new file mode 100644 index 0000000000..727c4b6166 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/mod.rs @@ -0,0 +1,2 @@ +pub mod model_downloaders; +pub mod vall_e_x_inference_command; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs new file mode 100644 index 0000000000..ea843fdaec --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs @@ -0,0 +1 @@ +use crate::util::model_downloader::ModelDownloader; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs new file mode 100644 index 0000000000..0ae673cd39 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs @@ -0,0 +1,99 @@ +use std::collections::HashSet; +use std::env; +use std::ffi::OsString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::anyhow; +use log::info; +use once_cell::sync::Lazy; +use subprocess::{Popen, PopenConfig, Redirection}; + +use container_common::anyhow_result::AnyhowResult; +use filesys::path_to_string::path_to_string; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; + +use crate::job::job_loop::command_exit_status::CommandExitStatus; + +#[derive(Clone)] +pub enum ExecutableOrCommand { + /// Eg. `inference.py` + Executable(PathBuf), + + /// Eg. `python3 inference.py` + Command(String), +} + + +pub struct InferenceArgs<'s, P: AsRef> { + /// --driven_audio: path to the input audio + pub input_audio: P, + + /// --source_image: path to the input image (or video) + pub input_image: P, + + /// --result_dir: path to directory work is performed + pub work_dir: P, + + /// --result_file: path to final file output + pub output_file: P, + + pub stderr_output_file: P, + + /// --still: less animation + pub make_still: bool, + + /// --enhancer: "gfpgan" + pub maybe_enhancer: Option<&'s str>, + + /// --preprocess: "crop", etc. + pub maybe_preprocess: Option<&'s str>, + + // TODO: Other SadTalker args + } + + +pub impl VallEXInferenceCommand { + pub fn new>( + root_code_directory: P, + executable_or_command: ExecutableOrCommand, + maybe_virtual_env_activation_command: Option<&str>, + maybe_default_config_path: Option

, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + alternate_checkpoint_dir: Option + ) -> Self { + Self { + root_code_directory: root_code_directory.as_ref().to_path_buf(), + executable_or_command, + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + } + } +} + +pub impl VallEXCreateEmbeddingCommand { + pub fn new>( + root_code_directory: P, + executable_or_command: ExecutableOrCommand, + maybe_virtual_env_activation_command: Option<&str>, + maybe_default_config_path: Option

, + maybe_docker_options: Option, + maybe_execution_timeout: Option, + alternate_checkpoint_dir: Option + ) -> Self { + Self { + root_code_directory: root_code_directory.as_ref().to_path_buf(), + executable_or_command, + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), + maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), + maybe_docker_options, + maybe_execution_timeout, + alternate_checkpoint_dir, + } + } +} \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index a7ceb4b90e..340ed78c50 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -23,6 +23,10 @@ use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDown use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; + +use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXInferenceCommand; +use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; + use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::SoVitsSvcInferenceCommand; @@ -203,4 +207,4 @@ pub struct SadTalkerDetails { pub struct VallEXDetails { pub inference_command: VallEXInferenceCommand, pub create_embedding_command: VallEXCreateEmbeddingCommand -} \ No newline at end of file +} diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 0123051b98..d9af8260bb 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -53,10 +53,14 @@ use crate::job::job_loop::main_loop::main_loop; use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; + use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; + use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::SoVitsSvcInferenceCommand; + use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, PretrainedModels, RvcV2Details, SadTalkerDetails, SoVitsSvcDetails, Tacotron2VocodesDetails, VitsDetails, VallEXDetails}; use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand; use crate::util::scoped_execution::ScopedExecution; @@ -308,6 +312,7 @@ async fn main() -> AnyhowResult<()> { hifigan_superres_vocoder_model_filename, }, vallEX: VallEXDetails { + downloaders: VallEXDownloaders::build_all_from_env(), inference_command: "", create_embedding_command: "" },