diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 8830f96ae1..65abdfed72 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -77,7 +77,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result) -> Result return Err(ProcessSingleJobError::Other(anyhow!("tts model not found: {:?}", tts_model_token))), Some(model) => model, }; - let job_success_result = match tts_model.tts_model_type { + let job_success_result: JobSuccessResult = match tts_model.tts_model_type { TtsModelType::Tacotron2 => { tacotron2_v2_early_fakeyou::process_job::process_job(ProcessJobArgs { job_dependencies, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/download_embedding_file.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/download_embedding_file.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs index ea843fdaec..aaa6906e10 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/model_downloaders.rs @@ -1 +1,91 @@ -use crate::util::model_downloader::ModelDownloader; \ No newline at end of file +use crate::util::model_downloader::ModelDownloader; + +pub struct VallEXDownloaders { + pub vall_e_x: String, + pub whisper_lg: String, + pub whisper_md: String, + pub vocos_encodec_24khz: String +} + +crate::impl_model_downloader!( + VallEXModelMapping, + "Vall-e-x Model Mapping", + "VALL_E_X_MAPPING_BUCKET_PATH", + "/dependencies/zero_shot_tts/vall-e-x_1.0", + "VALL_E_X_MAPPING_FILESYSTEM_PATH", + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0" + ); + + crate::impl_model_downloader!( + WhisperLargeModelMapping, + "whisper Large Model Mapping", + "WHISPER_LARGE_MAPPING_BUCKET_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/whisper-large", + "WHISPER_LARGE_MAPPING_FILESYSTEM_PATH", + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0/whisper-large" + ); + + crate::impl_model_downloader!( + WhisperMediumModelMapping, + "whisper Medium Model Mapping", + "WHISPER_MEDIUM_TENSOR_BUCKET_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/whisper-medium", + "WHISPER_MEDIUM_TENSOR_FILESYSTEM_PATH", + "/tmp/downloads/zero_shot_tts/vall-e-x_1.0/whisper-medium" + ); + + crate::impl_model_downloader!( + VocosEncodecModelMapping, + "Vocos Encodec Model Mapping", + "VOCOS_ENCODEC_TENSOR_BUCKET_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin", + "VOCOS_ENCODEC_TENSOR_FILESYSTEM_PATH", + "dependencies/zero_shot_tts/vall-e-x_1.0/encodec_pytorch_model.bin" + ); + +impl VallEXDownloaders { + pub fn build_all_from_env() -> Self { + Self { + vall_e_x: VallEXModelMapping::from_env(), + whisper_lg: WhisperLargeModelMapping::from_env(), + whisper_md: WhisperMediumModelMapping::from_env(), + vocos_encodec_24khz: VocosEncodecModelMapping::from_env() + } + } + + pub fn all_downloaders(&self) -> Vec<&dyn ModelDownloader> { + vec![ + &self.vall_e_x, + &self.whisper_lg, + &self.whisper_md, + &self.vocos_encodec_24khz, + ] + } + } + + + + mod test { + // use std::path::Path; + + // use crate::job::job_types::lipsync::sad_talker::model_downloaders::{SadTalkerModelFirstMapping, SadTalkerModelSecondMapping}; + // use crate::util::model_downloader::ModelDownloader; + + // #[test] + // fn test_downloader_model_first_mapping() { + // let model = SadTalkerModelFirstMapping::default(); + // assert_eq!(model.get_model_name(), "sad talker model first mapping"); + // assert_eq!(model.get_cloud_bucket_path(), "/animation_sadtalker/model/mapping_00109-model.pth.tar"); + // assert_eq!(model.get_filesystem_path(), Path::new("/tmp/downloads/sadtalker/mapping_00109-model.pth.tar")); + // } + + // #[test] + // fn test_downloader_model_second_mapping() { + // let model = SadTalkerModelSecondMapping::default(); + // assert_eq!(model.get_model_name(), "sad talker model second mapping"); + // assert_eq!(model.get_cloud_bucket_path(), "/animation_sadtalker/model/mapping_00229-model.pth.tar"); + // assert_eq!(model.get_filesystem_path(), Path::new("/tmp/downloads/sadtalker/mapping_00229-model.pth.tar")); + // } + + } + \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs new file mode 100644 index 0000000000..2afc067c12 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/process_job.rs @@ -0,0 +1,73 @@ +use std::fs::read_to_string; +use std::time::Instant; + +use anyhow::anyhow; +use log::{error, info, warn}; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use container_common::filesystem::check_file_exists::check_file_exists; +use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; +use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::file_size::file_size; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_face_animation::{insert_media_file_from_face_animation, InsertArgs}; +use tokens::users::user::UserToken; + +use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +//use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; +//use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; +//use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; + +use crate::job_dependencies::JobDependencies; + +pub struct VALL_E_XProcessJobArgs<'a> { + pub job_dependencies: &'a JobDependencies, + pub job: &'a AvailableInferenceJob, + } + + pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result { + + // get job args and dependencies and execute them with the inputs + let job = args.job; + let deps = args.job_dependencies; + + let mut job_progress_reporter = args.job_dependencies + .job_progress_reporter + .new_generic_inference(job.inference_job_token.as_str()) + .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; + + // validate the inputs on the job + let job_args = validate_job(job)?; + + + // Need to download the models + info!("Download models (if not present)..."); + let mut i : usize = 0; + for downloader in deps.job_type_details.sad_talker.downloaders.all_downloaders() { + + // Temporary debugging + info!("Downloader {}", i); + i = i + 1; + + let result = downloader.download_if_not_on_filesystem( + &args.job_dependencies.private_bucket_client, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_downloads, + ).await; + + if let Err(e) = result { + error!("could not download: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + } + + // Download embeddings + + // + + + } \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/validate_job.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs index 0ae673cd39..38e272a3e5 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs @@ -20,80 +20,73 @@ use crate::job::job_loop::command_exit_status::CommandExitStatus; pub enum ExecutableOrCommand { /// Eg. `inference.py` Executable(PathBuf), - /// Eg. `python3 inference.py` Command(String), } + pub struct InferenceArgs<'s, P: AsRef> { /// --driven_audio: path to the input audio - pub input_audio: P, + pub input_audio: P, // Change to files /// --source_image: path to the input image (or video) - pub input_image: P, - + pub input_embedding: P, + pub input_text: P, /// --result_dir: path to directory work is performed pub work_dir: P, - /// --result_file: path to final file output pub output_file: P, - pub stderr_output_file: P, - - /// --still: less animation - pub make_still: bool, - - /// --enhancer: "gfpgan" - pub maybe_enhancer: Option<&'s str>, - - /// --preprocess: "crop", etc. - pub maybe_preprocess: Option<&'s str>, - - // TODO: Other SadTalker args } +pub struct VallEXInferenceCommand { -pub impl VallEXInferenceCommand { - pub fn new>( - root_code_directory: P, - executable_or_command: ExecutableOrCommand, - maybe_virtual_env_activation_command: Option<&str>, - maybe_default_config_path: Option

, - maybe_docker_options: Option, - maybe_execution_timeout: Option, - alternate_checkpoint_dir: Option - ) -> Self { - Self { - root_code_directory: root_code_directory.as_ref().to_path_buf(), - executable_or_command, - maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), - maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), - maybe_docker_options, - maybe_execution_timeout, - alternate_checkpoint_dir, - } - } +} +// pub impl VallEXInferenceCommand { +// pub fn new>( +// root_code_directory: P, +// executable_or_command: ExecutableOrCommand, +// maybe_virtual_env_activation_command: Option<&str>, +// maybe_default_config_path: Option

, +// maybe_docker_options: Option, +// maybe_execution_timeout: Option, +// alternate_checkpoint_dir: Option +// ) -> Self { +// Self { +// root_code_directory: root_code_directory.as_ref().to_path_buf(), +// executable_or_command, +// maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), +// maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), +// maybe_docker_options, +// maybe_execution_timeout, +// alternate_checkpoint_dir, +// } +// } +// } + +pub struct VallEXCreateEmbeddingCommand { + } -pub impl VallEXCreateEmbeddingCommand { - pub fn new>( - root_code_directory: P, - executable_or_command: ExecutableOrCommand, - maybe_virtual_env_activation_command: Option<&str>, - maybe_default_config_path: Option

, - maybe_docker_options: Option, - maybe_execution_timeout: Option, - alternate_checkpoint_dir: Option - ) -> Self { - Self { - root_code_directory: root_code_directory.as_ref().to_path_buf(), - executable_or_command, - maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), - maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), - maybe_docker_options, - maybe_execution_timeout, - alternate_checkpoint_dir, - } - } -} \ No newline at end of file +// pub impl VallEXCreateEmbeddingCommand { +// pub fn new>( +// root_code_directory: P, +// executable_or_command: ExecutableOrCommand, +// maybe_virtual_env_activation_command: Option<&str>, +// maybe_default_config_path: Option

, +// maybe_docker_options: Option, +// maybe_execution_timeout: Option, +// alternate_checkpoint_dir: Option +// ) -> Self { +// Self { +// root_code_directory: root_code_directory.as_ref().to_path_buf(), +// executable_or_command, +// maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), +// maybe_default_config_path: maybe_default_config_path.map(|p| p.as_ref().to_path_buf()), +// maybe_docker_options, +// maybe_execution_timeout, +// alternate_checkpoint_dir, +// } +// } +// } \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 340ed78c50..9f87447404 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -24,8 +24,8 @@ use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::Sa use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; -use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXInferenceCommand; -use crate::job::job_types::tts::vallEX::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; @@ -165,7 +165,7 @@ pub struct JobTypeDetails { pub rvc_v2: RvcV2Details, pub so_vits_svc: SoVitsSvcDetails, pub sad_talker: SadTalkerDetails, - pub vallEX: VallEXDetails + pub vall_e_x: VallEXDetails //pub tacotron2_modern: ..., //pub softvc: ..., } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index d9af8260bb..f25760e7f0 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -55,7 +55,10 @@ use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::Sa use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; + use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; +use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; +use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders; use crate::job::job_types::vc::rvc_v2::pretrained_hubert_model::PretrainedHubertModel; use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::RvcV2InferenceCommand; @@ -311,10 +314,10 @@ async fn main() -> AnyhowResult<()> { hifigan_vocoder_model_filename, hifigan_superres_vocoder_model_filename, }, - vallEX: VallEXDetails { + vall_e_x: VallEXDetails { downloaders: VallEXDownloaders::build_all_from_env(), - inference_command: "", - create_embedding_command: "" + inference_command: VallEXInferenceCommand, + create_embedding_command: VallEXCreateEmbeddingCommand, }, vits: VitsDetails { inference_command: vits_inference_command()?,