From 2856be7aa96348395f304cf264f70e32fff07b03 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Nov 2023 21:34:32 -0500 Subject: [PATCH] SadTalker dependencies --- .../job/job_types/lipsync/sad_talker/mod.rs | 1 + .../lipsync/sad_talker/process_job.rs | 17 ++++++++------- .../sad_talker/sad_talker_dependencies.rs | 21 +++++++++++++++++++ .../job/inference_job/src/job_dependencies.rs | 12 +---------- .../src/job_specific_dependencies.rs | 9 ++++++++ crates/service/job/inference_job/src/main.rs | 10 +-------- 6 files changed, 43 insertions(+), 27 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_dependencies.rs diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs index bb306e90c4..fa6391096a 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs @@ -4,5 +4,6 @@ pub mod download_image_file; pub mod model_downloaders; pub mod process_job; pub mod resize_image; +pub mod sad_talker_dependencies; pub mod sad_talker_inference_command; pub mod validate_job; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 03783a796e..c0fa86cf4e 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -47,6 +47,13 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result AnyhowResult { + Ok(Self { + downloaders: SadTalkerDownloaders::build_all_from_env(), + ffmpeg_watermark_command: FfmpegLogoWatermarkCommand::from_env()?, + inference_command: SadTalkerInferenceCommand::from_env()?, + }) + } +} diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index c3df91e6f9..0a2ad77659 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -24,15 +24,12 @@ use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improve use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; use newrelic_telemetry::Client as NewRelicClient; -use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders; -use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; use crate::job_specific_dependencies::JobSpecificDependencies; -use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand; use crate::util::scoped_execution::ScopedExecution; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; @@ -163,7 +160,6 @@ pub struct JobCaches { pub struct JobTypeDetails { pub tacotron2_old_vocodes: Tacotron2VocodesDetails, pub vits: VitsDetails, - pub sad_talker: SadTalkerDetails, pub vall_e_x: VallEXDetails //pub tacotron2_modern: ..., //pub softvc: ..., @@ -187,13 +183,7 @@ pub struct VitsDetails { pub inference_command: VitsInferenceCommand, } -pub struct SadTalkerDetails { - pub downloaders: SadTalkerDownloaders, - pub inference_command: SadTalkerInferenceCommand, - pub ffmpeg_watermark_command: FfmpegLogoWatermarkCommand, -} - -// TODO: we will probably want a command type of some kind that implements +// TODO: we will probably want a command type of some kind that implements // some kind of interface that we can just pass strings to. pub struct VallEXDetails { pub downloaders: VallEXDownloaders, diff --git a/crates/service/job/inference_job/src/job_specific_dependencies.rs b/crates/service/job/inference_job/src/job_specific_dependencies.rs index 5b91b198d4..b278054453 100644 --- a/crates/service/job/inference_job/src/job_specific_dependencies.rs +++ b/crates/service/job/inference_job/src/job_specific_dependencies.rs @@ -3,12 +3,14 @@ use log::info; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use errors::AnyhowResult; +use crate::job::job_types::lipsync::sad_talker::sad_talker_dependencies::SadTalkerDependencies; use crate::job::job_types::vc::rvc_v2::rvc_v2_dependencies::RvcV2Dependencies; use crate::job::job_types::vc::so_vits_svc::svc_dependencies::SvcDependencies; use crate::util::scoped_execution::ScopedExecution; pub struct JobSpecificDependencies { pub maybe_rvc_v2_dependencies: Option, + pub maybe_sad_talker_dependencies: Option, pub maybe_svc_dependencies: Option, } @@ -16,6 +18,7 @@ impl JobSpecificDependencies { pub fn setup_for_jobs(scoped_execution: &ScopedExecution) -> AnyhowResult { let mut maybe_rvc_v2_dependencies = None; + let mut maybe_sad_talker_dependencies = None; let mut maybe_svc_dependencies = None; if scoped_execution.can_run_job(InferenceModelType::RvcV2) { @@ -23,6 +26,11 @@ impl JobSpecificDependencies { maybe_rvc_v2_dependencies = Some(RvcV2Dependencies::setup()?); } + if scoped_execution.can_run_job(InferenceModelType::SadTalker) { + info!("Setting SadTalker dependencies..."); + maybe_sad_talker_dependencies = Some(SadTalkerDependencies::setup()?); + } + if scoped_execution.can_run_job(InferenceModelType::SoVitsSvc) { info!("Setting SVC dependencies..."); maybe_svc_dependencies = Some(SvcDependencies::setup()?); @@ -30,6 +38,7 @@ impl JobSpecificDependencies { Ok(JobSpecificDependencies { maybe_rvc_v2_dependencies, + maybe_sad_talker_dependencies, maybe_svc_dependencies }) } diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index c4267b3493..90c034c169 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -50,16 +50,13 @@ use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, Doc use crate::http_server::run_http_server::CreateServerArgs; use crate::http_server::run_http_server::launch_http_server; use crate::job::job_loop::main_loop::main_loop; -use crate::job::job_types::lipsync::sad_talker::model_downloaders::SadTalkerDownloaders; -use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::SadTalkerInferenceCommand; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_inference_command::Tacotron2InferenceCommand; use crate::job::job_types::tts::vall_e_x::model_downloaders::VallEXDownloaders; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXCreateEmbeddingCommand; use crate::job::job_types::tts::vall_e_x::vall_e_x_inference_command::VallEXInferenceCommand; use crate::job::job_types::tts::vits::vits_inference_command::VitsInferenceCommand; -use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, SadTalkerDetails, Tacotron2VocodesDetails, VallEXDetails, VitsDetails}; +use crate::job_dependencies::{FileSystemDetails, JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, Tacotron2VocodesDetails, VallEXDetails, VitsDetails}; use crate::job_specific_dependencies::JobSpecificDependencies; -use crate::util::common_commands::ffmpeg_logo_watermark_command::FfmpegLogoWatermarkCommand; use crate::util::scoped_execution::ScopedExecution; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; @@ -322,11 +319,6 @@ async fn main() -> AnyhowResult<()> { vits: VitsDetails { inference_command: vits_inference_command()?, }, - sad_talker: SadTalkerDetails { - downloaders: SadTalkerDownloaders::build_all_from_env(), - inference_command: SadTalkerInferenceCommand::from_env()?, - ffmpeg_watermark_command: FfmpegLogoWatermarkCommand::from_env()?, - }, }, container: container_environment.clone(), container_db: ContainerEnvironmentArg {