diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index a54eb32408..a5d3317a66 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -21,6 +21,10 @@ pub enum InferenceModelType { #[serde(rename = "rvc_v2")] RvcV2, + // NB: sad_talker does use user-supplied models, so there is no "model token" + #[serde(rename = "sad_talker")] + SadTalker, + #[serde(rename = "so_vits_svc")] SoVitsSvc, @@ -41,6 +45,7 @@ impl InferenceModelType { pub fn to_str(&self) -> &'static str { match self { Self::RvcV2 => "rvc_v2", + Self::SadTalker => "sad_talker", Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", Self::Vits => "vits", @@ -50,6 +55,7 @@ impl InferenceModelType { pub fn from_str(value: &str) -> Result { match value { "rvc_v2" => Ok(Self::RvcV2), + "sad_talker" => Ok(Self::SadTalker), "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vits" => Ok(Self::Vits), @@ -62,6 +68,7 @@ impl InferenceModelType { // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ InferenceModelType::RvcV2, + InferenceModelType::SadTalker, InferenceModelType::SoVitsSvc, InferenceModelType::Tacotron2, InferenceModelType::Vits, @@ -77,6 +84,7 @@ mod tests { #[test] fn test_serialization() { assert_serialization(InferenceModelType::RvcV2, "rvc_v2"); + assert_serialization(InferenceModelType::SadTalker, "sad_talker"); assert_serialization(InferenceModelType::SoVitsSvc, "so_vits_svc"); assert_serialization(InferenceModelType::Tacotron2, "tacotron2"); assert_serialization(InferenceModelType::Vits, "vits"); @@ -85,6 +93,7 @@ mod tests { #[test] fn to_str() { assert_eq!(InferenceModelType::RvcV2.to_str(), "rvc_v2"); + assert_eq!(InferenceModelType::SadTalker.to_str(), "sad_talker"); assert_eq!(InferenceModelType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(InferenceModelType::Vits.to_str(), "vits"); @@ -93,6 +102,7 @@ mod tests { #[test] fn from_str() { assert_eq!(InferenceModelType::from_str("rvc_v2").unwrap(), InferenceModelType::RvcV2); + assert_eq!(InferenceModelType::from_str("sad_talker").unwrap(), InferenceModelType::SadTalker); assert_eq!(InferenceModelType::from_str("so_vits_svc").unwrap(), InferenceModelType::SoVitsSvc); assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2); assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); @@ -102,8 +112,9 @@ mod tests { fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 4); + assert_eq!(variants.len(), 5); assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2)); + assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 22dfffa9df..4d616476ed 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -26,7 +26,7 @@ pub struct AvailableInferenceJob { // Inference class pub inference_category: InferenceCategory, - pub maybe_model_type: Option, // TODO: Strongly type. + pub maybe_model_type: Option, pub maybe_model_token: Option, pub maybe_input_source_token: Option, @@ -115,7 +115,11 @@ pub async fn list_available_generic_inference_jobs( .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. inference_category: InferenceCategory::from_str(&record.inference_category) .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. - maybe_model_type: record.maybe_model_type, + maybe_model_type: record.maybe_model_type + .as_deref() + .map(|model_type| InferenceModelType::from_str(model_type)) + .transpose() + .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. maybe_model_token: record.maybe_model_token, maybe_input_source_token: record.maybe_input_source_token, maybe_input_source_token_type: record.maybe_input_source_token_type, diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 19e0ff85ad..436272ca31 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -14,6 +14,7 @@ use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_s use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; use std::time::Instant; +use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; pub async fn process_single_job( job_dependencies: &JobDependencies, @@ -168,8 +169,7 @@ async fn do_process_single_job( let job_success_result = match job.inference_category { InferenceCategory::LipsyncAnimation => { - // TODO: This is not implemented. - return Err(ProcessSingleJobError::Other(anyhow!("not implemented".to_string()))); + process_single_lipsync_job(job_dependencies, job).await? } InferenceCategory::TextToSpeech => { process_single_tts_job(job_dependencies, job).await? diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs new file mode 100644 index 0000000000..d3bc28b0bb --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs @@ -0,0 +1,2 @@ +pub mod process_single_lipsync_job; +pub mod sad_talker; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs new file mode 100644 index 0000000000..3bfb76d5dd --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs @@ -0,0 +1,36 @@ +use anyhow::anyhow; +use crate::job::job_loop::job_success_result::JobSuccessResult; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs; +use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs; +use crate::job::job_types::vc::{rvc_v2, so_vits_svc}; +use crate::job_dependencies::JobDependencies; +use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference; +use std::time::Duration; +use log::{error, info}; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::files::media_upload::MediaUploadToken; + +pub async fn process_single_lipsync_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { + + let job_success_result = match job.maybe_model_type { + Some(InferenceModelType::SadTalker) => { + //rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { + // job_dependencies, + // job, + // vc_model: &vc_model, + // media_upload_token: &media_upload_token, + // media_upload: &media_upload, + //}).await? + return Err(ProcessSingleJobError::Other(anyhow!("not yet implemented"))) + } + Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), + None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), + }; + + Ok(job_success_result) +} + diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/job/inference_job/src/job/job_types/mod.rs b/crates/service/job/inference_job/src/job/job_types/mod.rs index 2a49a1b0f9..f21a93a492 100644 --- a/crates/service/job/inference_job/src/job/job_types/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/mod.rs @@ -1,2 +1,3 @@ +pub mod lipsync; pub mod tts; pub mod vc; \ No newline at end of file