diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index a54eb32408..a5d3317a66 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -21,6 +21,10 @@ pub enum InferenceModelType { #[serde(rename = "rvc_v2")] RvcV2, + // NB: sad_talker does use user-supplied models, so there is no "model token" + #[serde(rename = "sad_talker")] + SadTalker, + #[serde(rename = "so_vits_svc")] SoVitsSvc, @@ -41,6 +45,7 @@ impl InferenceModelType { pub fn to_str(&self) -> &'static str { match self { Self::RvcV2 => "rvc_v2", + Self::SadTalker => "sad_talker", Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", Self::Vits => "vits", @@ -50,6 +55,7 @@ impl InferenceModelType { pub fn from_str(value: &str) -> Result { match value { "rvc_v2" => Ok(Self::RvcV2), + "sad_talker" => Ok(Self::SadTalker), "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vits" => Ok(Self::Vits), @@ -62,6 +68,7 @@ impl InferenceModelType { // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ InferenceModelType::RvcV2, + InferenceModelType::SadTalker, InferenceModelType::SoVitsSvc, InferenceModelType::Tacotron2, InferenceModelType::Vits, @@ -77,6 +84,7 @@ mod tests { #[test] fn test_serialization() { assert_serialization(InferenceModelType::RvcV2, "rvc_v2"); + assert_serialization(InferenceModelType::SadTalker, "sad_talker"); assert_serialization(InferenceModelType::SoVitsSvc, "so_vits_svc"); assert_serialization(InferenceModelType::Tacotron2, "tacotron2"); assert_serialization(InferenceModelType::Vits, "vits"); @@ -85,6 +93,7 @@ mod tests { #[test] fn to_str() { assert_eq!(InferenceModelType::RvcV2.to_str(), "rvc_v2"); + assert_eq!(InferenceModelType::SadTalker.to_str(), "sad_talker"); assert_eq!(InferenceModelType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(InferenceModelType::Vits.to_str(), "vits"); @@ -93,6 +102,7 @@ mod tests { #[test] fn from_str() { assert_eq!(InferenceModelType::from_str("rvc_v2").unwrap(), InferenceModelType::RvcV2); + assert_eq!(InferenceModelType::from_str("sad_talker").unwrap(), InferenceModelType::SadTalker); assert_eq!(InferenceModelType::from_str("so_vits_svc").unwrap(), InferenceModelType::SoVitsSvc); assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2); assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); @@ -102,8 +112,9 @@ mod tests { fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 4); + assert_eq!(variants.len(), 5); assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2)); + assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs similarity index 90% rename from crates/lib/mysql_queries/src/payloads/generic_inference_args.rs rename to crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index 4a0f386419..99021de4e3 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -1,3 +1,4 @@ +use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs}; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use errors::AnyhowResult; @@ -47,15 +48,8 @@ pub enum FundamentalFrequencyMethodForJob { pub enum PolymorphicInferenceArgs { /// Lipsync Animation (Short name to save space when serializing) /// This is SadTalker, not Wav2Lip. - La { - #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. - #[serde(skip_serializing_if = "Option::is_none")] - maybe_audio_media_upload_token: Option, + La (LipsyncArgs), - #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. - #[serde(skip_serializing_if = "Option::is_none")] - maybe_image_media_upload_token: Option, - }, /// Text to speech. (Short name to save space when serializing.) Tts { // No arguments yet. @@ -86,6 +80,7 @@ pub enum PolymorphicInferenceArgs { }, } + impl GenericInferenceArgs { pub fn from_json(json: &str) -> AnyhowResult { @@ -117,22 +112,23 @@ impl InferenceCategoryAbbreviated { #[cfg(test)] mod tests { - use crate::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; + use crate::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; + use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs}; #[test] fn typical_lipsync_animation_args_serialize() { let args = GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation), - args: Some(PolymorphicInferenceArgs::La { - maybe_audio_media_upload_token: Some("foo".to_string()), - maybe_image_media_upload_token: Some("bar".to_string()), - }), + args: Some(PolymorphicInferenceArgs::La(LipsyncArgs { + maybe_audio_source: Some(LipsyncAnimationAudioSource::U("foo".to_string())), + maybe_image_source: Some(LipsyncAnimationImageSource::F("bar".to_string())), + })), }; let json = serde_json::ser::to_string(&args).unwrap(); // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating. - assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":"foo","i":"bar"}}}"#.to_string()); + assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":{"U":"foo"},"i":{"F":"bar"}}}}"#.to_string()); // NB: Make sure we don't overflow the DB field capacity (TEXT column). assert!(json.len() < 1000); diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs new file mode 100644 index 0000000000..8871ee399a --- /dev/null +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs @@ -0,0 +1,125 @@ +//! Arguments for lipsync inferences + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LipsyncArgs { + #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_audio_source: Option, + + #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_image_source: Option, +} + +/// Audio sources can be one of several: +/// - F: media_files (todo) +/// - U: media_uploads (legacy) +/// - T: tts_results (legacy) +/// - V: voice_conversion_results (legacy) +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum LipsyncAnimationAudioSource { + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media File Token (media_files table) + /// Serde cannot yet rename enum variants. + F(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media Upload Token (media_uploads table) + /// Serde cannot yet rename enum variants. + U(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// TTS Result Token (tts_results table) + /// Serde cannot yet rename enum variants. + T(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Voice Conversion Result Token (voice_conversion_results table) + /// Serde cannot yet rename enum variants. + V(String), +} + +// TODO: Hydrate to a public enum with easier to read names. +impl LipsyncAnimationAudioSource { + pub fn media_file_token(token: &str) -> Self { + LipsyncAnimationAudioSource::F(token.to_string()) + } + pub fn media_upload_token(token: &str) -> Self { + LipsyncAnimationAudioSource::U(token.to_string()) + } + pub fn tts_result_token(token: &str) -> Self { + LipsyncAnimationAudioSource::T(token.to_string()) + } + pub fn voice_conversion_result_token(token: &str) -> Self { + LipsyncAnimationAudioSource::V(token.to_string()) + } +} + +/// Video sources can be one of several: +/// - F: media_files (todo) +/// - U: media_uploads (legacy) +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum LipsyncAnimationImageSource { + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media File Token (media_files table) + /// Serde cannot yet rename enum variants. + F(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media Upload Token (media_uploads table) + /// Serde cannot yet rename enum variants. + U(String), +} + +impl LipsyncAnimationImageSource { + pub fn media_file_token(token: &str) -> Self { + LipsyncAnimationImageSource::F(token.to_string()) + } + pub fn media_upload_token(token: &str) -> Self { + LipsyncAnimationImageSource::U(token.to_string()) + } +} + +#[cfg(test)] +mod tests { + use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs}; + + #[test] + fn test_media_file() { + let args = LipsyncArgs { + maybe_audio_source: Some(LipsyncAnimationAudioSource::media_file_token("audio_media_file")), + maybe_image_source: Some(LipsyncAnimationImageSource::media_file_token("image_media_file")), + }; + let json = serde_json::ser::to_string(&args).unwrap(); + assert_eq!(json, r#"{"a":{"F":"audio_media_file"},"i":{"F":"image_media_file"}}"#.to_string()); + } + + #[test] + fn test_media_upload() { + let args = LipsyncArgs { + maybe_audio_source: Some(LipsyncAnimationAudioSource::media_upload_token("audio_media_upload")), + maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")), + }; + let json = serde_json::ser::to_string(&args).unwrap(); + assert_eq!(json, r#"{"a":{"U":"audio_media_upload"},"i":{"U":"image_media_upload"}}"#.to_string()); + } + + #[test] + fn test_tts_result() { + let args = LipsyncArgs { + maybe_audio_source: Some(LipsyncAnimationAudioSource::tts_result_token("audio_tts_result")), + maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")), + }; + let json = serde_json::ser::to_string(&args).unwrap(); + assert_eq!(json, r#"{"a":{"T":"audio_tts_result"},"i":{"U":"image_media_upload"}}"#.to_string()); + } + #[test] + fn test_voice_conversion_result() { + let args = LipsyncArgs { + maybe_audio_source: Some(LipsyncAnimationAudioSource::voice_conversion_result_token("audio_voice_conversion_result")), + maybe_image_source: Some(LipsyncAnimationImageSource::media_upload_token("image_media_upload")), + }; + let json = serde_json::ser::to_string(&args).unwrap(); + assert_eq!(json, r#"{"a":{"V":"audio_voice_conversion_result"},"i":{"U":"image_media_upload"}}"#.to_string()); + } +} diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs new file mode 100644 index 0000000000..2e526cc04c --- /dev/null +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs @@ -0,0 +1,2 @@ +pub mod generic_inference_args; +pub mod lipsync_payload; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 3feaa53738..4d616476ed 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -13,7 +13,7 @@ use std::future::Future; use std::path::Path; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use tokens::jobs::inference::InferenceJobToken; -use crate::payloads::generic_inference_args::GenericInferenceArgs; +use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs; /// table: generic_inference_jobs #[derive(Debug)] @@ -26,7 +26,7 @@ pub struct AvailableInferenceJob { // Inference class pub inference_category: InferenceCategory, - pub maybe_model_type: Option, // TODO: Strongly type. + pub maybe_model_type: Option, pub maybe_model_token: Option, pub maybe_input_source_token: Option, @@ -115,7 +115,11 @@ pub async fn list_available_generic_inference_jobs( .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. inference_category: InferenceCategory::from_str(&record.inference_category) .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. - maybe_model_type: record.maybe_model_type, + maybe_model_type: record.maybe_model_type + .as_deref() + .map(|model_type| InferenceModelType::from_str(model_type)) + .transpose() + .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. maybe_model_token: record.maybe_model_token, maybe_input_source_token: record.maybe_input_source_token, maybe_input_source_token_type: record.maybe_input_source_token_type, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs index 95f9a51689..a57994fb54 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use crate::payloads::generic_inference_args::GenericInferenceArgs; +use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index a1998e3361..436272ca31 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -14,6 +14,7 @@ use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_s use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; use std::time::Instant; +use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; pub async fn process_single_job( job_dependencies: &JobDependencies, @@ -167,6 +168,9 @@ async fn do_process_single_job( let mut maybe_result_entity : Option; let job_success_result = match job.inference_category { + InferenceCategory::LipsyncAnimation => { + process_single_lipsync_job(job_dependencies, job).await? + } InferenceCategory::TextToSpeech => { process_single_tts_job(job_dependencies, job).await? } diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs new file mode 100644 index 0000000000..d3bc28b0bb --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/mod.rs @@ -0,0 +1,2 @@ +pub mod process_single_lipsync_job; +pub mod sad_talker; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs new file mode 100644 index 0000000000..3bfb76d5dd --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/process_single_lipsync_job.rs @@ -0,0 +1,36 @@ +use anyhow::anyhow; +use crate::job::job_loop::job_success_result::JobSuccessResult; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs; +use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs; +use crate::job::job_types::vc::{rvc_v2, so_vits_svc}; +use crate::job_dependencies::JobDependencies; +use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference; +use std::time::Duration; +use log::{error, info}; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::files::media_upload::MediaUploadToken; + +pub async fn process_single_lipsync_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { + + let job_success_result = match job.maybe_model_type { + Some(InferenceModelType::SadTalker) => { + //rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { + // job_dependencies, + // job, + // vc_model: &vc_model, + // media_upload_token: &media_upload_token, + // media_upload: &media_upload, + //}).await? + return Err(ProcessSingleJobError::Other(anyhow!("not yet implemented"))) + } + Some(model_type) => return Err(ProcessSingleJobError::Other(anyhow!("wrong model type: {:?}", model_type))), + None => return Err(ProcessSingleJobError::Other(anyhow!("no model type in record"))), + }; + + Ok(job_success_result) +} + diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/job/inference_job/src/job/job_types/mod.rs b/crates/service/job/inference_job/src/job/job_types/mod.rs index 2a49a1b0f9..f21a93a492 100644 --- a/crates/service/job/inference_job/src/job/job_types/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/mod.rs @@ -1,2 +1,3 @@ +pub mod lipsync; pub mod tts; pub mod vc; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index 510501868b..f8fb450ba8 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -15,9 +15,9 @@ use filesys::file_size::file_size; use log::{error, info, warn}; use media::decode_basic_audio_info::decode_basic_audio_file_info; use mimetypes::mimetype_for_file::get_mimetype_for_file; -use mysql_queries::payloads::generic_inference_args::PolymorphicInferenceArgs; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_uploads::get_media_upload_for_inference::{get_media_upload_for_inference, MediaUploadRecordForInference}; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs}; use std::time::Instant; @@ -149,6 +149,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, + PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0, }) .flatten() @@ -157,6 +158,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, + PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, }) .flatten(); @@ -164,6 +166,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, + PolymorphicInferenceArgs::La(_) => None, PolymorphicInferenceArgs::Vc { override_f0_method, .. } => *override_f0_method, }) diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs index 674650af5d..214ee69b78 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs @@ -1,18 +1,17 @@ use anyhow::anyhow; use container_common::anyhow_result::AnyhowResult; +use crate::job::job_loop::command_exit_status::CommandExitStatus; use filesys::path_to_string::path_to_string; use log::info; -use mysql_queries::payloads::generic_inference_args::FundamentalFrequencyMethodForJob; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::FundamentalFrequencyMethodForJob; use once_cell::sync::Lazy; use std::collections::HashSet; use std::env; use std::ffi::OsString; -use std::fs::OpenOptions; use std::path::{Path, PathBuf}; use std::time::Duration; -use subprocess::{ExitStatus, Popen, PopenConfig, Redirection}; +use subprocess::{Popen, PopenConfig}; use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions}; -use crate::job::job_loop::command_exit_status::CommandExitStatus; // These environment vars are not copied over to the subprocess // TODO/FIXME(bt, 2023-05-28): This is horrific security! diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs index c1d09d485c..c360f03f3a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs @@ -15,11 +15,14 @@ use enums::common::visibility::Visibility; use http_server_common::request::get_request_header_optional::get_request_header_optional; use http_server_common::request::get_request_ip::get_request_ip; use log::{info, warn}; -use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs}; use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job}; use std::sync::Arc; use tokens::files::media_upload::MediaUploadToken; use tokens::jobs::inference::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::voice_conversion_results::VoiceConversionResultToken; use tokens::users::user::UserToken; use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL; @@ -35,9 +38,8 @@ const ROUTING_TAG_HEADER_NAME : &'static str = "routing-tag"; pub struct EnqueueLipsyncAnimationRequest { uuid_idempotency_token: String, - // TODO: Eventually accept result tokens for audio. - source_image_media_upload_token: MediaUploadToken, - source_audio_media_upload_token: MediaUploadToken, + audio_source: AudioSource, + image_source: ImageSource, creator_set_visibility: Option, is_storyteller_demo: Option, @@ -46,6 +48,22 @@ pub struct EnqueueLipsyncAnimationRequest { animate_full_body: Option, } +/// Treated as an enum. Only one of these may be set. +#[derive(Deserialize)] +pub struct AudioSource { + maybe_media_file_token: Option, + maybe_media_upload_token: Option, + maybe_tts_result_token: Option, + maybe_voice_conversion_result_token: Option, +} + +/// Treated as an enum. Only one of these may be set. +#[derive(Deserialize)] +pub struct ImageSource { + maybe_media_file_token: Option, + maybe_media_upload_token: Option, +} + #[derive(Serialize)] pub struct EnqueueLipsyncAnimationSuccessResponse { pub success: bool, @@ -190,9 +208,31 @@ pub async fn enqueue_lipsync_animation_handler( // ==================== LOOK UP MODEL INFO ==================== // - // TODO(bt): CHECK DATABASE! - let audio_media_token = request.source_audio_media_upload_token.clone(); - let image_media_token = request.source_image_media_upload_token.clone(); + // TODO(bt): CHECK DATABASE FOR TOKENS! + + let audio_source = { + if let Some(ref token) = request.audio_source.maybe_media_file_token { + LipsyncAnimationAudioSource::media_file_token(token.as_str()) + } else if let Some(ref token) = request.audio_source.maybe_media_upload_token { + LipsyncAnimationAudioSource::media_upload_token(token.as_str()) + } else if let Some(ref token) = request.audio_source.maybe_tts_result_token { + LipsyncAnimationAudioSource::tts_result_token(token) + } else if let Some(ref token) = request.audio_source.maybe_voice_conversion_result_token { + LipsyncAnimationAudioSource::voice_conversion_result_token(token.as_str()) + } else { + return Err(EnqueueLipsyncAnimationError::BadInput("audio source not fully specified".to_string())); + } + }; + + let image_source = { + if let Some(ref token) = request.image_source.maybe_media_file_token { + LipsyncAnimationImageSource::media_file_token(token.as_str()) + } else if let Some(ref token) = request.image_source.maybe_media_upload_token { + LipsyncAnimationImageSource::media_upload_token(token.as_str()) + } else { + return Err(EnqueueLipsyncAnimationError::BadInput("image source not fully specified".to_string())); + } + }; // ==================== CHECK AND ENQUEUE ANIMATION ==================== // @@ -224,28 +264,11 @@ pub async fn enqueue_lipsync_animation_handler( info!("Creating lipsync animation job record..."); - // TODO: Cleanup. - let maybe_args = Some(PolymorphicInferenceArgs::La { - maybe_audio_media_upload_token: Some(audio_media_token.to_string()), - maybe_image_media_upload_token: Some(image_media_token.to_string()), - }); - - // TODO: SadTalker arguments. - //let set_args = request.auto_predict_f0.is_some() - // || request.transpose.is_some() - // || request.override_f0_method.is_some(); - - //if set_args { - // maybe_args = Some(PolymorphicInferenceArgs::Vc { - // auto_predict_f0: request.auto_predict_f0, - // override_f0_method: request.override_f0_method.map(|f| match f { - // FundamentalFrequencyMethod::Crepe => FundamentalFrequencyMethodForJob::Crepe, - // FundamentalFrequencyMethod::Dio => FundamentalFrequencyMethodForJob::Dio, - // FundamentalFrequencyMethod::Harvest => FundamentalFrequencyMethodForJob::Harvest, - // }), - // transpose: request.transpose, - // }); - //} + // TODO: other SadTalker arguments. + let maybe_args = Some(PolymorphicInferenceArgs::La(LipsyncArgs { + maybe_audio_source: Some(audio_source), + maybe_image_source: Some(image_source), + })); let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { uuid_idempotency_token: &request.uuid_idempotency_token, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index 6fa62842a6..a12ad5aadd 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -20,7 +20,7 @@ use http_server_common::request::get_request_api_token::get_request_api_token; use http_server_common::request::get_request_header_optional::get_request_header_optional; use http_server_common::request::get_request_ip::get_request_ip; use log::{info, warn}; -use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated}; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated}; use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use mysql_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; use mysql_queries::queries::tts::tts_models::get_tts_model::TtsModelRecord; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index d94707679e..a163179f0e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -19,7 +19,7 @@ use enums::common::visibility::Visibility; use http_server_common::request::get_request_header_optional::get_request_header_optional; use http_server_common::request::get_request_ip::get_request_ip; use log::{error, info, warn}; -use mysql_queries::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job}; use mysql_queries::queries::voice_conversion::model_info_lite::get_voice_conversion_model_info_lite::get_voice_conversion_model_info_lite_with_connection; use r2d2_redis::redis::Commands;