diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index 19bf5a74c7..c776b40dd5 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -28,6 +28,9 @@ pub enum InferenceCategory { #[serde(rename = "video_filter")] VideoFilter, + + #[serde(rename = "mocap")] + Mocap, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -42,6 +45,7 @@ impl InferenceCategory { Self::TextToSpeech => "text_to_speech", Self::VoiceConversion => "voice_conversion", Self::VideoFilter => "video_filter", + Self::Mocap => "mocap", } } @@ -51,6 +55,7 @@ impl InferenceCategory { "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), "video_filter" => Ok(Self::VideoFilter), + "mocap" => Ok(Self::Mocap), _ => Err(format!("invalid value: {:?}", value)), } } @@ -63,6 +68,7 @@ impl InferenceCategory { Self::TextToSpeech, Self::VoiceConversion, Self::VideoFilter, + Self::Mocap, ]) } } @@ -78,6 +84,7 @@ mod tests { assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech"); assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); assert_serialization(InferenceCategory::VideoFilter, "video_filter"); + assert_serialization(InferenceCategory::Mocap, "mocap") } #[test] @@ -86,6 +93,7 @@ mod tests { assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(InferenceCategory::VideoFilter.to_str(), "video_filter"); + assert_eq!(InferenceCategory::Mocap.to_str(), "mocap"); } #[test] @@ -94,17 +102,19 @@ mod tests { assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); assert_eq!(InferenceCategory::from_str("video_filter").unwrap(), InferenceCategory::VideoFilter); + assert_eq!(InferenceCategory::from_str("mocap").unwrap(), InferenceCategory::Mocap); } #[test] fn all_variants() { // Static check let mut variants = InferenceCategory::all_variants(); - assert_eq!(variants.len(), 4); + assert_eq!(variants.len(), 5); assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VideoFilter)); + assert_eq!(variants.pop_first(), Some(InferenceCategory::Mocap)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index a3ea0bcd29..cd71c3246a 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -33,6 +33,8 @@ pub enum InferenceModelType { VallEX, #[serde(rename = "rerender_a_video")] RerenderAVideo, + #[serde(rename = "mocap_net")] + MocapNet, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -50,6 +52,7 @@ impl InferenceModelType { Self::Vits => "vits", Self::VallEX => "vall_e_x", Self::RerenderAVideo => "rerender_a_video", + Self::MocapNet => "mocap_net", } } @@ -62,6 +65,7 @@ impl InferenceModelType { "vits" => Ok(Self::Vits), "vall_e_x" => Ok(Self::VallEX), "rerender_a_video" => Ok(Self::RerenderAVideo), + "mocap_net" => Ok(Self::MocapNet), _ => Err(format!("invalid value: {:?}", value)), } } @@ -76,7 +80,8 @@ impl InferenceModelType { InferenceModelType::Tacotron2, InferenceModelType::Vits, InferenceModelType::VallEX, - InferenceModelType::RerenderAVideo + InferenceModelType::RerenderAVideo, + InferenceModelType::MocapNet, ]) } } @@ -95,6 +100,7 @@ mod tests { assert_serialization(InferenceModelType::Vits, "vits"); assert_serialization(InferenceModelType::VallEX, "vall_e_x"); assert_serialization(InferenceModelType::RerenderAVideo, "rerender_a_video"); + assert_serialization(InferenceModelType::MocapNet, "mocap_net"); } #[test] @@ -106,6 +112,7 @@ mod tests { assert_eq!(InferenceModelType::Vits.to_str(), "vits"); assert_eq!(InferenceModelType::VallEX.to_str(), "vall_e_x"); assert_eq!(InferenceModelType::RerenderAVideo.to_str(), "rerender_a_video"); + assert_eq!(InferenceModelType::MocapNet.to_str(), "mocap_net"); } #[test] @@ -117,13 +124,14 @@ mod tests { assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); assert_eq!(InferenceModelType::from_str("vall_e_x").unwrap(), InferenceModelType::VallEX); assert_eq!(InferenceModelType::from_str("rerender_a_video").unwrap(), InferenceModelType::RerenderAVideo); + assert_eq!(InferenceModelType::from_str("mocap_net").unwrap(), InferenceModelType::MocapNet); } #[test] fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 7); + assert_eq!(variants.len(), 8); assert_eq!(variants.pop_first(), Some(InferenceModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); @@ -131,6 +139,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); assert_eq!(variants.pop_first(), Some(InferenceModelType::VallEX)); assert_eq!(variants.pop_first(), Some(InferenceModelType::RerenderAVideo)); + assert_eq!(variants.pop_first(), Some(InferenceModelType::MocapNet)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index 821a66f0ce..4b258cb124 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -25,6 +25,10 @@ pub enum IdCategory { #[serde(rename = "video_filter")] VideoFilterResult, + // Results from mocap + #[serde(rename = "mocap")] + MocapResult, + /// Results from tacotron2 /// Applies for RVC and SVC #[serde(rename = "tts_result")] @@ -73,6 +77,7 @@ impl IdCategory { Self::VideoFilterResult => "video_filter", Self::ModelWeights => "model_weights", Self::FileUpload => "file_upload", + Self::MocapResult => "mocap", } } @@ -88,6 +93,7 @@ impl IdCategory { "video_filter" => Ok(Self::VideoFilterResult), "model_weights" => Ok(Self::ModelWeights), "file_upload" => Ok(Self::FileUpload), + "mocap" => Ok(Self::MocapResult), _ => Err(format!("invalid value: {:?}", value)), } } @@ -130,6 +136,7 @@ mod tests { assert_serialization(IdCategory::ZeroShotTtsResult, "zs_tts_result"); assert_serialization(IdCategory::ModelWeights, "model_weights"); assert_serialization(IdCategory::FileUpload, "file_upload"); + assert_serialization(IdCategory::MocapResult, "mocap"); } #[test] @@ -144,6 +151,7 @@ mod tests { assert_eq!(IdCategory::ZeroShotTtsResult.to_str(), "zs_tts_result"); assert_eq!(IdCategory::ModelWeights.to_str(), "model_weights"); assert_eq!(IdCategory::FileUpload.to_str(), "file_upload"); + assert_eq!(IdCategory::MocapResult.to_str(), "mocap"); } #[test] @@ -158,13 +166,14 @@ mod tests { assert_eq!(IdCategory::from_str("zs_tts_result").unwrap(), IdCategory::ZeroShotTtsResult); assert_eq!(IdCategory::from_str("model_weights").unwrap(), IdCategory::ModelWeights); assert_eq!(IdCategory::from_str("file_upload").unwrap(), IdCategory::FileUpload); + assert_eq!(IdCategory::from_str("mocap").unwrap(), IdCategory::MocapResult); } #[test] fn all_variants() { // Static check let mut variants = IdCategory::all_variants(); - assert_eq!(variants.len(), 10); + assert_eq!(variants.len(), 11); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VideoFilterResult)); @@ -175,6 +184,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotVoiceEmbedding)); assert_eq!(variants.pop_first(), Some(IdCategory::ModelWeights)); assert_eq!(variants.pop_first(), Some(IdCategory::FileUpload)); + assert_eq!(variants.pop_first(), Some(IdCategory::MocapResult)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index 3fc35a9bb1..1e07888167 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -34,6 +34,9 @@ pub enum MediaFileOriginModelType { #[serde(rename = "rerender")] Rerender, + + #[serde(rename = "mocapnet")] + MocapNet, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -51,6 +54,7 @@ impl MediaFileOriginModelType { Self::Tacotron2 => "tacotron2", Self::VallEX => "vall_e_x", Self::Rerender => "rerender", + Self::MocapNet => "mocapnet", } } @@ -62,6 +66,7 @@ impl MediaFileOriginModelType { "tacotron2" => Ok(Self::Tacotron2), "vall_e_x" => Ok(Self::VallEX), "rerender" => Ok(Self::Rerender), + "mocapnet" => Ok(Self::MocapNet), _ => Err(format!("invalid value: {:?}", value)), } } @@ -76,6 +81,7 @@ impl MediaFileOriginModelType { Self::Tacotron2, Self::VallEX, Self::Rerender, + Self::MocapNet, ]) } } @@ -101,6 +107,7 @@ mod tests { assert_eq!(MediaFileOriginModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(MediaFileOriginModelType::VallEX.to_str(), "vall_e_x"); assert_eq!(MediaFileOriginModelType::Rerender.to_str(), "rerender"); + assert_eq!(MediaFileOriginModelType::MocapNet.to_str(), "mocapnet"); } #[test] @@ -111,19 +118,21 @@ mod tests { assert_eq!(MediaFileOriginModelType::from_str("tacotron2").unwrap(), MediaFileOriginModelType::Tacotron2); assert_eq!(MediaFileOriginModelType::from_str("vall_e_x").unwrap(), MediaFileOriginModelType::VallEX); assert_eq!(MediaFileOriginModelType::from_str("rerender").unwrap(), MediaFileOriginModelType::Rerender); + assert_eq!(MediaFileOriginModelType::from_str("mocapnet").unwrap(), MediaFileOriginModelType::MocapNet); assert!(MediaFileOriginModelType::from_str("foo").is_err()); } #[test] fn all_variants() { let mut variants = MediaFileOriginModelType::all_variants(); - assert_eq!(variants.len(), 6); + assert_eq!(variants.len(), 7); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::VallEX)); assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::Rerender)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::MocapNet)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index ccf1266246..bc99d59e07 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -43,6 +43,10 @@ pub enum MediaFileOriginProductCategory { // Media files for video filters #[serde(rename = "video_filter")] VideoFilter, + + // Mocap + #[serde(rename = "mocap")] + Mocap, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -60,6 +64,7 @@ impl MediaFileOriginProductCategory { Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", Self::VideoFilter => "video_filter", + Self::Mocap => "mocap", } } @@ -71,6 +76,7 @@ impl MediaFileOriginProductCategory { "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), "video_filter" => Ok(Self::VideoFilter), + "mocap" => Ok(Self::Mocap), _ => Err(format!("invalid value: {:?}", value)), } } @@ -85,6 +91,7 @@ impl MediaFileOriginProductCategory { Self::Unknown, Self::VoiceConversion, Self::ZeroShotVoice, + Self::Mocap, ]) } } @@ -104,7 +111,8 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); - assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter") + assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter"); + assert_serialization(MediaFileOriginProductCategory::Mocap, "mocap") } #[test] @@ -115,6 +123,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter"); + assert_eq!(MediaFileOriginProductCategory::Mocap.to_str(), "mocap"); } #[test] @@ -125,18 +134,20 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter); + assert_eq!(MediaFileOriginProductCategory::from_str("mocap").unwrap(), MediaFileOriginProductCategory::Mocap); } #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 6); + assert_eq!(variants.len(), 7); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Mocap)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_type.rs index 8ba5f0ecb9..539844602d 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_type.rs @@ -21,6 +21,9 @@ pub enum MediaFileType { /// Video files: mp4, etc. Video, + + /// Mocap files: BVH, etc. + Mocap, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -35,6 +38,7 @@ impl MediaFileType { Self::Audio => "audio", Self::Image => "image", Self::Video => "video", + Self::Mocap => "mocap", } } @@ -43,6 +47,7 @@ impl MediaFileType { "audio" => Ok(Self::Audio), "image" => Ok(Self::Image), "video" => Ok(Self::Video), + "mocap" => Ok(Self::Mocap), _ => Err(format!("invalid value: {:?}", value)), } } @@ -54,6 +59,7 @@ impl MediaFileType { Self::Audio, Self::Image, Self::Video, + Self::Mocap, ]) } } @@ -71,6 +77,7 @@ mod tests { assert_serialization(MediaFileType::Audio, "audio"); assert_serialization(MediaFileType::Image, "image"); assert_serialization(MediaFileType::Video, "video"); + assert_serialization(MediaFileType::Mocap, "mocap"); } } @@ -82,6 +89,7 @@ mod tests { assert_eq!(MediaFileType::Audio.to_str(), "audio"); assert_eq!(MediaFileType::Image.to_str(), "image"); assert_eq!(MediaFileType::Video.to_str(), "video"); + assert_eq!(MediaFileType::Mocap.to_str(), "mocap"); } #[test] @@ -89,6 +97,7 @@ mod tests { assert_eq!(MediaFileType::from_str("audio").unwrap(), MediaFileType::Audio); assert_eq!(MediaFileType::from_str("image").unwrap(), MediaFileType::Image); assert_eq!(MediaFileType::from_str("video").unwrap(), MediaFileType::Video); + assert_eq!(MediaFileType::from_str("mocap").unwrap(), MediaFileType::Mocap); assert!(MediaFileType::from_str("foo").is_err()); } } @@ -99,10 +108,11 @@ mod tests { #[test] fn all_variants() { let mut variants = MediaFileType::all_variants(); - assert_eq!(variants.len(), 3); + assert_eq!(variants.len(), 4); assert_eq!(variants.pop_first(), Some(MediaFileType::Audio)); assert_eq!(variants.pop_first(), Some(MediaFileType::Image)); assert_eq!(variants.pop_first(), Some(MediaFileType::Video)); + assert_eq!(variants.pop_first(), Some(MediaFileType::Mocap)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs index 513c0dfe98..a57b89f97f 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs @@ -1,17 +1,18 @@ +use anyhow::anyhow; use errors::AnyhowResult; use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; -use crate::remote_file_manager::bucket_orchestration::BucketOrchestrationCore; +use crate::remote_file_manager::bucket_orchestration::{BucketOrchestration, BucketOrchestrationCore}; use crate::remote_file_manager::file_directory::FileBucketDirectory; use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails; use super::file_descriptor::FileDescriptor; use super::file_meta_data::FileMetaData; -struct RemoteCloudFileClient { +pub struct RemoteCloudFileClient { bucket_orchestration_client: Box } @@ -22,6 +23,19 @@ impl RemoteCloudFileClient { } } + pub async fn get_remote_cloud_file_client() -> AnyhowResult { + let bucket_orchestration = match BucketOrchestration::new_bucket_client_from_existing_env() { + Ok(client) => client, + Err(e) => { + return Err(anyhow!("Error creating bucket orchestration client: {:?}", e)); + } + }; + + Ok(Self { + bucket_orchestration_client: Box::new(bucket_orchestration) + }) + } + pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyhowResult<()> { let file_descriptor = remote_cloud_bucket_details.file_descriptor_from_bucket_details(); let file_bucket_directory = FileBucketDirectory::from_existing_bucket_details(remote_cloud_bucket_details); diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index a67dd0bd91..c7ff50488b 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -2,6 +2,7 @@ use enums::by_table::generic_inference_jobs::inference_category::InferenceCatego use errors::AnyhowResult; use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs; +use crate::payloads::generic_inference_args::mocap_payload::MocapArgs; use crate::payloads::generic_inference_args::tts_payload::TTSArgs; use crate::payloads::generic_inference_args::videofilter_payload::{RerenderArgs}; @@ -40,6 +41,10 @@ pub enum InferenceCategoryAbbreviated { #[serde(rename = "vf")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. #[serde(alias = "video_filter")] VideoFilter, + + #[serde(rename = "mc")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(alias = "mocap")] + Mocap, } #[derive(Serialize, Deserialize, Clone, Copy, Debug)] @@ -91,7 +96,10 @@ pub enum PolymorphicInferenceArgs { }, /// Rerender a video. (Short name to save space when serializing.) - Rr(RerenderArgs) + Rr(RerenderArgs), + + // Mocap (Short name to save space when serializing.) + Mc(MocapArgs) } @@ -113,6 +121,7 @@ impl InferenceCategoryAbbreviated { InferenceCategory::TextToSpeech => Self::TextToSpeech, InferenceCategory::VoiceConversion => Self::VoiceConversion, InferenceCategory::VideoFilter => Self::VideoFilter, + InferenceCategory::Mocap => Self::Mocap, } } @@ -122,6 +131,7 @@ impl InferenceCategoryAbbreviated { Self::TextToSpeech => InferenceCategory::TextToSpeech, Self::VoiceConversion => InferenceCategory::VoiceConversion, Self::VideoFilter => InferenceCategory::VideoFilter, + Self::Mocap => InferenceCategory::Mocap, } } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mocap_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mocap_payload.rs new file mode 100644 index 0000000000..ca061c3b7e --- /dev/null +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mocap_payload.rs @@ -0,0 +1,60 @@ +/// Video sources can be one of several: +/// - F: media_files (todo) +/// - U: media_uploads (legacy) +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub enum MocapVideoSource { + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media File Token (media_files table) + /// Serde cannot yet rename enum variants. + F(String), + + // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + /// Media Upload Token (media_uploads table) + /// Serde cannot yet rename enum variants. + U(String), +} + + +impl MocapVideoSource { + pub fn media_file_token(token: &str) -> Self { + MocapVideoSource::F(token.to_string()) + } + pub fn media_upload_token(token: &str) -> Self { + MocapVideoSource::U(token.to_string()) + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] +pub struct MocapArgs { + #[serde(rename = "vs")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_video_source: Option, + + #[serde(rename = "ik1")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_ik1: Option, + + #[serde(rename = "ik2")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_ik2: Option, + + #[serde(rename = "ik3")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_ik3: Option, + + #[serde(rename = "sm1")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_smoothing1: Option, + + #[serde(rename = "sm2")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_smoothing2: Option, + + #[serde(rename = "si1")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_size1: Option, + + #[serde(rename = "si2")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_size2: Option, +} diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs index e68fc5a71c..97fdf2c3fa 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/mod.rs @@ -1,4 +1,5 @@ pub mod generic_inference_args; pub mod lipsync_payload; pub mod tts_payload; -pub mod videofilter_payload; \ No newline at end of file +pub mod videofilter_payload; +pub mod mocap_payload; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index e2a9b7be9c..c36ae1d225 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -179,6 +179,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), InferenceCategory::VideoFilter => Some("Video Filter"), + InferenceCategory::Mocap => Some("Mocap"), }; // NB: A bit of a hack. We store TTS results with a full path. @@ -188,6 +189,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), + InferenceCategory::Mocap => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), }; // NB: We've moved voice conversion out of their own table and into media_files diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_from_mocapnet.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_from_mocapnet.rs new file mode 100644 index 0000000000..4cc96f4866 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_from_mocapnet.rs @@ -0,0 +1,155 @@ +use anyhow::anyhow; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id; + +pub struct InsertArgs<'a> { + pub pool: &'a MySqlPool, + pub job: &'a AvailableInferenceJob, + + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub sha256_checksum: &'a str, + // TODO: Media duration. + //pub duration_millis: u64, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, + + pub is_on_prem: bool, + pub worker_hostname: &'a str, + pub worker_cluster: &'a str, +} + +pub async fn insert_media_file_from_mocapnet( + args: InsertArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let result_token = MediaFileToken::generate(); + + let mut maybe_creator_file_synthetic_id : Option = None; + let mut maybe_creator_category_synthetic_id : Option = None; + + let mut transaction = args.pool.begin().await?; + + if let Some(creator_user_token) = args.job.maybe_creator_user_token.as_deref() { + let user_token = UserToken::new_from_str(creator_user_token); + + let next_media_file_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MediaFile, + &mut transaction + ).await?; + + let next_mocapnet_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MocapResult, + &mut transaction + ).await?; + + maybe_creator_file_synthetic_id = Some(next_media_file_id); + maybe_creator_category_synthetic_id = Some(next_mocapnet_id); + } + + const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Mocap; + const ORIGIN_MODEL_TYPE : MediaFileOriginModelType = MediaFileOriginModelType::MocapNet; + const MEDIA_TYPE : MediaFileType = MediaFileType::Mocap; + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + maybe_origin_model_type = ?, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ?, + + maybe_creator_file_synthetic_id = ?, + maybe_creator_category_synthetic_id = ?, + + is_generated_on_prem = ?, + generated_by_worker = ?, + generated_by_cluster = ? + + "#, + result_token.as_str(), + + ORIGIN_CATEGORY.to_str(), + ORIGIN_PRODUCT_CATEGORY.to_str(), + ORIGIN_MODEL_TYPE.to_str(), + + MEDIA_TYPE.to_str(), + args.maybe_mime_type, + args.file_size_bytes, + + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + args.job.maybe_creator_user_token, + args.job.maybe_creator_anonymous_visitor_token, + args.job.creator_ip_address, + + args.job.creator_set_visibility.to_str(), + + maybe_creator_file_synthetic_id, + maybe_creator_category_synthetic_id, + + args.is_on_prem, + args.worker_hostname, + args.worker_cluster + ) + .execute(&mut *transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((result_token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs index b01e5bb277..177ae2eda4 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/create/mod.rs @@ -5,3 +5,4 @@ pub mod insert_media_file_from_rerender; pub mod insert_media_file_from_tacotron2; pub mod insert_media_file_from_voice_conversion; pub mod insert_media_file_from_zero_shot_tts; +pub mod insert_media_file_from_mocapnet; diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/get_total_bookmark_count_for_entity.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_total_bookmark_count_for_entity.rs index 214686c8c4..c366eab630 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/get_total_bookmark_count_for_entity.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/get_total_bookmark_count_for_entity.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use log::warn; -use sqlx::{Executor, MySql}; +use sqlx::{MySql}; use sqlx::pool::PoolConnection; use errors::AnyhowResult; diff --git a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs index 5781de057c..a16ef234b1 100644 --- a/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs +++ b/crates/lib/mysql_queries/src/queries/user_bookmarks/list_user_bookmarks_result.rs @@ -145,6 +145,7 @@ impl RawUserBookmarkRecord { MediaFileType::Audio => Some("audio media file".to_string()), MediaFileType::Image => Some("image media file".to_string()), MediaFileType::Video => Some("video media file".to_string()), + MediaFileType::Mocap => Some("mocap da file".to_string()), }), UserBookmarkEntityType::VoiceConversionModel => self.maybe_descriptive_text_voice_conversion_model_title, UserBookmarkEntityType::ZsVoice => self.maybe_descriptive_text_zs_voice_title, diff --git a/crates/lib/mysql_queries/src/queries/users/user_ratings/get_total_user_rating_count_for_entity.rs b/crates/lib/mysql_queries/src/queries/users/user_ratings/get_total_user_rating_count_for_entity.rs index c9afcc3dc6..39a4961607 100644 --- a/crates/lib/mysql_queries/src/queries/users/user_ratings/get_total_user_rating_count_for_entity.rs +++ b/crates/lib/mysql_queries/src/queries/users/user_ratings/get_total_user_rating_count_for_entity.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; use log::warn; -use sqlx::{Executor, MySql}; +use sqlx::{MySql}; use sqlx::pool::PoolConnection; use errors::AnyhowResult; diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index b911d80f6e..5ed3a07dad 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -104,6 +104,14 @@ VITS_INFERENCE_MAYBE_DOCKER_IMAGE_SHA="03017896367b" RERENDER_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/rerender-lora/" RERENDER_INFERENCE_EXECUTABLE_OR_COMMAND="venv/bin/python rerender.py" +# ================================= # +# # +# MocapNET # +# # +# ================================= # +MOCAPNET_INFERENCE_ROOT_DIRECTORY="/home/salt/clone/storyteller-ml/animation/MocapNET/src/python/mnet4/" +MOCAPNET_INFERENCE_EXECUTABLE_OR_COMMAND="pythonVirtualEnvironment/bin/python -m mediapipeHolisticWebcamMocapNET" + # ================================= # # # # CACHE SETTINGS # diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 1aab84a477..cca0d768c8 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -17,6 +17,7 @@ use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_l use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job; use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job; use crate::job::job_types::videofilter::process_single_vf_job::process_single_vf_job; +use crate::job::job_types::mocap::process_single_mc_job::process_single_mc_job; use crate::job_dependencies::JobDependencies; pub async fn process_single_job( @@ -186,6 +187,9 @@ async fn do_process_single_job( InferenceCategory::VideoFilter => { process_single_vf_job(job_dependencies, job).await? } + InferenceCategory::Mocap => { + process_single_mc_job(job_dependencies, job).await? + } }; let maybe_entity_type = job_success_result.maybe_result_entity diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs new file mode 100644 index 0000000000..3a12c84363 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/download_video_file.rs @@ -0,0 +1,109 @@ +use std::path::PathBuf; + +use anyhow::anyhow; +use log::{error, warn}; +use sqlx::MySqlPool; +use tempdir::TempDir; +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath; +use cloud_storage::bucket_client::BucketClient; +use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; +use mysql_queries::payloads::generic_inference_args::mocap_payload::MocapVideoSource; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::media_uploads::MediaUploadToken; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; +use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; + +pub struct VideoFile { + pub filesystem_path: PathBuf, +} + +// TODO: Replace with remote cloud client +pub async fn download_video_file( + video_source: &MocapVideoSource, + public_bucket_client: &BucketClient, + job_progress_reporter: &mut Box, + job: &AvailableInferenceJob, + temp_dir_creator: &ScopedTempDirCreator, + work_temp_dir: &TempDir, + mysql_pool: &MySqlPool, +) -> Result { + + let bucket_object_path; + + match video_source { + MocapVideoSource::F(media_file_token) => { + let media_file_token = MediaFileToken::new_from_str(media_file_token); + let media_file_result = get_media_file( + &media_file_token, + false, + mysql_pool + ).await; + match media_file_result { + Ok(Some(result)) => { + let media_file_bucket_path = + MediaFileBucketPath::from_object_hash( + &result.public_bucket_directory_hash, + result.maybe_public_bucket_prefix.as_deref(), + result.maybe_public_bucket_extension.as_deref(), + ); + bucket_object_path = media_file_bucket_path.to_full_object_pathbuf(); + } + Ok(None) => { + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media file: {:?}", media_file_token))) + } + Err(e) => { + error!("could not query media file: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + } + } + MocapVideoSource::U(media_upload_token) => { + let media_upload_token = MediaUploadToken::new_from_str(media_upload_token); + let media_upload_result = get_media_upload_for_inference( + &media_upload_token, + mysql_pool + ).await; + + let media_upload_result = match media_upload_result { + Ok(Some(result)) => result, + Ok(None) => { + warn!("could not find media upload: {:?}", media_upload_token); + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find media upload: {:?}", media_upload_token))) + } + Err(e) => { + error!("could not query media upload: {:?}", e); + return Err(ProcessSingleJobError::from_anyhow_error(e)) + } + }; + + let media_upload_bucket_path = MediaUploadOriginalFilePath::from_object_hash(&media_upload_result.public_bucket_directory_hash); + bucket_object_path = media_upload_bucket_path.to_full_object_pathbuf(); + } + } + + let downloaded_filesystem_path = work_temp_dir.path().join("video.mp4"); + + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "video file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, + job_progress_reporter, + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; + + Ok(VideoFile { + filesystem_path: downloaded_filesystem_path, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_dependencies.rs new file mode 100644 index 0000000000..f642fcb40e --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_dependencies.rs @@ -0,0 +1,15 @@ +use errors::AnyhowResult; + +use crate::job::job_types::mocap::mocap_net::mocapnet_inference_command::MocapnetInferenceCommand; + +pub struct MocapNetDependencies { + pub inference_command: MocapnetInferenceCommand, +} + +impl MocapNetDependencies { + pub fn setup() -> AnyhowResult { + Ok(Self { + inference_command: MocapnetInferenceCommand::from_env()?, + }) + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_inference_command.rs new file mode 100644 index 0000000000..44180f6c6c --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_inference_command.rs @@ -0,0 +1,240 @@ +use std::collections::HashSet; +use std::env; +use std::ffi::OsString; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use anyhow::anyhow; +use log::info; +use once_cell::sync::Lazy; +use subprocess::{Popen, PopenConfig, Redirection}; +use errors::AnyhowResult; +use filesys::path_to_string::path_to_string; +use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; +use crate::job::job_loop::command_exit_status::CommandExitStatus; + + +// These environment vars are not copied over to the subprocess +// TODO/FIXME(bt, 2023-05-28): This is horrific security! +static IGNORED_ENVIRONMENT_VARS : Lazy> = Lazy::new(|| { + let env_var_names= [ + "MYSQL_URL", + "ACCESS_KEY", + "SECRET_KEY", + "NEWRELIC_API_KEY", + ]; + + env_var_names.iter() + .map(|value| value.to_string()) + .collect::>() +}); + + +#[derive(Clone)] +pub struct MocapnetInferenceCommand { + /// Where the code lives + pub(crate) mocapnet_root_code_directory: PathBuf, + + /// A single executable script or a much larger bash command. + executable_or_command: ExecutableOrCommand, + + /// eg. `source python/bin/activate` + maybe_virtual_env_activation_command: Option, + + /// If this is run under Docker (eg. in development), these are the options. + maybe_docker_options: Option, + + /// If the execution should be ended after a certain point. + maybe_execution_timeout: Option, + +} + +#[derive(Clone)] +pub enum ExecutableOrCommand { + /// Eg. `inference.py` + Executable(PathBuf), + + /// Eg. `python3 inference.py` + Command(String), +} + +pub struct InferenceArgs<'s, P: AsRef> { + pub video_file: &'s P, + pub maybe_ik1: &'s Option, + pub maybe_ik2: &'s Option, + pub maybe_ik3: &'s Option, + pub maybe_smoothing1: &'s Option, + pub maybe_smoothing2: &'s Option, + pub maybe_size1: &'s Option, + pub maybe_size2: &'s Option, + pub stderr_output_file: &'s Path, +} + +impl MocapnetInferenceCommand { + pub fn from_env() -> AnyhowResult { + let mocapnet_root_code_directory = easyenv::get_env_pathbuf_required( + "MOCAPNET_INFERENCE_ROOT_DIRECTORY")?; + + let executable_or_command = match easyenv::get_env_string_optional( + "MOCAPNET_INFERENCE_EXECUTABLE_OR_COMMAND") { + None => { + return Err(anyhow!("MOCAPNET_INFERENCE_EXECUTABLE_OR_COMMAND is required")); + } + Some(executable_or_command) => { + if executable_or_command.contains(" ") { + ExecutableOrCommand::Command(executable_or_command) + } else { + ExecutableOrCommand::Executable(PathBuf::from(executable_or_command)) + } + } + }; + + let maybe_virtual_env_activation_command = easyenv::get_env_string_optional( + "MOCAPNET_INFERENCE_MAYBE_VENV_COMMAND"); + + let maybe_execution_timeout = + easyenv::get_env_duration_seconds_optional("MOCAPNET_TIMEOUT_SECONDS"); + + let maybe_docker_options = easyenv::get_env_string_optional( + "MOCAPNET_INFERENCE_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_environment_variables: None, + maybe_gpu: Some(DockerGpu::All), + } + }); + + Ok(Self { + mocapnet_root_code_directory, + executable_or_command, + maybe_virtual_env_activation_command, + maybe_docker_options, + maybe_execution_timeout, + }) + } + + pub fn execute_inference>( + &self, + args: InferenceArgs

, + ) -> CommandExitStatus { + match self.do_execute_inference(args) { + Ok(exit_status) => exit_status, + Err(error) => CommandExitStatus::FailureWithReason { reason: format!("error: {:?}", error) }, + } + } + + fn do_execute_inference>( + &self, + args: InferenceArgs

, + ) -> AnyhowResult { + + let mut command = String::new(); + command.push_str(&format!("cd {}", path_to_string(&self.mocapnet_root_code_directory))); + + if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() { + command.push_str(" && "); + command.push_str(venv_command); + command.push_str(" "); + } + + command.push_str(" && "); + + match self.executable_or_command { + ExecutableOrCommand::Executable(ref executable) => { + command.push_str(&path_to_string(executable)); + command.push_str(" "); + } + ExecutableOrCommand::Command(ref cmd) => { + command.push_str(cmd); + command.push_str(" "); + } + } + + // ===== Begin Python Args ===== + + command.push_str(" -m mediapipeHolisticWebcamMocapNET "); + + let video_path = args.video_file.as_ref(); + command.push_str(&format!(" --from {}", path_to_string(video_path))); + if let Some(ik1) = args.maybe_ik1 { + command.push_str(&format!(" --ik {} {} {}", ik1, args.maybe_ik2.unwrap(), args.maybe_ik3.unwrap())); + } + if let Some(smooth1) = args.maybe_smoothing1 { + command.push_str(&format!(" --smooth {} {}", smooth1, args.maybe_smoothing2.unwrap())); + } + if let Some(size1) = args.maybe_size1 { + command.push_str(&format!(" --size {} {}", size1, args.maybe_size2.unwrap())); + } + command.push_str(" --all --save --headless 2"); + + // ===== End Python Args ===== + + if let Some(docker_options) = self.maybe_docker_options.as_ref() { + command = docker_options.to_command_string(&command); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + let mut env_vars = Vec::new(); + + // Copy all environment variables from the parent process. + // This is necessary to send all the kubernetes settings for Nvidia / CUDA. + for (env_key, env_value) in env::vars() { + if IGNORED_ENVIRONMENT_VARS.contains(&env_key) { + continue; + } + env_vars.push(( + OsString::from(env_key), + OsString::from(env_value), + )); + } + + let mut config = PopenConfig::default(); + + info!("stderr will be written to file: {:?}", args.stderr_output_file.as_os_str()); + + let stderr_file = File::create(&args.stderr_output_file)?; + config.stderr = Redirection::File(stderr_file); + + if !env_vars.is_empty() { + config.env = Some(env_vars); + } + + let mut p = Popen::create(&command_parts, config)?; + + info!("Subprocess PID: {:?}", p.pid()); + + match self.maybe_execution_timeout { + None => { + let exit_status = p.wait()?; + info!("Subprocess exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + Some(timeout) => { + info!("Executing with timeout: {:?}", &timeout); + let exit_status = p.wait_timeout(timeout.clone())?; + + match exit_status { + None => { + // NB: If the program didn't successfully terminate, kill it. + info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout); + let _r = p.terminate()?; + Ok(CommandExitStatus::Timeout) + } + Some(exit_status) => { + info!("Subprocess timed wait exit status: {:?}", exit_status); + Ok(CommandExitStatus::from_exit_status(exit_status)) + } + } + } + } + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mod.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mod.rs new file mode 100644 index 0000000000..f3095aabc4 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mod.rs @@ -0,0 +1,5 @@ +mod validate_job; +pub(crate) mod process_job; +pub(crate) mod mocapnet_dependencies; +mod mocapnet_inference_command; +mod download_video_file; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/process_job.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/process_job.rs new file mode 100644 index 0000000000..e1d4216249 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/process_job.rs @@ -0,0 +1,258 @@ +use std::fs::read_to_string; +use std::time::Instant; + +use anyhow::{anyhow}; +use log::{error, info, warn}; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use filesys::check_file_exists::check_file_exists; +use filesys::file_size::file_size; +use filesys::safe_delete_temp_directory::safe_delete_temp_directory; +use filesys::safe_delete_temp_file::safe_delete_temp_file; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Mc; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::create::insert_media_file_from_mocapnet::{insert_media_file_from_mocapnet, InsertArgs}; +use crate::job::job_types::mocap::mocap_net::mocapnet_inference_command::{InferenceArgs, MocapnetInferenceCommand}; + +use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::mocap::mocap_net::download_video_file::download_video_file; +use crate::job::job_types::mocap::mocap_net::validate_job::validate_job; +use crate::job_dependencies::JobDependencies; + +const BUCKET_FILE_PREFIX: &str = "fakeyou_"; +const BUCKET_FILE_EXTENSION: &str = ".bvh"; + +pub struct MocapNetProcessJobArgs<'a> { + pub job_dependencies: &'a JobDependencies, + pub job: &'a AvailableInferenceJob, +} + +pub async fn process_job(args: MocapNetProcessJobArgs<'_>) -> Result { + let job = args.job; + let deps = args.job_dependencies; + + let mut job_progress_reporter = args + .job_dependencies + .clients + .job_progress_reporter + .new_generic_inference(job.inference_job_token.as_str()) + .map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?; + + let model_dependencies = args + .job_dependencies + .job + .job_specific_dependencies + .maybe_mocapnet_dependencies + .as_ref() + .ok_or_else(|| ProcessSingleJobError::JobSystemMisconfiguration(Some("missing Mocap dependencies".to_string())))?; + + // ==================== UNPACK + VALIDATE INFERENCE ARGS ==================== // + + let job_args = validate_job(job)?; + + // ==================== TEMP DIR ==================== // + + let work_temp_dir = format!("temp_mocapnet_inference_{}", job.id.0); + + // NB: TempDir exists until it goes out of scope, at which point it should delete from filesystem. + let work_temp_dir = args.job_dependencies + .fs + .scoped_temp_dir_creator_for_work + .new_tempdir(&work_temp_dir) + .map_err(|e| ProcessSingleJobError::from_io_error(e))?; + + // ==================== QUERY AND DOWNLOAD FILES ==================== // + + let video_path = download_video_file( + &job_args.video_source, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir, + &deps.db.mysql_pool + ).await?; + + info!("Downloaded video file: {:?}", video_path.filesystem_path); + + // ==================== TRANSCODE MEDIA (IF NECESSARY) ==================== // + + let usable_video_path = video_path.filesystem_path.clone(); + + //TODO: re encode with ffmpeg + + info!("Used video file: {:?}", usable_video_path); + + // ==================== SETUP FOR INFERENCE ==================== // + + info!("Ready for MocapNET inference..."); + + job_progress_reporter.log_status("running inference") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let output_bvh_path = model_dependencies.inference_command.mocapnet_root_code_directory.clone() + .join("out.bvh"); + + info!("Running MocapNET inference..."); + + info!("Expected output bvh filename: {:?}", &output_bvh_path); + + // TODO: Limit output length for non-premium (???) + + let maybe_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten(); + + let poly_args = match maybe_args { + None => return Err(ProcessSingleJobError::Other(anyhow!("Mocap args not found"))), + Some(args) => args, + }; + + let mc_args = match poly_args { + Mc(args) => args, + _ => return Err(ProcessSingleJobError::Other(anyhow!("Mocap args not found"))), + }; + + // ==================== RUN INFERENCE SCRIPT ==================== // + let stderr_output_file = work_temp_dir.path().join("stderr.txt"); + let inference_start_time = Instant::now(); + + let command_exit_status = model_dependencies + .inference_command + .execute_inference(InferenceArgs { + video_file: &usable_video_path, + maybe_ik1: &mc_args.maybe_ik1, + maybe_ik2: &mc_args.maybe_ik2, + maybe_ik3: &mc_args.maybe_ik3, + maybe_smoothing1: &mc_args.maybe_smoothing1, + maybe_smoothing2: &mc_args.maybe_smoothing2, + maybe_size1: &mc_args.maybe_size1, + maybe_size2: &mc_args.maybe_size2, + stderr_output_file: &stderr_output_file, + }); + + let inference_duration = Instant::now().duration_since(inference_start_time); + + info!("Inference took duration to complete: {:?}", &inference_duration); + + if !command_exit_status.is_success() { + error!("Inference failed: {:?}", command_exit_status); + + let error = ProcessSingleJobError::Other(anyhow!("CommandExitStatus: {:?}", command_exit_status)); + + if let Ok(contents) = read_to_string(&stderr_output_file) { + warn!("Captured stderr output: {}", contents); + } + + safe_delete_temp_file(&video_path.filesystem_path); + safe_delete_temp_file(&output_bvh_path); + safe_delete_temp_file(&stderr_output_file); + safe_delete_temp_directory(&work_temp_dir); + + return Err(error); + } + + // ==================== CHECK NON-WATERMARKED RESULT ==================== // + + info!("Checking that output file exists: {:?} ...", output_bvh_path); + + check_file_exists(&output_bvh_path).map_err(|e| ProcessSingleJobError::Other(e))?; + + // ==================== OPTIONAL WATERMARK ==================== // + + let finished_file = output_bvh_path.clone(); + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + info!("Checking that output watermark file exists: {:?} ...", finished_file); + check_file_exists(&finished_file).map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Interrogating result file size ..."); + + let file_size_bytes = file_size(&finished_file) + .map_err(|err| ProcessSingleJobError::Other(err))?; + + info!("Interrogating result mimetype ..."); + + let mimetype = "application/octet-stream"; + + info!("Calculating sha256..."); + + let file_checksum = sha256_hash_file(&finished_file) + .map_err(|err| { + ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err)) + })?; + + // ==================== UPLOAD VIDEO TO BUCKET ==================== // + + job_progress_reporter.log_status("uploading result") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + let result_bucket_location = MediaFileBucketPath::generate_new( + Some(BUCKET_FILE_PREFIX), + Some(BUCKET_FILE_EXTENSION)); + + let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf(); + + info!("Video destination bucket path: {:?}", &result_bucket_object_pathbuf); + + info!("Uploading BVH ..."); + + args.job_dependencies.buckets.public_bucket_client.upload_filename_with_content_type( + &result_bucket_object_pathbuf, + &finished_file, + &mimetype) // TODO: We should check the mimetype to make sure bad payloads can't get uploaded + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + // ==================== DELETE TEMP FILES ==================== // + + safe_delete_temp_file(&video_path.filesystem_path); + safe_delete_temp_file(&output_bvh_path); + safe_delete_temp_file(&stderr_output_file); + safe_delete_temp_directory(&work_temp_dir); + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&work_temp_dir); + + // ==================== SAVE RECORDS ==================== // + + info!("Saving MocapNET result (media_files table record) ..."); + + let (media_file_token, id) = insert_media_file_from_mocapnet(InsertArgs { + pool: &args.job_dependencies.db.mysql_pool, + job: &job, + maybe_mime_type: Some(&mimetype), + file_size_bytes, + sha256_checksum: &file_checksum, + public_bucket_directory_hash: result_bucket_location.get_object_hash(), + maybe_public_bucket_prefix: Some(BUCKET_FILE_PREFIX), + maybe_public_bucket_extension: Some(BUCKET_FILE_EXTENSION), + is_on_prem: args.job_dependencies.job.info.container.is_on_prem, + worker_hostname: &args.job_dependencies.job.info.container.hostname, + worker_cluster: &args.job_dependencies.job.info.container.cluster_name, + }) + .await + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Mocap Done."); + + job_progress_reporter.log_status("done") + .map_err(|e| ProcessSingleJobError::Other(e))?; + + info!("Job {:?} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}, Result Token: {}", + job.id, id, &media_file_token); + + Ok(JobSuccessResult { + maybe_result_entity: Some(ResultEntity { + entity_type: InferenceResultType::MediaFile, + entity_token: media_file_token.to_string(), + }), + inference_duration, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/validate_job.rs new file mode 100644 index 0000000000..5499bb9dad --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/validate_job.rs @@ -0,0 +1,72 @@ +use anyhow::anyhow; + +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::mocap_payload::{MocapVideoSource}; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; + +pub struct JobArgs<'a> { + pub video_source: &'a MocapVideoSource, + pub maybe_ik1: Option<&'a f32>, + pub maybe_ik2: Option<&'a i32>, + pub maybe_ik3: Option<&'a i32>, + pub maybe_smoothing1: Option<&'a f32>, + pub maybe_smoothing2: Option<&'a f32>, + pub maybe_size1: Option<&'a i32>, + pub maybe_size2: Option<&'a i32>, +} + +pub fn validate_job(job: &AvailableInferenceJob) -> Result { + let inference_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten(); + + let inference_category = job.maybe_inference_args + .as_ref() + .map(|args| args.inference_category) + .flatten(); + + match inference_category { + Some(InferenceCategoryAbbreviated::Mocap) => {}, // Valid + Some(category) => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job: {:?}", category))); + }, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference category for job!"))); + } + }; + + let inference_args = match inference_args { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference args for job!"))); + } + }; + + let inference_args = match inference_args { + PolymorphicInferenceArgs::Mc(inference_args) => inference_args, + _ => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inner args for job!"))); + } + }; + + let video_source = match &inference_args.maybe_video_source { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no video source!"))); + } + }; + + Ok(JobArgs { + video_source, + maybe_ik1: inference_args.maybe_ik1.as_ref(), + maybe_ik2: inference_args.maybe_ik2.as_ref(), + maybe_ik3: inference_args.maybe_ik3.as_ref(), + maybe_smoothing1: inference_args.maybe_smoothing1.as_ref(), + maybe_smoothing2: inference_args.maybe_smoothing2.as_ref(), + maybe_size1: inference_args.maybe_size1.as_ref(), + maybe_size2: inference_args.maybe_size2.as_ref(), + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/mod.rs b/crates/service/job/inference_job/src/job/job_types/mocap/mod.rs new file mode 100644 index 0000000000..259316ba3b --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod process_single_mc_job; +pub(crate) mod mocap_net; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/mocap/process_single_mc_job.rs b/crates/service/job/inference_job/src/job/job_types/mocap/process_single_mc_job.rs new file mode 100644 index 0000000000..b755586697 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/mocap/process_single_mc_job.rs @@ -0,0 +1,16 @@ +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use crate::job::job_loop::job_success_result::JobSuccessResult; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::mocap::mocap_net; +use crate::job_dependencies::JobDependencies; + +pub async fn process_single_mc_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { + let job_success_result = mocap_net::process_job::process_job( + mocap_net::process_job::MocapNetProcessJobArgs { + job_dependencies, + job, + } + ).await?; + + Ok(job_success_result) +} diff --git a/crates/service/job/inference_job/src/job/job_types/mod.rs b/crates/service/job/inference_job/src/job/job_types/mod.rs index a688e281ba..3270d976e8 100644 --- a/crates/service/job/inference_job/src/job/job_types/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/mod.rs @@ -1,4 +1,5 @@ pub mod lipsync; pub mod tts; pub mod vc; -pub(crate) mod videofilter; \ No newline at end of file +pub(crate) mod videofilter; +pub(crate) mod mocap; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 0d2f22e34b..bcf3f0999c 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -260,6 +260,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::Vc { override_f0_method, .. } => *override_f0_method, PolymorphicInferenceArgs::Rr(_) => None, + PolymorphicInferenceArgs::Mc(_) => None, }) .flatten(); @@ -269,6 +270,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, PolymorphicInferenceArgs::Rr(_) => None, + PolymorphicInferenceArgs::Mc(_) => None, }) .flatten(); diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index a46e8c89f5..b6f94edf27 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -180,6 +180,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0, PolymorphicInferenceArgs::Rr(_) => None, + PolymorphicInferenceArgs::Mc(_) => None, }) .flatten() .unwrap_or(false); @@ -190,6 +191,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result None, PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose, PolymorphicInferenceArgs::Rr(_) => None, + PolymorphicInferenceArgs::Mc(_) => None, }) .flatten(); diff --git a/crates/service/job/inference_job/src/job_specific_dependencies.rs b/crates/service/job/inference_job/src/job_specific_dependencies.rs index a863b3872c..a17b4ef472 100644 --- a/crates/service/job/inference_job/src/job_specific_dependencies.rs +++ b/crates/service/job/inference_job/src/job_specific_dependencies.rs @@ -8,6 +8,7 @@ use crate::job::job_types::tts::vits::vits_dependencies::VitsDependencies; use crate::job::job_types::vc::rvc_v2::rvc_v2_dependencies::RvcV2Dependencies; use crate::job::job_types::vc::so_vits_svc::svc_dependencies::SvcDependencies; use crate::job::job_types::videofilter::rerender_a_video::rerender_dependencies::RerenderDependencies; +use crate::job::job_types::mocap::mocap_net::mocapnet_dependencies::MocapNetDependencies; use crate::util::scoped_execution::ScopedExecution; pub struct JobSpecificDependencies { @@ -18,6 +19,7 @@ pub struct JobSpecificDependencies { pub maybe_vall_e_x_dependencies: Option, pub maybe_vits_dependencies: Option, pub maybe_rerender_dependencies: Option, + pub maybe_mocapnet_dependencies: Option, } impl JobSpecificDependencies { @@ -30,6 +32,7 @@ impl JobSpecificDependencies { let mut maybe_vall_e_x_dependencies = None; let mut maybe_vits_dependencies = None; let mut maybe_rerender_dependencies = None; + let mut maybe_mocapnet_dependencies = None; if scoped_execution.can_run_job(InferenceModelType::RvcV2) { print_with_space("Setting RVCv2 dependencies..."); @@ -66,6 +69,11 @@ impl JobSpecificDependencies { maybe_rerender_dependencies = Some(RerenderDependencies::setup()?); } + if scoped_execution.can_run_job(InferenceModelType::MocapNet) { + print_with_space("Setting MocapNet dependencies..."); + maybe_mocapnet_dependencies = Some(MocapNetDependencies::setup()?); + } + Ok(JobSpecificDependencies { maybe_rvc_v2_dependencies, maybe_sad_talker_dependencies, @@ -74,6 +82,7 @@ impl JobSpecificDependencies { maybe_vall_e_x_dependencies, maybe_vits_dependencies, maybe_rerender_dependencies, + maybe_mocapnet_dependencies }) } } diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan.rs b/crates/service/web/storyteller_web/src/configs/plans/plan.rs index 80d100a9fe..86dd798f6f 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan.rs @@ -52,6 +52,9 @@ pub struct Plan { // ========== Features for Videofilters ========== rerender_requires_frontend_keepalive: bool, + // ========== Features for Mocap ========== + mocapnet_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -95,6 +98,7 @@ impl Plan { is_synthetic_plan: builder.is_synthetic_plan, lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive, rerender_requires_frontend_keepalive: builder.rerender_requires_frontend_keepalive, + mocapnet_requires_frontend_keepalive: builder.mocapnet_requires_frontend_keepalive, can_remove_visual_watermarks: builder.can_remove_visual_watermarks, tts_base_priority_level: builder.tts_base_priority_level, tts_max_duration: builder.tts_max_duration, @@ -154,6 +158,10 @@ impl Plan { self.rerender_requires_frontend_keepalive } + pub fn mocapnet_requires_frontend_keepalive(&self) -> bool { + self.mocapnet_requires_frontend_keepalive + } + pub fn can_remove_visual_watermarks(&self) -> bool { self.can_remove_visual_watermarks } @@ -232,6 +240,7 @@ pub struct PlanBuilder { w2l_max_duration: Duration, w2l_can_turn_off_watermark: bool, + mocapnet_requires_frontend_keepalive: bool, } impl PlanBuilder { @@ -253,6 +262,9 @@ impl PlanBuilder { // Videofilters rerender_requires_frontend_keepalive: true, + // Mocapnet + mocapnet_requires_frontend_keepalive: true, + // TTS tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL, tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS), @@ -275,7 +287,7 @@ impl PlanBuilder { // W2L w2l_max_duration: Duration::seconds(W2L_DEFAULT_TIME_LIMIT_SECONDS), - w2l_can_turn_off_watermark: false + w2l_can_turn_off_watermark: false, } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 071b3795a6..ea21e9f8cd 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -256,6 +256,14 @@ fn record_to_payload( .get_full_object_path_str() .to_string() } + InferenceCategory::Mocap => { + MediaFileBucketPath::from_object_hash( + &result_details.public_bucket_location_or_hash, + result_details.maybe_media_file_public_bucket_prefix.as_deref(), + result_details.maybe_media_file_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string() + } InferenceCategory::VideoFilter => { MediaFileBucketPath::from_object_hash( &result_details.public_bucket_location_or_hash, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mocap/enqueue_mocapnet.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mocap/enqueue_mocapnet.rs new file mode 100644 index 0000000000..029305590b --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mocap/enqueue_mocapnet.rs @@ -0,0 +1,231 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use log::{info, warn}; + +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use enums::common::visibility::Visibility; +use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::request::get_request_ip::get_request_ip; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::mocap_payload::{MocapArgs, MocapVideoSource}; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// Debug requests can get routed to special "debug-only" workers, which can +/// be used to trial new code, run debugging, etc. +const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; + +/// The routing tag header can send workloads to particular k8s hosts. +/// This is useful for catching the live logs or intercepting the job. +const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; + +#[derive(Deserialize)] +pub struct EnqueueMocapnetRequest { + uuid_idempotency_token: String, + + video_source: Option, + creator_set_visibility: Option, +} + +/// Treated as an enum. Only one of these may be set. + +#[derive(Serialize)] +pub struct EnqueueMocapnetSuccessResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} + +#[derive(Debug)] +pub enum EnqueueMocapnetError { + BadInput(String), + NotAuthorized, + ServerError, + RateLimited, +} + +impl ResponseError for EnqueueMocapnetError { + fn status_code(&self) -> StatusCode { + match *self { + EnqueueMocapnetError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueMocapnetError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueMocapnetError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueMocapnetError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EnqueueMocapnetError::BadInput(reason) => reason.to_string(), + EnqueueMocapnetError::NotAuthorized => "unauthorized".to_string(), + EnqueueMocapnetError::ServerError => "server error".to_string(), + EnqueueMocapnetError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl std::fmt::Display for EnqueueMocapnetError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn enqueue_mocapnet_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let mut maybe_user_token : Option = None; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueMocapnetError::ServerError + })?; + + let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request); + + // ==================== USER SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EnqueueMocapnetError::ServerError + })?; + + if let Some(user_session) = maybe_user_session.as_ref() { + maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token)); + } + + // TODO: Plan should handle "first anonymous use" and "investor" cases. + let plan = get_correct_plan_for_session( + server_state.server_environment, + maybe_user_session.as_ref()); + + // TODO: Separate priority for animation. + let priority_level = plan.web_vc_base_priority_level(); + + // ==================== DEBUG MODE + ROUTING TAG ==================== // + + let is_debug_request = + get_request_header_optional(&http_request, DEBUG_HEADER_NAME) + .is_some(); + + let maybe_routing_tag= + get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) + .map(|routing_tag| routing_tag.trim().to_string()); + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.role.is_banned { + return Err(EnqueueMocapnetError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(EnqueueMocapnetError::RateLimited); + } + + // ==================== LOOK UP MODEL INFO ==================== // + + // TODO(bt): CHECK DATABASE FOR TOKENS! + + let video_source: MocapVideoSource = request.video_source + .as_ref() + .map(|token| MocapVideoSource::media_file_token(token.as_str())) + .ok_or_else(|| EnqueueMocapnetError::BadInput("video source not fully specified".to_string()))?; + + let ip_address = get_request_ip(&http_request); + + let maybe_user_preferred_visibility : Option = maybe_user_session + .as_ref() + .map(|user_session: &users_component::utils::user_session_extended::UserSessionExtended| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc + + let set_visibility = request.creator_set_visibility + .or(maybe_user_preferred_visibility) + .unwrap_or(Visibility::Public); + + let inference_args = MocapArgs { + maybe_video_source: Some(video_source), + maybe_ik1: Some(0.001f32), + maybe_ik2: Some(99i32), + maybe_ik3: Some(99i32), + maybe_smoothing1: Some(60f32), + maybe_smoothing2: Some(10f32), + maybe_size1: Some(1920i32), + maybe_size2: Some(1080i32), + }; + + info!("Creating mocapnet job record..."); + + let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + inference_category: InferenceCategory::Mocap, + maybe_model_type: Some(InferenceModelType::MocapNet), // NB: Model is static during inference + maybe_model_token: None, // NB: Model is static during inference + maybe_input_source_token: None, // TODO: Introduce a second foreign key ? + maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? + maybe_raw_inference_text: None, // No text + maybe_max_duration_seconds: None, + maybe_inference_args: Some(GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::Mocap), + args: Some(PolymorphicInferenceArgs::Mc(inference_args)), + }), + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: set_visibility, + priority_level, + requires_keepalive: plan.mocapnet_requires_frontend_keepalive(), + is_debug_request, + maybe_routing_tag: maybe_routing_tag.as_deref(), + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match query_result { + Ok((job_token, _id)) => job_token, + Err(err) => { + warn!("New generic inference job creation DB error: {:?}", err); + return Err(EnqueueMocapnetError::ServerError); + } + }; + + let response: EnqueueMocapnetSuccessResponse = EnqueueMocapnetSuccessResponse { + success: true, + inference_job_token: job_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| EnqueueMocapnetError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mocap/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mocap/mod.rs new file mode 100644 index 0000000000..4545280f48 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mocap/mod.rs @@ -0,0 +1 @@ +pub(crate) mod enqueue_mocapnet; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 0a6ed9375d..6ad279af5d 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -26,4 +26,5 @@ pub mod voice_conversion; pub mod voice_designer; pub mod w2l; pub mod user_bookmarks; -pub mod weights; \ No newline at end of file +pub mod weights; +pub(crate) mod mocap; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index c63903237e..d74734dbd7 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -13,6 +13,7 @@ use users_component::endpoints::get_profile_handler::get_profile_handler; use crate::http_server::endpoints::animation::enqueue_face_animation::enqueue_face_animation_handler; use crate::http_server::endpoints::animation::enqueue_rerender_animation::enqueue_rerender_animation_handler; +use crate::http_server::endpoints::mocap::enqueue_mocapnet::enqueue_mocapnet_handler; use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler; use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler; use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler; @@ -226,6 +227,11 @@ pub fn add_routes (app: App, server_environment: ServerEnvironment) -> .add_post("/v1/animation/rerender/create", enqueue_rerender_animation_handler) .into_app(); + // ==================== Mocap ======================== + let mut app = RouteBuilder::from_app(app) + .add_post("/v1/mocap/mocapnet/create", enqueue_mocapnet_handler) + .into_app(); + // ==================== "Generic" Inference ==================== let mut app = RouteBuilder::from_app(app)