diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 8c90d4f8b4..0d02ba61d7 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -20,6 +20,7 @@ hifigan hparams huggingface + lipsync pathbuf replayable retryably diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index 2a3b93420b..2c20fdf1aa 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -16,6 +16,10 @@ use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] pub enum InferenceCategory { + /// Eg. SadTalker and possibly Wav2Lip + #[serde(rename = "lipsync_animation")] + LipsyncAnimation, + #[serde(rename = "text_to_speech")] TextToSpeech, @@ -31,6 +35,7 @@ impl_mysql_enum_coders!(InferenceCategory); impl InferenceCategory { pub fn to_str(&self) -> &'static str { match self { + Self::LipsyncAnimation => "lipsync_animation", Self::TextToSpeech => "text_to_speech", Self::VoiceConversion => "voice_conversion", } @@ -38,6 +43,7 @@ impl InferenceCategory { pub fn from_str(value: &str) -> Result { match value { + "lipsync_animation" => Ok(Self::LipsyncAnimation), "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), _ => Err(format!("invalid value: {:?}", value)), @@ -48,6 +54,7 @@ impl InferenceCategory { // NB: BTreeSet is sorted // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ + Self::LipsyncAnimation, Self::TextToSpeech, Self::VoiceConversion, ]) @@ -61,18 +68,21 @@ mod tests { #[test] fn test_serialization() { + assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation"); assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech"); assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); } #[test] fn to_str() { + assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation"); assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); } #[test] fn from_str() { + assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation); assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); } @@ -81,7 +91,8 @@ mod tests { fn all_variants() { // Static check let mut variants = InferenceCategory::all_variants(); - assert_eq!(variants.len(), 2); + assert_eq!(variants.len(), 3); + assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); assert_eq!(variants.pop_first(), None); diff --git a/crates/lib/mysql_queries/src/mediators/firehose_publisher.rs b/crates/lib/mysql_queries/src/mediators/firehose_publisher.rs index 5eef188769..a5e2346843 100644 --- a/crates/lib/mysql_queries/src/mediators/firehose_publisher.rs +++ b/crates/lib/mysql_queries/src/mediators/firehose_publisher.rs @@ -32,6 +32,10 @@ enum FirehoseEvent { VcInferenceStarted, VcInferenceCompleted, + // SadTalker, not Wav2Lip + LipsyncAnimationStarted, + LipsyncAnimationCompleted, + GenericDownloadStarted, GenericDownloadCompleted, @@ -64,6 +68,8 @@ impl FirehoseEvent { FirehoseEvent::W2lInferenceCompleted => "w2l_inference_completed", FirehoseEvent::VcInferenceStarted => "vc_inference_started", FirehoseEvent::VcInferenceCompleted => "vc_inference_completed", + FirehoseEvent::LipsyncAnimationStarted => "lipsync_animation_started", + FirehoseEvent::LipsyncAnimationCompleted => "lipsync_animation_completed", FirehoseEvent::GenericDownloadStarted => "generic_download_started", FirehoseEvent::GenericDownloadCompleted => "generic_download_completed", FirehoseEvent::MediaUploaded => "media_uploaded", @@ -220,6 +226,16 @@ impl FirehosePublisher { Ok(()) } + pub async fn enqueue_lipsync_animation(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::LipsyncAnimationStarted, + maybe_user_token.map(|u| u.as_str()), + Some(inference_job_token.as_str()), + Some(inference_job_token.as_str()), + ).await?; + Ok(()) + } + pub async fn enqueue_generic_download(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( FirehoseEvent::GenericDownloadStarted, diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs index e7236d6041..4a0f386419 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs @@ -1,8 +1,5 @@ use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use errors::AnyhowResult; -use tokens::files::media_upload::MediaUploadToken; -use tokens::tokens::tts_models::TtsModelToken; -use tokens::voice_conversion::model::VoiceConversionModelToken; /// Used to encode extra state for the `generic_inference_jobs` table. /// This should act somewhat like a serialized protobuf stored inside a record. @@ -23,6 +20,10 @@ pub struct GenericInferenceArgs { /// Do not change the values. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] pub enum InferenceCategoryAbbreviated { + #[serde(rename = "la")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(alias = "lipsync_animation")] + LipsyncAnimation, + #[serde(rename = "tts")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. #[serde(alias = "text_to_speech")] TextToSpeech, @@ -44,6 +45,17 @@ pub enum FundamentalFrequencyMethodForJob { #[derive(Clone, Debug, Serialize, Deserialize)] pub enum PolymorphicInferenceArgs { + /// Lipsync Animation (Short name to save space when serializing) + /// This is SadTalker, not Wav2Lip. + La { + #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + maybe_audio_media_upload_token: Option, + + #[serde(rename = "i")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + maybe_image_media_upload_token: Option, + }, /// Text to speech. (Short name to save space when serializing.) Tts { // No arguments yet. @@ -88,6 +100,7 @@ impl GenericInferenceArgs { impl InferenceCategoryAbbreviated { pub fn from_inference_category(category: InferenceCategory) -> Self { match category { + InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation, InferenceCategory::TextToSpeech => Self::TextToSpeech, InferenceCategory::VoiceConversion => Self::VoiceConversion, } @@ -95,6 +108,7 @@ impl InferenceCategoryAbbreviated { pub fn to_inference_category(self) -> InferenceCategory { match self { + Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation, Self::TextToSpeech => InferenceCategory::TextToSpeech, Self::VoiceConversion => InferenceCategory::VoiceConversion, } @@ -104,10 +118,25 @@ impl InferenceCategoryAbbreviated { #[cfg(test)] mod tests { use crate::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; - use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; - use tokens::files::media_upload::MediaUploadToken; - use tokens::tokens::tts_models::TtsModelToken; - use tokens::voice_conversion::model::VoiceConversionModelToken; + + #[test] + fn typical_lipsync_animation_args_serialize() { + let args = GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation), + args: Some(PolymorphicInferenceArgs::La { + maybe_audio_media_upload_token: Some("foo".to_string()), + maybe_image_media_upload_token: Some("bar".to_string()), + }), + }; + + let json = serde_json::ser::to_string(&args).unwrap(); + + // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating. + assert_eq!(json, r#"{"cat":"la","args":{"La":{"a":"foo","i":"bar"}}}"#.to_string()); + + // NB: Make sure we don't overflow the DB field capacity (TEXT column). + assert!(json.len() < 1000); + } #[test] fn typical_tts_args_serialize() { diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 880ca3d88e..7e3cd0486f 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -120,6 +120,7 @@ WHERE jobs.token = ? }; let maybe_model_title = match record.inference_category { + InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), }; @@ -127,6 +128,7 @@ WHERE jobs.token = ? // NB: A bit of a hack. We store TTS results with a full path. // Going forward, all other record types will store a hash. let (bucket_path_is_hash, maybe_public_bucket_hash) = match record.inference_category { + InferenceCategory::LipsyncAnimation => (true, Some("todo")), // TODO - these values are wrong InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), }; diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan.rs b/crates/service/web/storyteller_web/src/configs/plans/plan.rs index 6879f5fdd9..cc87e400a0 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan.rs @@ -42,6 +42,10 @@ pub struct Plan { /// Whether this plan does not exist in Stripe and only exists as defined by this server. is_synthetic_plan: bool, + // ========== Features for lipsync (SadTalker, not Wav2Lip) ========== + + lipsync_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -83,6 +87,7 @@ impl Plan { stripe_price_id: builder.stripe_price_id.clone(), is_development_plan: builder.is_development_plan, is_synthetic_plan: builder.is_synthetic_plan, + lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive, tts_base_priority_level: builder.tts_base_priority_level, tts_max_duration: builder.tts_max_duration, tts_max_character_length: builder.tts_max_character_length, @@ -133,6 +138,10 @@ impl Plan { self.is_synthetic_plan } + pub fn lipsync_requires_frontend_keepalive(&self) -> bool { + self.lipsync_requires_frontend_keepalive + } + pub fn tts_base_priority_level(&self) -> u8 { self.tts_base_priority_level } @@ -169,6 +178,10 @@ pub struct PlanBuilder { is_development_plan: bool, is_synthetic_plan: bool, + // ========== Features for lipsync (SadTalker, not Wav2Lip) ========== + + lipsync_requires_frontend_keepalive: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -212,6 +225,9 @@ impl PlanBuilder { is_development_plan: false, is_synthetic_plan: false, + // Lipsync (SadTalker, not Wav2Lip) + lipsync_requires_frontend_keepalive: true, + // TTS tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL, tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS), @@ -272,6 +288,11 @@ impl PlanBuilder { self } + pub fn lipsync_requires_frontend_keepalive(mut self, value: bool) -> Self { + self.lipsync_requires_frontend_keepalive = value; + self + } + pub fn tts_base_priority_level(mut self, value: u8) -> Self { self.tts_base_priority_level = value; self diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs b/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs index be5ac989b3..cc01e2af83 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs @@ -15,6 +15,7 @@ pub static FREE_LOGGED_OUT_PLAN : Lazy = Lazy::new(|| { .tts_max_character_length(1024) .web_vc_base_priority_level(FAKEYOU_ANONYMOUS_PRIORITY_LEVEL) .web_vc_requires_frontend_keepalive(true) + .lipsync_requires_frontend_keepalive(true) .build() }); @@ -28,6 +29,7 @@ pub static FREE_LOGGED_OUT_FIRST_TRY_PLAN : Lazy = Lazy::new(|| { .tts_max_character_length(1024) .web_vc_base_priority_level(FAKEYOU_LOGGED_IN_PRIORITY_LEVEL) // NB: Same as logged-in free users. .web_vc_requires_frontend_keepalive(true) + .lipsync_requires_frontend_keepalive(true) .build() }); @@ -41,6 +43,7 @@ pub static FREE_LOGGED_IN_PLAN : Lazy = Lazy::new(|| { .tts_max_character_length(1024) .web_vc_base_priority_level(FAKEYOU_LOGGED_IN_PRIORITY_LEVEL) .web_vc_requires_frontend_keepalive(true) + .lipsync_requires_frontend_keepalive(true) .build() }); @@ -56,6 +59,7 @@ pub static LOYALTY_PLANS : Lazy> = Lazy::new(|| { .tts_max_duration_seconds(30) .web_vc_base_priority_level(2) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); plans @@ -77,6 +81,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_max_character_length(2048) .tts_max_duration_seconds(30) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); plans.insert(PlanBuilder::new("development_fakeyou_pro") @@ -89,6 +94,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_max_character_length(2048) .tts_max_duration_seconds(60 * 2) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); plans.insert(PlanBuilder::new("development_fakeyou_elite") @@ -101,6 +107,7 @@ pub static DEVELOPMENT_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_max_character_length(2048) .tts_max_duration_seconds(60 * 5) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); plans @@ -122,6 +129,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_max_duration_seconds(30) .web_vc_base_priority_level(20) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); plans.insert(PlanBuilder::new("fakeyou_pro") @@ -134,6 +142,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_max_duration_seconds(60 * 2) .web_vc_base_priority_level(30) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); plans.insert(PlanBuilder::new("fakeyou_elite") @@ -146,6 +155,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_max_duration_seconds(60 * 5) .web_vc_base_priority_level(40) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .build()); // ========== Spanish plans ========== @@ -158,6 +168,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .tts_base_priority_level(10) .web_vc_base_priority_level(10) .web_vc_requires_frontend_keepalive(false) + .lipsync_requires_frontend_keepalive(false) .tts_max_duration_seconds(30) .build()); @@ -405,6 +416,7 @@ mod test { fn test_assert_web_vc_keepalive_forced() { FREE_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| { assert!(plan.web_vc_requires_frontend_keepalive()); + assert!(plan.lipsync_requires_frontend_keepalive()); }); } @@ -412,9 +424,11 @@ mod test { fn test_assert_web_vc_no_keepalive() { LOYALTY_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| { assert!(!plan.web_vc_requires_frontend_keepalive()); + assert!(!plan.lipsync_requires_frontend_keepalive()); }); PRODUCTION_PREMIUM_PLANS_BY_SLUG.iter().for_each(|(price_id, plan)| { assert!(!plan.web_vc_requires_frontend_keepalive()); + assert!(!plan.lipsync_requires_frontend_keepalive()); }); } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs new file mode 100644 index 0000000000..c1d09d485c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs @@ -0,0 +1,300 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{web, HttpResponse, HttpRequest}; +use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; +use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::common::visibility::Visibility; +use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{info, warn}; +use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job}; +use std::sync::Arc; +use tokens::files::media_upload::MediaUploadToken; +use tokens::jobs::inference::InferenceJobToken; +use tokens::users::user::UserToken; +use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL; + +/// Debug requests can get routed to special "debug-only" workers, which can +/// be used to trial new code, run debugging, etc. +const DEBUG_HEADER_NAME : &'static str = "enable-debug-mode"; + +/// The routing tag header can send workloads to particular k8s hosts. +/// This is useful for catching the live logs or intercepting the job. +const ROUTING_TAG_HEADER_NAME : &'static str = "routing-tag"; + +#[derive(Deserialize)] +pub struct EnqueueLipsyncAnimationRequest { + uuid_idempotency_token: String, + + // TODO: Eventually accept result tokens for audio. + source_image_media_upload_token: MediaUploadToken, + source_audio_media_upload_token: MediaUploadToken, + + creator_set_visibility: Option, + is_storyteller_demo: Option, + + /// SadTalker: parameter to animate the full body. + animate_full_body: Option, +} + +#[derive(Serialize)] +pub struct EnqueueLipsyncAnimationSuccessResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} + +#[derive(Debug)] +pub enum EnqueueLipsyncAnimationError { + BadInput(String), + NotAuthorized, + ServerError, + RateLimited, +} + +impl ResponseError for EnqueueLipsyncAnimationError { + fn status_code(&self) -> StatusCode { + match *self { + EnqueueLipsyncAnimationError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueLipsyncAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueLipsyncAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueLipsyncAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EnqueueLipsyncAnimationError::BadInput(reason) => reason.to_string(), + EnqueueLipsyncAnimationError::NotAuthorized => "unauthorized".to_string(), + EnqueueLipsyncAnimationError::ServerError => "server error".to_string(), + EnqueueLipsyncAnimationError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl std::fmt::Display for EnqueueLipsyncAnimationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn enqueue_lipsync_animation_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let mut maybe_user_token : Option = None; + let mut priority_level ; + let disable_rate_limiter = false; // NB: Careful! + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueLipsyncAnimationError::ServerError + })?; + + // ==================== USER SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EnqueueLipsyncAnimationError::ServerError + })?; + + if let Some(user_session) = maybe_user_session.as_ref() { + maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token)); + } + + // TODO: Plan should handle "first anonymous use" and "investor" cases. + let plan = get_correct_plan_for_session( + server_state.server_environment, + maybe_user_session.as_ref()); + + // TODO: Separate priority for animation. + priority_level = plan.web_vc_base_priority_level(); + + // ==================== INVESTOR PRIORITY ==================== // + + // TODO/TEMP: Give investors even more priority + let mut is_investor = false; + + { + // TODO/TEMP: The storyteller.io website's AJAX calls will set this. + // This is just for the YCombinator demo. + match request.is_storyteller_demo { + Some(true) => { + is_investor = true; + }, + _ => {}, + }; + + // TODO/TEMP: The storyteller.io website will redirect and establish this cookie. + // This is just for the YCombinator demo. + if request_has_demo_cookie(&http_request) { + is_investor = true; + } + + if is_investor { + priority_level = FAKEYOU_INVESTOR_PRIORITY_LEVEL; + } + } + + // ==================== DEBUG MODE + ROUTING TAG ==================== // + + let is_debug_request = + get_request_header_optional(&http_request, DEBUG_HEADER_NAME) + .is_some(); + + let maybe_routing_tag= + get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME) + .map(|routing_tag| routing_tag.trim().to_string()); + + // ==================== RATE LIMIT ==================== // + + if !disable_rate_limiter { + let mut rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.role.is_banned { + return Err(EnqueueLipsyncAnimationError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + // TODO/TEMP + if is_investor { + rate_limiter = &server_state.redis_rate_limiters.logged_in; + } + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(EnqueueLipsyncAnimationError::RateLimited); + } + } + + // ==================== LOOK UP MODEL INFO ==================== // + + // TODO(bt): CHECK DATABASE! + let audio_media_token = request.source_audio_media_upload_token.clone(); + let image_media_token = request.source_image_media_upload_token.clone(); + + // ==================== CHECK AND ENQUEUE ANIMATION ==================== // + + //let mut redis = server_state.redis_pool + // .get() + // .map_err(|e| { + // error!("redis error: {:?}", e); + // EnqueueLipsyncAnimationError::ServerError + // })?; + + //let redis_count_key = RedisKeys::web_vc_model_usage_count(model_token.as_str()); + + //redis.incr(&redis_count_key, 1) + // .map_err(|e| { + // warn!("redis error: {:?}", e); + // EnqueueLipsyncAnimationError::ServerError + // })?; + + let ip_address = get_request_ip(&http_request); + + let maybe_user_preferred_visibility : Option = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc + + let set_visibility = request.creator_set_visibility + .or(maybe_user_preferred_visibility) + .unwrap_or(Visibility::Public); + + + info!("Creating lipsync animation job record..."); + + // TODO: Cleanup. + let maybe_args = Some(PolymorphicInferenceArgs::La { + maybe_audio_media_upload_token: Some(audio_media_token.to_string()), + maybe_image_media_upload_token: Some(image_media_token.to_string()), + }); + + // TODO: SadTalker arguments. + //let set_args = request.auto_predict_f0.is_some() + // || request.transpose.is_some() + // || request.override_f0_method.is_some(); + + //if set_args { + // maybe_args = Some(PolymorphicInferenceArgs::Vc { + // auto_predict_f0: request.auto_predict_f0, + // override_f0_method: request.override_f0_method.map(|f| match f { + // FundamentalFrequencyMethod::Crepe => FundamentalFrequencyMethodForJob::Crepe, + // FundamentalFrequencyMethod::Dio => FundamentalFrequencyMethodForJob::Dio, + // FundamentalFrequencyMethod::Harvest => FundamentalFrequencyMethodForJob::Harvest, + // }), + // transpose: request.transpose, + // }); + //} + + let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + inference_category: InferenceCategory::LipsyncAnimation, + maybe_model_type: None, // NB: Model is static during inference + maybe_model_token: None, // NB: Model is static during inference + maybe_input_source_token: None, // TODO: Introduce a second foreign key ? + maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? + maybe_raw_inference_text: None, // No text + maybe_inference_args: Some(GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::LipsyncAnimation), + args: maybe_args, + }), + maybe_creator_user_token: maybe_user_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: set_visibility, + priority_level, + requires_keepalive: plan.lipsync_requires_frontend_keepalive(), + is_debug_request, + maybe_routing_tag: maybe_routing_tag.as_deref(), + mysql_pool: &server_state.mysql_pool, + }).await; + + let job_token = match query_result { + Ok((job_token, _id)) => job_token, + Err(err) => { + warn!("New generic inference job creation DB error: {:?}", err); + return Err(EnqueueLipsyncAnimationError::ServerError); + } + }; + + server_state.firehose_publisher.enqueue_lipsync_animation( + maybe_user_token.as_ref(), + &job_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + EnqueueLipsyncAnimationError::ServerError + })?; + + let response = EnqueueLipsyncAnimationSuccessResponse { + success: true, + inference_job_token: job_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| EnqueueLipsyncAnimationError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs new file mode 100644 index 0000000000..1102d9357d --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/mod.rs @@ -0,0 +1 @@ +pub mod enqueue_lipsync_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index eee9882534..93a81c3501 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -216,6 +216,10 @@ pub async fn get_inference_job_status_handler( }, maybe_result: record.maybe_result_details.map(|result_details| { let public_bucket_media_path = match inference_category { + InferenceCategory::LipsyncAnimation => { + // TODO + "TODO".to_string() + } InferenceCategory::TextToSpeech => { // NB: TTS results receive the legacy treatment where their table only reports the full bucket path result_details.public_bucket_location_or_hash diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 6f730c94c5..e32479e65b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -1,3 +1,4 @@ +pub mod animation; pub mod api_tokens; pub mod categories; pub mod comments;