From ebed7267681674a4ea83007a2d9f17a61636d740 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 3 Oct 2023 12:28:24 -0400 Subject: [PATCH 1/2] Resize SadTalker to "twitter" dimensions --- Cargo.lock | 2 + crates/lib/files/images/src/lib.rs | 4 + .../resize_image_file_preserving_aspect.rs | 43 +++++++ .../images/src/resize_preserving_aspect.rs | 81 ++++++++++--- .../generic_inference_args.rs | 2 + .../generic_inference_args/lipsync_payload.rs | 26 ++++- crates/service/job/inference_job/Cargo.toml | 2 + .../lipsync/sad_talker/process_job.rs | 54 ++++++++- .../lipsync/sad_talker/resize_image.rs | 26 ++++- .../lipsync/sad_talker/validate_job.rs | 12 +- .../animation/enqueue_lipsync_animation.rs | 108 +++++++++--------- 11 files changed, 280 insertions(+), 80 deletions(-) create mode 100644 crates/lib/files/images/src/resize_image_file_preserving_aspect.rs diff --git a/Cargo.lock b/Cargo.lock index 3a68aa5e70..c1772edad7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2941,6 +2941,7 @@ dependencies = [ "hashing", "hostname", "hyper 0.14.24", + "images", "jobs_common", "lfu", "log", @@ -2959,6 +2960,7 @@ dependencies = [ "subprocess", "subprocess_common", "tempdir", + "testing", "tokens", "tokio 1.25.0", "tts_common", diff --git a/crates/lib/files/images/src/lib.rs b/crates/lib/files/images/src/lib.rs index dbb0728069..4d4a55b5dd 100644 --- a/crates/lib/files/images/src/lib.rs +++ b/crates/lib/files/images/src/lib.rs @@ -19,4 +19,8 @@ #![allow(dead_code)] #![allow(non_snake_case)] +pub mod resize_image_file_preserving_aspect; pub mod resize_preserving_aspect; + +// Re-export image to everyone importing this library +pub use image; diff --git a/crates/lib/files/images/src/resize_image_file_preserving_aspect.rs b/crates/lib/files/images/src/resize_image_file_preserving_aspect.rs new file mode 100644 index 0000000000..6b57fe491f --- /dev/null +++ b/crates/lib/files/images/src/resize_image_file_preserving_aspect.rs @@ -0,0 +1,43 @@ +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +use image::DynamicImage; +use image::io::Reader; + +use errors::AnyhowResult; + +use crate::resize_preserving_aspect::resize_preserving_aspect; + +pub fn resize_image_file_preserving_aspect>( + source_image_file: P, + new_width: u32, + new_height: u32, + exact_fit: bool, +) -> AnyhowResult { + let file = File::open(source_image_file)?; + let reader = BufReader::new(file); + + let reader = Reader::new(reader) + .with_guessed_format()?; + + let img = reader.decode()?; + + Ok(resize_preserving_aspect(&img, new_width, new_height, exact_fit)) +} + +#[cfg(test)] +mod tests { + use testing::test_file_path::test_file_path; + + use crate::resize_image_file_preserving_aspect::resize_image_file_preserving_aspect; + + #[test] + fn test_happy_path() { + // NB: The other function does most of the testing + let path = test_file_path("test_data/image/mochi.jpg").unwrap(); + let image = resize_image_file_preserving_aspect(path, 100, 100, true).unwrap(); + assert_eq!(image.width(), 100); + assert_eq!(image.height(), 100); + } +} \ No newline at end of file diff --git a/crates/lib/files/images/src/resize_preserving_aspect.rs b/crates/lib/files/images/src/resize_preserving_aspect.rs index b0487eed46..ee4eb2078b 100644 --- a/crates/lib/files/images/src/resize_preserving_aspect.rs +++ b/crates/lib/files/images/src/resize_preserving_aspect.rs @@ -1,8 +1,21 @@ use image::DynamicImage; use image::imageops::FilterType; -pub fn resize_preserving_aspect(source_image: &DynamicImage, new_width: u32, new_height: u32) -> DynamicImage { - source_image.resize(new_width, new_height, FilterType::Lanczos3) +/// Resize an image preserving the exact aspect ratio of the source image. +/// +/// If `exact_fit` is true, the image will be resized to fit the largest dimension, +/// then cropped to fit the precise dimensions specified. +pub fn resize_preserving_aspect( + source_image: &DynamicImage, + new_width: u32, + new_height: u32, + exact_fit: bool, +) -> DynamicImage { + if exact_fit { + source_image.resize_to_fill(new_width, new_height, FilterType::Lanczos3) + } else { + source_image.resize(new_width, new_height, FilterType::Lanczos3) + } } #[cfg(test)] @@ -38,9 +51,7 @@ mod tests { #[test] fn downsize_wide_image_preserving_aspect() { let image = open_wide_image().unwrap(); - assert!(image.width() > image.height()); - - let resized = resize_preserving_aspect(&image, 500, 500); + let resized = resize_preserving_aspect(&image, 500, 500, false); assert!(resized.width() > resized.height()); assert_eq!(resized.width(), 500); assert_eq!(resized.height(), 375); @@ -49,9 +60,7 @@ mod tests { #[test] fn downsize_tall_image_preserving_aspect() { let image = open_tall_image().unwrap(); - assert!(image.width() < image.height()); - - let resized = resize_preserving_aspect(&image, 500, 500); + let resized = resize_preserving_aspect(&image, 500, 500, false); assert!(resized.width() < resized.height()); assert_eq!(resized.width(), 377); assert_eq!(resized.height(), 500); @@ -60,9 +69,7 @@ mod tests { #[test] fn enlarge_wide_image_preserving_aspect() { let image = open_wide_image().unwrap(); - assert!(image.width() > image.height()); - - let resized = resize_preserving_aspect(&image, 1200, 1200); + let resized = resize_preserving_aspect(&image, 1200, 1200, false); assert!(resized.width() > resized.height()); assert_eq!(resized.width(), 1200); assert_eq!(resized.height(), 900); @@ -71,11 +78,57 @@ mod tests { #[test] fn enlarge_tall_image_preserving_aspect() { let image = open_tall_image().unwrap(); - assert!(image.width() < image.height()); - - let resized = resize_preserving_aspect(&image, 1200, 1200); + let resized = resize_preserving_aspect(&image, 1200, 1200, false); assert!(resized.width() < resized.height()); assert_eq!(resized.width(), 904); assert_eq!(resized.height(), 1200); } + + #[test] + fn resize_wide_image_preserving_aspect_exact() { + // Downscale + let image = open_wide_image().unwrap(); + let resized = resize_preserving_aspect(&image, 500, 500, true); + assert_eq!(resized.width(), 500); + assert_eq!(resized.height(), 500); + + // Upscale + let resized = resize_preserving_aspect(&image, 1200, 1200, true); + assert_eq!(resized.width(), 1200); + assert_eq!(resized.height(), 1200); + + // Really wide + let resized = resize_preserving_aspect(&image, 1200, 100, true); + assert_eq!(resized.width(), 1200); + assert_eq!(resized.height(), 100); + + // Really tall + let resized = resize_preserving_aspect(&image, 100, 1200, true); + assert_eq!(resized.width(), 100); + assert_eq!(resized.height(), 1200); + } + + #[test] + fn resize_tall_image_preserving_aspect_exact() { + // Downscale + let image = open_tall_image().unwrap(); + let resized = resize_preserving_aspect(&image, 500, 500, true); + assert_eq!(resized.width(), 500); + assert_eq!(resized.height(), 500); + + // Upscale + let resized = resize_preserving_aspect(&image, 1200, 1200, true); + assert_eq!(resized.width(), 1200); + assert_eq!(resized.height(), 1200); + + // Really wide + let resized = resize_preserving_aspect(&image, 1200, 100, true); + assert_eq!(resized.width(), 1200); + assert_eq!(resized.height(), 100); + + // Really tall + let resized = resize_preserving_aspect(&image, 100, 1200, true); + assert_eq!(resized.width(), 100); + assert_eq!(resized.height(), 1200); + } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index eb26c9865b..089115ff67 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -128,6 +128,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, })), }; diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs index 43186b0b8d..ddb46b4804 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/lipsync_payload.rs @@ -34,9 +34,21 @@ pub struct LipsyncArgs { pub maybe_make_still: Option, /// Omit adding the watermark - #[serde(rename = "w")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(rename = "m")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. #[serde(skip_serializing_if = "Option::is_none")] pub maybe_remove_watermark: Option, + + /// Resize width + /// NB: FOR TESTING ONLY + #[serde(rename = "w")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_resize_width: Option, + + /// Resize height + /// NB: FOR TESTING ONLY + #[serde(rename = "h")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_resize_height: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] @@ -152,6 +164,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; let json = serde_json::ser::to_string(&args).unwrap(); assert_eq!(json, r#"{"a":{"F":"audio_media_file"},"i":{"F":"image_media_file"}}"#.to_string()); @@ -167,6 +181,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; let json = serde_json::ser::to_string(&args).unwrap(); assert_eq!(json, r#"{"a":{"U":"audio_media_upload"},"i":{"U":"image_media_upload"}}"#.to_string()); @@ -182,6 +198,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; let json = serde_json::ser::to_string(&args).unwrap(); assert_eq!(json, r#"{"a":{"T":"audio_tts_result"},"i":{"U":"image_media_upload"}}"#.to_string()); @@ -197,6 +215,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; let json = serde_json::ser::to_string(&args).unwrap(); assert_eq!(json, r#"{"a":{"V":"audio_voice_conversion_result"},"i":{"U":"image_media_upload"}}"#.to_string()); @@ -212,6 +232,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; let json = serde_json::ser::to_string(&args).unwrap(); assert_eq!(json, r#"{"f":"G"}"#.to_string()); @@ -227,6 +249,8 @@ mod tests { maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; let json = serde_json::ser::to_string(&args).unwrap(); assert_eq!(json, r#"{"f":"R"}"#.to_string()); diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 004cb29847..097c30f857 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -28,6 +28,7 @@ envvar = { path = "../../../lib/envvar" } errors = { path = "../../../lib/errors" } filesys = { path = "../../../lib/files/filesys" } hashing = { path = "../../../lib/files/hashing" } +images = { path = "../../../lib/files/images" } jobs_common = { path = "../../../lib/jobs_common" } media = { path = "../../../lib/files/media" } memory_caching = { path = "../../../lib/caching/memory_caching" } @@ -35,6 +36,7 @@ mimetypes = { path = "../../../lib/files/mimetypes" } mysql_queries = { path = "../../../lib/mysql_queries" } redis_common = { path = "../../../lib/redis_common" } subprocess_common = { path = "../../../lib/subprocess_common" } +testing = { path = "../../../lib/testing" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index 8830f96ae1..20497e11b1 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -20,11 +20,15 @@ use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::lipsync::sad_talker::download_audio_file::download_audio_file; use crate::job::job_types::lipsync::sad_talker::download_image_file::download_image_file; +use crate::job::job_types::lipsync::sad_talker::resize_image::resize_image; use crate::job::job_types::lipsync::sad_talker::sad_talker_inference_command::InferenceArgs; use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; use crate::job_dependencies::JobDependencies; use crate::util::common_commands::ffmpeg_logo_watermark_command; +/// The maximum that either width or height can be +const MAX_DIMENSION : u32 = 1500; + pub struct SadTalkerProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, @@ -102,11 +106,50 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result { + error!("Image resize failure: {:?}", err); + safe_delete_temp_file(&audio_path.filesystem_path); + safe_delete_temp_file(&image_path.filesystem_path); + safe_delete_temp_directory(&work_temp_dir); + return Err(ProcessSingleJobError::Other(anyhow!("image resize failure: {:?}", err))); + } + Ok(resized_image_path) => { + usable_image_path = resized_image_path; + } + } + } + + info!("Used image file: {:?}", usable_image_path); // ==================== SETUP FOR INFERENCE ==================== // @@ -122,7 +165,7 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result, Q: AsRef>(input_file: P, output_file: Q) -> AnyhowResult<()> { - +pub fn resize_image>( + input_file: P, + tempdir: &TempDir, + new_width: u32, + new_height: u32 +) -> AnyhowResult { + let image = resize_image_file_preserving_aspect( + input_file, + new_width, + new_height, + true + )?; + let output_file = tempdir.path().join("resized_image.png"); + info!("File successfully resized to {}x{} !", image.width(), image.height()); + image.save_with_format(&output_file, ImageFormat::Png)?; + Ok(output_file) } diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/validate_job.rs index a87310136d..7f3c7954a0 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/validate_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/validate_job.rs @@ -15,6 +15,9 @@ pub struct JobArgs<'a> { pub preprocess: Option, // NB: Enhancer controls quality pub enhancer: Option, + + pub width: Option, + pub height: Option, } pub fn validate_job(job: &AvailableInferenceJob) -> Result { @@ -70,7 +73,7 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result None, + None => Some("full".to_string()), Some(Preprocess::F) => Some("full".to_string()), Some(Preprocess::EF) => Some("extfull".to_string()), Some(Preprocess::C) => Some("crop".to_string()), @@ -83,6 +86,9 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result Some("RestoreFormer".to_string()), }; + let mut width = inference_args.maybe_resize_width; + let mut height = inference_args.maybe_resize_height; + Ok(JobArgs { audio_source, image_source, @@ -90,5 +96,7 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result, - is_storyteller_demo: Option, - /// SadTalker: cropping - crop: Option, + ///// SadTalker: cropping + //crop: Option, /// SadTalker: parameter to make the animation more still. make_still: Option, @@ -59,17 +56,48 @@ pub struct EnqueueLipsyncAnimationRequest { /// Remove the watermark remove_watermark: Option, + + /// Size of the video to generate + dimensions: Option, } +//#[derive(Deserialize, Copy, Clone)] +//#[serde(rename_all = "snake_case")] +//pub enum CropMode { +// /// SadTalker: "extcrop", which appears to be a wider crop than the "crop" option +// Crop, +// /// SadTalker: "crop" (the default) +// CloseCrop, +// /// SadTalker: "full" (can't tell that "extfull" does any differently) +// Full, +//} + #[derive(Deserialize, Copy, Clone)] #[serde(rename_all = "snake_case")] -pub enum CropMode { - /// SadTalker: "extcrop", which appears to be a wider crop than the "crop" option - Crop, - /// SadTalker: "crop" (the default) - CloseCrop, - /// SadTalker: "full" (can't tell that "extfull" does any differently) - Full, +pub enum FrameSize { + /// Twitter horizontal = 1280x720 + /// See: https://developer.twitter.com/en/docs/twitter-api/v1/media/upload-media/uploading-media/media-best-practices + TwitterLandscape, + + /// Twitter vertical = 720x1280 + TwitterPortrait, + + /// Twitter square = 720x720 + TwitterSquare, + + // D-ID: 1000x1000 + // Runway: 896x512 + // Pika: 1024x576 +} + +impl FrameSize { + pub fn get_width_and_height(&self) -> (u32, u32) { + match self { + FrameSize::TwitterLandscape => (1280, 720), + FrameSize::TwitterPortrait => (720, 1280), + FrameSize::TwitterSquare => (720, 720), + } + } } /// Treated as an enum. Only one of these may be set. @@ -137,7 +165,7 @@ pub async fn enqueue_lipsync_animation_handler( server_state: web::Data>) -> Result { let mut maybe_user_token : Option = None; - let mut priority_level ; + let priority_level ; let disable_rate_limiter = false; // NB: Careful! let mut mysql_connection = server_state.mysql_pool @@ -171,32 +199,6 @@ pub async fn enqueue_lipsync_animation_handler( // TODO: Separate priority for animation. priority_level = plan.web_vc_base_priority_level(); - // ==================== INVESTOR PRIORITY ==================== // - - // TODO/TEMP: Give investors even more priority - let mut is_investor = false; - - { - // TODO/TEMP: The storyteller.io website's AJAX calls will set this. - // This is just for the YCombinator demo. - match request.is_storyteller_demo { - Some(true) => { - is_investor = true; - }, - _ => {}, - }; - - // TODO/TEMP: The storyteller.io website will redirect and establish this cookie. - // This is just for the YCombinator demo. - if request_has_demo_cookie(&http_request) { - is_investor = true; - } - - if is_investor { - priority_level = FAKEYOU_INVESTOR_PRIORITY_LEVEL; - } - } - // ==================== DEBUG MODE + ROUTING TAG ==================== // let is_debug_request = @@ -210,7 +212,7 @@ pub async fn enqueue_lipsync_animation_handler( // ==================== RATE LIMIT ==================== // if !disable_rate_limiter { - let mut rate_limiter = match maybe_user_session { + let rate_limiter = match maybe_user_session { None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { if user.role.is_banned { @@ -220,11 +222,6 @@ pub async fn enqueue_lipsync_animation_handler( }, }; - // TODO/TEMP - if is_investor { - rate_limiter = &server_state.redis_rate_limiters.logged_in; - } - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { return Err(EnqueueLipsyncAnimationError::RateLimited); } @@ -276,22 +273,14 @@ pub async fn enqueue_lipsync_animation_handler( maybe_preprocess: None, maybe_make_still: None, maybe_remove_watermark: None, + maybe_resize_width: None, + maybe_resize_height: None, }; if request.high_quality.unwrap_or(false) { inference_args.maybe_face_enhancer = Some(FaceEnhancer::G); // "--enhancer gfpgan" } - if let Some(crop_mode) = request.crop { - let crop_mode = match crop_mode { - CropMode::Crop => Preprocess::EC, // NB: "extcrop" is less of a close crop than "crop" - CropMode::CloseCrop => Preprocess::C, - CropMode::Full => Preprocess::F, - }; - - inference_args.maybe_preprocess = Some(crop_mode); - } - if request.make_still.unwrap_or(false) { inference_args.maybe_make_still = Some(true); } @@ -300,6 +289,13 @@ pub async fn enqueue_lipsync_animation_handler( inference_args.maybe_remove_watermark = Some(true); } + let (width, height) = request.dimensions + .unwrap_or(FrameSize::TwitterLandscape) + .get_width_and_height(); + + inference_args.maybe_resize_width = Some(width); + inference_args.maybe_resize_height = Some(height); + info!("Creating lipsync animation job record..."); let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { From e676df0bf4cfa2291d4c93df7988df3e142dfa37 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 4 Oct 2023 00:25:21 -0400 Subject: [PATCH 2/2] premium plan features --- .../storyteller_web/src/configs/plans/plan.rs | 15 ++++++ .../src/configs/plans/plan_list.rs | 2 + .../animation/enqueue_lipsync_animation.rs | 46 +++++++++---------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan.rs b/crates/service/web/storyteller_web/src/configs/plans/plan.rs index 38d6a2b42b..67dd178cc1 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan.rs @@ -47,6 +47,8 @@ pub struct Plan { lipsync_requires_frontend_keepalive: bool, + can_remove_visual_watermarks: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -89,6 +91,7 @@ impl Plan { is_development_plan: builder.is_development_plan, is_synthetic_plan: builder.is_synthetic_plan, lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive, + can_remove_visual_watermarks: builder.can_remove_visual_watermarks, tts_base_priority_level: builder.tts_base_priority_level, tts_max_duration: builder.tts_max_duration, tts_max_character_length: builder.tts_max_character_length, @@ -143,6 +146,10 @@ impl Plan { self.lipsync_requires_frontend_keepalive } + pub fn can_remove_visual_watermarks(&self) -> bool { + self.can_remove_visual_watermarks + } + pub fn tts_base_priority_level(&self) -> u8 { self.tts_base_priority_level } @@ -183,6 +190,8 @@ pub struct PlanBuilder { lipsync_requires_frontend_keepalive: bool, + can_remove_visual_watermarks: bool, + // ========== Features for TTS ========== tts_base_priority_level: u8, @@ -228,6 +237,7 @@ impl PlanBuilder { // Lipsync (SadTalker, not Wav2Lip) lipsync_requires_frontend_keepalive: true, + can_remove_visual_watermarks: false, // TTS tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL, @@ -294,6 +304,11 @@ impl PlanBuilder { self } + pub fn can_remove_visual_watermarks(mut self, value: bool) -> Self { + self.can_remove_visual_watermarks = value; + self + } + pub fn tts_base_priority_level(mut self, value: u8) -> Self { self.tts_base_priority_level = value; self diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs b/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs index ccd5a0c90a..40a345c0cd 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan_list.rs @@ -146,6 +146,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .web_vc_base_priority_level(30) .web_vc_requires_frontend_keepalive(false) .lipsync_requires_frontend_keepalive(false) + .can_remove_visual_watermarks(true) .build()); plans.insert(PlanBuilder::new("fakeyou_elite") @@ -159,6 +160,7 @@ pub static PRODUCTION_PREMIUM_PLANS: Lazy> = Lazy::new(|| { .web_vc_base_priority_level(40) .web_vc_requires_frontend_keepalive(false) .lipsync_requires_frontend_keepalive(false) + .can_remove_visual_watermarks(true) .build()); // ========== Spanish plans ========== diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs index 26d3ac3d28..e636d8605c 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_lipsync_animation.rs @@ -45,20 +45,20 @@ pub struct EnqueueLipsyncAnimationRequest { creator_set_visibility: Option, - ///// SadTalker: cropping - //crop: Option, + /// Width/height of the video to generate (presets) + dimensions: Option, /// SadTalker: parameter to make the animation more still. make_still: Option, - /// SadTalker: parameter to enhance the face using gfpgan. - high_quality: Option, + /// SadTalker: we use gfpgan face enhancement by default; this disables it. + disable_face_enhancement: Option, - /// Remove the watermark + /// Remove the watermark (premium only) remove_watermark: Option, - /// Size of the video to generate - dimensions: Option, + ///// SadTalker: cropping + //crop: Option, } //#[derive(Deserialize, Copy, Clone)] @@ -165,8 +165,6 @@ pub async fn enqueue_lipsync_animation_handler( server_state: web::Data>) -> Result { let mut maybe_user_token : Option = None; - let priority_level ; - let disable_rate_limiter = false; // NB: Careful! let mut mysql_connection = server_state.mysql_pool .acquire() @@ -197,7 +195,7 @@ pub async fn enqueue_lipsync_animation_handler( maybe_user_session.as_ref()); // TODO: Separate priority for animation. - priority_level = plan.web_vc_base_priority_level(); + let priority_level = plan.web_vc_base_priority_level(); // ==================== DEBUG MODE + ROUTING TAG ==================== // @@ -211,20 +209,18 @@ pub async fn enqueue_lipsync_animation_handler( // ==================== RATE LIMIT ==================== // - if !disable_rate_limiter { - let rate_limiter = match maybe_user_session { - None => &server_state.redis_rate_limiters.logged_out, - Some(ref user) => { - if user.role.is_banned { - return Err(EnqueueLipsyncAnimationError::NotAuthorized); - } - &server_state.redis_rate_limiters.logged_in - }, - }; + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.role.is_banned { + return Err(EnqueueLipsyncAnimationError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; - if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(EnqueueLipsyncAnimationError::RateLimited); - } + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(EnqueueLipsyncAnimationError::RateLimited); } // ==================== LOOK UP MODEL INFO ==================== // @@ -277,7 +273,9 @@ pub async fn enqueue_lipsync_animation_handler( maybe_resize_height: None, }; - if request.high_quality.unwrap_or(false) { + let enable_face_enhancement = !request.disable_face_enhancement.unwrap_or(false); + + if enable_face_enhancement { inference_args.maybe_face_enhancer = Some(FaceEnhancer::G); // "--enhancer gfpgan" }