diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index d1a420544b..19bf5a74c7 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -26,8 +26,8 @@ pub enum InferenceCategory { #[serde(rename = "voice_conversion")] VoiceConversion, - #[serde(rename = "rerender_a_video")] - RerenderAVideo, + #[serde(rename = "video_filter")] + VideoFilter, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -41,7 +41,7 @@ impl InferenceCategory { Self::LipsyncAnimation => "lipsync_animation", Self::TextToSpeech => "text_to_speech", Self::VoiceConversion => "voice_conversion", - Self::RerenderAVideo => "rerender_a_video", + Self::VideoFilter => "video_filter", } } @@ -50,7 +50,7 @@ impl InferenceCategory { "lipsync_animation" => Ok(Self::LipsyncAnimation), "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), - "rerender_a_video" => Ok(Self::RerenderAVideo), + "video_filter" => Ok(Self::VideoFilter), _ => Err(format!("invalid value: {:?}", value)), } } @@ -62,7 +62,7 @@ impl InferenceCategory { Self::LipsyncAnimation, Self::TextToSpeech, Self::VoiceConversion, - Self::RerenderAVideo, + Self::VideoFilter, ]) } } @@ -77,7 +77,7 @@ mod tests { assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation"); assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech"); assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); - assert_serialization(InferenceCategory::RerenderAVideo, "rerender_a_video"); + assert_serialization(InferenceCategory::VideoFilter, "video_filter"); } #[test] @@ -85,7 +85,7 @@ mod tests { assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation"); assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); - assert_eq!(InferenceCategory::RerenderAVideo.to_str(), "rerender_a_video"); + assert_eq!(InferenceCategory::VideoFilter.to_str(), "video_filter"); } #[test] @@ -93,7 +93,7 @@ mod tests { assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation); assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); - assert_eq!(InferenceCategory::from_str("rerender_a_video").unwrap(), InferenceCategory::RerenderAVideo); + assert_eq!(InferenceCategory::from_str("video_filter").unwrap(), InferenceCategory::VideoFilter); } #[test] @@ -104,7 +104,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); - assert_eq!(variants.pop_first(), Some(InferenceCategory::RerenderAVideo)); + assert_eq!(variants.pop_first(), Some(InferenceCategory::VideoFilter)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index e5b2e90771..c751d5f5cc 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -21,9 +21,9 @@ pub enum IdCategory { #[serde(rename = "lipsync_animation")] LipsyncAnimationResult, - // Results from rerender a video - #[serde(rename = "rerender")] - RerenderResult, + // Results from video filters + #[serde(rename = "video_filter")] + VideoFilterResult, /// Results from tacotron2 /// Applies for RVC and SVC @@ -66,7 +66,7 @@ impl IdCategory { Self::ZeroShotVoiceDataset => "zs_dataset", Self::ZeroShotVoiceEmbedding => "zs_voice", Self::ZeroShotTtsResult => "zs_tts_result", - Self::RerenderResult => "rerender", + Self::VideoFilterResult => "video_filter", Self::ModelWeights => "model_weights", } } @@ -80,7 +80,7 @@ impl IdCategory { "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), "zs_tts_result" => Ok(Self::ZeroShotTtsResult), - "rerender" => Ok(Self::RerenderResult), + "video_filter" => Ok(Self::VideoFilterResult), "model_weights" => Ok(Self::ModelWeights), _ => Err(format!("invalid value: {:?}", value)), } @@ -92,7 +92,7 @@ impl IdCategory { BTreeSet::from([ Self::MediaFile, Self::LipsyncAnimationResult, - Self::RerenderResult, + Self::VideoFilterResult, Self::TtsResult, Self::VoiceConversionResult, Self::ZeroShotTtsResult, @@ -115,7 +115,7 @@ mod tests { fn test_serialization() { assert_serialization(IdCategory::MediaFile, "media_file"); assert_serialization(IdCategory::LipsyncAnimationResult, "lipsync_animation"); - assert_serialization(IdCategory::RerenderResult, "rerender"); + assert_serialization(IdCategory::VideoFilterResult, "video_filter"); assert_serialization(IdCategory::TtsResult, "tts_result"); assert_serialization(IdCategory::VoiceConversionResult, "voice_conversion"); assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset"); @@ -128,7 +128,7 @@ mod tests { fn to_str() { assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); assert_eq!(IdCategory::LipsyncAnimationResult.to_str(), "lipsync_animation"); - assert_eq!(IdCategory::RerenderResult.to_str(), "rerender"); + assert_eq!(IdCategory::VideoFilterResult.to_str(), "video_filter"); assert_eq!(IdCategory::TtsResult.to_str(), "tts_result"); assert_eq!(IdCategory::VoiceConversionResult.to_str(), "voice_conversion"); assert_eq!(IdCategory::ZeroShotVoiceDataset.to_str(), "zs_dataset"); @@ -141,7 +141,7 @@ mod tests { fn from_str() { assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimationResult); - assert_eq!(IdCategory::from_str("rerender").unwrap(), IdCategory::RerenderResult); + assert_eq!(IdCategory::from_str("video_filter").unwrap(), IdCategory::VideoFilterResult); assert_eq!(IdCategory::from_str("tts_result").unwrap(), IdCategory::TtsResult); assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversionResult); assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZeroShotVoiceDataset); @@ -157,7 +157,7 @@ mod tests { assert_eq!(variants.len(), 9); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); - assert_eq!(variants.pop_first(), Some(IdCategory::RerenderResult)); + assert_eq!(variants.pop_first(), Some(IdCategory::VideoFilterResult)); assert_eq!(variants.pop_first(), Some(IdCategory::TtsResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotTtsResult)); diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 103bab0042..e74da29ecf 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -39,9 +39,9 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "zs_voice")] ZeroShotVoice, - // Media files for Rerender - #[serde(rename = "rerender")] - Rerender, + // Media files for video filters + #[serde(rename = "video_filter")] + VideoFilter, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -58,7 +58,7 @@ impl MediaFileOriginProductCategory { Self::Unknown => "unknown", Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", - Self::Rerender => "rerender", + Self::VideoFilter => "video_filter", } } @@ -69,7 +69,7 @@ impl MediaFileOriginProductCategory { "unknown" => Ok(Self::Unknown), "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), - "rerender" => Ok(Self::Rerender), + "video_filter" => Ok(Self::VideoFilter), _ => Err(format!("invalid value: {:?}", value)), } } @@ -80,7 +80,7 @@ impl MediaFileOriginProductCategory { BTreeSet::from([ Self::FaceAnimator, Self::TextToSpeech, - Self::Rerender, + Self::VideoFilter, Self::Unknown, Self::VoiceConversion, Self::ZeroShotVoice, @@ -103,7 +103,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); - assert_serialization(MediaFileOriginProductCategory::Rerender, "rerender") + assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter") } #[test] @@ -113,7 +113,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); - assert_eq!(MediaFileOriginProductCategory::Rerender.to_str(), "rerender"); + assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter"); } #[test] @@ -123,7 +123,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); - assert_eq!(MediaFileOriginProductCategory::from_str("rerender").unwrap(), MediaFileOriginProductCategory::Rerender); + assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter); } #[test] @@ -135,7 +135,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); - assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Rerender)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index f34a4469c3..a67dd0bd91 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -37,9 +37,9 @@ pub enum InferenceCategoryAbbreviated { #[serde(alias = "voice_conversion")] VoiceConversion, - #[serde(rename = "rr")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. - #[serde(alias = "rerender_a_video")] - RerenderAVideo, + #[serde(rename = "vf")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(alias = "video_filter")] + VideoFilter, } #[derive(Serialize, Deserialize, Clone, Copy, Debug)] @@ -112,7 +112,7 @@ impl InferenceCategoryAbbreviated { InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation, InferenceCategory::TextToSpeech => Self::TextToSpeech, InferenceCategory::VoiceConversion => Self::VoiceConversion, - InferenceCategory::RerenderAVideo => Self::RerenderAVideo, + InferenceCategory::VideoFilter => Self::VideoFilter, } } @@ -121,7 +121,7 @@ impl InferenceCategoryAbbreviated { Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation, Self::TextToSpeech => InferenceCategory::TextToSpeech, Self::VoiceConversion => InferenceCategory::VoiceConversion, - Self::RerenderAVideo => InferenceCategory::RerenderAVideo, + Self::VideoFilter => InferenceCategory::VideoFilter, } } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs index 12d316933f..1fbaeaab0a 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs @@ -1,3 +1,5 @@ +use tokens::tokens::model_weights::ModelWeightToken; + /// Video sources can be one of several: /// - F: media_files (todo) /// - U: media_uploads (legacy) @@ -32,5 +34,25 @@ pub struct RerenderArgs { #[serde(rename = "sd")] #[serde(skip_serializing_if = "Option::is_none")] - pub maybe_sd_model: Option, + pub maybe_sd_model_token: Option, + + #[serde(rename = "lora")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_lora_model_token: Option, + + #[serde(rename = "p")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_prompt: Option, + + #[serde(rename = "ap")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_a_prompt: Option, + + #[serde(rename = "np")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_n_prompt: Option, + + #[serde(rename = "se")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_seed: Option, } diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 489ddd8391..e2a9b7be9c 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -178,7 +178,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), - InferenceCategory::RerenderAVideo => Some("rerender animation"), + InferenceCategory::VideoFilter => Some("Video Filter"), }; // NB: A bit of a hack. We store TTS results with a full path. @@ -187,7 +187,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), - InferenceCategory::RerenderAVideo => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), + InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), }; // NB: We've moved voice conversion out of their own table and into media_files diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs index c599a4fad0..b1adc3d6c3 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs @@ -55,7 +55,7 @@ pub async fn insert_media_file_from_rerender( let next_rerender_id = transactional_increment_generic_synthetic_id( &user_token, - IdCategory::RerenderResult, + IdCategory::VideoFilterResult, &mut transaction ).await?; @@ -67,7 +67,7 @@ pub async fn insert_media_file_from_rerender( let creator_set_visibility = args.job.creator_set_visibility.clone(); const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; - const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Rerender; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::VideoFilter; const ORIGIN_MODEL_TYPE : MediaFileOriginModelType = MediaFileOriginModelType::Rerender; const MEDIA_TYPE : MediaFileType = MediaFileType::Video; diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs index 050f4de251..01a128f9a7 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs @@ -33,7 +33,7 @@ pub struct RetrivedModelWeight { pub file_size_bytes: i32, pub file_checksum_sha2: String, pub private_bucket_hash: String, - pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_prefix: Option, // TODO crossproduct rename these to public thx ^-^ pub maybe_private_bucket_extension: Option, pub cached_user_ratings_negative_count: u32, pub cached_user_ratings_positive_count: u32, diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index c7edcdbb7f..1aab84a477 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -16,7 +16,7 @@ use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSucce use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job; use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job; -use crate::job::job_types::videofilter::process_single_vf_job::process_single_rr_job; +use crate::job::job_types::videofilter::process_single_vf_job::process_single_vf_job; use crate::job_dependencies::JobDependencies; pub async fn process_single_job( @@ -183,8 +183,8 @@ async fn do_process_single_job( InferenceCategory::VoiceConversion => { process_single_vc_job(job_dependencies, job).await? } - InferenceCategory::RerenderAVideo => { - process_single_rr_job(job_dependencies, job).await? + InferenceCategory::VideoFilter => { + process_single_vf_job(job_dependencies, job).await? } }; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index d5ab7b371c..076a723ef9 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -6,59 +6,7 @@ use crate::job::job_types::videofilter::rerender_a_video; use crate::job::job_types::videofilter::rerender_a_video::process_job::RerenderProcessJobArgs; use crate::job_dependencies::JobDependencies; -pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { - // let maybe_inference_args = job.maybe_inference_args.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Inference args not found")))?; - // - // let maybe_rerender_args = maybe_inference_args.args.as_ref().map(|args| match args { - // Rr(args) => Some(args), - // _ => None, - // }).flatten(); - // - // let rerender_args = match maybe_rerender_args { - // None => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), - // Some(args) => args, - // }; - // - // let videofilter_source = rerender_args.maybe_video_source.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Video source not found")))?; - // - // let media_file_token = match videofilter_source { - // VideofilterVideoSource::F(token) => token, - // _ => return Err(ProcessSingleJobError::Other(anyhow!("Video source not found"))), - // }; - // - // let media_file_token = MediaFileToken::new_from_str(media_file_token); - // - // info!("media_file_token: {:?}", media_file_token); - // - // - // let media_file = match media_file_result { - // Ok(Some(result)) => result, - // Ok(None) => { - // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); - // } - // Err(e) => { - // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); - // } - // }; - - // info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); - // info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); - // info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); - - - - // let job_success_result = match vc_model.model_type { - // VoiceConversionModelType::RvcV2 => { - // rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { - // job_dependencies, - // job, - // vc_model: &vc_model, - // media_upload_token: &media_upload_token, - // media_upload: &media_upload, - // }).await? - // } - // }; - +pub async fn process_single_vf_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { let job_success_result = rerender_a_video::process_job::process_job( RerenderProcessJobArgs { job_dependencies, diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs new file mode 100644 index 0000000000..ee848d3361 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs @@ -0,0 +1,64 @@ +use std::path::PathBuf; + +use anyhow::anyhow; +use tempdir::TempDir; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use cloud_storage::bucket_client::BucketClient; +use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::model_weights::get_weight::RetrivedModelWeight; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; +use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; + +pub struct ModelFile { + pub filesystem_path: PathBuf, +} + +pub async fn download_model_file( + model_record: &Option, + public_bucket_client: &BucketClient, + job_progress_reporter: &mut Box, + job: &AvailableInferenceJob, + temp_dir_creator: &ScopedTempDirCreator, + work_temp_dir: &TempDir, +) -> Result { + + let bucket_object_path; + let model_filename; + match model_record { + Some(model_record) => { + let model_bucket_path = + MediaFileBucketPath::from_object_hash( + &model_record.private_bucket_hash, + model_record.maybe_private_bucket_prefix.as_deref(), + model_record.maybe_private_bucket_extension.as_deref(), + ); + model_filename = model_record.private_bucket_hash.clone() + ".".to_owned() + model_record.maybe_private_bucket_extension.as_deref().unwrap_or("bin"); + bucket_object_path = model_bucket_path.to_full_object_pathbuf(); + } + None => { + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find model file"))) + } + }; + let downloaded_filesystem_path = work_temp_dir.path().join(model_filename); + + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "model file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, + job_progress_reporter, + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; + + Ok(ModelFile { + filesystem_path: downloaded_filesystem_path, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 28c9dea48a..dc00dc3412 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -4,3 +4,4 @@ mod download_video_file; mod rerender_inference_command; pub(crate) mod rerender_dependencies; mod categorize_error; +mod download_model_file; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs index 705671b1c1..cce8a0b8b5 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs @@ -16,9 +16,11 @@ use mimetypes::mimetype_for_file::get_mimetype_for_file; use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs}; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::videofilter::rerender_a_video::download_model_file::download_model_file; use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file; use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; @@ -122,22 +124,69 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), }; - let sd_model = rr_args.maybe_sd_model.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?; + + // =================== LOAD MODELS AND LORAS ==================== // + + let sd_model_token = rr_args.maybe_sd_model_token.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?; + + let retrieved_sd_record = get_weight_by_token( + sd_model_token, + false, + &deps.db.mysql_pool + ).await?; + + let model_path = download_model_file( + &retrieved_sd_record, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir).await?; + + info!("Downloaded model file: {:?}", model_path.filesystem_path); + let sd_model = model_path.filesystem_path.to_str().unwrap().to_string(); + + let lora_path = match rr_args.maybe_lora_model_token.clone() { + Some(lora_model_token) => { + let retrieved_lora_record = get_weight_by_token( + &lora_model_token, + false, + &deps.db.mysql_pool + ).await?; + + let lora_path = download_model_file( + &retrieved_lora_record, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir).await?; + + info!("Downloaded lora file: {:?}", lora_path.filesystem_path); + + Some(lora_path.filesystem_path) + } + None => None + }; + let sd_lora = match lora_path { + Some(lora_path) => Some(lora_path.to_str().unwrap().to_string()), + None => None + }; // ============== CREATE AND WRITE CONFIG TO DISK ============== // // create seed if not exists - // let seed = rr_args.seed.unwrap_or_else(|| rand::random::()); - let seed = rand::random::(); + let seed = rr_args.maybe_seed.unwrap_or_else(|| rand::random::()); let config = RerenderConfig { input: Some(usable_video_path.to_str().unwrap().to_string()), output: Some(output_video_fs_path.to_str().unwrap().to_string()), work_dir: Some(work_temp_dir.path().to_str().unwrap().to_string()), sd_model: Some(sd_model.to_string()), - prompt: Some("1girl, anime style, red hair".to_string()), - a_prompt: Some("1girl, 3d, realistic, read hair".to_string()), - n_prompt: Some("Bad quality, low quality, photo, artifacts, watermark, signature, faceless".to_string()), + lora_path: sd_lora, + prompt: rr_args.maybe_prompt.clone(), + a_prompt: rr_args.maybe_a_prompt.clone(), + n_prompt: rr_args.maybe_n_prompt.clone(), seed: Some(seed as i64), ..RerenderConfig::new() }; @@ -170,14 +219,6 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { - // warn!("Face not detected in source image"); - // error = ProcessSingleJobError::FaceDetectionFailure; - // } - // _ => {} - // } } safe_delete_temp_file(&video_path.filesystem_path); @@ -234,7 +275,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { pub video_source: &'a VideofilterVideoSource, - pub sd_model: &'a str, + pub sd_model_token: &'a ModelWeightToken, + pub lora_model_token: &'a Option, + pub prompt: &'a str, + pub a_prompt: &'a str, + pub n_prompt: &'a str, + pub seed: Option, } pub fn validate_job(job: &AvailableInferenceJob) -> Result { @@ -23,7 +29,7 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result {}, // Valid + Some(InferenceCategoryAbbreviated::VideoFilter) => {}, // Valid Some(category) => { return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job: {:?}", category))); }, @@ -53,33 +59,41 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result args, None => { return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!"))); } }; + let prompt = match &inference_args.maybe_prompt { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no prompt!"))); + } + }; - // let preprocess = match inference_args.maybe_preprocess { - // None => Some("full".to_string()), - // Some(Preprocess::F) => Some("full".to_string()), - // Some(Preprocess::EF) => Some("extfull".to_string()), - // Some(Preprocess::C) => Some("crop".to_string()), - // Some(Preprocess::EC) => Some("extcrop".to_string()), - // }; - // - // let enhancer = match inference_args.maybe_face_enhancer { - // None => None, - // Some(FaceEnhancer::G) => Some("gfpgan".to_string()), - // Some(FaceEnhancer::R) => Some("RestoreFormer".to_string()), - // }; - // - // let mut width = inference_args.maybe_resize_width; - // let mut height = inference_args.maybe_resize_height; + let a_prompt = match &inference_args.maybe_a_prompt { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no a prompt!"))); + } + }; + + let n_prompt = match &inference_args.maybe_n_prompt { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no n prompt!"))); + } + }; Ok(JobArgs { video_source, - sd_model, + sd_model_token, + lora_model_token: &inference_args.maybe_lora_model_token, + prompt, + a_prompt, + n_prompt, + seed: inference_args.maybe_seed, }) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs index d90caa3b8b..7cff133c25 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs @@ -20,6 +20,7 @@ use mysql_queries::payloads::generic_inference_args::videofilter_payload::{Reren use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; @@ -38,8 +39,13 @@ const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; pub struct EnqueueRerenderAnimationRequest { uuid_idempotency_token: String, + prompt: Option, // prompt for the new video + a_prompt: Option, // prompt describing the old video + n_prompt: Option, // negative prompt video_source: Option, - sd_model: String, + sd_model_token: Option, + lora_model_token: Option, + seed: Option, creator_set_visibility: Option, } @@ -158,13 +164,15 @@ pub async fn enqueue_rerender_animation_handler( // TODO(bt): CHECK DATABASE FOR TOKENS! - let video_source: VideofilterVideoSource = { - if let Some(ref token) = request.video_source { - VideofilterVideoSource::media_file_token(token.as_str()) - } else { - return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string())); - } - }; + let video_source: VideofilterVideoSource = request.video_source + .as_ref() + .map(|token| VideofilterVideoSource::media_file_token(token.as_str())) + .ok_or_else(|| EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string()))?; + + let sd_model_token: ModelWeightToken = request.sd_model_token + .as_ref() + .map(|token| ModelWeightToken::new_from_str(token.as_str())) + .ok_or_else(|| EnqueueRerenderAnimationError::BadInput("sd model not fully specified".to_string()))?; let ip_address = get_request_ip(&http_request); @@ -178,14 +186,19 @@ pub async fn enqueue_rerender_animation_handler( let inference_args = RerenderArgs { maybe_video_source: Some(video_source), - maybe_sd_model: Some(request.sd_model.clone()), + maybe_sd_model_token: Some(sd_model_token), + maybe_lora_model_token: request.lora_model_token.clone(), + maybe_prompt: request.prompt.clone(), + maybe_a_prompt: request.a_prompt.clone(), + maybe_n_prompt: request.n_prompt.clone(), + maybe_seed: request.seed.clone(), }; info!("Creating rerender animation job record..."); let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { uuid_idempotency_token: &request.uuid_idempotency_token, - inference_category: InferenceCategory::RerenderAVideo, + inference_category: InferenceCategory::VideoFilter, maybe_model_type: Some(InferenceModelType::RerenderAVideo), // NB: Model is static during inference maybe_model_token: None, // NB: Model is static during inference maybe_input_source_token: None, // TODO: Introduce a second foreign key ? @@ -193,7 +206,7 @@ pub async fn enqueue_rerender_animation_handler( maybe_raw_inference_text: None, // No text maybe_max_duration_seconds: None, maybe_inference_args: Some(GenericInferenceArgs { - inference_category: Some(InferenceCategoryAbbreviated::RerenderAVideo), + inference_category: Some(InferenceCategoryAbbreviated::VideoFilter), args: Some(PolymorphicInferenceArgs::Rr(inference_args)), }), maybe_creator_user_token: maybe_user_token.as_ref(), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 5befd6c91e..071b3795a6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -256,7 +256,7 @@ fn record_to_payload( .get_full_object_path_str() .to_string() } - InferenceCategory::RerenderAVideo => { + InferenceCategory::VideoFilter => { MediaFileBucketPath::from_object_hash( &result_details.public_bucket_location_or_hash, result_details.maybe_media_file_public_bucket_prefix.as_deref(),