From 2f971dda60e2b6705d6c3788ac12687d099a819b Mon Sep 17 00:00:00 2001 From: saltacc Date: Wed, 13 Dec 2023 21:17:44 -0800 Subject: [PATCH 1/6] add model downloading --- .../inference_category.rs | 18 ++--- .../generic_synthetic_ids/id_category.rs | 20 +++--- .../media_file_origin_product_category.rs | 20 +++--- .../generic_inference_args.rs | 10 +-- .../videofilter_payload.rs | 24 ++++++- .../web/get_inference_job_status.rs | 4 +- .../insert_media_file_from_rerender.rs | 4 +- .../src/queries/model_weights/get_weight.rs | 2 +- .../src/job/job_loop/process_single_job.rs | 6 +- .../videofilter/process_single_vf_job.rs | 61 +--------------- .../rerender_a_video/download_model_file.rs | 64 +++++++++++++++++ .../videofilter/rerender_a_video/mod.rs | 2 + .../rerender_a_video/process_job.rs | 70 +++++++++++++++---- .../rerender_a_video/validate_job.rs | 54 ++++++++------ .../animation/enqueue_rerender_animation.rs | 35 +++++++--- .../inference_job/get_inference_job_status.rs | 2 +- 16 files changed, 247 insertions(+), 149 deletions(-) create mode 100644 crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs index d1a420544b..19bf5a74c7 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_category.rs @@ -26,8 +26,8 @@ pub enum InferenceCategory { #[serde(rename = "voice_conversion")] VoiceConversion, - #[serde(rename = "rerender_a_video")] - RerenderAVideo, + #[serde(rename = "video_filter")] + VideoFilter, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -41,7 +41,7 @@ impl InferenceCategory { Self::LipsyncAnimation => "lipsync_animation", Self::TextToSpeech => "text_to_speech", Self::VoiceConversion => "voice_conversion", - Self::RerenderAVideo => "rerender_a_video", + Self::VideoFilter => "video_filter", } } @@ -50,7 +50,7 @@ impl InferenceCategory { "lipsync_animation" => Ok(Self::LipsyncAnimation), "text_to_speech" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), - "rerender_a_video" => Ok(Self::RerenderAVideo), + "video_filter" => Ok(Self::VideoFilter), _ => Err(format!("invalid value: {:?}", value)), } } @@ -62,7 +62,7 @@ impl InferenceCategory { Self::LipsyncAnimation, Self::TextToSpeech, Self::VoiceConversion, - Self::RerenderAVideo, + Self::VideoFilter, ]) } } @@ -77,7 +77,7 @@ mod tests { assert_serialization(InferenceCategory::LipsyncAnimation, "lipsync_animation"); assert_serialization(InferenceCategory::TextToSpeech, "text_to_speech"); assert_serialization(InferenceCategory::VoiceConversion, "voice_conversion"); - assert_serialization(InferenceCategory::RerenderAVideo, "rerender_a_video"); + assert_serialization(InferenceCategory::VideoFilter, "video_filter"); } #[test] @@ -85,7 +85,7 @@ mod tests { assert_eq!(InferenceCategory::LipsyncAnimation.to_str(), "lipsync_animation"); assert_eq!(InferenceCategory::TextToSpeech.to_str(), "text_to_speech"); assert_eq!(InferenceCategory::VoiceConversion.to_str(), "voice_conversion"); - assert_eq!(InferenceCategory::RerenderAVideo.to_str(), "rerender_a_video"); + assert_eq!(InferenceCategory::VideoFilter.to_str(), "video_filter"); } #[test] @@ -93,7 +93,7 @@ mod tests { assert_eq!(InferenceCategory::from_str("lipsync_animation").unwrap(), InferenceCategory::LipsyncAnimation); assert_eq!(InferenceCategory::from_str("text_to_speech").unwrap(), InferenceCategory::TextToSpeech); assert_eq!(InferenceCategory::from_str("voice_conversion").unwrap(), InferenceCategory::VoiceConversion); - assert_eq!(InferenceCategory::from_str("rerender_a_video").unwrap(), InferenceCategory::RerenderAVideo); + assert_eq!(InferenceCategory::from_str("video_filter").unwrap(), InferenceCategory::VideoFilter); } #[test] @@ -104,7 +104,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(InferenceCategory::LipsyncAnimation)); assert_eq!(variants.pop_first(), Some(InferenceCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(InferenceCategory::VoiceConversion)); - assert_eq!(variants.pop_first(), Some(InferenceCategory::RerenderAVideo)); + assert_eq!(variants.pop_first(), Some(InferenceCategory::VideoFilter)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index e5b2e90771..c751d5f5cc 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -21,9 +21,9 @@ pub enum IdCategory { #[serde(rename = "lipsync_animation")] LipsyncAnimationResult, - // Results from rerender a video - #[serde(rename = "rerender")] - RerenderResult, + // Results from video filters + #[serde(rename = "video_filter")] + VideoFilterResult, /// Results from tacotron2 /// Applies for RVC and SVC @@ -66,7 +66,7 @@ impl IdCategory { Self::ZeroShotVoiceDataset => "zs_dataset", Self::ZeroShotVoiceEmbedding => "zs_voice", Self::ZeroShotTtsResult => "zs_tts_result", - Self::RerenderResult => "rerender", + Self::VideoFilterResult => "video_filter", Self::ModelWeights => "model_weights", } } @@ -80,7 +80,7 @@ impl IdCategory { "zs_dataset" => Ok(Self::ZeroShotVoiceDataset), "zs_voice" => Ok(Self::ZeroShotVoiceEmbedding), "zs_tts_result" => Ok(Self::ZeroShotTtsResult), - "rerender" => Ok(Self::RerenderResult), + "video_filter" => Ok(Self::VideoFilterResult), "model_weights" => Ok(Self::ModelWeights), _ => Err(format!("invalid value: {:?}", value)), } @@ -92,7 +92,7 @@ impl IdCategory { BTreeSet::from([ Self::MediaFile, Self::LipsyncAnimationResult, - Self::RerenderResult, + Self::VideoFilterResult, Self::TtsResult, Self::VoiceConversionResult, Self::ZeroShotTtsResult, @@ -115,7 +115,7 @@ mod tests { fn test_serialization() { assert_serialization(IdCategory::MediaFile, "media_file"); assert_serialization(IdCategory::LipsyncAnimationResult, "lipsync_animation"); - assert_serialization(IdCategory::RerenderResult, "rerender"); + assert_serialization(IdCategory::VideoFilterResult, "video_filter"); assert_serialization(IdCategory::TtsResult, "tts_result"); assert_serialization(IdCategory::VoiceConversionResult, "voice_conversion"); assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset"); @@ -128,7 +128,7 @@ mod tests { fn to_str() { assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); assert_eq!(IdCategory::LipsyncAnimationResult.to_str(), "lipsync_animation"); - assert_eq!(IdCategory::RerenderResult.to_str(), "rerender"); + assert_eq!(IdCategory::VideoFilterResult.to_str(), "video_filter"); assert_eq!(IdCategory::TtsResult.to_str(), "tts_result"); assert_eq!(IdCategory::VoiceConversionResult.to_str(), "voice_conversion"); assert_eq!(IdCategory::ZeroShotVoiceDataset.to_str(), "zs_dataset"); @@ -141,7 +141,7 @@ mod tests { fn from_str() { assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimationResult); - assert_eq!(IdCategory::from_str("rerender").unwrap(), IdCategory::RerenderResult); + assert_eq!(IdCategory::from_str("video_filter").unwrap(), IdCategory::VideoFilterResult); assert_eq!(IdCategory::from_str("tts_result").unwrap(), IdCategory::TtsResult); assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversionResult); assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZeroShotVoiceDataset); @@ -157,7 +157,7 @@ mod tests { assert_eq!(variants.len(), 9); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); - assert_eq!(variants.pop_first(), Some(IdCategory::RerenderResult)); + assert_eq!(variants.pop_first(), Some(IdCategory::VideoFilterResult)); assert_eq!(variants.pop_first(), Some(IdCategory::TtsResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversionResult)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotTtsResult)); diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 103bab0042..e74da29ecf 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -39,9 +39,9 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "zs_voice")] ZeroShotVoice, - // Media files for Rerender - #[serde(rename = "rerender")] - Rerender, + // Media files for video filters + #[serde(rename = "video_filter")] + VideoFilter, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -58,7 +58,7 @@ impl MediaFileOriginProductCategory { Self::Unknown => "unknown", Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", - Self::Rerender => "rerender", + Self::VideoFilter => "video_filter", } } @@ -69,7 +69,7 @@ impl MediaFileOriginProductCategory { "unknown" => Ok(Self::Unknown), "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), - "rerender" => Ok(Self::Rerender), + "video_filter" => Ok(Self::VideoFilter), _ => Err(format!("invalid value: {:?}", value)), } } @@ -80,7 +80,7 @@ impl MediaFileOriginProductCategory { BTreeSet::from([ Self::FaceAnimator, Self::TextToSpeech, - Self::Rerender, + Self::VideoFilter, Self::Unknown, Self::VoiceConversion, Self::ZeroShotVoice, @@ -103,7 +103,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); - assert_serialization(MediaFileOriginProductCategory::Rerender, "rerender") + assert_serialization(MediaFileOriginProductCategory::VideoFilter, "video_filter") } #[test] @@ -113,7 +113,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); - assert_eq!(MediaFileOriginProductCategory::Rerender.to_str(), "rerender"); + assert_eq!(MediaFileOriginProductCategory::VideoFilter.to_str(), "video_filter"); } #[test] @@ -123,7 +123,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); - assert_eq!(MediaFileOriginProductCategory::from_str("rerender").unwrap(), MediaFileOriginProductCategory::Rerender); + assert_eq!(MediaFileOriginProductCategory::from_str("video_filter").unwrap(), MediaFileOriginProductCategory::VideoFilter); } #[test] @@ -135,7 +135,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); - assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Rerender)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoFilter)); assert_eq!(variants.pop_first(), None); } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index f34a4469c3..a67dd0bd91 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -37,9 +37,9 @@ pub enum InferenceCategoryAbbreviated { #[serde(alias = "voice_conversion")] VoiceConversion, - #[serde(rename = "rr")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. - #[serde(alias = "rerender_a_video")] - RerenderAVideo, + #[serde(rename = "vf")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(alias = "video_filter")] + VideoFilter, } #[derive(Serialize, Deserialize, Clone, Copy, Debug)] @@ -112,7 +112,7 @@ impl InferenceCategoryAbbreviated { InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation, InferenceCategory::TextToSpeech => Self::TextToSpeech, InferenceCategory::VoiceConversion => Self::VoiceConversion, - InferenceCategory::RerenderAVideo => Self::RerenderAVideo, + InferenceCategory::VideoFilter => Self::VideoFilter, } } @@ -121,7 +121,7 @@ impl InferenceCategoryAbbreviated { Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation, Self::TextToSpeech => InferenceCategory::TextToSpeech, Self::VoiceConversion => InferenceCategory::VoiceConversion, - Self::RerenderAVideo => InferenceCategory::RerenderAVideo, + Self::VideoFilter => InferenceCategory::VideoFilter, } } } diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs index 12d316933f..1fbaeaab0a 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/videofilter_payload.rs @@ -1,3 +1,5 @@ +use tokens::tokens::model_weights::ModelWeightToken; + /// Video sources can be one of several: /// - F: media_files (todo) /// - U: media_uploads (legacy) @@ -32,5 +34,25 @@ pub struct RerenderArgs { #[serde(rename = "sd")] #[serde(skip_serializing_if = "Option::is_none")] - pub maybe_sd_model: Option, + pub maybe_sd_model_token: Option, + + #[serde(rename = "lora")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_lora_model_token: Option, + + #[serde(rename = "p")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_prompt: Option, + + #[serde(rename = "ap")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_a_prompt: Option, + + #[serde(rename = "np")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_n_prompt: Option, + + #[serde(rename = "se")] + #[serde(skip_serializing_if = "Option::is_none")] + pub maybe_seed: Option, } diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 489ddd8391..e2a9b7be9c 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -178,7 +178,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => Some("lipsync animation"), InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(), InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(), - InferenceCategory::RerenderAVideo => Some("rerender animation"), + InferenceCategory::VideoFilter => Some("Video Filter"), }; // NB: A bit of a hack. We store TTS results with a full path. @@ -187,7 +187,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), - InferenceCategory::RerenderAVideo => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), + InferenceCategory::VideoFilter => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), }; // NB: We've moved voice conversion out of their own table and into media_files diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs index c599a4fad0..b1adc3d6c3 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_rerender.rs @@ -55,7 +55,7 @@ pub async fn insert_media_file_from_rerender( let next_rerender_id = transactional_increment_generic_synthetic_id( &user_token, - IdCategory::RerenderResult, + IdCategory::VideoFilterResult, &mut transaction ).await?; @@ -67,7 +67,7 @@ pub async fn insert_media_file_from_rerender( let creator_set_visibility = args.job.creator_set_visibility.clone(); const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; - const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Rerender; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::VideoFilter; const ORIGIN_MODEL_TYPE : MediaFileOriginModelType = MediaFileOriginModelType::Rerender; const MEDIA_TYPE : MediaFileType = MediaFileType::Video; diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs index 050f4de251..01a128f9a7 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get_weight.rs @@ -33,7 +33,7 @@ pub struct RetrivedModelWeight { pub file_size_bytes: i32, pub file_checksum_sha2: String, pub private_bucket_hash: String, - pub maybe_private_bucket_prefix: Option, + pub maybe_private_bucket_prefix: Option, // TODO crossproduct rename these to public thx ^-^ pub maybe_private_bucket_extension: Option, pub cached_user_ratings_negative_count: u32, pub cached_user_ratings_positive_count: u32, diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index c7edcdbb7f..1aab84a477 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -16,7 +16,7 @@ use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSucce use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job; use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job; use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job; -use crate::job::job_types::videofilter::process_single_vf_job::process_single_rr_job; +use crate::job::job_types::videofilter::process_single_vf_job::process_single_vf_job; use crate::job_dependencies::JobDependencies; pub async fn process_single_job( @@ -183,8 +183,8 @@ async fn do_process_single_job( InferenceCategory::VoiceConversion => { process_single_vc_job(job_dependencies, job).await? } - InferenceCategory::RerenderAVideo => { - process_single_rr_job(job_dependencies, job).await? + InferenceCategory::VideoFilter => { + process_single_vf_job(job_dependencies, job).await? } }; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs index 9d7cc0163c..076a723ef9 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/process_single_vf_job.rs @@ -1,11 +1,4 @@ -use anyhow::anyhow; -use log::info; - -use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; -use mysql_queries::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::media_files::get_media_file::get_media_file; -use tokens::tokens::media_files::MediaFileToken; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -13,59 +6,7 @@ use crate::job::job_types::videofilter::rerender_a_video; use crate::job::job_types::videofilter::rerender_a_video::process_job::RerenderProcessJobArgs; use crate::job_dependencies::JobDependencies; -pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { - // let maybe_inference_args = job.maybe_inference_args.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Inference args not found")))?; - // - // let maybe_rerender_args = maybe_inference_args.args.as_ref().map(|args| match args { - // Rr(args) => Some(args), - // _ => None, - // }).flatten(); - // - // let rerender_args = match maybe_rerender_args { - // None => return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), - // Some(args) => args, - // }; - // - // let videofilter_source = rerender_args.maybe_video_source.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("Video source not found")))?; - // - // let media_file_token = match videofilter_source { - // VideofilterVideoSource::F(token) => token, - // _ => return Err(ProcessSingleJobError::Other(anyhow!("Video source not found"))), - // }; - // - // let media_file_token = MediaFileToken::new_from_str(media_file_token); - // - // info!("media_file_token: {:?}", media_file_token); - // - // - // let media_file = match media_file_result { - // Ok(Some(result)) => result, - // Ok(None) => { - // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); - // } - // Err(e) => { - // return Err(ProcessSingleJobError::Other(anyhow!("Media file not found"))); - // } - // }; - - // info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes); - // info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis); - // info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0)); - - - - // let job_success_result = match vc_model.model_type { - // VoiceConversionModelType::RvcV2 => { - // rvc_v2::process_job::process_job(RvcV2ProcessJobArgs { - // job_dependencies, - // job, - // vc_model: &vc_model, - // media_upload_token: &media_upload_token, - // media_upload: &media_upload, - // }).await? - // } - // }; - +pub async fn process_single_vf_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result { let job_success_result = rerender_a_video::process_job::process_job( RerenderProcessJobArgs { job_dependencies, diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs new file mode 100644 index 0000000000..42e4298f77 --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs @@ -0,0 +1,64 @@ +use std::path::PathBuf; + +use anyhow::anyhow; +use tempdir::TempDir; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use cloud_storage::bucket_client::BucketClient; +use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter; +use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::model_weights::get_weight::RetrivedModelWeight; + +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; +use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; + +pub struct ModelFile { + pub filesystem_path: PathBuf, +} + +pub async fn download_model_file( + model_record: &Option, + public_bucket_client: &BucketClient, + job_progress_reporter: &mut Box, + job: &AvailableInferenceJob, + temp_dir_creator: &ScopedTempDirCreator, + work_temp_dir: &TempDir, +) -> Result { + + let bucket_object_path; + + match model_record { + Some(model_record) => { + let model_bucket_path = + MediaFileBucketPath::from_object_hash( + &model_record.private_bucket_hash, + model_record.maybe_private_bucket_prefix.as_deref(), + model_record.maybe_private_bucket_extension.as_deref(), + ); + bucket_object_path = model_bucket_path.to_full_object_pathbuf(); + } + None => { + return Err(ProcessSingleJobError::from_anyhow_error( + anyhow!("could not find model file"))) + } + } + + let downloaded_filesystem_path = work_temp_dir.path().join("video.mp4"); + + maybe_download_file_from_bucket(MaybeDownloadArgs { + name_or_description_of_file: "video file", + final_filesystem_file_path: &downloaded_filesystem_path, + bucket_object_path: &bucket_object_path, + bucket_client: public_bucket_client, + job_progress_reporter, + job_progress_update_description: "downloading", + job_id: job.id.0, + scoped_tempdir_creator: &temp_dir_creator, + maybe_existing_file_minimum_size_required: None, + }).await?; + + Ok(ModelFile { + filesystem_path: downloaded_filesystem_path, + }) +} diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 28c9dea48a..6b30de9473 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -4,3 +4,5 @@ mod download_video_file; mod rerender_inference_command; pub(crate) mod rerender_dependencies; mod categorize_error; +mod download_model_file; +mod download_model_file; diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs index 705671b1c1..3cb8598978 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs @@ -16,13 +16,16 @@ use mimetypes::mimetype_for_file::get_mimetype_for_file; use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Rr; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::media_files::insert_media_file_from_rerender::{insert_media_file_from_rerender, InsertArgs}; +use mysql_queries::queries::model_weights::get_weight::get_weight_by_token; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; +use crate::job::job_types::videofilter::rerender_a_video::download_model_file::download_model_file; use crate::job::job_types::videofilter::rerender_a_video::download_video_file::download_video_file; use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; use crate::job_dependencies::JobDependencies; +use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; /// The maximum that either width or height can be const MAX_DIMENSION : u32 = 1500; @@ -122,22 +125,69 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result return Err(ProcessSingleJobError::Other(anyhow!("Rerender args not found"))), }; - let sd_model = rr_args.maybe_sd_model.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?; + + // =================== LOAD MODELS AND LORAS ==================== // + + let sd_model_token = rr_args.maybe_sd_model_token.as_ref().ok_or(ProcessSingleJobError::Other(anyhow!("SD model not found")))?; + + let retrieved_sd_record = get_weight_by_token( + sd_model_token, + false, + &deps.db.mysql_pool + ).await?; + + let model_path = download_model_file( + &retrieved_sd_record, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir).await?; + + info!("Downloaded model file: {:?}", model_path.filesystem_path); + let sd_model = model_path.filesystem_path.to_str().unwrap().to_string(); + + let lora_path = match rr_args.maybe_lora_model_token.clone() { + Some(lora_model_token) => { + let retrieved_lora_record = get_weight_by_token( + &lora_model_token, + false, + &deps.db.mysql_pool + ).await?; + + let lora_path = download_model_file( + &retrieved_lora_record, + &args.job_dependencies.buckets.public_bucket_client, + &mut job_progress_reporter, + job, + &args.job_dependencies.fs.scoped_temp_dir_creator_for_work, + &work_temp_dir).await?; + + info!("Downloaded lora file: {:?}", lora_path.filesystem_path); + + Some(lora_path.filesystem_path) + } + None => None + }; + let sd_lora = match lora_path { + Some(lora_path) => Some(lora_path.to_str().unwrap().to_string()), + None => None + }; // ============== CREATE AND WRITE CONFIG TO DISK ============== // // create seed if not exists - // let seed = rr_args.seed.unwrap_or_else(|| rand::random::()); - let seed = rand::random::(); + let seed = rr_args.maybe_seed.unwrap_or_else(|| rand::random::()); let config = RerenderConfig { input: Some(usable_video_path.to_str().unwrap().to_string()), output: Some(output_video_fs_path.to_str().unwrap().to_string()), work_dir: Some(work_temp_dir.path().to_str().unwrap().to_string()), sd_model: Some(sd_model.to_string()), - prompt: Some("1girl, anime style, red hair".to_string()), - a_prompt: Some("1girl, 3d, realistic, read hair".to_string()), - n_prompt: Some("Bad quality, low quality, photo, artifacts, watermark, signature, faceless".to_string()), + lora_path: sd_lora, + prompt: rr_args.maybe_prompt.clone(), + a_prompt: rr_args.maybe_a_prompt.clone(), + n_prompt: rr_args.maybe_n_prompt.clone(), seed: Some(seed as i64), ..RerenderConfig::new() }; @@ -170,14 +220,6 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result { - // warn!("Face not detected in source image"); - // error = ProcessSingleJobError::FaceDetectionFailure; - // } - // _ => {} - // } } safe_delete_temp_file(&video_path.filesystem_path); diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs index dd9bc75c2f..6a1856899c 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/validate_job.rs @@ -3,12 +3,18 @@ use anyhow::anyhow; use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use mysql_queries::payloads::generic_inference_args::videofilter_payload::{VideofilterVideoSource}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use tokens::tokens::model_weights::ModelWeightToken; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; pub struct JobArgs<'a> { pub video_source: &'a VideofilterVideoSource, - pub sd_model: &'a str, + pub sd_model_token: &'a ModelWeightToken, + pub lora_model_token: &'a Option, + pub prompt: &'a str, + pub a_prompt: &'a str, + pub n_prompt: &'a str, + pub seed: Option, } pub fn validate_job(job: &AvailableInferenceJob) -> Result { @@ -23,7 +29,7 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result {}, // Valid + Some(InferenceCategoryAbbreviated::VideoFilter) => {}, // Valid Some(category) => { return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inference category for job: {:?}", category))); }, @@ -53,33 +59,41 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result args, None => { return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!"))); } }; + let prompt = match &inference_args.maybe_prompt { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no prompt!"))); + } + }; - // let preprocess = match inference_args.maybe_preprocess { - // None => Some("full".to_string()), - // Some(Preprocess::F) => Some("full".to_string()), - // Some(Preprocess::EF) => Some("extfull".to_string()), - // Some(Preprocess::C) => Some("crop".to_string()), - // Some(Preprocess::EC) => Some("extcrop".to_string()), - // }; - // - // let enhancer = match inference_args.maybe_face_enhancer { - // None => None, - // Some(FaceEnhancer::G) => Some("gfpgan".to_string()), - // Some(FaceEnhancer::R) => Some("RestoreFormer".to_string()), - // }; - // - // let mut width = inference_args.maybe_resize_width; - // let mut height = inference_args.maybe_resize_height; + let a_prompt = match &inference_args.maybe_a_prompt { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no a prompt!"))); + } + }; + + let n_prompt = match &inference_args.maybe_n_prompt { + Some(args) => args, + None => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no n prompt!"))); + } + }; Ok(JobArgs { video_source, - sd_model, + sd_model_token, + lora_model_token: &inference_args.maybe_lora_model_token, + prompt, + a_prompt, + n_prompt, + seed: inference_args.maybe_seed, }) } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs index d90caa3b8b..7cff133c25 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/animation/enqueue_rerender_animation.rs @@ -20,6 +20,7 @@ use mysql_queries::payloads::generic_inference_args::videofilter_payload::{Reren use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; use tokens::tokens::users::UserToken; use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; @@ -38,8 +39,13 @@ const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; pub struct EnqueueRerenderAnimationRequest { uuid_idempotency_token: String, + prompt: Option, // prompt for the new video + a_prompt: Option, // prompt describing the old video + n_prompt: Option, // negative prompt video_source: Option, - sd_model: String, + sd_model_token: Option, + lora_model_token: Option, + seed: Option, creator_set_visibility: Option, } @@ -158,13 +164,15 @@ pub async fn enqueue_rerender_animation_handler( // TODO(bt): CHECK DATABASE FOR TOKENS! - let video_source: VideofilterVideoSource = { - if let Some(ref token) = request.video_source { - VideofilterVideoSource::media_file_token(token.as_str()) - } else { - return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string())); - } - }; + let video_source: VideofilterVideoSource = request.video_source + .as_ref() + .map(|token| VideofilterVideoSource::media_file_token(token.as_str())) + .ok_or_else(|| EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string()))?; + + let sd_model_token: ModelWeightToken = request.sd_model_token + .as_ref() + .map(|token| ModelWeightToken::new_from_str(token.as_str())) + .ok_or_else(|| EnqueueRerenderAnimationError::BadInput("sd model not fully specified".to_string()))?; let ip_address = get_request_ip(&http_request); @@ -178,14 +186,19 @@ pub async fn enqueue_rerender_animation_handler( let inference_args = RerenderArgs { maybe_video_source: Some(video_source), - maybe_sd_model: Some(request.sd_model.clone()), + maybe_sd_model_token: Some(sd_model_token), + maybe_lora_model_token: request.lora_model_token.clone(), + maybe_prompt: request.prompt.clone(), + maybe_a_prompt: request.a_prompt.clone(), + maybe_n_prompt: request.n_prompt.clone(), + maybe_seed: request.seed.clone(), }; info!("Creating rerender animation job record..."); let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { uuid_idempotency_token: &request.uuid_idempotency_token, - inference_category: InferenceCategory::RerenderAVideo, + inference_category: InferenceCategory::VideoFilter, maybe_model_type: Some(InferenceModelType::RerenderAVideo), // NB: Model is static during inference maybe_model_token: None, // NB: Model is static during inference maybe_input_source_token: None, // TODO: Introduce a second foreign key ? @@ -193,7 +206,7 @@ pub async fn enqueue_rerender_animation_handler( maybe_raw_inference_text: None, // No text maybe_max_duration_seconds: None, maybe_inference_args: Some(GenericInferenceArgs { - inference_category: Some(InferenceCategoryAbbreviated::RerenderAVideo), + inference_category: Some(InferenceCategoryAbbreviated::VideoFilter), args: Some(PolymorphicInferenceArgs::Rr(inference_args)), }), maybe_creator_user_token: maybe_user_token.as_ref(), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 5befd6c91e..071b3795a6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -256,7 +256,7 @@ fn record_to_payload( .get_full_object_path_str() .to_string() } - InferenceCategory::RerenderAVideo => { + InferenceCategory::VideoFilter => { MediaFileBucketPath::from_object_hash( &result_details.public_bucket_location_or_hash, result_details.maybe_media_file_public_bucket_prefix.as_deref(), From c137e1ace39bdb3366a4f9af64dbd8b31545be38 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 15 Dec 2023 19:28:25 -0500 Subject: [PATCH 2/6] migration tool binary bootstrap --- .gitignore | 4 +++ Cargo.lock | 23 ++++++++++++++ Cargo.toml | 1 + crates/cli/migration-tool/Cargo.toml | 44 +++++++++++++++++++++++++++ crates/cli/migration-tool/src/main.rs | 23 ++++++++++++++ 5 files changed, 95 insertions(+) create mode 100644 crates/cli/migration-tool/Cargo.toml create mode 100644 crates/cli/migration-tool/src/main.rs diff --git a/.gitignore b/.gitignore index d9d031aab3..efa47b920d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .env .env-aichatbot-secrets .env-elasticsearch-secrets +.env-migration-tool-secrets .env-secrets _develop/tools/.secrets _develop/tools/db_secrets.toml @@ -17,6 +18,9 @@ secrets.toml secrets.yaml twitch_secrets.toml +# Old env secrets +.*-secrets + # New env secrets (per-app) *-secrets.env diff --git a/Cargo.lock b/Cargo.lock index 7af174d2be..a6a2c3ec3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3480,6 +3480,29 @@ dependencies = [ "my-workspace-hack", ] +[[package]] +name = "migration-tool" +version = "0.0.1" +dependencies = [ + "buckets", + "clap", + "config", + "easyenv", + "enums", + "errors", + "log", + "my-workspace-hack", + "mysql_queries", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "storyteller_root", + "strum", + "tokens", + "tokio", +] + [[package]] name = "mime" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 411f131e53..e0a47f555a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/cli/elasticsearch-cli", "crates/cli/kubectl-pod-cleanup", "crates/cli/lru-delete-models", + "crates/cli/migration-tool", "crates/cli/scrape_test", # Shared libraries diff --git a/crates/cli/migration-tool/Cargo.toml b/crates/cli/migration-tool/Cargo.toml new file mode 100644 index 0000000000..2c7d4b065c --- /dev/null +++ b/crates/cli/migration-tool/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "migration-tool" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "migration-tool" +path = "src/main.rs" + +[dependencies] +# Internal +buckets = { path = "../../_types/buckets" } +config = { path = "../../lib/deprecated/config" } +easyenv = { path = "../../lib/easyenv" } +enums = { path = "../../_types/enums" } +errors = { path = "../../lib/errors" } +mysql_queries = { path = "../../lib/mysql_queries" } +storyteller_root = { path = "../../lib/storyteller_root" } +tokens = { path = "../../_types/tokens" } + +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } + +# External +clap = "4.4.8" +log = "0.4.14" +strum = { version = "0.25", features = ["derive"] } + +# Async + clients +tokio = { version = "1.34.0", features = ["macros"] } +sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.108" + +[dev-dependencies] +# None yet diff --git a/crates/cli/migration-tool/src/main.rs b/crates/cli/migration-tool/src/main.rs new file mode 100644 index 0000000000..dd6af54df8 --- /dev/null +++ b/crates/cli/migration-tool/src/main.rs @@ -0,0 +1,23 @@ +//! migration-tool +//! +//! Migrate database records. +//! + + +use config::shared_constants::DEFAULT_RUST_LOG; +use errors::AnyhowResult; + +#[tokio::main] +pub async fn main() -> AnyhowResult<()> { + println!("migration-tool: migrate database records"); + + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: This secrets file differs from the rest because we might actually want to cross + // development/production boundaries for migration. We don't want to pull in secrets + // from other sources. (Hopefully this isn't getting out of hand at this point.) + easyenv::from_filename(".env-migration-tool-secrets")?; + + + Ok(()) +} From 732dd27d0f853512161e8150aeb980e527475f49 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 15 Dec 2023 19:39:06 -0500 Subject: [PATCH 3/6] fix build errors; kebab-case all CLI apps --- Cargo.toml | 8 ++++---- .../{database_backup => database-backup}/Cargo.toml | 0 .../{database_backup => database-backup}/src/main.rs | 0 .../{database_cleanup => database-cleanup}/Cargo.toml | 0 .../{database_cleanup => database-cleanup}/src/main.rs | 0 .../Cargo.toml | 0 .../src/bucket_clients.rs | 0 .../src/cli_args.rs | 0 .../src/main.rs | 0 .../src/seeding/mod.rs | 0 .../src/seeding/model_weights.rs | 0 .../src/seeding/pythonex.py | 0 .../src/seeding/seed_media_files.rs | 0 .../src/seeding/seed_model_weights.rs | 10 ++++------ .../src/seeding/tts_tacotron2.rs | 0 .../src/seeding/users.rs | 0 .../src/seeding/voice_conversion.rs | 0 .../src/seeding/zero_shot_tts.rs | 0 .../Cargo.toml | 0 .../src/cli_args.rs | 0 .../src/main.rs | 0 21 files changed, 8 insertions(+), 10 deletions(-) rename crates/cli/{database_backup => database-backup}/Cargo.toml (100%) rename crates/cli/{database_backup => database-backup}/src/main.rs (100%) rename crates/cli/{database_cleanup => database-cleanup}/Cargo.toml (100%) rename crates/cli/{database_cleanup => database-cleanup}/src/main.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/Cargo.toml (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/bucket_clients.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/cli_args.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/main.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/mod.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/model_weights.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/pythonex.py (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/seed_media_files.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/seed_model_weights.rs (96%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/tts_tacotron2.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/users.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/voice_conversion.rs (100%) rename crates/cli/{dev_database_seed => dev-database-seed}/src/seeding/zero_shot_tts.rs (100%) rename crates/cli/{dev_upload_media_file => dev-upload-media-file}/Cargo.toml (100%) rename crates/cli/{dev_upload_media_file => dev-upload-media-file}/src/cli_args.rs (100%) rename crates/cli/{dev_upload_media_file => dev-upload-media-file}/src/main.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index e0a47f555a..78728f6407 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,10 +10,10 @@ members = [ "crates/_types/tokens", # CLI tools for operations, etc. - "crates/cli/database_backup", - "crates/cli/database_cleanup", - "crates/cli/dev_database_seed", - "crates/cli/dev_upload_media_file", + "crates/cli/database-backup", + "crates/cli/database-cleanup", + "crates/cli/dev-database-seed", + "crates/cli/dev-upload-media-file", "crates/cli/elasticsearch-cli", "crates/cli/kubectl-pod-cleanup", "crates/cli/lru-delete-models", diff --git a/crates/cli/database_backup/Cargo.toml b/crates/cli/database-backup/Cargo.toml similarity index 100% rename from crates/cli/database_backup/Cargo.toml rename to crates/cli/database-backup/Cargo.toml diff --git a/crates/cli/database_backup/src/main.rs b/crates/cli/database-backup/src/main.rs similarity index 100% rename from crates/cli/database_backup/src/main.rs rename to crates/cli/database-backup/src/main.rs diff --git a/crates/cli/database_cleanup/Cargo.toml b/crates/cli/database-cleanup/Cargo.toml similarity index 100% rename from crates/cli/database_cleanup/Cargo.toml rename to crates/cli/database-cleanup/Cargo.toml diff --git a/crates/cli/database_cleanup/src/main.rs b/crates/cli/database-cleanup/src/main.rs similarity index 100% rename from crates/cli/database_cleanup/src/main.rs rename to crates/cli/database-cleanup/src/main.rs diff --git a/crates/cli/dev_database_seed/Cargo.toml b/crates/cli/dev-database-seed/Cargo.toml similarity index 100% rename from crates/cli/dev_database_seed/Cargo.toml rename to crates/cli/dev-database-seed/Cargo.toml diff --git a/crates/cli/dev_database_seed/src/bucket_clients.rs b/crates/cli/dev-database-seed/src/bucket_clients.rs similarity index 100% rename from crates/cli/dev_database_seed/src/bucket_clients.rs rename to crates/cli/dev-database-seed/src/bucket_clients.rs diff --git a/crates/cli/dev_database_seed/src/cli_args.rs b/crates/cli/dev-database-seed/src/cli_args.rs similarity index 100% rename from crates/cli/dev_database_seed/src/cli_args.rs rename to crates/cli/dev-database-seed/src/cli_args.rs diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev-database-seed/src/main.rs similarity index 100% rename from crates/cli/dev_database_seed/src/main.rs rename to crates/cli/dev-database-seed/src/main.rs diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev-database-seed/src/seeding/mod.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/mod.rs rename to crates/cli/dev-database-seed/src/seeding/mod.rs diff --git a/crates/cli/dev_database_seed/src/seeding/model_weights.rs b/crates/cli/dev-database-seed/src/seeding/model_weights.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/model_weights.rs rename to crates/cli/dev-database-seed/src/seeding/model_weights.rs diff --git a/crates/cli/dev_database_seed/src/seeding/pythonex.py b/crates/cli/dev-database-seed/src/seeding/pythonex.py similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/pythonex.py rename to crates/cli/dev-database-seed/src/seeding/pythonex.py diff --git a/crates/cli/dev_database_seed/src/seeding/seed_media_files.rs b/crates/cli/dev-database-seed/src/seeding/seed_media_files.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/seed_media_files.rs rename to crates/cli/dev-database-seed/src/seeding/seed_media_files.rs diff --git a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs b/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs similarity index 96% rename from crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs rename to crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs index 66f222c408..5af2266431 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs @@ -1,12 +1,10 @@ use std::path::Path; -use log::{ info, warn }; -use sqlx::{ MySql, Pool }; +use log::{info, warn}; +use rand::Rng; +use sqlx::{MySql, Pool}; -use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self, weight_files::bucket_file_path::WeightFileBucketPath }, private }; - - -use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath; use enums::{ by_table::model_weights::{weights_category::WeightsCategory, weights_types::WeightsType}, common::visibility::Visibility, diff --git a/crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs b/crates/cli/dev-database-seed/src/seeding/tts_tacotron2.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs rename to crates/cli/dev-database-seed/src/seeding/tts_tacotron2.rs diff --git a/crates/cli/dev_database_seed/src/seeding/users.rs b/crates/cli/dev-database-seed/src/seeding/users.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/users.rs rename to crates/cli/dev-database-seed/src/seeding/users.rs diff --git a/crates/cli/dev_database_seed/src/seeding/voice_conversion.rs b/crates/cli/dev-database-seed/src/seeding/voice_conversion.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/voice_conversion.rs rename to crates/cli/dev-database-seed/src/seeding/voice_conversion.rs diff --git a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs b/crates/cli/dev-database-seed/src/seeding/zero_shot_tts.rs similarity index 100% rename from crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs rename to crates/cli/dev-database-seed/src/seeding/zero_shot_tts.rs diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev-upload-media-file/Cargo.toml similarity index 100% rename from crates/cli/dev_upload_media_file/Cargo.toml rename to crates/cli/dev-upload-media-file/Cargo.toml diff --git a/crates/cli/dev_upload_media_file/src/cli_args.rs b/crates/cli/dev-upload-media-file/src/cli_args.rs similarity index 100% rename from crates/cli/dev_upload_media_file/src/cli_args.rs rename to crates/cli/dev-upload-media-file/src/cli_args.rs diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev-upload-media-file/src/main.rs similarity index 100% rename from crates/cli/dev_upload_media_file/src/main.rs rename to crates/cli/dev-upload-media-file/src/main.rs From 05789802056c79ed06988f499d301e239b15379a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 16 Dec 2023 23:26:16 -0500 Subject: [PATCH 4/6] re-home nessu's models (per vegito) --- _sql/actions/tts_models.sql | 6 ++++++ _sql/actions/voice_conversion_models.sql | 3 +++ .../cli/dev-database-seed/src/seeding/seed_model_weights.rs | 3 +-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 _sql/actions/tts_models.sql diff --git a/_sql/actions/tts_models.sql b/_sql/actions/tts_models.sql new file mode 100644 index 0000000000..eadec00f93 --- /dev/null +++ b/_sql/actions/tts_models.sql @@ -0,0 +1,6 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Re-home Nessu's models +update tts_models set creator_user_token = "user_7z1h6sykg4r62" where creator_user_token = "U:C00NRSW1AE8T4" limit 5; diff --git a/_sql/actions/voice_conversion_models.sql b/_sql/actions/voice_conversion_models.sql index 7f97f2da68..ddff34378a 100644 --- a/_sql/actions/voice_conversion_models.sql +++ b/_sql/actions/voice_conversion_models.sql @@ -13,6 +13,9 @@ where token IN ( 'vcm_502wkt2pca3y' ); +-- Re-home Nessu's models +update voice_conversion_models set creator_user_token = "user_7z1h6sykg4r62" where creator_user_token = "U:C00NRSW1AE8T4" limit 5; + -- Remove broken models update voice_conversion_models set diff --git a/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs b/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs index 5af2266431..566624561a 100644 --- a/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs @@ -17,8 +17,7 @@ use hashing::sha256::sha256_hash_file::sha256_hash_file; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::model_weights::create_weight::{ create_weight, - CreateModelWeightsArgs - , + CreateModelWeightsArgs, }; use storyteller_root::get_seed_tool_data_root; use tokens::tokens::model_weights::ModelWeightToken; From 25b90c07ce4ed4382d4f8d1f750e3b65b692b6dc Mon Sep 17 00:00:00 2001 From: saltacc Date: Mon, 18 Dec 2023 15:49:55 -0800 Subject: [PATCH 5/6] add model downloading --- .../rerender_a_video/download_model_file.rs | 10 +++++----- .../job/job_types/videofilter/rerender_a_video/mod.rs | 3 +-- .../videofilter/rerender_a_video/process_job.rs | 3 +-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs index 42e4298f77..ee848d3361 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/download_model_file.rs @@ -27,7 +27,7 @@ pub async fn download_model_file( ) -> Result { let bucket_object_path; - + let model_filename; match model_record { Some(model_record) => { let model_bucket_path = @@ -36,18 +36,18 @@ pub async fn download_model_file( model_record.maybe_private_bucket_prefix.as_deref(), model_record.maybe_private_bucket_extension.as_deref(), ); + model_filename = model_record.private_bucket_hash.clone() + ".".to_owned() + model_record.maybe_private_bucket_extension.as_deref().unwrap_or("bin"); bucket_object_path = model_bucket_path.to_full_object_pathbuf(); } None => { return Err(ProcessSingleJobError::from_anyhow_error( anyhow!("could not find model file"))) } - } - - let downloaded_filesystem_path = work_temp_dir.path().join("video.mp4"); + }; + let downloaded_filesystem_path = work_temp_dir.path().join(model_filename); maybe_download_file_from_bucket(MaybeDownloadArgs { - name_or_description_of_file: "video file", + name_or_description_of_file: "model file", final_filesystem_file_path: &downloaded_filesystem_path, bucket_object_path: &bucket_object_path, bucket_client: public_bucket_client, diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs index 6b30de9473..dc00dc3412 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/mod.rs @@ -4,5 +4,4 @@ mod download_video_file; mod rerender_inference_command; pub(crate) mod rerender_dependencies; mod categorize_error; -mod download_model_file; -mod download_model_file; +mod download_model_file; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs index 3cb8598978..cce8a0b8b5 100644 --- a/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/process_job.rs @@ -25,7 +25,6 @@ use crate::job::job_types::videofilter::rerender_a_video::download_video_file::d use crate::job::job_types::videofilter::rerender_a_video::rerender_inference_command::InferenceArgs; use crate::job::job_types::videofilter::rerender_a_video::validate_job::validate_job; use crate::job_dependencies::JobDependencies; -use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs}; /// The maximum that either width or height can be const MAX_DIMENSION : u32 = 1500; @@ -276,7 +275,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result Date: Mon, 18 Dec 2023 21:08:43 -0500 Subject: [PATCH 6/6] Seed Model weights fix --- .../src/seeding/seed_model_weights.rs | 65 ++++++++----------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs b/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs index 566624561a..e479f275da 100644 --- a/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev-database-seed/src/seeding/seed_model_weights.rs @@ -1,5 +1,5 @@ -use std::path::Path; - +use std::{path::Path, string}; +use errors::{anyhow, AnyhowResult}; use log::{info, warn}; use rand::Rng; use sqlx::{MySql, Pool}; @@ -9,7 +9,6 @@ use enums::{ by_table::model_weights::{weights_category::WeightsCategory, weights_types::WeightsType}, common::visibility::Visibility, }; -use errors::AnyhowResult; use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; use filesys::path_to_string::path_to_string; @@ -94,9 +93,9 @@ pub async fn seed_weights_files( let original_download_url = "https://example.com/model.safetensors"; let thumbnail_token = format!("{}", n); - let public_bucket_hash: String = weights_file_token.to_string(); - let public_bucket_prefix = weights_type.to_str(); - let public_bucket_extension = weights_category.to_str(); + // let public_bucket_hash: String = weights_file_token.to_string(); + // let public_bucket_prefix = weights_type.to_string(); + // let public_bucket_extension = weights_category.to_string(); let cached_user_ratings_total_count = n.clone(); let cached_user_ratings_positive_count = n.clone(); @@ -111,7 +110,7 @@ pub async fn seed_weights_files( let file_size_bytes = file_size(weight_file_path.clone())?; let sha256_checksum = sha256_hash_file(weight_file_path.clone())?; - let result = seed_model( + let bucket_details = seed_model( &mysql_pool, &model_weight_token, &user_token, @@ -119,6 +118,11 @@ pub async fn seed_weights_files( &weight_file_path, maybe_bucket_clients ).await; + let bucket_details_data = bucket_details.unwrap(); + + let bucket_hash = bucket_details_data.bucket_hash; + let maybe_bucket_prefix = bucket_details_data.maybe_bucket_prefix.as_deref().unwrap(); + let maybe_bucket_extension = bucket_details_data.maybe_bucket_extension.as_deref().unwrap(); let create_model_weights_args = CreateModelWeightsArgs { token: &model_weight_token, @@ -136,9 +140,9 @@ pub async fn seed_weights_files( original_filename: Some(original_filename.to_string()), file_size_bytes: file_size_bytes as i32, // Assuming the file size is 0 file_checksum_sha2: sha256_checksum.to_string(), // Assuming the file checksum is an empty string - private_bucket_hash: public_bucket_hash, - maybe_private_bucket_prefix: Some(public_bucket_prefix.to_string()), - maybe_private_bucket_extension: Some(public_bucket_extension.to_string()), + private_bucket_hash: bucket_hash, + maybe_private_bucket_prefix: Some(maybe_bucket_prefix.to_string()), + maybe_private_bucket_extension: Some(maybe_bucket_extension.to_string()), cached_user_ratings_total_count, cached_user_ratings_positive_count, cached_user_ratings_negative_count, @@ -149,24 +153,15 @@ pub async fn seed_weights_files( create_weight(create_model_weights_args).await?; - match result { - Ok(_) => info!("Seeded {:?}", weight_file_path), - Err(err) => - warn!( - r#" - Could not seed media file {} , {:?} : {:?} - (No worries: if there's a duplicate key error, we probably already - seeded the media file on a previous invocation!) - "#, - model_weight_token, - subdirectory_path, - err - ), - } + + info!("Seeded {:?}", weight_file_path) + } Ok(()) } + +#[derive(Clone)] struct BucketDetails { bucket_hash: String, maybe_bucket_prefix: Option, @@ -180,30 +175,26 @@ async fn seed_model( weight_type: WeightsType, weight_file_path: &Path, maybe_bucket_clients: Option<&BucketClients> -) -> AnyhowResult<()> { +) -> AnyhowResult { info!("Seeding Media file {:?} ...", weight_file_path); // mutable variables - let mut bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); - let mut maybe_bucket_prefix = ""; - let mut maybe_bucket_extension = ""; - if let Some(bucket_clients) = maybe_bucket_clients { - let bucket_details = seed_file_to_bucket(weight_file_path, bucket_clients).await?; - bucket_hash = bucket_details.bucket_hash; - maybe_bucket_prefix = bucket_details.maybe_bucket_prefix.as_deref().unwrap(); - maybe_bucket_extension = bucket_details.maybe_bucket_extension.as_deref().unwrap(); + let bucket_details = seed_file_to_bucket(weight_file_path, bucket_clients,weight_type).await?; + return Ok(bucket_details); } - - Ok(()) + Err(anyhow!("No bucket clients provided!")) + } async fn seed_file_to_bucket( weight_file_path: &Path, - bucket_clients: &BucketClients + bucket_clients: &BucketClients, + weight_type: WeightsType, ) -> AnyhowResult { info!("Uploading weights file {:?} ...", weight_file_path); - let maybe_bucket_prefix = Some("fakeyou_"); + let maybe_bucket_prefix = Some(format!("{}_",weight_type.to_str())); + let maybe_bucket_prefix = maybe_bucket_prefix.as_deref(); let maybe_bucket_extension = weight_file_path .extension()