From 2af57fcf3e382589592721690c1062e8c83f8527 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 20 Oct 2023 03:43:33 -0400 Subject: [PATCH 1/3] Write voice conversion results to media_files tables --- .../generic_synthetic_ids/id_category.rs | 101 ++++++---- .../media_file_origin_model_type.rs | 88 +++++++-- .../media_file_origin_product_category.rs | 13 +- ...insert_media_file_from_voice_conversion.rs | 173 ++++++++++++++++++ .../src/queries/media_files/mod.rs | 3 +- .../config/download-job.development.env | 8 +- .../config/inference-job.development.env | 3 +- .../job_types/vc/so_vits_svc/process_job.rs | 80 ++++---- 8 files changed, 382 insertions(+), 87 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_voice_conversion.rs diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index 2611d797ff..156f3e652d 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -21,6 +21,11 @@ pub enum IdCategory { #[serde(rename = "lipsync_animation")] LipsyncAnimation, + /// Results from voice conversion (which may live in the media_files table) + /// Applies for RVC and SVC + #[serde(rename = "voice_conversion")] + VoiceConversion, + /// Zs dataset which lives in the zs_voice_datasets table #[serde(rename = "zs_dataset")] ZsDataset, @@ -40,6 +45,7 @@ impl IdCategory { match self { Self::MediaFile => "media_file", Self::LipsyncAnimation => "lipsync_animation", + Self::VoiceConversion => "voice_conversion", Self::ZsDataset => "zs_dataset", Self::ZsVoice => "zs_voice", } @@ -49,6 +55,7 @@ impl IdCategory { match value { "media_file" => Ok(Self::MediaFile), "lipsync_animation" => Ok(Self::LipsyncAnimation), + "voice_conversion" => Ok(Self::VoiceConversion), "zs_dataset" => Ok(Self::ZsDataset), "zs_voice" => Ok(Self::ZsVoice), _ => Err(format!("invalid value: {:?}", value)), @@ -61,6 +68,7 @@ impl IdCategory { BTreeSet::from([ Self::MediaFile, Self::LipsyncAnimation, + Self::VoiceConversion, Self::ZsDataset, Self::ZsVoice, ]) @@ -72,41 +80,70 @@ mod tests { use crate::by_table::generic_synthetic_ids::id_category::IdCategory; use crate::test_helpers::assert_serialization; - #[test] - fn test_serialization() { - assert_serialization(IdCategory::MediaFile, "media_file"); - assert_serialization(IdCategory::LipsyncAnimation, "lipsync_animation"); + mod explicit_checks { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(IdCategory::MediaFile, "media_file"); + assert_serialization(IdCategory::LipsyncAnimation, "lipsync_animation"); + assert_serialization(IdCategory::VoiceConversion, "voice_conversion"); + assert_serialization(IdCategory::ZsDataset, "zs_dataset"); + assert_serialization(IdCategory::ZsVoice, "zs_voice"); + } + + #[test] + fn to_str() { + assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); + assert_eq!(IdCategory::LipsyncAnimation.to_str(), "lipsync_animation"); + assert_eq!(IdCategory::VoiceConversion.to_str(), "voice_conversion"); + assert_eq!(IdCategory::ZsDataset.to_str(), "zs_dataset"); + assert_eq!(IdCategory::ZsVoice.to_str(), "zs_voice"); + } + + #[test] + fn from_str() { + assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); + assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimation); + assert_eq!(IdCategory::from_str("voice_conversion").unwrap(), IdCategory::VoiceConversion); + assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZsDataset); + assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZsVoice); + } + + #[test] + fn all_variants() { + // Static check + let mut variants = IdCategory::all_variants(); + assert_eq!(variants.len(), 5); + assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); + assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimation)); + assert_eq!(variants.pop_first(), Some(IdCategory::VoiceConversion)); + assert_eq!(variants.pop_first(), Some(IdCategory::ZsDataset)); + assert_eq!(variants.pop_first(), Some(IdCategory::ZsVoice)); + assert_eq!(variants.pop_first(), None); + + // Generated check + use strum::IntoEnumIterator; + assert_eq!(IdCategory::all_variants().len(), IdCategory::iter().len()); + } } - #[test] - fn to_str() { - assert_eq!(IdCategory::MediaFile.to_str(), "media_file"); - assert_eq!(IdCategory::LipsyncAnimation.to_str(), "lipsync_animation"); - assert_eq!(IdCategory::ZsDataset.to_str(), "zs_dataset"); - assert_eq!(IdCategory::ZsVoice.to_str(), "zs_voice"); - } + mod mechanical_checks { + use super::*; - #[test] - fn from_str() { - assert_eq!(IdCategory::from_str("media_file").unwrap(), IdCategory::MediaFile); - assert_eq!(IdCategory::from_str("lipsync_animation").unwrap(), IdCategory::LipsyncAnimation); - assert_eq!(IdCategory::from_str("zs_dataset").unwrap(), IdCategory::ZsDataset); - assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZsVoice); - } + #[test] + fn variant_length() { + use strum::IntoEnumIterator; + assert_eq!(IdCategory::all_variants().len(), IdCategory::iter().len()); + } - #[test] - fn all_variants() { - // Static check - let mut variants = IdCategory::all_variants(); - assert_eq!(variants.len(), 4); - assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); - assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimation)); - assert_eq!(variants.pop_first(), Some(IdCategory::ZsDataset)); - assert_eq!(variants.pop_first(), Some(IdCategory::ZsVoice)); - assert_eq!(variants.pop_first(), None); - - // Generated check - use strum::IntoEnumIterator; - assert_eq!(IdCategory::all_variants().len(), IdCategory::iter().len()); + #[test] + fn round_trip() { + for variant in IdCategory::all_variants() { + assert_eq!(variant, IdCategory::from_str(variant.to_str()).unwrap()); + assert_eq!(variant, IdCategory::from_str(&format!("{}", variant)).unwrap()); + assert_eq!(variant, IdCategory::from_str(&format!("{:?}", variant)).unwrap()); + } + } } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs index b401758897..b713f014c5 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_model_type.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeSet; + use serde::Deserialize; use serde::Serialize; #[cfg(test)] @@ -9,11 +11,19 @@ use strum::EnumIter; /// /// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. #[cfg_attr(test, derive(EnumIter, EnumCount))] -#[derive(Clone, Copy, Eq, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] #[serde(rename_all = "snake_case")] pub enum MediaFileOriginModelType { + /// RVC (v2) voice conversion models + #[serde(rename = "rvc_v2")] + RvcV2, + /// SadTalker -- v1, we may add another enum value for future versions SadTalker, + + /// so-vits-svc voice conversion models + #[serde(rename = "so_vits_svc")] + SoVitsSvc, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -24,16 +34,30 @@ impl_mysql_enum_coders!(MediaFileOriginModelType); impl MediaFileOriginModelType { pub fn to_str(&self) -> &'static str { match self { + Self::RvcV2 => "rvc_v2", Self::SadTalker => "sad_talker", + Self::SoVitsSvc => "so_vits_svc", } } pub fn from_str(value: &str) -> Result { match value { + "rvc_v2" => Ok(Self::RvcV2), "sad_talker" => Ok(Self::SadTalker), + "so_vits_svc" => Ok(Self::SoVitsSvc), _ => Err(format!("invalid value: {:?}", value)), } } + + pub fn all_variants() -> BTreeSet { + // NB: BTreeSet is sorted + // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. + BTreeSet::from([ + Self::RvcV2, + Self::SadTalker, + Self::SoVitsSvc, + ]) + } } #[cfg(test)] @@ -41,19 +65,59 @@ mod tests { use crate::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; use crate::test_helpers::assert_serialization; - #[test] - fn test_serialization() { - assert_serialization(MediaFileOriginModelType::SadTalker, "sad_talker"); + mod explicit_checks { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(MediaFileOriginModelType::RvcV2, "rvc_v2"); + assert_serialization(MediaFileOriginModelType::SadTalker, "sad_talker"); + assert_serialization(MediaFileOriginModelType::SoVitsSvc, "so_vits_svc"); + } + + #[test] + fn test_to_str() { + assert_eq!(MediaFileOriginModelType::RvcV2.to_str(), "rvc_v2"); + assert_eq!(MediaFileOriginModelType::SadTalker.to_str(), "sad_talker"); + assert_eq!(MediaFileOriginModelType::SoVitsSvc.to_str(), "so_vits_svc"); + } + + #[test] + fn test_from_str() { + assert_eq!(MediaFileOriginModelType::from_str("rvc_v2").unwrap(), MediaFileOriginModelType::RvcV2); + assert_eq!(MediaFileOriginModelType::from_str("sad_talker").unwrap(), MediaFileOriginModelType::SadTalker); + assert_eq!(MediaFileOriginModelType::from_str("so_vits_svc").unwrap(), MediaFileOriginModelType::SoVitsSvc); + assert!(MediaFileOriginModelType::from_str("foo").is_err()); + } + + #[test] + fn all_variants() { + let mut variants = MediaFileOriginModelType::all_variants(); + assert_eq!(variants.len(), 3); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::RvcV2)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SadTalker)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginModelType::SoVitsSvc)); + assert_eq!(variants.pop_first(), None); + } } - #[test] - fn test_to_str() { - assert_eq!(MediaFileOriginModelType::SadTalker.to_str(), "sad_talker"); - } - #[test] - fn test_from_str() { - assert_eq!(MediaFileOriginModelType::from_str("sad_talker").unwrap(), MediaFileOriginModelType::SadTalker); - assert!(MediaFileOriginModelType::from_str("foo").is_err()); + mod mechanical_checks { + use super::*; + + #[test] + fn variant_length() { + use strum::IntoEnumIterator; + assert_eq!(MediaFileOriginModelType::all_variants().len(), MediaFileOriginModelType::iter().len()); + } + + #[test] + fn round_trip() { + for variant in MediaFileOriginModelType::all_variants() { + assert_eq!(variant, MediaFileOriginModelType::from_str(variant.to_str()).unwrap()); + assert_eq!(variant, MediaFileOriginModelType::from_str(&format!("{}", variant)).unwrap()); + assert_eq!(variant, MediaFileOriginModelType::from_str(&format!("{:?}", variant)).unwrap()); + } + } } } diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 6721b683d5..8bc2a4b81a 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -27,6 +27,10 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "unknown")] Unknown, + /// Voice conversion (either RVC or SVC) + #[serde(rename = "voice_conversion")] + VoiceConversion, + /// Media files created by (or uploaded for) the Zero Shot voice product. #[serde(rename = "zs_voice")] ZeroShotVoice, @@ -42,6 +46,7 @@ impl MediaFileOriginProductCategory { match self { Self::FaceAnimator => "face_animator", Self::Unknown => "unknown", + Self::VoiceConversion => "voice_conversion", Self::ZeroShotVoice => "zs_voice", } } @@ -50,6 +55,7 @@ impl MediaFileOriginProductCategory { match value { "face_animator" => Ok(Self::FaceAnimator), "unknown" => Ok(Self::Unknown), + "voice_conversion" => Ok(Self::VoiceConversion), "zs_voice" => Ok(Self::ZeroShotVoice), _ => Err(format!("invalid value: {:?}", value)), } @@ -61,6 +67,7 @@ impl MediaFileOriginProductCategory { BTreeSet::from([ Self::FaceAnimator, Self::Unknown, + Self::VoiceConversion, Self::ZeroShotVoice, ]) } @@ -78,6 +85,7 @@ mod tests { fn test_serialization() { assert_serialization(MediaFileOriginProductCategory::FaceAnimator, "face_animator"); assert_serialization(MediaFileOriginProductCategory::Unknown, "unknown"); + assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); assert_serialization(MediaFileOriginProductCategory::ZeroShotVoice, "zs_voice"); } @@ -85,6 +93,7 @@ mod tests { fn to_str() { assert_eq!(MediaFileOriginProductCategory::FaceAnimator.to_str(), "face_animator"); assert_eq!(MediaFileOriginProductCategory::Unknown.to_str(), "unknown"); + assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); assert_eq!(MediaFileOriginProductCategory::ZeroShotVoice.to_str(), "zs_voice"); } @@ -92,15 +101,17 @@ mod tests { fn from_str() { assert_eq!(MediaFileOriginProductCategory::from_str("face_animator").unwrap(), MediaFileOriginProductCategory::FaceAnimator); assert_eq!(MediaFileOriginProductCategory::from_str("unknown").unwrap(), MediaFileOriginProductCategory::Unknown); + assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); assert_eq!(MediaFileOriginProductCategory::from_str("zs_voice").unwrap(), MediaFileOriginProductCategory::ZeroShotVoice); } #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 3); + assert_eq!(variants.len(), 4); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ZeroShotVoice)); assert_eq!(variants.pop_first(), None); } diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_voice_conversion.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_voice_conversion.rs new file mode 100644 index 0000000000..db79e2b734 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_voice_conversion.rs @@ -0,0 +1,173 @@ +use anyhow::anyhow; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id; + +pub enum VoiceConversionModelType { + SoVitsSvc, + RvcV2, +} + +pub struct InsertMediaFileArgs<'a> { + pub pool: &'a MySqlPool, + pub job: &'a AvailableInferenceJob, + + pub voice_conversion_type: VoiceConversionModelType, + + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub duration_millis: u64, + pub sha256_checksum: &'a str, + // TODO: Media duration. + //pub duration_millis: u64, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, + + pub is_on_prem: bool, + pub worker_hostname: &'a str, + pub worker_cluster: &'a str, +} + +pub async fn insert_media_file_from_voice_conversion( + args: InsertMediaFileArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let result_token = MediaFileToken::generate(); + + let origin_model_type = match args.voice_conversion_type { + VoiceConversionModelType::SoVitsSvc => MediaFileOriginModelType::SoVitsSvc, + VoiceConversionModelType::RvcV2 => MediaFileOriginModelType::RvcV2, + }; + + let mut maybe_creator_file_synthetic_id : Option = None; + let mut maybe_creator_category_synthetic_id : Option = None; + + let mut transaction = args.pool.begin().await?; + + if let Some(creator_user_token) = args.job.maybe_creator_user_token.as_deref() { + let user_token = UserToken::new_from_str(creator_user_token); + + let next_media_file_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MediaFile, + &mut transaction + ).await?; + + let next_voice_conversion_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::VoiceConversion, + &mut transaction + ).await?; + + maybe_creator_file_synthetic_id = Some(next_media_file_id); + maybe_creator_category_synthetic_id = Some(next_voice_conversion_id); + } + + let vc_model_token = args.job.maybe_model_token.as_deref(); + let creator_ip_address = args.job.creator_ip_address.as_str(); + let creator_set_visibility = args.job.creator_set_visibility.clone(); + + const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Inference; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::VoiceConversion; + const MEDIA_TYPE : MediaFileType = MediaFileType::Audio; + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + + maybe_origin_model_type = ?, + maybe_origin_model_token = ?, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ?, + + maybe_creator_file_synthetic_id = ?, + maybe_creator_category_synthetic_id = ?, + + is_generated_on_prem = ?, + generated_by_worker = ?, + generated_by_cluster = ? + + "#, + result_token.as_str(), + + ORIGIN_CATEGORY.to_str(), + ORIGIN_PRODUCT_CATEGORY.to_str(), + + origin_model_type.to_str(), + args.job.maybe_model_token, + + MEDIA_TYPE.to_str(), + args.maybe_mime_type, + args.file_size_bytes, + + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + args.job.maybe_creator_user_token, + args.job.creator_ip_address, + + args.job.creator_set_visibility.to_str(), + + maybe_creator_file_synthetic_id, + maybe_creator_category_synthetic_id, + + args.is_on_prem, + args.worker_hostname, + args.worker_cluster + ) + .execute(&mut transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((result_token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index cab9cbf24c..2e48f83061 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -1,3 +1,4 @@ pub mod get_media_file; +pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; -pub mod insert_media_file_from_cli_tool; \ No newline at end of file +pub mod insert_media_file_from_voice_conversion; \ No newline at end of file diff --git a/crates/service/job/download_job/config/download-job.development.env b/crates/service/job/download_job/config/download-job.development.env index bd1d27eb0d..48c09604a6 100644 --- a/crates/service/job/download_job/config/download-job.development.env +++ b/crates/service/job/download_job/config/download-job.development.env @@ -43,13 +43,11 @@ SOFTVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE=a79106991ce3 # # # ================================= # -SO_VITS_SVC_MODEL_CHECK_ROOT_DIRECTORY="/models/voice_conversion/so-vits-svc" -SO_VITS_SVC_MODEL_CHECK_COMMAND="python3 -m so_vits_svc_fork.fakeyou_infer" -SO_VITS_SVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE="8acacf30bb2e" +SO_VITS_SVC_MODEL_CHECK_ROOT_DIRECTORY="/models/voice_conversion/so-vits-svc/src" +SO_VITS_SVC_MODEL_CHECK_COMMAND="/install/python/bin/python3.10 fakeyou_infer.py" # NB: so-vits-svc-paul-2 +SO_VITS_SVC_MODEL_CHECK_MAYBE_DOCKER_IMAGE="1136fe8b50d5" SO_VITS_SVC_MODEL_CHECK_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vits-svc/example_config.json" SO_VITS_SVC_MODEL_CHECK_MAYBE_DEFAULT_TEST_WAV_PATH="/models/voice_conversion/so-vits-svc/example.wav" -#SO_VITS_SVC_MODEL_CHECK_COMMAND="/common/so-vits-svc-code/install/python/bin/python3.10 -m so_vits_svc_fork.fakeyou_infer" -#SO_VITS_SVC_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate" # ================================= # # # diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 5ac8debcda..47d2468fbe 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -60,10 +60,9 @@ FFMPEG_LOGO_LOGO_PATH="%storyteller_rust%/includes/container_includes/image_asse HUBERT_PATH="/tmp/hubert/checkpoint_best_legacy_500.pt" SO_VITS_SVC_INFERENCE_ROOT_DIRECTORY="/models/voice_conversion/so-vits-svc/src" #NB: so-vits-svc-paul-2 -SO_VITS_SVC_INFERENCE_MAYBE_DOCKER_IMAGE="247952d65c82" # so-vits-svc-paul-2: --f0-method +SO_VITS_SVC_INFERENCE_MAYBE_DOCKER_IMAGE="1136fe8b50d5" # so-vits-svc-paul-2: --f0-method SO_VITS_SVC_INFERENCE_COMMAND="/install/python/bin/python3.10 fakeyou_infer.py" # NB: so-vits-svc-paul-2 SO_VITS_SVC_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH="/models/voice_conversion/so-vits-svc/example_config.json" -#SO_VITS_SVC_INFERENCE_MAYBE_VENV_COMMAND="source python/bin/activate" # ================================= # # # diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs index 666571dbfe..1a771fe7d6 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs @@ -2,6 +2,7 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; @@ -11,13 +12,14 @@ use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_size::file_size; +use hashing::sha256::sha256_hash_file::sha256_hash_file; use media::decode_basic_audio_info::decode_basic_audio_file_info; use mimetypes::mimetype_for_file::get_mimetype_for_file; use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType}; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; -use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs}; use tokens::tokens::media_uploads::MediaUploadToken; use tokens::tokens::users::UserToken; @@ -27,6 +29,9 @@ use crate::job::job_types::vc::so_vits_svc::so_vits_svc_inference_command::{Devi use crate::job_dependencies::JobDependencies; use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +const BUCKET_FILE_PREFIX : &str = "fakeyou_svc_"; +const BUCKET_FILE_EXTENSION : &str = ".wav"; + pub struct SoVitsSvcProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, @@ -131,14 +136,10 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result Date: Fri, 20 Oct 2023 04:50:07 -0400 Subject: [PATCH 2/3] RVC -> media_files --- .../job/job_types/vc/rvc_v2/process_job.rs | 70 ++++++++++++------- 1 file changed, 44 insertions(+), 26 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs index 19a8d08b7a..f1c0c0d7c7 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs @@ -3,6 +3,7 @@ use std::time::Instant; use anyhow::anyhow; use log::{error, info, warn}; +use buckets::public::media_files::original_file::MediaFileBucketPath; use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath; @@ -12,13 +13,14 @@ use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use filesys::create_dir_all_if_missing::create_dir_all_if_missing; use filesys::file_size::file_size; +use hashing::sha256::sha256_hash_file::sha256_hash_file; use media::decode_basic_audio_info::decode_basic_audio_file_info; use mimetypes::mimetype_for_file::get_mimetype_for_file; use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::media_files::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType}; use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference; use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInference; -use mysql_queries::queries::voice_conversion::results::insert_voice_conversion_result::{insert_voice_conversion_result, InsertArgs}; use tokens::tokens::media_uploads::MediaUploadToken; use tokens::tokens::users::UserToken; @@ -28,6 +30,9 @@ use crate::job::job_types::vc::rvc_v2::rvc_v2_inference_command::InferenceArgs; use crate::job_dependencies::JobDependencies; use crate::util::maybe_download_file_from_bucket::maybe_download_file_from_bucket; +const BUCKET_FILE_PREFIX : &str = "fakeyou_rvc_"; +const BUCKET_FILE_EXTENSION : &str = ".wav"; + pub struct RvcV2ProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, @@ -267,8 +272,6 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result) -> Result) -> Result) -> Result) -> Result Date: Fri, 20 Oct 2023 05:37:51 -0400 Subject: [PATCH 3/3] fix inference job status links --- .../web/get_inference_job_status.rs | 16 ++++++++++++++- .../inference_job/get_inference_job_status.rs | 20 ++++++++++++++++--- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index 8bef81c017..2089bda8df 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -10,6 +10,7 @@ use tokens::tokens::generic_inference_jobs::InferenceJobToken; use crate::helpers::boolean_converters::i8_to_bool; +#[derive(Debug)] pub struct GenericInferenceJobStatus { pub job_token: InferenceJobToken, @@ -34,6 +35,7 @@ pub struct GenericInferenceJobStatus { /// Details about the user's original inference request /// (We may want to present it in the "pending" UI.) +#[derive(Debug)] pub struct RequestDetails { pub inference_category: InferenceCategory, pub maybe_model_type: Option, // TODO: Strongly type @@ -45,6 +47,7 @@ pub struct RequestDetails { } /// Details about the generated result +#[derive(Debug)] pub struct ResultDetails { pub entity_type: String, pub entity_token: String, @@ -94,6 +97,7 @@ SELECT tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path, voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash, + media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash, media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix, media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension, @@ -144,12 +148,22 @@ WHERE jobs.token = ? // NB: A bit of a hack. We store TTS results with a full path. // Going forward, all other record types will store a hash. - let (bucket_path_is_hash, maybe_public_bucket_hash) = match record.inference_category { + let (mut bucket_path_is_hash, mut maybe_public_bucket_hash) = match record.inference_category { InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()), InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()), InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()), }; + // NB: We've moved voice conversion out of their own table and into media_files + // This check should allow for graceful migration to the new end-state. + match record.maybe_result_entity_type.as_deref() { + Some("media_file") => { + bucket_path_is_hash = true; + maybe_public_bucket_hash = record.maybe_media_file_public_bucket_directory_hash.as_deref(); + } + _ => {} + } + let maybe_result_details = record .maybe_result_entity_type .as_deref() diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs index 7a9f26eafe..8f0ca0decf 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/get_inference_job_status.rs @@ -233,9 +233,23 @@ pub async fn get_inference_job_status_handler( result_details.public_bucket_location_or_hash } InferenceCategory::VoiceConversion => { - VoiceConversionResultOriginalFilePath::from_object_hash(&result_details.public_bucket_location_or_hash) - .get_full_object_path_str() - .to_string() + match result_details.entity_type.as_str() { + "media_file" => { + // NB: We're migrating voice conversion to media_files. + MediaFileBucketPath::from_object_hash( + &result_details.public_bucket_location_or_hash, + result_details.maybe_media_file_public_bucket_prefix.as_deref(), + result_details.maybe_media_file_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string() + } + _ => { + // NB: This is the old voice conversion result pathing. + VoiceConversionResultOriginalFilePath::from_object_hash(&result_details.public_bucket_location_or_hash) + .get_full_object_path_str() + .to_string() + } + } } };