From 4672102da4e148df9a7a6fc21ae4df80c9ba52b7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 23 Apr 2023 05:03:37 -0700 Subject: [PATCH] code changes stemming from schema changes --- .../inference_input_source_token_type.rs | 79 +++++++++++++++++++ .../inference_model_type.rs | 13 ++- .../by_table/generic_inference_jobs/mod.rs | 1 + .../web/insert_generic_inference_job.rs | 25 ++++-- .../get_media_upload_for_inference.rs | 2 +- .../media_uploads/insert_media_upload.rs | 17 ++-- .../results/insert_voice_conversion_result.rs | 2 +- .../endpoints/media_uploads/upload_media.rs | 3 +- .../enqueue_infer_tts_handler.rs | 12 ++- .../enqueue_voice_conversion_inference.rs | 10 ++- 10 files changed, 142 insertions(+), 22 deletions(-) create mode 100644 crates/_types/enums/src/by_table/generic_inference_jobs/inference_input_source_token_type.rs diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_input_source_token_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_input_source_token_type.rs new file mode 100644 index 0000000000..e86470596f --- /dev/null +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_input_source_token_type.rs @@ -0,0 +1,79 @@ +use std::collections::BTreeSet; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + + +/// Used in the `generic_inference_jobs` table in `VARCHAR(32)` field `maybe_input_source_token`. +/// +/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] +pub enum InferenceInputSourceTokenType { + #[serde(rename = "media_upload")] + MediaUpload, +} + +// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(InferenceInputSourceTokenType); +impl_mysql_enum_coders!(InferenceInputSourceTokenType); + +/// NB: Legacy API for older code. +impl InferenceInputSourceTokenType { + pub fn to_str(&self) -> &'static str { + match self { + Self::MediaUpload => "media_upload", + } + } + + pub fn from_str(value: &str) -> Result { + match value { + "media_upload" => Ok(Self::MediaUpload), + _ => Err(format!("invalid value: {:?}", value)), + } + } + + pub fn all_variants() -> BTreeSet { + // NB: BTreeSet is sorted + // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. + BTreeSet::from([ + InferenceInputSourceTokenType::MediaUpload, + ]) + } +} + +#[cfg(test)] +mod tests { + use crate::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType; + use crate::test_helpers::assert_serialization; + + #[test] + fn test_serialization() { + assert_serialization(InferenceInputSourceTokenType::MediaUpload, "media_upload"); + } + + #[test] + fn to_str() { + assert_eq!(InferenceInputSourceTokenType::MediaUpload.to_str(), "media_upload"); + } + + #[test] + fn from_str() { + assert_eq!(InferenceInputSourceTokenType::from_str("media_upload").unwrap(), InferenceInputSourceTokenType::MediaUpload); + } + + #[test] + fn all_variants() { + // Static check + let mut variants = InferenceInputSourceTokenType::all_variants(); + assert_eq!(variants.len(), 1); + assert_eq!(variants.pop_first(), Some(InferenceInputSourceTokenType::MediaUpload)); + assert_eq!(variants.pop_first(), None); + + // Generated check + use strum::IntoEnumIterator; + assert_eq!(InferenceInputSourceTokenType::all_variants().len(), InferenceInputSourceTokenType::iter().len()); + } +} diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs index 56f4bae83e..957ad4d2f0 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/inference_model_type.rs @@ -18,6 +18,10 @@ use strum::EnumIter; #[cfg_attr(test, derive(EnumIter, EnumCount))] #[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize)] pub enum InferenceModelType { + #[serde(rename = "so_vits_svc")] + SoVitsSvc, + + // TODO: Does this need to be "legacy_tacotron2" ? #[serde(rename = "tacotron2")] Tacotron2, @@ -33,6 +37,7 @@ impl_mysql_enum_coders!(InferenceModelType); impl InferenceModelType { pub fn to_str(&self) -> &'static str { match self { + Self::SoVitsSvc => "so_vits_svc", Self::Tacotron2 => "tacotron2", Self::Vits => "vits", } @@ -40,6 +45,7 @@ impl InferenceModelType { pub fn from_str(value: &str) -> Result { match value { + "so_vits_svc" => Ok(Self::SoVitsSvc), "tacotron2" => Ok(Self::Tacotron2), "vits" => Ok(Self::Vits), _ => Err(format!("invalid value: {:?}", value)), @@ -50,6 +56,7 @@ impl InferenceModelType { // NB: BTreeSet is sorted // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ + InferenceModelType::SoVitsSvc, InferenceModelType::Tacotron2, InferenceModelType::Vits, ]) @@ -63,18 +70,21 @@ mod tests { #[test] fn test_serialization() { + assert_serialization(InferenceModelType::SoVitsSvc, "so_vits_svc"); assert_serialization(InferenceModelType::Tacotron2, "tacotron2"); assert_serialization(InferenceModelType::Vits, "vits"); } #[test] fn to_str() { + assert_eq!(InferenceModelType::SoVitsSvc.to_str(), "so_vits_svc"); assert_eq!(InferenceModelType::Tacotron2.to_str(), "tacotron2"); assert_eq!(InferenceModelType::Vits.to_str(), "vits"); } #[test] fn from_str() { + assert_eq!(InferenceModelType::from_str("so_vits_svc").unwrap(), InferenceModelType::SoVitsSvc); assert_eq!(InferenceModelType::from_str("tacotron2").unwrap(), InferenceModelType::Tacotron2); assert_eq!(InferenceModelType::from_str("vits").unwrap(), InferenceModelType::Vits); } @@ -83,7 +93,8 @@ mod tests { fn all_variants() { // Static check let mut variants = InferenceModelType::all_variants(); - assert_eq!(variants.len(), 2); + assert_eq!(variants.len(), 3); + assert_eq!(variants.pop_first(), Some(InferenceModelType::SoVitsSvc)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Tacotron2)); assert_eq!(variants.pop_first(), Some(InferenceModelType::Vits)); assert_eq!(variants.pop_first(), None); diff --git a/crates/_types/enums/src/by_table/generic_inference_jobs/mod.rs b/crates/_types/enums/src/by_table/generic_inference_jobs/mod.rs index a18aabd692..6a6ded26cf 100644 --- a/crates/_types/enums/src/by_table/generic_inference_jobs/mod.rs +++ b/crates/_types/enums/src/by_table/generic_inference_jobs/mod.rs @@ -1,3 +1,4 @@ pub mod inference_category; +pub mod inference_input_source_token_type; pub mod inference_model_type; pub mod inference_result_type; diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs index 2dce02baa6..57003933b0 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs @@ -1,6 +1,8 @@ use anyhow::anyhow; use crate::payloads::generic_inference_args::GenericInferenceArgs; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use sqlx::MySqlPool; @@ -11,12 +13,16 @@ pub struct InsertGenericInferenceArgs<'a> { pub uuid_idempotency_token: &'a str, pub inference_category: InferenceCategory, - pub maybe_model_type: Option<&'a str>, + pub maybe_model_type: Option, pub maybe_model_token: Option<&'a str>, - pub maybe_inference_args: Option, + pub maybe_input_source_token: Option<&'a str>, + pub maybe_input_source_token_type: Option, + pub maybe_raw_inference_text: Option<&'a str>, + pub maybe_inference_args: Option, + pub maybe_creator_user_token: Option<&'a UserToken>, pub creator_ip_address: &'a str, pub creator_set_visibility: Visibility, @@ -44,9 +50,13 @@ SET maybe_model_type = ?, maybe_model_token = ?, - maybe_inference_args = ?, + maybe_input_source_token = ?, + maybe_input_source_token_type = ?, + maybe_raw_inference_text = ?, + maybe_inference_args = ?, + maybe_creator_user_token = ?, creator_ip_address = ?, creator_set_visibility = ?, @@ -60,12 +70,17 @@ SET args.uuid_idempotency_token, args.inference_category.to_str(), - args.maybe_model_type, + + args.maybe_model_type.map(|t| t.to_str()), args.maybe_model_token, - serialized_args_payload, + args.maybe_input_source_token, + args.maybe_input_source_token_type, + args.maybe_raw_inference_text, + serialized_args_payload, + args.maybe_creator_user_token.map(|t| t.to_string()), args.creator_ip_address, args.creator_set_visibility.to_str(), diff --git a/crates/lib/mysql_queries/src/queries/media_uploads/get_media_upload_for_inference.rs b/crates/lib/mysql_queries/src/queries/media_uploads/get_media_upload_for_inference.rs index 7857e9bfd7..e5bb039c06 100644 --- a/crates/lib/mysql_queries/src/queries/media_uploads/get_media_upload_for_inference.rs +++ b/crates/lib/mysql_queries/src/queries/media_uploads/get_media_upload_for_inference.rs @@ -59,7 +59,7 @@ SELECT mu.public_bucket_directory_hash, mu.original_file_size_bytes, mu.original_duration_millis, - mu.extra_file_modification_info as maybe_extra_file_modification_info, + mu.maybe_extra_file_modification_info, mu.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, mu.created_at, mu.updated_at diff --git a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs index 5c25b159d8..f3b8d3505f 100644 --- a/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs +++ b/crates/lib/mysql_queries/src/queries/media_uploads/insert_media_upload.rs @@ -29,15 +29,13 @@ pub struct Args <'a> { pub checksum_sha2: &'a str, pub public_upload_path: &'a MediaUploadOriginalFilePath, - pub extra_file_modification_info: MediaUploadModificationDetails, + pub maybe_extra_file_modification_info: Option, pub maybe_creator_user_token: Option<&'a UserToken>, pub maybe_creator_anonymous_visitor_token: Option<&'a str>, pub creator_ip_address: &'a str, pub creator_set_visibility: Visibility, - pub maybe_creator_synthetic_id: Option, - pub mysql_pool: &'a MySqlPool, } @@ -103,6 +101,11 @@ LIMIT 1 let media_token = MediaUploadToken::generate(); + let maybe_extra_file_modification_info = args.maybe_extra_file_modification_info + .clone() // FIXME + .map(|info| info.to_json()) + .transpose()?; + let query = sqlx::query!( r#" INSERT INTO media_uploads @@ -122,8 +125,8 @@ SET checksum_sha2 = ?, public_bucket_directory_hash = ?, - public_bucket_directory_full_path = ?, - extra_file_modification_info = ?, + + maybe_extra_file_modification_info = ?, maybe_creator_user_token = ?, maybe_creator_synthetic_id = ?, @@ -147,8 +150,8 @@ SET args.checksum_sha2, args.public_upload_path.get_object_hash(), - args.public_upload_path.get_full_object_path_str(), - "", // TODO: args.extra_file_modification_info, + + maybe_extra_file_modification_info, args.maybe_creator_user_token, maybe_creator_synthetic_id, diff --git a/crates/lib/mysql_queries/src/queries/voice_conversion/results/insert_voice_conversion_result.rs b/crates/lib/mysql_queries/src/queries/voice_conversion/results/insert_voice_conversion_result.rs index d16703b074..33f4681cbd 100644 --- a/crates/lib/mysql_queries/src/queries/voice_conversion/results/insert_voice_conversion_result.rs +++ b/crates/lib/mysql_queries/src/queries/voice_conversion/results/insert_voice_conversion_result.rs @@ -102,7 +102,7 @@ SET model_token = ?, media_token = ?, - media_token_type= ?, + media_token_type = ?, maybe_creator_user_token = ?, maybe_creator_synthetic_id = ?, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs index a201a1b66a..9e92c80a81 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs @@ -273,12 +273,11 @@ pub async fn upload_media_handler( maybe_original_frame_height: None, checksum_sha2: &hash, public_upload_path: &public_upload_path, - extra_file_modification_info: MediaUploadModificationDetails {}, // TODO + maybe_extra_file_modification_info: None, maybe_creator_user_token: maybe_user_token.as_ref(), maybe_creator_anonymous_visitor_token: None, creator_ip_address: &ip_address, creator_set_visibility, - maybe_creator_synthetic_id: None, // TODO: Don't forget about this. mysql_pool: &server_state.mysql_pool, }) .await diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index 76eabc1983..619ed54914 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -12,6 +12,7 @@ use crate::http_server::endpoints::tts::enqueue_infer_tts_handler::get_model_wit use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::common::visibility::Visibility; use http_server_common::request::get_request_api_token::get_request_api_token; @@ -353,6 +354,11 @@ pub async fn enqueue_infer_tts_handler( let job_token; let job_token_type; + let model_type = match tts_model.tts_model_type { + TtsModelType::Tacotron2 => InferenceModelType::Tacotron2, + TtsModelType::Vits => InferenceModelType::Vits, + }; + if use_new_job_system { // This branch uses the `generic-inference-job` service and tables. info!("Creating tts inference job record (new generic job system)..."); @@ -364,13 +370,15 @@ pub async fn enqueue_infer_tts_handler( let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { uuid_idempotency_token: &request.uuid_idempotency_token, inference_category: InferenceCategory::TextToSpeech, - maybe_model_type: None, // TODO(bt, 2023-04-08): Add this + maybe_model_type: Some(model_type), maybe_model_token: Some(request.tts_model_token.as_str()), + maybe_input_source_token: None, // NB: TTS doesn't have input media + maybe_input_source_token_type: None, // NB: TTS doesn't have input media + maybe_raw_inference_text: Some(inference_text.as_str()), maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::TextToSpeech), args: None, // NB: We don't need to encode any args yet. }), - maybe_raw_inference_text: Some(inference_text.as_str()), maybe_creator_user_token: maybe_creator_user_token_typed.as_ref(), creator_ip_address: &ip_address, creator_set_visibility: set_visibility, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index 8fa968e702..3b1f21b420 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -21,6 +21,8 @@ use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; use std::fmt; use std::sync::Arc; +use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType; +use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use tokens::files::media_upload::MediaUploadToken; use tokens::jobs::inference::InferenceJobToken; use tokens::users::user::UserToken; @@ -224,15 +226,17 @@ pub async fn enqueue_voice_conversion_inference_handler( let query_result = insert_generic_inference_job(InsertGenericInferenceArgs { uuid_idempotency_token: &request.uuid_idempotency_token, inference_category: InferenceCategory::VoiceConversion, - maybe_model_type: None, // TODO(bt, 2023-04-08): Add this. + maybe_model_type: Some(InferenceModelType::SoVitsSvc), maybe_model_token: Some(model_token.as_str()), + maybe_input_source_token: Some(media_token.as_str()), + maybe_input_source_token_type: Some(InferenceInputSourceTokenType::MediaUpload), + maybe_raw_inference_text: None, // NB: Voice conversion isn't TTS, so there's no text. maybe_inference_args: Some(GenericInferenceArgs { inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), args: Some(PolymorphicInferenceArgs::VoiceConversionInferenceArgs { - maybe_media_token: Some(media_token), + maybe_media_token: Some(media_token.clone()), }), }), - maybe_raw_inference_text: None, maybe_creator_user_token: maybe_user_token.as_ref(), creator_ip_address: &ip_address, creator_set_visibility: set_visibility,