diff --git a/crates/lib/migration/src/text_to_speech/get_tts_model_for_inference_migration.rs b/crates/lib/migration/src/text_to_speech/get_tts_model_for_enqueue_inference_migration.rs similarity index 81% rename from crates/lib/migration/src/text_to_speech/get_tts_model_for_inference_migration.rs rename to crates/lib/migration/src/text_to_speech/get_tts_model_for_enqueue_inference_migration.rs index 9768b6eaf2..350ba67781 100644 --- a/crates/lib/migration/src/text_to_speech/get_tts_model_for_inference_migration.rs +++ b/crates/lib/migration/src/text_to_speech/get_tts_model_for_enqueue_inference_migration.rs @@ -4,18 +4,18 @@ use sqlx::pool::PoolConnection; use enums::common::visibility::Visibility; use errors::AnyhowResult; -use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_enqueue::{get_weight_for_legacy_tts_enqueue_with_connection, ModelWeightForLegacyTtsInference}; +use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_enqueue::{get_weight_for_legacy_tts_enqueue_with_connection, ModelWeightForLegacyTtsEnqueue}; use mysql_queries::queries::tts::tts_models::get_tts_model::{get_tts_model_by_token_using_connection, TtsModelRecord}; use tokens::tokens::model_weights::ModelWeightToken; /// Get TTS model /// This is for the tts inference page. -pub async fn get_tts_model_for_inference_migration( +pub async fn get_tts_model_for_enqueue_inference_migration( token: &str, mysql_connection: &mut PoolConnection, can_see_deleted: bool, use_weights_table: bool, -) -> AnyhowResult> { +) -> AnyhowResult> { // NB: This is temporary migration code as we switch from the `tts_models` table to the `model_weights` table. if use_weights_table { let token = ModelWeightToken::new_from_str(token); @@ -26,7 +26,7 @@ pub async fn get_tts_model_for_inference_migration( mysql_connection ).await?; - Ok(maybe_model.map(|model| TtsModelForInferenceMigration::ModelWeight(model))) + Ok(maybe_model.map(|model| TtsModelForEnqueueInferenceMigration::ModelWeight(model))) } else { @@ -36,21 +36,21 @@ pub async fn get_tts_model_for_inference_migration( mysql_connection ).await?; - Ok(maybe_model.map(|model| TtsModelForInferenceMigration::LegacyTts(model))) + Ok(maybe_model.map(|model| TtsModelForEnqueueInferenceMigration::LegacyTts(model))) } } /// Union over the legacy table and the new table to support an easier migration. /// This enum can hold a record of either type and present a unified accessor interface. #[derive(Clone, Serialize, Deserialize)] -pub enum TtsModelForInferenceMigration { +pub enum TtsModelForEnqueueInferenceMigration { /// Old type from the `tts_models` table, on the way out LegacyTts(TtsModelRecord), /// New type, replacing the `tts_models` table. - ModelWeight(ModelWeightForLegacyTtsInference), + ModelWeight(ModelWeightForLegacyTtsEnqueue), } -impl TtsModelForInferenceMigration { +impl TtsModelForEnqueueInferenceMigration { pub fn token(&self) -> &str { match self { Self::LegacyTts(ref model) => model.model_token.as_str(), diff --git a/crates/lib/migration/src/text_to_speech/get_tts_model_for_run_inference_migration.rs b/crates/lib/migration/src/text_to_speech/get_tts_model_for_run_inference_migration.rs new file mode 100644 index 0000000000..0d39aed382 --- /dev/null +++ b/crates/lib/migration/src/text_to_speech/get_tts_model_for_run_inference_migration.rs @@ -0,0 +1,200 @@ +use std::path::PathBuf; + +use chrono::{DateTime, Utc}; +use sqlx::MySqlPool; + +use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath; +use cloud_storage::bucket_path_unifier::BucketPathUnifier; +use enums::by_table::tts_models::tts_model_type::TtsModelType; +use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_inference::{get_weight_for_legacy_tts_inference, ModelWeightForLegacyTtsInference}; +use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{CustomVocoderFields, get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord}; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::users::UserToken; + +/// Get TTS model +/// This is for the inference inference job. +pub async fn get_tts_model_for_run_inference_migration( + tts_model_token: &str, + mysql_pool: &MySqlPool, +) -> Result, TtsModelForInferenceError> { + // NB: This is temporary migration code as we switch from the `tts_models` table to the `model_weights` table. + if tts_model_token.starts_with(ModelWeightToken::token_prefix()) { + let token = ModelWeightToken::new_from_str(tts_model_token); + + let maybe_model = get_weight_for_legacy_tts_inference( + &token, + mysql_pool + ).await?; + + Ok(maybe_model.map(|model| TtsModelForRunInferenceMigration::ModelWeight(model))) + } else { + + let maybe_model = get_tts_model_for_inference_improved( + mysql_pool, + &tts_model_token, + ).await?; + + Ok(maybe_model.map(|model| TtsModelForRunInferenceMigration::LegacyTts(model))) + } +} + +/// Migration wrapper type. +/// Union over the legacy table and the new table to support an easier migration. +/// This enum can hold a record of either type and present a unified accessor interface. +#[derive(Clone)] +pub enum TtsModelForRunInferenceMigration { + /// Old type from the `tts_models` table, on the way out + LegacyTts(TtsModelForInferenceRecord), + /// New type, replacing the `tts_models` table. + ModelWeight(ModelWeightForLegacyTtsInference), +} + +impl TtsModelForRunInferenceMigration { + pub fn token(&self) -> &str { + match self { + Self::LegacyTts(ref model) => model.model_token.as_str(), + Self::ModelWeight(ref model) => model.token.as_str(), + } + } + + pub fn title(&self) -> &str { + match self { + Self::LegacyTts(ref model) => &model.title, + Self::ModelWeight(ref model) => &model.title, + } + } + + pub fn creator_user_token(&self) -> &UserToken { + match self { + Self::LegacyTts(ref model) => &model.creator_user_token, + Self::ModelWeight(ref model) => &model.creator_user_token, + } + } + + pub fn creator_username(&self) -> &str { + match self { + Self::LegacyTts(ref model) => &model.creator_username, + Self::ModelWeight(ref model) => &model.creator_username, + } + } + + pub fn creator_display_name(&self) -> &str { + match self { + Self::LegacyTts(ref model) => &model.creator_display_name, + Self::ModelWeight(ref model) => &model.creator_display_name, + } + } + + pub fn created_at(&self) -> &DateTime { + match self { + Self::LegacyTts(ref model) => &model.created_at, + Self::ModelWeight(ref model) => &model.created_at, + } + } + + pub fn updated_at(&self) -> &DateTime { + match self { + Self::LegacyTts(ref model) => &model.updated_at, + Self::ModelWeight(ref model) => &model.updated_at, + } + } + + pub fn mod_deleted_at(&self) -> Option<&DateTime> { + match self { + Self::LegacyTts(ref model) => model.mod_deleted_at.as_ref(), + Self::ModelWeight(ref model) => model.mod_deleted_at.as_ref(), + } + } + + pub fn user_deleted_at(&self) -> Option<&DateTime> { + match self { + Self::LegacyTts(ref model) => model.user_deleted_at.as_ref(), + Self::ModelWeight(ref model) => model.user_deleted_at.as_ref(), + } + } + + pub fn tts_model_type(&self) -> TtsModelType { + match self { + Self::LegacyTts(ref model) => model.tts_model_type, + Self::ModelWeight(ref _model) => TtsModelType::Tacotron2, // NB: Always TT2 + } + } + + pub fn maybe_default_pretrained_vocoder(&self) -> Option<&str> { + match self { + Self::LegacyTts(ref model) => model.maybe_default_pretrained_vocoder.as_deref(), + Self::ModelWeight(ref model) => model.maybe_default_pretrained_vocoder.as_deref(), + } + } + + /* NB(bt, 2024-01-20): Breakdown of the text_pipeline_type values in the DB: + mysql> select text_pipeline_type , count(*) from tts_models group by text_pipeline_type ; + +--------------------+----------+ + | text_pipeline_type | count(*) | + +--------------------+----------+ + | NULL | 4406 | + | legacy_fakeyou | 2572 | + | english_v1 | 1548 | + | spanish_v2 | 194 | + +--------------------+----------+ + */ + pub fn text_pipeline_type(&self) -> Option<&str> { + match self { + Self::LegacyTts(ref model) => model.text_pipeline_type.as_deref(), + Self::ModelWeight(ref model) => model.maybe_text_pipeline_type.as_deref(), + } + } + + pub fn use_default_mel_multiply_factor(&self) -> bool { + match self { + Self::LegacyTts(ref model) => model.use_default_mel_multiply_factor, + Self::ModelWeight(ref model) => model.use_default_mel_multiply_factor, + } + } + + pub fn maybe_custom_mel_multiply_factor(&self) -> Option { + match self { + Self::LegacyTts(ref model) => model.maybe_custom_mel_multiply_factor, + Self::ModelWeight(ref model) => model.maybe_custom_mel_multiply_factor, + } + } + + pub fn maybe_custom_vocoder(&self) -> Option<&CustomVocoderFields> { + match self { + Self::LegacyTts(ref model) => model.maybe_custom_vocoder.as_ref(), + Self::ModelWeight(ref model) => model.maybe_custom_vocoder.as_ref(), + } + } + + /// Which bucket to download the weights from + pub fn is_private_bucket(&self) -> bool { + match self { + Self::LegacyTts(ref _model) => true, + Self::ModelWeight(ref _model) => false, + } + } + + /// Path to the weights in the bucket + /// The old and new schemes are quite different. + pub fn bucket_object_path(&self, bucket_path_unifier: &BucketPathUnifier) -> PathBuf { + match self { + Self::LegacyTts(ref model) => bucket_path_unifier.tts_synthesizer_path(&model.private_bucket_hash), + Self::ModelWeight(ref model) => { + let path = WeightFileBucketPath::from_object_hash( + &model.public_bucket_hash, + model.maybe_public_bucket_prefix.as_deref(), + model.maybe_public_bucket_extension.as_deref()); + + PathBuf::from(path.get_full_object_path_str()) + }, + } + } + + // NB: Just for vits, which we never supported anyway + pub fn vits_traced_synthesizer_object_path(&self, bucket_path_unifier: &BucketPathUnifier) -> PathBuf { + match self { + Self::LegacyTts(ref model) => bucket_path_unifier.tts_traced_synthesizer_path(&model.private_bucket_hash), + Self::ModelWeight(ref _model) => PathBuf::from("VITS_UNSUPPORTED_FOR_MODEL_WEIGHTS"), + } + } +} diff --git a/crates/lib/migration/src/text_to_speech/mod.rs b/crates/lib/migration/src/text_to_speech/mod.rs index a26935efbd..ad2fdb08ed 100644 --- a/crates/lib/migration/src/text_to_speech/mod.rs +++ b/crates/lib/migration/src/text_to_speech/mod.rs @@ -1,3 +1,4 @@ -pub mod get_tts_model_for_inference_migration; +pub mod get_tts_model_for_enqueue_inference_migration; +pub mod get_tts_model_for_run_inference_migration; pub mod get_tts_model_info_migration; pub mod list_tts_models_for_migration; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_enqueue.rs b/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_enqueue.rs index 1136440065..cf81bb63a2 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_enqueue.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_enqueue.rs @@ -11,7 +11,7 @@ use enums::common::visibility::Visibility; use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken}; #[derive(Clone, Serialize, Deserialize)] -pub struct ModelWeightForLegacyTtsInference { +pub struct ModelWeightForLegacyTtsEnqueue { pub token: ModelWeightToken, pub weights_type: WeightsType, pub weights_category: WeightsCategory, @@ -27,7 +27,7 @@ pub async fn get_weight_for_legacy_tts_enqueue( weight_token: &ModelWeightToken, can_see_deleted: bool, mysql_pool: &MySqlPool -) -> AnyhowResult> { +) -> AnyhowResult> { let mut connection = mysql_pool.acquire().await?; get_weight_for_legacy_tts_enqueue_with_connection(weight_token, can_see_deleted, &mut connection).await } @@ -36,7 +36,7 @@ pub async fn get_weight_for_legacy_tts_enqueue_with_connection( weight_token: &ModelWeightToken, can_see_deleted: bool, mysql_connection: &mut PoolConnection -) -> AnyhowResult> { +) -> AnyhowResult> { let maybe_result = if can_see_deleted { select_include_deleted(weight_token, mysql_connection).await } else { @@ -54,19 +54,15 @@ pub async fn get_weight_for_legacy_tts_enqueue_with_connection( } }; - // unwrap the result - - Ok( - Some(ModelWeightForLegacyTtsInference { - token: record.token, - weights_type: record.weights_type, - weights_category: record.weights_category, - creator_user_token: record.creator_user_token, - creator_set_visibility: record.creator_set_visibility, - user_deleted_at: record.user_deleted_at, - mod_deleted_at: record.mod_deleted_at, - }) - ) + Ok(Some(ModelWeightForLegacyTtsEnqueue { + token: record.token, + weights_type: record.weights_type, + weights_category: record.weights_category, + creator_user_token: record.creator_user_token, + creator_set_visibility: record.creator_set_visibility, + user_deleted_at: record.user_deleted_at, + mod_deleted_at: record.mod_deleted_at, + })) } async fn select_include_deleted( diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_inference.rs b/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_inference.rs new file mode 100644 index 0000000000..16ed295c8f --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_inference.rs @@ -0,0 +1,306 @@ +use chrono::{DateTime, Utc}; +use log::error; +use sqlx::{MySql, MySqlPool}; +use sqlx::pool::PoolConnection; + +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::by_table::model_weights::weights_types::WeightsType; +use enums::common::visibility::Visibility; +use enums::common::vocoder_type::VocoderType; +use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken}; + +use crate::helpers::boolean_converters::nullable_i8_to_bool_default_false; +use crate::queries::tts::tts_models::get_tts_model_for_inference_improved::{CustomVocoderFields, TtsModelForInferenceError}; + +#[derive(Clone)] +pub struct ModelWeightForLegacyTtsInference { + pub token: ModelWeightToken, + pub title: String, + pub weights_type: WeightsType, + pub weights_category: WeightsCategory, + pub maybe_description_markdown: Option, + pub maybe_description_rendered_html: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + pub creator_gravatar_hash: String, + + pub creator_ip_address: String, + pub creator_set_visibility: Visibility, + + pub maybe_last_update_user_token: Option, + pub original_download_url: Option, + pub original_filename: Option, + pub file_size_bytes: i32, + pub file_checksum_sha2: String, + pub public_bucket_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub maybe_cover_image_public_bucket_hash: Option, + pub maybe_cover_image_public_bucket_prefix: Option, + pub maybe_cover_image_public_bucket_extension: Option, + + pub maybe_ratings_positive_count: Option, + pub maybe_ratings_negative_count: Option, + pub maybe_bookmark_count: Option, + + pub maybe_ietf_language_tag: Option, + pub maybe_ietf_primary_language_subtag: Option, + pub maybe_default_pretrained_vocoder: Option, + pub maybe_text_pipeline_type: Option, + + // Joined custom vocoder fields + pub maybe_custom_vocoder: Option, + + //// Joined custom vocoder fields + //pub maybe_custom_vocoder_token: Option, + //pub maybe_custom_vocoder_type: Option, + //pub maybe_custom_vocoder_title: Option, + //pub maybe_custom_vocoder_private_bucket_hash: Option, + + pub use_default_mel_multiply_factor: bool, + pub maybe_custom_mel_multiply_factor: Option, + + pub version: i32, + pub created_at: DateTime, + pub updated_at: DateTime, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn get_weight_for_legacy_tts_inference( + weight_token: &ModelWeightToken, + mysql_pool: &MySqlPool +) -> Result, TtsModelForInferenceError> { + let mut connection = mysql_pool.acquire() + .await + .map_err(|err| { + error!("Error acquiring db connection from pool: {:?}", err); + return TtsModelForInferenceError::DatabaseError { + reason: format!("Mysql connection error: {:?}", err) + }; + })?; + + get_weight_for_legacy_tts_inference_with_connection(weight_token, &mut connection).await +} + +pub async fn get_weight_for_legacy_tts_inference_with_connection( + weight_token: &ModelWeightToken, + mysql_connection: &mut PoolConnection +) -> Result, TtsModelForInferenceError> { + let maybe_result = select_include_deleted(weight_token, mysql_connection).await; + + let record: RawWeight = match maybe_result { + Ok(record) => record, + Err(sqlx::Error::RowNotFound) => { + return Ok(None); + } + Err(err) => { + error!("Error fetching weights by token: {:?}", err); + return Err(TtsModelForInferenceError::DatabaseError { + reason: format!("Mysql error: {:?}", err) + }); + } + }; + + if record.mod_deleted_at.is_some() || record.user_deleted_at.is_some() { + return Err(TtsModelForInferenceError::ModelDeleted); + } + + Ok(Some(ModelWeightForLegacyTtsInference { + token: record.token, + title: record.title, + weights_type: record.weights_type, + weights_category: record.weights_category, + maybe_description_markdown: record.maybe_description_markdown, + maybe_description_rendered_html: record.maybe_description_rendered_html, + creator_user_token: record.creator_user_token, + creator_username: record.creator_username, + creator_display_name: record.creator_display_name, + creator_gravatar_hash: record.creator_gravatar_hash, + creator_ip_address: record.creator_ip_address, + creator_set_visibility: record.creator_set_visibility, + maybe_last_update_user_token: record.maybe_last_update_user_token, + original_download_url: record.original_download_url, + original_filename: record.original_filename, + file_size_bytes: record.file_size_bytes, + file_checksum_sha2: record.file_checksum_sha2, + public_bucket_hash: record.public_bucket_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + maybe_cover_image_public_bucket_hash: record.maybe_cover_image_public_bucket_hash, + maybe_cover_image_public_bucket_prefix: record.maybe_cover_image_public_bucket_prefix, + maybe_cover_image_public_bucket_extension: record.maybe_cover_image_public_bucket_extension, + maybe_ratings_positive_count: record.maybe_ratings_positive_count, + maybe_ratings_negative_count: record.maybe_ratings_negative_count, + maybe_bookmark_count: record.maybe_bookmark_count, + maybe_ietf_language_tag: record.maybe_ietf_language_tag, + maybe_ietf_primary_language_subtag: record.maybe_ietf_primary_language_subtag, + maybe_default_pretrained_vocoder: record.maybe_default_pretrained_vocoder, + maybe_text_pipeline_type: record.maybe_text_pipeline_type, + //maybe_custom_vocoder_token: record.maybe_custom_vocoder_token, + //maybe_custom_vocoder_type: record.maybe_custom_vocoder_type, + //maybe_custom_vocoder_title: record.maybe_custom_vocoder_title, + //maybe_custom_vocoder_private_bucket_hash: record.maybe_custom_vocoder_private_bucket_hash, + maybe_custom_vocoder: match record.maybe_custom_vocoder_token { + // NB: We're relying on a single field's presence to infer that the others vocoder fields + // are also there. If for some reason they aren't, fail open. + None => None, + Some(vocoder_token) => Some(CustomVocoderFields { + vocoder_token, + vocoder_type: record.maybe_custom_vocoder_type.ok_or( + TtsModelForInferenceError::DatabaseError { reason: "custom_vocoder_type field error".to_string() })?, + vocoder_title: record.maybe_custom_vocoder_title.ok_or( + TtsModelForInferenceError::DatabaseError { reason: "custom_vocoder_title field error".to_string() })?, + vocoder_private_bucket_hash: record.maybe_custom_vocoder_private_bucket_hash.ok_or( + TtsModelForInferenceError::DatabaseError { reason: "vocoder_private_bucket_hash field error".to_string() })?, + }) + }, + use_default_mel_multiply_factor: nullable_i8_to_bool_default_false(record.use_default_mel_multiply_factor), + maybe_custom_mel_multiply_factor: record.maybe_custom_mel_multiply_factor, + version: record.version, + created_at: record.created_at, + updated_at: record.updated_at, + user_deleted_at: record.user_deleted_at, + mod_deleted_at: record.mod_deleted_at, + })) +} + +async fn select_include_deleted( + weight_token: &ModelWeightToken, + mysql_connection: &mut PoolConnection +) -> Result { + sqlx + ::query_as!( + RawWeight, + r#" + SELECT + wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`, + wt.title, + wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`, + wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`, + wt.maybe_description_markdown, + wt.maybe_description_rendered_html, + + wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`, + users.username as creator_username, + users.display_name as creator_display_name, + users.email_gravatar_hash AS creator_gravatar_hash, + + wt.creator_ip_address, + wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`, + wt.original_download_url, + wt.original_filename, + wt.file_size_bytes, + wt.file_checksum_sha2, + + wt.public_bucket_hash, + wt.maybe_public_bucket_prefix, + wt.maybe_public_bucket_extension, + + cover_image.public_bucket_directory_hash as maybe_cover_image_public_bucket_hash, + cover_image.maybe_public_bucket_prefix as maybe_cover_image_public_bucket_prefix, + cover_image.maybe_public_bucket_extension as maybe_cover_image_public_bucket_extension, + + entity_stats.ratings_positive_count as maybe_ratings_positive_count, + entity_stats.ratings_negative_count as maybe_ratings_negative_count, + entity_stats.bookmark_count as maybe_bookmark_count, + + w_extension.ietf_language_tag as maybe_ietf_language_tag, + w_extension.ietf_primary_language_subtag as maybe_ietf_primary_language_subtag, + w_extension.maybe_default_pretrained_vocoder, + w_extension.text_pipeline_type as maybe_text_pipeline_type, + w_extension.use_default_mel_multiply_factor, + w_extension.maybe_custom_mel_multiply_factor, + + w_extension.maybe_custom_vocoder_token, + + vocoder.vocoder_type as `maybe_custom_vocoder_type: enums::common::vocoder_type::VocoderType`, + vocoder.title as maybe_custom_vocoder_title, + vocoder.private_bucket_hash as maybe_custom_vocoder_private_bucket_hash, + + wt.version, + wt.created_at, + wt.updated_at, + wt.user_deleted_at, + wt.mod_deleted_at + + FROM model_weights as wt + JOIN users + ON users.token = wt.creator_user_token + LEFT OUTER JOIN media_files as cover_image + ON cover_image.token = wt.maybe_cover_image_media_file_token + LEFT OUTER JOIN entity_stats + ON entity_stats.entity_type = "model_weight" + AND entity_stats.entity_token = wt.token + + LEFT OUTER JOIN model_weights_extension_tts_details as w_extension + ON w_extension.model_weights_token = wt.token + + LEFT OUTER JOIN vocoder_models AS vocoder + ON vocoder.token = w_extension.maybe_custom_vocoder_token + + WHERE + wt.token = ? + "#, + weight_token.as_str() + ) + .fetch_one(&mut **mysql_connection).await +} + +#[derive(Serialize)] +pub struct RawWeight { + token: ModelWeightToken, + title: String, + weights_type: WeightsType, + weights_category: WeightsCategory, + maybe_description_markdown: Option, + maybe_description_rendered_html: Option, + + creator_user_token: UserToken, + creator_username: String, + creator_display_name: String, + creator_gravatar_hash: String, + + creator_ip_address: String, + creator_set_visibility: Visibility, + maybe_last_update_user_token: Option, + original_download_url: Option, + original_filename: Option, + file_size_bytes: i32, + file_checksum_sha2: String, + public_bucket_hash: String, + maybe_public_bucket_prefix: Option, + maybe_public_bucket_extension: Option, + + maybe_cover_image_public_bucket_hash: Option, + maybe_cover_image_public_bucket_prefix: Option, + maybe_cover_image_public_bucket_extension: Option, + + maybe_ratings_positive_count: Option, + maybe_ratings_negative_count: Option, + maybe_bookmark_count: Option, + + maybe_ietf_language_tag: Option, + maybe_ietf_primary_language_subtag: Option, + maybe_default_pretrained_vocoder: Option, + maybe_text_pipeline_type: Option, + + // Joined custom vocoder fields + maybe_custom_vocoder_token: Option, + maybe_custom_vocoder_type: Option, + maybe_custom_vocoder_title: Option, + maybe_custom_vocoder_private_bucket_hash: Option, + + use_default_mel_multiply_factor: Option, // bool; note this was *not* optional before, but is because of our join + maybe_custom_mel_multiply_factor: Option, + + version: i32, + created_at: DateTime, + updated_at: DateTime, + user_deleted_at: Option>, + mod_deleted_at: Option>, +} diff --git a/crates/lib/mysql_queries/src/queries/model_weights/get/mod.rs b/crates/lib/mysql_queries/src/queries/model_weights/get/mod.rs index 2a6378f982..998f00bea6 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/get/mod.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/get/mod.rs @@ -1,3 +1,4 @@ pub mod get_weight; pub mod get_weight_for_legacy_tts_enqueue; +pub mod get_weight_for_legacy_tts_inference; pub mod get_weight_for_legacy_tts_info; diff --git a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs index 57e99ae5b5..ee440a992a 100644 --- a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs +++ b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs @@ -6,6 +6,7 @@ use log::{info, warn}; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::tts_models::tts_model_type::TtsModelType; use filesys::file_exists::file_exists; +use migration::text_to_speech::get_tts_model_for_run_inference_migration::{get_tts_model_for_run_inference_migration, TtsModelForRunInferenceMigration}; use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelError, VcModelType}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord}; @@ -27,7 +28,7 @@ pub struct DependencyStatus { pub enum MaybeInferenceModel { None, - TtsModel(TtsModelForInferenceRecord), + TtsModel(TtsModelForRunInferenceMigration), VcModel(VcModel) } @@ -64,13 +65,13 @@ fn get_model_token_and_path(job_dependencies: &JobDependencies, maybe_model: &Ma // TODO(bt,2023-05-01): Also check other paths (user-supplied vocoders, etc.) let maybe_filesystem_path = match maybe_model { MaybeInferenceModel::TtsModel(ref model) => { - match model.tts_model_type { + match model.tts_model_type() { TtsModelType::Tacotron2 => { - maybe_model_token = Some(model.model_token.to_string()); + maybe_model_token = Some(model.token().to_string()); Some(job_dependencies .fs .semi_persistent_cache - .tts_synthesizer_model_path(model.model_token.as_str())) + .tts_synthesizer_model_path(model.token())) } TtsModelType::Vits => { None @@ -123,8 +124,8 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie match maybe_model { Some(model) => MaybeInferenceModel::TtsModel(model), None => { - let maybe_tts_model = get_tts_model_for_inference_improved( - &job_dependencies.db.mysql_pool, token) + let maybe_tts_model = get_tts_model_for_run_inference_migration( + token, &job_dependencies.db.mysql_pool) .await .map_err(|err| match err { TtsModelForInferenceError::ModelDeleted => ProcessSingleJobError::ModelDeleted, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 495901d1dc..9b5c3a06c2 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -3,8 +3,8 @@ use log::warn; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use enums::by_table::tts_models::tts_model_type::TtsModelType; +use migration::text_to_speech::get_tts_model_for_run_inference_migration::get_tts_model_for_run_inference_migration; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::get_tts_model_for_inference_improved; use crate::job::job_loop::job_success_result::JobSuccessResult; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; @@ -88,9 +88,9 @@ async fn dispatch_fine_tuned_weights_model( .ok_or(ProcessSingleJobError::Other(anyhow!("no model token on job")))?; // TODO(bt,2023-10-09): Interrogate TTS model cache before querying database. - let maybe_tts_model = get_tts_model_for_inference_improved( - &job_dependencies.db.mysql_pool, + let maybe_tts_model = get_tts_model_for_run_inference_migration( tts_model_token, + &job_dependencies.db.mysql_pool, ).await.map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?; let tts_model = match maybe_tts_model { @@ -100,7 +100,7 @@ async fn dispatch_fine_tuned_weights_model( Some(tts_model) => tts_model, }; - match tts_model.tts_model_type { + match tts_model.tts_model_type() { TtsModelType::Vits => { vits::process_job::process_job(VitsProcessJobArgs { job_dependencies, diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index c2e16d1e72..572ae20156 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -11,9 +11,9 @@ use errors::AnyhowResult; use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; +use migration::text_to_speech::get_tts_model_for_run_inference_migration::TtsModelForRunInferenceMigration; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; use tts_common::clean_symbols::clean_symbols; use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; use tts_common::text_pipelines::text_pipeline_type::TextPipelineType; @@ -36,7 +36,7 @@ const TEST_REQUEST_TEXT: &str = "This is a test request."; pub struct ProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, - pub tts_model: &'a TtsModelForInferenceRecord, + pub tts_model: &'a TtsModelForRunInferenceMigration, pub raw_inference_text: &'a str, } @@ -111,7 +111,7 @@ async fn process_job_with_cleanup( // ==================== CONFIRM OR DOWNLOAD OPTIONAL CUSTOM VOCODER MODEL ==================== // - let custom_vocoder_fs_path = match &tts_model.maybe_custom_vocoder { + let custom_vocoder_fs_path = match tts_model.maybe_custom_vocoder() { None => None, Some(vocoder) => { let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token); @@ -136,14 +136,20 @@ async fn process_job_with_cleanup( // ==================== CONFIRM OR DOWNLOAD TTS SYNTHESIZER MODEL ==================== // let tts_synthesizer_fs_path = { - let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str()); - let tts_synthesizer_object_path = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash); + let bucket_client = if tts_model.is_private_bucket() { + &args.job_dependencies.buckets.private_bucket_client + } else { + &args.job_dependencies.buckets.public_bucket_client + }; + + let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.token()); + let tts_synthesizer_object_path = tts_model.bucket_object_path(&args.job_dependencies.buckets.bucket_path_unifier); maybe_download_file_from_bucket(MaybeDownloadArgs { name_or_description_of_file: "synthesizer", final_filesystem_file_path: & tts_synthesizer_fs_path, bucket_object_path: &tts_synthesizer_object_path, - bucket_client: &args.job_dependencies.buckets.private_bucket_client, + bucket_client, job_progress_reporter: &mut job_progress_reporter, job_progress_update_description: "downloading synthesizer", job_id: job.id.0, @@ -183,7 +189,7 @@ async fn process_job_with_cleanup( info!("Expected output metadata filename: {:?}", &output_metadata_fs_path); let mut pretrained_vocoder = VocoderType::HifiGanSuperResolution; - if let Some(default_vocoder) = tts_model.maybe_default_pretrained_vocoder.as_deref() { + if let Some(default_vocoder) = tts_model.maybe_default_pretrained_vocoder().as_deref() { pretrained_vocoder = VocoderType::from_str(default_vocoder) .map_err(|e| ProcessSingleJobError::Other(e))?; } @@ -214,14 +220,14 @@ async fn process_job_with_cleanup( }; }; - let text_pipeline_type_or_guess = tts_model.text_pipeline_type + let text_pipeline_type_or_guess = tts_model.text_pipeline_type() .as_deref() // NB: If there's an error deserializing, turn it to None. .and_then(|pipeline_type| TextPipelineType::from_str(pipeline_type).ok()) - .unwrap_or_else(|| guess_text_pipeline_heuristic(Some(tts_model.created_at))); + .unwrap_or_else(|| guess_text_pipeline_heuristic(Some(tts_model.created_at().clone()))); info!("With text pipeline type `{:?} ` (or guess: {:?})", - &tts_model.text_pipeline_type, + &tts_model.text_pipeline_type(), &text_pipeline_type_or_guess); // NB: Tacotron operates on decoder steps. 1000 steps is the default and correlates to @@ -232,9 +238,9 @@ async fn process_job_with_cleanup( let mut maybe_mel_multiply_factor = None; - if let Some(factor) = tts_model.maybe_custom_mel_multiply_factor { + if let Some(factor) = tts_model.maybe_custom_mel_multiply_factor() { maybe_mel_multiply_factor = Some(MelMultiplyFactor::CustomMultiplyFactor(factor)); - } else if tts_model.use_default_mel_multiply_factor { + } else if tts_model.use_default_mel_multiply_factor() { maybe_mel_multiply_factor = Some(MelMultiplyFactor::DefaultMultiplyFactor); } @@ -262,8 +268,8 @@ async fn process_job_with_cleanup( &output_spectrogram_fs_path, &output_metadata_fs_path, maybe_unload_model_path, - tts_model.use_default_mel_multiply_factor, - tts_model.maybe_custom_mel_multiply_factor, + tts_model.use_default_mel_multiply_factor(), + tts_model.maybe_custom_mel_multiply_factor(), ).await.map_err(|e| ProcessSingleJobError::Other(e))?; } else { info!("Shelling out for inference..."); @@ -322,7 +328,7 @@ async fn process_job_with_cleanup( args.job_dependencies.clients.firehose_publisher.tts_inference_finished( job.maybe_creator_user_token.as_deref(), - tts_model.model_token.as_str(), + tts_model.token(), &inference_result.entity_token) .await .map_err(|e| { diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs index 0b24e58b74..76dfc03b35 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs @@ -21,6 +21,7 @@ pub struct Tacotron2Dependencies { /// dependencies that are for the sidecar version of TT2 pub sidecar: SidecarDeps, + /// Rollout flag for uploading results as `media_files` instead of `tts_results`. pub upload_as_media_file: bool, } diff --git a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs index 336631315d..f6025f8a04 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs @@ -13,9 +13,9 @@ use filesys::check_file_exists::check_file_exists; use filesys::safe_delete_temp_directory::safe_delete_temp_directory; use filesys::safe_delete_temp_file::safe_delete_temp_file; use hashing::sha256::sha256_hash_string::sha256_hash_string; +use migration::text_to_speech::get_tts_model_for_run_inference_migration::TtsModelForRunInferenceMigration; use mysql_queries::column_types::vocoder_type::VocoderType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; -use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord; use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType}; use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity}; @@ -32,7 +32,7 @@ const TEST_REQUEST_TEXT: &str = "This is a test request."; pub struct VitsProcessJobArgs<'a> { pub job_dependencies: &'a JobDependencies, pub job: &'a AvailableInferenceJob, - pub tts_model: &'a TtsModelForInferenceRecord, + pub tts_model: &'a TtsModelForRunInferenceMigration, pub raw_inference_text: &'a str, } @@ -63,12 +63,12 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result) -> Result, + pub tts_model_record_cache: MultiItemTtlCache, pub vc_model_record_cache: MultiItemTtlCache, /// Skip processing models if they're not on the filesystem. diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs index 9b57494881..a84eaced54 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs @@ -20,7 +20,7 @@ use enums::common::visibility::Visibility; use http_server_common::request::get_request_api_token::get_request_api_token; use http_server_common::request::get_request_header_optional::get_request_header_optional; use http_server_common::request::get_request_ip::get_request_ip; -use migration::text_to_speech::get_tts_model_for_inference_migration::TtsModelForInferenceMigration; +use migration::text_to_speech::get_tts_model_for_enqueue_inference_migration::TtsModelForEnqueueInferenceMigration; use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated}; use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use mysql_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder; @@ -553,7 +553,7 @@ pub fn validate_inference_text(text: &str, plan: &Plan) -> Result<(), String> { /// This is designed to fail closed (read: actually open!) rather than hit resources. async fn check_if_authorized_to_use_model( maybe_user_session: Option<&UserSessionExtended>, - tts_model: &TtsModelForInferenceMigration, + tts_model: &TtsModelForEnqueueInferenceMigration, ) -> bool { match tts_model.creator_set_visibility() { diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/get_tts_model_with_caching.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/get_tts_model_with_caching.rs index 45905e0af4..5529212d7e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/get_tts_model_with_caching.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/get_tts_model_with_caching.rs @@ -3,7 +3,7 @@ use sqlx::MySql; use sqlx::pool::PoolConnection; use errors::AnyhowResult; -use migration::text_to_speech::get_tts_model_for_inference_migration::{get_tts_model_for_inference_migration, TtsModelForInferenceMigration}; +use migration::text_to_speech::get_tts_model_for_enqueue_inference_migration::{get_tts_model_for_enqueue_inference_migration, TtsModelForEnqueueInferenceMigration}; use redis_caching::redis_ttl_cache::RedisTtlCache; use redis_common::redis_cache_keys::RedisCacheKeys; @@ -14,14 +14,14 @@ pub async fn get_tts_model_with_caching( redis_ttl_cache: &RedisTtlCache, mysql_connection: &mut PoolConnection, use_weights_table: bool, -) -> AnyhowResult> { +) -> AnyhowResult> { // NB: Copy due to move. Not a cloned ref due to clippy lint let model_token2 = model_token.to_string(); let get_tts_model = move || { // NB: async closures are not yet stable in Rust, so we include an async block. async move { - get_tts_model_for_inference_migration( + get_tts_model_for_enqueue_inference_migration( &model_token2, mysql_connection, true,