Merge pull request #289 from storytold/brandon-migrate-tts-inference

migrate TTS inference to weights

crates/lib/migration/src/text_to_speech/get_tts_model_for_enqueue_inference_migration.rs

@@ -4,18 +4,18 @@ use sqlx::pool::PoolConnection;
 
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
-use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_enqueue::{get_weight_for_legacy_tts_enqueue_with_connection, ModelWeightForLegacyTtsInference};
+use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_enqueue::{get_weight_for_legacy_tts_enqueue_with_connection, ModelWeightForLegacyTtsEnqueue};
 use mysql_queries::queries::tts::tts_models::get_tts_model::{get_tts_model_by_token_using_connection, TtsModelRecord};
 use tokens::tokens::model_weights::ModelWeightToken;
 
 /// Get TTS model
 /// This is for the tts inference page.
-pub async fn get_tts_model_for_inference_migration(
+pub async fn get_tts_model_for_enqueue_inference_migration(
   token: &str,
   mysql_connection: &mut PoolConnection<MySql>,
   can_see_deleted: bool,
   use_weights_table: bool,
-) -> AnyhowResult<Option<TtsModelForInferenceMigration>> {
+) -> AnyhowResult<Option<TtsModelForEnqueueInferenceMigration>> {
   // NB: This is temporary migration code as we switch from the `tts_models` table to the `model_weights` table.
   if use_weights_table {
     let token = ModelWeightToken::new_from_str(token);
@@ -26,7 +26,7 @@ pub async fn get_tts_model_for_inference_migration(
       mysql_connection
     ).await?;
 
-    Ok(maybe_model.map(|model| TtsModelForInferenceMigration::ModelWeight(model)))
+    Ok(maybe_model.map(|model| TtsModelForEnqueueInferenceMigration::ModelWeight(model)))
 
   } else {
 
@@ -36,21 +36,21 @@ pub async fn get_tts_model_for_inference_migration(
       mysql_connection
     ).await?;
 
-    Ok(maybe_model.map(|model| TtsModelForInferenceMigration::LegacyTts(model)))
+    Ok(maybe_model.map(|model| TtsModelForEnqueueInferenceMigration::LegacyTts(model)))
   }
 }
 
 /// Union over the legacy table and the new table to support an easier migration.
 /// This enum can hold a record of either type and present a unified accessor interface.
 #[derive(Clone, Serialize, Deserialize)]
-pub enum TtsModelForInferenceMigration {
+pub enum TtsModelForEnqueueInferenceMigration {
   /// Old type from the `tts_models` table, on the way out
   LegacyTts(TtsModelRecord),
   /// New type, replacing the `tts_models` table.
-  ModelWeight(ModelWeightForLegacyTtsInference),
+  ModelWeight(ModelWeightForLegacyTtsEnqueue),
 }
 
-impl TtsModelForInferenceMigration {
+impl TtsModelForEnqueueInferenceMigration {
   pub fn token(&self) -> &str {
     match self {
       Self::LegacyTts(ref model) => model.model_token.as_str(),

crates/lib/migration/src/text_to_speech/get_tts_model_for_run_inference_migration.rs

@@ -0,0 +1,200 @@
+use std::path::PathBuf;
+
+use chrono::{DateTime, Utc};
+use sqlx::MySqlPool;
+
+use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
+use cloud_storage::bucket_path_unifier::BucketPathUnifier;
+use enums::by_table::tts_models::tts_model_type::TtsModelType;
+use mysql_queries::queries::model_weights::get::get_weight_for_legacy_tts_inference::{get_weight_for_legacy_tts_inference, ModelWeightForLegacyTtsInference};
+use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{CustomVocoderFields, get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord};
+use tokens::tokens::model_weights::ModelWeightToken;
+use tokens::tokens::users::UserToken;
+
+/// Get TTS model
+/// This is for the inference inference job.
+pub async fn get_tts_model_for_run_inference_migration(
+  tts_model_token: &str,
+  mysql_pool: &MySqlPool,
+) -> Result<Option<TtsModelForRunInferenceMigration>, TtsModelForInferenceError> {
+  // NB: This is temporary migration code as we switch from the `tts_models` table to the `model_weights` table.
+  if tts_model_token.starts_with(ModelWeightToken::token_prefix()) {
+    let token = ModelWeightToken::new_from_str(tts_model_token);
+
+    let maybe_model = get_weight_for_legacy_tts_inference(
+      &token,
+      mysql_pool
+    ).await?;
+
+    Ok(maybe_model.map(|model| TtsModelForRunInferenceMigration::ModelWeight(model)))
+  } else {
+
+    let maybe_model = get_tts_model_for_inference_improved(
+      mysql_pool,
+      &tts_model_token,
+    ).await?;
+
+    Ok(maybe_model.map(|model| TtsModelForRunInferenceMigration::LegacyTts(model)))
+  }
+}
+
+/// Migration wrapper type.
+/// Union over the legacy table and the new table to support an easier migration.
+/// This enum can hold a record of either type and present a unified accessor interface.
+#[derive(Clone)]
+pub enum TtsModelForRunInferenceMigration {
+  /// Old type from the `tts_models` table, on the way out
+  LegacyTts(TtsModelForInferenceRecord),
+  /// New type, replacing the `tts_models` table.
+  ModelWeight(ModelWeightForLegacyTtsInference),
+}
+
+impl TtsModelForRunInferenceMigration {
+  pub fn token(&self) -> &str {
+    match self {
+      Self::LegacyTts(ref model) => model.model_token.as_str(),
+      Self::ModelWeight(ref model) => model.token.as_str(),
+    }
+  }
+
+  pub fn title(&self) -> &str {
+    match self {
+      Self::LegacyTts(ref model) => &model.title,
+      Self::ModelWeight(ref model) => &model.title,
+    }
+  }
+
+  pub fn creator_user_token(&self) -> &UserToken {
+    match self {
+      Self::LegacyTts(ref model) => &model.creator_user_token,
+      Self::ModelWeight(ref model) => &model.creator_user_token,
+    }
+  }
+
+  pub fn creator_username(&self) -> &str {
+    match self {
+      Self::LegacyTts(ref model) => &model.creator_username,
+      Self::ModelWeight(ref model) => &model.creator_username,
+    }
+  }
+
+  pub fn creator_display_name(&self) -> &str {
+    match self {
+      Self::LegacyTts(ref model) => &model.creator_display_name,
+      Self::ModelWeight(ref model) => &model.creator_display_name,
+    }
+  }
+
+  pub fn created_at(&self) -> &DateTime<Utc> {
+    match self {
+      Self::LegacyTts(ref model) => &model.created_at,
+      Self::ModelWeight(ref model) => &model.created_at,
+    }
+  }
+
+  pub fn updated_at(&self) -> &DateTime<Utc> {
+    match self {
+      Self::LegacyTts(ref model) => &model.updated_at,
+      Self::ModelWeight(ref model) => &model.updated_at,
+    }
+  }
+
+  pub fn mod_deleted_at(&self) -> Option<&DateTime<Utc>> {
+    match self {
+      Self::LegacyTts(ref model) => model.mod_deleted_at.as_ref(),
+      Self::ModelWeight(ref model) => model.mod_deleted_at.as_ref(),
+    }
+  }
+
+  pub fn user_deleted_at(&self) -> Option<&DateTime<Utc>> {
+    match self {
+      Self::LegacyTts(ref model) => model.user_deleted_at.as_ref(),
+      Self::ModelWeight(ref model) => model.user_deleted_at.as_ref(),
+    }
+  }
+
+  pub fn tts_model_type(&self) -> TtsModelType {
+    match self {
+      Self::LegacyTts(ref model) => model.tts_model_type,
+      Self::ModelWeight(ref _model) => TtsModelType::Tacotron2, // NB: Always TT2
+    }
+  }
+
+  pub fn maybe_default_pretrained_vocoder(&self) -> Option<&str> {
+    match self {
+      Self::LegacyTts(ref model) => model.maybe_default_pretrained_vocoder.as_deref(),
+      Self::ModelWeight(ref model) => model.maybe_default_pretrained_vocoder.as_deref(),
+    }
+  }
+
+  /* NB(bt, 2024-01-20): Breakdown of the text_pipeline_type values in the DB:
+  mysql> select text_pipeline_type , count(*) from tts_models group by text_pipeline_type ;
+    +--------------------+----------+
+    | text_pipeline_type | count(*) |
+    +--------------------+----------+
+    | NULL               |     4406 |
+    | legacy_fakeyou     |     2572 |
+    | english_v1         |     1548 |
+    | spanish_v2         |      194 |
+    +--------------------+----------+
+   */
+  pub fn text_pipeline_type(&self) -> Option<&str> {
+    match self {
+      Self::LegacyTts(ref model) => model.text_pipeline_type.as_deref(),
+      Self::ModelWeight(ref model) => model.maybe_text_pipeline_type.as_deref(),
+    }
+  }
+
+  pub fn use_default_mel_multiply_factor(&self) -> bool {
+    match self {
+      Self::LegacyTts(ref model) => model.use_default_mel_multiply_factor,
+      Self::ModelWeight(ref model) => model.use_default_mel_multiply_factor,
+    }
+  }
+
+  pub fn maybe_custom_mel_multiply_factor(&self) -> Option<f64> {
+    match self {
+      Self::LegacyTts(ref model) => model.maybe_custom_mel_multiply_factor,
+      Self::ModelWeight(ref model) => model.maybe_custom_mel_multiply_factor,
+    }
+  }
+
+  pub fn maybe_custom_vocoder(&self) -> Option<&CustomVocoderFields> {
+    match self {
+      Self::LegacyTts(ref model) => model.maybe_custom_vocoder.as_ref(),
+      Self::ModelWeight(ref model) => model.maybe_custom_vocoder.as_ref(),
+    }
+  }
+
+  /// Which bucket to download the weights from
+  pub fn is_private_bucket(&self) -> bool {
+    match self {
+      Self::LegacyTts(ref _model) => true,
+      Self::ModelWeight(ref _model) => false,
+    }
+  }
+
+  /// Path to the weights in the bucket
+  /// The old and new schemes are quite different.
+  pub fn bucket_object_path(&self, bucket_path_unifier: &BucketPathUnifier) -> PathBuf {
+    match self {
+      Self::LegacyTts(ref model) => bucket_path_unifier.tts_synthesizer_path(&model.private_bucket_hash),
+      Self::ModelWeight(ref model) => {
+        let path = WeightFileBucketPath::from_object_hash(
+          &model.public_bucket_hash,
+          model.maybe_public_bucket_prefix.as_deref(),
+          model.maybe_public_bucket_extension.as_deref());
+
+        PathBuf::from(path.get_full_object_path_str())
+      },
+    }
+  }
+
+  // NB: Just for vits, which we never supported anyway
+  pub fn vits_traced_synthesizer_object_path(&self, bucket_path_unifier: &BucketPathUnifier) -> PathBuf {
+    match self {
+      Self::LegacyTts(ref model) => bucket_path_unifier.tts_traced_synthesizer_path(&model.private_bucket_hash),
+      Self::ModelWeight(ref _model) => PathBuf::from("VITS_UNSUPPORTED_FOR_MODEL_WEIGHTS"),
+    }
+  }
+}

crates/lib/migration/src/text_to_speech/mod.rs

@@ -1,3 +1,4 @@
-pub mod get_tts_model_for_inference_migration;
+pub mod get_tts_model_for_enqueue_inference_migration;
+pub mod get_tts_model_for_run_inference_migration;
 pub mod get_tts_model_info_migration;
 pub mod list_tts_models_for_migration;

crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_enqueue.rs

@@ -11,7 +11,7 @@ use enums::common::visibility::Visibility;
 use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
 
 #[derive(Clone, Serialize, Deserialize)]
-pub struct ModelWeightForLegacyTtsInference {
+pub struct ModelWeightForLegacyTtsEnqueue {
     pub token: ModelWeightToken,
     pub weights_type: WeightsType,
     pub weights_category: WeightsCategory,
@@ -27,7 +27,7 @@ pub async fn get_weight_for_legacy_tts_enqueue(
     weight_token: &ModelWeightToken,
     can_see_deleted: bool,
     mysql_pool: &MySqlPool
-) -> AnyhowResult<Option<ModelWeightForLegacyTtsInference>> {
+) -> AnyhowResult<Option<ModelWeightForLegacyTtsEnqueue>> {
     let mut connection = mysql_pool.acquire().await?;
     get_weight_for_legacy_tts_enqueue_with_connection(weight_token, can_see_deleted, &mut connection).await
 }
@@ -36,7 +36,7 @@ pub async fn get_weight_for_legacy_tts_enqueue_with_connection(
     weight_token: &ModelWeightToken,
     can_see_deleted: bool,
     mysql_connection: &mut PoolConnection<MySql>
-) -> AnyhowResult<Option<ModelWeightForLegacyTtsInference>> {
+) -> AnyhowResult<Option<ModelWeightForLegacyTtsEnqueue>> {
     let maybe_result = if can_see_deleted {
         select_include_deleted(weight_token, mysql_connection).await
     } else {
@@ -54,19 +54,15 @@ pub async fn get_weight_for_legacy_tts_enqueue_with_connection(
         }
     };
 
-    // unwrap the result
-
-    Ok(
-        Some(ModelWeightForLegacyTtsInference {
-            token: record.token,
-            weights_type: record.weights_type,
-            weights_category: record.weights_category,
-            creator_user_token: record.creator_user_token,
-            creator_set_visibility: record.creator_set_visibility,
-            user_deleted_at: record.user_deleted_at,
-            mod_deleted_at: record.mod_deleted_at,
-        })
-    )
+    Ok(Some(ModelWeightForLegacyTtsEnqueue {
+        token: record.token,
+        weights_type: record.weights_type,
+        weights_category: record.weights_category,
+        creator_user_token: record.creator_user_token,
+        creator_set_visibility: record.creator_set_visibility,
+        user_deleted_at: record.user_deleted_at,
+        mod_deleted_at: record.mod_deleted_at,
+    }))
 }
 
 async fn select_include_deleted(

crates/lib/mysql_queries/src/queries/model_weights/get/get_weight_for_legacy_tts_inference.rs

@@ -0,0 +1,306 @@
+use chrono::{DateTime, Utc};
+use log::error;
+use sqlx::{MySql, MySqlPool};
+use sqlx::pool::PoolConnection;
+
+use enums::by_table::model_weights::weights_category::WeightsCategory;
+use enums::by_table::model_weights::weights_types::WeightsType;
+use enums::common::visibility::Visibility;
+use enums::common::vocoder_type::VocoderType;
+use tokens::tokens::{model_weights::ModelWeightToken, users::UserToken};
+
+use crate::helpers::boolean_converters::nullable_i8_to_bool_default_false;
+use crate::queries::tts::tts_models::get_tts_model_for_inference_improved::{CustomVocoderFields, TtsModelForInferenceError};
+
+#[derive(Clone)]
+pub struct ModelWeightForLegacyTtsInference {
+    pub token: ModelWeightToken,
+    pub title: String,
+    pub weights_type: WeightsType,
+    pub weights_category: WeightsCategory,
+    pub maybe_description_markdown: Option<String>,
+    pub maybe_description_rendered_html: Option<String>,
+
+    pub creator_user_token: UserToken,
+    pub creator_username: String,
+    pub creator_display_name: String,
+    pub creator_gravatar_hash: String,
+
+    pub creator_ip_address: String,
+    pub creator_set_visibility: Visibility,
+
+    pub maybe_last_update_user_token: Option<UserToken>,
+    pub original_download_url: Option<String>,
+    pub original_filename: Option<String>,
+    pub file_size_bytes: i32,
+    pub file_checksum_sha2: String,
+    pub public_bucket_hash: String,
+    pub maybe_public_bucket_prefix: Option<String>,
+    pub maybe_public_bucket_extension: Option<String>,
+
+    pub maybe_cover_image_public_bucket_hash: Option<String>,
+    pub maybe_cover_image_public_bucket_prefix: Option<String>,
+    pub maybe_cover_image_public_bucket_extension: Option<String>,
+
+    pub maybe_ratings_positive_count: Option<u32>,
+    pub maybe_ratings_negative_count: Option<u32>,
+    pub maybe_bookmark_count: Option<u32>,
+
+    pub maybe_ietf_language_tag: Option<String>,
+    pub maybe_ietf_primary_language_subtag: Option<String>,
+    pub maybe_default_pretrained_vocoder: Option<String>,
+    pub maybe_text_pipeline_type: Option<String>,
+
+    // Joined custom vocoder fields
+    pub maybe_custom_vocoder: Option<CustomVocoderFields>,
+
+    //// Joined custom vocoder fields
+    //pub maybe_custom_vocoder_token: Option<String>,
+    //pub maybe_custom_vocoder_type: Option<VocoderType>,
+    //pub maybe_custom_vocoder_title: Option<String>,
+    //pub maybe_custom_vocoder_private_bucket_hash: Option<String>,
+
+    pub use_default_mel_multiply_factor: bool,
+    pub maybe_custom_mel_multiply_factor: Option<f64>,
+
+    pub version: i32,
+    pub created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    pub user_deleted_at: Option<DateTime<Utc>>,
+    pub mod_deleted_at: Option<DateTime<Utc>>,
+}
+
+pub async fn get_weight_for_legacy_tts_inference(
+    weight_token: &ModelWeightToken,
+    mysql_pool: &MySqlPool
+) -> Result<Option<ModelWeightForLegacyTtsInference>, TtsModelForInferenceError> {
+    let mut connection = mysql_pool.acquire()
+        .await
+        .map_err(|err| {
+            error!("Error acquiring db connection from pool: {:?}", err);
+            return TtsModelForInferenceError::DatabaseError {
+                reason: format!("Mysql connection error: {:?}", err)
+            };
+        })?;
+
+    get_weight_for_legacy_tts_inference_with_connection(weight_token, &mut connection).await
+}
+
+pub async fn get_weight_for_legacy_tts_inference_with_connection(
+    weight_token: &ModelWeightToken,
+    mysql_connection: &mut PoolConnection<MySql>
+) -> Result<Option<ModelWeightForLegacyTtsInference>, TtsModelForInferenceError> {
+    let maybe_result = select_include_deleted(weight_token, mysql_connection).await;
+
+    let record: RawWeight = match maybe_result {
+        Ok(record) => record,
+        Err(sqlx::Error::RowNotFound) => {
+            return Ok(None);
+        }
+        Err(err) => {
+            error!("Error fetching weights by token: {:?}", err);
+            return Err(TtsModelForInferenceError::DatabaseError {
+                reason: format!("Mysql error: {:?}", err)
+            });
+        }
+    };
+
+    if record.mod_deleted_at.is_some() || record.user_deleted_at.is_some() {
+        return Err(TtsModelForInferenceError::ModelDeleted);
+    }
+
+    Ok(Some(ModelWeightForLegacyTtsInference {
+        token: record.token,
+        title: record.title,
+        weights_type: record.weights_type,
+        weights_category: record.weights_category,
+        maybe_description_markdown: record.maybe_description_markdown,
+        maybe_description_rendered_html: record.maybe_description_rendered_html,
+        creator_user_token: record.creator_user_token,
+        creator_username: record.creator_username,
+        creator_display_name: record.creator_display_name,
+        creator_gravatar_hash: record.creator_gravatar_hash,
+        creator_ip_address: record.creator_ip_address,
+        creator_set_visibility: record.creator_set_visibility,
+        maybe_last_update_user_token: record.maybe_last_update_user_token,
+        original_download_url: record.original_download_url,
+        original_filename: record.original_filename,
+        file_size_bytes: record.file_size_bytes,
+        file_checksum_sha2: record.file_checksum_sha2,
+        public_bucket_hash: record.public_bucket_hash,
+        maybe_public_bucket_prefix: record.maybe_public_bucket_prefix,
+        maybe_public_bucket_extension: record.maybe_public_bucket_extension,
+        maybe_cover_image_public_bucket_hash: record.maybe_cover_image_public_bucket_hash,
+        maybe_cover_image_public_bucket_prefix: record.maybe_cover_image_public_bucket_prefix,
+        maybe_cover_image_public_bucket_extension: record.maybe_cover_image_public_bucket_extension,
+        maybe_ratings_positive_count: record.maybe_ratings_positive_count,
+        maybe_ratings_negative_count: record.maybe_ratings_negative_count,
+        maybe_bookmark_count: record.maybe_bookmark_count,
+        maybe_ietf_language_tag: record.maybe_ietf_language_tag,
+        maybe_ietf_primary_language_subtag: record.maybe_ietf_primary_language_subtag,
+        maybe_default_pretrained_vocoder: record.maybe_default_pretrained_vocoder,
+        maybe_text_pipeline_type: record.maybe_text_pipeline_type,
+        //maybe_custom_vocoder_token: record.maybe_custom_vocoder_token,
+        //maybe_custom_vocoder_type: record.maybe_custom_vocoder_type,
+        //maybe_custom_vocoder_title: record.maybe_custom_vocoder_title,
+        //maybe_custom_vocoder_private_bucket_hash: record.maybe_custom_vocoder_private_bucket_hash,
+        maybe_custom_vocoder: match record.maybe_custom_vocoder_token {
+            // NB: We're relying on a single field's presence to infer that the others vocoder fields
+            // are also there. If for some reason they aren't, fail open.
+            None => None,
+            Some(vocoder_token) => Some(CustomVocoderFields {
+                vocoder_token,
+                vocoder_type: record.maybe_custom_vocoder_type.ok_or(
+                    TtsModelForInferenceError::DatabaseError { reason: "custom_vocoder_type field error".to_string() })?,
+                vocoder_title: record.maybe_custom_vocoder_title.ok_or(
+                    TtsModelForInferenceError::DatabaseError { reason: "custom_vocoder_title field error".to_string() })?,
+                vocoder_private_bucket_hash: record.maybe_custom_vocoder_private_bucket_hash.ok_or(
+                    TtsModelForInferenceError::DatabaseError { reason: "vocoder_private_bucket_hash field error".to_string() })?,
+            })
+        },
+        use_default_mel_multiply_factor: nullable_i8_to_bool_default_false(record.use_default_mel_multiply_factor),
+        maybe_custom_mel_multiply_factor: record.maybe_custom_mel_multiply_factor,
+        version: record.version,
+        created_at: record.created_at,
+        updated_at: record.updated_at,
+        user_deleted_at: record.user_deleted_at,
+        mod_deleted_at: record.mod_deleted_at,
+    }))
+}
+
+async fn select_include_deleted(
+    weight_token: &ModelWeightToken,
+    mysql_connection: &mut PoolConnection<MySql>
+) -> Result<RawWeight, sqlx::Error> {
+    sqlx
+        ::query_as!(
+            RawWeight,
+            r#"
+        SELECT
+        wt.token as `token: tokens::tokens::model_weights::ModelWeightToken`,
+        wt.title,
+        wt.weights_type as `weights_type: enums::by_table::model_weights::weights_types::WeightsType`,
+        wt.weights_category as `weights_category: enums::by_table::model_weights::weights_category::WeightsCategory`,
+        wt.maybe_description_markdown,
+        wt.maybe_description_rendered_html,
+
+        wt.creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
+        users.username as creator_username,
+        users.display_name as creator_display_name,
+        users.email_gravatar_hash AS creator_gravatar_hash,
+
+        wt.creator_ip_address,
+        wt.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
+        wt.maybe_last_update_user_token as `maybe_last_update_user_token: tokens::tokens::users::UserToken`,
+        wt.original_download_url,
+        wt.original_filename,
+        wt.file_size_bytes,
+        wt.file_checksum_sha2,
+
+        wt.public_bucket_hash,
+        wt.maybe_public_bucket_prefix,
+        wt.maybe_public_bucket_extension,
+
+        cover_image.public_bucket_directory_hash as maybe_cover_image_public_bucket_hash,
+        cover_image.maybe_public_bucket_prefix as maybe_cover_image_public_bucket_prefix,
+        cover_image.maybe_public_bucket_extension as maybe_cover_image_public_bucket_extension,
+
+        entity_stats.ratings_positive_count as maybe_ratings_positive_count,
+        entity_stats.ratings_negative_count as maybe_ratings_negative_count,
+        entity_stats.bookmark_count as maybe_bookmark_count,
+
+        w_extension.ietf_language_tag as maybe_ietf_language_tag,
+        w_extension.ietf_primary_language_subtag as maybe_ietf_primary_language_subtag,
+        w_extension.maybe_default_pretrained_vocoder,
+        w_extension.text_pipeline_type as maybe_text_pipeline_type,
+        w_extension.use_default_mel_multiply_factor,
+        w_extension.maybe_custom_mel_multiply_factor,
+
+        w_extension.maybe_custom_vocoder_token,
+
+        vocoder.vocoder_type as `maybe_custom_vocoder_type: enums::common::vocoder_type::VocoderType`,
+        vocoder.title as maybe_custom_vocoder_title,
+        vocoder.private_bucket_hash as maybe_custom_vocoder_private_bucket_hash,
+
+        wt.version,
+        wt.created_at,
+        wt.updated_at,
+        wt.user_deleted_at,
+        wt.mod_deleted_at
+
+        FROM model_weights as wt
+        JOIN users
+            ON users.token = wt.creator_user_token
+        LEFT OUTER JOIN media_files as cover_image
+            ON cover_image.token = wt.maybe_cover_image_media_file_token
+        LEFT OUTER JOIN entity_stats
+            ON entity_stats.entity_type = "model_weight"
+            AND entity_stats.entity_token = wt.token
+
+        LEFT OUTER JOIN model_weights_extension_tts_details as w_extension
+            ON w_extension.model_weights_token = wt.token
+
+        LEFT OUTER JOIN vocoder_models AS vocoder
+            ON vocoder.token = w_extension.maybe_custom_vocoder_token
+
+        WHERE
+            wt.token = ?
+            "#,
+            weight_token.as_str()
+        )
+        .fetch_one(&mut **mysql_connection).await
+}
+
+#[derive(Serialize)]
+pub struct RawWeight {
+    token: ModelWeightToken,
+    title: String,
+    weights_type: WeightsType,
+    weights_category: WeightsCategory,
+    maybe_description_markdown: Option<String>,
+    maybe_description_rendered_html: Option<String>,
+
+    creator_user_token: UserToken,
+    creator_username: String,
+    creator_display_name: String,
+    creator_gravatar_hash: String,
+
+    creator_ip_address: String,
+    creator_set_visibility: Visibility,
+    maybe_last_update_user_token: Option<UserToken>,
+    original_download_url: Option<String>,
+    original_filename: Option<String>,
+    file_size_bytes: i32,
+    file_checksum_sha2: String,
+    public_bucket_hash: String,
+    maybe_public_bucket_prefix: Option<String>,
+    maybe_public_bucket_extension: Option<String>,
+
+    maybe_cover_image_public_bucket_hash: Option<String>,
+    maybe_cover_image_public_bucket_prefix: Option<String>,
+    maybe_cover_image_public_bucket_extension: Option<String>,
+
+    maybe_ratings_positive_count: Option<u32>,
+    maybe_ratings_negative_count: Option<u32>,
+    maybe_bookmark_count: Option<u32>,
+
+    maybe_ietf_language_tag: Option<String>,
+    maybe_ietf_primary_language_subtag: Option<String>,
+    maybe_default_pretrained_vocoder: Option<String>,
+    maybe_text_pipeline_type: Option<String>,
+
+    // Joined custom vocoder fields
+    maybe_custom_vocoder_token: Option<String>,
+    maybe_custom_vocoder_type: Option<VocoderType>,
+    maybe_custom_vocoder_title: Option<String>,
+    maybe_custom_vocoder_private_bucket_hash: Option<String>,
+
+    use_default_mel_multiply_factor: Option<i8>, // bool; note this was *not* optional before, but is because of our join
+    maybe_custom_mel_multiply_factor: Option<f64>,
+
+    version: i32,
+    created_at: DateTime<Utc>,
+    updated_at: DateTime<Utc>,
+    user_deleted_at: Option<DateTime<Utc>>,
+    mod_deleted_at: Option<DateTime<Utc>>,
+}

crates/lib/mysql_queries/src/queries/model_weights/get/mod.rs

@@ -1,3 +1,4 @@
 pub mod get_weight;
 pub mod get_weight_for_legacy_tts_enqueue;
+pub mod get_weight_for_legacy_tts_inference;
 pub mod get_weight_for_legacy_tts_info;

crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs

@@ -6,6 +6,7 @@ use log::{info, warn};
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use enums::by_table::tts_models::tts_model_type::TtsModelType;
 use filesys::file_exists::file_exists;
+use migration::text_to_speech::get_tts_model_for_run_inference_migration::{get_tts_model_for_run_inference_migration, TtsModelForRunInferenceMigration};
 use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelError, VcModelType};
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord};
@@ -27,7 +28,7 @@ pub struct DependencyStatus {
 
 pub enum MaybeInferenceModel {
   None,
-  TtsModel(TtsModelForInferenceRecord),
+  TtsModel(TtsModelForRunInferenceMigration),
   VcModel(VcModel)
 }
 
@@ -64,13 +65,13 @@ fn get_model_token_and_path(job_dependencies: &JobDependencies, maybe_model: &Ma
   // TODO(bt,2023-05-01): Also check other paths (user-supplied vocoders, etc.)
   let maybe_filesystem_path = match maybe_model {
     MaybeInferenceModel::TtsModel(ref model) => {
-      match model.tts_model_type {
+      match model.tts_model_type() {
         TtsModelType::Tacotron2 => {
-          maybe_model_token = Some(model.model_token.to_string());
+          maybe_model_token = Some(model.token().to_string());
           Some(job_dependencies
               .fs
               .semi_persistent_cache
-              .tts_synthesizer_model_path(model.model_token.as_str()))
+              .tts_synthesizer_model_path(model.token()))
         }
         TtsModelType::Vits => {
           None
@@ -123,8 +124,8 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie
       match maybe_model {
         Some(model) => MaybeInferenceModel::TtsModel(model),
         None => {
-          let maybe_tts_model = get_tts_model_for_inference_improved(
-            &job_dependencies.db.mysql_pool, token)
+          let maybe_tts_model = get_tts_model_for_run_inference_migration(
+            token, &job_dependencies.db.mysql_pool)
               .await
               .map_err(|err| match err {
                 TtsModelForInferenceError::ModelDeleted => ProcessSingleJobError::ModelDeleted,

crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs

@@ -3,8 +3,8 @@ use log::warn;
 
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use enums::by_table::tts_models::tts_model_type::TtsModelType;
+use migration::text_to_speech::get_tts_model_for_run_inference_migration::get_tts_model_for_run_inference_migration;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::get_tts_model_for_inference_improved;
 
 use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -88,9 +88,9 @@ async fn dispatch_fine_tuned_weights_model(
       .ok_or(ProcessSingleJobError::Other(anyhow!("no model token on job")))?;
 
   // TODO(bt,2023-10-09): Interrogate TTS model cache before querying database.
-  let maybe_tts_model = get_tts_model_for_inference_improved(
-    &job_dependencies.db.mysql_pool,
+  let maybe_tts_model = get_tts_model_for_run_inference_migration(
     tts_model_token,
+    &job_dependencies.db.mysql_pool,
   ).await.map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?;
 
   let tts_model = match maybe_tts_model {
@@ -100,7 +100,7 @@ async fn dispatch_fine_tuned_weights_model(
     Some(tts_model) => tts_model,
   };
 
-  match tts_model.tts_model_type {
+  match tts_model.tts_model_type() {
     TtsModelType::Vits => {
       vits::process_job::process_job(VitsProcessJobArgs {
         job_dependencies,

crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs

@@ -11,9 +11,9 @@ use errors::AnyhowResult;
 use filesys::check_file_exists::check_file_exists;
 use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
 use filesys::safe_delete_temp_file::safe_delete_temp_file;
+use migration::text_to_speech::get_tts_model_for_run_inference_migration::TtsModelForRunInferenceMigration;
 use mysql_queries::column_types::vocoder_type::VocoderType;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord;
 use tts_common::clean_symbols::clean_symbols;
 use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic;
 use tts_common::text_pipelines::text_pipeline_type::TextPipelineType;
@@ -36,7 +36,7 @@ const TEST_REQUEST_TEXT: &str = "This is a test request.";
 pub struct ProcessJobArgs<'a> {
   pub job_dependencies: &'a JobDependencies,
   pub job: &'a AvailableInferenceJob,
-  pub tts_model: &'a TtsModelForInferenceRecord,
+  pub tts_model: &'a TtsModelForRunInferenceMigration,
   pub raw_inference_text: &'a str,
 }
 
@@ -111,7 +111,7 @@ async fn process_job_with_cleanup(
 
   // ==================== CONFIRM OR DOWNLOAD OPTIONAL CUSTOM VOCODER MODEL ==================== //
 
-  let custom_vocoder_fs_path = match &tts_model.maybe_custom_vocoder {
+  let custom_vocoder_fs_path = match tts_model.maybe_custom_vocoder() {
     None => None,
     Some(vocoder) => {
       let custom_vocoder_fs_path = args.job_dependencies.fs.semi_persistent_cache.custom_vocoder_model_path(&vocoder.vocoder_token);
@@ -136,14 +136,20 @@ async fn process_job_with_cleanup(
   // ==================== CONFIRM OR DOWNLOAD TTS SYNTHESIZER MODEL ==================== //
 
   let tts_synthesizer_fs_path = {
-    let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str());
-    let tts_synthesizer_object_path  = args.job_dependencies.buckets.bucket_path_unifier.tts_synthesizer_path(&tts_model.private_bucket_hash);
+    let bucket_client = if tts_model.is_private_bucket() {
+      &args.job_dependencies.buckets.private_bucket_client
+    } else {
+      &args.job_dependencies.buckets.public_bucket_client
+    };
+
+    let tts_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.token());
+    let tts_synthesizer_object_path  = tts_model.bucket_object_path(&args.job_dependencies.buckets.bucket_path_unifier);
 
     maybe_download_file_from_bucket(MaybeDownloadArgs {
       name_or_description_of_file: "synthesizer",
       final_filesystem_file_path: & tts_synthesizer_fs_path,
       bucket_object_path: &tts_synthesizer_object_path,
-      bucket_client: &args.job_dependencies.buckets.private_bucket_client,
+      bucket_client,
       job_progress_reporter: &mut job_progress_reporter,
       job_progress_update_description: "downloading synthesizer",
       job_id: job.id.0,
@@ -183,7 +189,7 @@ async fn process_job_with_cleanup(
   info!("Expected output metadata filename: {:?}", &output_metadata_fs_path);
 
   let mut pretrained_vocoder = VocoderType::HifiGanSuperResolution;
-  if let Some(default_vocoder) = tts_model.maybe_default_pretrained_vocoder.as_deref() {
+  if let Some(default_vocoder) = tts_model.maybe_default_pretrained_vocoder().as_deref() {
     pretrained_vocoder = VocoderType::from_str(default_vocoder)
         .map_err(|e| ProcessSingleJobError::Other(e))?;
   }
@@ -214,14 +220,14 @@ async fn process_job_with_cleanup(
       };
   };
 
-  let text_pipeline_type_or_guess = tts_model.text_pipeline_type
+  let text_pipeline_type_or_guess = tts_model.text_pipeline_type()
       .as_deref()
       // NB: If there's an error deserializing, turn it to None.
       .and_then(|pipeline_type| TextPipelineType::from_str(pipeline_type).ok())
-      .unwrap_or_else(|| guess_text_pipeline_heuristic(Some(tts_model.created_at)));
+      .unwrap_or_else(|| guess_text_pipeline_heuristic(Some(tts_model.created_at().clone())));
 
   info!("With text pipeline type `{:?} ` (or guess: {:?})",
-    &tts_model.text_pipeline_type,
+    &tts_model.text_pipeline_type(),
     &text_pipeline_type_or_guess);
 
   // NB: Tacotron operates on decoder steps. 1000 steps is the default and correlates to
@@ -232,9 +238,9 @@ async fn process_job_with_cleanup(
 
   let mut maybe_mel_multiply_factor = None;
 
-  if let Some(factor) = tts_model.maybe_custom_mel_multiply_factor {
+  if let Some(factor) = tts_model.maybe_custom_mel_multiply_factor() {
     maybe_mel_multiply_factor = Some(MelMultiplyFactor::CustomMultiplyFactor(factor));
-  } else if tts_model.use_default_mel_multiply_factor {
+  } else if tts_model.use_default_mel_multiply_factor() {
     maybe_mel_multiply_factor = Some(MelMultiplyFactor::DefaultMultiplyFactor);
   }
 
@@ -262,8 +268,8 @@ async fn process_job_with_cleanup(
       &output_spectrogram_fs_path,
       &output_metadata_fs_path,
       maybe_unload_model_path,
-      tts_model.use_default_mel_multiply_factor,
-      tts_model.maybe_custom_mel_multiply_factor,
+      tts_model.use_default_mel_multiply_factor(),
+      tts_model.maybe_custom_mel_multiply_factor(),
     ).await.map_err(|e| ProcessSingleJobError::Other(e))?;
   } else {
     info!("Shelling out for inference...");
@@ -322,7 +328,7 @@ async fn process_job_with_cleanup(
 
   args.job_dependencies.clients.firehose_publisher.tts_inference_finished(
     job.maybe_creator_user_token.as_deref(),
-    tts_model.model_token.as_str(),
+    tts_model.token(),
     &inference_result.entity_token)
       .await
       .map_err(|e| {

crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/tacotron2_dependencies.rs

@@ -21,6 +21,7 @@ pub struct Tacotron2Dependencies {
   /// dependencies that are for the sidecar version of TT2
   pub sidecar: SidecarDeps,
 
+  /// Rollout flag for uploading results as `media_files` instead of `tts_results`.
   pub upload_as_media_file: bool,
 }
 

crates/service/job/inference_job/src/job/job_types/tts/vits/process_job.rs

@@ -13,9 +13,9 @@ use filesys::check_file_exists::check_file_exists;
 use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
 use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_string::sha256_hash_string;
+use migration::text_to_speech::get_tts_model_for_run_inference_migration::TtsModelForRunInferenceMigration;
 use mysql_queries::column_types::vocoder_type::VocoderType;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
-use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord;
 use mysql_queries::queries::tts::tts_results::insert_tts_result::{insert_tts_result, JobType};
 
 use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -32,7 +32,7 @@ const TEST_REQUEST_TEXT: &str = "This is a test request.";
 pub struct VitsProcessJobArgs<'a> {
   pub job_dependencies: &'a JobDependencies,
   pub job: &'a AvailableInferenceJob,
-  pub tts_model: &'a TtsModelForInferenceRecord,
+  pub tts_model: &'a TtsModelForRunInferenceMigration,
   pub raw_inference_text: &'a str,
 }
 
@@ -63,12 +63,12 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result<JobSuccessResul
   // ==================== CONFIRM OR DOWNLOAD VITS SYNTHESIZER MODEL ==================== //
 
   let vits_traced_synthesizer_fs_path = {
-    let vits_traced_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.model_token.as_str());
+    let vits_traced_synthesizer_fs_path = args.job_dependencies.fs.semi_persistent_cache.tts_synthesizer_model_path(tts_model.token());
 
     // NB: We're using traced models, not the original model files.
     // We generate these at time of upload from the original model files.
     // In the future we may need to "repair" broken models.
-    let vits_traced_synthesizer_object_path  = args.job_dependencies.buckets.bucket_path_unifier.tts_traced_synthesizer_path(&tts_model.private_bucket_hash);
+    let vits_traced_synthesizer_object_path  = tts_model.vits_traced_synthesizer_object_path(&args.job_dependencies.buckets.bucket_path_unifier);
 
     maybe_download_file_from_bucket(MaybeDownloadArgs {
       name_or_description_of_file: "vits traced tts model",
@@ -233,7 +233,7 @@ pub async fn process_job(args: VitsProcessJobArgs<'_>) -> Result<JobSuccessResul
 
   args.job_dependencies.clients.firehose_publisher.tts_inference_finished(
     job.maybe_creator_user_token.as_deref(),
-    tts_model.model_token.as_str(),
+    tts_model.token(),
     &inference_result_token)
       .await
       .map_err(|e| {

crates/service/job/inference_job/src/job_dependencies.rs

@@ -18,10 +18,10 @@ use jobs_common::job_stats::JobStats;
 use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
 use memory_caching::multi_item_ttl_cache::MultiItemTtlCache;
 use memory_caching::ttl_key_counter::TtlKeyCounter;
+use migration::text_to_speech::get_tts_model_for_run_inference_migration::TtsModelForRunInferenceMigration;
 use migration::voice_conversion::query_vc_model_for_migration::VcModel;
 use mysql_queries::common_inputs::container_environment_arg::ContainerEnvironmentArg;
 use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
-use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::TtsModelForInferenceRecord;
 
 use crate::job_specific_dependencies::JobSpecificDependencies;
 use crate::util::scoped_execution::ScopedExecution;
@@ -163,7 +163,7 @@ pub struct FileSystemDetails {
 
 // TODO: Move into the appropriate job-specific dependencies object.
 pub struct JobCaches {
-  pub tts_model_record_cache: MultiItemTtlCache<String, TtsModelForInferenceRecord>,
+  pub tts_model_record_cache: MultiItemTtlCache<String, TtsModelForRunInferenceMigration>,
   pub vc_model_record_cache: MultiItemTtlCache<String, VcModel>,
 
   /// Skip processing models if they're not on the filesystem.

crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/enqueue_infer_tts_handler.rs

@@ -20,7 +20,7 @@ use enums::common::visibility::Visibility;
 use http_server_common::request::get_request_api_token::get_request_api_token;
 use http_server_common::request::get_request_header_optional::get_request_header_optional;
 use http_server_common::request::get_request_ip::get_request_ip;
-use migration::text_to_speech::get_tts_model_for_inference_migration::TtsModelForInferenceMigration;
+use migration::text_to_speech::get_tts_model_for_enqueue_inference_migration::TtsModelForEnqueueInferenceMigration;
 use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated};
 use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
 use mysql_queries::queries::tts::tts_inference_jobs::insert_tts_inference_job::TtsInferenceJobInsertBuilder;
@@ -553,7 +553,7 @@ pub fn validate_inference_text(text: &str, plan: &Plan) -> Result<(), String> {
 /// This is designed to fail closed (read: actually open!) rather than hit resources.
 async fn check_if_authorized_to_use_model(
   maybe_user_session: Option<&UserSessionExtended>,
-  tts_model: &TtsModelForInferenceMigration,
+  tts_model: &TtsModelForEnqueueInferenceMigration,
 ) -> bool {
 
   match tts_model.creator_set_visibility() {

crates/service/web/storyteller_web/src/http_server/endpoints/tts/enqueue_infer_tts_handler/get_tts_model_with_caching.rs

@@ -3,7 +3,7 @@ use sqlx::MySql;
 use sqlx::pool::PoolConnection;
 
 use errors::AnyhowResult;
-use migration::text_to_speech::get_tts_model_for_inference_migration::{get_tts_model_for_inference_migration, TtsModelForInferenceMigration};
+use migration::text_to_speech::get_tts_model_for_enqueue_inference_migration::{get_tts_model_for_enqueue_inference_migration, TtsModelForEnqueueInferenceMigration};
 use redis_caching::redis_ttl_cache::RedisTtlCache;
 use redis_common::redis_cache_keys::RedisCacheKeys;
 
@@ -14,14 +14,14 @@ pub async fn get_tts_model_with_caching(
   redis_ttl_cache: &RedisTtlCache,
   mysql_connection: &mut PoolConnection<MySql>,
   use_weights_table: bool,
-) -> AnyhowResult<Option<TtsModelForInferenceMigration>> {
+) -> AnyhowResult<Option<TtsModelForEnqueueInferenceMigration>> {
   // NB: Copy due to move. Not a cloned ref due to clippy lint
   let model_token2 = model_token.to_string();
 
   let get_tts_model = move || {
     // NB: async closures are not yet stable in Rust, so we include an async block.
     async move {
-      get_tts_model_for_inference_migration(
+      get_tts_model_for_enqueue_inference_migration(
         &model_token2,
         mysql_connection,
         true,