diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 9d1924eeac..25bb38c41c 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -83,9 +83,9 @@ pub async fn list_available_generic_inference_jobs( creator_ip_address: record.creator_ip_address, maybe_creator_user_token: record.maybe_creator_user_token, creator_set_visibility: Visibility::from_str(&record.creator_set_visibility) - .map_err(|e| anyhow!("error: {:?}", e))?, + .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. inference_type: GenericInferenceType::from_str(&record.inference_type) - .map_err(|e| anyhow!("error: {:?}", e))?, + .map_err(|e| anyhow!("error: {:?}", e))?, // TODO/FIXME: This is a gross fix. maybe_inference_args: record.maybe_inference_args, maybe_raw_inference_text: record.maybe_raw_inference_text, maybe_model_token: record.maybe_model_token, diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_models/get_tts_model_for_inference_improved.rs b/crates/lib/mysql_queries/src/queries/tts/tts_models/get_tts_model_for_inference_improved.rs new file mode 100644 index 0000000000..02561c1b7d --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/tts/tts_models/get_tts_model_for_inference_improved.rs @@ -0,0 +1,216 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use chrono::{Utc, DateTime}; +use enums::common::vocoder_type::VocoderType; +use log::warn; +use sqlx::MySqlPool; +use sqlx; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::users::user::UserToken; +use crate::column_types::tts_model_type::TtsModelType; +use crate::helpers::boolean_converters::i8_to_bool; + +// TODO: Can probably just reuse another query. +// TODO(bt, 2023-03-20): There are now three copies of this query. Kill with fire. + +#[derive(Clone)] +pub struct TtsModelForInferenceRecord { + pub model_token: TtsModelToken, + pub tts_model_type: TtsModelType, + + /// NB: text_pipeline_type may not always be present in the database. + pub text_pipeline_type: Option, + + /// Whether to multiply the mel outputs before being vocoded. + pub use_default_mel_multiply_factor: bool, + pub maybe_custom_mel_multiply_factor: Option, + + /// [vocoders 1] + /// This is the new type of vocoder configuration. Users can choose a custom trained + /// vocoder to associate with their model. The tokens reference the `vocoder_models` + /// table. + pub maybe_custom_vocoder: Option, + + /// [vocoders 2] + /// This is the old type of vocoder configuration, which leverages old pretrained + /// vocoders that we manually uploaded. There aren't many good options for users to + /// choose here, so this should be treated as a legacy option going forward. We'll + /// likely be stuck with this configuration for some time, however, due to the large + /// collection of legacy models we have. + pub maybe_default_pretrained_vocoder: Option, + + pub creator_user_token: UserToken, + pub creator_username: String, + pub creator_display_name: String, + + pub title: String, + pub private_bucket_hash: String, + pub created_at: DateTime, + pub updated_at: DateTime, + pub mod_deleted_at: Option>, + pub user_deleted_at: Option>, +} + +#[derive(Clone)] +pub struct CustomVocoderFields { + pub vocoder_token: String, + pub vocoder_type: VocoderType, + pub vocoder_title: String, + pub vocoder_private_bucket_hash: String, +} + +#[derive(Clone, Debug)] +pub enum TtsModelForInferenceError { + ModelDeleted, + DatabaseError { reason: String }, +} + +impl std::fmt::Display for TtsModelForInferenceError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TtsModelForInferenceError::ModelDeleted => write!(f, "ModelDeleted"), + TtsModelForInferenceError::DatabaseError { reason} => + write!(f, "Database error: {:?}", reason), + } + } +} + +impl std::error::Error for TtsModelForInferenceError {} + +pub async fn get_tts_model_for_inference_improved( + pool: &MySqlPool, + model_token: &str +) -> Result, TtsModelForInferenceError> +{ + // NB: Lookup failure is Err(RowNotFound). + // NB: Since this is publicly exposed, we don't query sensitive data. + let maybe_model = sqlx::query_as!( + InternalTtsModelForInferenceRecordRaw, + r#" +SELECT + tts.token as `model_token: tokens::tokens::tts_models::TtsModelToken`, + tts.tts_model_type as `tts_model_type: crate::column_types::tts_model_type::TtsModelType`, + tts.text_pipeline_type, + + tts.use_default_mel_multiply_factor, + tts.maybe_custom_mel_multiply_factor, + + tts.maybe_default_pretrained_vocoder, + + tts.maybe_custom_vocoder_token, + vocoder.vocoder_type as `maybe_custom_vocoder_type: enums::common::vocoder_type::VocoderType`, + vocoder.title as maybe_custom_vocoder_title, + vocoder.private_bucket_hash as maybe_custom_vocoder_private_bucket_hash, + + tts.creator_user_token as `creator_user_token: tokens::users::user::UserToken`, + users.username as creator_username, + users.display_name as creator_display_name, + tts.title, + tts.private_bucket_hash, + tts.created_at, + tts.updated_at, + tts.user_deleted_at, + tts.mod_deleted_at + +FROM tts_models as tts + +JOIN users + ON users.token = tts.creator_user_token + +LEFT OUTER JOIN vocoder_models AS vocoder + ON vocoder.token = tts.maybe_custom_vocoder_token + +WHERE tts.token = ? + "#, + &model_token + ) + .fetch_one(pool) + .await; // TODO: This will return error if it doesn't exist + + let model : InternalTtsModelForInferenceRecordRaw = match maybe_model { + Ok(model) => model, + Err(err) => { + match err { + sqlx::Error::RowNotFound => { + return Ok(None); + }, + _ => { + warn!("tts model query error: {:?}", err); + return Err(TtsModelForInferenceError::DatabaseError { + reason: format!("Mysql error: {:?}", err) + }); + } + } + } + }; + + if model.mod_deleted_at.is_some() || model.user_deleted_at.is_some() { + return Err(TtsModelForInferenceError::ModelDeleted); + } + + Ok(Some(TtsModelForInferenceRecord { + model_token: model.model_token, + tts_model_type: model.tts_model_type, + text_pipeline_type: model.text_pipeline_type, + use_default_mel_multiply_factor: i8_to_bool(model.use_default_mel_multiply_factor), + maybe_custom_mel_multiply_factor: model.maybe_custom_mel_multiply_factor, + maybe_custom_vocoder: match model.maybe_custom_vocoder_token { + // NB: We're relying on a single field's presence to infer that the others vocoder fields + // are also there. If for some reason they aren't, fail open. + None => None, + Some(vocoder_token) => Some(CustomVocoderFields { + vocoder_token, + vocoder_type: model.maybe_custom_vocoder_type.ok_or( + TtsModelForInferenceError::DatabaseError { reason: "custom_vocoder_type field error".to_string() })?, + vocoder_title: model.maybe_custom_vocoder_title.ok_or( + TtsModelForInferenceError::DatabaseError { reason: "custom_vocoder_title field error".to_string() })?, + vocoder_private_bucket_hash: model.maybe_custom_vocoder_private_bucket_hash.ok_or( + TtsModelForInferenceError::DatabaseError { reason: "vocoder_private_bucket_hash field error".to_string() })?, + }) + }, + maybe_default_pretrained_vocoder: model.maybe_default_pretrained_vocoder, + creator_user_token: model.creator_user_token, + creator_username: model.creator_username, + creator_display_name: model.creator_display_name, + title: model.title, + private_bucket_hash: model.private_bucket_hash, + created_at: model.created_at, + updated_at: model.updated_at, + mod_deleted_at: model.mod_deleted_at, + user_deleted_at: model.user_deleted_at, + })) +} + +struct InternalTtsModelForInferenceRecordRaw { + model_token: TtsModelToken, + tts_model_type: TtsModelType, + + text_pipeline_type: Option, + + use_default_mel_multiply_factor: i8, // NB: bool + maybe_custom_mel_multiply_factor: Option, + + // Joined custom vocoder fields + maybe_custom_vocoder_token: Option, + maybe_custom_vocoder_type: Option, + maybe_custom_vocoder_title: Option, + maybe_custom_vocoder_private_bucket_hash: Option, + + // Legacy vocoder config + maybe_default_pretrained_vocoder: Option, + + creator_user_token: UserToken, + creator_username: String, + creator_display_name: String, + + title: String, + private_bucket_hash: String, + created_at: DateTime, + updated_at: DateTime, + mod_deleted_at: Option>, + user_deleted_at: Option>, +} + diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_models/mod.rs b/crates/lib/mysql_queries/src/queries/tts/tts_models/mod.rs index 334a9f9dd3..418b501882 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_models/mod.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_models/mod.rs @@ -5,6 +5,7 @@ pub mod edit_tts_model_details; pub mod edit_tts_model_moderator_details; pub mod get_tts_model; pub mod get_tts_model_for_inference; +pub mod get_tts_model_for_inference_improved; pub mod insert_tts_model_from_download_job; pub mod list_all_tts_model_tokens; pub mod list_tts_models; diff --git a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs index 61a813d743..965a93c655 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/process_single_tts_job.rs @@ -1,8 +1,22 @@ +use anyhow::anyhow; use errors::AnyhowResult; +use mysql_queries::column_types::tts_model_type::TtsModelType; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; +use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference::get_tts_model_for_inference; +use tokens::tokens::tts_models::TtsModelToken; use crate::job::job_steps::job_dependencies::JobDependencies; pub async fn process_single_tts_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> AnyhowResult<()> { + + let tts_model_token = match job.maybe_model_token.as_deref() { + None => return Err(anyhow!("no model token on job")), + Some(model_token) => TtsModelToken::new_from_str(model_token), + }; + + // TODO: Interrogate cache + let tts_model = get_tts_model_for_inference(&job_dependencies.mysql_pool, tts_model_token.as_str()).await?; + + Ok(()) }