diff --git a/Cargo.lock b/Cargo.lock index 6470c56070..67d13e5ca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4464,6 +4464,8 @@ dependencies = [ "log", "once_cell", "regex", + "serde", + "serde_derive", "unicode-segmentation", "user_input_common", ] diff --git a/crates/lib/database_queries/src/queries/tts/tts_models/edit_tts_model_moderator_details.rs b/crates/lib/database_queries/src/queries/tts/tts_models/edit_tts_model_moderator_details.rs index dedef1b834..484995077c 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_models/edit_tts_model_moderator_details.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_models/edit_tts_model_moderator_details.rs @@ -45,4 +45,4 @@ LIMIT 1 ) .execute(mysql_pool) .await -} \ No newline at end of file +} diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/tts_common/Cargo.toml index 8b37fa3462..a8131c7693 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/tts_common/Cargo.toml @@ -26,3 +26,6 @@ once_cell = "1.9.0" regex = "1.5.4" unicode-segmentation = "1.7.1" +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" diff --git a/crates/lib/tts_common/src/lib.rs b/crates/lib/tts_common/src/lib.rs index 682bfdfe17..a582e68ef4 100644 --- a/crates/lib/tts_common/src/lib.rs +++ b/crates/lib/tts_common/src/lib.rs @@ -1,4 +1,5 @@ +#[macro_use] extern crate serde_derive; + pub mod clean_symbols; pub mod priority; -pub mod text_pipeline_defaults; pub mod text_pipelines; \ No newline at end of file diff --git a/crates/lib/tts_common/src/text_pipelines.rs b/crates/lib/tts_common/src/text_pipelines.rs deleted file mode 100644 index 09bf798157..0000000000 --- a/crates/lib/tts_common/src/text_pipelines.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! We support a variety of text pipeline systems. - -use std::collections::HashSet; -use once_cell::sync::Lazy; - -// TODO: Introduce old vocodes models. -//pub const LEGACY_VOCODES : &'static str = "legacy_vocodes"; - -/// Introduction date: 2021. -/// Legacy FakeYou models use graphemes by default. They "can" support arpabet segments manually -/// specified by the user with curly brackets, but there is no guarantee that the model supports -/// this. By default the model will be sent grapheme symbols converted to integers. -pub const LEGACY_FAKEYOU : &'static str = "legacy_fakeyou"; - -/// Introduction date: July 2022, though models trained earlier on arpabet should support this. -/// Unlike "legacy_fakeyou", this forces arpabet lookup of graphemes in all cases possible. The -/// integer encoding is the same, but there are some additional normalization routines. -pub const ENGLISH_V1 : &'static str = "english_v1"; - -/// Introduction date: approx July 2022. -/// This was developed by Ezequiel and Mathias using a modified arpabet scheme similar to -/// "english_v1". -/// NB(2022-07-05): Technically not landed yet. -pub const SPANISH_V1 : &'static str = "spanish_v1"; - -/// Introduction date: approx July 2022. -/// An improvement upon "spanish_v1" that uses an entirely different phonetic system. -/// NB(2022-07-05): Technically not landed yet. -pub const SPANISH_V2 : &'static str = "spanish_v2"; - -static SUPPORTED_TEXT_PIPELINES : Lazy> = Lazy::new(|| { - let text_pipeline_names = [ - LEGACY_FAKEYOU, - ENGLISH_V1, - SPANISH_V1, - SPANISH_V2, - ]; - text_pipeline_names.iter() - .map(|tag| tag.to_string()) - .collect::>() -}); - -/// Check if the text pipeline name (must be lowercase) is valid and supported. -pub fn is_valid_text_pipeline(text_pipeline_name: &str) -> bool { - SUPPORTED_TEXT_PIPELINES.contains(text_pipeline_name) -} - -#[cfg(test)] -mod tests { - use crate::text_pipelines::is_valid_text_pipeline; - - #[test] - fn valid_text_pipelines() { - assert!(is_valid_text_pipeline("legacy_fakeyou")); - assert!(is_valid_text_pipeline("english_v1")); - assert!(is_valid_text_pipeline("spanish_v1")); - assert!(is_valid_text_pipeline("spanish_v2")); - } - - #[test] - fn invalid_text_pipelines() { - // Garbage - assert!(!is_valid_text_pipeline("")); - assert!(!is_valid_text_pipeline("asdf")); - - // NB: Must be lower case - assert!(!is_valid_text_pipeline("LEGACY_FAKEYOU")); - assert!(!is_valid_text_pipeline("ENGLISH_V1")); - assert!(!is_valid_text_pipeline("SPANISH_V1")); - assert!(!is_valid_text_pipeline("SPANISH_V2")); - - // NB: Not yet supported - assert!(!is_valid_text_pipeline("legacy_vocodes")); - assert!(!is_valid_text_pipeline("spanish_v3")); - assert!(!is_valid_text_pipeline("english_v2")); - - // Wrong names - assert!(!is_valid_text_pipeline("english")); - assert!(!is_valid_text_pipeline("spanish")); - assert!(!is_valid_text_pipeline("vocodes")); - } -} diff --git a/crates/lib/tts_common/src/text_pipeline_defaults.rs b/crates/lib/tts_common/src/text_pipelines/guess_pipeline.rs similarity index 72% rename from crates/lib/tts_common/src/text_pipeline_defaults.rs rename to crates/lib/tts_common/src/text_pipelines/guess_pipeline.rs index f57c663993..2b3e6257f3 100644 --- a/crates/lib/tts_common/src/text_pipeline_defaults.rs +++ b/crates/lib/tts_common/src/text_pipelines/guess_pipeline.rs @@ -1,7 +1,7 @@ use std::ops::Deref; use chrono::{DateTime, Utc}; use once_cell::sync::Lazy; -use crate::text_pipelines::{ENGLISH_V1, LEGACY_FAKEYOU}; +use crate::text_pipelines::text_pipeline::TtsTextPipeline; const ENGLISH_V1_EPOCH_STR : &'static str = "2022-05-01T00:00:00.00Z"; @@ -20,27 +20,28 @@ static ENGLISH_V1_EPOCH : Lazy> = Lazy::new(|| { /// There will always be a guess returned. /// /// maybe_model_created_at - `created_at` timestamp from database -pub fn guess_text_pipeline_heuristic(maybe_model_created_at: Option>) -> &'static str { +pub fn guess_text_pipeline_heuristic(maybe_model_created_at: Option>) -> TtsTextPipeline { // TODO: Use language to infer as well. if let Some(created_at) = maybe_model_created_at { if created_at < *ENGLISH_V1_EPOCH { - return LEGACY_FAKEYOU; + return TtsTextPipeline::LegacyFakeYou; } } - ENGLISH_V1 + TtsTextPipeline::EnglishV1 } #[cfg(test)] mod tests { use chrono::{DateTime, Utc}; - use crate::text_pipeline_defaults::guess_text_pipeline_heuristic; + use crate::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; + use crate::text_pipelines::text_pipeline::TtsTextPipeline; #[test] fn new_models_just_created_with_no_date_use_english_v1() { - assert_eq!(guess_text_pipeline_heuristic(None), "english_v1"); + assert_eq!(guess_text_pipeline_heuristic(None), TtsTextPipeline::EnglishV1); } #[test] @@ -48,7 +49,7 @@ mod tests { let datetime = DateTime::parse_from_rfc3339("2022-07-01T00:00:00.00Z") .expect("should parse") .with_timezone(&Utc); - assert_eq!(guess_text_pipeline_heuristic(Some(datetime)), "english_v1"); + assert_eq!(guess_text_pipeline_heuristic(Some(datetime)), TtsTextPipeline::EnglishV1); } #[test] @@ -56,6 +57,6 @@ mod tests { let datetime = DateTime::parse_from_rfc3339("2022-01-01T00:00:00.00Z") .expect("should parse") .with_timezone(&Utc); - assert_eq!(guess_text_pipeline_heuristic(Some(datetime)), "legacy_fakeyou"); + assert_eq!(guess_text_pipeline_heuristic(Some(datetime)), TtsTextPipeline::LegacyFakeYou); } } diff --git a/crates/lib/tts_common/src/text_pipelines/mod.rs b/crates/lib/tts_common/src/text_pipelines/mod.rs new file mode 100644 index 0000000000..e118ebd584 --- /dev/null +++ b/crates/lib/tts_common/src/text_pipelines/mod.rs @@ -0,0 +1,2 @@ +pub mod guess_pipeline; +pub mod text_pipeline; diff --git a/crates/lib/tts_common/src/text_pipelines/text_pipeline.rs b/crates/lib/tts_common/src/text_pipelines/text_pipeline.rs new file mode 100644 index 0000000000..e89de46e2c --- /dev/null +++ b/crates/lib/tts_common/src/text_pipelines/text_pipeline.rs @@ -0,0 +1,112 @@ +use anyhow::anyhow; +use container_common::anyhow_result::AnyhowResult; + +/// Text Pipelines for TTS +/// The system only tolerates the following values for TTS text pipelines. +/// +/// This is *not* a database enum, but the text serializations get stored in the database in a +/// varchar field and communicated over the API. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize)] +pub enum TtsTextPipeline { + + // TODO: Introduce old vocodes models. + //#[serde(rename = "legacy_vocodes")] + //LegacyVocodes, + + /// Introduction date: 2021. + /// Legacy FakeYou models use graphemes by default. They "can" support arpabet segments manually + /// specified by the user with curly brackets, but there is no guarantee that the model supports + /// this. By default the model will be sent grapheme symbols converted to integers. + #[serde(rename = "legacy_fakeyou")] + LegacyFakeYou, + + /// Introduction date: July 2022, though models trained earlier on arpabet should support this. + /// Unlike "legacy_fakeyou", this forces arpabet lookup of graphemes in all cases possible. The + /// integer encoding is the same, but there are some additional normalization routines. + #[serde(rename = "english_v1")] + EnglishV1, + + /// Introduction date: approx July 2022. + /// This was developed by Ezequiel and Mathias using a modified arpabet scheme similar to + /// "english_v1". + /// NB(2022-07-05): Technically not landed yet. + #[serde(rename = "spanish_v1")] + SpanishV1, + + /// Introduction date: approx July 2022. + /// An improvement upon "spanish_v1" that uses an entirely different phonetic system. + /// NB(2022-07-05): Technically not landed yet. + #[serde(rename = "spanish_v2")] + SpanishV2, +} + +// TODO: Sucks to redefine enum variants. `serde_variant` looks like a fix, but it's GPL3. +const LEGACY_FAKEYOU : &'static str = "legacy_fakeyou"; +const ENGLISH_V1 : &'static str = "english_v1"; +const SPANISH_V1 : &'static str = "spanish_v1"; +const SPANISH_V2 : &'static str = "spanish_v2"; + +impl TtsTextPipeline { + + /// Check if the text pipeline name is valid and supported. + pub fn is_valid_name(tts_text_pipeline: &str) -> bool { + Self::from_str(tts_text_pipeline).is_ok() + } + + // TODO: There has to be a better way to extract serde variant info. + // `serde_variant` looks like a fix, but it's GPL3. + pub fn to_str(&self) -> &'static str { + match self { + TtsTextPipeline::LegacyFakeYou => LEGACY_FAKEYOU, + TtsTextPipeline::EnglishV1 => ENGLISH_V1, + TtsTextPipeline::SpanishV1 => SPANISH_V1, + TtsTextPipeline::SpanishV2 => SPANISH_V2, + } + } + + pub fn from_str(tts_text_pipeline: &str) -> AnyhowResult { + match tts_text_pipeline { + LEGACY_FAKEYOU => Ok(Self::LegacyFakeYou), + ENGLISH_V1 => Ok(Self::EnglishV1), + SPANISH_V1 => Ok(Self::SpanishV1), + SPANISH_V2 => Ok(Self::SpanishV2), + _ => Err(anyhow!("invalid variant: {}", tts_text_pipeline)), + } + } +} + +#[cfg(test)] +mod tests { + use crate::text_pipelines::text_pipeline::TtsTextPipeline; + + #[test] + fn valid_text_pipeline_names() { + assert!(TtsTextPipeline::is_valid_name("legacy_fakeyou")); + assert!(TtsTextPipeline::is_valid_name("english_v1")); + assert!(TtsTextPipeline::is_valid_name("spanish_v1")); + assert!(TtsTextPipeline::is_valid_name("spanish_v2")); + } + + #[test] + fn invalid_text_pipeline_names() { + // Garbage + assert!(!TtsTextPipeline::is_valid_name("")); + assert!(!TtsTextPipeline::is_valid_name("asdf")); + + // NB: Must be lower case + assert!(!TtsTextPipeline::is_valid_name("LEGACY_FAKEYOU")); + assert!(!TtsTextPipeline::is_valid_name("ENGLISH_V1")); + assert!(!TtsTextPipeline::is_valid_name("SPANISH_V1")); + assert!(!TtsTextPipeline::is_valid_name("SPANISH_V2")); + + // NB: Not yet supported + assert!(!TtsTextPipeline::is_valid_name("legacy_vocodes")); + assert!(!TtsTextPipeline::is_valid_name("spanish_v3")); + assert!(!TtsTextPipeline::is_valid_name("english_v2")); + + // Wrong names + assert!(!TtsTextPipeline::is_valid_name("english")); + assert!(!TtsTextPipeline::is_valid_name("spanish")); + assert!(!TtsTextPipeline::is_valid_name("vocodes")); + } +} diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs index f9df623fcb..dacec55916 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/edit_tts_model.rs @@ -50,9 +50,10 @@ pub struct EditTtsModelRequest { // NB: We calculate 'ietf_primary_language_subtag' from this value. pub ietf_language_tag: Option, + pub text_pipeline_type: Option, + //pub updatable_slug: Option, //pub tts_model_type: Option, - //pub text_preprocessing_algorithm: Option, //pub vocoder_token: Option, // ========== Moderator options (protection) ========== @@ -306,8 +307,8 @@ pub async fn edit_tts_model_handler( async fn update_mod_details( request: &Json, - mod_user_token: &str, - model_token: &str, + moderator_user_token: &str, + tts_model_token: &str, mysql_pool: &MySqlPool ) -> Result<(), EditTtsModelError> { @@ -333,14 +334,14 @@ async fn update_mod_details( let query_result = edit_tts_model_moderator_details( &mysql_pool, - model_token, + tts_model_token, is_public_listing_approved, is_locked_from_user_modification, is_locked_from_use, maybe_suggested_unique_bot_command.as_deref(), is_front_page_featured, is_twitch_featured, - mod_user_token, + moderator_user_token, request.maybe_mod_comments.as_deref(), ).await; diff --git a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs index 1f5b1c356f..234518d8f1 100644 --- a/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs +++ b/crates/service/storyteller_web/src/http_server/endpoints/tts/get_tts_model.rs @@ -21,7 +21,8 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; -use tts_common::text_pipeline_defaults::guess_text_pipeline_heuristic; +use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; +use tts_common::text_pipelines::text_pipeline::TtsTextPipeline; // =============== Request =============== @@ -49,8 +50,8 @@ pub struct TtsModelInfo { /// /// >> NB: text_pipeline_type may not always be present in the database, but if absent we'll /// inform the frontend (and inference pipeline) of our best guess according to a heuristic. - pub text_pipeline_type: Option, - pub text_pipeline_type_guess: String, + pub text_pipeline_type: Option, + pub text_pipeline_type_guess: TtsTextPipeline, pub maybe_default_pretrained_vocoder: Option, pub text_preprocessing_algorithm: String, @@ -183,6 +184,11 @@ pub async fn get_tts_model_handler( model.maybe_moderator_fields = None; } + // If there's an error deserializing, turn it to None. + let text_pipeline_type = model.text_pipeline_type + .as_deref() + .and_then(|pipeline_type| TtsTextPipeline::from_str(pipeline_type).ok()); + // TODO: Use language to infer as well. let text_pipeline_type_guess = guess_text_pipeline_heuristic(Some(model.created_at.clone())); @@ -193,8 +199,8 @@ pub async fn get_tts_model_handler( model: TtsModelInfo { model_token: model.model_token, tts_model_type: model.tts_model_type, - text_pipeline_type: model.text_pipeline_type, - text_pipeline_type_guess: text_pipeline_type_guess.to_string(), + text_pipeline_type, + text_pipeline_type_guess, maybe_default_pretrained_vocoder: model.maybe_default_pretrained_vocoder, text_preprocessing_algorithm: model.text_preprocessing_algorithm, creator_user_token: model.creator_user_token, diff --git a/crates/service/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/tts_inference_job/src/job_steps/process_single_job.rs index 54675b87c5..270a48756a 100644 --- a/crates/service/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/tts_inference_job/src/job_steps/process_single_job.rs @@ -50,7 +50,8 @@ use storage_buckets_common::bucket_client::BucketClient; use storage_buckets_common::bucket_path_unifier::BucketPathUnifier; use tempdir::TempDir; use tts_common::clean_symbols::clean_symbols; -use tts_common::text_pipeline_defaults::guess_text_pipeline_heuristic; +use tts_common::text_pipelines::guess_pipeline::guess_text_pipeline_heuristic; +use tts_common::text_pipelines::text_pipeline::TtsTextPipeline; #[derive(Deserialize, Default)] struct FileMetadata { @@ -324,10 +325,13 @@ pub async fn process_single_job( info!("With pretrained vocoder: {:?}", pretrained_vocoder); let text_pipeline_type_or_guess = model_record.text_pipeline_type - .clone() - .unwrap_or_else(|| guess_text_pipeline_heuristic(Some(model_record.created_at)).to_string()); + .as_deref() + .and_then(|pipeline_type| + TtsTextPipeline::from_str(pipeline_type).ok())// NB: If there's an error deserializing, turn it to None. + .unwrap_or_else(|| + guess_text_pipeline_heuristic(Some(model_record.created_at))); - info!("With text pipeline type `{:?} ` (or guess: {})", + info!("With text pipeline type `{:?} ` (or guess: {:?})", &model_record.text_pipeline_type, &text_pipeline_type_or_guess); @@ -335,7 +339,7 @@ pub async fn process_single_job( &cleaned_inference_text, &tts_synthesizer_fs_path, pretrained_vocoder, - &text_pipeline_type_or_guess, + &text_pipeline_type_or_guess.to_str(), &hifigan_vocoder_model_fs_path, &hifigan_superres_vocoder_model_fs_path, &waveglow_vocoder_model_fs_path,