diff --git a/_sql/actions/truncate.sql b/_sql/actions/truncate.sql index 1b864beb14..bd0ba63527 100644 --- a/_sql/actions/truncate.sql +++ b/_sql/actions/truncate.sql @@ -2,10 +2,13 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- DO NOT DO THIS IN PRODUCTION. +-- Just placing this here to be easy to copy/paste in development. truncate zs_voices; truncate zs_voice_datasets; truncate zs_voice_dataset_samples; truncate media_files; truncate media_uploads; truncate generic_inference_jobs; +truncate tts_models; diff --git a/crates/cli/dev_database_seed/src/bucket_clients.rs b/crates/cli/dev_database_seed/src/bucket_clients.rs new file mode 100644 index 0000000000..724b4f9cb0 --- /dev/null +++ b/crates/cli/dev_database_seed/src/bucket_clients.rs @@ -0,0 +1,51 @@ +use std::time::Duration; + +use log::info; + +use cloud_storage::bucket_client::BucketClient; +use errors::AnyhowResult; + +pub struct BucketClients { + pub public: BucketClient, + pub private: BucketClient, +} + +pub fn get_bucket_clients() -> AnyhowResult { + let access_key = easyenv::get_env_string_required("ACCESS_KEY")?; + let secret_key = easyenv::get_env_string_required("SECRET_KEY")?; + let region_name = easyenv::get_env_string_required("REGION_NAME")?; + let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?; + let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?; + + // NB: Long timeout for dev rust builds to upload to cloud buckets. + // Unoptimized binaries sometimes take a lot of time to upload, presumably due to unoptimized code. + let bucket_timeout = easyenv::get_env_duration_seconds_or_default( + "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10)); + + info!("Configuring public GCS bucket..."); + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + Some(bucket_timeout), + )?; + + info!("Configuring private GCS bucket..."); + + let private_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + Some(bucket_timeout), + )?; + + Ok(BucketClients { + public: public_bucket_client, + private: private_bucket_client, + }) +} diff --git a/crates/cli/dev_database_seed/src/main.rs b/crates/cli/dev_database_seed/src/main.rs index 415e01a2bc..24bc186d2b 100644 --- a/crates/cli/dev_database_seed/src/main.rs +++ b/crates/cli/dev_database_seed/src/main.rs @@ -1,18 +1,18 @@ -use std::time::Duration; use elasticsearch::Elasticsearch; - use log::info; use sqlx::mysql::MySqlPoolOptions; -use cloud_storage::bucket_client::BucketClient; use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; use errors::AnyhowResult; -use crate::cli_args::parse_cli_args; +use crate::bucket_clients::get_bucket_clients; +use crate::cli_args::parse_cli_args; +use crate::seeding::tts_tacotron2::seed_tts_tacotron2; use crate::seeding::users::seed_user_accounts; use crate::seeding::voice_conversion::seed_voice_conversion; use crate::seeding::zero_shot_tts::seed_zero_shot_tts; +pub mod bucket_clients; pub mod cli_args; pub mod seeding; @@ -37,10 +37,10 @@ pub async fn main() -> AnyhowResult<()> { let args = parse_cli_args()?; - let mut maybe_public_bucket_client = None; + let mut maybe_bucket_clients = None; if args.seed_cloud_bucket { - maybe_public_bucket_client = Some(get_bucket_client()?); + maybe_bucket_clients = Some(get_bucket_clients()?); } let mut maybe_elasticsearch = None; @@ -50,36 +50,14 @@ pub async fn main() -> AnyhowResult<()> { } seed_user_accounts(&pool).await?; - seed_zero_shot_tts(&pool, maybe_public_bucket_client.as_ref()).await?; + seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?; seed_voice_conversion(&pool).await?; + seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?; info!("Done!"); Ok(()) } -fn get_bucket_client() -> AnyhowResult { - let access_key = easyenv::get_env_string_required("ACCESS_KEY")?; - let secret_key = easyenv::get_env_string_required("SECRET_KEY")?; - let region_name = easyenv::get_env_string_required("REGION_NAME")?; - let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?; - - let bucket_timeout = easyenv::get_env_duration_seconds_or_default( - "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); - - info!("Configuring GCS bucket..."); - - let public_bucket_client = BucketClient::create( - &access_key, - &secret_key, - ®ion_name, - &public_bucket_name, - None, - Some(bucket_timeout), - )?; - - Ok(public_bucket_client) -} - fn get_elasticsearch_client() -> AnyhowResult { // TODO(bt,2023-10-26): Allow connecting to instances by URL instead of the default dev URL. let client = Elasticsearch::default(); diff --git a/crates/cli/dev_database_seed/src/seeding/mod.rs b/crates/cli/dev_database_seed/src/seeding/mod.rs index ed87c42c9e..241ec08005 100644 --- a/crates/cli/dev_database_seed/src/seeding/mod.rs +++ b/crates/cli/dev_database_seed/src/seeding/mod.rs @@ -1,3 +1,4 @@ +pub mod tts_tacotron2; pub mod users; pub mod voice_conversion; pub mod zero_shot_tts; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs b/crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs new file mode 100644 index 0000000000..f36d2165ad --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs @@ -0,0 +1,181 @@ +use std::path::Path; + +use log::{info, warn}; +use sqlx::{MySql, Pool}; + +use cloud_storage::bucket_path_unifier::BucketPathUnifier; +use enums::by_table::tts_models::tts_model_type::TtsModelType; +use enums::common::visibility::Visibility; +use errors::{anyhow, AnyhowResult}; +use filesys::file_read_bytes::file_read_bytes; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_file::sha256_hash_file; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token; +use mysql_queries::queries::tts::tts_models::insert_tts_model_from_download_job::{insert_tts_model_from_download_job, InsertTtsModelFromDownloadJobArgs}; +use storyteller_root::get_seed_tool_data_root; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::users::UserToken; + +use crate::bucket_clients::BucketClients; +use crate::seeding::users::HANASHI_USER_TOKEN; + +pub async fn seed_tts_tacotron2(mysql_pool: &Pool, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> { + info!("Seeding TTS (Tacotron2)..."); + + // NB: This is idempotent and will only install the models once. + let tts_models = [ + ("Mai Valentine from Yugioh (by Vegito)", HANASHI_USER_TOKEN, "TM:cknxtj6mx3t3", "models/tts/tacotron2/MaiYugioh.pth"), + ("Nagito (by Vegito)", HANASHI_USER_TOKEN, "TM:tmhxapghwaf3", "models/tts/tacotron2/Nagito.pth"), + ("Son Goku V2 (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "TM:kbvaa4726jdw", "models/tts/tacotron2/SonGokuV2.pth"), + ("Vegeta (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "TM:kvv9xqashp6n", "models/tts/tacotron2/VegetaNew.pth"), + ]; + + let seed_tool_data_root = get_seed_tool_data_root(); + + // TODO(bt,2023-11-27): DO NOT USE BucketPathUnifier for any new code. Only for TT2 and super legacy systems. + let bucket_path_unifier = BucketPathUnifier::default_paths(); + + for (model_title, user_token, tts_model_token, subdirectory_path) in tts_models { + let user_token = UserToken::new_from_str(user_token); + let tts_model_token = TtsModelToken::new_from_str(tts_model_token); + + let mut model_file_path = seed_tool_data_root.clone(); + model_file_path.push(subdirectory_path); + + let result = seed_model( + &mysql_pool, + &tts_model_token, + model_title, + &user_token, + &model_file_path, + maybe_bucket_clients, + &bucket_path_unifier + ).await; + + match result { + Ok(_) => info!("Seeded {}", model_title), + Err(err) => warn!(r#" + Could not TTS model {} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the TTS model on a previous invocation!) + "#, model_title, err), + } + } + + Ok(()) +} + +async fn seed_model( + mysql_pool: &Pool, + tts_model_token: &TtsModelToken, + model_title: &str, + user_token: &UserToken, + model_file_path: &Path, + maybe_bucket_clients: Option<&BucketClients>, + bucket_path_unifier: &BucketPathUnifier, +) -> AnyhowResult<()> { + info!("Seeding TTS model {} ...", model_title); + + let mut private_bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + let mut private_bucket_object_name = "NOT_UPLOADED_BY_SEED_TOOL".to_string(); + + if tts_model_exists(mysql_pool, tts_model_token).await? { + info!("Model already seeded: {:?}", tts_model_token); + return Ok(()) + } + + if let Some(bucket_clients) = maybe_bucket_clients { + let bucket_details = seed_file_to_bucket(model_file_path, bucket_clients, bucket_path_unifier).await?; + private_bucket_hash = bucket_details.bucket_hash; + private_bucket_object_name = bucket_details.bucket_object_name; + } + + let filename = model_file_path + .file_name() + .map(|name| name.to_str()) + .flatten() + .unwrap_or(""); + + insert_tts_model_from_download_job(InsertTtsModelFromDownloadJobArgs { + tts_model_type: TtsModelType::Tacotron2, + title: model_title, + original_download_url: "https://example.com", + original_filename: filename, + file_size_bytes: 0, + creator_user_token: user_token, + creator_ip_address: "127.0.0.1", + creator_set_visibility: Visibility::Public, + private_bucket_hash: &private_bucket_hash, + private_bucket_object_name: &private_bucket_object_name, + maybe_model_token: Some(&tts_model_token), + mysql_pool, + }).await?; + + Ok(()) +} + +async fn tts_model_exists( + mysql_pool: &Pool, + tts_model_token: &TtsModelToken, +) -> AnyhowResult { + + const CAN_SEE_DELETED : bool = true; + + let maybe_model = get_tts_model_by_token( + tts_model_token.as_str(), + CAN_SEE_DELETED, + mysql_pool + ).await?; + + Ok(maybe_model.is_some()) +} + +struct BucketDetails { + bucket_hash: String, + bucket_object_name: String, +} + +async fn seed_file_to_bucket( + model_file_path: &Path, + bucket_clients: &BucketClients, + bucket_path_unifier: &BucketPathUnifier +) -> AnyhowResult { + + info!("Uploading Tacotron2 model file {:?} ...", model_file_path); + + // TODO(bt,2023-11-27): DO NOT DO THIS OR COPY THIS PATTERN + // This method of uploading model files is super deprecated and results in inconsistent pathing. + // Try to standardize on something resembling media_files going forward. + let private_bucket_hash = sha256_hash_file(&model_file_path)?; + + info!("File hash: {}", private_bucket_hash); + + // FIXME(bt,2023-11-27): DO NOT COPY THIS PATTERN! + // 'bucket_path_unifier' is super deprecated. Do not use it anymore or for anything except TT2! + let model_bucket_path = bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash); + + //let public_upload_path = MediaFileBucketPath::generate_new( + // Some("sample_"), + // Some(".bin")); + + let model_bucket_path = path_to_string(model_bucket_path); + + info!("Uploading TT2 model to bucket path: {:?}", model_bucket_path); + + info!("Reading TT2 model file: {:?}", model_file_path); + + let bytes = file_read_bytes(model_file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); + + let _r = bucket_clients.private.upload_file_with_content_type( + &model_bucket_path, + bytes.as_ref(), + mimetype) + .await?; + + Ok(BucketDetails { + bucket_hash: private_bucket_hash, + bucket_object_name: model_bucket_path, + }) +} diff --git a/crates/cli/dev_database_seed/src/seeding/users.rs b/crates/cli/dev_database_seed/src/seeding/users.rs index 1ab5590757..a15707a7fd 100644 --- a/crates/cli/dev_database_seed/src/seeding/users.rs +++ b/crates/cli/dev_database_seed/src/seeding/users.rs @@ -5,25 +5,35 @@ use errors::{anyhow, AnyhowResult}; use hashing::bcrypt::bcrypt_password_hash::bcrypt_password_hash; use hashing::md5::email_to_gravatar_hash::email_to_gravatar_hash; use mysql_queries::queries::users::user::create_account::{create_account, CreateAccountArgs}; +use tokens::tokens::users::UserToken; pub const ADMIN_USERNAME : &str = "admin"; pub const HANASHI_USERNAME : &str = "hanashi"; +// We want deterministic tokens for some records +pub const ADMIN_USER_TOKEN : &str = "user_4jgh4dk9tmr6t"; +pub const HANASHI_USER_TOKEN : &str = "user_fcfteany0ds60"; + pub async fn seed_user_accounts(mysql_pool: &Pool) -> AnyhowResult<()> { info!("Seeding user accounts..."); // NB: This is idempotent and will only install the accounts once. let users = [ - (ADMIN_USERNAME, "password"), - (HANASHI_USERNAME, "password"), - ("test", "password"), + (ADMIN_USERNAME, "password", Some(ADMIN_USER_TOKEN)), + (HANASHI_USERNAME, "password", Some(HANASHI_USER_TOKEN)), + ("test", "password", None), ]; - for (username, password) in users { - let result = seed_user(username, password, &mysql_pool).await; + for (username, password, maybe_user_token) in users { + let maybe_user_token = maybe_user_token.map(|token| UserToken::new_from_str(token)); + let result = seed_user(username, password, maybe_user_token, &mysql_pool).await; match result { Ok(_) => info!("Seeded {}", username), - Err(err) => warn!("Could not seed user {} : {:?}", username, err), + Err(err) => warn!(r#" + Could not seed user {} : {:?} + (No worries: if there's a duplicate key error, we probably already + seeded the user on a previous invocation!) + "#, username, err), } } @@ -33,6 +43,7 @@ pub async fn seed_user_accounts(mysql_pool: &Pool) -> AnyhowResult<()> { async fn seed_user( username: &str, password: &str, + maybe_user_token: Option, mysql_pool: &Pool, ) -> AnyhowResult<()> { info!("Seeding user {} ...", username); @@ -50,6 +61,7 @@ async fn seed_user( email_gravatar_hash: &email_gravatar_hash, password_hash: &password_hash, ip_address: "127.0.0.1", + maybe_user_token: maybe_user_token.as_ref(), }).await.map_err(|err| anyhow!("err: {:?}", err))?; Ok(()) diff --git a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs index d0004aa182..d01c359d13 100644 --- a/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs +++ b/crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs @@ -22,9 +22,10 @@ use tokens::tokens::users::UserToken; use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; use tokens::tokens::zs_voices::ZsVoiceToken; +use crate::bucket_clients::BucketClients; use crate::seeding::users::HANASHI_USERNAME; -pub async fn seed_zero_shot_tts(mysql_pool: &Pool, maybe_bucket_client: Option<&BucketClient>) -> AnyhowResult<()> { +pub async fn seed_zero_shot_tts(mysql_pool: &Pool, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> { info!("Seeding zero shot TTS..."); let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? { @@ -46,7 +47,7 @@ pub async fn seed_zero_shot_tts(mysql_pool: &Pool, maybe_bucket_client: O ]; for (voice_name, bucket_hash, wav_file, user_token) in records { - create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, maybe_bucket_client).await?; + create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, maybe_bucket_clients).await?; } Ok(()) @@ -58,7 +59,7 @@ async fn create_voice_records( wav_file: &str, creator_user_token: &UserToken, mysql_pool: &Pool, - maybe_bucket_client: Option<&BucketClient>, + maybe_bucket_clients: Option<&BucketClients>, ) -> AnyhowResult<(ZsVoiceToken, ZsVoiceDatasetToken)> { info!("Creating voice records for voice {} ...", voice_name); @@ -76,8 +77,8 @@ async fn create_voice_records( let public_upload_path; - if let Some(bucket_client) = maybe_bucket_client { - public_upload_path = seed_file_to_bucket(wav_file, bucket_client).await?; + if let Some(bucket_clients) = maybe_bucket_clients { + public_upload_path = seed_file_to_bucket(wav_file, &bucket_clients.public).await?; } else { public_upload_path = MediaFileBucketPath::from_object_hash("fake", None, None); } diff --git a/crates/lib/mysql_queries/src/mediators/badge_granter.rs b/crates/lib/mysql_queries/src/mediators/badge_granter.rs index 299a67b71a..ec85cce537 100644 --- a/crates/lib/mysql_queries/src/mediators/badge_granter.rs +++ b/crates/lib/mysql_queries/src/mediators/badge_granter.rs @@ -1,11 +1,12 @@ use anyhow::anyhow; use log::warn; -use sqlx::MySqlPool; use sqlx::error::Error::Database; use sqlx::mysql::MySqlQueryResult; +use sqlx::MySqlPool; use errors::AnyhowResult; use reusable_types::achievements::user_badge::UserBadge; +use tokens::tokens::users::UserToken; use crate::mediators::firehose_publisher::FirehosePublisher; @@ -34,8 +35,8 @@ impl BadgeGranter { } /// This needs to be called *after* successful upload. - pub async fn maybe_grant_tts_model_uploads_badge(&self, user_token: &str) -> AnyhowResult<()> { - let count = self.count_tts_models_uploaded(user_token).await?; + pub async fn maybe_grant_tts_model_uploads_badge(&self, user_token: &UserToken) -> AnyhowResult<()> { + let count = self.count_tts_models_uploaded(user_token.as_str()).await?; let mut maybe_badge = None; @@ -68,16 +69,16 @@ impl BadgeGranter { None => return Ok(()), }; - if self.has_badge(user_token, badge).await? { + if self.has_badge(user_token.as_str(), badge).await? { return Ok(()) } let _record_id = self.insert( badge, - user_token, + user_token.as_str(), ).await?; - self.firehose_publisher.publish_user_badge_granted(user_token, badge.to_db_value()) + self.firehose_publisher.publish_user_badge_granted(user_token.as_str(), badge.to_db_value()) .await?; Ok(()) diff --git a/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs index 2fb380c5cf..ccd5875e53 100644 --- a/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs @@ -8,6 +8,7 @@ use enums::common::job_status::JobStatus; use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::generic_download_jobs::DownloadJobToken; +use tokens::tokens::users::UserToken; use crate::queries::generic_download::job::_keys::GenericDownloadJobId; @@ -17,7 +18,7 @@ pub struct AvailableDownloadJob { pub id: GenericDownloadJobId, pub download_job_token: DownloadJobToken, - pub creator_user_token: String, + pub creator_user_token: UserToken, pub creator_ip_address: String, pub creator_set_visibility: Visibility, @@ -51,7 +52,7 @@ SELECT id, token, - creator_user_token, + creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`, creator_ip_address, creator_set_visibility, @@ -143,7 +144,7 @@ struct AvailableDownloadJobRawInternal { pub id: i64, pub token: String, - pub creator_user_token: String, + pub creator_user_token: UserToken, pub creator_ip_address: String, pub creator_set_visibility: String, diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_models/insert_tts_model_from_download_job.rs b/crates/lib/mysql_queries/src/queries/tts/tts_models/insert_tts_model_from_download_job.rs index fcfef3a0f0..17754c6be5 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_models/insert_tts_model_from_download_job.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_models/insert_tts_model_from_download_job.rs @@ -7,6 +7,7 @@ use enums::by_table::tts_models::tts_model_type::TtsModelType; use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::users::UserToken; pub struct InsertTtsModelFromDownloadJobArgs<'a, P: AsRef> { pub tts_model_type: TtsModelType, @@ -17,22 +18,30 @@ pub struct InsertTtsModelFromDownloadJobArgs<'a, P: AsRef> { pub original_filename: &'a str, pub file_size_bytes: u64, - pub creator_user_token: &'a str, + pub creator_user_token: &'a UserToken, pub creator_ip_address: &'a str, pub creator_set_visibility: Visibility, pub private_bucket_hash: &'a str, pub private_bucket_object_name: P, + /// In production code, send this as `None`. + /// Only provide an external model token for db integration tests and db seeding tools. + /// This allows for knowing the model token a priori. + pub maybe_model_token: Option<&'a TtsModelToken>, + pub mysql_pool: &'a MySqlPool, } pub async fn insert_tts_model_from_download_job>( args: InsertTtsModelFromDownloadJobArgs<'_, P>, -) -> AnyhowResult<(u64, String)> { +) -> AnyhowResult<(u64, TtsModelToken)> { - let model_token = TtsModelToken::generate().to_string(); + let model_token = match args.maybe_model_token { + None => TtsModelToken::generate(), + Some(model_token) => model_token.clone(), + }; let private_bucket_object_name = &args.private_bucket_object_name .as_ref() diff --git a/crates/lib/mysql_queries/src/queries/users/user/create_account.rs b/crates/lib/mysql_queries/src/queries/users/user/create_account.rs index d8e9094828..c240001df3 100644 --- a/crates/lib/mysql_queries/src/queries/users/user/create_account.rs +++ b/crates/lib/mysql_queries/src/queries/users/user/create_account.rs @@ -16,6 +16,11 @@ pub struct CreateAccountArgs<'a> { pub email_gravatar_hash: &'a str, pub password_hash: &'a str, pub ip_address: &'a str, + + /// In production code, send this as `None`. + /// Only provide an external user token for db integration tests and db seeding tools. + /// This allows for knowing the user token a priori. + pub maybe_user_token: Option<&'a UserToken>, } pub struct CreateAccountSuccessResult { @@ -40,7 +45,10 @@ pub async fn create_account( const INITIAL_PROFILE_RENDERED_HTML : &str = ""; const INITIAL_USER_ROLE: &str = "user"; - let user_token = UserToken::generate(); + let user_token = match args.maybe_user_token { + None => UserToken::generate(), + Some(user_token) => user_token.clone(), + }; let query_result = sqlx::query!( r#" diff --git a/crates/lib/storyteller_root/src/lib.rs b/crates/lib/storyteller_root/src/lib.rs index 009fc5f5a7..6b8dffe735 100644 --- a/crates/lib/storyteller_root/src/lib.rs +++ b/crates/lib/storyteller_root/src/lib.rs @@ -31,6 +31,7 @@ mod paths; mod substituted_path; +pub use paths::seed_tool_data_root::get_seed_tool_data_root; pub use paths::storyteller_frontend_root::get_storyteller_frontend_root; pub use paths::storyteller_ml_root::get_storyteller_ml_root; pub use paths::storyteller_root::get_storyteller_root; diff --git a/crates/lib/storyteller_root/src/paths/mod.rs b/crates/lib/storyteller_root/src/paths/mod.rs index 2d2a0798cc..10469280c6 100644 --- a/crates/lib/storyteller_root/src/paths/mod.rs +++ b/crates/lib/storyteller_root/src/paths/mod.rs @@ -1,4 +1,5 @@ pub (crate) mod env_get_path; +pub (crate) mod seed_tool_data_root; pub (crate) mod storyteller_frontend_root; pub (crate) mod storyteller_ml_root; pub (crate) mod storyteller_root; diff --git a/crates/lib/storyteller_root/src/paths/seed_tool_data_root.rs b/crates/lib/storyteller_root/src/paths/seed_tool_data_root.rs new file mode 100644 index 0000000000..434182e77f --- /dev/null +++ b/crates/lib/storyteller_root/src/paths/seed_tool_data_root.rs @@ -0,0 +1,122 @@ +use std::path::PathBuf; + +use crate::get_storyteller_root; +use crate::paths::env_get_path::env_get_path; + +/// The env var that declares where the seed tool data repo lives. +pub const SEED_TOOL_DATA_ROOT : &str = "SEED_TOOL_DATA_ROOT"; + +// DO NOT LEAK THIS. THIS IS FOR TESTING ONLY. +pub (crate) const TEST_SEED_TOOL_DATA_ROOT : &str = "ENV_TEST_SEED_TOOL_DATA_ROOT_DO_NOT_LEAK_"; + +/// Get the root of the seed-tool-data repo +pub fn get_seed_tool_data_root() -> PathBuf { + // 1) Try "STORYTELLER_RUST_ROOT" env var override. Do not check for path existence. + let root = if cfg!(test) { TEST_SEED_TOOL_DATA_ROOT } else { SEED_TOOL_DATA_ROOT }; + + if let Ok(Some(path)) = env_get_path(root) { + return path; + } + + // 2) Relative to "storyteller root" + let mut dir = get_storyteller_root(); + dir.push("seed-tool-data"); + dir +} + +#[cfg(test)] +mod tests { + use std::env::{current_dir, temp_dir}; + use std::fs::{create_dir, remove_dir}; + use std::path::PathBuf; + + use serial_test::serial; + + use crate::paths::storyteller_root::TEST_HOME; + use crate::paths::seed_tool_data_root::{get_seed_tool_data_root, TEST_SEED_TOOL_DATA_ROOT}; + + #[test] + #[serial] + fn test_seed_tool_data_home_env_var() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_SEED_TOOL_DATA_ROOT, "/testing/storyteller/root/custom-seed-tool-data"); + assert_eq!(get_seed_tool_data_root(), PathBuf::from("/testing/storyteller/root/custom-seed-tool-data")); + std::env::remove_var(TEST_SEED_TOOL_DATA_ROOT); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_no_directories_exist() { + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, "/testing/fake_home"); + assert_eq!(get_seed_tool_data_root(), PathBuf::from("/testing/fake_home/code/storyteller/seed-tool-data")); + std::env::remove_var(TEST_HOME); + } + + #[test] + #[serial] + fn test_relative_to_home_directory_with_existing_directory() { + let mut fake_home = temp_dir(); + fake_home.push("temp_dir_for_testing"); + + let mut fake_dev = fake_home.clone(); + fake_dev.push("dev"); + + // NB: Final path should be /tmp/temp_dir_for_testing/dev/storyteller + let mut fake_storyteller = fake_dev.clone(); + fake_storyteller.push("storyteller"); + + // NB: Ordered for repeatability + let _r = remove_dir(&fake_home).ok(); + let _r = create_dir(&fake_home).ok(); + let _r = create_dir(&fake_dev).ok(); + let _r = create_dir(&fake_storyteller).ok(); + + // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters + // more complicated, tests run in parallel and in any order. We're using a special + // test variable so we never infect other tests. + std::env::set_var(TEST_HOME, fake_home.to_str().expect("should be string")); + + let mut expected = fake_storyteller.clone(); + expected.push("seed-tool-data"); + + let actual = get_seed_tool_data_root(); + + if actual.starts_with("/private") && !expected.starts_with("/private") { + // NB: For Mac, TempDir creates in /var/... which is an alias of /private/var/... + // We'll make sure to canonicalize both paths. + let mut corrected = PathBuf::from("/private"); + + expected.iter() + .filter(|component| !component.to_string_lossy().eq("/")) + .for_each(|component| corrected.push(component)); + + expected = corrected; + } + + assert_eq!(actual, expected); + + std::env::remove_var(TEST_HOME); + + // NB: Must remove in reverse order, or we leave dangling pointers + let _r = remove_dir(&fake_storyteller).ok(); + let _r = remove_dir(&fake_dev).ok(); + let _r = remove_dir(&fake_home).ok(); + } + + #[test] + #[serial] + fn test_relative_to_current_working_directory() { + // NB: In this test we have no "TEST_HOME" directory. + let current_working_dir = current_dir().expect("Should have working dir."); + + let mut expected_dir = current_working_dir.clone(); + expected_dir.push("code/storyteller/seed-tool-data"); + + assert_eq!(get_seed_tool_data_root(), expected_dir); + } +} diff --git a/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs b/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs index 0747caaec9..3f17a79862 100644 --- a/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs +++ b/crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs @@ -76,12 +76,15 @@ pub async fn process_tacotron_model<'a, 'b>( info!("File hash: {}", private_bucket_hash); + // FIXME(bt,2023-11-27): 'bucket_path_unifier' is super deprecated. Do not use it anymore or for anything except TT2! let model_bucket_path = job_state.bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash); info!("Destination bucket path: {:?}", &model_bucket_path); redis_logger.log_status("uploading tacotron TTS model")?; + // TODO(bt,2023-11-27): This method of uploading model files is super deprecated. + // Try to standardize on something resembling media_files going forward. if let Err(e) = job_state.bucket_client.upload_filename(&model_bucket_path, &file_path).await { safe_delete_temp_file(&output_metadata_fs_path); safe_delete_temp_file(&file_path); @@ -111,6 +114,7 @@ pub async fn process_tacotron_model<'a, 'b>( creator_set_visibility: Visibility::Public, // TODO: All models default to public at start private_bucket_hash: &private_bucket_hash, private_bucket_object_name: &model_bucket_path, + maybe_model_token: None, // NB: This parameter is for internal testing only mysql_pool: &job_state.mysql_pool, }).await?; @@ -122,7 +126,7 @@ pub async fn process_tacotron_model<'a, 'b>( })?; Ok(JobResults { - entity_token: Some(model_token), + entity_token: Some(model_token.to_string()), entity_type: Some(TtsModelType::Tacotron2.to_string()), // NB: This may be different from `GenericDownloadType` in the future! }) } diff --git a/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs b/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs index 31b7194f9a..715f94f454 100644 --- a/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs +++ b/crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs @@ -144,6 +144,7 @@ pub async fn process_vits_model<'a, 'b>( creator_set_visibility: Visibility::Public, // TODO: All models default to public at start private_bucket_hash: &private_bucket_hash, private_bucket_object_name: "", // TODO: This should go away. + maybe_model_token: None, // NB: This parameter is for internal testing only mysql_pool: &job_state.mysql_pool, }).await?; @@ -155,7 +156,7 @@ pub async fn process_vits_model<'a, 'b>( })?; Ok(JobResults { - entity_token: Some(model_token), + entity_token: Some(model_token.to_string()), entity_type: Some(TtsModelType::Vits.to_string()), // NB: This may be different from `GenericDownloadType` in the future! }) } diff --git a/crates/service/plugins/users/src/endpoints/create_account_handler.rs b/crates/service/plugins/users/src/endpoints/create_account_handler.rs index a0ea2a2ccf..5d0d17a95a 100644 --- a/crates/service/plugins/users/src/endpoints/create_account_handler.rs +++ b/crates/service/plugins/users/src/endpoints/create_account_handler.rs @@ -160,6 +160,7 @@ pub async fn create_account_handler( email_gravatar_hash: &email_gravatar_hash, password_hash: &password_hash, ip_address: &ip_address, + maybe_user_token: None, // NB: This parameter is for internal testing only } ).await;