Merge pull request #170 from storytold/brandon-seed-tt2-models

seed tool: TT2 models (with idempotency + early exit)

_sql/actions/truncate.sql

@@ -2,10 +2,13 @@
 -- noinspection SqlNoDataSourceInspectionForFile
 -- noinspection SqlResolveForFile
 
+-- DO NOT DO THIS IN PRODUCTION.
+-- Just placing this here to be easy to copy/paste in development.
 truncate zs_voices;
 truncate zs_voice_datasets;
 truncate zs_voice_dataset_samples;
 truncate media_files;
 truncate media_uploads;
 truncate generic_inference_jobs;
+truncate tts_models;
 

crates/cli/dev_database_seed/src/bucket_clients.rs

@@ -0,0 +1,51 @@
+use std::time::Duration;
+
+use log::info;
+
+use cloud_storage::bucket_client::BucketClient;
+use errors::AnyhowResult;
+
+pub struct BucketClients {
+  pub public: BucketClient,
+  pub private: BucketClient,
+}
+
+pub fn get_bucket_clients() -> AnyhowResult<BucketClients> {
+  let access_key = easyenv::get_env_string_required("ACCESS_KEY")?;
+  let secret_key = easyenv::get_env_string_required("SECRET_KEY")?;
+  let region_name = easyenv::get_env_string_required("REGION_NAME")?;
+  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
+  let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?;
+
+  // NB: Long timeout for dev rust builds to upload to cloud buckets.
+  // Unoptimized binaries sometimes take a lot of time to upload, presumably due to unoptimized code.
+  let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
+    "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10));
+
+  info!("Configuring public GCS bucket...");
+
+  let public_bucket_client = BucketClient::create(
+    &access_key,
+    &secret_key,
+    &region_name,
+    &public_bucket_name,
+    None,
+    Some(bucket_timeout),
+  )?;
+
+  info!("Configuring private GCS bucket...");
+
+  let private_bucket_client = BucketClient::create(
+    &access_key,
+    &secret_key,
+    &region_name,
+    &private_bucket_name,
+    None,
+    Some(bucket_timeout),
+  )?;
+
+  Ok(BucketClients {
+    public: public_bucket_client,
+    private: private_bucket_client,
+  })
+}

crates/cli/dev_database_seed/src/main.rs

@@ -1,18 +1,18 @@
-use std::time::Duration;
 use elasticsearch::Elasticsearch;
-
 use log::info;
 use sqlx::mysql::MySqlPoolOptions;
 
-use cloud_storage::bucket_client::BucketClient;
 use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
 use errors::AnyhowResult;
-use crate::cli_args::parse_cli_args;
 
+use crate::bucket_clients::get_bucket_clients;
+use crate::cli_args::parse_cli_args;
+use crate::seeding::tts_tacotron2::seed_tts_tacotron2;
 use crate::seeding::users::seed_user_accounts;
 use crate::seeding::voice_conversion::seed_voice_conversion;
 use crate::seeding::zero_shot_tts::seed_zero_shot_tts;
 
+pub mod bucket_clients;
 pub mod cli_args;
 pub mod seeding;
 
@@ -37,10 +37,10 @@ pub async fn main() -> AnyhowResult<()> {
 
   let args = parse_cli_args()?;
 
-  let mut maybe_public_bucket_client = None;
+  let mut maybe_bucket_clients = None;
 
   if args.seed_cloud_bucket {
-    maybe_public_bucket_client = Some(get_bucket_client()?);
+    maybe_bucket_clients = Some(get_bucket_clients()?);
   }
 
   let mut maybe_elasticsearch = None;
@@ -50,36 +50,14 @@ pub async fn main() -> AnyhowResult<()> {
   }
 
   seed_user_accounts(&pool).await?;
-  seed_zero_shot_tts(&pool, maybe_public_bucket_client.as_ref()).await?;
+  seed_zero_shot_tts(&pool, maybe_bucket_clients.as_ref()).await?;
   seed_voice_conversion(&pool).await?;
+  seed_tts_tacotron2(&pool, maybe_bucket_clients.as_ref()).await?;
 
   info!("Done!");
   Ok(())
 }
 
-fn get_bucket_client() -> AnyhowResult<BucketClient> {
-  let access_key = easyenv::get_env_string_required("ACCESS_KEY")?;
-  let secret_key = easyenv::get_env_string_required("SECRET_KEY")?;
-  let region_name = easyenv::get_env_string_required("REGION_NAME")?;
-  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
-
-  let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
-    "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5));
-
-  info!("Configuring GCS bucket...");
-
-  let public_bucket_client = BucketClient::create(
-    &access_key,
-    &secret_key,
-    &region_name,
-    &public_bucket_name,
-    None,
-    Some(bucket_timeout),
-  )?;
-
-  Ok(public_bucket_client)
-}
-
 fn get_elasticsearch_client() -> AnyhowResult<Elasticsearch> {
   // TODO(bt,2023-10-26): Allow connecting to instances by URL instead of the default dev URL.
   let client = Elasticsearch::default();

crates/cli/dev_database_seed/src/seeding/mod.rs

@@ -1,3 +1,4 @@
+pub mod tts_tacotron2;
 pub mod users;
 pub mod voice_conversion;
 pub mod zero_shot_tts;

crates/cli/dev_database_seed/src/seeding/tts_tacotron2.rs

@@ -0,0 +1,181 @@
+use std::path::Path;
+
+use log::{info, warn};
+use sqlx::{MySql, Pool};
+
+use cloud_storage::bucket_path_unifier::BucketPathUnifier;
+use enums::by_table::tts_models::tts_model_type::TtsModelType;
+use enums::common::visibility::Visibility;
+use errors::{anyhow, AnyhowResult};
+use filesys::file_read_bytes::file_read_bytes;
+use filesys::path_to_string::path_to_string;
+use hashing::sha256::sha256_hash_file::sha256_hash_file;
+use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
+use mysql_queries::queries::tts::tts_models::get_tts_model::get_tts_model_by_token;
+use mysql_queries::queries::tts::tts_models::insert_tts_model_from_download_job::{insert_tts_model_from_download_job, InsertTtsModelFromDownloadJobArgs};
+use storyteller_root::get_seed_tool_data_root;
+use tokens::tokens::tts_models::TtsModelToken;
+use tokens::tokens::users::UserToken;
+
+use crate::bucket_clients::BucketClients;
+use crate::seeding::users::HANASHI_USER_TOKEN;
+
+pub async fn seed_tts_tacotron2(mysql_pool: &Pool<MySql>, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> {
+  info!("Seeding TTS (Tacotron2)...");
+
+  // NB: This is idempotent and will only install the models once.
+  let tts_models = [
+    ("Mai Valentine from Yugioh (by Vegito)", HANASHI_USER_TOKEN, "TM:cknxtj6mx3t3", "models/tts/tacotron2/MaiYugioh.pth"),
+    ("Nagito (by Vegito)", HANASHI_USER_TOKEN, "TM:tmhxapghwaf3", "models/tts/tacotron2/Nagito.pth"),
+    ("Son Goku V2 (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "TM:kbvaa4726jdw", "models/tts/tacotron2/SonGokuV2.pth"),
+    ("Vegeta (by Vegito, Arpabet version)", HANASHI_USER_TOKEN, "TM:kvv9xqashp6n", "models/tts/tacotron2/VegetaNew.pth"),
+  ];
+
+  let seed_tool_data_root = get_seed_tool_data_root();
+
+  // TODO(bt,2023-11-27): DO NOT USE BucketPathUnifier for any new code. Only for TT2 and super legacy systems.
+  let bucket_path_unifier = BucketPathUnifier::default_paths();
+
+  for (model_title, user_token, tts_model_token, subdirectory_path) in tts_models {
+    let user_token = UserToken::new_from_str(user_token);
+    let tts_model_token = TtsModelToken::new_from_str(tts_model_token);
+
+    let mut model_file_path = seed_tool_data_root.clone();
+    model_file_path.push(subdirectory_path);
+
+    let result = seed_model(
+      &mysql_pool,
+      &tts_model_token,
+      model_title,
+      &user_token,
+      &model_file_path,
+      maybe_bucket_clients,
+      &bucket_path_unifier
+    ).await;
+
+    match result {
+      Ok(_) => info!("Seeded {}", model_title),
+      Err(err) => warn!(r#"
+        Could not TTS model {} : {:?}
+        (No worries: if there's a duplicate key error, we probably already
+        seeded the TTS model on a previous invocation!)
+      "#, model_title, err),
+    }
+  }
+
+  Ok(())
+}
+
+async fn seed_model(
+  mysql_pool: &Pool<MySql>,
+  tts_model_token: &TtsModelToken,
+  model_title: &str,
+  user_token: &UserToken,
+  model_file_path: &Path,
+  maybe_bucket_clients: Option<&BucketClients>,
+  bucket_path_unifier: &BucketPathUnifier,
+) -> AnyhowResult<()> {
+  info!("Seeding TTS model {} ...", model_title);
+
+  let mut private_bucket_hash = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
+  let mut private_bucket_object_name = "NOT_UPLOADED_BY_SEED_TOOL".to_string();
+
+  if tts_model_exists(mysql_pool, tts_model_token).await? {
+    info!("Model already seeded: {:?}", tts_model_token);
+    return Ok(())
+  }
+
+  if let Some(bucket_clients) = maybe_bucket_clients {
+    let bucket_details = seed_file_to_bucket(model_file_path, bucket_clients, bucket_path_unifier).await?;
+    private_bucket_hash = bucket_details.bucket_hash;
+    private_bucket_object_name = bucket_details.bucket_object_name;
+  }
+
+  let filename = model_file_path
+      .file_name()
+      .map(|name| name.to_str())
+      .flatten()
+      .unwrap_or("");
+
+  insert_tts_model_from_download_job(InsertTtsModelFromDownloadJobArgs {
+    tts_model_type: TtsModelType::Tacotron2,
+    title: model_title,
+    original_download_url: "https://example.com",
+    original_filename: filename,
+    file_size_bytes: 0,
+    creator_user_token: user_token,
+    creator_ip_address: "127.0.0.1",
+    creator_set_visibility: Visibility::Public,
+    private_bucket_hash: &private_bucket_hash,
+    private_bucket_object_name: &private_bucket_object_name,
+    maybe_model_token: Some(&tts_model_token),
+    mysql_pool,
+  }).await?;
+
+  Ok(())
+}
+
+async fn tts_model_exists(
+  mysql_pool: &Pool<MySql>,
+  tts_model_token: &TtsModelToken,
+) -> AnyhowResult<bool> {
+
+  const CAN_SEE_DELETED : bool = true;
+
+  let maybe_model = get_tts_model_by_token(
+    tts_model_token.as_str(),
+    CAN_SEE_DELETED,
+    mysql_pool
+  ).await?;
+
+  Ok(maybe_model.is_some())
+}
+
+struct BucketDetails {
+  bucket_hash: String,
+  bucket_object_name: String,
+}
+
+async fn seed_file_to_bucket(
+  model_file_path: &Path,
+  bucket_clients: &BucketClients,
+  bucket_path_unifier: &BucketPathUnifier
+) -> AnyhowResult<BucketDetails> {
+
+  info!("Uploading Tacotron2 model file {:?} ...", model_file_path);
+
+  // TODO(bt,2023-11-27): DO NOT DO THIS OR COPY THIS PATTERN
+  //  This method of uploading model files is super deprecated and results in inconsistent pathing.
+  //  Try to standardize on something resembling media_files going forward.
+  let private_bucket_hash = sha256_hash_file(&model_file_path)?;
+
+  info!("File hash: {}", private_bucket_hash);
+
+  // FIXME(bt,2023-11-27): DO NOT COPY THIS PATTERN!
+  //  'bucket_path_unifier' is super deprecated. Do not use it anymore or for anything except TT2!
+  let model_bucket_path = bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash);
+
+  //let public_upload_path = MediaFileBucketPath::generate_new(
+  //  Some("sample_"),
+  //  Some(".bin"));
+
+  let model_bucket_path = path_to_string(model_bucket_path);
+
+  info!("Uploading TT2 model to bucket path: {:?}", model_bucket_path);
+
+  info!("Reading TT2 model file: {:?}", model_file_path);
+
+  let bytes = file_read_bytes(model_file_path)?;
+  let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream");
+
+  let _r = bucket_clients.private.upload_file_with_content_type(
+    &model_bucket_path,
+    bytes.as_ref(),
+    mimetype)
+      .await?;
+
+  Ok(BucketDetails {
+    bucket_hash: private_bucket_hash,
+    bucket_object_name: model_bucket_path,
+  })
+}

crates/cli/dev_database_seed/src/seeding/users.rs

@@ -5,25 +5,35 @@ use errors::{anyhow, AnyhowResult};
 use hashing::bcrypt::bcrypt_password_hash::bcrypt_password_hash;
 use hashing::md5::email_to_gravatar_hash::email_to_gravatar_hash;
 use mysql_queries::queries::users::user::create_account::{create_account, CreateAccountArgs};
+use tokens::tokens::users::UserToken;
 
 pub const ADMIN_USERNAME : &str = "admin";
 pub const HANASHI_USERNAME : &str = "hanashi";
 
+// We want deterministic tokens for some records
+pub const ADMIN_USER_TOKEN : &str = "user_4jgh4dk9tmr6t";
+pub const HANASHI_USER_TOKEN : &str = "user_fcfteany0ds60";
+
 pub async fn seed_user_accounts(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
   info!("Seeding user accounts...");
 
   // NB: This is idempotent and will only install the accounts once.
   let users = [
-    (ADMIN_USERNAME, "password"),
-    (HANASHI_USERNAME, "password"),
-    ("test", "password"),
+    (ADMIN_USERNAME, "password", Some(ADMIN_USER_TOKEN)),
+    (HANASHI_USERNAME, "password", Some(HANASHI_USER_TOKEN)),
+    ("test", "password", None),
   ];
 
-  for (username, password) in users {
-    let result = seed_user(username, password, &mysql_pool).await;
+  for (username, password, maybe_user_token) in users {
+    let maybe_user_token = maybe_user_token.map(|token| UserToken::new_from_str(token));
+    let result = seed_user(username, password, maybe_user_token, &mysql_pool).await;
     match result {
       Ok(_) => info!("Seeded {}", username),
-      Err(err) => warn!("Could not seed user {} : {:?}", username, err),
+      Err(err) => warn!(r#"
+        Could not seed user {} : {:?}
+        (No worries: if there's a duplicate key error, we probably already
+        seeded the user on a previous invocation!)
+      "#, username, err),
     }
   }
 
@@ -33,6 +43,7 @@ pub async fn seed_user_accounts(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
 async fn seed_user(
   username: &str,
   password: &str,
+  maybe_user_token: Option<UserToken>,
   mysql_pool: &Pool<MySql>,
 ) -> AnyhowResult<()> {
   info!("Seeding user {} ...", username);
@@ -50,6 +61,7 @@ async fn seed_user(
     email_gravatar_hash: &email_gravatar_hash,
     password_hash: &password_hash,
     ip_address: "127.0.0.1",
+    maybe_user_token: maybe_user_token.as_ref(),
   }).await.map_err(|err| anyhow!("err: {:?}", err))?;
 
   Ok(())

crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs

@@ -22,9 +22,10 @@ use tokens::tokens::users::UserToken;
 use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken;
 use tokens::tokens::zs_voices::ZsVoiceToken;
 
+use crate::bucket_clients::BucketClients;
 use crate::seeding::users::HANASHI_USERNAME;
 
-pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, maybe_bucket_client: Option<&BucketClient>) -> AnyhowResult<()> {
+pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, maybe_bucket_clients: Option<&BucketClients>) -> AnyhowResult<()> {
   info!("Seeding zero shot TTS...");
 
   let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? {
@@ -46,7 +47,7 @@ pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, maybe_bucket_client: O
   ];
 
   for (voice_name, bucket_hash, wav_file, user_token) in records {
-    create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, maybe_bucket_client).await?;
+    create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, maybe_bucket_clients).await?;
   }
 
   Ok(())
@@ -58,7 +59,7 @@ async fn create_voice_records(
   wav_file: &str,
   creator_user_token: &UserToken,
   mysql_pool: &Pool<MySql>,
-  maybe_bucket_client: Option<&BucketClient>,
+  maybe_bucket_clients: Option<&BucketClients>,
 ) -> AnyhowResult<(ZsVoiceToken, ZsVoiceDatasetToken)> {
   info!("Creating voice records for voice {} ...", voice_name);
 
@@ -76,8 +77,8 @@ async fn create_voice_records(
 
   let public_upload_path;
 
-  if let Some(bucket_client) = maybe_bucket_client {
-    public_upload_path = seed_file_to_bucket(wav_file, bucket_client).await?;
+  if let Some(bucket_clients) = maybe_bucket_clients {
+    public_upload_path = seed_file_to_bucket(wav_file, &bucket_clients.public).await?;
   } else {
     public_upload_path = MediaFileBucketPath::from_object_hash("fake", None, None);
   }

crates/lib/mysql_queries/src/mediators/badge_granter.rs

@@ -1,11 +1,12 @@
 use anyhow::anyhow;
 use log::warn;
-use sqlx::MySqlPool;
 use sqlx::error::Error::Database;
 use sqlx::mysql::MySqlQueryResult;
+use sqlx::MySqlPool;
 
 use errors::AnyhowResult;
 use reusable_types::achievements::user_badge::UserBadge;
+use tokens::tokens::users::UserToken;
 
 use crate::mediators::firehose_publisher::FirehosePublisher;
 
@@ -34,8 +35,8 @@ impl BadgeGranter {
   }
 
   /// This needs to be called *after* successful upload.
-  pub async fn maybe_grant_tts_model_uploads_badge(&self, user_token: &str) -> AnyhowResult<()> {
-    let count = self.count_tts_models_uploaded(user_token).await?;
+  pub async fn maybe_grant_tts_model_uploads_badge(&self, user_token: &UserToken) -> AnyhowResult<()> {
+    let count = self.count_tts_models_uploaded(user_token.as_str()).await?;
 
     let mut maybe_badge = None;
 
@@ -68,16 +69,16 @@ impl BadgeGranter {
       None => return Ok(()),
     };
 
-    if self.has_badge(user_token, badge).await? {
+    if self.has_badge(user_token.as_str(), badge).await? {
       return Ok(())
     }
 
     let _record_id = self.insert(
       badge,
-      user_token,
+      user_token.as_str(),
     ).await?;
 
-    self.firehose_publisher.publish_user_badge_granted(user_token, badge.to_db_value())
+    self.firehose_publisher.publish_user_badge_granted(user_token.as_str(), badge.to_db_value())
         .await?;
 
     Ok(())

crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs

@@ -8,6 +8,7 @@ use enums::common::job_status::JobStatus;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use tokens::tokens::generic_download_jobs::DownloadJobToken;
+use tokens::tokens::users::UserToken;
 
 use crate::queries::generic_download::job::_keys::GenericDownloadJobId;
 
@@ -17,7 +18,7 @@ pub struct AvailableDownloadJob {
   pub id: GenericDownloadJobId,
   pub download_job_token: DownloadJobToken,
 
-  pub creator_user_token: String,
+  pub creator_user_token: UserToken,
   pub creator_ip_address: String,
   pub creator_set_visibility: Visibility,
 
@@ -51,7 +52,7 @@ SELECT
   id,
   token,
 
-  creator_user_token,
+  creator_user_token as `creator_user_token: tokens::tokens::users::UserToken`,
   creator_ip_address,
   creator_set_visibility,
 
@@ -143,7 +144,7 @@ struct AvailableDownloadJobRawInternal {
   pub id: i64,
   pub token: String,
 
-  pub creator_user_token: String,
+  pub creator_user_token: UserToken,
   pub creator_ip_address: String,
   pub creator_set_visibility: String,
 

crates/lib/mysql_queries/src/queries/tts/tts_models/insert_tts_model_from_download_job.rs

@@ -7,6 +7,7 @@ use enums::by_table::tts_models::tts_model_type::TtsModelType;
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
 use tokens::tokens::tts_models::TtsModelToken;
+use tokens::tokens::users::UserToken;
 
 pub struct InsertTtsModelFromDownloadJobArgs<'a, P: AsRef<Path>> {
   pub tts_model_type: TtsModelType,
@@ -17,22 +18,30 @@ pub struct InsertTtsModelFromDownloadJobArgs<'a, P: AsRef<Path>> {
   pub original_filename: &'a str,
   pub file_size_bytes: u64,
 
-  pub creator_user_token: &'a str,
+  pub creator_user_token: &'a UserToken,
   pub creator_ip_address: &'a str,
   pub creator_set_visibility: Visibility,
 
   pub private_bucket_hash: &'a str,
   pub private_bucket_object_name: P,
 
+  /// In production code, send this as `None`.
+  /// Only provide an external model token for db integration tests and db seeding tools.
+  /// This allows for knowing the model token a priori.
+  pub maybe_model_token: Option<&'a TtsModelToken>,
+
   pub mysql_pool: &'a MySqlPool,
 }
 
 
 pub async fn insert_tts_model_from_download_job<P: AsRef<Path>>(
   args: InsertTtsModelFromDownloadJobArgs<'_, P>,
-) -> AnyhowResult<(u64, String)> {
+) -> AnyhowResult<(u64, TtsModelToken)> {
 
-  let model_token = TtsModelToken::generate().to_string();
+  let model_token = match args.maybe_model_token {
+    None => TtsModelToken::generate(),
+    Some(model_token) => model_token.clone(),
+  };
 
   let private_bucket_object_name = &args.private_bucket_object_name
       .as_ref()

crates/lib/mysql_queries/src/queries/users/user/create_account.rs

@@ -16,6 +16,11 @@ pub struct CreateAccountArgs<'a> {
   pub email_gravatar_hash: &'a str,
   pub password_hash: &'a str,
   pub ip_address: &'a str,
+
+  /// In production code, send this as `None`.
+  /// Only provide an external user token for db integration tests and db seeding tools.
+  /// This allows for knowing the user token a priori.
+  pub maybe_user_token: Option<&'a UserToken>,
 }
 
 pub struct CreateAccountSuccessResult {
@@ -40,7 +45,10 @@ pub async fn create_account(
   const INITIAL_PROFILE_RENDERED_HTML : &str = "";
   const INITIAL_USER_ROLE: &str = "user";
 
-  let user_token = UserToken::generate();
+  let user_token = match args.maybe_user_token {
+    None => UserToken::generate(),
+    Some(user_token) => user_token.clone(),
+  };
 
   let query_result = sqlx::query!(
         r#"

crates/lib/storyteller_root/src/lib.rs

@@ -31,6 +31,7 @@
 mod paths;
 mod substituted_path;
 
+pub use paths::seed_tool_data_root::get_seed_tool_data_root;
 pub use paths::storyteller_frontend_root::get_storyteller_frontend_root;
 pub use paths::storyteller_ml_root::get_storyteller_ml_root;
 pub use paths::storyteller_root::get_storyteller_root;

crates/lib/storyteller_root/src/paths/mod.rs

@@ -1,4 +1,5 @@
 pub (crate) mod env_get_path;
+pub (crate) mod seed_tool_data_root;
 pub (crate) mod storyteller_frontend_root;
 pub (crate) mod storyteller_ml_root;
 pub (crate) mod storyteller_root;

crates/lib/storyteller_root/src/paths/seed_tool_data_root.rs

@@ -0,0 +1,122 @@
+use std::path::PathBuf;
+
+use crate::get_storyteller_root;
+use crate::paths::env_get_path::env_get_path;
+
+/// The env var that declares where the seed tool data repo lives.
+pub const SEED_TOOL_DATA_ROOT : &str = "SEED_TOOL_DATA_ROOT";
+
+// DO NOT LEAK THIS. THIS IS FOR TESTING ONLY.
+pub (crate) const TEST_SEED_TOOL_DATA_ROOT : &str = "ENV_TEST_SEED_TOOL_DATA_ROOT_DO_NOT_LEAK_";
+
+/// Get the root of the seed-tool-data repo
+pub fn get_seed_tool_data_root() -> PathBuf {
+  // 1) Try "STORYTELLER_RUST_ROOT" env var override. Do not check for path existence.
+  let root = if cfg!(test) { TEST_SEED_TOOL_DATA_ROOT } else { SEED_TOOL_DATA_ROOT };
+
+  if let Ok(Some(path)) = env_get_path(root) {
+    return path;
+  }
+
+  // 2) Relative to "storyteller root"
+  let mut dir = get_storyteller_root();
+  dir.push("seed-tool-data");
+  dir
+}
+
+#[cfg(test)]
+mod tests {
+  use std::env::{current_dir, temp_dir};
+  use std::fs::{create_dir, remove_dir};
+  use std::path::PathBuf;
+
+  use serial_test::serial;
+
+  use crate::paths::storyteller_root::TEST_HOME;
+  use crate::paths::seed_tool_data_root::{get_seed_tool_data_root, TEST_SEED_TOOL_DATA_ROOT};
+
+  #[test]
+  #[serial]
+  fn test_seed_tool_data_home_env_var() {
+    // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters
+    // more complicated, tests run in parallel and in any order. We're using a special
+    // test variable so we never infect other tests.
+    std::env::set_var(TEST_SEED_TOOL_DATA_ROOT, "/testing/storyteller/root/custom-seed-tool-data");
+    assert_eq!(get_seed_tool_data_root(), PathBuf::from("/testing/storyteller/root/custom-seed-tool-data"));
+    std::env::remove_var(TEST_SEED_TOOL_DATA_ROOT);
+  }
+
+  #[test]
+  #[serial]
+  fn test_relative_to_home_directory_no_directories_exist() {
+    // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters
+    // more complicated, tests run in parallel and in any order. We're using a special
+    // test variable so we never infect other tests.
+    std::env::set_var(TEST_HOME, "/testing/fake_home");
+    assert_eq!(get_seed_tool_data_root(), PathBuf::from("/testing/fake_home/code/storyteller/seed-tool-data"));
+    std::env::remove_var(TEST_HOME);
+  }
+
+  #[test]
+  #[serial]
+  fn test_relative_to_home_directory_with_existing_directory() {
+    let mut fake_home = temp_dir();
+    fake_home.push("temp_dir_for_testing");
+
+    let mut fake_dev = fake_home.clone();
+    fake_dev.push("dev");
+
+    // NB: Final path should be /tmp/temp_dir_for_testing/dev/storyteller
+    let mut fake_storyteller = fake_dev.clone();
+    fake_storyteller.push("storyteller");
+
+    // NB: Ordered for repeatability
+    let _r = remove_dir(&fake_home).ok();
+    let _r = create_dir(&fake_home).ok();
+    let _r = create_dir(&fake_dev).ok();
+    let _r = create_dir(&fake_storyteller).ok();
+
+    // NB: Careful with `set_var` as it escapes into *all* other tests! To make matters
+    // more complicated, tests run in parallel and in any order. We're using a special
+    // test variable so we never infect other tests.
+    std::env::set_var(TEST_HOME, fake_home.to_str().expect("should be string"));
+
+    let mut expected = fake_storyteller.clone();
+    expected.push("seed-tool-data");
+
+    let actual = get_seed_tool_data_root();
+
+    if actual.starts_with("/private") && !expected.starts_with("/private") {
+      // NB: For Mac, TempDir creates in /var/... which is an alias of /private/var/...
+      // We'll make sure to canonicalize both paths.
+      let mut corrected = PathBuf::from("/private");
+
+      expected.iter()
+          .filter(|component| !component.to_string_lossy().eq("/"))
+          .for_each(|component| corrected.push(component));
+
+      expected = corrected;
+    }
+
+    assert_eq!(actual, expected);
+
+    std::env::remove_var(TEST_HOME);
+
+    // NB: Must remove in reverse order, or we leave dangling pointers
+    let _r = remove_dir(&fake_storyteller).ok();
+    let _r = remove_dir(&fake_dev).ok();
+    let _r = remove_dir(&fake_home).ok();
+  }
+
+  #[test]
+  #[serial]
+  fn test_relative_to_current_working_directory() {
+    // NB: In this test we have no "TEST_HOME" directory.
+    let current_working_dir = current_dir().expect("Should have working dir.");
+
+    let mut expected_dir = current_working_dir.clone();
+    expected_dir.push("code/storyteller/seed-tool-data");
+
+    assert_eq!(get_seed_tool_data_root(), expected_dir);
+  }
+}

crates/service/job/download_job/src/job_types/tts/tacotron/process_tacotron_model.rs

@@ -76,12 +76,15 @@ pub async fn process_tacotron_model<'a, 'b>(
 
   info!("File hash: {}", private_bucket_hash);
 
+  // FIXME(bt,2023-11-27): 'bucket_path_unifier' is super deprecated. Do not use it anymore or for anything except TT2!
   let model_bucket_path = job_state.bucket_path_unifier.tts_synthesizer_path(&private_bucket_hash);
 
   info!("Destination bucket path: {:?}", &model_bucket_path);
 
   redis_logger.log_status("uploading tacotron TTS model")?;
 
+  // TODO(bt,2023-11-27): This method of uploading model files is super deprecated.
+  //  Try to standardize on something resembling media_files going forward.
   if let Err(e) = job_state.bucket_client.upload_filename(&model_bucket_path, &file_path).await {
     safe_delete_temp_file(&output_metadata_fs_path);
     safe_delete_temp_file(&file_path);
@@ -111,6 +114,7 @@ pub async fn process_tacotron_model<'a, 'b>(
     creator_set_visibility: Visibility::Public, // TODO: All models default to public at start
     private_bucket_hash: &private_bucket_hash,
     private_bucket_object_name: &model_bucket_path,
+    maybe_model_token: None, // NB: This parameter is for internal testing only
     mysql_pool: &job_state.mysql_pool,
   }).await?;
 
@@ -122,7 +126,7 @@ pub async fn process_tacotron_model<'a, 'b>(
       })?;
 
   Ok(JobResults {
-    entity_token: Some(model_token),
+    entity_token: Some(model_token.to_string()),
     entity_type: Some(TtsModelType::Tacotron2.to_string()), // NB: This may be different from `GenericDownloadType` in the future!
   })
 }

crates/service/job/download_job/src/job_types/tts/vits/process_vits_model.rs

@@ -144,6 +144,7 @@ pub async fn process_vits_model<'a, 'b>(
     creator_set_visibility: Visibility::Public, // TODO: All models default to public at start
     private_bucket_hash: &private_bucket_hash,
     private_bucket_object_name: "", // TODO: This should go away.
+    maybe_model_token: None, // NB: This parameter is for internal testing only
     mysql_pool: &job_state.mysql_pool,
   }).await?;
 
@@ -155,7 +156,7 @@ pub async fn process_vits_model<'a, 'b>(
       })?;
 
   Ok(JobResults {
-    entity_token: Some(model_token),
+    entity_token: Some(model_token.to_string()),
     entity_type: Some(TtsModelType::Vits.to_string()), // NB: This may be different from `GenericDownloadType` in the future!
   })
 }

crates/service/plugins/users/src/endpoints/create_account_handler.rs

@@ -160,6 +160,7 @@ pub async fn create_account_handler(
       email_gravatar_hash: &email_gravatar_hash,
       password_hash: &password_hash,
       ip_address: &ip_address,
+      maybe_user_token: None, // NB: This parameter is for internal testing only
     }
   ).await;