Merge pull request #112 from storytold/echelon-seed-samples

Update seed tool to seed and upload samples

Cargo.lock

@@ -1829,14 +1829,20 @@ dependencies = [
 name = "dev-database-seed"
 version = "0.0.1"
 dependencies = [
+ "buckets",
+ "cloud_storage",
  "config",
  "easyenv",
  "enums",
  "errors",
+ "filesys",
  "hashing",
+ "idempotency",
  "log",
+ "mimetypes",
  "mysql_queries",
  "sqlx",
+ "storyteller_root",
  "tokens",
  "tokio 1.25.0",
 ]

crates/cli/dev_database_seed/Cargo.toml

@@ -14,12 +14,18 @@ path = "src/main.rs"
 
 [dependencies]
 # Internal
+buckets = { path = "../../_types/buckets" }
+cloud_storage = { path = "../../lib/api_clients/cloud_storage" }
 config = { path = "../../lib/deprecated/config" }
 easyenv = { path = "../../lib/easyenv" }
 enums = { path = "../../_types/enums" }
 errors = { path = "../../lib/errors" }
+filesys = { path = "../../lib/files/filesys" }
 hashing = { path = "../../lib/files/hashing" }
+idempotency = { path = "../../lib/idempotency" }
+mimetypes = { path = "../../lib/files/mimetypes" }
 mysql_queries = { path = "../../lib/mysql_queries" }
+storyteller_root = { path = "../../lib/storyteller_root" }
 tokens = { path = "../../_types/tokens" }
 
 # External

crates/cli/dev_database_seed/src/main.rs

@@ -1,6 +1,9 @@
+use std::time::Duration;
+
 use log::info;
 use sqlx::mysql::MySqlPoolOptions;
 
+use cloud_storage::bucket_client::BucketClient;
 use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
 use errors::AnyhowResult;
 
@@ -16,6 +19,9 @@ pub async fn main() -> AnyhowResult<()> {
 
   info!("Database seed CLI script.");
 
+  // NB: Read secrets (eg. ACCESS_KEY)
+  easyenv::from_filename(".env-secrets")?;
+
   let db_connection_string =
       easyenv::get_env_string_or_default(
         "MYSQL_URL",
@@ -26,10 +32,35 @@ pub async fn main() -> AnyhowResult<()> {
       .connect(&db_connection_string)
       .await?;
 
+  let public_bucket_client = get_bucket_client()?;
+
   seed_user_accounts(&pool).await?;
-  seed_zero_shot_tts(&pool).await?;
+  seed_zero_shot_tts(&pool, &public_bucket_client).await?;
   seed_voice_conversion(&pool).await?;
 
   info!("Done!");
   Ok(())
 }
+
+fn get_bucket_client() -> AnyhowResult<BucketClient> {
+  let access_key = easyenv::get_env_string_required("ACCESS_KEY")?;
+  let secret_key = easyenv::get_env_string_required("SECRET_KEY")?;
+  let region_name = easyenv::get_env_string_required("REGION_NAME")?;
+  let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
+
+  let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
+    "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5));
+
+  info!("Configuring GCS bucket...");
+
+  let public_bucket_client = BucketClient::create(
+    &access_key,
+    &secret_key,
+    &region_name,
+    &public_bucket_name,
+    None,
+    Some(bucket_timeout),
+  )?;
+
+  Ok(public_bucket_client)
+}

crates/cli/dev_database_seed/src/seeding/zero_shot_tts.rs

@@ -1,15 +1,28 @@
+use std::path::PathBuf;
 use log::info;
 use sqlx::{MySql, Pool};
 
+use buckets::public::media_files::original_file::MediaFileBucketPath;
+use cloud_storage::bucket_client::BucketClient;
+use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
+use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
+use enums::common::visibility::Visibility;
 use errors::{anyhow, AnyhowResult};
+use filesys::file_read_bytes::file_read_bytes;
+use idempotency::uuid::generate_random_uuid;
+use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
 use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username;
 use mysql_queries::queries::voice_designer::datasets::create_dataset::{create_dataset, CreateDatasetArgs};
+use mysql_queries::queries::voice_designer::voice_samples::insert_dataset_sample_and_media_file::{insert_dataset_sample_and_media_file, InsertDatasetSampleAndMediaFileArgs};
 use mysql_queries::queries::voice_designer::voices::create_voice::{create_voice, CreateVoiceArgs};
+use storyteller_root::get_storyteller_rust_root;
 use tokens::tokens::users::UserToken;
+use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken;
+use tokens::tokens::zs_voices::ZsVoiceToken;
 
 use crate::seeding::users::HANASHI_USERNAME;
 
-pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
+pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>, bucket_client: &BucketClient) -> AnyhowResult<()> {
   info!("Seeding zero shot TTS...");
 
   let user_token = match get_user_token_by_username(HANASHI_USERNAME, mysql_pool).await? {
@@ -20,16 +33,16 @@ pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
   let records = [
     // NB: The bucket hashes here are already uploaded to the development Google Cloud Storage
     // bucket and should be usable if you have the development secrets on your machine.
-    ("Alice", "qtqaprnd5shtybve4fqpvcfp50yjw238", &user_token),
-    ("Biden", "n945w0xsq15xrh16hc147a5mc1a91gwh", &user_token),
-    ("Goku", "cnnv05yjst2m737dpmxazgfpksjf4y7c", &user_token),
-    ("Hilary", "7wav68ba2yy86491jk36cgk36tkmzesr", &user_token),
-    ("Obama", "z3gy4v56sgtfrxfrpvaj7v74sqc67rcq", &user_token),
-    ("Trump", "qcy7pv3rph0ntkqnpz5cfg9ksyh7kkz5", &user_token),
+    ("Alice", "qtqaprnd5shtybve4fqpvcfp50yjw238", "alice.wav", &user_token),
+    ("Biden", "n945w0xsq15xrh16hc147a5mc1a91gwh", "biden.wav", &user_token),
+    ("Goku", "cnnv05yjst2m737dpmxazgfpksjf4y7c", "goku.wav", &user_token),
+    ("Hilary", "7wav68ba2yy86491jk36cgk36tkmzesr", "hilary.wav", &user_token),
+    ("Obama", "z3gy4v56sgtfrxfrpvaj7v74sqc67rcq", "obama.wav", &user_token),
+    ("Trump", "qcy7pv3rph0ntkqnpz5cfg9ksyh7kkz5", "trump.wav", &user_token),
   ];
 
-  for (voice_name, bucket_hash, user_token) in records {
-    create_voice_records(voice_name, bucket_hash, user_token, mysql_pool).await?;
+  for (voice_name, bucket_hash, wav_file, user_token) in records {
+    create_voice_records(voice_name, bucket_hash, wav_file, user_token, mysql_pool, bucket_client).await?;
   }
 
   Ok(())
@@ -38,12 +51,14 @@ pub async fn seed_zero_shot_tts(mysql_pool: &Pool<MySql>) -> AnyhowResult<()> {
 async fn create_voice_records(
   voice_name: &str,
   bucket_hash: &str,
+  wav_file: &str,
   creator_user_token: &UserToken,
   mysql_pool: &Pool<MySql>,
-) -> AnyhowResult<()> {
+  bucket_client: &BucketClient,
+) -> AnyhowResult<(ZsVoiceToken, ZsVoiceDatasetToken)> {
   info!("Creating voice records for voice {} ...", voice_name);
 
-  let dataset_title = format!("{} dataset", voice_name);
+  let dataset_title = format!("{} Dataset", voice_name);
 
   info!("Creating dataset...");
 
@@ -51,11 +66,53 @@ async fn create_voice_records(
     dataset_title: &dataset_title,
     maybe_creator_user_token: Some(creator_user_token.as_str()),
     creator_ip_address: "127.0.0.1",
-    creator_set_visibility: &Default::default(),
+    creator_set_visibility: Visibility::Public,
     mysql_pool,
   }).await?;
 
-  info!("Creating voice...");
+  info!("Uploading wav file {} ...", wav_file);
+
+  let public_upload_path = MediaFileBucketPath::generate_new(
+    Some("dev_zs_"),
+    Some(".bin"));
+
+  info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
+
+  let mut file_path = get_storyteller_rust_root();
+  file_path.push(format!("assets/seed/{}", wav_file));
+
+  info!("Reading seed file: {:?}", file_path);
+
+  let bytes = file_read_bytes(file_path)?;
+  let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("audio/wav");
+
+  let _r = bucket_client.upload_file_with_content_type(
+    public_upload_path.get_full_object_path_str(),
+    bytes.as_ref(),
+    mimetype)
+      .await?;
+
+
+  info!("Creating dataset sample record...");
+
+  let uuid_idempotency_token = generate_random_uuid();
+
+  let (_sample_token, _media_file_token, _id) =
+      insert_dataset_sample_and_media_file(InsertDatasetSampleAndMediaFileArgs {
+        uuid_idempotency_token: &uuid_idempotency_token,
+        dataset_token: &dataset_token,
+        media_type: MediaUploadType::Audio,
+        origin_category: MediaFileOriginCategory::Upload,
+        checksum_sha2: "", // TODO
+        media_file_path: &public_upload_path,
+        maybe_creator_user_token: Some(&creator_user_token),
+        maybe_creator_anonymous_visitor_token: None,
+        creator_ip_address: "127.0.0.1",
+        creator_set_visibility: Visibility::Public,
+        mysql_pool,
+      }).await?;
+
+  info!("Creating voice record...");
 
   let voice_token = create_voice(CreateVoiceArgs {
     dataset_token: &dataset_token,
@@ -67,9 +124,9 @@ async fn create_voice_records(
     bucket_hash: &bucket_hash,
     maybe_creator_user_token: Some(&creator_user_token),
     creator_ip_address: "127.0.0.1",
-    creator_set_visibility: &Default::default(),
+    creator_set_visibility: Visibility::Public,
     mysql_pool,
   }).await?;
 
-  Ok(())
+  Ok((voice_token, dataset_token))
 }

crates/lib/mysql_queries/src/queries/voice_designer/datasets/create_dataset.rs

@@ -12,7 +12,7 @@ pub struct CreateDatasetArgs<'a> {
     pub dataset_title: &'a str,
     pub maybe_creator_user_token: Option<&'a str>,
     pub creator_ip_address: &'a str,
-    pub creator_set_visibility: &'a Visibility,
+    pub creator_set_visibility: Visibility,
     pub mysql_pool: &'a MySqlPool
 }
 

crates/lib/mysql_queries/src/queries/voice_designer/voice_samples/insert_dataset_sample_and_media_file.rs

@@ -17,11 +17,11 @@ pub struct SyntheticIdRecord {
   pub next_id: i64,
 }
 
-pub struct Args <'a> {
+pub struct InsertDatasetSampleAndMediaFileArgs<'a> {
   pub uuid_idempotency_token: &'a str,
 
   // Dataset to which this sample belongs
-  pub dataset_token: ZsVoiceDatasetToken,
+  pub dataset_token: &'a ZsVoiceDatasetToken,
 
   pub media_type: MediaUploadType,
 
@@ -51,7 +51,7 @@ pub struct Args <'a> {
 
 const ORIGIN_PRODUCT : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Unknown; // TODO: Zero shot voice enum
 
-pub async fn insert_dataset_sample_and_media_file(args: Args<'_>) -> AnyhowResult<(ZsVoiceDatasetSampleToken, MediaFileToken, u64)> {
+pub async fn insert_dataset_sample_and_media_file(args: InsertDatasetSampleAndMediaFileArgs<'_>) -> AnyhowResult<(ZsVoiceDatasetSampleToken, MediaFileToken, u64)> {
 
   let mut maybe_creator_synthetic_id : Option<u64> = None;
 

crates/lib/mysql_queries/src/queries/voice_designer/voices/create_voice.rs

@@ -26,7 +26,7 @@ pub struct CreateVoiceArgs<'a> {
   pub maybe_creator_user_token: Option<&'a UserToken>,
 
   pub creator_ip_address: &'a str,
-  pub creator_set_visibility: &'a Visibility,
+  pub creator_set_visibility: Visibility,
   pub mysql_pool: &'a MySqlPool
 }
 

crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_dataset_samples/upload_sample.rs

@@ -19,7 +19,7 @@ use media::decode_basic_audio_info::decode_basic_audio_bytes_info;
 use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
 use mysql_queries::queries::media_uploads::insert_media_upload::insert_media_upload;
 use mysql_queries::queries::voice_designer::voice_samples::get_dataset_sample_by_uuid::get_dataset_sample_by_uuid_with_connection;
-use mysql_queries::queries::voice_designer::voice_samples::insert_dataset_sample_and_media_file::{Args, insert_dataset_sample_and_media_file};
+use mysql_queries::queries::voice_designer::voice_samples::insert_dataset_sample_and_media_file::{InsertDatasetSampleAndMediaFileArgs, insert_dataset_sample_and_media_file};
 use tokens::tokens::media_files::MediaFileToken;
 use tokens::tokens::zs_voice_dataset_samples::ZsVoiceDatasetSampleToken;
 use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken;
@@ -325,11 +325,11 @@ pub async fn upload_sample_handler(
         UploadSampleError::ServerError
       })?;
 
-  let (dataset_sample_token, media_file_token, _record_id) = insert_dataset_sample_and_media_file(Args {
+  let (dataset_sample_token, media_file_token, _record_id) = insert_dataset_sample_and_media_file(InsertDatasetSampleAndMediaFileArgs {
     uuid_idempotency_token: &uuid_idempotency_token,
     media_type: media_upload_type,
     origin_category: media_file_origin,
-    dataset_token,
+    dataset_token: &dataset_token,
     //maybe_original_filename: upload_media_request.file_name.as_deref(),
     //original_file_size_bytes: file_size_bytes as u64,
     //maybe_original_duration_millis: maybe_duration_millis,

crates/service/web/storyteller_web/src/http_server/endpoints/voice_designer/voice_datasets/create_dataset.rs

@@ -88,7 +88,7 @@ pub async fn create_dataset_handler(http_request: HttpRequest, request: web::Jso
       dataset_title: &title,
       maybe_creator_user_token: Some(user_session.user_token.clone().as_ref()),
       creator_ip_address: &creator_ip_address,
-      creator_set_visibility: &creator_set_visibility,
+      creator_set_visibility,
       mysql_pool: &server_state.mysql_pool
   }).await;