upload audio to bucket

.env

@@ -17,6 +17,9 @@ TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models
 
 W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads
 W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public
+
+# Bucket roots
+AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
 W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates
 
 W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip

src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs

@@ -132,6 +132,7 @@ pub async fn enqueue_infer_w2l_with_uploads(
   let mut maybe_uuid_idempotency_token: Option<String> = None;
   let mut maybe_template_token: Option<String> = None;
   let mut maybe_audio_file_name : Option<String> = None;
+  let mut audio_bytes = BytesMut::with_capacity(0);
 
   while let Ok(Some(mut field)) = payload.try_next().await {
     let mut field_name = "".to_string();
@@ -173,7 +174,7 @@ pub async fn enqueue_infer_w2l_with_uploads(
             InferW2lWithUploadError::ServerError
           })?;
 
-        let bytes = match maybe_bytes {
+        audio_bytes = match maybe_bytes {
           Some(bytes) => bytes,
           None => {
             warn!("Empty file uploaded");
@@ -235,13 +236,20 @@ pub async fn enqueue_infer_w2l_with_uploads(
 
   let audio_upload_bucket_path = hash_to_bucket_path(
     &upload_uuid,
-    Some("/audio_uploads") // TODO: pull from configs
+    Some(&server_state.audio_uploads_bucket_root)
   ).map_err(|e| {
     warn!("Hash bucket path error: {:?}", e);
     InferW2lWithUploadError::ServerError
   })?;
 
-  // TODO: Upload audio to bucket!
+  info!("Uploading audio to bucket...");
+  server_state.private_bucket_client.upload_file(
+    &audio_upload_bucket_path,
+    audio_bytes.as_ref()).await
+    .map_err(|e| {
+      warn!("Upload audio bytes to bucket error: {:?}", e);
+      InferW2lWithUploadError::ServerError
+    })?;
 
   // ==================== SAVE JOB RECORD ==================== //
 

src/main.rs

@@ -48,18 +48,36 @@ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_h
 use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler;
 use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler;
 use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads;
+use crate::buckets::bucket_client::BucketClient;
 
 const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345";
 
 // NB: sqlx::query is spammy and logs all queries as "info"-level
 const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn";
 
+// Buckets (shared config)
+const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
+const ENV_SECRET_KEY : &'static str = "SECRET_KEY";
+const ENV_REGION_NAME : &'static str = "REGION_NAME";
+
+// Buckets (private data)
+const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME";
+// Buckets (public data)
+const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME";
+
+// Various bucket roots
+const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT";
+
 pub type AnyhowResult<T> = anyhow::Result<T>;
 
 #[actix_web::main]
 async fn main() -> AnyhowResult<()> {
   easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
 
+  // NB: Do not check this secrets-containing dotenv file into VCS.
+  // This file should only contain *development* secrets, never production.
+  let _ = dotenv::from_filename(".env-secrets").ok();
+
   info!("Obtaining hostname...");
 
   let server_hostname = hostname::get()
@@ -93,6 +111,33 @@ async fn main() -> AnyhowResult<()> {
   let cookie_manager = CookieManager::new(&cookie_domain, &hmac_secret);
   let session_checker = SessionChecker::new(&cookie_manager);
 
+  let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
+  let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
+  let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
+
+  // Private and Public Buckets
+  let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
+  let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
+
+  // Bucket roots
+  let audio_uploads_bucket_root= easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?;
+
+  let private_bucket_client = BucketClient::create(
+    &access_key,
+    &secret_key,
+    &region_name,
+    &private_bucket_name,
+    None,
+  )?;
+
+  let public_bucket_client = BucketClient::create(
+    &access_key,
+    &secret_key,
+    &region_name,
+    &public_bucket_name,
+    None,
+  )?;
+
   let server_state = ServerState {
     env_config: EnvConfig {
       num_workers,
@@ -105,6 +150,9 @@ async fn main() -> AnyhowResult<()> {
     mysql_pool: pool,
     cookie_manager,
     session_checker,
+    private_bucket_client,
+    public_bucket_client,
+    audio_uploads_bucket_root,
   };
 
   serve(server_state)

src/server_state.rs

@@ -1,6 +1,7 @@
 use crate::http_server::web_utils::cookie_manager::CookieManager;
 use crate::http_server::web_utils::session_checker::SessionChecker;
 use sqlx::MySqlPool;
+use crate::buckets::bucket_client::BucketClient;
 
 /// State that is injected into every endpoint.
 #[derive(Clone)]
@@ -15,6 +16,12 @@ pub struct ServerState {
   pub cookie_manager: CookieManager,
 
   pub session_checker: SessionChecker,
+
+  pub private_bucket_client: BucketClient,
+  pub public_bucket_client: BucketClient,
+
+  /// Where to store audio uploads for w2l
+  pub audio_uploads_bucket_root: String,
 }
 
 #[derive(Clone)]

src/w2l_download_job.rs

@@ -87,6 +87,8 @@ struct Downloader {
 async fn main() -> AnyhowResult<()> {
   easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
 
+  // NB: Do not check this secrets-containing dotenv file into VCS.
+  // This file should only contain *development* secrets, never production.
   let _ = dotenv::from_filename(".env-secrets").ok();
 
   info!("Obtaining hostname...");