diff --git a/.env b/.env index 91b73890b7..86389b51fb 100644 --- a/.env +++ b/.env @@ -17,6 +17,9 @@ TTS_DOWNLOAD_BUCKET_ROOT=/user_uploaded_tts_models W2L_PRIVATE_DOWNLOAD_BUCKET_NAME=dev-vocodes-private-uploads W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public + +# Bucket roots +AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index 2f688ac1d6..df185ada83 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -132,6 +132,7 @@ pub async fn enqueue_infer_w2l_with_uploads( let mut maybe_uuid_idempotency_token: Option = None; let mut maybe_template_token: Option = None; let mut maybe_audio_file_name : Option = None; + let mut audio_bytes = BytesMut::with_capacity(0); while let Ok(Some(mut field)) = payload.try_next().await { let mut field_name = "".to_string(); @@ -173,7 +174,7 @@ pub async fn enqueue_infer_w2l_with_uploads( InferW2lWithUploadError::ServerError })?; - let bytes = match maybe_bytes { + audio_bytes = match maybe_bytes { Some(bytes) => bytes, None => { warn!("Empty file uploaded"); @@ -235,13 +236,20 @@ pub async fn enqueue_infer_w2l_with_uploads( let audio_upload_bucket_path = hash_to_bucket_path( &upload_uuid, - Some("/audio_uploads") // TODO: pull from configs + Some(&server_state.audio_uploads_bucket_root) ).map_err(|e| { warn!("Hash bucket path error: {:?}", e); InferW2lWithUploadError::ServerError })?; - // TODO: Upload audio to bucket! + info!("Uploading audio to bucket..."); + server_state.private_bucket_client.upload_file( + &audio_upload_bucket_path, + audio_bytes.as_ref()).await + .map_err(|e| { + warn!("Upload audio bytes to bucket error: {:?}", e); + InferW2lWithUploadError::ServerError + })?; // ==================== SAVE JOB RECORD ==================== // diff --git a/src/main.rs b/src/main.rs index 628fea5b7c..c800d58f00 100644 --- a/src/main.rs +++ b/src/main.rs @@ -48,18 +48,36 @@ use crate::http_server::endpoints::w2l::list_w2l_templates::list_w2l_templates_h use crate::http_server::endpoints::w2l::get_w2l_template::get_w2l_template_handler; use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler; use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; +use crate::buckets::bucket_client::BucketClient; const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +// Buckets (shared config) +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT"; + pub type AnyhowResult = anyhow::Result; #[actix_web::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + info!("Obtaining hostname..."); let server_hostname = hostname::get() @@ -93,6 +111,33 @@ async fn main() -> AnyhowResult<()> { let cookie_manager = CookieManager::new(&cookie_domain, &hmac_secret); let session_checker = SessionChecker::new(&cookie_manager); + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + + // Private and Public Buckets + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + + // Bucket roots + let audio_uploads_bucket_root= easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; + + let private_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + )?; + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + )?; + let server_state = ServerState { env_config: EnvConfig { num_workers, @@ -105,6 +150,9 @@ async fn main() -> AnyhowResult<()> { mysql_pool: pool, cookie_manager, session_checker, + private_bucket_client, + public_bucket_client, + audio_uploads_bucket_root, }; serve(server_state) diff --git a/src/server_state.rs b/src/server_state.rs index 0a8f4f01c4..01824d453e 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,6 +1,7 @@ use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; use sqlx::MySqlPool; +use crate::buckets::bucket_client::BucketClient; /// State that is injected into every endpoint. #[derive(Clone)] @@ -15,6 +16,12 @@ pub struct ServerState { pub cookie_manager: CookieManager, pub session_checker: SessionChecker, + + pub private_bucket_client: BucketClient, + pub public_bucket_client: BucketClient, + + /// Where to store audio uploads for w2l + pub audio_uploads_bucket_root: String, } #[derive(Clone)] diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index c9b899e992..680b6942a6 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -87,6 +87,8 @@ struct Downloader { async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. let _ = dotenv::from_filename(".env-secrets").ok(); info!("Obtaining hostname...");