diff --git a/Cargo.lock b/Cargo.lock index 269452c5bf..85499b1cbf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1504,6 +1504,7 @@ version = "0.0.1" dependencies = [ "anyhow", "base64 0.13.0", + "buckets", "chrono", "config", "container_common", diff --git a/_migrations/2022-12-15-060826_media_uploads/up.sql b/_migrations/2022-12-15-060826_media_uploads/up.sql index 98f599199a..c5c8b50d61 100644 --- a/_migrations/2022-12-15-060826_media_uploads/up.sql +++ b/_migrations/2022-12-15-060826_media_uploads/up.sql @@ -60,6 +60,10 @@ CREATE TABLE media_uploads ( -- ========== UPLOAD, TRANSCODING, AND TRUNCATION DETAILS ========== + -- The hash for the bucket directory that contains the original upload + -- as well as any transcodings, downsamplings, etc. + public_bucket_directory_hash VARCHAR(32) NOT NULL, + -- TODO(bt, 2022-12-20): Before landing, rename this field be better and indicate this -- is a *directory*. Add a second field to contain the default file name. -- The directory this media is uploaded to will be exclusive for this file. diff --git a/crates/_types/buckets/src/public/media_uploads/directory.rs b/crates/_types/buckets/src/public/media_uploads/directory.rs index 405172a135..551ffa595d 100644 --- a/crates/_types/buckets/src/public/media_uploads/directory.rs +++ b/crates/_types/buckets/src/public/media_uploads/directory.rs @@ -1,6 +1,8 @@ use crate::public::public_path::PublicPath; use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; +// TODO: Generate these from a macro. + // TODO: Use a central path registry for quick reference const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media"; @@ -30,6 +32,10 @@ impl MediaUploadDirectory { pub fn get_directory_path_str(&self) -> &str { &self.directory } + + pub fn get_object_hash(&self) -> &str { + &self.object_hash + } } #[cfg(test)] @@ -47,4 +53,11 @@ mod tests { let directory = MediaUploadDirectory::from_object_hash("foo"); assert_eq!(directory.get_directory_path_str(), "/media/f/o/foo"); } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let directory = MediaUploadDirectory::from_object_hash(hash); + assert_eq!(directory.get_object_hash(), hash); + } } diff --git a/crates/_types/buckets/src/public/media_uploads/original_file.rs b/crates/_types/buckets/src/public/media_uploads/original_file.rs index 3ec71fac29..c27fb1cf80 100644 --- a/crates/_types/buckets/src/public/media_uploads/original_file.rs +++ b/crates/_types/buckets/src/public/media_uploads/original_file.rs @@ -3,6 +3,8 @@ use crate::public::public_path::PublicPath; const ORIGINAL_FILE_BASENAME : &'static str = "original_contents.bin"; +// TODO: Generate these from a macro. + /// The original user upload file. /// It may have derivative files (down samples, crops, etc.) that live alongside it. #[derive(Clone)] @@ -25,13 +27,21 @@ impl MediaUploadOriginalFilePath { } } - pub fn get_basename() -> &'static str { - ORIGINAL_FILE_BASENAME - } - pub fn get_full_object_path_str(&self) -> &str { &self.full_object_path } + + pub fn get_directory(&self) -> &MediaUploadDirectory { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.directory.get_object_hash() + } + + pub fn get_basename() -> &'static str { + ORIGINAL_FILE_BASENAME + } } #[cfg(test)] @@ -49,4 +59,18 @@ mod tests { let file = MediaUploadOriginalFilePath::from_object_hash("foo"); assert_eq!(file.get_full_object_path_str(), "/media/f/o/foo/original_contents.bin"); } + + #[test] + pub fn get_full_object_path_str_starts_with_directory() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + let full_path = file.get_full_object_path_str(); + assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let file = MediaUploadOriginalFilePath::from_object_hash(hash); + assert_eq!(file.get_object_hash(), hash); + } } diff --git a/crates/lib/database_queries/Cargo.toml b/crates/lib/database_queries/Cargo.toml index 17f0537c4a..c2ae02522e 100644 --- a/crates/lib/database_queries/Cargo.toml +++ b/crates/lib/database_queries/Cargo.toml @@ -15,6 +15,7 @@ path = "src/lib.rs" [dependencies] # Internal +buckets = { path = "../../_types/buckets" } config = { path = "../config" } container_common = { path = "../container_common" } enums = { path = "../../_types/enums" } diff --git a/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs b/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs index 9145110069..c028f437a1 100644 --- a/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs +++ b/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs @@ -1,4 +1,5 @@ use anyhow::anyhow; +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use container_common::anyhow_result::AnyhowResult; use enums::core::visibility::Visibility; use enums::files::media_upload_type::MediaUploadType; @@ -22,7 +23,7 @@ pub struct Args <'a> { pub maybe_original_frame_height: Option, pub checksum_sha2: &'a str, - pub public_bucket_directory_full_path: &'a str, + pub public_upload_path: &'a MediaUploadOriginalFilePath, pub extra_file_modification_info: MediaUploadDetails, pub maybe_creator_user_token: Option<&'a UserToken>, @@ -54,6 +55,7 @@ SET maybe_original_frame_height = ?, checksum_sha2 = ?, + public_bucket_directory_hash = ?, public_bucket_directory_full_path = ?, extra_file_modification_info = ?, @@ -78,7 +80,8 @@ SET args.maybe_original_frame_height, args.checksum_sha2, - args.public_bucket_directory_full_path, + args.public_upload_path.get_object_hash(), + args.public_upload_path.get_full_object_path_str(), "", // TODO: args.extra_file_modification_info, args.maybe_creator_user_token, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/mod.rs index 91cf062fc9..1c753941e4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/mod.rs @@ -1,3 +1,2 @@ pub mod drain_multipart_request; -pub mod upload_audio; -pub mod upload_video; \ No newline at end of file +pub mod upload_media; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs similarity index 83% rename from crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs rename to crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs index e713269cab..1df3559be2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs @@ -34,17 +34,18 @@ use symphonia::core::formats::FormatOptions; use symphonia::core::io::{MediaSourceStream, ReadOnlySource}; use symphonia::core::meta::MetadataOptions; use symphonia::core::probe::Hint; +use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath; use media::decode_basic_audio_info::decode_basic_audio_info; use tokens::files::media_upload::MediaUploadToken; #[derive(Serialize)] -pub struct UploadAudioSuccessResponse { +pub struct UploadMediaSuccessResponse { pub success: bool, pub upload_token: MediaUploadToken, } #[derive(Debug, Serialize)] -pub enum UploadAudioError { +pub enum UploadMediaError { BadInput(String), NotAuthorized, MustBeLoggedIn, @@ -52,14 +53,14 @@ pub enum UploadAudioError { RateLimited, } -impl ResponseError for UploadAudioError { +impl ResponseError for UploadMediaError { fn status_code(&self) -> StatusCode { match *self { - UploadAudioError::BadInput(_) => StatusCode::BAD_REQUEST, - UploadAudioError::NotAuthorized => StatusCode::UNAUTHORIZED, - UploadAudioError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, - UploadAudioError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - UploadAudioError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + UploadMediaError::BadInput(_) => StatusCode::BAD_REQUEST, + UploadMediaError::NotAuthorized => StatusCode::UNAUTHORIZED, + UploadMediaError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + UploadMediaError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + UploadMediaError::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } @@ -69,17 +70,17 @@ impl ResponseError for UploadAudioError { } // NB: Not using derive_more::Display since Clion doesn't understand it. -impl fmt::Display for UploadAudioError { +impl fmt::Display for UploadMediaError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?}", self) } } -pub async fn upload_audio_handler( +pub async fn upload_media_handler( http_request: HttpRequest, server_state: web::Data>, mut multipart_payload: Multipart, -) -> Result { +) -> Result { // ==================== READ SESSION ==================== // @@ -89,7 +90,7 @@ pub async fn upload_audio_handler( .await .map_err(|e| { warn!("Session checker error: {:?}", e); - UploadAudioError::ServerError + UploadMediaError::ServerError })?; // ==================== RATE LIMIT ==================== // @@ -98,18 +99,18 @@ pub async fn upload_audio_handler( None => &server_state.redis_rate_limiters.logged_out, Some(ref user) => { if user.is_banned { - return Err(UploadAudioError::NotAuthorized); + return Err(UploadMediaError::NotAuthorized); } &server_state.redis_rate_limiters.logged_in }, }; if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { - return Err(UploadAudioError::RateLimited); + return Err(UploadMediaError::RateLimited); } if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) { - return Err(UploadAudioError::RateLimited); + return Err(UploadMediaError::RateLimited); } // ==================== READ MULTIPART REQUEST ==================== // @@ -118,15 +119,15 @@ pub async fn upload_audio_handler( .await .map_err(|e| { // TODO: Error handling could be nicer. - UploadAudioError::BadInput("bad request".to_string()) + UploadMediaError::BadInput("bad request".to_string()) })?; let uuid_idempotency_token = upload_media_request.uuid_idempotency_token - .ok_or(UploadAudioError::BadInput("no uuid".to_string()))?; + .ok_or(UploadMediaError::BadInput("no uuid".to_string()))?; if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { - return Err(UploadAudioError::BadInput(reason)); + return Err(UploadMediaError::BadInput(reason)); } let creator_set_visibility = maybe_user_session @@ -152,16 +153,16 @@ pub async fn upload_audio_handler( .transpose() .map_err(|io_error| { warn!("Problem hashing bytes: {:?}", io_error); - return UploadAudioError::ServerError; + return UploadMediaError::ServerError; })?; let hash = match maybe_hash { - None => return Err(UploadAudioError::BadInput("invalid file".to_string())), + None => return Err(UploadMediaError::BadInput("invalid file".to_string())), Some(hash) => hash, }; let bytes = match upload_media_request.file_bytes { - None => return Err(UploadAudioError::BadInput("invalid file".to_string())), + None => return Err(UploadMediaError::BadInput("invalid file".to_string())), Some(bytes) => bytes, }; @@ -201,7 +202,7 @@ pub async fn upload_audio_handler( None ).map_err(|e| { warn!("file decoding error: {:?}", e); - UploadAudioError::BadInput("could not decode file".to_string()) + UploadMediaError::BadInput("could not decode file".to_string()) })?; maybe_duration_millis = basic_info.duration_millis; @@ -213,10 +214,13 @@ pub async fn upload_audio_handler( Some(m) => m, None => { warn!("Invalid mimetype: {:?}", maybe_mimetype); - return Err(UploadAudioError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype))) + return Err(UploadMediaError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype))) }, }; + let public_upload_path = + MediaUploadOriginalFilePath::from_object_hash("todo"); + let record_id = insert_media_upload(Args { token: &token, uuid_idempotency_token: &uuid_idempotency_token, @@ -230,7 +234,7 @@ pub async fn upload_audio_handler( maybe_original_frame_width: None, maybe_original_frame_height: None, checksum_sha2: &hash, - public_bucket_directory_full_path: "", + public_upload_path: &public_upload_path, extra_file_modification_info: MediaUploadDetails {}, // TODO maybe_creator_user_token: maybe_user_token.as_ref(), maybe_creator_anonymous_visitor_token: None, @@ -242,7 +246,7 @@ pub async fn upload_audio_handler( .await .map_err(|err| { warn!("New generic download creation DB error: {:?}", err); - UploadAudioError::ServerError + UploadMediaError::ServerError })?; info!("new media upload id: {}", record_id); @@ -253,16 +257,16 @@ pub async fn upload_audio_handler( .await .map_err(|e| { warn!("error publishing event: {:?}", e); - UploadAudioError::ServerError + UploadMediaError::ServerError })?; - let response = UploadAudioSuccessResponse { + let response = UploadMediaSuccessResponse { success: true, upload_token: token, }; let body = serde_json::to_string(&response) - .map_err(|e| UploadAudioError::ServerError)?; + .map_err(|e| UploadMediaError::ServerError)?; Ok(HttpResponse::Ok() .content_type("application/json") diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_video.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_video.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index c9f4db34b9..0a129dc2df 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -21,7 +21,7 @@ use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_ref use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler; use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; -use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; +use crate::http_server::endpoints::media_uploads::upload_media::upload_media_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler; use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler; @@ -791,9 +791,9 @@ fn add_media_upload_routes (app: App) -> App InitError = (), >, { - app.service(web::scope("/v1/media_upload") - .service(web::resource("/upload_audio") - .route(web::post().to(upload_audio_handler)) + app.service(web::scope("/v1/media_uploads") + .service(web::resource("/upload") + .route(web::post().to(upload_media_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) )