diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 2a67955851..0219bc9bf0 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -62,6 +62,7 @@ use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileU use crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::*; use crate::http_server::endpoints::media_files::upload::upload_video::upload_video_media_file_handler::*; use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::*; +use crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::*; use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError; use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*; use crate::http_server::endpoints::moderation::user_feature_flags::edit_user_feature_flags_handler::*; @@ -110,6 +111,7 @@ use crate::http_server::web_utils::response_success_helpers::*; crate::http_server::endpoints::media_files::list::list_media_files_handler::list_media_files_handler, crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler, crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler, + crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::upload_new_scene_media_file_handler, crate::http_server::endpoints::media_files::upload::upload_video::upload_video_media_file_handler::upload_video_media_file_handler, crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler, crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler, @@ -356,6 +358,8 @@ use crate::http_server::web_utils::response_success_helpers::*; UpdateWeightRequest, UploadEngineAssetMediaSuccessResponse, UploadMediaSuccessResponse, + UploadNewSceneMediaFileForm, + UploadNewSceneMediaFileSuccessResponse, UploadVideoMediaSuccessResponse, UserBookmarkDetailsForUserList, UserBookmarkEntityType, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs index 4bead6e942..0eabc8a7d3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs @@ -1,4 +1,5 @@ pub mod upload_engine_asset; pub mod upload_error; pub mod upload_generic; +pub mod upload_new_scene_media_file_handler; pub mod upload_video; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs index 3f07d4615f..e316f88bb6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs @@ -4,11 +4,13 @@ use actix_web::HttpResponse; use utoipa::ToSchema; use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError; #[derive(Debug, Serialize, ToSchema)] pub enum MediaFileUploadError { BadInput(String), NotAuthorized, + NotAuthorizedVerbose(String), MustBeLoggedIn, ServerError, RateLimited, @@ -17,11 +19,12 @@ pub enum MediaFileUploadError { impl ResponseError for MediaFileUploadError { fn status_code(&self) -> StatusCode { match *self { - MediaFileUploadError::BadInput(_) => StatusCode::BAD_REQUEST, - MediaFileUploadError::NotAuthorized => StatusCode::UNAUTHORIZED, - MediaFileUploadError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, - MediaFileUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, - MediaFileUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + Self::BadInput(_) => StatusCode::BAD_REQUEST, + Self::NotAuthorized => StatusCode::UNAUTHORIZED, + Self::NotAuthorizedVerbose(_) => StatusCode::UNAUTHORIZED, + Self::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + Self::RateLimited => StatusCode::TOO_MANY_REQUESTS, } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_scene_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_scene_media_file_handler.rs new file mode 100644 index 0000000000..1562ab3b26 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_scene_media_file_handler.rs @@ -0,0 +1,251 @@ +use std::collections::HashSet; +use std::io::{BufReader, Bytes, Cursor, Read}; +use std::path::PathBuf; +use std::sync::Arc; + +use actix_multipart::form::MultipartForm; +use actix_multipart::form::tempfile::TempFile; +use actix_multipart::form::text::Text; +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use log::{error, info, warn}; +use once_cell::sync::Lazy; +use stripe::CreatePaymentLinkShippingAddressCollectionAllowedCountries::Mf; +use utoipa::ToSchema; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory; +use enums::by_table::media_files::media_file_subtype::MediaFileSubtype; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_to_extension::mimetype_to_extension; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::create::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; +use mysql_queries::queries::media_files::get::get_media_file::get_media_file; +use tokens::tokens::media_files::MediaFileToken; +use videos::get_mp4_info::{get_mp4_info, get_mp4_info_for_bytes, get_mp4_info_for_bytes_and_len}; + +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::server_state::ServerState; +use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult}; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; + +#[derive(MultipartForm, ToSchema)] +#[multipart(duplicate_field = "deny")] +pub struct UploadNewSceneMediaFileForm { + /// UUID for request idempotency + #[multipart(limit = "2 KiB")] + uuid_idempotency_token: Text, + + /// The uploaded file + #[multipart(limit = "512 MiB")] + file: TempFile, + + /// Optional: Title (name) of the scene + #[multipart(limit = "2 KiB")] + title: Option>, + + /// Optional: Visibility of the scene + #[multipart(limit = "2 KiB")] + visibility: Option>, +} + +// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*. +#[derive(Serialize, ToSchema)] +pub struct UploadNewSceneMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} + +#[utoipa::path( + post, + tag = "Media Files", + path = "/v1/media_files/upload/new_scene", + responses( + (status = 200, description = "Success Update", body = UploadNewSceneMediaFileSuccessResponse), + (status = 400, description = "Bad input", body = MediaFileUploadError), + (status = 401, description = "Not authorized", body = MediaFileUploadError), + (status = 429, description = "Too many requests", body = MediaFileUploadError), + (status = 500, description = "Server error", body = MediaFileUploadError), + ), + params( + ("request" = (), description = "Ask Brandon. This is form-multipart."), + ) +)] +pub async fn upload_new_scene_media_file_handler( + http_request: HttpRequest, + server_state: web::Data>, + MultipartForm(mut form): MultipartForm, + mut multipart_payload: Multipart, +) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let maybe_user_token = maybe_user_session + .as_ref() + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(MediaFileUploadError::NotAuthorizedVerbose("user is banned".to_string())); + } + } + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(MediaFileUploadError::RateLimited); + } + + // ==================== HANDLE IDEMPOTENCY ==================== // + + // TODO(bt, 2024-02-26): This should be a transaction. + let uuid_idempotency_token = form.uuid_idempotency_token.as_ref(); + + if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) { + return Err(MediaFileUploadError::BadInput(reason)); + } + + insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + MediaFileUploadError::BadInput("invalid idempotency token".to_string()) + })?; + + // ==================== UPLOAD METADATA ==================== // + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. + .unwrap_or(Visibility::default()); + + // ==================== USER DATA ==================== // + + let ip_address = get_request_ip(&http_request); + + // ==================== FILE DATA ==================== // + + let maybe_filename = form.file.file_name.as_deref() + .as_deref() + .map(|filename| PathBuf::from(filename)); + + let maybe_file_extension = maybe_filename + .as_ref() + .and_then(|filename| filename.extension()) + .and_then(|ext| ext.to_str()); + + let mut file_bytes = Vec::new(); + form.file.file.read_to_end(&mut file_bytes) + .map_err(|e| { + error!("Problem reading file: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let file_size_bytes = file_bytes.len(); + + let hash = sha256_hash_bytes(&file_bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + MediaFileUploadError::ServerError + })?; + + // ==================== UPLOAD AND SAVE ==================== // + + // TODO(bt,2024-03-26): At first I thought we should map these to the existing file paths on upsert, + // but now I'm thinking we can just lead cruft in the bucket and clean it later. We don't have the + // benefit of restoring old versions (if we mapped to existing paths but had a versioning scheme), + // but we can move fast. + + const MIMETYPE: &str = "application/json"; + const PREFIX : Option<&str> = Some("upload_"); + const SUFFIX: &str = ".scn.json"; + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(SUFFIX)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + MIMETYPE) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let title = form.title.map(|title| title.to_string()); + + let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs { + maybe_media_class: Some(MediaFileClass::Dimensional), + media_file_type: MediaFileType::SceneJson, + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility, + upload_type: UploadType::Filesystem, + maybe_engine_category: Some(MediaFileEngineCategory::Scene), + maybe_animation_type: None, + maybe_mime_type: Some(MIMETYPE), + file_size_bytes: file_size_bytes as u64, + duration_millis: 0, + sha256_checksum: &hash, + maybe_title: title.as_deref(), + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix: PREFIX, + maybe_public_bucket_extension: Some(SUFFIX), + pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New file creation DB error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + info!("new media file id: {} token: {:?}", record_id, &token); + + let response = UploadNewSceneMediaFileSuccessResponse { + success: true, + media_file_token: token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| MediaFileUploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} diff --git a/crates/service/web/storyteller_web/src/routes/media_files_routes.rs b/crates/service/web/storyteller_web/src/routes/media_files_routes.rs index 11a7ac2a2c..4365f8acff 100644 --- a/crates/service/web/storyteller_web/src/routes/media_files_routes.rs +++ b/crates/service/web/storyteller_web/src/routes/media_files_routes.rs @@ -16,6 +16,7 @@ use crate::http_server::endpoints::media_files::list::list_media_files_by_batch_ use crate::http_server::endpoints::media_files::list::list_media_files_for_user_handler::list_media_files_for_user_handler; use crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler; use crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler; +use crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::upload_new_scene_media_file_handler; use crate::http_server::endpoints::media_files::upload::upload_video::upload_video_media_file_handler::upload_video_media_file_handler; use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler; use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler; @@ -92,5 +93,9 @@ pub fn add_media_file_routes (app: App) -> App .route(web::post().to(write_scene_file_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/upload/new_scene") + .route(web::post().to(upload_new_scene_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) }