New Actix Form Multipart upload endpoint with annotations

crates/service/web/storyteller_web/src/api_doc.rs

@@ -62,6 +62,7 @@ use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileU
 use crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::*;
 use crate::http_server::endpoints::media_files::upload::upload_video::upload_video_media_file_handler::*;
 use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::*;
+use crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::*;
 use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
 use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*;
 use crate::http_server::endpoints::moderation::user_feature_flags::edit_user_feature_flags_handler::*;
@@ -110,6 +111,7 @@ use crate::http_server::web_utils::response_success_helpers::*;
     crate::http_server::endpoints::media_files::list::list_media_files_handler::list_media_files_handler,
     crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler,
     crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler,
+    crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::upload_new_scene_media_file_handler,
     crate::http_server::endpoints::media_files::upload::upload_video::upload_video_media_file_handler::upload_video_media_file_handler,
     crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler,
     crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler,
@@ -356,6 +358,8 @@ use crate::http_server::web_utils::response_success_helpers::*;
     UpdateWeightRequest,
     UploadEngineAssetMediaSuccessResponse,
     UploadMediaSuccessResponse,
+    UploadNewSceneMediaFileForm,
+    UploadNewSceneMediaFileSuccessResponse,
     UploadVideoMediaSuccessResponse,
     UserBookmarkDetailsForUserList,
     UserBookmarkEntityType,

crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs

@@ -1,4 +1,5 @@
 pub mod upload_engine_asset;
 pub mod upload_error;
 pub mod upload_generic;
+pub mod upload_new_scene_media_file_handler;
 pub mod upload_video;

crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs

@@ -4,11 +4,13 @@ use actix_web::HttpResponse;
 use utoipa::ToSchema;
 
 use http_server_common::response::serialize_as_json_error::serialize_as_json_error;
+use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
 
 #[derive(Debug, Serialize, ToSchema)]
 pub enum MediaFileUploadError {
   BadInput(String),
   NotAuthorized,
+  NotAuthorizedVerbose(String),
   MustBeLoggedIn,
   ServerError,
   RateLimited,
@@ -17,11 +19,12 @@ pub enum MediaFileUploadError {
 impl ResponseError for MediaFileUploadError {
   fn status_code(&self) -> StatusCode {
     match *self {
-      MediaFileUploadError::BadInput(_) => StatusCode::BAD_REQUEST,
-      MediaFileUploadError::NotAuthorized => StatusCode::UNAUTHORIZED,
-      MediaFileUploadError::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
-      MediaFileUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
-      MediaFileUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
+      Self::BadInput(_) => StatusCode::BAD_REQUEST,
+      Self::NotAuthorized => StatusCode::UNAUTHORIZED,
+      Self::NotAuthorizedVerbose(_) => StatusCode::UNAUTHORIZED,
+      Self::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
+      Self::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
+      Self::RateLimited => StatusCode::TOO_MANY_REQUESTS,
     }
   }
 

crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_scene_media_file_handler.rs

@@ -0,0 +1,251 @@
+use std::collections::HashSet;
+use std::io::{BufReader, Bytes, Cursor, Read};
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use actix_multipart::form::MultipartForm;
+use actix_multipart::form::tempfile::TempFile;
+use actix_multipart::form::text::Text;
+use actix_multipart::Multipart;
+use actix_web::{HttpRequest, HttpResponse, web};
+use log::{error, info, warn};
+use once_cell::sync::Lazy;
+use stripe::CreatePaymentLinkShippingAddressCollectionAllowedCountries::Mf;
+use utoipa::ToSchema;
+
+use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
+use enums::by_table::media_files::media_file_class::MediaFileClass;
+use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
+use enums::by_table::media_files::media_file_subtype::MediaFileSubtype;
+use enums::by_table::media_files::media_file_type::MediaFileType;
+use enums::common::visibility::Visibility;
+use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
+use http_server_common::request::get_request_ip::get_request_ip;
+use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
+use mimetypes::mimetype_to_extension::mimetype_to_extension;
+use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
+use mysql_queries::queries::media_files::create::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
+use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
+use tokens::tokens::media_files::MediaFileToken;
+use videos::get_mp4_info::{get_mp4_info, get_mp4_info_for_bytes, get_mp4_info_for_bytes_and_len};
+
+use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
+use crate::server_state::ServerState;
+use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult};
+use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format;
+
+#[derive(MultipartForm, ToSchema)]
+#[multipart(duplicate_field = "deny")]
+pub struct UploadNewSceneMediaFileForm {
+  /// UUID for request idempotency
+  #[multipart(limit = "2 KiB")]
+  uuid_idempotency_token: Text<String>,
+
+  /// The uploaded file
+  #[multipart(limit = "512 MiB")]
+  file: TempFile,
+
+  /// Optional: Title (name) of the scene
+  #[multipart(limit = "2 KiB")]
+  title: Option<Text<String>>,
+
+  /// Optional: Visibility of the scene
+  #[multipart(limit = "2 KiB")]
+  visibility: Option<Text<Visibility>>,
+}
+
+// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*.
+#[derive(Serialize, ToSchema)]
+pub struct UploadNewSceneMediaFileSuccessResponse {
+  pub success: bool,
+  pub media_file_token: MediaFileToken,
+}
+
+#[utoipa::path(
+  post,
+  tag = "Media Files",
+  path = "/v1/media_files/upload/new_scene",
+  responses(
+    (status = 200, description = "Success Update", body = UploadNewSceneMediaFileSuccessResponse),
+    (status = 400, description = "Bad input", body = MediaFileUploadError),
+    (status = 401, description = "Not authorized", body = MediaFileUploadError),
+    (status = 429, description = "Too many requests", body = MediaFileUploadError),
+    (status = 500, description = "Server error", body = MediaFileUploadError),
+  ),
+  params(
+    ("request" = (), description = "Ask Brandon. This is form-multipart."),
+  )
+)]
+pub async fn upload_new_scene_media_file_handler(
+  http_request: HttpRequest,
+  server_state: web::Data<Arc<ServerState>>,
+  MultipartForm(mut form): MultipartForm<UploadNewSceneMediaFileForm>,
+  mut multipart_payload: Multipart,
+) -> Result<HttpResponse, MediaFileUploadError> {
+
+  let mut mysql_connection = server_state.mysql_pool
+      .acquire()
+      .await
+      .map_err(|err| {
+        error!("MySql pool error: {:?}", err);
+        MediaFileUploadError::ServerError
+      })?;
+
+  // ==================== READ SESSION ==================== //
+
+  let maybe_user_session = server_state
+      .session_checker
+      .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
+      .await
+      .map_err(|e| {
+        error!("Session checker error: {:?}", e);
+        MediaFileUploadError::ServerError
+      })?;
+
+  let maybe_user_token = maybe_user_session
+      .as_ref()
+      .map(|session| session.get_strongly_typed_user_token());
+
+  let maybe_avt_token = server_state
+      .avt_cookie_manager
+      .get_avt_token_from_request(&http_request);
+
+  // ==================== BANNED USERS ==================== //
+
+  if let Some(ref user) = maybe_user_session {
+    if user.is_banned {
+      return Err(MediaFileUploadError::NotAuthorizedVerbose("user is banned".to_string()));
+    }
+  }
+
+  // ==================== RATE LIMIT ==================== //
+
+  let rate_limiter = match maybe_user_session {
+    None => &server_state.redis_rate_limiters.file_upload_logged_out,
+    Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in,
+  };
+
+  if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
+    return Err(MediaFileUploadError::RateLimited);
+  }
+
+  // ==================== HANDLE IDEMPOTENCY ==================== //
+
+  // TODO(bt, 2024-02-26): This should be a transaction.
+  let uuid_idempotency_token = form.uuid_idempotency_token.as_ref();
+
+  if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) {
+    return Err(MediaFileUploadError::BadInput(reason));
+  }
+
+  insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection)
+      .await
+      .map_err(|err| {
+        error!("Error inserting idempotency token: {:?}", err);
+        MediaFileUploadError::BadInput("invalid idempotency token".to_string())
+      })?;
+
+  // ==================== UPLOAD METADATA ==================== //
+
+  let creator_set_visibility = maybe_user_session
+      .as_ref()
+      .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control.
+      .unwrap_or(Visibility::default());
+
+  // ==================== USER DATA ==================== //
+
+  let ip_address = get_request_ip(&http_request);
+
+  // ==================== FILE DATA ==================== //
+
+  let maybe_filename = form.file.file_name.as_deref()
+      .as_deref()
+      .map(|filename| PathBuf::from(filename));
+
+  let maybe_file_extension = maybe_filename
+      .as_ref()
+      .and_then(|filename| filename.extension())
+      .and_then(|ext| ext.to_str());
+
+  let mut file_bytes = Vec::new();
+  form.file.file.read_to_end(&mut file_bytes)
+      .map_err(|e| {
+        error!("Problem reading file: {:?}", e);
+        MediaFileUploadError::ServerError
+      })?;
+
+  let file_size_bytes = file_bytes.len();
+
+  let hash = sha256_hash_bytes(&file_bytes)
+      .map_err(|io_error| {
+        error!("Problem hashing bytes: {:?}", io_error);
+        MediaFileUploadError::ServerError
+      })?;
+
+  // ==================== UPLOAD AND SAVE ==================== //
+
+  // TODO(bt,2024-03-26): At first I thought we should map these to the existing file paths on upsert,
+  //  but now I'm thinking we can just lead cruft in the bucket and clean it later. We don't have the
+  //  benefit of restoring old versions (if we mapped to existing paths but had a versioning scheme),
+  //  but we can move fast.
+
+  const MIMETYPE: &str = "application/json";
+  const PREFIX : Option<&str> = Some("upload_");
+  const SUFFIX: &str = ".scn.json";
+
+  let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(SUFFIX));
+
+  info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
+
+  server_state.public_bucket_client.upload_file_with_content_type(
+    public_upload_path.get_full_object_path_str(),
+    file_bytes.as_ref(),
+    MIMETYPE)
+      .await
+      .map_err(|e| {
+        warn!("Upload media bytes to bucket error: {:?}", e);
+        MediaFileUploadError::ServerError
+      })?;
+
+  let title = form.title.map(|title| title.to_string());
+
+  let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
+    maybe_media_class: Some(MediaFileClass::Dimensional),
+    media_file_type: MediaFileType::SceneJson,
+    maybe_creator_user_token: maybe_user_token.as_ref(),
+    maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
+    creator_ip_address: &ip_address,
+    creator_set_visibility,
+    upload_type: UploadType::Filesystem,
+    maybe_engine_category: Some(MediaFileEngineCategory::Scene),
+    maybe_animation_type: None,
+    maybe_mime_type: Some(MIMETYPE),
+    file_size_bytes: file_size_bytes as u64,
+    duration_millis: 0,
+    sha256_checksum: &hash,
+    maybe_title: title.as_deref(),
+    public_bucket_directory_hash: public_upload_path.get_object_hash(),
+    maybe_public_bucket_prefix: PREFIX,
+    maybe_public_bucket_extension: Some(SUFFIX),
+    pool: &server_state.mysql_pool,
+  })
+      .await
+      .map_err(|err| {
+        warn!("New file creation DB error: {:?}", err);
+        MediaFileUploadError::ServerError
+      })?;
+
+  info!("new media file id: {} token: {:?}", record_id, &token);
+
+  let response = UploadNewSceneMediaFileSuccessResponse {
+    success: true,
+    media_file_token: token,
+  };
+
+  let body = serde_json::to_string(&response)
+      .map_err(|e| MediaFileUploadError::ServerError)?;
+
+  return Ok(HttpResponse::Ok()
+      .content_type("application/json")
+      .body(body));
+}

crates/service/web/storyteller_web/src/routes/media_files_routes.rs

@@ -16,6 +16,7 @@ use crate::http_server::endpoints::media_files::list::list_media_files_by_batch_
 use crate::http_server::endpoints::media_files::list::list_media_files_for_user_handler::list_media_files_for_user_handler;
 use crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler;
 use crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler;
+use crate::http_server::endpoints::media_files::upload::upload_new_scene_media_file_handler::upload_new_scene_media_file_handler;
 use crate::http_server::endpoints::media_files::upload::upload_video::upload_video_media_file_handler::upload_video_media_file_handler;
 use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler;
 use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler;
@@ -92,5 +93,9 @@ pub fn add_media_file_routes<T, B> (app: App<T>) -> App<T>
           .route(web::post().to(write_scene_file_media_file_handler))
           .route(web::head().to(|| HttpResponse::Ok()))
       )
+      .service(web::resource("/upload/new_scene")
+          .route(web::post().to(upload_new_scene_media_file_handler))
+          .route(web::head().to(|| HttpResponse::Ok()))
+      )
   )
 }