diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 57cdf97f79..ae2fc15751 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -56,6 +56,7 @@ use crate::http_server::endpoints::media_files::list::list_featured_media_files_ use crate::http_server::endpoints::media_files::list::list_media_files_by_batch_token_handler::*; use crate::http_server::endpoints::media_files::list::list_media_files_for_user_handler::*; use crate::http_server::endpoints::media_files::list::list_media_files_handler::*; +use crate::http_server::endpoints::media_files::upload::upload_audio_media_file_handler::*; use crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::*; use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; use crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::*; @@ -111,6 +112,7 @@ use crate::http_server::web_utils::response_success_helpers::*; crate::http_server::endpoints::media_files::list::list_media_files_by_batch_token_handler::list_media_files_by_batch_token_handler, crate::http_server::endpoints::media_files::list::list_media_files_for_user_handler::list_media_files_for_user_handler, crate::http_server::endpoints::media_files::list::list_media_files_handler::list_media_files_handler, + crate::http_server::endpoints::media_files::upload::upload_audio_media_file_handler::upload_audio_media_file_handler, crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler, crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler, crate::http_server::endpoints::media_files::upload::upload_new_engine_asset_media_file_handler::upload_new_engine_asset_media_file_handler, @@ -359,6 +361,8 @@ use crate::http_server::web_utils::response_success_helpers::*; UpdateWeightError, UpdateWeightPathInfo, UpdateWeightRequest, + UploadAudioMediaFileForm, + UploadAudioMediaFileSuccessResponse, UploadEngineAssetMediaSuccessResponse, UploadMediaSuccessResponse, UploadNewEngineAssetFileForm, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs index 25b889b7f2..04474f85c6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs @@ -1,3 +1,4 @@ +pub mod upload_audio_media_file_handler; pub mod upload_engine_asset; pub mod upload_error; pub mod upload_generic; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_audio_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_audio_media_file_handler.rs new file mode 100644 index 0000000000..8eb4ca0d55 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_audio_media_file_handler.rs @@ -0,0 +1,360 @@ +use std::collections::HashSet; +use std::io::{BufReader, Bytes, Cursor, Read}; +use std::path::PathBuf; +use std::sync::Arc; + +use actix_multipart::form::MultipartForm; +use actix_multipart::form::tempfile::TempFile; +use actix_multipart::form::text::Text; +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use log::{error, info, warn}; +use once_cell::sync::Lazy; +use stripe::CreatePaymentLinkShippingAddressCollectionAllowedCountries::Mf; +use utoipa::ToSchema; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory; +use enums::by_table::media_files::media_file_subtype::MediaFileSubtype; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_to_extension::mimetype_to_extension; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::create::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; +use mysql_queries::queries::media_files::get::get_media_file::get_media_file; +use tokens::tokens::media_files::MediaFileToken; +use videos::get_mp4_info::{get_mp4_info, get_mp4_info_for_bytes, get_mp4_info_for_bytes_and_len}; + +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::server_state::ServerState; +use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult}; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; + +/// Form-multipart request fields. +/// +/// IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `UploadAudioMediaFileForm` (Under "Schema") FOR DETAILS ON FIELDS AND NULLABILITY. +#[derive(MultipartForm, ToSchema)] +#[multipart(duplicate_field = "deny")] +pub struct UploadAudioMediaFileForm { + /// UUID for request idempotency + #[multipart(limit = "2 KiB")] + #[schema(value_type = String, format = Binary)] + uuid_idempotency_token: Text, + + // TODO: is MultipartBytes better than TempFile ? + /// The uploaded file + #[multipart(limit = "512 MiB")] + #[schema(value_type = Vec, format = Binary)] + file: TempFile, + + /// Optional: Title (name) of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_title: Option>, + + /// Optional: Visibility of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_visibility: Option>, +} + +// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*. +#[derive(Serialize, ToSchema)] +pub struct UploadAudioMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + // Audio + "audio/aac", + "audio/m4a", + "audio/mpeg", + "audio/ogg", + "audio/opus", + "audio/x-flac", + "audio/x-wav", + // Mixed + "audio/mp4", // iPhone seems to upload these as audio + // Video + "video/mp4", // TODO(bt,2024-05-01): Do we want to allow these as audio? + "video/webm", // TODO(bt,2024-05-01): Do we want to allow these as audio? + ]) +}); + +/// This endpoint is for uploading audio files. +#[utoipa::path( + post, + tag = "Media Files", + path = "/v1/media_files/upload/audio", + responses( + (status = 200, description = "Success Update", body = UploadAudioMediaFileSuccessResponse), + (status = 400, description = "Bad input", body = MediaFileUploadError), + (status = 401, description = "Not authorized", body = MediaFileUploadError), + (status = 429, description = "Too many requests", body = MediaFileUploadError), + (status = 500, description = "Server error", body = MediaFileUploadError), + ), + params( + ( + "request" = UploadAudioMediaFileForm, + description = "IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `UploadAudioMediaFileForm` (Under 'Schema') FOR DETAILS ON FIELDS AND NULLABILITY." + ), + ) +)] +pub async fn upload_audio_media_file_handler( + http_request: HttpRequest, + server_state: web::Data>, + MultipartForm(mut form): MultipartForm, +) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let maybe_user_token = maybe_user_session + .as_ref() + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(MediaFileUploadError::NotAuthorizedVerbose("user is banned".to_string())); + } + } + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(MediaFileUploadError::RateLimited); + } + + // ==================== HANDLE IDEMPOTENCY ==================== // + + // TODO(bt, 2024-02-26): This should be a transaction. + let uuid_idempotency_token = form.uuid_idempotency_token.as_ref(); + + if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) { + return Err(MediaFileUploadError::BadInput(reason)); + } + + insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + MediaFileUploadError::BadInput("invalid idempotency token".to_string()) + })?; + + // ==================== UPLOAD METADATA ==================== // + + let maybe_title = form.maybe_title.map(|title| title.to_string()); + + let creator_set_visibility = form.maybe_visibility + .map(|visibility| visibility.0) + .or_else(|| { + maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) + }) + .unwrap_or(Visibility::default()); + + // ==================== USER DATA ==================== // + + let ip_address = get_request_ip(&http_request); + + // ==================== FILE VALIDATION ==================== // + + let mut file_bytes = Vec::new(); + form.file.file.read_to_end(&mut file_bytes) + .map_err(|e| { + error!("Problem reading file: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let mimetype = get_mimetype_for_bytes(file_bytes.as_ref()) + .map(|mimetype| mimetype.to_string()) + .ok_or_else(|| { + warn!("Could not determine mimetype for file"); + MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string()) + })?; + + if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // ==================== DURATION DETECTION ==================== // + + let do_audio_decode = match mimetype.as_ref() { + // TODO: Revisit when Safari can send us this metadata consistently + "audio/mp4" | "video/mp4" => false, + "audio/opus" => { + // TODO/FIXME(bt, 2023-05-19): Symphonia is currently broken for Firefox's opus. + // We're on an off-master branch that may resolve the problem in the future, but for now + // it panics as follows: + // + // [2023-05-19T10:25:34Z INFO symphonia_core::probe] found a possible format marker within [4f, 67, 67, 53, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, d4, c5] @ 0+2 bytes. + // [2023-05-19T10:25:34Z INFO symphonia_core::probe] found the format marker [4f, 67, 67, 53] @ 0+2 bytes. + // [2023-05-19T10:25:34Z DEBUG symphonia_format_ogg::page] grow page buffer to 8192 bytes + // [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] starting new physical stream + // [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] selected opus mapper for stream with serial=0x19aac5d4 + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Probed! + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Find audio track... + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Found audio track (maybe) + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe track duration: None + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe codec short name: Some("opus") + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Opus handler + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Media source stream + // [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : options... + // [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : try_read... + // [2023-05-19T10:25:34Z DEBUG symphonia_format_mkv::ebml] element with tag: 4F67 + // thread 'actix-rt|system:0|arbiter:1' panicked at 'assertion failed: `(left == right)` + // left: `Unknown`, + // right: `Ebml`: EBML element type must be checked before calling this function', /Users/bt/.cargo/git/checkouts/symphonia-8fbe6c90fc095688/e1a7009/symphonia-format-mkv/src/ebml.rs:335:9 + false + } + // Also, don't decode images + "image/jpeg" => false, + "image/png" => false, + "image/webp" => false, + _ => true, + }; + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + + if do_audio_decode { + let basic_info = decode_basic_audio_bytes_info( + file_bytes.as_ref(), + Some(&mimetype), + None + ).map_err(|e| { + warn!("file decoding error: {:?}", e); + MediaFileUploadError::BadInput("could not decode file".to_string()) + })?; + + maybe_duration_millis = basic_info.duration_millis; + maybe_codec_name = basic_info.codec_name; + } + + // ==================== OTHER FILE METADATA ==================== // + + let maybe_filename = form.file.file_name.as_deref() + .as_deref() + .map(|filename| PathBuf::from(filename)); + + let extension = mimetype_to_extension(&mimetype) + .or_else(|| { + maybe_filename + .as_ref() + .and_then(|filename| filename.extension()) // TODO needs dot prefix + .and_then(|ext| ext.to_str()) + }) + .ok_or_else(|| { + warn!("Could not determine file extension for mimetype: {}", &mimetype); + MediaFileUploadError::ServerError + })?; + + let extension = format!(".{extension}"); // NB: needs dot prefix + + let file_size_bytes = file_bytes.len(); + + let hash = sha256_hash_bytes(&file_bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + MediaFileUploadError::ServerError + })?; + + // ==================== UPLOAD AND SAVE ==================== // + + const PREFIX : Option<&str> = Some("aud_"); + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + &mimetype) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs { + maybe_media_class: Some(MediaFileClass::Audio), + media_file_type: MediaFileType::Audio, + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility, + upload_type: UploadType::Filesystem, // TODO(bt,2024-05-02): This should be a parameter and a well-known enum. + maybe_engine_category: None, + maybe_animation_type: None, + maybe_mime_type: Some(&mimetype), + file_size_bytes: file_size_bytes as u64, + duration_millis: maybe_duration_millis.unwrap_or(0), + sha256_checksum: &hash, + maybe_title: maybe_title.as_deref(), + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix: PREFIX, + maybe_public_bucket_extension: Some(&extension), + pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New file creation DB error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + info!("new media file id: {} token: {:?}", record_id, &token); + + let response = UploadAudioMediaFileSuccessResponse { + success: true, + media_file_token: token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| MediaFileUploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} diff --git a/crates/service/web/storyteller_web/src/routes/media_files_routes.rs b/crates/service/web/storyteller_web/src/routes/media_files_routes.rs index fc2454757d..d484942fb4 100644 --- a/crates/service/web/storyteller_web/src/routes/media_files_routes.rs +++ b/crates/service/web/storyteller_web/src/routes/media_files_routes.rs @@ -14,6 +14,7 @@ use crate::http_server::endpoints::media_files::list::list_featured_media_files_ use crate::http_server::endpoints::media_files::list::list_media_files_handler::list_media_files_handler; use crate::http_server::endpoints::media_files::list::list_media_files_by_batch_token_handler::list_media_files_by_batch_token_handler; use crate::http_server::endpoints::media_files::list::list_media_files_for_user_handler::list_media_files_for_user_handler; +use crate::http_server::endpoints::media_files::upload::upload_audio_media_file_handler::upload_audio_media_file_handler; use crate::http_server::endpoints::media_files::upload::upload_engine_asset::upload_engine_asset_media_file_handler::upload_engine_asset_media_file_handler; use crate::http_server::endpoints::media_files::upload::upload_generic::upload_media_file_handler::upload_media_file_handler; use crate::http_server::endpoints::media_files::upload::upload_new_engine_asset_media_file_handler::upload_new_engine_asset_media_file_handler; @@ -107,5 +108,9 @@ pub fn add_media_file_routes (app: App) -> App .route(web::post().to(upload_saved_scene_media_file_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/upload/audio") + .route(web::post().to(upload_audio_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) }