From dd41c268e44b6c29a5e606bd24b7c63b732fa97c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 21 Dec 2023 00:38:04 -0500 Subject: [PATCH] media file upload --- ...d4076cc0bd7809b6c2cb5740151d6583a6d11.json | 12 + .../generic_synthetic_ids/id_category.rs | 15 +- .../insert_media_file_from_file_upload.rs | 158 ++++++++ .../src/queries/media_files/mod.rs | 1 + .../http_server/endpoints/media_files/mod.rs | 2 + .../upload/drain_multipart_request.rs | 91 +++++ .../endpoints/media_files/upload/mod.rs | 3 + .../upload/process_upload_media_file.rs | 344 ++++++++++++++++++ .../media_files/upload/upload_error.rs | 36 ++ .../media_files/upload_media_file.rs | 64 ++++ .../service/web/storyteller_web/src/routes.rs | 5 + 11 files changed, 729 insertions(+), 2 deletions(-) create mode 100644 .sqlx/query-fed337e365f0bd36cfa543e0ce9d4076cc0bd7809b6c2cb5740151d6583a6d11.json create mode 100644 crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_file_upload.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/drain_multipart_request.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/process_upload_media_file.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload_media_file.rs diff --git a/.sqlx/query-fed337e365f0bd36cfa543e0ce9d4076cc0bd7809b6c2cb5740151d6583a6d11.json b/.sqlx/query-fed337e365f0bd36cfa543e0ce9d4076cc0bd7809b6c2cb5740151d6583a6d11.json new file mode 100644 index 0000000000..05eba3a4c9 --- /dev/null +++ b/.sqlx/query-fed337e365f0bd36cfa543e0ce9d4076cc0bd7809b6c2cb5740151d6583a6d11.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n maybe_origin_model_type = NULL,\n maybe_origin_model_token = NULL,\n\n media_type = ?,\n maybe_mime_type = ?,\n file_size_bytes = ?,\n\n checksum_sha2 = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n\n maybe_creator_file_synthetic_id = ?,\n maybe_creator_category_synthetic_id = ?\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 16 + }, + "nullable": [] + }, + "hash": "fed337e365f0bd36cfa543e0ce9d4076cc0bd7809b6c2cb5740151d6583a6d11" +} diff --git a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs index c751d5f5cc..821a66f0ce 100644 --- a/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs +++ b/crates/_types/enums/src/by_table/generic_synthetic_ids/id_category.rs @@ -49,6 +49,10 @@ pub enum IdCategory { #[serde(rename = "model_weights")] ModelWeights, + + /// Files that are uploaded with no general product area they belong to. (Eg. local dev testing) + #[serde(rename = "file_upload")] + FileUpload, } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -68,6 +72,7 @@ impl IdCategory { Self::ZeroShotTtsResult => "zs_tts_result", Self::VideoFilterResult => "video_filter", Self::ModelWeights => "model_weights", + Self::FileUpload => "file_upload", } } @@ -82,6 +87,7 @@ impl IdCategory { "zs_tts_result" => Ok(Self::ZeroShotTtsResult), "video_filter" => Ok(Self::VideoFilterResult), "model_weights" => Ok(Self::ModelWeights), + "file_upload" => Ok(Self::FileUpload), _ => Err(format!("invalid value: {:?}", value)), } } @@ -99,6 +105,7 @@ impl IdCategory { Self::ZeroShotVoiceDataset, Self::ZeroShotVoiceEmbedding, Self::ModelWeights, + Self::FileUpload, ]) } } @@ -121,7 +128,8 @@ mod tests { assert_serialization(IdCategory::ZeroShotVoiceDataset, "zs_dataset"); assert_serialization(IdCategory::ZeroShotVoiceEmbedding, "zs_voice"); assert_serialization(IdCategory::ZeroShotTtsResult, "zs_tts_result"); - assert_serialization(IdCategory::ModelWeights, "model_weights") + assert_serialization(IdCategory::ModelWeights, "model_weights"); + assert_serialization(IdCategory::FileUpload, "file_upload"); } #[test] @@ -135,6 +143,7 @@ mod tests { assert_eq!(IdCategory::ZeroShotVoiceEmbedding.to_str(), "zs_voice"); assert_eq!(IdCategory::ZeroShotTtsResult.to_str(), "zs_tts_result"); assert_eq!(IdCategory::ModelWeights.to_str(), "model_weights"); + assert_eq!(IdCategory::FileUpload.to_str(), "file_upload"); } #[test] @@ -148,13 +157,14 @@ mod tests { assert_eq!(IdCategory::from_str("zs_voice").unwrap(), IdCategory::ZeroShotVoiceEmbedding); assert_eq!(IdCategory::from_str("zs_tts_result").unwrap(), IdCategory::ZeroShotTtsResult); assert_eq!(IdCategory::from_str("model_weights").unwrap(), IdCategory::ModelWeights); + assert_eq!(IdCategory::from_str("file_upload").unwrap(), IdCategory::FileUpload); } #[test] fn all_variants() { // Static check let mut variants = IdCategory::all_variants(); - assert_eq!(variants.len(), 9); + assert_eq!(variants.len(), 10); assert_eq!(variants.pop_first(), Some(IdCategory::MediaFile)); assert_eq!(variants.pop_first(), Some(IdCategory::LipsyncAnimationResult)); assert_eq!(variants.pop_first(), Some(IdCategory::VideoFilterResult)); @@ -164,6 +174,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotVoiceDataset)); assert_eq!(variants.pop_first(), Some(IdCategory::ZeroShotVoiceEmbedding)); assert_eq!(variants.pop_first(), Some(IdCategory::ModelWeights)); + assert_eq!(variants.pop_first(), Some(IdCategory::FileUpload)); assert_eq!(variants.pop_first(), None); // Generated check diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_file_upload.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_file_upload.rs new file mode 100644 index 0000000000..724118c80b --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_file_upload.rs @@ -0,0 +1,158 @@ +use anyhow::anyhow; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::generic_synthetic_ids::id_category::IdCategory; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +use crate::queries::generic_synthetic_ids::transactional_increment_generic_synthetic_id::transactional_increment_generic_synthetic_id; + +pub enum UploadType { + Filesystem, + DeviceCaptureApi, +} + +pub struct InsertMediaFileFromUploadArgs<'a> { + pub pool: &'a MySqlPool, + + pub maybe_creator_user_token: Option<&'a UserToken>, + pub maybe_creator_anonymous_visitor_token: Option<&'a AnonymousVisitorTrackingToken>, + pub creator_ip_address: &'a str, + pub creator_set_visibility: Visibility, + + pub upload_type: UploadType, + + pub media_file_type: MediaFileType, + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub duration_millis: u64, + pub sha256_checksum: &'a str, + + // TODO: Media duration. + //pub duration_millis: u64, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, +} + +pub async fn insert_media_file_from_file_upload( + args: InsertMediaFileFromUploadArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let token = MediaFileToken::generate(); + + let origin_category = match args.upload_type { + UploadType::Filesystem => MediaFileOriginCategory::Upload, + UploadType::DeviceCaptureApi => MediaFileOriginCategory::DeviceApi, + }; + + let mut maybe_creator_file_synthetic_id : Option = None; + let mut maybe_creator_category_synthetic_id : Option = None; + + let mut transaction = args.pool.begin().await?; + + if let Some(user_token) = args.maybe_creator_user_token.as_deref() { + + let next_media_file_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::MediaFile, + &mut transaction + ).await?; + + let next_voice_conversion_id = transactional_increment_generic_synthetic_id( + &user_token, + IdCategory::FileUpload, + &mut transaction + ).await?; + + maybe_creator_file_synthetic_id = Some(next_media_file_id); + maybe_creator_category_synthetic_id = Some(next_voice_conversion_id); + } + + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Unknown; + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + + maybe_origin_model_type = NULL, + maybe_origin_model_token = NULL, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + maybe_creator_anonymous_visitor_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ?, + + maybe_creator_file_synthetic_id = ?, + maybe_creator_category_synthetic_id = ? + "#, + token.as_str(), + + origin_category.to_str(), + ORIGIN_PRODUCT_CATEGORY.to_str(), + + args.media_file_type.to_str(), + args.maybe_mime_type, + args.file_size_bytes, + + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + args.maybe_creator_user_token, + args.maybe_creator_anonymous_visitor_token, + args.creator_ip_address, + + args.creator_set_visibility.to_str(), + + maybe_creator_file_synthetic_id, + maybe_creator_category_synthetic_id, + ) + .execute(&mut *transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index cedc16583c..cfdc3eb3c8 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -2,6 +2,7 @@ pub mod delete_media_file; pub mod get_media_file; pub mod insert_media_file_from_cli_tool; pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_file_upload; pub mod insert_media_file_from_rerender; pub mod insert_media_file_from_tacotron2; pub mod insert_media_file_from_voice_conversion; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs index 8365645e1f..321debe164 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -5,3 +5,5 @@ pub mod list_featured_media_files; pub mod list_media_files; pub mod list_media_files_for_user; pub mod update_media_file; +pub mod upload; +pub mod upload_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/drain_multipart_request.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/drain_multipart_request.rs new file mode 100644 index 0000000000..708eb6cc86 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/drain_multipart_request.rs @@ -0,0 +1,91 @@ +use actix_multipart::Multipart; +use actix_web::web::BytesMut; +use futures::TryStreamExt; +use log::warn; + +use errors::AnyhowResult; +use tokens::tokens::zs_voice_datasets::ZsVoiceDatasetToken; + +use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text}; + +pub struct MediaFileUploadData { + pub uuid_idempotency_token: Option, + pub file_name: Option, + pub file_bytes: Option, + pub media_source: MediaFileUploadSource, +} + +/// Where the frontend tells us the file came from. +#[derive(Eq, PartialEq, Clone, Copy)] +pub enum MediaFileUploadSource { + Unknown, + /// Eg. from the user's filesystem + UserFile, + /// Eg. the web audio API in Javascript. + UserDeviceApi, +} + +/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads. +pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> AnyhowResult { + let mut uuid_idempotency_token = None; + let mut file_bytes = None; + let mut file_name = None; + let mut media_source = None; + + while let Ok(Some(mut field)) = multipart_payload.try_next().await { + let mut field_name = None; + let mut field_filename = None; + + if let content_disposition = field.content_disposition() { + field_name = content_disposition.get_name() + .map(|s| s.to_string()); + field_filename = content_disposition.get_filename() // NB: Only used for the file bytes. + .map(|s| s.to_string()); + } + + match field_name.as_deref() { + Some("uuid_idempotency_token") => { + uuid_idempotency_token = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading idempotency token: {:}", &e); + e + })?; + }, + Some("file") => { + file_name = field_filename.clone(); + file_bytes = checked_read_multipart_bytes(&mut field).await + .map_err(|e| { + warn!("Error reading audio upload: {:}", &e); + e + })?; + }, + Some("source") => { + media_source = read_multipart_field_as_text(&mut field).await + .map_err(|e| { + warn!("Error reading source: {:}", &e); + e + })?; + }, + _ => continue, + } + } + + let media_source = match media_source.as_deref() { + Some("device") => MediaFileUploadSource::UserDeviceApi, + Some("file") => MediaFileUploadSource::UserFile, + _ => { + if file_name.as_deref() == Some("blob") { + MediaFileUploadSource::UserDeviceApi + } else { + MediaFileUploadSource::Unknown + } + }, + }; + + Ok(MediaFileUploadData { + uuid_idempotency_token, + file_name, + file_bytes, + media_source, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs new file mode 100644 index 0000000000..d98cfbc5af --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/mod.rs @@ -0,0 +1,3 @@ +pub mod drain_multipart_request; +pub mod process_upload_media_file; +pub mod upload_error; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/process_upload_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/process_upload_media_file.rs new file mode 100644 index 0000000000..df23ab1240 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/process_upload_media_file.rs @@ -0,0 +1,344 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, web}; +use log::{error, info, warn}; + +use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_files::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::endpoints::media_files::upload::drain_multipart_request::{drain_multipart_request, MediaFileUploadSource}; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::server_state::ServerState; +use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; + +// TODO(bt,2023-12-20): THIS CODE NEEDS CLEANUP. This has been cargo culted three+ times. +// It's ridiculous and complicated. + +pub enum SuccessCase { + MediaAlreadyUploaded { + existing_media_file_token: MediaFileToken, + }, + MediaSuccessfullyUploaded { + media_file_token: MediaFileToken, + } +} + +impl SuccessCase { + pub fn to_media_token(self) -> MediaFileToken { + match self { + SuccessCase::MediaAlreadyUploaded { existing_media_file_token } => existing_media_file_token, + SuccessCase::MediaSuccessfullyUploaded { media_file_token } => media_file_token, + } + } +} + +pub async fn process_upload_media_file( + http_request: &HttpRequest, + server_state: &web::Data>, + mut multipart_payload: Multipart, + allowed_mimetypes: &HashSet<&'static str>, +) -> Result { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + // ==================== READ SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== BANNED USERS ==================== // + + if let Some(ref user) = maybe_user_session { + if user.is_banned { + return Err(MediaFileUploadError::NotAuthorized); + } + } + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.file_upload_logged_out, + Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in, + }; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(MediaFileUploadError::RateLimited); + } + + // ==================== READ MULTIPART REQUEST ==================== // + + let upload_media_request = drain_multipart_request(multipart_payload) + .await + .map_err(|e| { + // TODO: Error handling could be nicer. + MediaFileUploadError::BadInput("bad request".to_string()) + })?; + + let uuid_idempotency_token = upload_media_request.uuid_idempotency_token + .ok_or(MediaFileUploadError::BadInput("no uuid".to_string()))?; + + // TODO(bt,2023-12-20): Generic table for this. + //let maybe_existing_upload = + // get_media_upload_by_uuid_with_connection(&uuid_idempotency_token, &mut mysql_connection) + // .await; + + //match maybe_existing_upload { + // Err(err) => { + // error!("Error checking for previous upload: {:?}", err); + // return Err(MediaFileUploadError::ServerError); + // } + // Ok(Some(upload)) => { + // // File already uploaded and frontend didn't protect us. + // return Ok(SuccessCase::MediaAlreadyUploaded { + // existing_media_file_token: upload.token, + // }); + // } + // Ok(None) => { + // // Proceed. + // } + //} + + if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) { + return Err(MediaFileUploadError::BadInput(reason)); + } + + let creator_set_visibility = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control. + .unwrap_or(Visibility::default()); + + let ip_address = get_request_ip(&http_request); + + let maybe_user_token = maybe_user_session + .map(|session| session.get_strongly_typed_user_token()); + + let maybe_file_size_bytes = upload_media_request.file_bytes + .as_ref() + .map(|bytes| bytes.len()); + + info!("Upload maybe filesize: {:?}", maybe_file_size_bytes); + + let maybe_mimetype = upload_media_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + + let bytes = match upload_media_request.file_bytes { + None => return Err(MediaFileUploadError::BadInput("missing file contents".to_string())), + Some(bytes) => bytes, + }; + + let hash = sha256_hash_bytes(&bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + MediaFileUploadError::ServerError + })?; + + let file_size_bytes = bytes.len(); + + let mut maybe_duration_millis = None; + let mut maybe_codec_name = None; + let mut media_file_type = None; + + if let Some(mimetype) = maybe_mimetype.as_deref() { + + if !allowed_mimetypes.contains(mimetype) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // NB: .aiff (audio/aiff) isn't supported by Symphonia: + // It contains uncompressed PCM-encoded audio similar to wav. + // See: https://github.com/pdeljanov/Symphonia/issues/75 + // NB: The following formats are not supported by Symphonia and + // do not have any open issues filed. They may simply be too old: + // - .wma (audio/x-ms-wma) + // - .avi (video/x-msvideo) + media_file_type = match mimetype { + // Audio + "audio/aac" /* .aac */ => Some(MediaFileType::Audio), + "audio/m4a" /* .m4a */ => Some(MediaFileType::Audio), + "audio/mpeg" /* .mp3 */ => Some(MediaFileType::Audio), + "audio/ogg" /* .ogg */ => Some(MediaFileType::Audio), + "audio/opus" /* .opus */ => Some(MediaFileType::Audio), + "audio/x-flac" /* .flac */ => Some(MediaFileType::Audio), + "audio/x-wav" /* .wav */ => Some(MediaFileType::Audio), + // Image + "image/jpeg" /* .jpg */ => Some(MediaFileType::Image), + "image/png" /* .png */ => Some(MediaFileType::Image), + "image/webp" /* .webp */ => Some(MediaFileType::Image), + // Video + "video/mp4" /* .mp4 */ => Some(MediaFileType::Video), + "video/quicktime" /* .mov */ => Some(MediaFileType::Video), + "video/webm" /* .webm */ => Some(MediaFileType::Video), + _ => None, + }; + + let do_audio_decode = match mimetype { + // TODO: Revisit when Safari can send us this metadata consistently + "audio/mp4" | "video/mp4" => false, + "audio/opus" => { + // TODO/FIXME(bt, 2023-05-19): Symphonia is currently broken for Firefox's opus. + // We're on an off-master branch that may resolve the problem in the future, but for now + // it panics as follows: + // + // [2023-05-19T10:25:34Z INFO symphonia_core::probe] found a possible format marker within [4f, 67, 67, 53, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, d4, c5] @ 0+2 bytes. + // [2023-05-19T10:25:34Z INFO symphonia_core::probe] found the format marker [4f, 67, 67, 53] @ 0+2 bytes. + // [2023-05-19T10:25:34Z DEBUG symphonia_format_ogg::page] grow page buffer to 8192 bytes + // [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] starting new physical stream + // [2023-05-19T10:25:34Z INFO symphonia_format_ogg::demuxer] selected opus mapper for stream with serial=0x19aac5d4 + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Probed! + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Find audio track... + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Found audio track (maybe) + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe track duration: None + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Maybe codec short name: Some("opus") + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Opus handler + // [2023-05-19T10:25:34Z INFO media::decode_basic_audio_info] Media source stream + // [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : options... + // [2023-05-19T10:25:34Z INFO media::decode_webm_opus_info] decode_mkv : try_read... + // [2023-05-19T10:25:34Z DEBUG symphonia_format_mkv::ebml] element with tag: 4F67 + // thread 'actix-rt|system:0|arbiter:1' panicked at 'assertion failed: `(left == right)` + // left: `Unknown`, + // right: `Ebml`: EBML element type must be checked before calling this function', /Users/bt/.cargo/git/checkouts/symphonia-8fbe6c90fc095688/e1a7009/symphonia-format-mkv/src/ebml.rs:335:9 + false + } + // Also, don't decode images + "image/jpeg" => false, + "image/png" => false, + "image/webp" => false, + _ => true, + }; + + if do_audio_decode && media_file_type.is_some() { + let basic_info = decode_basic_audio_bytes_info( + bytes.as_ref(), + Some(mimetype), + None + ).map_err(|e| { + warn!("file decoding error: {:?}", e); + MediaFileUploadError::BadInput("could not decode file".to_string()) + })?; + + maybe_duration_millis = basic_info.duration_millis; + maybe_codec_name = basic_info.codec_name; + } + } + + let media_file_type = match media_file_type { + Some(m) => m, + None => { + warn!("Invalid mimetype: {:?}", maybe_mimetype); + return Err(MediaFileUploadError::BadInput(format!("unknown mimetype: {:?}", maybe_mimetype))) + }, + }; + + let upload_type = match upload_media_request.media_source { + MediaFileUploadSource::Unknown => UploadType::Filesystem, + MediaFileUploadSource::UserFile => UploadType::Filesystem, + MediaFileUploadSource::UserDeviceApi => UploadType::DeviceCaptureApi, + }; + + // TODO: Clean up code + let mime_type = match maybe_mimetype { + Some(m) => m, + None => { + warn!("Missing mimetype!"); + return Err(MediaFileUploadError::BadInput("Missing mimetype".to_string())); + }, + }; + + // TODO(bt,2023-12-20): Reasonable extensions. + let extension = match mime_type { + // Audio + "audio/aac" /* .aac */ => Some(".aac"), + "audio/m4a" /* .m4a */ => Some(".m4a"), + "audio/mpeg" /* .mp3 */ => Some(".mp3"), + "audio/ogg" /* .ogg */ => Some(".ogg"), + "audio/opus" /* .opus */ => Some(".opus"), + "audio/x-flac" /* .flac */ => Some(".flac"), + "audio/x-wav" /* .wav */ => Some(".wav"), + // Image + "image/jpeg" /* .jpg */ => Some(".jpg"), + "image/png" /* .png */ => Some(".png"), + "image/webp" /* .webp */ => Some(".webp"), + // Video + "video/mp4" /* .mp4 */ => Some(".mp4"), + "video/quicktime" /* .mov */ => Some(".mov"), + "video/webm" /* .webm */ => Some(".webm"), + _ => None, + }; + + const PREFIX : Option<&str> = Some("upload_"); + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, extension); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mime_type) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs { + maybe_creator_user_token: maybe_user_token.as_ref(), + maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility, + upload_type, + media_file_type, + maybe_mime_type: Some(mime_type), + file_size_bytes: file_size_bytes as u64, + duration_millis: 0, // TODO + sha256_checksum: &hash, + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix: PREFIX, + maybe_public_bucket_extension: extension, + pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New generic download creation DB error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + info!("new media file id: {} token: {:?}", record_id, &token); + + Ok(SuccessCase::MediaSuccessfullyUploaded { + media_file_token: token, + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs new file mode 100644 index 0000000000..39db17e7ed --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_error.rs @@ -0,0 +1,36 @@ +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::HttpResponse; + +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; + +#[derive(Debug, Serialize)] +pub enum MediaFileUploadError { + BadInput(String), + NotAuthorized, + MustBeLoggedIn, + ServerError, + RateLimited, +} + +impl ResponseError for MediaFileUploadError { + fn status_code(&self) -> StatusCode { + match *self { + MediaFileUploadError::BadInput(_) => StatusCode::BAD_REQUEST, + MediaFileUploadError::NotAuthorized => StatusCode::UNAUTHORIZED, + MediaFileUploadError::MustBeLoggedIn => StatusCode::UNAUTHORIZED, + MediaFileUploadError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + MediaFileUploadError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +impl std::fmt::Display for MediaFileUploadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload_media_file.rs new file mode 100644 index 0000000000..118a1f267c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload_media_file.rs @@ -0,0 +1,64 @@ +use std::collections::HashSet; +use std::sync::Arc; + +use actix_multipart::Multipart; +use actix_web::{HttpRequest, HttpResponse, web}; +use once_cell::sync::Lazy; +use tokens::tokens::media_files::MediaFileToken; +use crate::http_server::endpoints::media_files::upload::process_upload_media_file::process_upload_media_file; + + +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::server_state::ServerState; + +#[derive(Serialize)] +pub struct UploadMediaSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + // Audio + "audio/aac", + "audio/m4a", + "audio/mpeg", + "audio/ogg", + "audio/opus", + "audio/x-flac", + "audio/x-wav", + // Mixed + "audio/mp4", // iPhone seems to upload these as audio + // Video + "video/mp4", + "video/webm", + //"video/quicktime", + ]) +}); + +pub async fn upload_media_file_handler( + http_request: HttpRequest, + server_state: web::Data>, + mut multipart_payload: Multipart, +) -> Result { + + let response = process_upload_media_file( + &http_request, + &server_state, + multipart_payload, + &ALLOWED_MIME_TYPES).await?; + + let media_file_token = response.to_media_token(); + + let response = UploadMediaSuccessResponse { + success: true, + media_file_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| MediaFileUploadError::ServerError)?; + + return Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)); +} diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index aafc1d1442..7b5dfcea51 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -44,6 +44,7 @@ use crate::http_server::endpoints::media_files::list_featured_media_files::list_ use crate::http_server::endpoints::media_files::list_media_files::list_media_files_handler; use crate::http_server::endpoints::media_files::list_media_files_for_user::list_media_files_for_user_handler; use crate::http_server::endpoints::media_files::update_media_file::update_media_file_handler; +use crate::http_server::endpoints::media_files::upload_media_file::upload_media_file_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; use crate::http_server::endpoints::media_uploads::upload_image::upload_image_handler; @@ -1035,6 +1036,10 @@ fn add_media_file_routes (app: App) -> App .route(web::get().to(list_media_files_for_user_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/upload") + .route(web::post().to(upload_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) ) }