From fafb485a6073fac09301808cd5ffda0dd5d1ba4b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Jun 2026 22:37:11 -0400 Subject: [PATCH] omni upload APIs (#1653) --- .../artcraft/artcraft_api_defs/src/lib.rs | 1 + .../artcraft_api_defs/src/omni_api/mod.rs | 3 + .../src/omni_api/omni_upload_audio.rs | 15 + .../src/omni_api/omni_upload_image.rs | 15 + .../src/omni_api/omni_upload_video.rs | 15 + .../web/storyteller_web/src/docs/api_doc.rs | 15 + .../src/http_server/endpoints/omni_api/mod.rs | 1 + .../endpoints/omni_api/upload/mod.rs | 3 + .../omni_upload_audio_media_file_handler.rs | 330 +++++++++++ .../omni_upload_image_media_file_handler.rs | 341 +++++++++++ .../omni_upload_video_media_file_handler.rs | 560 ++++++++++++++++++ .../application_routes/omni_api_routes.rs | 17 + 12 files changed, 1316 insertions(+) create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/mod.rs create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_audio.rs create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_image.rs create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_video.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_audio_media_file_handler.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_image_media_file_handler.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_video_media_file_handler.rs diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs index a9d2efc0fb..156cbd21c5 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs @@ -8,6 +8,7 @@ pub mod generate; pub mod jobs; pub mod media_file; pub mod moderation; +pub mod omni_api; pub mod omni_gen; pub mod prompts; pub mod stripe_artcraft; diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/mod.rs new file mode 100644 index 0000000000..4cd11c41e5 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/mod.rs @@ -0,0 +1,3 @@ +pub mod omni_upload_audio; +pub mod omni_upload_image; +pub mod omni_upload_video; diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_audio.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_audio.rs new file mode 100644 index 0000000000..8c4bb2588f --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_audio.rs @@ -0,0 +1,15 @@ +use serde_derive::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use tokens::tokens::media_files::MediaFileToken; + +/// Success response for the API-key authenticated audio upload endpoint +/// (`POST /v1/omni_api/upload/audio`). +/// +/// NB: The request is a `multipart/form-data` body. Because the multipart form type is bound to the +/// web framework, it is declared alongside the handler in `storyteller_web` rather than here. +#[derive(Clone, Serialize, Deserialize, ToSchema, Debug)] +pub struct OmniUploadAudioMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_image.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_image.rs new file mode 100644 index 0000000000..b8e968d81f --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_image.rs @@ -0,0 +1,15 @@ +use serde_derive::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use tokens::tokens::media_files::MediaFileToken; + +/// Success response for the API-key authenticated image upload endpoint +/// (`POST /v1/omni_api/upload/image`). +/// +/// NB: The request is a `multipart/form-data` body. Because the multipart form type is bound to the +/// web framework, it is declared alongside the handler in `storyteller_web` rather than here. +#[derive(Clone, Serialize, Deserialize, ToSchema, Debug)] +pub struct OmniUploadImageMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_video.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_video.rs new file mode 100644 index 0000000000..da2c606ee5 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/omni_api/omni_upload_video.rs @@ -0,0 +1,15 @@ +use serde_derive::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use tokens::tokens::media_files::MediaFileToken; + +/// Success response for the API-key authenticated video upload endpoint +/// (`POST /v1/omni_api/upload/video`). +/// +/// NB: The request is a `multipart/form-data` body. Because the multipart form type is bound to the +/// web framework, it is declared alongside the handler in `storyteller_web` rather than here. +#[derive(Clone, Serialize, Deserialize, ToSchema, Debug)] +pub struct OmniUploadVideoMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index 2faf9fe2c9..1eedb13c78 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -80,6 +80,9 @@ use crate::http_server::endpoints::media_files::upload::upload_spz_media_file_ha use crate::http_server::endpoints::media_files::upload::upload_studio_shot::upload_studio_shot_media_file_handler::*; use crate::http_server::endpoints::media_files::upload::upload_video_new::upload_new_video_media_file_handler::*; use crate::http_server::endpoints::media_files::upload::upload_video_old::upload_video_media_file_handler::*; +use crate::http_server::endpoints::omni_api::upload::omni_upload_audio_media_file_handler::*; +use crate::http_server::endpoints::omni_api::upload::omni_upload_image_media_file_handler::*; +use crate::http_server::endpoints::omni_api::upload::omni_upload_video_media_file_handler::*; use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::*; use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError; use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*; @@ -343,6 +346,9 @@ use artcraft_api_defs::omni_gen::generate_response::omni_gen_image_generate_resp use artcraft_api_defs::omni_gen::generate_response::omni_gen_video_generate_response::*; use artcraft_api_defs::omni_gen::models::omni_gen_image_models::*; use artcraft_api_defs::omni_gen::models::omni_gen_video_models::*; +use artcraft_api_defs::omni_api::omni_upload_audio::*; +use artcraft_api_defs::omni_api::omni_upload_image::*; +use artcraft_api_defs::omni_api::omni_upload_video::*; use artcraft_api_defs::moderation::user::list_subscribing_users_by_signup_date::*; use artcraft_api_defs::moderation::user::list_users_by_signup_date::*; use artcraft_api_defs::moderation::user::user_lookup::*; @@ -553,6 +559,9 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux // Omni API (API-key authenticated generate) crate::http_server::endpoints::omni_api::generate::image::omni_api_image_generate_handler::omni_api_image_generate_handler, crate::http_server::endpoints::omni_api::generate::video::omni_api_video_generate_handler::omni_api_video_generate_handler, + crate::http_server::endpoints::omni_api::upload::omni_upload_audio_media_file_handler::omni_upload_audio_media_file_handler, + crate::http_server::endpoints::omni_api::upload::omni_upload_image_media_file_handler::omni_upload_image_media_file_handler, + crate::http_server::endpoints::omni_api::upload::omni_upload_video_media_file_handler::omni_upload_video_media_file_handler, // Moderation crate::http_server::endpoints::moderation::alerts::moderation_send_alert_handler::moderation_send_alert_handler, crate::http_server::endpoints::moderation::info::moderator_token_info_handler::moderator_get_token_info_handler, @@ -1080,6 +1089,12 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux UploadEngineAssetMediaSuccessResponse, UploadImageMediaFileForm, UploadImageMediaFileSuccessResponse, + OmniUploadAudioMediaFileForm, + OmniUploadAudioMediaFileSuccessResponse, + OmniUploadImageMediaFileForm, + OmniUploadImageMediaFileSuccessResponse, + OmniUploadVideoMediaFileForm, + OmniUploadVideoMediaFileSuccessResponse, UserImpersonationRequestResponse, UploadMediaSuccessResponse, UploadNewEngineAssetFileForm, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/mod.rs index 5cd6f81640..46c5a37d41 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/mod.rs @@ -1,2 +1,3 @@ pub mod generate; pub mod shared_utils; +pub mod upload; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/mod.rs new file mode 100644 index 0000000000..84dfcb12f9 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/mod.rs @@ -0,0 +1,3 @@ +pub mod omni_upload_audio_media_file_handler; +pub mod omni_upload_image_media_file_handler; +pub mod omni_upload_video_media_file_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_audio_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_audio_media_file_handler.rs new file mode 100644 index 0000000000..b1e444adae --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_audio_media_file_handler.rs @@ -0,0 +1,330 @@ +use std::collections::HashSet; +use std::io::Read; +use std::path::PathBuf; +use std::sync::Arc; + +use actix_multipart::form::tempfile::TempFile; +use actix_multipart::form::text::Text; +use actix_multipart::form::MultipartForm; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use log::{error, info, warn}; +use once_cell::sync::Lazy; +use utoipa::ToSchema; + +use artcraft_api_defs::omni_api::omni_upload_audio::OmniUploadAudioMediaFileSuccessResponse; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use media::decode_basic_audio_info::decode_basic_audio_bytes_info; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_to_extension::mimetype_to_extension; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::endpoints::media_files::upload::common_utils::try_parse_generation_provider::try_parse_generation_provider; +use crate::http_server::user_lookup::api_keys::require_api_key_user::require_api_key_user; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; + +/// Form-multipart request fields. +/// +/// IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `OmniUploadAudioMediaFileForm` (Under "Schema") FOR DETAILS ON FIELDS AND NULLABILITY. +#[derive(MultipartForm, ToSchema)] +#[multipart(duplicate_field = "deny")] +pub struct OmniUploadAudioMediaFileForm { + /// UUID for request idempotency + #[multipart(limit = "2 KiB")] + #[schema(value_type = String, format = Binary)] + uuid_idempotency_token: Text, + + /// The uploaded file + #[multipart(limit = "512 MiB")] + #[schema(value_type = Vec, format = Binary)] + file: TempFile, + + /// Optional: Title (name) of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_title: Option>, + + /// Optional: Visibility of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_visibility: Option>, + + /// Optional: The third-party generation provider (e.g. "fal", "replicate"). + /// If set, `is_user_upload` will be false and `is_intermediate_system_file` will be forced false. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_generation_provider: Option>, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + // Audio + "audio/aac", + "audio/m4a", + "audio/mpeg", + "audio/ogg", + "audio/opus", + "audio/x-flac", + "audio/x-wav", + // Mixed + "audio/mp4", // iPhone seems to upload these as audio + // Video + "video/mp4", + "video/webm", + ]) +}); + +/// API-key authenticated audio upload (Omni API). Identity is read from the `Authorization` header +/// API key rather than a session cookie. +#[utoipa::path( + post, + tag = "Omni API", + path = "/v1/omni_api/upload/audio", + responses( + (status = 200, description = "Success Update", body = OmniUploadAudioMediaFileSuccessResponse), + (status = 400, description = "Bad input", body = MediaFileUploadError), + (status = 401, description = "Not authorized", body = MediaFileUploadError), + (status = 429, description = "Too many requests", body = MediaFileUploadError), + (status = 500, description = "Server error", body = MediaFileUploadError), + ), + params( + ( + "request" = OmniUploadAudioMediaFileForm, + description = "IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `OmniUploadAudioMediaFileForm` (Under 'Schema') FOR DETAILS ON FIELDS AND NULLABILITY." + ), + ) +)] +pub async fn omni_upload_audio_media_file_handler( + http_request: HttpRequest, + server_state: web::Data>, + MultipartForm(mut form): MultipartForm, +) -> Result, MediaFileUploadError> { + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + // ==================== API KEY USER ==================== // + + // API-key authentication (Authorization header) instead of a session cookie. Never cached, and a + // banned owner is rejected inside `require_api_key_user`. + let api_session = require_api_key_user(&http_request, &mut *mysql_connection) + .await + .map_err(map_api_key_auth_error)?; + + let maybe_user_token = Some(&api_session.user_token); + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = &server_state.redis_rate_limiters.file_upload_logged_in; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request).await { + return Err(MediaFileUploadError::RateLimited); + } + + // ==================== HANDLE IDEMPOTENCY ==================== // + + let uuid_idempotency_token = form.uuid_idempotency_token.as_ref(); + + if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) { + return Err(MediaFileUploadError::BadInput(reason)); + } + + insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + MediaFileUploadError::BadInput("invalid idempotency token".to_string()) + })?; + + // ==================== UPLOAD METADATA ==================== // + + let maybe_title = form.maybe_title + .map(|title| title.trim().to_string()) + .filter(|title| !title.is_empty()); + + let creator_set_visibility = form.maybe_visibility + .map(|visibility| visibility.0) + .unwrap_or(Visibility::default()); + + // ==================== USER DATA ==================== // + + let ip_address = get_request_ip(&http_request); + + // ==================== FILE VALIDATION ==================== // + + let mut file_bytes = Vec::new(); + form.file.file.read_to_end(&mut file_bytes) + .map_err(|e| { + error!("Problem reading file: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let mimetype = get_mimetype_for_bytes(file_bytes.as_ref()) + .map(|mimetype| mimetype.to_string()) + .ok_or_else(|| { + warn!("Could not determine mimetype for file"); + MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string()) + })?; + + if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // ==================== DURATION DETECTION ==================== // + + let do_audio_decode = match mimetype.as_ref() { + // TODO: Revisit when Safari can send us this metadata consistently + "audio/mp4" | "video/mp4" => false, + // Symphonia is currently broken for Firefox's opus; skip decoding it. + "audio/opus" => false, + // Also, don't decode images + "image/jpeg" => false, + "image/png" => false, + "image/webp" => false, + _ => true, + }; + + let mut maybe_duration_millis = None; + + if do_audio_decode { + let basic_info = decode_basic_audio_bytes_info( + file_bytes.as_ref(), + Some(&mimetype), + None + ).map_err(|e| { + warn!("file decoding error: {:?}", e); + MediaFileUploadError::BadInput("could not decode file".to_string()) + })?; + + maybe_duration_millis = basic_info.duration_millis; + } + + // ==================== OTHER FILE METADATA ==================== // + + let maybe_filename = form.file.file_name.as_deref() + .as_deref() + .map(|filename| PathBuf::from(filename)); + + let extension = mimetype_to_extension(&mimetype) + .or_else(|| { + maybe_filename + .as_ref() + .and_then(|filename| filename.extension()) + .and_then(|ext| ext.to_str()) + }) + .ok_or_else(|| { + warn!("Could not determine file extension for mimetype: {}", &mimetype); + MediaFileUploadError::ServerError + })?; + + let extension = format!(".{extension}"); // NB: needs dot prefix + + let file_size_bytes = file_bytes.len(); + + let hash = sha256_hash_bytes(&file_bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + MediaFileUploadError::ServerError + })?; + + // ==================== UPLOAD AND SAVE ==================== // + + const PREFIX : Option<&str> = Some("aud_"); + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + &mimetype) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let maybe_generation_provider = form.maybe_generation_provider + .as_ref() + .and_then(|text| try_parse_generation_provider(text.as_ref())); + + let upload_type = if maybe_generation_provider.is_some() { + UploadType::ThirdPartyInference + } else { + UploadType::Filesystem + }; + + let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs { + maybe_media_class: Some(MediaFileClass::Audio), + media_file_type: MediaFileType::Audio, + maybe_creator_user_token: maybe_user_token, + maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility, + upload_type, + maybe_engine_category: None, + maybe_animation_type: None, + maybe_prompt_token: None, + maybe_batch_token: None, + maybe_mime_type: Some(&mimetype), + file_size_bytes: file_size_bytes as u64, + maybe_duration_millis, + sha256_checksum: &hash, + maybe_title: maybe_title.as_deref(), + maybe_scene_source_media_file_token: None, + is_intermediate_system_file: false, // NB: is_user_upload = true + maybe_generation_provider, + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix: PREFIX, + maybe_public_bucket_extension: Some(&extension), + pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New file creation DB error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + info!("new media file id: {} token: {:?}", record_id, &token); + + Ok(Json(OmniUploadAudioMediaFileSuccessResponse { + success: true, + media_file_token: token, + })) +} + +/// Map an API-key authentication failure onto this endpoint's error type. A genuine 401 stays a +/// 401; anything else (e.g. a DB error during lookup) becomes a 500. +fn map_api_key_auth_error(err: CommonWebError) -> MediaFileUploadError { + match err { + CommonWebError::NotAuthorized => { + MediaFileUploadError::NotAuthorizedVerbose("invalid or missing API key".to_string()) + } + _ => MediaFileUploadError::ServerError, + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_image_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_image_media_file_handler.rs new file mode 100644 index 0000000000..e36f5b63a6 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_image_media_file_handler.rs @@ -0,0 +1,341 @@ +use std::collections::HashSet; +use std::io::Read; +use std::path::PathBuf; +use std::sync::Arc; + +use actix_multipart::form::tempfile::TempFile; +use actix_multipart::form::text::Text; +use actix_multipart::form::MultipartForm; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use log::{error, info, warn}; +use once_cell::sync::Lazy; +use utoipa::ToSchema; + +use artcraft_api_defs::omni_api::omni_upload_image::OmniUploadImageMediaFileSuccessResponse; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_to_extension::mimetype_to_extension; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; +use tokens::tokens::batch_generations::BatchGenerationToken; +use tokens::tokens::prompts::PromptToken; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::endpoints::media_files::upload::common_utils::try_parse_generation_provider::try_parse_generation_provider; +use crate::http_server::user_lookup::api_keys::require_api_key_user::require_api_key_user; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; + +/// Form-multipart request fields. +/// +/// IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `OmniUploadImageMediaFileForm` (Under "Schema") FOR DETAILS ON FIELDS AND NULLABILITY. +#[derive(MultipartForm, ToSchema)] +#[multipart(duplicate_field = "deny")] +pub struct OmniUploadImageMediaFileForm { + /// UUID for request idempotency + #[multipart(limit = "2 KiB")] + #[schema(value_type = String, format = Binary)] + uuid_idempotency_token: Text, + + /// The uploaded file + #[multipart(limit = "512 MiB")] + #[schema(value_type = Vec, format = Binary)] + file: TempFile, + + /// Optional: Title (name) of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_title: Option>, + + /// Optional: Visibility of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_visibility: Option>, + + /// Optional: Prompt associated with this image + /// NOTE: Cannot set `is_intermediate_system_file = true` if this is set. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_prompt_token: Option>, + + /// Optional: Batch associated with this image + /// NOTE: Cannot set `is_intermediate_system_file = true` if this is set. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_batch_token: Option>, + + /// Optional: Whether this is a system file (e.g. cover files we should hide) + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + is_intermediate_system_file: Option>, + + /// Optional: The third-party generation provider (e.g. "fal", "replicate"). + /// If set, `is_user_upload` will be false and `is_intermediate_system_file` will be forced false. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_generation_provider: Option>, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + ]) +}); + +/// API-key authenticated image upload (Omni API). Identity is read from the `Authorization` header +/// API key rather than a session cookie. +#[utoipa::path( + post, + tag = "Omni API", + path = "/v1/omni_api/upload/image", + responses( + (status = 200, description = "Success Update", body = OmniUploadImageMediaFileSuccessResponse), + (status = 400, description = "Bad input", body = MediaFileUploadError), + (status = 401, description = "Not authorized", body = MediaFileUploadError), + (status = 429, description = "Too many requests", body = MediaFileUploadError), + (status = 500, description = "Server error", body = MediaFileUploadError), + ), + params( + ( + "request" = OmniUploadImageMediaFileForm, + description = "IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `OmniUploadImageMediaFileForm` (Under 'Schema') FOR DETAILS ON FIELDS AND NULLABILITY." + ), + ) +)] +pub async fn omni_upload_image_media_file_handler( + http_request: HttpRequest, + server_state: web::Data>, + MultipartForm(mut form): MultipartForm, +) -> Result, MediaFileUploadError> { + + validate_form(&form)?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + // ==================== API KEY USER ==================== // + + // API-key authentication (Authorization header) instead of a session cookie. Never cached, and a + // banned owner is rejected inside `require_api_key_user`. + let api_session = require_api_key_user(&http_request, &mut *mysql_connection) + .await + .map_err(map_api_key_auth_error)?; + + let maybe_user_token = Some(&api_session.user_token); + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== HANDLE IDEMPOTENCY ==================== // + + let uuid_idempotency_token = form.uuid_idempotency_token.as_ref(); + + if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) { + return Err(MediaFileUploadError::BadInput(reason)); + } + + insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + MediaFileUploadError::BadInput("invalid idempotency token".to_string()) + })?; + + // ==================== UPLOAD METADATA ==================== // + + let maybe_title = form.maybe_title.map(|title| title.to_string()); + + let creator_set_visibility = form.maybe_visibility + .map(|visibility| visibility.0) + .unwrap_or(Visibility::default()); + + // ==================== USER DATA ==================== // + + let ip_address = get_request_ip(&http_request); + + // ==================== FILE VALIDATION ==================== // + + let mut file_bytes = Vec::new(); + form.file.file.read_to_end(&mut file_bytes) + .map_err(|e| { + error!("Problem reading file: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let mimetype = get_mimetype_for_bytes(file_bytes.as_ref()) + .map(|mimetype| mimetype.to_string()) + .ok_or_else(|| { + warn!("Could not determine mimetype for file"); + MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string()) + })?; + + if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // ==================== OTHER FILE METADATA ==================== // + + let maybe_filename = form.file.file_name.as_deref() + .as_deref() + .map(|filename| PathBuf::from(filename)); + + let extension = mimetype_to_extension(&mimetype) + .or_else(|| { + maybe_filename + .as_ref() + .and_then(|filename| filename.extension()) + .and_then(|ext| ext.to_str()) + }) + .ok_or_else(|| { + warn!("Could not determine file extension for mimetype: {}", &mimetype); + MediaFileUploadError::ServerError + })?; + + let extension = format!(".{extension}"); // NB: needs dot prefix + + let file_size_bytes = file_bytes.len(); + + let hash = sha256_hash_bytes(&file_bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + MediaFileUploadError::ServerError + })?; + + // ==================== UPLOAD AND SAVE ==================== // + + const PREFIX : Option<&str> = Some("image_"); + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + &mimetype) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let is_intermediate_system_file = form.is_intermediate_system_file + .map(|b| b.0) + .unwrap_or(false); + + let maybe_prompt_token = form.maybe_prompt_token + .as_ref() + .map(|token| token.0.clone()); + + let maybe_batch_token = form.maybe_batch_token + .as_ref() + .map(|token| token.0.clone()); + + let maybe_generation_provider = form.maybe_generation_provider + .as_ref() + .and_then(|text| try_parse_generation_provider(text.as_ref())); + + let upload_type = if maybe_generation_provider.is_some() { + UploadType::ThirdPartyInference + } else { + UploadType::Filesystem + }; + + let media_file_type = match mimetype.as_str() { + "image/jpeg" => MediaFileType::Jpg, + "image/png" => MediaFileType::Png, + "image/gif" => MediaFileType::Gif, + "image/webp" => MediaFileType::Image, // Fallback + _ => MediaFileType::Image, // fallback + }; + + let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs { + maybe_media_class: Some(MediaFileClass::Image), + media_file_type, + maybe_creator_user_token: maybe_user_token, + maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility, + upload_type, + maybe_engine_category: None, + maybe_animation_type: None, + maybe_mime_type: Some(&mimetype), + maybe_prompt_token: maybe_prompt_token.as_ref(), + maybe_batch_token: maybe_batch_token.as_ref(), + file_size_bytes: file_size_bytes as u64, + maybe_duration_millis: None, + sha256_checksum: &hash, + maybe_title: maybe_title.as_deref(), + maybe_scene_source_media_file_token: None, + is_intermediate_system_file, + maybe_generation_provider, + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix: PREFIX, + maybe_public_bucket_extension: Some(&extension), + pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New file creation DB error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + info!("new media file id: {} token: {:?}", record_id, &token); + + Ok(Json(OmniUploadImageMediaFileSuccessResponse { + success: true, + media_file_token: token, + })) +} + +fn validate_form(form: &OmniUploadImageMediaFileForm) -> Result<(), MediaFileUploadError> { + if form.uuid_idempotency_token.is_empty() { + return Err(MediaFileUploadError::BadInput("Idempotency token is required".to_string())); + } + + let is_intermediate_system_file = form.is_intermediate_system_file + .as_ref() + .map(|b| b.0) + .unwrap_or(false); + + if is_intermediate_system_file && form.maybe_prompt_token.is_some() { + return Err(MediaFileUploadError::BadInput( + "Cannot set `is_intermediate_system_file` to true if `maybe_prompt_token` is provided." + .to_string())); + } + + Ok(()) +} + +/// Map an API-key authentication failure onto this endpoint's error type. A genuine 401 stays a +/// 401; anything else (e.g. a DB error during lookup) becomes a 500. +fn map_api_key_auth_error(err: CommonWebError) -> MediaFileUploadError { + match err { + CommonWebError::NotAuthorized => { + MediaFileUploadError::NotAuthorizedVerbose("invalid or missing API key".to_string()) + } + _ => MediaFileUploadError::ServerError, + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_video_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_video_media_file_handler.rs new file mode 100644 index 0000000000..66b23f6b95 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_video_media_file_handler.rs @@ -0,0 +1,560 @@ +use std::collections::HashSet; +use std::io::Read; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use actix_multipart::form::tempfile::TempFile; +use actix_multipart::form::text::Text; +use actix_multipart::form::MultipartForm; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use log::{debug, error, info, warn}; +use once_cell::sync::Lazy; +use utoipa::ToSchema; + +use artcraft_api_defs::omni_api::omni_upload_video::OmniUploadVideoMediaFileSuccessResponse; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use ffmpeg_utils::ffmpeg::ffmpeg_transcode_to_mp4::{ffmpeg_transcode_to_mp4, FfmpegTranscodeToMp4Args}; +use ffmpeg_utils::ffmpeg::ffmpeg_trim_and_resample::{ffmpeg_trim_and_resample, Args}; +use ffmpeg_utils::ffprobe::ffprobe_get_info::ffprobe_get_info; +use filesys::file_read_bytes::file_read_bytes; +use filesys::path_to_string::path_to_string; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use http_server_common::request::get_request_ip::get_request_ip; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mimetypes::mimetype_to_extension::mimetype_to_extension; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; +use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::prompts::PromptToken; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::endpoints::media_files::upload::common_utils::try_parse_generation_provider::try_parse_generation_provider; +use crate::http_server::user_lookup::api_keys::require_api_key_user::require_api_key_user; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; + +/// Form-multipart request fields. +/// +/// IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `OmniUploadVideoMediaFileForm` (Under "Schema") FOR DETAILS ON FIELDS AND NULLABILITY. +#[derive(MultipartForm, ToSchema)] +#[multipart(duplicate_field = "deny")] +pub struct OmniUploadVideoMediaFileForm { + /// UUID for request idempotency + #[multipart(limit = "2 KiB")] + #[schema(value_type = String, format = Binary)] + uuid_idempotency_token: Text, + + /// The uploaded file + #[multipart(limit = "512 MiB")] + #[schema(value_type = Vec, format = Binary)] + file: TempFile, + + /// Optional: Title (name) of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_title: Option>, + + /// Optional: Visibility of the scene + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_visibility: Option>, + + /// Optional: Prompt associated with this image + /// NOTE: Cannot set `is_intermediate_system_file = true` if this is set. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_prompt_token: Option>, + + /// Optional: If an engine scene was used to generate this video, provide it here to create a link. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_scene_source_media_file_token: Option>, + + /// Optional: Whether this is a system file (eg. cover files we should hide) + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + is_intermediate_system_file: Option>, + + /// Optional: Trim start offset in milliseconds. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_trim_start_millis: Option>, + + /// Optional: Trim end offset in milliseconds. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_trim_end_millis: Option>, + + /// Optional: Resample the video to this FPS. + /// Only certain values equal or under 24fps are allowed. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_resample_fps: Option>, + + /// Optional: The third-party generation provider (e.g. "fal", "replicate"). + /// If set, `is_user_upload` will be false and `is_intermediate_system_file` will be forced false. + #[multipart(limit = "2 KiB")] + #[schema(value_type = Option, format = Binary)] + maybe_generation_provider: Option>, +} + +static ALLOWED_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + // Video + "video/mp4", // NB: Only mp4 for now. + ]) +}); + +static TRANSCODE_MIME_TYPES : Lazy> = Lazy::new(|| { + HashSet::from([ + // Video + "video/quicktime", // .qt quicktime files + ]) +}); + +/// API-key authenticated video upload (Omni API). Identity is read from the `Authorization` header +/// API key rather than a session cookie. +#[utoipa::path( + post, + tag = "Omni API", + path = "/v1/omni_api/upload/video", + responses( + (status = 200, description = "Success Update", body = OmniUploadVideoMediaFileSuccessResponse), + (status = 400, description = "Bad input", body = MediaFileUploadError), + (status = 401, description = "Not authorized", body = MediaFileUploadError), + (status = 429, description = "Too many requests", body = MediaFileUploadError), + (status = 500, description = "Server error", body = MediaFileUploadError), + ), + params( + ( + "request" = OmniUploadVideoMediaFileForm, + description = "IF VIEWING DOCS, PLEASE SEE BOTTOM OF PAGE `OmniUploadVideoMediaFileForm` (Under 'Schema') FOR DETAILS ON FIELDS AND NULLABILITY." + ), + ) +)] +pub async fn omni_upload_video_media_file_handler( + http_request: HttpRequest, + server_state: web::Data>, + MultipartForm(mut form): MultipartForm, +) -> Result, MediaFileUploadError> { + + fast_form_validations(&form)?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + error!("MySql pool error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + // ==================== API KEY USER ==================== // + + // API-key authentication (Authorization header) instead of a session cookie. Never cached, and a + // banned owner is rejected inside `require_api_key_user`. + let api_session = require_api_key_user(&http_request, &mut *mysql_connection) + .await + .map_err(map_api_key_auth_error)?; + + let maybe_user_token = Some(&api_session.user_token); + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // ==================== RATE LIMIT ==================== // + + let rate_limiter = &server_state.redis_rate_limiters.file_upload_logged_in; + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request).await { + return Err(MediaFileUploadError::RateLimited); + } + + // ==================== HANDLE IDEMPOTENCY ==================== // + + let uuid_idempotency_token = form.uuid_idempotency_token.as_ref(); + + if let Err(reason) = validate_idempotency_token_format(uuid_idempotency_token) { + return Err(MediaFileUploadError::BadInput(reason)); + } + + insert_idempotency_token(uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + MediaFileUploadError::BadInput("invalid idempotency token".to_string()) + })?; + + // ==================== UPLOAD METADATA ==================== // + + let maybe_title = form.maybe_title + .map(|title| title.trim().to_string()) + .filter(|title| !title.is_empty()); + + let creator_set_visibility = form.maybe_visibility + .map(|visibility| visibility.0) + .unwrap_or(Visibility::default()); + + // ==================== USER DATA ==================== // + + let ip_address = get_request_ip(&http_request); + + // ==================== FILE VALIDATION ==================== // + + let mut file_bytes = Vec::new(); + form.file.file.read_to_end(&mut file_bytes) + .map_err(|e| { + error!("Problem reading file: {:?}", e); + MediaFileUploadError::ServerError + })?; + + let mut mimetype = get_mimetype_for_bytes(file_bytes.as_ref()) + .map(|mimetype| mimetype.to_string()) + .ok_or_else(|| { + warn!("Could not determine mimetype for file"); + MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string()) + })?; + + let needs_transcode = TRANSCODE_MIME_TYPES.contains(mimetype.as_str()); + + if !ALLOWED_MIME_TYPES.contains(mimetype.as_str()) && !needs_transcode { + // NB: Don't let our error message inject malicious strings + let filtered_mimetype = mimetype + .chars() + .filter(|c| c.is_ascii()) + .filter(|c| c.is_alphanumeric() || *c == '/') + .collect::(); + return Err(MediaFileUploadError::BadInput(format!("unpermitted mime type: {}", &filtered_mimetype))); + } + + // ==================== ORIGINAL FILE BOOKKEEPING ==================== // + // Hold on to the user's original file (path + mimetype + extension) so we + // can upload it as a `_original.{ext}` sibling later if we end up + // transcoding and/or resampling it. + + let original_file_path: PathBuf = form.file.file.path().to_path_buf(); + let original_mimetype: String = mimetype.clone(); + let original_filename_extension: Option = form.file.file_name.as_deref() + .and_then(|filename| std::path::Path::new(filename).extension()) + .and_then(|ext| ext.to_str()) + .map(|ext| ext.to_string()); + let original_extension: String = mimetype_to_extension(&original_mimetype) + .map(|ext| ext.to_string()) + .or(original_filename_extension) + .ok_or_else(|| { + warn!("Could not determine file extension for original mimetype: {}", &original_mimetype); + MediaFileUploadError::ServerError + })?; + + // ==================== OPTIONAL TRANSCODE TO MP4 ==================== // + // Some accepted source mime types (e.g. QuickTime .mov) are not the canonical + // distribution format we want for the media file record. Transcode them to + // mp4 in a tempdir so we can resample / upload mp4 instead. + + let mut transcode_tempdir_ref = None; + let mut resample_tempdir_ref = None; + + let mut final_upload_file_path = form.file.file.path().to_path_buf(); + + if needs_transcode { + let transcode_tempdir = server_state.temp_dir_creator.new_tempdir("ffmpeg_transcode") + .map_err(|err| { + error!("Problem creating transcode temp dir: {:?}", err); + MediaFileUploadError::ServerError + })?; + + let transcoded_path = transcode_tempdir.path().join("transcoded.mp4"); + + info!("Transcoding {} → mp4", &original_mimetype); + ffmpeg_transcode_to_mp4(FfmpegTranscodeToMp4Args { + video_input_path: form.file.file.path(), + video_output_path: &transcoded_path, + }).map_err(|err| { + error!("Problem transcoding video to mp4: {:?}", err); + MediaFileUploadError::ServerError + })?; + + file_bytes = file_read_bytes(&transcoded_path) + .map_err(|e| { + error!("Problem reading transcoded mp4: {:?}", e); + MediaFileUploadError::ServerError + })?; + mimetype = "video/mp4".to_string(); + final_upload_file_path = transcoded_path; + transcode_tempdir_ref = Some(transcode_tempdir); + } + + // ==================== OPTIONAL VIDEO RESAMPLE ==================== // + + let should_resample = form.maybe_resample_fps.is_some() + || form.maybe_trim_start_millis.is_some() + || form.maybe_trim_end_millis.is_some(); + + if should_resample { + let frame_temp_dir = server_state.temp_dir_creator.new_tempdir("ffmpeg") + .map_err(|err| { + error!("Problem creating temp dir: {:?}", err); + MediaFileUploadError::ServerError + })?; + + let video_output_path = frame_temp_dir.path().join("output.mp4"); + + let maybe_new_frame_rate = form.maybe_resample_fps.map(|fps| fps.0); + let maybe_start_offset = form.maybe_trim_start_millis.map(|millis| Duration::from_millis(millis.0)); + let maybe_end_offset = form.maybe_trim_end_millis.map(|millis| Duration::from_millis(millis.0)); + + ffmpeg_trim_and_resample(Args { + video_input_path: &final_upload_file_path, + video_output_path: &video_output_path, + maybe_new_frame_rate, + maybe_start_offset, + maybe_end_offset, + }).map_err(|err| { + error!("Problem resampling video: {:?}", err); + MediaFileUploadError::ServerError + })?; + + file_bytes = file_read_bytes(&video_output_path) + .map_err(|e| { + error!("Problem reading file: {:?}", e); + MediaFileUploadError::ServerError + })?; + + mimetype = get_mimetype_for_bytes(file_bytes.as_ref()) + .map(|mimetype| mimetype.to_string()) + .ok_or_else(|| { + warn!("Could not determine mimetype for file"); + MediaFileUploadError::BadInput("Could not determine mimetype for file".to_string()) + })?; + + final_upload_file_path = video_output_path; + resample_tempdir_ref = Some(frame_temp_dir); // NB: Keep from going out of scope + } + + // True if we touched the user's file in any way (transcoded, resampled, or both). + let should_upload_original = needs_transcode || should_resample; + + // ==================== OTHER FILE METADATA ==================== // + + let mut maybe_duration_millis = None; + + match ffprobe_get_info(&final_upload_file_path) { + Ok(video_info) => { + maybe_duration_millis = video_info.duration + .map(|duration| duration.millis as u64); + } + Err(error) => { + warn!("Error reading video dimensions with ffprobe: {:?}", error); + } + } + + let maybe_filename = form.file.file_name.as_deref() + .as_deref() + .map(|filename| PathBuf::from(filename)); + + let extension = mimetype_to_extension(&mimetype) + .or_else(|| { + maybe_filename + .as_ref() + .and_then(|filename| filename.extension()) + .and_then(|ext| ext.to_str()) + }) + .ok_or_else(|| { + warn!("Could not determine file extension for mimetype: {}", &mimetype); + MediaFileUploadError::ServerError + })?; + + let extension = format!(".{extension}"); // NB: needs dot prefix + + let file_size_bytes = file_bytes.len(); + + let hash = sha256_hash_bytes(&file_bytes) + .map_err(|io_error| { + error!("Problem hashing bytes: {:?}", io_error); + MediaFileUploadError::ServerError + })?; + + // ==================== UPLOAD AND SAVE ==================== // + + const PREFIX : Option<&str> = Some("video_"); + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + &mimetype) + .await + .map_err(|e| { + warn!("Upload media bytes to bucket error: {:?}", e); + MediaFileUploadError::ServerError + })?; + + // ==================== UPLOAD ORIGINAL SIBLING ==================== // + // If we transcoded and/or resampled the user's upload, also upload the + // unmodified original alongside the canonical file as `_original.{ext}`. + + if should_upload_original { + let original_suffix = format!("_original.{}", original_extension); + let original_bucket_path = MediaFileBucketPath::from_object_hash( + public_upload_path.get_object_hash(), + PREFIX, + Some(&original_suffix), + ); + + info!( + "Uploading original sibling to bucket path: {}", + original_bucket_path.get_full_object_path_str(), + ); + + let original_bytes = file_read_bytes(&original_file_path) + .map_err(|e| { + error!("Problem reading original file for sibling upload: {:?}", e); + MediaFileUploadError::ServerError + })?; + + server_state.public_bucket_client.upload_file_with_content_type( + original_bucket_path.get_full_object_path_str(), + original_bytes.as_ref(), + &original_mimetype, + ) + .await + .map_err(|e| { + // NB: Sibling failure is non-fatal — the canonical upload already + // succeeded and the user can still use the media. Log loudly. + warn!("Upload original sibling to bucket error: {:?}", e); + }) + .ok(); + } + + let maybe_scene_source_media_file_token = form.maybe_scene_source_media_file_token + .as_ref() + .map(|token| &token.0); + + let maybe_prompt_token = form.maybe_prompt_token + .as_ref() + .map(|token| token.0.clone()); + + // NB: If we're uploading a video file that references an engine scene, then this is an engine + // render video, and we should mark it as a system (hidden) file. + let is_intermediate_system_file = + maybe_scene_source_media_file_token.is_some() || + form.is_intermediate_system_file.map(|b| b.0).unwrap_or(false); + + let maybe_generation_provider = form.maybe_generation_provider + .as_ref() + .and_then(|text| try_parse_generation_provider(text.as_ref())); + + let upload_type = if maybe_generation_provider.is_some() { + UploadType::ThirdPartyInference + } else { + UploadType::Filesystem + }; + + let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs { + maybe_media_class: Some(MediaFileClass::Video), + media_file_type: MediaFileType::Video, + maybe_creator_user_token: maybe_user_token, + maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility, + upload_type, + maybe_engine_category: None, + maybe_animation_type: None, + maybe_prompt_token: maybe_prompt_token.as_ref(), + maybe_batch_token: None, + maybe_mime_type: Some(&mimetype), + file_size_bytes: file_size_bytes as u64, + maybe_duration_millis, + sha256_checksum: &hash, + maybe_title: maybe_title.as_deref(), + maybe_scene_source_media_file_token, + is_intermediate_system_file, // NB: is_user_upload = TRUE + maybe_generation_provider, + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix: PREFIX, + maybe_public_bucket_extension: Some(&extension), + pool: &server_state.mysql_pool, + }) + .await + .map_err(|err| { + warn!("New file creation DB error: {:?}", err); + MediaFileUploadError::ServerError + })?; + + info!("new media file id: {} token: {:?}", record_id, &token); + + let thumbnail_task_result = ThumbnailTaskBuilder::new_for_source_mimetype(ThumbnailTaskInputMimeType::MP4) + .with_bucket(server_state.public_bucket_client.bucket_name().as_str()) + .with_path(&*path_to_string(public_upload_path.to_full_object_pathbuf())) + .with_output_suffix("thumb") + .with_event_id(&token.to_string()) + .send_all() + .await; + + match thumbnail_task_result { + Ok(thumbnail_task) => { + debug!("Thumbnail tasks sent: {:?}", thumbnail_task); + }, + Err(e) => { + error!("Failed to create some/all thumbnail tasks: {:?}", e); + } + } + + Ok(Json(OmniUploadVideoMediaFileSuccessResponse { + success: true, + media_file_token: token, + })) +} + +fn fast_form_validations(form: &OmniUploadVideoMediaFileForm) -> Result<(), MediaFileUploadError> { + if let Some(resample_fps) = form.maybe_resample_fps.as_ref() { + if **resample_fps > 24 { + return Err(MediaFileUploadError::BadInput("Resample FPS must be 24 or lower".to_string())); + } else if **resample_fps == 0 { + return Err(MediaFileUploadError::BadInput("Resample FPS must be greater than 0".to_string())); + } + } + + form.maybe_trim_start_millis.as_ref().zip(form.maybe_trim_end_millis.as_ref()) + .map(|(start, end)| { + if **start >= **end { + return Err(MediaFileUploadError::BadInput("Trim start must be less than trim end".to_string())); + } + Ok(()) + }) + .transpose()?; + + let is_intermediate_system_file = form.is_intermediate_system_file + .as_ref() + .map(|b| b.0) + .unwrap_or(false); + + if is_intermediate_system_file && form.maybe_prompt_token.is_some() { + return Err(MediaFileUploadError::BadInput( + "Cannot set `is_intermediate_system_file` to true if `maybe_prompt_token` is provided." + .to_string())); + } + + Ok(()) +} + +/// Map an API-key authentication failure onto this endpoint's error type. A genuine 401 stays a +/// 401; anything else (e.g. a DB error during lookup) becomes a 500. +fn map_api_key_auth_error(err: CommonWebError) -> MediaFileUploadError { + match err { + CommonWebError::NotAuthorized => { + MediaFileUploadError::NotAuthorizedVerbose("invalid or missing API key".to_string()) + } + _ => MediaFileUploadError::ServerError, + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/omni_api_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/omni_api_routes.rs index a889d7b66d..72a1f15082 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/omni_api_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/omni_api_routes.rs @@ -5,6 +5,9 @@ use actix_web::{web, App, Error, HttpResponse}; use crate::http_server::endpoints::omni_api::generate::image::omni_api_image_generate_handler::omni_api_image_generate_handler; use crate::http_server::endpoints::omni_api::generate::video::omni_api_video_generate_handler::omni_api_video_generate_handler; +use crate::http_server::endpoints::omni_api::upload::omni_upload_audio_media_file_handler::omni_upload_audio_media_file_handler; +use crate::http_server::endpoints::omni_api::upload::omni_upload_image_media_file_handler::omni_upload_image_media_file_handler; +use crate::http_server::endpoints::omni_api::upload::omni_upload_video_media_file_handler::omni_upload_video_media_file_handler; /// API-key authenticated generation endpoints. These mirror `/v1/omni_gen/generate/*` but read the /// caller's identity from the `Authorization` header API key instead of a session cookie. The @@ -32,5 +35,19 @@ where .route(web::head().to(|| HttpResponse::Ok())) ) ) + .service(web::scope("/upload") + .service(web::resource("/audio") + .route(web::post().to(omni_upload_audio_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/image") + .route(web::post().to(omni_upload_image_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/video") + .route(web::post().to(omni_upload_video_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) ) }