From aa594b773e41efd1f92ea37123f3a0c781db66d3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 27 Apr 2025 02:17:53 -0400 Subject: [PATCH] upload media files from gpt-image-1 --- .../media_file_origin_product_category.rs | 13 +- .../media_file_insert_builder.rs | 57 +++++-- .../media_file_insert_builder_error.rs | 27 ++++ .../media_files/create/insert_builder/mod.rs | 3 +- crates/service/web/storyteller_web/Cargo.toml | 3 +- .../update_gpt_image_job_status_handler.rs | 140 ++++++++++++++++-- 6 files changed, 208 insertions(+), 35 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs diff --git a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs index 7813f50ba7..8ffc554344 100644 --- a/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs +++ b/crates/_types/enums/src/by_table/media_files/media_file_origin_product_category.rs @@ -39,6 +39,10 @@ pub enum MediaFileOriginProductCategory { #[serde(rename = "vst")] VideoStyleTransfer, + /// Image Studio + #[serde(rename = "image_studio")] + ImageStudio, + /// Storyteller Studio #[serde(rename = "studio")] StorytellerStudio, @@ -87,6 +91,7 @@ impl MediaFileOriginProductCategory { Self::FaceFusion => "face_fusion", Self::FaceMirror => "face_mirror", Self::VideoStyleTransfer => "vst", + Self::ImageStudio => "image_studio", Self::StorytellerStudio => "studio", Self::TextToSpeech => "tts", Self::VoiceConversion => "voice_conversion", @@ -105,6 +110,7 @@ impl MediaFileOriginProductCategory { "face_fusion" => Ok(Self::FaceFusion), "face_mirror" => Ok(Self::FaceMirror), "vst" => Ok(Self::VideoStyleTransfer), + "image_studio" => Ok(Self::ImageStudio), "studio" => Ok(Self::StorytellerStudio), "tts" => Ok(Self::TextToSpeech), "voice_conversion" => Ok(Self::VoiceConversion), @@ -126,6 +132,7 @@ impl MediaFileOriginProductCategory { Self::FaceFusion, Self::FaceMirror, Self::VideoStyleTransfer, + Self::ImageStudio, Self::StorytellerStudio, Self::TextToSpeech, Self::VoiceConversion, @@ -153,6 +160,7 @@ mod tests { assert_serialization(MediaFileOriginProductCategory::FaceFusion, "face_fusion"); assert_serialization(MediaFileOriginProductCategory::FaceMirror, "face_mirror"); assert_serialization(MediaFileOriginProductCategory::VideoStyleTransfer, "vst"); + assert_serialization(MediaFileOriginProductCategory::ImageStudio, "image_studio"); assert_serialization(MediaFileOriginProductCategory::StorytellerStudio, "studio"); assert_serialization(MediaFileOriginProductCategory::TextToSpeech, "tts"); assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion"); @@ -170,6 +178,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::FaceFusion.to_str(), "face_fusion"); assert_eq!(MediaFileOriginProductCategory::FaceMirror.to_str(), "face_mirror"); assert_eq!(MediaFileOriginProductCategory::VideoStyleTransfer.to_str(), "vst"); + assert_eq!(MediaFileOriginProductCategory::ImageStudio.to_str(), "image_studio"); assert_eq!(MediaFileOriginProductCategory::StorytellerStudio.to_str(), "studio"); assert_eq!(MediaFileOriginProductCategory::TextToSpeech.to_str(), "tts"); assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion"); @@ -187,6 +196,7 @@ mod tests { assert_eq!(MediaFileOriginProductCategory::from_str("face_fusion").unwrap(), MediaFileOriginProductCategory::FaceFusion); assert_eq!(MediaFileOriginProductCategory::from_str("face_mirror").unwrap(), MediaFileOriginProductCategory::FaceMirror); assert_eq!(MediaFileOriginProductCategory::from_str("vst").unwrap(), MediaFileOriginProductCategory::VideoStyleTransfer); + assert_eq!(MediaFileOriginProductCategory::from_str("image_studio").unwrap(), MediaFileOriginProductCategory::ImageStudio); assert_eq!(MediaFileOriginProductCategory::from_str("studio").unwrap(), MediaFileOriginProductCategory::StorytellerStudio); assert_eq!(MediaFileOriginProductCategory::from_str("tts").unwrap(), MediaFileOriginProductCategory::TextToSpeech); assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion); @@ -200,12 +210,13 @@ mod tests { #[test] fn all_variants() { let mut variants = MediaFileOriginProductCategory::all_variants(); - assert_eq!(variants.len(), 13); + assert_eq!(variants.len(), 14); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceFusion)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceMirror)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoStyleTransfer)); + assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ImageStudio)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::StorytellerStudio)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech)); assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion)); diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs index 0afd0bd61d..8d02c6ec49 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder.rs @@ -12,13 +12,15 @@ use sqlx::MySqlPool; use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::users::UserToken; +use crate::queries::media_files::create::insert_builder::media_file_insert_builder_error::MediaFileInsertBuilderError; /// Builder for inserting a media files into the database. /// Resembles an ORM, but can still use statically type checked queries (which is probably /// really important since we don't have tests on database functionality yet). #[derive(Clone)] pub struct MediaFileInsertBuilder { - token: MediaFileToken, + // TODO: Allow specifying the token externally + // token: MediaFileToken, // Creator info maybe_creator_user_token: Option, @@ -65,18 +67,17 @@ pub struct MediaFileInsertBuilder { // Storage details public_bucket_directory_hash: Option, // NB: Non-nullable field(s) - - + // Remaining fields... } impl MediaFileInsertBuilder { - pub fn generate_token() -> Self { - Self::with_token(&MediaFileToken::generate()) - } + //pub fn generate_token() -> Self { + // Self::with_token(&MediaFileToken::generate()) + //} - pub fn with_token(token: &MediaFileToken) -> Self { + pub fn new() -> Self { MediaFileInsertBuilder { - token: token.clone(), + //token: token.clone(), maybe_creator_user_token: None, maybe_creator_anonymous_visitor_token: None, creator_ip_address: None, @@ -101,11 +102,21 @@ impl MediaFileInsertBuilder { self } - pub fn anonymous_visitor_token(mut self, avt: &AnonymousVisitorTrackingToken) -> Self { + pub fn maybe_creator_user(mut self, maybe_user: Option<&UserToken>) -> Self { + self.maybe_creator_user_token = maybe_user.map(|token| token.clone()); + self + } + + pub fn creator_anonymous_visitor(mut self, avt: &AnonymousVisitorTrackingToken) -> Self { self.maybe_creator_anonymous_visitor_token = Some(avt.clone()); self } + pub fn maybe_creator_anonymous_visitor(mut self, maybe_avt: Option<&AnonymousVisitorTrackingToken>) -> Self { + self.maybe_creator_anonymous_visitor_token = maybe_avt.map(|token| token.clone()); + self + } + pub fn creator_ip_address(mut self, ip_address: &str) -> Self { self.creator_ip_address = Some(ip_address.to_string()); self @@ -187,21 +198,30 @@ impl MediaFileInsertBuilder { // TODO: maybe_prompt_token // TODO: maybe_batch_token + pub fn public_bucket_directory_hash(mut self, public_bucket_directory_hash: &MediaFileBucketPath) -> Self { + self.public_bucket_directory_hash = Some(public_bucket_directory_hash.clone()); + self + } + // TODO(bt,2025-04-26): Other connector options. - pub async fn insert_pool(self, mysql_pool: &MySqlPool) -> AnyhowResult<(MediaFileToken, u64)> { + pub async fn insert_pool(self, mysql_pool: &MySqlPool) -> Result { let media_file_type = self.media_file_type - .ok_or_else(|| anyhow!("Media file type is required"))?; + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Media file type is required".to_string()))?; let origin_category = self.origin_category - .ok_or_else(|| anyhow!("Origin category is required"))?; + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Origin category is required".to_string()))?; let checksum_sha2 = self.checksum_sha2 - .ok_or_else(|| anyhow!("Checksum SHA2 is required"))?; + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Checksum SHA2 is required".to_string()))?; let bucket_path = self.public_bucket_directory_hash - .ok_or_else(|| anyhow!("Public bucket directory hash is required"))?; + .ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField( + "Public bucket directory hash is required".to_string()))?; - insert_media_file_generic(InsertArgs { + let result = insert_media_file_generic(InsertArgs { pool: mysql_pool, maybe_creator_user_token: self.maybe_creator_user_token.as_ref(), maybe_creator_anonymous_visitor_token: self.maybe_creator_anonymous_visitor_token.as_ref(), @@ -239,6 +259,11 @@ impl MediaFileInsertBuilder { generated_by_worker: None, // TODO generated_by_cluster: None, // TODO maybe_mod_user_token: None, // TODO - }).await + }).await; + + match result { + Ok(result) => Ok(result.0), + Err(err) => Err(MediaFileInsertBuilderError::ProbablyDatabaseError(err)), + } } } diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs new file mode 100644 index 0000000000..f7dc467a17 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/media_file_insert_builder_error.rs @@ -0,0 +1,27 @@ +use std::error::Error; +use std::fmt::{Debug, Display, Formatter}; +use errors::AnyhowError; + +#[derive(Debug)] +pub enum MediaFileInsertBuilderError { + /// A user-provided field is missing (400 error). + MissingRequiredField(String), + + // TODO: We need to refactor to ensure these are actually DB errors (500s). + ProbablyDatabaseError(AnyhowError), +} + +impl Error for MediaFileInsertBuilderError {} + +impl Display for MediaFileInsertBuilderError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + MediaFileInsertBuilderError::MissingRequiredField(reason) => { + write!(f, "Missing required field: {}", reason) + } + MediaFileInsertBuilderError::ProbablyDatabaseError(reason) => { + write!(f, "Probably database error: {:?}", reason) + } + } + } +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs index e656db47cd..c86603dcf7 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_builder/mod.rs @@ -1 +1,2 @@ -mod media_file_insert_builder; \ No newline at end of file +pub mod media_file_insert_builder; +mod media_file_insert_builder_error; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index c925dcc9d8..95260d8987 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -51,11 +51,12 @@ google_sign_in = { path = "../../../lib/api_clients/google_sign_in" } hashing = { path = "../../../lib/files/hashing" } http_server_common = { path = "../../../lib/deprecated/http_server_common" } idempotency = { path = "../../../lib/idempotency" } +images = { workspace = true } markdown = { path = "../../../lib/markdown" } media = { path = "../../../lib/files/media" } memory_caching = { path = "../../../lib/caching/memory_caching" } migration = { path = "../../../lib/migration" } -mimetypes = { path = "../../../lib/files/mimetypes" } +mimetypes = { workspace = true } mysql_queries = { path = "../../../lib/mysql_queries" } openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" } password = { path = "../../../lib/password" } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs index 1471bfc6bb..b21c689ac6 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/update_gpt_image_job_status_handler.rs @@ -1,25 +1,45 @@ use std::fmt; use std::sync::Arc; +use crate::http_server::endpoints::engine::create_scene_handler::CreateSceneError; +use crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::GetInferenceJobStatusError; +use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::http_server::web_utils::user_session::require_user_session::require_user_session; +use crate::state::server_state::ServerState; use actix_web::error::ResponseError; use actix_web::http::StatusCode; use actix_web::web::{Json, Path}; use actix_web::{web, HttpMessage, HttpRequest, HttpResponse}; -use log::{error, warn}; -use utoipa::ToSchema; +use base64::alphabet::STANDARD; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use cloud_storage::bucket_client::BucketClient; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::job_status::JobStatus; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; -use crate::http_server::web_utils::user_session::require_user_session::require_user_session; -use crate::state::server_state::ServerState; +use enums::common::job_status_plus::JobStatusPlus; +use errors::AnyhowResult; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; use http_server_common::request::get_request_ip::get_request_ip; +use images::image_info::image_info::ImageInfo; +use log::{error, info, warn}; use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed; use mysql_queries::queries::generic_inference::web::dismiss_finished_jobs_for_user::dismiss_finished_jobs_for_user; use mysql_queries::queries::generic_inference::web::get_inference_job_status::{get_inference_job_status, get_inference_job_status_from_connection}; +use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus; use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_cancelled_by_user::mark_generic_inference_job_cancelled_by_user; +use mysql_queries::queries::media_files::create::generic_insert::insert_media_file_generic::{insert_media_file_generic, InsertArgs}; +use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType}; use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead; +use sqlx::{MySqlConnection, MySqlPool}; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; -use crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::GetInferenceJobStatusError; +use utoipa::ToSchema; #[derive(Deserialize, ToSchema)] pub struct UpdateGptImageJobStatusRequest { @@ -105,20 +125,20 @@ pub async fn update_gpt_image_job_status_handler( request: Json, server_state: web::Data>) -> Result, UpdateGptImageJobStatusError> { - // TODO(bt,2024-06-16): Reuse connection - let mut mysql_connection = server_state.mysql_pool.acquire() - .await - .map_err(|e| { - warn!("Could not acquire DB pool: {:?}", e); - UpdateGptImageJobStatusError::ServerError - })?; + //// TODO(bt,2024-06-16): Reuse connection + //let mut mysql_connection = server_state.mysql_pool.acquire() + // .await + // .map_err(|e| { + // warn!("Could not acquire DB pool: {:?}", e); + // UpdateGptImageJobStatusError::ServerError + // })?; - let maybe_status = get_inference_job_status( + let inference_job = get_inference_job_status( &request.job_token, &server_state.mysql_pool ).await; - let record = match maybe_status { + let inference_job = match inference_job { Ok(Some(record)) => record, Ok(None) => return Err(UpdateGptImageJobStatusError::NotFound), Err(err) => { @@ -127,7 +147,37 @@ pub async fn update_gpt_image_job_status_handler( } }; - // TODO(bt): Upload images if job was successful and create relevant media files + match inference_job.status { + JobStatusPlus::CompleteSuccess => { + return Ok(Json(UpdateGptImageJobStatusSuccessResponse { success: true })) + }, + // TODO: Handle other states as terminal states? + //JobStatusPlus::CompleteFailure => return Err(UpdateGptImageJobStatusError::ServerError), + //JobStatusPlus::AttemptFailed => {} + //JobStatusPlus::Dead => {} + //JobStatusPlus::CancelledByUser => return Err(UpdateGptImageJobStatusError::ServerError), + //JobStatusPlus::CancelledBySystem => {} + _ => {} // Intentional fall through. + } + + if let Some(images) = &request.images { + for image in images.iter() { + let result = upload_and_save_image( + image, + &inference_job, + &server_state.mysql_pool, + &server_state.public_bucket_client, + ).await; + + let media_token = match result { + Ok(media_token) => media_token, + Err(err) => { + error!("Error uploading image: {:?}", err); + return Err(UpdateGptImageJobStatusError::ServerError); + } + }; + } + } // TODO(bt): Update job status @@ -136,3 +186,61 @@ pub async fn update_gpt_image_job_status_handler( })) } +async fn upload_and_save_image( + base64_image: &str, + inference_job: &GenericInferenceJobStatus, + mysql_pool: &MySqlPool, + public_bucket_client: &BucketClient, +) -> AnyhowResult { + + let image_bytes = BASE64_STANDARD.decode(base64_image)?; + + // Read file metadata + let file_size_bytes = image_bytes.len(); + let file_hash = sha256_hash_bytes(&image_bytes)?; + let image_info = ImageInfo::decode_image_from_bytes(&image_bytes)?; + let mimetype = image_info.mime_type(); + + // Upload file + const PREFIX : Option<&str> = Some("image_"); + + let extension_with_period = image_info.file_extension() + .map(|ext| ext.extension_with_period()) + .unwrap_or_else(|| ".png"); + + let media_file_type = match mimetype { + "image/png" => MediaFileType::Png, + "image/jpeg" => MediaFileType::Jpg, + _ => MediaFileType::Png, + }; + + let bucket_upload_path = MediaFileBucketPath::generate_new( + PREFIX, Some(extension_with_period)); + + info!("Uploading media to bucket path: {}", bucket_upload_path.get_full_object_path_str()); + + public_bucket_client.upload_file_with_content_type( + bucket_upload_path.get_full_object_path_str(), + image_bytes.as_ref(), + mimetype + ).await?; + + let media_token = MediaFileInsertBuilder::new() + .maybe_creator_user(inference_job.user_details.maybe_creator_user_token.as_ref()) + .maybe_creator_anonymous_visitor(inference_job.user_details.maybe_creator_anonymous_visitor_token.as_ref()) + .creator_ip_address(&inference_job.user_details.creator_ip_address) + .public_bucket_directory_hash(&bucket_upload_path) + .media_file_class(MediaFileClass::Image) + .media_file_type(media_file_type) + .media_file_origin_category(MediaFileOriginCategory::Inference) + .media_file_origin_product_category(MediaFileOriginProductCategory::ImageStudio) + .mime_type(mimetype) + .file_size_bytes(file_size_bytes as u64) + .frame_width(image_info.width()) + .frame_height(image_info.height()) + .checksum_sha2(&file_hash) + .insert_pool(mysql_pool) + .await?; + + Ok(media_token) +}