upload media files from gpt-image-1

This commit is contained in:
Brandon Thomas committed 2025-04-27 02:17:53 -04:00
1 parent ac209f8f4f
commit aa594b773e
6 files changed
+208 -35

No files matched your search

@@ -39,6 +39,10 @@ pub enum MediaFileOriginProductCategory {
#[serde(rename = "vst")]
VideoStyleTransfer,
/// Image Studio
#[serde(rename = "image_studio")]
ImageStudio,
/// Storyteller Studio
#[serde(rename = "studio")]
StorytellerStudio,
@@ -87,6 +91,7 @@ impl MediaFileOriginProductCategory {
Self::FaceFusion => "face_fusion",
Self::FaceMirror => "face_mirror",
Self::VideoStyleTransfer => "vst",
Self::ImageStudio => "image_studio",
Self::StorytellerStudio => "studio",
Self::TextToSpeech => "tts",
Self::VoiceConversion => "voice_conversion",
@@ -105,6 +110,7 @@ impl MediaFileOriginProductCategory {
"face_fusion" => Ok(Self::FaceFusion),
"face_mirror" => Ok(Self::FaceMirror),
"vst" => Ok(Self::VideoStyleTransfer),
"image_studio" => Ok(Self::ImageStudio),
"studio" => Ok(Self::StorytellerStudio),
"tts" => Ok(Self::TextToSpeech),
"voice_conversion" => Ok(Self::VoiceConversion),
@@ -126,6 +132,7 @@ impl MediaFileOriginProductCategory {
Self::FaceFusion,
Self::FaceMirror,
Self::VideoStyleTransfer,
Self::ImageStudio,
Self::StorytellerStudio,
Self::TextToSpeech,
Self::VoiceConversion,
@@ -153,6 +160,7 @@ mod tests {
assert_serialization(MediaFileOriginProductCategory::FaceFusion, "face_fusion");
assert_serialization(MediaFileOriginProductCategory::FaceMirror, "face_mirror");
assert_serialization(MediaFileOriginProductCategory::VideoStyleTransfer, "vst");
assert_serialization(MediaFileOriginProductCategory::ImageStudio, "image_studio");
assert_serialization(MediaFileOriginProductCategory::StorytellerStudio, "studio");
assert_serialization(MediaFileOriginProductCategory::TextToSpeech, "tts");
assert_serialization(MediaFileOriginProductCategory::VoiceConversion, "voice_conversion");
@@ -170,6 +178,7 @@ mod tests {
assert_eq!(MediaFileOriginProductCategory::FaceFusion.to_str(), "face_fusion");
assert_eq!(MediaFileOriginProductCategory::FaceMirror.to_str(), "face_mirror");
assert_eq!(MediaFileOriginProductCategory::VideoStyleTransfer.to_str(), "vst");
assert_eq!(MediaFileOriginProductCategory::ImageStudio.to_str(), "image_studio");
assert_eq!(MediaFileOriginProductCategory::StorytellerStudio.to_str(), "studio");
assert_eq!(MediaFileOriginProductCategory::TextToSpeech.to_str(), "tts");
assert_eq!(MediaFileOriginProductCategory::VoiceConversion.to_str(), "voice_conversion");
@@ -187,6 +196,7 @@ mod tests {
assert_eq!(MediaFileOriginProductCategory::from_str("face_fusion").unwrap(), MediaFileOriginProductCategory::FaceFusion);
assert_eq!(MediaFileOriginProductCategory::from_str("face_mirror").unwrap(), MediaFileOriginProductCategory::FaceMirror);
assert_eq!(MediaFileOriginProductCategory::from_str("vst").unwrap(), MediaFileOriginProductCategory::VideoStyleTransfer);
assert_eq!(MediaFileOriginProductCategory::from_str("image_studio").unwrap(), MediaFileOriginProductCategory::ImageStudio);
assert_eq!(MediaFileOriginProductCategory::from_str("studio").unwrap(), MediaFileOriginProductCategory::StorytellerStudio);
assert_eq!(MediaFileOriginProductCategory::from_str("tts").unwrap(), MediaFileOriginProductCategory::TextToSpeech);
assert_eq!(MediaFileOriginProductCategory::from_str("voice_conversion").unwrap(), MediaFileOriginProductCategory::VoiceConversion);
@@ -200,12 +210,13 @@ mod tests {
#[test]
fn all_variants() {
let mut variants = MediaFileOriginProductCategory::all_variants();
assert_eq!(variants.len(), 13);
assert_eq!(variants.len(), 14);
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::Unknown));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceAnimator));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceFusion));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::FaceMirror));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VideoStyleTransfer));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::ImageStudio));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::StorytellerStudio));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::TextToSpeech));
assert_eq!(variants.pop_first(), Some(MediaFileOriginProductCategory::VoiceConversion));
@@ -12,13 +12,15 @@ use sqlx::MySqlPool;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::users::UserToken;
use crate::queries::media_files::create::insert_builder::media_file_insert_builder_error::MediaFileInsertBuilderError;
/// Builder for inserting a media files into the database.
/// Resembles an ORM, but can still use statically type checked queries (which is probably
/// really important since we don't have tests on database functionality yet).
#[derive(Clone)]
pub struct MediaFileInsertBuilder {
token: MediaFileToken,
// TODO: Allow specifying the token externally
// token: MediaFileToken,
// Creator info
maybe_creator_user_token: Option<UserToken>,
@@ -65,18 +67,17 @@ pub struct MediaFileInsertBuilder {
// Storage details
public_bucket_directory_hash: Option<MediaFileBucketPath>, // NB: Non-nullable field(s)
// Remaining fields...
}
impl MediaFileInsertBuilder {
pub fn generate_token() -> Self {
Self::with_token(&MediaFileToken::generate())
}
//pub fn generate_token() -> Self {
// Self::with_token(&MediaFileToken::generate())
//}
pub fn with_token(token: &MediaFileToken) -> Self {
pub fn new() -> Self {
MediaFileInsertBuilder {
token: token.clone(),
//token: token.clone(),
maybe_creator_user_token: None,
maybe_creator_anonymous_visitor_token: None,
creator_ip_address: None,
@@ -101,11 +102,21 @@ impl MediaFileInsertBuilder {
self
}
pub fn anonymous_visitor_token(mut self, avt: &AnonymousVisitorTrackingToken) -> Self {
pub fn maybe_creator_user(mut self, maybe_user: Option<&UserToken>) -> Self {
self.maybe_creator_user_token = maybe_user.map(|token| token.clone());
self
}
pub fn creator_anonymous_visitor(mut self, avt: &AnonymousVisitorTrackingToken) -> Self {
self.maybe_creator_anonymous_visitor_token = Some(avt.clone());
self
}
pub fn maybe_creator_anonymous_visitor(mut self, maybe_avt: Option<&AnonymousVisitorTrackingToken>) -> Self {
self.maybe_creator_anonymous_visitor_token = maybe_avt.map(|token| token.clone());
self
}
pub fn creator_ip_address(mut self, ip_address: &str) -> Self {
self.creator_ip_address = Some(ip_address.to_string());
self
@@ -187,21 +198,30 @@ impl MediaFileInsertBuilder {
// TODO: maybe_prompt_token
// TODO: maybe_batch_token
pub fn public_bucket_directory_hash(mut self, public_bucket_directory_hash: &MediaFileBucketPath) -> Self {
self.public_bucket_directory_hash = Some(public_bucket_directory_hash.clone());
self
}
// TODO(bt,2025-04-26): Other connector options.
pub async fn insert_pool(self, mysql_pool: &MySqlPool) -> AnyhowResult<(MediaFileToken, u64)> {
pub async fn insert_pool(self, mysql_pool: &MySqlPool) -> Result<MediaFileToken, MediaFileInsertBuilderError> {
let media_file_type = self.media_file_type
.ok_or_else(|| anyhow!("Media file type is required"))?;
.ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField(
"Media file type is required".to_string()))?;
let origin_category = self.origin_category
.ok_or_else(|| anyhow!("Origin category is required"))?;
.ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField(
"Origin category is required".to_string()))?;
let checksum_sha2 = self.checksum_sha2
.ok_or_else(|| anyhow!("Checksum SHA2 is required"))?;
.ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField(
"Checksum SHA2 is required".to_string()))?;
let bucket_path = self.public_bucket_directory_hash
.ok_or_else(|| anyhow!("Public bucket directory hash is required"))?;
.ok_or_else(|| MediaFileInsertBuilderError::MissingRequiredField(
"Public bucket directory hash is required".to_string()))?;
insert_media_file_generic(InsertArgs {
let result = insert_media_file_generic(InsertArgs {
pool: mysql_pool,
maybe_creator_user_token: self.maybe_creator_user_token.as_ref(),
maybe_creator_anonymous_visitor_token: self.maybe_creator_anonymous_visitor_token.as_ref(),
@@ -239,6 +259,11 @@ impl MediaFileInsertBuilder {
generated_by_worker: None, // TODO
generated_by_cluster: None, // TODO
maybe_mod_user_token: None, // TODO
}).await
}).await;
match result {
Ok(result) => Ok(result.0),
Err(err) => Err(MediaFileInsertBuilderError::ProbablyDatabaseError(err)),
}
}
}
@@ -0,0 +1,27 @@
use std::error::Error;
use std::fmt::{Debug, Display, Formatter};
use errors::AnyhowError;
#[derive(Debug)]
pub enum MediaFileInsertBuilderError {
/// A user-provided field is missing (400 error).
MissingRequiredField(String),
// TODO: We need to refactor to ensure these are actually DB errors (500s).
ProbablyDatabaseError(AnyhowError),
}
impl Error for MediaFileInsertBuilderError {}
impl Display for MediaFileInsertBuilderError {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
match self {
MediaFileInsertBuilderError::MissingRequiredField(reason) => {
write!(f, "Missing required field: {}", reason)
}
MediaFileInsertBuilderError::ProbablyDatabaseError(reason) => {
write!(f, "Probably database error: {:?}", reason)
}
}
}
}
@@ -1 +1,2 @@
mod media_file_insert_builder;
pub mod media_file_insert_builder;
mod media_file_insert_builder_error;
@@ -51,11 +51,12 @@ google_sign_in = { path = "../../../lib/api_clients/google_sign_in" }
hashing = { path = "../../../lib/files/hashing" }
http_server_common = { path = "../../../lib/deprecated/http_server_common" }
idempotency = { path = "../../../lib/idempotency" }
images = { workspace = true }
markdown = { path = "../../../lib/markdown" }
media = { path = "../../../lib/files/media" }
memory_caching = { path = "../../../lib/caching/memory_caching" }
migration = { path = "../../../lib/migration" }
mimetypes = { path = "../../../lib/files/mimetypes" }
mimetypes = { workspace = true }
mysql_queries = { path = "../../../lib/mysql_queries" }
openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" }
password = { path = "../../../lib/password" }
@@ -1,25 +1,45 @@
use std::fmt;
use std::sync::Arc;
use crate::http_server::endpoints::engine::create_scene_handler::CreateSceneError;
use crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::GetInferenceJobStatusError;
use crate::http_server::endpoints::media_files::upload::upload_error::MediaFileUploadError;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::http_server::web_utils::user_session::require_user_session::require_user_session;
use crate::state::server_state::ServerState;
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use actix_web::web::{Json, Path};
use actix_web::{web, HttpMessage, HttpRequest, HttpResponse};
use log::{error, warn};
use utoipa::ToSchema;
use base64::alphabet::STANDARD;
use base64::prelude::BASE64_STANDARD;
use base64::Engine;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::job_status::JobStatus;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::http_server::web_utils::user_session::require_user_session::require_user_session;
use crate::state::server_state::ServerState;
use enums::common::job_status_plus::JobStatusPlus;
use errors::AnyhowResult;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use images::image_info::image_info::ImageInfo;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed;
use mysql_queries::queries::generic_inference::web::dismiss_finished_jobs_for_user::dismiss_finished_jobs_for_user;
use mysql_queries::queries::generic_inference::web::get_inference_job_status::{get_inference_job_status, get_inference_job_status_from_connection};
use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus;
use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_cancelled_by_user::mark_generic_inference_job_cancelled_by_user;
use mysql_queries::queries::media_files::create::generic_insert::insert_media_file_generic::{insert_media_file_generic, InsertArgs};
use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use mysql_queries::queries::tts::tts_inference_jobs::mark_tts_inference_job_permanently_dead::mark_tts_inference_job_permanently_dead;
use sqlx::{MySqlConnection, MySqlPool};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::inference_job::get::get_inference_job_status_handler::GetInferenceJobStatusError;
use utoipa::ToSchema;
#[derive(Deserialize, ToSchema)]
pub struct UpdateGptImageJobStatusRequest {
@@ -105,20 +125,20 @@ pub async fn update_gpt_image_job_status_handler(
request: Json<UpdateGptImageJobStatusRequest>,
server_state: web::Data<Arc<ServerState>>) -> Result<Json<UpdateGptImageJobStatusSuccessResponse>, UpdateGptImageJobStatusError>
{
// TODO(bt,2024-06-16): Reuse connection
let mut mysql_connection = server_state.mysql_pool.acquire()
.await
.map_err(|e| {
warn!("Could not acquire DB pool: {:?}", e);
UpdateGptImageJobStatusError::ServerError
})?;
//// TODO(bt,2024-06-16): Reuse connection
//let mut mysql_connection = server_state.mysql_pool.acquire()
// .await
// .map_err(|e| {
// warn!("Could not acquire DB pool: {:?}", e);
// UpdateGptImageJobStatusError::ServerError
// })?;
let maybe_status = get_inference_job_status(
let inference_job = get_inference_job_status(
&request.job_token,
&server_state.mysql_pool
).await;
let record = match maybe_status {
let inference_job = match inference_job {
Ok(Some(record)) => record,
Ok(None) => return Err(UpdateGptImageJobStatusError::NotFound),
Err(err) => {
@@ -127,7 +147,37 @@ pub async fn update_gpt_image_job_status_handler(
}
};
// TODO(bt): Upload images if job was successful and create relevant media files
match inference_job.status {
JobStatusPlus::CompleteSuccess => {
return Ok(Json(UpdateGptImageJobStatusSuccessResponse { success: true }))
},
// TODO: Handle other states as terminal states?
//JobStatusPlus::CompleteFailure => return Err(UpdateGptImageJobStatusError::ServerError),
//JobStatusPlus::AttemptFailed => {}
//JobStatusPlus::Dead => {}
//JobStatusPlus::CancelledByUser => return Err(UpdateGptImageJobStatusError::ServerError),
//JobStatusPlus::CancelledBySystem => {}
_ => {} // Intentional fall through.
}
if let Some(images) = &request.images {
for image in images.iter() {
let result = upload_and_save_image(
image,
&inference_job,
&server_state.mysql_pool,
&server_state.public_bucket_client,
).await;
let media_token = match result {
Ok(media_token) => media_token,
Err(err) => {
error!("Error uploading image: {:?}", err);
return Err(UpdateGptImageJobStatusError::ServerError);
}
};
}
}
// TODO(bt): Update job status
@@ -136,3 +186,61 @@ pub async fn update_gpt_image_job_status_handler(
}))
}
async fn upload_and_save_image(
base64_image: &str,
inference_job: &GenericInferenceJobStatus,
mysql_pool: &MySqlPool,
public_bucket_client: &BucketClient,
) -> AnyhowResult<MediaFileToken> {
let image_bytes = BASE64_STANDARD.decode(base64_image)?;
// Read file metadata
let file_size_bytes = image_bytes.len();
let file_hash = sha256_hash_bytes(&image_bytes)?;
let image_info = ImageInfo::decode_image_from_bytes(&image_bytes)?;
let mimetype = image_info.mime_type();
// Upload file
const PREFIX : Option<&str> = Some("image_");
let extension_with_period = image_info.file_extension()
.map(|ext| ext.extension_with_period())
.unwrap_or_else(|| ".png");
let media_file_type = match mimetype {
"image/png" => MediaFileType::Png,
"image/jpeg" => MediaFileType::Jpg,
_ => MediaFileType::Png,
};
let bucket_upload_path = MediaFileBucketPath::generate_new(
PREFIX, Some(extension_with_period));
info!("Uploading media to bucket path: {}", bucket_upload_path.get_full_object_path_str());
public_bucket_client.upload_file_with_content_type(
bucket_upload_path.get_full_object_path_str(),
image_bytes.as_ref(),
mimetype
).await?;
let media_token = MediaFileInsertBuilder::new()
.maybe_creator_user(inference_job.user_details.maybe_creator_user_token.as_ref())
.maybe_creator_anonymous_visitor(inference_job.user_details.maybe_creator_anonymous_visitor_token.as_ref())
.creator_ip_address(&inference_job.user_details.creator_ip_address)
.public_bucket_directory_hash(&bucket_upload_path)
.media_file_class(MediaFileClass::Image)
.media_file_type(media_file_type)
.media_file_origin_category(MediaFileOriginCategory::Inference)
.media_file_origin_product_category(MediaFileOriginProductCategory::ImageStudio)
.mime_type(mimetype)
.file_size_bytes(file_size_bytes as u64)
.frame_width(image_info.width())
.frame_height(image_info.height())
.checksum_sha2(&file_hash)
.insert_pool(mysql_pool)
.await?;
Ok(media_token)
}