From be7315b28d5fc255578ea6305577c1845e0eeb2d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 22 Feb 2024 21:36:43 -0500 Subject: [PATCH] insert prompt records in SD inference job --- .../list_available_generic_inference_jobs.rs | 15 ++++-- .../create/insert_media_file_generic.rs | 10 ++-- .../src/queries/prompts/insert_prompt.rs | 4 +- .../bvh_to_workflow/process_job.rs | 1 + .../fbx_to_gltf/process_job.rs | 1 + .../sd/process_job_inference.rs | 54 ++++++++++++++----- 6 files changed, 64 insertions(+), 21 deletions(-) diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 221e10b170..6199023d16 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -11,6 +11,7 @@ use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::generic_inference_jobs::InferenceJobToken; use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; use crate::helpers::boolean_converters::i8_to_bool; use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs; @@ -42,6 +43,8 @@ pub struct AvailableInferenceJob { // User information to propagate downstream pub maybe_creator_user_token: Option, + /// Same user token, but a `UserToken`. + pub maybe_creator_user_token_typed: Option, pub maybe_creator_anonymous_visitor_token: Option, pub creator_ip_address: String, pub creator_set_visibility: Visibility, @@ -102,12 +105,17 @@ pub async fn list_available_generic_inference_jobs( let job_records : Vec = job_records.into_iter() .map(|record : AvailableInferenceJobRawInternal| { - let record = AvailableInferenceJob { + let maybe_creator_user_token_typed = record.maybe_creator_user_token + .as_ref() + .map(|s| UserToken::new_from_str(s)); + + Ok(AvailableInferenceJob { id: GenericInferenceJobId(record.id), inference_job_token: InferenceJobToken::new(record.inference_job_token), uuid_idempotency_token: record.uuid_idempotency_token, creator_ip_address: record.creator_ip_address, - maybe_creator_user_token: record.maybe_creator_user_token, + maybe_creator_user_token: record.maybe_creator_user_token.clone(), + maybe_creator_user_token_typed, maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token, maybe_cover_image_media_file_token: record.maybe_cover_image_media_file_token .map(|s| MediaFileToken::new_from_str(&s)), @@ -143,8 +151,7 @@ pub async fn list_available_generic_inference_jobs( updated_at: record.updated_at, retry_at: record.retry_at, database_clock: record.database_clock, - }; - Ok(record) + }) }) // NB: Magic Vec -> Result> // https://stackoverflow.com/a/63798748 diff --git a/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_generic.rs b/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_generic.rs index 36bb1f6b9a..de1d15ce70 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_generic.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/create/insert_media_file_generic.rs @@ -11,6 +11,7 @@ use errors::AnyhowResult; use tokens::tokens::batch_generations::BatchGenerationToken; use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::prompts::PromptToken; use tokens::tokens::users::UserToken; use crate::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; @@ -41,6 +42,7 @@ pub struct InsertArgs<'a> { pub maybe_video_encoding: Option<&'a str>, pub maybe_frame_width: Option, pub maybe_frame_height: Option, + pub maybe_prompt_token: Option<&'a PromptToken>, pub checksum_sha2: &'a str, // Storage details @@ -114,7 +116,8 @@ pub async fn insert_media_file_generic( maybe_audio_encoding = ?, maybe_video_encoding = ?, maybe_frame_width = ?, - maybe_frame_height = ?, + maybe_frame_height = ?, + maybe_prompt_token = ?, checksum_sha2 = ?, public_bucket_directory_hash = ?, @@ -146,7 +149,7 @@ pub async fn insert_media_file_generic( args.maybe_origin_filename, is_batch_generated, - args.maybe_batch_token.map(|e| e.to_string()), + args.maybe_batch_token.map(|e| e.as_str()), args.maybe_mime_type, args.file_size_bytes, @@ -154,7 +157,8 @@ pub async fn insert_media_file_generic( args.maybe_audio_encoding, args.maybe_video_encoding, args.maybe_frame_width, - args.maybe_frame_height, + args.maybe_frame_height, + args.maybe_prompt_token.map(|e| e.as_str()), args.checksum_sha2, args.public_bucket_directory_hash, diff --git a/crates/lib/mysql_queries/src/queries/prompts/insert_prompt.rs b/crates/lib/mysql_queries/src/queries/prompts/insert_prompt.rs index 3ca887a922..918d3db0ba 100644 --- a/crates/lib/mysql_queries/src/queries/prompts/insert_prompt.rs +++ b/crates/lib/mysql_queries/src/queries/prompts/insert_prompt.rs @@ -8,7 +8,7 @@ use errors::AnyhowResult; use tokens::tokens::prompts::PromptToken; use tokens::tokens::users::UserToken; -pub struct InsertArgs<'e, 'c, E> +pub struct InsertPromptArgs<'e, 'c, E> where E: 'e + Executor<'c, Database = MySql> { /// If we need to generate the prompt token upfront, this will be the token to use for the insert. @@ -34,7 +34,7 @@ pub struct InsertArgs<'e, 'c, E> pub phantom: PhantomData<&'c E>, } -pub async fn insert_prompt<'e, 'c : 'e, E>(args: InsertArgs<'e, 'c, E>) +pub async fn insert_prompt<'e, 'c : 'e, E>(args: InsertPromptArgs<'e, 'c, E>) -> AnyhowResult where E: 'e + Executor<'c, Database = MySql> { diff --git a/crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/process_job.rs b/crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/process_job.rs index 7c44972c3e..0296b55fa2 100644 --- a/crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/process_job.rs @@ -246,6 +246,7 @@ pub async fn process_job(args: BvhToWorkflowJobArgs<'_>) -> Result) -> Result return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model token for job!"))), @@ -81,7 +83,7 @@ pub async fn process_job_inference( let sd_model_weight = get_weight_by_token( sd_model_weight_token, false, - &deps.db.mysql_pool + &deps.db.mysql_pool, ).await?; let sd_model_weight = match sd_model_weight { @@ -97,7 +99,8 @@ pub async fn process_job_inference( info!("sd_checkpoint_path: {:?}", sd_checkpoint_path); - // origin file name needs to be just the file name /tmp/downloads_long_lived/temp_stable_diffusion_inference_32.8qJJljxWWZeD/output_0.png + // origin file name needs to be just the file name + // /tmp/downloads_long_lived/temp_stable_diffusion_inference_32.8qJJljxWWZeD/output_0.png // ignore if no lora token let mut maybe_lora_record = None; @@ -106,7 +109,7 @@ pub async fn process_job_inference( maybe_lora_record = get_weight_by_token( &token, false, - &deps.db.mysql_pool + &deps.db.mysql_pool, ).await?; } @@ -151,10 +154,10 @@ pub async fn process_job_inference( let stderr_output_file = work_temp_dir.path().join("sd_err.txt"); let stdout_output_file = work_temp_dir.path().join("sd_out.txt"); - let number_of_samples = match sd_args.maybe_number_of_samples { - Some(val) => val, - None => 20, - }; + let number_of_samples = sd_args.maybe_number_of_samples.unwrap_or(20); + + let positive_prompt = prompt.clone(); + let maybe_negative_prompt = sd_args.maybe_n_prompt.clone(); let inference_start_time = Instant::now(); @@ -163,8 +166,8 @@ pub async fn process_job_inference( output_file: output_path.clone(), stderr_output_file: &stderr_output_file, stdout_output_file: &stdout_output_file, - prompt: prompt.clone(), - negative_prompt: sd_args.maybe_n_prompt.clone().unwrap_or_default(), + prompt: positive_prompt.clone(), + negative_prompt: maybe_negative_prompt.clone().unwrap_or_default(), number_of_samples, samplers: sd_args.maybe_sampler.clone().unwrap_or(String::from("Euler a")), width: sd_args.maybe_width.unwrap_or(512), @@ -232,6 +235,7 @@ pub async fn process_job_inference( }; let batch_token = BatchGenerationToken::generate(); + let prompt_token = PromptToken::generate(); let mut maybe_first_media_file_token = None; @@ -244,7 +248,7 @@ pub async fn process_job_inference( let metadata = remote_cloud_file_client.upload_file( Box::new(MediaImagePngDescriptor {}), - file_path.as_ref() + file_path.as_ref(), ).await?; let bucket_details = match metadata.bucket_details { @@ -267,6 +271,7 @@ pub async fn process_job_inference( maybe_origin_model_token: Some(sd_model_weight_token.clone()), maybe_origin_filename: Some(file_path), maybe_batch_token: Some(&batch_token), + maybe_prompt_token: Some(&prompt_token), maybe_mime_type: Some(metadata.mimetype.as_ref()), file_size_bytes: metadata.file_size_bytes, maybe_duration_millis: Some(inference_duration.as_millis() as u64), @@ -298,7 +303,11 @@ pub async fn process_job_inference( entries.push(batch_generation_entity); } - let mut transaction = mysql_pool.begin().await.unwrap(); + // TODO(bt,2024-02-22): This transaction should wrap everything + let mut transaction = mysql_pool + .begin() + .await + .map_err(|e| ProcessSingleJobError::from_anyhow_error(anyhow!(e)))?; let batch_token_result = insert_batch_generation_records(InsertBatchArgs { entries, @@ -317,6 +326,27 @@ pub async fn process_job_inference( } }; + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: Some(&prompt_token), + prompt_type: PromptType::StableDiffusion, + maybe_creator_user_token: job.maybe_creator_user_token_typed.as_ref(), + maybe_positive_prompt: Some(&positive_prompt), + maybe_negative_prompt: maybe_negative_prompt.as_deref(), + maybe_other_args: None, // TODO(bt,2024-02-22): Support other arguments + creator_ip_address: &job.creator_ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + match prompt_result { + Ok(_token) => {} + Err(err) => { + error!("No prompt result token? something has failed: {:?}", err); + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No prompt result token? something has failed."))); + } + } + // TODO(bt,2024-02-12): Return the batch token instead. (We're not ready for that.) Ok(JobSuccessResult { inference_duration,