From 89b1936fb319d0510ad34ceca95ecba2533abd58 Mon Sep 17 00:00:00 2001 From: salt Date: Fri, 15 Mar 2024 11:07:12 -0700 Subject: [PATCH] mapping code --- .../style_transfer/style_transfer_name.rs | 256 +++++++++++++----- .../workflow_payload.rs | 15 + .../config/inference-job.development.env | 4 + .../comfy_ui/comfy_ui_inference_command.rs | 18 ++ .../workflow/comfy_ui/process_job.rs | 126 ++++++--- .../workflow/comfy_ui/validate_job.rs | 22 +- .../enqueue_video_style_transfer_handler.rs | 14 +- 7 files changed, 334 insertions(+), 121 deletions(-) diff --git a/crates/_types/enums/src/no_table/style_transfer/style_transfer_name.rs b/crates/_types/enums/src/no_table/style_transfer/style_transfer_name.rs index 39d2044e7a..9a21f29a69 100644 --- a/crates/_types/enums/src/no_table/style_transfer/style_transfer_name.rs +++ b/crates/_types/enums/src/no_table/style_transfer/style_transfer_name.rs @@ -14,12 +14,38 @@ use utoipa::ToSchema; #[derive(Clone, Copy, Eq, PartialEq, Ord, PartialOrd, Hash, Deserialize, Serialize, ToSchema)] #[serde(rename_all = "snake_case")] pub enum StyleTransferName { - #[serde(rename = "anime_2d")] - Anime2d, + #[serde(rename = "anime_2_5d")] + Anime2_5D, + #[serde(rename = "anime_2d_flat")] + Anime2DFlat, #[serde(rename = "cartoon_3d")] - Cartoon3d, - Ink, - Origami, + Cartoon3D, + #[serde(rename = "comic_book")] + ComicBook, + #[serde(rename = "anime_ghibli")] + AnimeGhibli, + #[serde(rename = "ink_punk")] + InkPunk, + #[serde(rename = "ink_splash")] + InkSplash, + #[serde(rename = "ink_bw_style")] + InkBWStyle, + #[serde(rename = "jojo_style")] + JojoStyle, + #[serde(rename = "paper_origami")] + PaperOrigami, + #[serde(rename = "pixel_art")] + PixelArt, + #[serde(rename = "pop_art")] + PopArt, + #[serde(rename = "realistic_1")] + Realistic1, + #[serde(rename = "realistic_2")] + Realistic2, + #[serde(rename = "anime_retro_neon")] + AnimeRetroNeon, + #[serde(rename = "anime_standard")] + AnimeStandard } // TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! @@ -31,20 +57,65 @@ impl_enum_display_and_debug_using_to_str!(StyleTransferName); impl StyleTransferName { pub fn to_str(&self) -> &'static str { match self { - Self::Anime2d => "anime_2d", - Self::Cartoon3d => "cartoon_3d", - Self::Ink => "ink", - Self::Origami => "origami", + Self::Anime2_5D => "anime_2_5d", + Self::Anime2DFlat => "anime_2d_flat", + Self::Cartoon3D => "cartoon_3d", + Self::ComicBook => "comic_book", + Self::AnimeGhibli => "anime_ghibli", + Self::InkPunk => "ink_punk", + Self::InkSplash => "ink_splash", + Self::InkBWStyle => "ink_bw_style", + Self::JojoStyle => "jojo_style", + Self::PaperOrigami => "paper_origami", + Self::PixelArt => "pixel_art", + Self::PopArt => "pop_art", + Self::Realistic1 => "realistic_1", + Self::Realistic2 => "realistic_2", + Self::AnimeRetroNeon => "anime_retro_neon", + Self::AnimeStandard => "anime_standard", } } pub fn from_str(value: &str) -> Result { match value { - "anime_2d" => Ok(Self::Anime2d), - "cartoon_3d" => Ok(Self::Cartoon3d), - "ink" => Ok(Self::Ink), - "origami" => Ok(Self::Origami), - _ => Err(format!("invalid value: {:?}", value)), + "anime_2_5d" => Ok(Self::Anime2_5D), + "anime_2d_flat" => Ok(Self::Anime2DFlat), + "cartoon_3d" => Ok(Self::Cartoon3D), + "comic_book" => Ok(Self::ComicBook), + "anime_ghibli" => Ok(Self::AnimeGhibli), + "ink_punk" => Ok(Self::InkPunk), + "ink_splash" => Ok(Self::InkSplash), + "ink_bw_style" => Ok(Self::InkBWStyle), + "jojo_style" => Ok(Self::JojoStyle), + "paper_origami" => Ok(Self::PaperOrigami), + "pixel_art" => Ok(Self::PixelArt), + "pop_art" => Ok(Self::PopArt), + "realistic_1" => Ok(Self::Realistic1), + "realistic_2" => Ok(Self::Realistic2), + "anime_retro_neon" => Ok(Self::AnimeRetroNeon), + "anime_standard" => Ok(Self::AnimeStandard), + _ => Err(format!("Unknown StyleTransferName: {}", value)), + } + } + + pub fn to_filename(&self) -> &'static str { + match self { + Self::Anime2_5D => "1_2.5d_anime_model.json", + Self::Anime2DFlat => "2_2d_flat_anime_model.json", + Self::Cartoon3D => "3_3d_cartoon_style.json", + Self::ComicBook => "4_comic_book_model.json", + Self::AnimeGhibli => "5_ghibli_anime_model.json", + Self::InkPunk => "6_ink_punk.json", + Self::InkSplash => "7_ink_splash.json", + Self::InkBWStyle => "8_ink_w_and_b_style.json", + Self::JojoStyle => "9_jojo_style.json", + Self::PaperOrigami => "10_paper_origami.json", + Self::PixelArt => "11_pixel_art.json", + Self::PopArt => "12_pop_art.json", + Self::Realistic1 => "13_realistic_1.json", + Self::Realistic2 => "14_realistic_2.json", + Self::AnimeRetroNeon => "15_retro_neon_anime_90.json", + Self::AnimeStandard => "16_standard_anime_model.json", } } @@ -52,10 +123,22 @@ impl StyleTransferName { // NB: BTreeSet is sorted // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. BTreeSet::from([ - Self::Anime2d, - Self::Cartoon3d, - Self::Ink, - Self::Origami, + Self::Anime2_5D, + Self::Anime2DFlat, + Self::Cartoon3D, + Self::ComicBook, + Self::AnimeGhibli, + Self::InkPunk, + Self::InkSplash, + Self::InkBWStyle, + Self::JojoStyle, + Self::PaperOrigami, + Self::PixelArt, + Self::PopArt, + Self::Realistic1, + Self::Realistic2, + Self::AnimeRetroNeon, + Self::AnimeStandard, ]) } } @@ -70,65 +153,96 @@ mod tests { #[test] fn test_serialization() { - assert_serialization(StyleTransferName::Anime2d, "anime_2d"); - assert_serialization(StyleTransferName::Cartoon3d, "cartoon_3d"); - assert_serialization(StyleTransferName::Ink, "ink"); - assert_serialization(StyleTransferName::Origami, "origami"); - } - } - - mod impl_methods { - use super::*; - - #[test] - fn to_str() { - assert_eq!(StyleTransferName::Anime2d.to_str(), "anime_2d"); - assert_eq!(StyleTransferName::Cartoon3d.to_str(), "cartoon_3d"); - assert_eq!(StyleTransferName::Ink.to_str(), "ink"); - assert_eq!(StyleTransferName::Origami.to_str(), "origami"); + assert_serialization(StyleTransferName::Anime2_5D, r#"anime_2_5d"#); + assert_serialization(StyleTransferName::Anime2DFlat, r#"anime_2d_flat"#); + assert_serialization(StyleTransferName::Cartoon3D, r#"cartoon_3d"#); + assert_serialization(StyleTransferName::ComicBook, r#"comic_book"#); + assert_serialization(StyleTransferName::AnimeGhibli, r#"anime_ghibli"#); + assert_serialization(StyleTransferName::InkPunk, r#"ink_punk"#); + assert_serialization(StyleTransferName::InkSplash, r#"ink_splash"#); + assert_serialization(StyleTransferName::InkBWStyle, r#"ink_bw_style"#); + assert_serialization(StyleTransferName::JojoStyle, r#"jojo_style"#); + assert_serialization(StyleTransferName::PaperOrigami, r#"paper_origami"#); + assert_serialization(StyleTransferName::PixelArt, r#"pixel_art"#); + assert_serialization(StyleTransferName::PopArt, r#"pop_art"#); + assert_serialization(StyleTransferName::Realistic1, r#"realistic_1"#); + assert_serialization(StyleTransferName::Realistic2, r#"realistic_2"#); + assert_serialization(StyleTransferName::AnimeRetroNeon, r#"anime_retro_neon"#); + assert_serialization(StyleTransferName::AnimeStandard, r#"anime_standard"#); } - #[test] - fn from_str() { - assert_eq!(StyleTransferName::from_str("anime_2d").unwrap(), StyleTransferName::Anime2d); - assert_eq!(StyleTransferName::from_str("cartoon_3d").unwrap(), StyleTransferName::Cartoon3d); - assert_eq!(StyleTransferName::from_str("ink").unwrap(), StyleTransferName::Ink); - assert_eq!(StyleTransferName::from_str("origami").unwrap(), StyleTransferName::Origami); - assert!(StyleTransferName::from_str("foo").is_err()); - } - } + mod impl_methods { + use super::*; - mod manual_variant_checks { - use super::*; + #[test] + fn to_str() { + assert_eq!(StyleTransferName::Anime2_5D.to_str(), "anime_2_5d"); + assert_eq!(StyleTransferName::Anime2DFlat.to_str(), "anime_2d_flat"); + assert_eq!(StyleTransferName::Cartoon3D.to_str(), "cartoon_3d"); + assert_eq!(StyleTransferName::ComicBook.to_str(), "comic_book"); + assert_eq!(StyleTransferName::AnimeGhibli.to_str(), "anime_ghibli"); + assert_eq!(StyleTransferName::InkPunk.to_str(), "ink_punk"); + assert_eq!(StyleTransferName::InkSplash.to_str(), "ink_splash"); + assert_eq!(StyleTransferName::InkBWStyle.to_str(), "ink_bw_style"); + assert_eq!(StyleTransferName::JojoStyle.to_str(), "jojo_style"); + assert_eq!(StyleTransferName::PaperOrigami.to_str(), "paper_origami"); + assert_eq!(StyleTransferName::PixelArt.to_str(), "pixel_art"); + assert_eq!(StyleTransferName::PopArt.to_str(), "pop_art"); + assert_eq!(StyleTransferName::Realistic1.to_str(), "realistic_1"); + assert_eq!(StyleTransferName::Realistic2.to_str(), "realistic_2"); + assert_eq!(StyleTransferName::AnimeRetroNeon.to_str(), "anime_retro_neon"); + assert_eq!(StyleTransferName::AnimeStandard.to_str(), "anime_standard"); + } - #[test] - fn all_variants() { - let mut variants = StyleTransferName::all_variants(); - assert_eq!(variants.len(), 4); - assert_eq!(variants.pop_first(), Some(StyleTransferName::Anime2d)); - assert_eq!(variants.pop_first(), Some(StyleTransferName::Cartoon3d)); - assert_eq!(variants.pop_first(), Some(StyleTransferName::Ink)); - assert_eq!(variants.pop_first(), Some(StyleTransferName::Origami)); - assert_eq!(variants.pop_first(), None); - } - } - - mod mechanical_checks { - use super::*; - - #[test] - fn variant_length() { - use strum::IntoEnumIterator; - assert_eq!(StyleTransferName::all_variants().len(), StyleTransferName::iter().len()); + #[test] + fn from_str() { + assert_eq!(StyleTransferName::from_str("anime_2_5d").unwrap(), StyleTransferName::Anime2_5D); + assert_eq!(StyleTransferName::from_str("anime_2d_flat").unwrap(), StyleTransferName::Anime2DFlat); + assert_eq!(StyleTransferName::from_str("cartoon_3d").unwrap(), StyleTransferName::Cartoon3D); + assert_eq!(StyleTransferName::from_str("comic_book").unwrap(), StyleTransferName::ComicBook); + assert_eq!(StyleTransferName::from_str("anime_ghibli").unwrap(), StyleTransferName::AnimeGhibli); + assert_eq!(StyleTransferName::from_str("ink_punk").unwrap(), StyleTransferName::InkPunk); + assert_eq!(StyleTransferName::from_str("ink_splash").unwrap(), StyleTransferName::InkSplash); + assert_eq!(StyleTransferName::from_str("ink_bw_style").unwrap(), StyleTransferName::InkBWStyle); + assert_eq!(StyleTransferName::from_str("jojo_style").unwrap(), StyleTransferName::JojoStyle); + assert_eq!(StyleTransferName::from_str("paper_origami").unwrap(), StyleTransferName::PaperOrigami); + assert_eq!(StyleTransferName::from_str("pixel_art").unwrap(), StyleTransferName::PixelArt); + assert_eq!(StyleTransferName::from_str("pop_art").unwrap(), StyleTransferName::PopArt); + assert_eq!(StyleTransferName::from_str("realistic_1").unwrap(), StyleTransferName::Realistic1); + assert_eq!(StyleTransferName::from_str("realistic_2").unwrap(), StyleTransferName::Realistic2); + assert_eq!(StyleTransferName::from_str("anime_retro_neon").unwrap(), StyleTransferName::AnimeRetroNeon); + assert_eq!(StyleTransferName::from_str("anime_standard").unwrap(), StyleTransferName::AnimeStandard); + assert!(StyleTransferName::from_str("foo").is_err()); + } } - #[test] - fn round_trip() { - for variant in StyleTransferName::all_variants() { - assert_eq!(variant, StyleTransferName::from_str(variant.to_str()).unwrap()); - assert_eq!(variant, StyleTransferName::from_str(&format!("{}", variant)).unwrap()); - assert_eq!(variant, StyleTransferName::from_str(&format!("{:?}", variant)).unwrap()); + mod manual_variant_checks { + use super::*; + + #[test] + fn all_variants() { + let variants = StyleTransferName::all_variants(); + assert_eq!(variants.len(), 16); + } + } + + mod mechanical_checks { + use super::*; + + #[test] + fn variant_length() { + use strum::IntoEnumIterator; + assert_eq!(StyleTransferName::all_variants().len(), StyleTransferName::iter().len()); + } + + #[test] + fn round_trip() { + for variant in StyleTransferName::all_variants() { + assert_eq!(variant, StyleTransferName::from_str(variant.to_str()).unwrap()); + assert_eq!(variant, StyleTransferName::from_str(&format!("{}", variant)).unwrap()); + assert_eq!(variant, StyleTransferName::from_str(&format!("{:?}", variant)).unwrap()); + } } } } -} +} \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/workflow_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/workflow_payload.rs index fea00d4421..ade78b212a 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/workflow_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/workflow_payload.rs @@ -22,6 +22,21 @@ impl NewValue { NewValue::Bool(s) => s.to_string(), } } + + pub fn from_json(json: &serde_json::Value) -> Self { + match json { + serde_json::Value::String(s) => NewValue::String(s.to_string()), + serde_json::Value::Number(n) => { + if n.is_f64() { + NewValue::Float(n.as_f64().unwrap() as f32) + } else { + NewValue::Int(n.as_i64().unwrap() as i32) + } + }, + serde_json::Value::Bool(b) => NewValue::Bool(*b), + _ => panic!("Invalid json type for NewValue"), + } + } } #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)] diff --git a/crates/service/job/inference_job/config/inference-job.development.env b/crates/service/job/inference_job/config/inference-job.development.env index 88dac0d340..56e73443a6 100644 --- a/crates/service/job/inference_job/config/inference-job.development.env +++ b/crates/service/job/inference_job/config/inference-job.development.env @@ -155,6 +155,10 @@ COMFY_MOUNTS_DIRECTORY="/home/salt/clone/storyteller/storyteller-ml/workflows/co COMFY_VIDEO_PROCESSING_SCRIPT="/home/salt/clone/storyteller/storyteller-ml/workflows/comfy/format_video.py" COMFY_SETUP_SCRIPT="/home/salt/clone/storyteller/storyteller-ml/workflows/comfy/install.py" +COMFY_STYLES_DIRECTORY="/home/salt/clone/storyteller/V2V-StylesList/V2VWorkflows/styles" +COMFY_WORKFLOWS_DIRECTORY="/home/salt/clone/storyteller/V2V-StylesList/V2VWorkflows/workflows" +COMFY_MAPPINGS_DIRECTORY="/home/salt/clone/storyteller/V2V-StylesList/V2VWorkflows/mappings" + # ================================= # # # # Fbx2Gltf # diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/comfy_ui_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/comfy_ui_inference_command.rs index a0ec7c627f..e61138d5db 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/comfy_ui_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/comfy_ui_inference_command.rs @@ -48,6 +48,12 @@ pub struct ComfyInferenceCommand { pub(crate) comfy_launch_command: PathBuf, + pub(crate) styles_directory: PathBuf, + + pub(crate) workflows_directory: PathBuf, + + pub(crate) mappings_directory: PathBuf, + /// Config file to use config_path: Option, @@ -128,6 +134,15 @@ impl ComfyInferenceCommand { let processing_script = easyenv::get_env_pathbuf_required( "COMFY_VIDEO_PROCESSING_SCRIPT")?; + let styles_directory = easyenv::get_env_pathbuf_required( + "COMFY_STYLES_DIRECTORY")?; + + let workflows_directory = easyenv::get_env_pathbuf_required( + "COMFY_WORKFLOWS_DIRECTORY")?; + + let mappings_directory = easyenv::get_env_pathbuf_required( + "COMFY_MAPPINGS_DIRECTORY")?; + Ok(Self { comfy_root_code_directory, executable_or_command, @@ -139,6 +154,9 @@ impl ComfyInferenceCommand { maybe_virtual_env_activation_command, maybe_docker_options, maybe_execution_timeout, + styles_directory, + workflows_directory, + mappings_directory, }) } diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/process_job.rs b/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/process_job.rs index 9bd4428439..6290d34579 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/process_job.rs @@ -5,9 +5,12 @@ use std::process::Command; use std::time::Instant; use anyhow::{anyhow, Result}; +use hostname::get; use log::{debug, error, info, warn}; +use r2d2_redis::redis::Commands; use serde_json::Value; use tokio::io::AsyncWriteExt; +use tokio::runtime::Handle; use walkdir::WalkDir; use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; @@ -87,6 +90,7 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result) -> Result { + let retrieved_workflow_record = get_weight_by_token( + workflow_token, + false, + &deps.db.mysql_pool + ).await?.ok_or_else(|| ProcessSingleJobError::Other(anyhow!("Workflow not found")))?; - let bucket_details = RemoteCloudBucketDetails { - object_hash: retrieved_workflow_record.public_bucket_hash, - prefix: retrieved_workflow_record.maybe_public_bucket_prefix.unwrap(), - suffix: retrieved_workflow_record.maybe_public_bucket_extension.unwrap(), - }; - remote_cloud_file_client.download_file(bucket_details, workflow_path.clone()).await?; - info!("Downloaded workflow to {:?}", workflow_path); + let bucket_details = RemoteCloudBucketDetails { + object_hash: retrieved_workflow_record.public_bucket_hash, + prefix: retrieved_workflow_record.maybe_public_bucket_prefix.unwrap(), + suffix: retrieved_workflow_record.maybe_public_bucket_extension.unwrap(), + }; + remote_cloud_file_client.download_file(bucket_details, workflow_path.clone()).await?; + info!("Downloaded workflow to {:?}", workflow_path); + } + _ => { } + } - // download_file(comfy_deps.workflow_bucket_path.clone(), PathBuf::from(workflow_path.clone())).await.map_err(|e| ProcessSingleJobError::Other(e))?; let maybe_args = job.maybe_inference_args .as_ref() @@ -167,18 +176,43 @@ pub async fn process_job(args: ComfyProcessJobArgs<'_>) -> Result) -> Result AnyhowResult<()> { - let modifications = match comfy_args.maybe_json_modifications.as_ref() { - None => return Err(anyhow!("no modifications dictionary")), - Some(modifications) => modifications, - }; +fn apply_jsonpath_modifications(modifications: HashMap, workflow_path: &str) -> AnyhowResult<(String)> { info!("Prompt modifications: #{:?}", modifications); @@ -650,17 +680,49 @@ fn apply_manual_modifications_from_override_map(comfy_args: &WorkflowArgs, workf } // Save prompt.json - info!("Saving prompt file: {:?}", workflow_path); - let prompt_file = File::create(workflow_path)?; + let workflow_parent_dir = Path::new(workflow_path).parent().unwrap(); + let prompt_filepath = workflow_parent_dir.join("prompt.json"); + let prompt_file = File::create(&prompt_filepath) + .map_err(|e| anyhow!("error creating prompt file: {:?}", e))?; + info!("Saving prompt file: {:?}", prompt_file); serde_json::to_writer(prompt_file, &prompt_json)?; - Ok(()) + Ok(prompt_filepath.to_str().unwrap().to_string()) } -fn build_prompt_modifications_from_style_name(comfy_args: &WorkflowArgs, workflow_path: &str) -> AnyhowResult<()> { - // TODO: New code - Ok(()) +fn get_style_modifications(style_json: &Value, mapping_json: &Value) -> HashMap { + let mut modifications = HashMap::new(); + let mut new_style_json = style_json.clone(); + + // Loras have to be processed differently + if let Some(loras) = style_json.get("loras").and_then(|l| l.as_array()) { + if loras.len() > 8 { + panic!("Too many loras, max is 8"); + } + + for (index, lora) in loras.iter().enumerate() { + if let (Some(name), Some(strength)) = (lora.get("name"), lora.get("strength")) { + new_style_json[format!("lora_{}_strength", index + 1)] = strength.clone(); + new_style_json[format!("lora_{}_name", index + 1)] = name.clone(); + } + } + } + + for (key, value) in new_style_json.as_object().unwrap() { + if key == "loras" { continue; } + + let mapping_key = format!("$.{}", key); + if let Ok(mapping_values) = jsonpath_lib::select(mapping_json, &mapping_key) { + if let Some(mapping_value) = mapping_values.get(0).and_then(|v| v.as_str()) { + modifications.insert(mapping_value.to_string(), NewValue::from_json(value)); + } else { + println!("No mapping found for key '{}'", key); + } + } + } + + modifications } diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/validate_job.rs b/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/validate_job.rs index accbc0963e..cba7baeb9d 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/validate_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/validate_job.rs @@ -11,8 +11,8 @@ use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; #[derive(Serialize, Debug)] pub struct JobArgs<'a> { - pub workflow_source: &'a ModelWeightToken, - pub output_path: &'a String, + pub workflow_source: &'a Option, + pub output_path: &'a str, pub maybe_json_modifications: &'a Option>, pub maybe_lora_model: &'a Option, pub maybe_input_file: &'a Option, @@ -53,23 +53,23 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result args, - None => { - return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No workflow source provided!"))); - } - }; - let output_path = match &inference_args.maybe_output_path { + // check if job is legacy + let is_legacy = inference_args.maybe_json_modifications.is_some(); + + let output_path = match inference_args.maybe_output_path.as_deref() { Some(args) => args, + None if is_legacy => { + return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No output path provided!"))); + }, None => { - return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No output file provided!"))); + "vid2vid/SparseUpscaleInterp_00001.mp4" } }; Ok(JobArgs { - workflow_source, + workflow_source: &inference_args.maybe_workflow_config, output_path, maybe_lora_model: &inference_args.maybe_lora_model, maybe_json_modifications: &inference_args.maybe_json_modifications, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/workflows/enqueue_video_style_transfer_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/workflows/enqueue_video_style_transfer_handler.rs index ce8c4ed4c8..348a2eeba5 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/workflows/enqueue_video_style_transfer_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/workflows/enqueue_video_style_transfer_handler.rs @@ -12,7 +12,6 @@ use log::{error, info, warn}; use utoipa::ToSchema; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; -use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType; use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; use enums::common::visibility::Visibility; @@ -46,12 +45,12 @@ pub struct EnqueueVideoStyleTransferRequest { /// Entropy for request de-duplication (required) uuid_idempotency_token: String, - /// The video to use as input (required) - input_media_file_token: MediaFileToken, - /// The name of the style to invoke (required) style: StyleTransferName, + // The input file (required) + input_file: MediaFileToken, + /// The positive prompt (optional) prompt: Option, @@ -249,7 +248,6 @@ pub async fn enqueue_video_style_transfer_handler( } let inference_args = WorkflowArgs { - maybe_input_file: Some(request.input_media_file_token.clone()), style_name: Some(request.style), creator_visibility: Some(set_visibility), trim_start_milliseconds: Some(trim_start_millis), @@ -257,10 +255,12 @@ pub async fn enqueue_video_style_transfer_handler( positive_prompt: request.prompt.clone(), negative_prompt: request.negative_prompt.clone(), enable_lipsync: request.enable_lipsync.clone(), + maybe_input_file: Some(request.input_file.clone()), // The new, simplified enqueuing doesn't care about the following parameters: maybe_lora_model: None, maybe_json_modifications: None, maybe_workflow_config: None, + maybe_input_file: None, maybe_output_path: None, maybe_google_drive_link: None, maybe_title: None, @@ -279,8 +279,8 @@ pub async fn enqueue_video_style_transfer_handler( inference_category: InferenceCategory::Workflow, maybe_model_type: Some(InferenceModelType::ComfyUi), // NB: Model is static during inference maybe_model_token: None, // NB: Model is static during inference - maybe_input_source_token: Some(request.input_media_file_token.as_str()), - maybe_input_source_token_type: Some(InferenceInputSourceTokenType::MediaFile), + maybe_input_source_token: None, // TODO: Introduce a second foreign key ? + maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ? maybe_download_url: None, maybe_cover_image_media_file_token: None, maybe_raw_inference_text: None, // No text