engine render job - process camera + speed args

crates/service/job/inference_job/src/job/job_loop/process_single_job.rs

@@ -14,11 +14,11 @@ use redis_common::redis_keys::RedisKeys;
 use crate::job::job_loop::determine_dependency_status::determine_dependency_status;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job::job_loop::process_single_job_success_case::ProcessSingleJobSuccessCase;
-use crate::job::job_types::bevy_to_workflow::process_single_bevy_to_workflow_conversion_job::process_single_bevy_to_workflow_conversion_job;
 use crate::job::job_types::format_conversion::process_single_format_conversion_job::process_single_format_conversion_job;
 use crate::job::job_types::image_generation::process_single_ig_job::process_single_ig_job;
 use crate::job::job_types::lipsync::process_single_lipsync_job::process_single_lipsync_job;
 use crate::job::job_types::mocap::process_single_mc_job::process_single_mc_job;
+use crate::job::job_types::render_engine_scene::process_render_engine_scene::process_single_render_engine_scene_job;
 use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job;
 use crate::job::job_types::vc::process_single_vc_job::process_single_vc_job;
 use crate::job::job_types::videofilter::process_single_vf_job::process_single_vf_job;
@@ -205,7 +205,7 @@ async fn do_process_single_job(
       process_single_format_conversion_job(job_dependencies, job).await?
     }
     InferenceCategory::ConvertBvhToWorkflow => {
-      process_single_bevy_to_workflow_conversion_job(job_dependencies, job).await?
+      process_single_render_engine_scene_job(job_dependencies, job).await?
     }
   };
 

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/command_args.rs

@@ -1,23 +0,0 @@
-use std::path::Path;
-
-use filesys::path_to_string::path_to_string;
-use subprocess_common::command_runner::command_args::CommandArgs;
-
-pub struct BvhToWorkflowCommandArgs<'a> {
-    pub input_file: &'a Path,
-    pub output_directory: &'a Path,
-}
-
-impl CommandArgs for BvhToWorkflowCommandArgs<'_> {
-    fn to_command_string(&self) -> String {
-        let mut command = String::new();
-
-        command.push_str(" -i ");
-        command.push_str(&path_to_string(self.input_file));
-
-        command.push_str(" -o ");
-        command.push_str(&path_to_string(self.output_directory));
-
-        command
-    }
-}

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/bvh_to_workflow/mod.rs

@@ -1,3 +0,0 @@
-pub mod process_job;
-pub mod command_args;
-pub mod dependencies;

crates/service/job/inference_job/src/job/job_types/bevy_to_workflow/mod.rs

@@ -1,2 +0,0 @@
-pub mod bvh_to_workflow;
-pub mod process_single_bevy_to_workflow_conversion_job;

crates/service/job/inference_job/src/job/job_types/mod.rs

@@ -1,9 +1,9 @@
+pub mod format_conversion;
+pub mod image_generation;
 pub mod lipsync;
+pub mod mocap;
+pub mod render_engine_scene;
 pub mod tts;
 pub mod vc;
-pub(crate) mod videofilter;
-pub mod image_generation;
-pub(crate) mod mocap;
-pub(crate) mod workflow;
-pub mod format_conversion;
-pub mod bevy_to_workflow;
+pub mod videofilter;
+pub mod workflow;

crates/service/job/inference_job/src/job/job_types/render_engine_scene/mod.rs

@@ -0,0 +1,2 @@
+pub mod render_engine_scene_to_video;
+pub mod process_render_engine_scene;

crates/service/job/inference_job/src/job/job_types/render_engine_scene/process_render_engine_scene.rs

@@ -7,12 +7,11 @@ use tokens::tokens::media_files::MediaFileToken;
 
 use crate::job::job_loop::job_success_result::JobSuccessResult;
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
-use crate::job::job_types::bevy_to_workflow::bvh_to_workflow;
-use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::process_job::BvhToWorkflowJobArgs;
+use crate::job::job_types::render_engine_scene::{process_render_engine_scene, render_engine_scene_to_video};
+use crate::job::job_types::render_engine_scene::render_engine_scene_to_video::process_job::BvhToWorkflowJobArgs;
 use crate::job_dependencies::JobDependencies;
 
-
-pub async fn process_single_bevy_to_workflow_conversion_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
+pub async fn process_single_render_engine_scene_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
 
   let maybe_media_file_token = job.maybe_input_source_token
       .as_deref()
@@ -38,7 +37,7 @@ pub async fn process_single_bevy_to_workflow_conversion_job(job_dependencies: &J
     }
   };
 
-  let job_success_result = bvh_to_workflow::process_job::process_job(BvhToWorkflowJobArgs {
+  let job_success_result = render_engine_scene_to_video::process_job::process_job(BvhToWorkflowJobArgs {
         job_dependencies,
         job,
         media_file: &media_file,

crates/service/job/inference_job/src/job/job_types/render_engine_scene/render_engine_scene_to_video/command_args.rs

@@ -0,0 +1,37 @@
+use std::path::Path;
+
+use filesys::path_to_string::path_to_string;
+use subprocess_common::command_runner::command_args::CommandArgs;
+
+pub struct RenderEngineSceneToVideoCommandArgs<'a> {
+    pub input_file: &'a Path,
+    pub output_directory: &'a Path,
+    pub maybe_camera: Option<&'a str>,
+    pub maybe_camera_speed: Option<f32>,
+}
+
+impl CommandArgs for RenderEngineSceneToVideoCommandArgs<'_> {
+    fn to_command_string(&self) -> String {
+        let mut command = String::new();
+
+        command.push_str(" -i ");
+        command.push_str(&path_to_string(self.input_file));
+
+        command.push_str(" -o ");
+        command.push_str(&path_to_string(self.output_directory));
+
+        if let Some(camera) = self.maybe_camera {
+            command.push_str(" -c ");
+            command.push_str(camera);
+            command.push_str(" ");
+        }
+
+        if let Some(camera_speed) = self.maybe_camera_speed {
+            command.push_str(" -s ");
+            command.push_str(&camera_speed.to_string());
+            command.push_str(" ");
+        }
+
+        command
+    }
+}

crates/service/job/inference_job/src/job/job_types/render_engine_scene/render_engine_scene_to_video/dependencies.rs

@@ -6,11 +6,11 @@ use subprocess_common::command_runner::env_var_policy::EnvVarPolicy;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerOptions};
 use subprocess_common::executable_or_command::ExecutableOrShellCommand;
 
-pub struct BvhToWorkflowDependencies {
+pub struct RenderEngineSceneToVideoDependencies {
     pub command_runner: CommandRunner,
 }
 
-impl BvhToWorkflowDependencies {
+impl RenderEngineSceneToVideoDependencies {
     pub fn setup() -> AnyhowResult<Self> {
         let maybe_execution_directory = easyenv::get_env_pathbuf_optional("BVH2WORKFLOW_DIRECTORY");
 

crates/service/job/inference_job/src/job/job_types/render_engine_scene/render_engine_scene_to_video/mod.rs

@@ -0,0 +1,3 @@
+pub mod command_args;
+pub mod dependencies;
+pub mod process_job;

crates/service/job/inference_job/src/job/job_types/render_engine_scene/render_engine_scene_to_video/process_job.rs

@@ -16,6 +16,8 @@ use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
 use filesys::safe_delete_temp_file::safe_delete_temp_file;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, PolymorphicInferenceArgs};
+use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Cu;
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::media_files::create::insert_media_file_generic::{insert_media_file_generic, InsertArgs};
 use mysql_queries::queries::media_files::get_media_file_for_inference::MediaFileForInference;
@@ -23,7 +25,7 @@ use subprocess_common::command_runner::command_runner_args::{FileOrCreate, RunAs
 
 use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
-use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::command_args::BvhToWorkflowCommandArgs;
+use crate::job::job_types::render_engine_scene::render_engine_scene_to_video::command_args::RenderEngineSceneToVideoCommandArgs;
 use crate::job_dependencies::JobDependencies;
 use crate::util::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
 
@@ -101,11 +103,28 @@ pub async fn process_job(args: BvhToWorkflowJobArgs<'_>) -> Result<JobSuccessRes
     original_media_file_fs_path
   };
 
-  // ==================== SETUP FOR CONVERSION ==================== //
+  // ==================== SETUP FOR CONVERSION / HANDLE ARGS ==================== //
 
   job_progress_reporter.log_status("running conversion")
       .map_err(|e| ProcessSingleJobError::Other(e))?;
 
+  let maybe_args = job.maybe_inference_args
+      .as_ref()
+      .map(|args| args.args.as_ref())
+      .flatten()
+      .map(|args| match args {
+        PolymorphicInferenceArgs::Es(args) => Some(args),
+        _ => None,
+      })
+      .flatten();
+
+  let mut maybe_camera = None;
+  let mut maybe_camera_speed = None;
+
+  if let Some(engine_args) = maybe_args {
+    maybe_camera = engine_args.camera_animation.clone();
+    maybe_camera_speed = engine_args.camera_speed.clone();
+  }
 
   // ==================== RUN INFERENCE SCRIPT ==================== //
 
@@ -119,9 +138,11 @@ pub async fn process_job(args: BvhToWorkflowJobArgs<'_>) -> Result<JobSuccessRes
     model_dependencies
         .command_runner
         .run_with_subprocess(RunAsSubprocessArgs {
-          args: Box::new(&BvhToWorkflowCommandArgs {
+          args: Box::new(&RenderEngineSceneToVideoCommandArgs {
             input_file: &original_media_upload_fs_path,
             output_directory: &output_directory,
+            maybe_camera: maybe_camera.as_deref(),
+            maybe_camera_speed,
           }),
           //maybe_stderr_output_file: Some(FileOrCreate::NewFileWithName(&stderr_output_file)),
           // NB(bt,2024-02-29): Bevy's stdout goes to stderr, so we can't capture the semantics we want

crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/process_job.rs

@@ -263,6 +263,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
         PolymorphicInferenceArgs::Ig(_) => None,
         PolymorphicInferenceArgs::Mc(_) => None,
         PolymorphicInferenceArgs::Cu(_) => None,
+        PolymorphicInferenceArgs::Es(_) => None,
       })
       .flatten();
 
@@ -275,6 +276,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
         PolymorphicInferenceArgs::Ig(_) => None,
         PolymorphicInferenceArgs::Mc(_) => None,
         PolymorphicInferenceArgs::Cu(_) => None,
+        PolymorphicInferenceArgs::Es(_) => None,
       })
       .flatten();
 

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/process_job.rs

@@ -183,6 +183,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
         PolymorphicInferenceArgs::Ig(_) => None,
         PolymorphicInferenceArgs::Mc(_) => None,
         PolymorphicInferenceArgs::Cu(_) => None,
+        PolymorphicInferenceArgs::Es(_) => None,
       })
       .flatten()
       .unwrap_or(false);
@@ -196,6 +197,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
         PolymorphicInferenceArgs::Ig(_) => None,
         PolymorphicInferenceArgs::Mc(_) => None,
         PolymorphicInferenceArgs::Cu(_) => None,
+        PolymorphicInferenceArgs::Es(_) => None,
       })
       .flatten();
 

crates/service/job/inference_job/src/job_specific_dependencies.rs

@@ -1,11 +1,11 @@
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use errors::AnyhowResult;
-use crate::job::job_types::bevy_to_workflow::bvh_to_workflow::dependencies::BvhToWorkflowDependencies;
 
 use crate::job::job_types::format_conversion::fbx_to_gltf::dependencies::FbxToGltfDependencies;
 use crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies;
 use crate::job::job_types::lipsync::sad_talker::sad_talker_dependencies::SadTalkerDependencies;
 use crate::job::job_types::mocap::mocap_net::mocapnet_dependencies::MocapNetDependencies;
+use crate::job::job_types::render_engine_scene::render_engine_scene_to_video::dependencies::RenderEngineSceneToVideoDependencies;
 use crate::job::job_types::tts::styletts2::styletts2_dependencies::StyleTTS2Dependencies;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies;
 use crate::job::job_types::tts::vall_e_x::vall_e_x_dependencies::VallExDependencies;
@@ -29,7 +29,7 @@ pub struct JobSpecificDependencies {
   pub maybe_styletts2_dependencies: Option<StyleTTS2Dependencies>,
   pub maybe_comfy_ui_dependencies: Option<ComfyDependencies>,
   pub maybe_convert_fbx_to_gltf_dependencies: Option<FbxToGltfDependencies>,
-  pub maybe_convert_bvh_to_workflow_dependencies: Option<BvhToWorkflowDependencies>,
+  pub maybe_convert_bvh_to_workflow_dependencies: Option<RenderEngineSceneToVideoDependencies>,
 }
 
 impl JobSpecificDependencies {
@@ -111,7 +111,7 @@ impl JobSpecificDependencies {
 
     if scoped_execution.can_run_job(InferenceModelType::BvhToWorkflow) {
       print_with_space("Setting ConvertBvhToWorkflow dependencies...");
-      maybe_convert_bvh_to_workflow_dependencies = Some(BvhToWorkflowDependencies::setup()?);
+      maybe_convert_bvh_to_workflow_dependencies = Some(RenderEngineSceneToVideoDependencies::setup()?);
     }
 
     Ok(JobSpecificDependencies {

crates/service/web/storyteller_web/src/http_server/endpoints/conversion/enqueue_render_engine_scene_to_video_handler.rs

@@ -13,6 +13,7 @@ use serde::Serialize;
 use utoipa::ToSchema;
 
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
+use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType;
 use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType;
 use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
 use enums::common::visibility::Visibility;
@@ -243,7 +244,7 @@ pub async fn enqueue_render_engine_scene_to_video_handler(
         maybe_model_type: Some(InferenceModelType::BvhToWorkflow),
         maybe_model_token: None,
         maybe_input_source_token: Some(&request.media_file_token.as_str()),
-        maybe_input_source_token_type: None,
+        maybe_input_source_token_type: Some(InferenceInputSourceTokenType::MediaFile),
         maybe_download_url: None,
         maybe_cover_image_media_file_token: None,
         maybe_raw_inference_text: None,