Merge pull request #292 from storytold/bt-fbx-to-gltf

fbx to gltf (wip)

Cargo.lock

@@ -6210,7 +6210,11 @@ dependencies = [
 name = "subprocess_common"
 version = "0.0.1"
 dependencies = [
+ "errors",
+ "filesys",
+ "log",
  "my-workspace-hack",
+ "subprocess",
 ]
 
 [[package]]

crates/lib/subprocess_common/Cargo.toml

@@ -14,9 +14,16 @@ path = "src/lib.rs"
 
 [dependencies]
 
-# External
+# Internal
+errors = { path = "../errors" }
+filesys = { path = "../files/filesys" }
+
 my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }
 
+# External
+log = "0.4.14"
+subprocess = "0.2.9"
+
 # Serialization
 #serde = "1.0.125"
 #serde_derive = "1.0.125"

crates/lib/subprocess_common/src/command_adapter/command_adapter.rs

@@ -0,0 +1,166 @@
+use std::env;
+use std::ffi::OsString;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use log::info;
+use subprocess::{Popen, PopenConfig, Redirection};
+
+use errors::AnyhowResult;
+use filesys::path_to_string::path_to_string;
+
+use crate::command_exit_status::CommandExitStatus;
+use crate::docker_options::DockerOptions;
+use crate::executable_or_command::ExecutableOrShellCommand;
+
+#[derive(Clone)]
+pub struct CommandAdapter {
+  /// The script, executable, or command to run.
+  pub executable_or_command: ExecutableOrShellCommand,
+
+  /// The directory to change to before running the command.
+  /// The parent process won't change directory here - only the subprocess will.
+  pub maybe_execution_directory: Option<PathBuf>,
+
+  /// If the command needs to run with a Python virtual env, this can be run.
+  /// eg. `source python/bin/activate`
+  pub maybe_virtual_env_activation_command: Option<String>,
+
+  /// If this is run under Docker (eg. in development), these are the options.
+  /// Typically, this is not done in production.
+  pub maybe_docker_options: Option<DockerOptions>,
+
+  /// If the execution should be ended after a certain point.
+  pub maybe_execution_timeout: Option<Duration>,
+}
+
+pub struct RunAsSubprocessArgs<'a> {
+  /// If set, write stderr to this file.
+  /// The file must not already exist and must not already be open.
+  pub maybe_stderr_output_file: Option<&'a Path>,
+}
+
+impl CommandAdapter {
+
+  /// Run the command with the `subprocess` crate utilities
+  pub fn run_with_subprocess(&self, args: RunAsSubprocessArgs<'_>) -> AnyhowResult<CommandExitStatus> {
+
+    let mut command = String::new();
+
+    if let Some(execution_directory) = self.maybe_execution_directory.as_deref() {
+      command.push_str("cd ");
+      command.push_str(&path_to_string(execution_directory));
+      command.push_str(" && ");
+    }
+
+    if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() {
+      command.push_str(venv_command);
+      command.push_str(" && ");
+    }
+
+    match self.executable_or_command {
+      ExecutableOrShellCommand::Executable(ref executable) => {
+        command.push_str(&path_to_string(executable));
+        command.push_str(" ");
+      }
+      ExecutableOrShellCommand::ShShellCommand(ref cmd) => {
+        command.push_str(cmd);
+        command.push_str(" ");
+      }
+      ExecutableOrShellCommand::BashShellCommand(ref cmd) => {
+        command.push_str(cmd);
+        command.push_str(" ");
+      }
+    }
+
+    // TODO: Command arguments
+
+    if let Some(docker_options) = self.maybe_docker_options.as_ref() {
+      command = docker_options.to_command_string(&command);
+    }
+
+    info!("Command: {:?}", command);
+
+    let command_parts = [
+      "bash",
+      "-c",
+      &command
+    ];
+
+    let mut env_vars = Vec::new();
+
+    // Copy all environment variables from the parent process.
+    // This is necessary to send all the kubernetes settings for Nvidia / CUDA.
+    for (env_key, env_value) in env::vars() {
+      // TODO(bt,2024-01-27): Env var sanitization
+      //if IGNORED_ENVIRONMENT_VARS.contains(&env_key) {
+      //  continue;
+      //}
+      env_vars.push((
+        OsString::from(env_key),
+        OsString::from(env_value),
+      ));
+    }
+
+    let mut config = PopenConfig::default();
+
+    if !env_vars.is_empty() {
+      config.env = Some(env_vars);
+    }
+
+    if let Some(stderr_output_file) = args.maybe_stderr_output_file {
+      info!("stderr will be written to file: {:?}", stderr_output_file);
+
+      let stderr_file = File::create(stderr_output_file)?;
+      config.stderr = Redirection::File(stderr_file);
+    }
+
+    let mut popen_handle = Popen::create(&command_parts, config)?;
+
+    info!("Subprocess PID: {:?}", popen_handle.pid());
+
+    let maybe_exit_status = match self.maybe_execution_timeout {
+      None => {
+        let exit_status = popen_handle.wait()?;
+        info!("Subprocess exit status: {:?}", exit_status);
+        return Ok(CommandExitStatus::from_exit_status(exit_status));
+      }
+      Some(timeout) => {
+        info!("Executing with timeout: {:?}", &timeout);
+        popen_handle.wait_timeout(timeout.clone())?
+      }
+    };
+
+    //let mut popen = Popen::create(
+    //  &["echo", "hello"],
+    //  PopenConfig {
+    //    stdout: Redirection::Pipe,
+    //    ..Default::default()
+    //  }
+    //)?;
+
+    //let stdout = popen.stdout.take().unwrap();
+    //let mut stdout_reader = BufReader::new(stdout);
+
+    //let mut output = String::new();
+    //stdout_reader.read_to_string(&mut output)?;
+
+    //let status = popen.wait()?;
+    //println!("status: {:?}", status);
+    //println!("output: {:?}", output);
+
+    match maybe_exit_status {
+      None => {
+        // NB: If the program didn't successfully terminate, kill it.
+        info!("Subprocess didn't end after timeout; terminating...");
+        let _r = popen_handle.terminate()?;
+        Ok(CommandExitStatus::Timeout)
+      }
+      Some(exit_status) => {
+        info!("Subprocess timed wait exit status: {:?}", exit_status);
+        Ok(CommandExitStatus::from_exit_status(exit_status))
+      }
+    }
+  }
+}

crates/lib/subprocess_common/src/command_adapter/mod.rs

@@ -0,0 +1,2 @@
+pub mod command_adapter;
+mod command_adapter_args;

crates/lib/subprocess_common/src/command_exit_status.rs

@@ -46,7 +46,7 @@ impl CommandExitStatus {
 mod tests {
   use subprocess::ExitStatus;
 
-  use crate::job::job_loop::command_exit_status::CommandExitStatus;
+  use crate::command_exit_status::CommandExitStatus;
 
   #[test]
   fn is_success() {

crates/lib/subprocess_common/src/executable_or_command.rs

@@ -0,0 +1,17 @@
+use std::path::PathBuf;
+
+/// A single executable script or a string to run as a shell command.
+#[derive(Clone)]
+pub enum ExecutableOrShellCommand {
+  /// A system path to an executable.
+  /// Eg. `inference.py`
+  Executable(PathBuf),
+
+  /// A string to run as `sh` shell command.
+  /// Eg. `python3 inference.py`
+  ShShellCommand(String),
+
+  /// A string to run as a `bash` shell command.
+  /// Eg. `python3 inference.py`
+  BashShellCommand(String),
+}

crates/lib/subprocess_common/src/lib.rs

@@ -13,4 +13,7 @@
 #![allow(dead_code)]
 #![allow(non_snake_case)]
 
-pub mod docker_options;
+pub mod command_adapter;
+pub mod command_exit_status;
+pub mod docker_options;
+pub mod executable_or_command;

crates/service/job/inference_job/src/job/job_loop/mod.rs

@@ -1,5 +1,4 @@
 pub mod clear_full_filesystem;
-pub mod command_exit_status;
 pub mod determine_dependency_status;
 pub mod job_success_result;
 pub mod main_loop;

crates/service/job/inference_job/src/job/job_types/format_conversion/fbx_to_gltf/command.rs

@@ -0,0 +1,207 @@
+/*
+use std::env;
+use std::ffi::OsString;
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::time::Duration;
+
+use anyhow::anyhow;
+use log::info;
+use subprocess::{Popen, PopenConfig, Redirection};
+
+use errors::AnyhowResult;
+use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
+use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
+
+#[derive(Clone)]
+pub struct FbxToGltfCommand {
+  /// Where the code lives
+  pub maybe_root_code_directory: Option<PathBuf>,
+
+  /// A single executable script or a much larger bash command.
+  pub executable_or_command: ExecutableOrCommand,
+
+  /// If this is run under Docker (eg. in development), these are the options.
+  pub maybe_docker_options: Option<DockerOptions>,
+
+  /// If the execution should be ended after a certain point.
+  pub maybe_execution_timeout: Option<Duration>,
+}
+
+#[derive(Clone)]
+pub enum ExecutableOrCommand {
+  /// Eg. `inference.py`
+  Executable(PathBuf),
+
+  /// Eg. `python3 inference.py`
+  Command(String),
+}
+
+pub struct InferenceArgs<'s, P: AsRef<Path>> {
+  pub input_fbx_file: P,
+  pub output_directory: P,
+}
+
+impl FbxToGltfCommand {
+
+  pub fn from_env() -> AnyhowResult<Self> {
+    let root_code_directory = easyenv::get_env_pathbuf_optional(
+      "FBX2GLTF_ROOT_DIRECTORY")?;
+
+    let maybe_command = easyenv::get_env_string_optional(
+      "FBX2GLTF_COMMAND");
+
+    // Optional, eg. `./infer.py`. Typically we'll use the command form instead.
+    let maybe_executable = easyenv::get_env_pathbuf_optional(
+      "FBX2GLTF_EXECUTABLE");
+
+    let executable_or_command = match maybe_command {
+      Some(command) => ExecutableOrCommand::Command(command),
+      None => match maybe_executable {
+        Some(executable) => ExecutableOrCommand::Executable(executable),
+        None => return Err(anyhow!("neither command nor executable passed")),
+      },
+    };
+
+    let maybe_virtual_env_activation_command = easyenv::get_env_string_optional(
+      "SAD_TALKER_INFERENCE_MAYBE_VENV_COMMAND");
+
+    let maybe_default_config_path = easyenv::get_env_pathbuf_optional(
+      "SAD_TALKER_INFERENCE_MAYBE_DEFAULT_CONFIG_PATH");
+
+    let maybe_execution_timeout =
+        easyenv::get_env_duration_seconds_optional("SAD_TALKER_TIMEOUT_SECONDS");
+
+    let maybe_docker_options = easyenv::get_env_string_optional(
+      "SAD_TALKER_INFERENCE_MAYBE_DOCKER_IMAGE")
+        .map(|image_name| {
+          DockerOptions {
+            image_name,
+            maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+            maybe_environment_variables: None,
+            maybe_gpu: Some(DockerGpu::All),
+          }
+        });
+
+    // Override for --checkpoint_dir at inference time
+    let alternate_checkpoint_dir = easyenv::get_env_pathbuf_optional(
+      "SAD_TALKER_ALTERNATE_CHECKPOINT_PATH");
+
+    Ok(Self {
+      executable_or_command,
+      maybe_docker_options,
+      maybe_execution_timeout,
+    })
+  }
+
+  pub fn execute_inference<P: AsRef<Path>>(
+    &self,
+    args: InferenceArgs<P>,
+  ) -> CommandExitStatus {
+    match self.do_execute_inference(args) {
+      Ok(exit_status) => exit_status,
+      Err(error) => CommandExitStatus::FailureWithReason { reason: format!("error: {:?}", error) },
+    }
+  }
+
+  fn do_execute_inference<P: AsRef<Path>>(
+    &self,
+    args: InferenceArgs<P>,
+  ) -> AnyhowResult<CommandExitStatus> {
+
+    let mut command = String::new();
+    command.push_str(&format!("cd {}", path_to_string(&self.root_code_directory)));
+
+    command.push_str(" && ");
+
+    match self.executable_or_command {
+      ExecutableOrCommand::Executable(ref executable) => {
+        command.push_str(&path_to_string(executable));
+        command.push_str(" ");
+      }
+      ExecutableOrCommand::Command(ref cmd) => {
+        command.push_str(cmd);
+        command.push_str(" ");
+      }
+    }
+
+    // ===== Begin Python Args =====
+
+    command.push_str(" -i ");
+    command.push_str(&path_to_string(args.input_fbx_file));
+
+    command.push_str(" -o ");
+    command.push_str(&path_to_string(args.output_directory));
+
+
+    // ===== End Python Args =====
+
+    if let Some(docker_options) = self.maybe_docker_options.as_ref() {
+      command = docker_options.to_command_string(&command);
+    }
+
+    info!("Command: {:?}", command);
+
+    let command_parts = [
+      "bash",
+      "-c",
+      &command
+    ];
+
+    let mut env_vars = Vec::new();
+
+    // Copy all environment variables from the parent process.
+    // This is necessary to send all the kubernetes settings for Nvidia / CUDA.
+    for (env_key, env_value) in env::vars() {
+      if IGNORED_ENVIRONMENT_VARS.contains(&env_key) {
+        continue;
+      }
+      env_vars.push((
+        OsString::from(env_key),
+        OsString::from(env_value),
+      ));
+    }
+
+    let mut config = PopenConfig::default();
+
+    info!("stderr will be written to file: {:?}", args.stderr_output_file.as_ref());
+
+    let stderr_file = File::create(&args.stderr_output_file)?;
+    config.stderr = Redirection::File(stderr_file);
+
+    if !env_vars.is_empty() {
+      config.env = Some(env_vars);
+    }
+
+    let mut p = Popen::create(&command_parts, config)?;
+
+    info!("Subprocess PID: {:?}", p.pid());
+
+    match self.maybe_execution_timeout {
+      None => {
+        let exit_status = p.wait()?;
+        info!("Subprocess exit status: {:?}", exit_status);
+        Ok(CommandExitStatus::from_exit_status(exit_status))
+      }
+      Some(timeout) => {
+        info!("Executing with timeout: {:?}", &timeout);
+        let exit_status = p.wait_timeout(timeout.clone())?;
+
+        match exit_status {
+          None => {
+            // NB: If the program didn't successfully terminate, kill it.
+            info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout);
+            let _r = p.terminate()?;
+            Ok(CommandExitStatus::Timeout)
+          }
+          Some(exit_status) => {
+            info!("Subprocess timed wait exit status: {:?}", exit_status);
+            Ok(CommandExitStatus::from_exit_status(exit_status))
+          }
+        }
+      }
+    }
+  }
+}
+*/

crates/service/job/inference_job/src/job/job_types/format_conversion/fbx_to_gltf/mod.rs

@@ -1 +1,2 @@
+pub mod command;
 pub mod process_job;

crates/service/job/inference_job/src/job/job_types/image_generation/sd/sd_inference_command.rs

@@ -1,4 +1,3 @@
-
 use std::collections::HashSet;
 use std::env;
 use std::ffi::OsString;
@@ -13,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!
 static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {

crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/sad_talker_inference_command.rs

@@ -12,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!
 static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {

crates/service/job/inference_job/src/job/job_types/mocap/mocap_net/mocapnet_inference_command.rs

@@ -4,15 +4,16 @@ use std::ffi::OsString;
 use std::fs::File;
 use std::path::{Path, PathBuf};
 use std::time::Duration;
+
 use anyhow::anyhow;
 use log::info;
 use once_cell::sync::Lazy;
 use subprocess::{Popen, PopenConfig, Redirection};
+
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!

crates/service/job/inference_job/src/job/job_types/tts/styletts2/styletts2_inference_command.rs

@@ -12,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 #[derive(Clone)]
 pub enum ExecutableOrCommand {
     /// Eg. `inference.py`

crates/service/job/inference_job/src/job/job_types/tts/vall_e_x/vall_e_x_inference_command.rs

@@ -12,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 #[derive(Clone)]
 pub enum ExecutableOrCommand {
     /// Eg. `inference.py`

crates/service/job/inference_job/src/job/job_types/vc/rvc_v2/rvc_v2_inference_command.rs

@@ -13,10 +13,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
 use mysql_queries::payloads::generic_inference_args::generic_inference_args::FundamentalFrequencyMethodForJob;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!
 static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {

crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs

@@ -12,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!
 static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {

crates/service/job/inference_job/src/job/job_types/videofilter/rerender_a_video/rerender_inference_command.rs

@@ -12,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!
 static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {

crates/service/job/inference_job/src/job/job_types/workflow/comfy_ui/comfy_ui_inference_command.rs

@@ -12,10 +12,9 @@ use subprocess::{Popen, PopenConfig, Redirection};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
+use subprocess_common::command_exit_status::CommandExitStatus;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
 
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
-
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!
 static IGNORED_ENVIRONMENT_VARS : Lazy<HashSet<String>> = Lazy::new(|| {

crates/service/job/inference_job/src/util/common_commands/ffmpeg_logo_watermark_command.rs

@@ -9,8 +9,7 @@ use subprocess::{Popen, PopenConfig};
 
 use errors::AnyhowResult;
 use filesys::path_to_string::path_to_string;
-
-use crate::job::job_loop::command_exit_status::CommandExitStatus;
+use subprocess_common::command_exit_status::CommandExitStatus;
 
 // These environment vars are not copied over to the subprocess
 // TODO/FIXME(bt, 2023-05-28): This is horrific security!