From a2c48d74693bdeda59a4c131ccda67e8fa14c586 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 2 May 2023 22:19:03 -0700 Subject: [PATCH] kill subprocess if timeout elapses --- .../vc/so_vits_svc/so_vits_svc_inference_command.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs index 4c3c69fca0..6ccb4be258 100644 --- a/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs +++ b/crates/service/job/inference_job/src/job/job_types/vc/so_vits_svc/so_vits_svc_inference_command.rs @@ -7,7 +7,7 @@ use std::ffi::OsString; use std::fs::OpenOptions; use std::path::{Path, PathBuf}; use std::time::Duration; -use subprocess::{Popen, PopenConfig, Redirection}; +use subprocess::{ExitStatus, Popen, PopenConfig, Redirection}; use subprocess_common::docker_options::{DockerEnvVar, DockerFilesystemMount, DockerGpu, DockerOptions}; /// This command is used to check tacotron for being a real model @@ -324,7 +324,14 @@ impl SoVitsSvcInferenceCommand { Some(timeout) => { info!("Executing with timeout: {:?}", &timeout); let exit_status = p.wait_timeout(timeout.clone())?; - info!("Subprocess timed exit status: {:?}", exit_status); + + if exit_status.is_none() { + // NB: If the program didn't successfully terminate, kill it. + info!("Subprocess didn't end after timeout: {:?}; terminating...", &timeout); + let _r = p.terminate()?; + } else { + info!("Subprocess timed wait exit status: {:?}", exit_status); + } } }