From 2ee6822bac3796cdbdb12c79d01f3a83a40e503c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 7 Aug 2024 19:17:44 -0400 Subject: [PATCH] truncate audio --- .../generic_inference_args.rs | 32 ----------- .../gptsovits_payload.rs | 11 ++++ .../gpt_sovits/gpt_sovits_dependencies.rs | 3 + .../tts_inference/check_and_validate_job.rs | 43 +++++++------- .../process_single_gpt_sovits_tts_job.rs | 56 ++++++++++++------- .../ffmpeg/ffmpeg_audio_truncate_args.rs | 4 +- .../old/ffmpeg_logo_watermark_command.rs | 15 ----- 7 files changed, 72 insertions(+), 92 deletions(-) diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs index da790488be..5446ccff4f 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/generic_inference_args.rs @@ -154,38 +154,6 @@ impl GenericInferenceArgs { } } -//impl InferenceCategoryAbbreviated { -// pub fn from_inference_category(category: InferenceCategory) -> Self { -// match category { -// InferenceCategory::LipsyncAnimation => Self::LipsyncAnimation, -// InferenceCategory::TextToSpeech => Self::TextToSpeech, -// InferenceCategory::VoiceConversion => Self::VoiceConversion, -// InferenceCategory::VideoFilter => Self::VideoFilter, -// InferenceCategory::ImageGeneration => Self::ImageGeneration, -// InferenceCategory::Mocap => Self::Mocap, -// InferenceCategory::Workflow => Self::Workflow, -// InferenceCategory::FormatConversion => Self::FormatConversion, -// InferenceCategory::ConvertBvhToWorkflow => Self::ConvertBvhToWorkflow, -// InferenceCategory::DeprecatedField => Self::LivePortrait, // TODO(bt,2024-07-16): This isn't correct -// } -// } -// -// pub fn to_inference_category(self) -> InferenceCategory { -// match self { -// Self::LipsyncAnimation => InferenceCategory::LipsyncAnimation, -// Self::TextToSpeech => InferenceCategory::TextToSpeech, -// Self::VoiceConversion => InferenceCategory::VoiceConversion, -// Self::VideoFilter => InferenceCategory::VideoFilter, -// Self::ImageGeneration =>InferenceCategory::ImageGeneration, -// Self::Mocap => InferenceCategory::Mocap, -// Self::Workflow => InferenceCategory::Workflow, -// Self::LivePortrait => InferenceCategory::DeprecatedField, // TODO(bt,2024-07-16): This isn't correct -// Self::FormatConversion => InferenceCategory::FormatConversion, -// Self::ConvertBvhToWorkflow => InferenceCategory::ConvertBvhToWorkflow, -// } -// } -//} - #[cfg(test)] mod tests { use tokens::tokens::{media_files::MediaFileToken, model_weights::ModelWeightToken}; diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args/gptsovits_payload.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args/gptsovits_payload.rs index 928fcc705e..eeee52edc8 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args/gptsovits_payload.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args/gptsovits_payload.rs @@ -13,5 +13,16 @@ pub struct GptSovitsPayload { #[serde(rename = "cv")] #[serde(skip_serializing_if = "Option::is_none")] pub creator_visibility: Option, + + // For free users, we may truncate long TTS inferences. + #[serde(rename = "s")] + #[serde(skip_serializing_if = "Option::is_none")] + pub truncate_seconds: Option, + + // For free users, we may occasionally append an advertisement + // to sign in or upgrade. + #[serde(rename = "a")] + #[serde(skip_serializing_if = "Option::is_none")] + pub append_advertisement: Option, } diff --git a/crates/service/job/inference_job/src/job/job_types/gpt_sovits/gpt_sovits_dependencies.rs b/crates/service/job/inference_job/src/job/job_types/gpt_sovits/gpt_sovits_dependencies.rs index 304bd26866..413d133cf8 100644 --- a/crates/service/job/inference_job/src/job/job_types/gpt_sovits/gpt_sovits_dependencies.rs +++ b/crates/service/job/inference_job/src/job/job_types/gpt_sovits/gpt_sovits_dependencies.rs @@ -1,9 +1,11 @@ use errors::AnyhowResult; use crate::job::job_types::gpt_sovits::gpt_sovits_inference_command::GptSovitsInferenceCommand; +use crate::util::common_commands::ffmpeg::runner::ffmpeg_command_runner::FfmpegCommandRunner; pub struct GptSovitsDependencies { pub inference_command: GptSovitsInferenceCommand, + pub ffmpeg_command_runner: FfmpegCommandRunner, } @@ -11,6 +13,7 @@ impl GptSovitsDependencies { pub fn setup() -> AnyhowResult { Ok(Self { inference_command: GptSovitsInferenceCommand::from_env()?, + ffmpeg_command_runner: FfmpegCommandRunner::from_env()?, }) } } \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/gpt_sovits/tts_inference/check_and_validate_job.rs b/crates/service/job/inference_job/src/job/job_types/gpt_sovits/tts_inference/check_and_validate_job.rs index a67588e52d..1afcbce33a 100644 --- a/crates/service/job/inference_job/src/job/job_types/gpt_sovits/tts_inference/check_and_validate_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/gpt_sovits/tts_inference/check_and_validate_job.rs @@ -1,6 +1,6 @@ use anyhow::anyhow; -use mysql_queries::payloads::generic_inference_args::generic_inference_args::InferenceCategoryAbbreviated; +use mysql_queries::payloads::generic_inference_args::generic_inference_args::{InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use tokens::tokens::model_weights::ModelWeightToken; @@ -13,14 +13,11 @@ pub struct JobArgs { pub reference_free: Option, pub temperature: Option, pub target_language: Option, + pub maybe_truncate_seconds: Option, + pub maybe_append_advertisement: Option, } pub fn check_and_validate_job(job: &AvailableInferenceJob) -> Result { - let inference_args = job.maybe_inference_args - .as_ref() - .map(|args| args.args.as_ref()) - .flatten(); - let inference_category = job.maybe_inference_args .as_ref() .map(|args| args.inference_category) @@ -34,20 +31,17 @@ pub fn check_and_validate_job(job: &AvailableInferenceJob) -> Result args, - // None => { - // return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no inference args for job!"))); - // } - // }; - // - // let inference_args = match inference_args { - // PolymorphicInferenceArgs::Gs(inference_args) => inference_args, - // _ => { - // return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong inner args for job!"))); - // } - // }; - // + let inference_args = job.maybe_inference_args + .as_ref() + .map(|args| args.args.as_ref()) + .flatten() + .map(|args| match args { + PolymorphicInferenceArgs::Gs(args) => Ok(args), + _ => Err(ProcessSingleJobError::from_anyhow_error(anyhow!("wrong payload for job!"))), + }) + .transpose()? + .ok_or_else(|| ProcessSingleJobError::from_anyhow_error(anyhow!("no inference args for job!")))?; + let input_text = match &job.maybe_raw_inference_text { Some(text) => text.clone(), None => { @@ -62,12 +56,13 @@ pub fn check_and_validate_job(job: &AvailableInferenceJob) -> Result 0 { + let truncated_audio_output_path = output_dir.join("truncated.wav"); + + let command_exit_status = gpt_sovits_deps + .ffmpeg_command_runner + .run_with_subprocess(RunAsSubprocessArgs { + args: Box::new(&FfmpegAudioTruncateArgs { + input_audio_file: &output_file_path, + output_audio_file: &truncated_audio_output_path, + truncate_seconds: truncate_seconds as usize, + }), + stderr: StreamRedirection::Pipe, + stdout: StreamRedirection::Pipe, + }); + + if !command_exit_status.is_success() { + warn!("Error truncating audio file. Exit status: {:?}", command_exit_status); + } + + if file_exists(&truncated_audio_output_path) { + info!("Truncated audio file created successfully."); + audio_for_upload_path = truncated_audio_output_path; + } else { + warn!("Truncated audio file does not exist: {:?}", &truncated_audio_output_path); + } + } + } // ==================== DECODE AUDIO FILE ==================== // let maybe_audio_info = decode_basic_audio_file_info( - &output_file_path, Some(MIME_TYPE), Some("wav")) + &audio_for_upload_path, Some(MIME_TYPE), Some("wav")) .map_err(|err| { warn!("Error decoding audio info: {:?}", err); err @@ -210,13 +228,13 @@ pub async fn process_single_gpt_sovits_tts_job( let result_bucket_object_pathbuf = result_bucket_location.to_full_object_pathbuf(); // Finished file path + info!("Upload File Path: {:?}", &audio_for_upload_path); info!("Upload Bucket Path: {:?}", result_bucket_object_pathbuf); - info!("Upload File Path: {:?}", &output_file_path); job_dependencies.buckets.public_bucket_client .upload_filename_with_content_type( &result_bucket_object_pathbuf, - &output_file_path, + &audio_for_upload_path, &MIME_TYPE ) .await @@ -226,11 +244,11 @@ pub async fn process_single_gpt_sovits_tts_job( info!("Calculating sha256..."); - let file_checksum = sha256_hash_file(&output_file_path).map_err(|err| { + let file_checksum = sha256_hash_file(&audio_for_upload_path).map_err(|err| { ProcessSingleJobError::Other(anyhow!("Error hashing file: {:?}", err)) })?; - let file_size_bytes = file_size(&output_file_path).map_err(|err| + let file_size_bytes = file_size(&audio_for_upload_path).map_err(|err| ProcessSingleJobError::Other(err) )?; diff --git a/crates/service/job/inference_job/src/util/common_commands/ffmpeg/ffmpeg_audio_truncate_args.rs b/crates/service/job/inference_job/src/util/common_commands/ffmpeg/ffmpeg_audio_truncate_args.rs index 30f42c2b7b..f37e60895c 100644 --- a/crates/service/job/inference_job/src/util/common_commands/ffmpeg/ffmpeg_audio_truncate_args.rs +++ b/crates/service/job/inference_job/src/util/common_commands/ffmpeg/ffmpeg_audio_truncate_args.rs @@ -3,13 +3,13 @@ use std::path::Path; use filesys::path_to_string::path_to_string; use subprocess_common::command_runner::command_args::CommandArgs; -pub struct FfmpegAudioReplaceArgs<'a> { +pub struct FfmpegAudioTruncateArgs<'a> { pub input_audio_file: &'a Path, pub output_audio_file: &'a Path, pub truncate_seconds: usize, } -impl CommandArgs for FfmpegAudioReplaceArgs<'_> { +impl CommandArgs for FfmpegAudioTruncateArgs<'_> { // eg. ffmpeg -i input-audio.aac -t 15 -c copy output.aac // time is in seconds diff --git a/crates/service/job/inference_job/src/util/common_commands/ffmpeg/old/ffmpeg_logo_watermark_command.rs b/crates/service/job/inference_job/src/util/common_commands/ffmpeg/old/ffmpeg_logo_watermark_command.rs index 9085c1732c..8f54dbd74e 100644 --- a/crates/service/job/inference_job/src/util/common_commands/ffmpeg/old/ffmpeg_logo_watermark_command.rs +++ b/crates/service/job/inference_job/src/util/common_commands/ffmpeg/old/ffmpeg_logo_watermark_command.rs @@ -11,21 +11,6 @@ use errors::AnyhowResult; use filesys::path_to_string::path_to_string; use subprocess_common::command_exit_status::CommandExitStatus; -// These environment vars are not copied over to the subprocess -// TODO/FIXME(bt, 2023-05-28): This is horrific security! -static IGNORED_ENVIRONMENT_VARS : Lazy> = Lazy::new(|| { - let env_var_names= [ - "MYSQL_URL", - "ACCESS_KEY", - "SECRET_KEY", - "NEWRELIC_API_KEY", - ]; - - env_var_names.iter() - .map(|value| value.to_string()) - .collect::>() -}); - #[derive(Clone)] pub struct FfmpegLogoWatermarkCommand { /// A single executable script or a much larger bash command.