mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-16 06:42:14 +00:00
WIP on w2l inference job (doesn't compile)
This commit is contained in:
1 parent
226019b09f
commit
c11ccc36d7
11 files changed
+880
-19
No files matched your search
@@ -21,8 +21,11 @@ W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public
|
||||
# Bucket roots
|
||||
AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
|
||||
W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates
|
||||
W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output
|
||||
|
||||
W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip
|
||||
#W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth
|
||||
W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth
|
||||
W2L_SCRIPT_NAME=vocodes_process_upload.py
|
||||
|
||||
W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py
|
||||
W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py
|
||||
@@ -0,0 +1,25 @@
|
||||
|
||||
/// This is designed to make it centrally configurable where
|
||||
/// different types of objects are stored.
|
||||
pub struct BucketPathUnifier {
|
||||
|
||||
}
|
||||
|
||||
impl BucketPathUnifier {
|
||||
|
||||
pub fn user_audio_for_w2l_inference_path(audio_token: &str) -> String {
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn end_bump_video_for_w2l_path(end_bump_filename: &str) -> String {
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn precomputed_faces_for_w2l_path(token: &str) -> String {
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn pretrained_w2l_model_path(model_name: &str) -> String {
|
||||
"".to_string()
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod bucket_client;
|
||||
pub mod bucket_path_unifier;
|
||||
pub mod bucket_paths;
|
||||
pub mod file_hashing;
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod tts_download_job_queries;
|
||||
pub mod w2l_download_job_queries;
|
||||
pub mod w2l_download_job_queries;
|
||||
pub mod w2l_inference_job_queries;
|
||||
@@ -0,0 +1,204 @@
|
||||
//! NB: This seems required for sqlx to generate the cached queries.
|
||||
//! Sqlx's prepare needs a *single* binary to work against, so we need to
|
||||
//! include these in the main binary to generate all the queries.
|
||||
|
||||
use anyhow::anyhow;
|
||||
use chrono::Utc;
|
||||
use crate::util::anyhow_result::AnyhowResult;
|
||||
use crate::util::random_crockford_token::random_crockford_token;
|
||||
use sqlx::MySqlPool;
|
||||
use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
|
||||
|
||||
/// table: w2l_template_upload_jobs
|
||||
#[derive(Debug)]
|
||||
pub struct W2lInferenceJobRecord {
|
||||
pub id: i64,
|
||||
pub inference_job_token: String,
|
||||
pub uuid_idempotency_token: String,
|
||||
|
||||
// ===== FACE TEMPLATE OPTIONS =====
|
||||
pub maybe_w2l_template_token: Option<String>,
|
||||
pub maybe_public_image_bucket_location: Option<String>,
|
||||
|
||||
// ===== AUDIO SOURCE OPTIONS =====
|
||||
pub maybe_tts_inference_result_token: Option<String>,
|
||||
pub maybe_public_audio_bucket_location: Option<String>,
|
||||
|
||||
pub creator_ip_address: String,
|
||||
pub maybe_creator_user_token: Option<String>,
|
||||
pub creator_set_visibility: String, // TODO
|
||||
|
||||
pub disable_end_bump: bool,
|
||||
pub disable_watermark: bool,
|
||||
//pub maybe_subject_token: Option<String>,
|
||||
//pub maybe_actor_subject_token: Option<String>,
|
||||
pub status: String, // TODO
|
||||
pub attempt_count: i32,
|
||||
pub failure_reason: Option<String>,
|
||||
pub created_at: chrono::DateTime<Utc>,
|
||||
pub updated_at: chrono::DateTime<Utc>,
|
||||
pub retry_at: Option<chrono::DateTime<Utc>>,
|
||||
}
|
||||
|
||||
pub async fn query_w2l_inference_job_records(pool: &MySqlPool, num_records: u32)
|
||||
-> AnyhowResult<Vec<W2lInferenceJobRecord>>
|
||||
{
|
||||
let job_records = sqlx::query_as!(
|
||||
W2lInferenceJobRecord,
|
||||
r#"
|
||||
SELECT
|
||||
token AS inference_job_token,
|
||||
*
|
||||
FROM w2l_inference_jobs
|
||||
WHERE
|
||||
(
|
||||
status IN ("pending", "attempt_failed")
|
||||
)
|
||||
AND
|
||||
(
|
||||
retry_at IS NULL
|
||||
OR
|
||||
retry_at < CURRENT_TIMESTAMP
|
||||
)
|
||||
LIMIT ?
|
||||
"#,
|
||||
num_records,
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
Ok(job_records)
|
||||
}
|
||||
|
||||
pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool,
|
||||
job: &W2lInferenceJobRecord,
|
||||
failure_reason: &str)
|
||||
-> AnyhowResult<()>
|
||||
{
|
||||
// statuses: "attempt_failed", "complete_failure", "dead"
|
||||
let status = "attempt_failed";
|
||||
|
||||
let query_result = sqlx::query!(
|
||||
r#"
|
||||
UPDATE w2l_inference_jobs
|
||||
SET
|
||||
status = ?,
|
||||
failure_reason = ?,
|
||||
retry_at = NOW() + interval 2 minute
|
||||
WHERE id = ?
|
||||
"#,
|
||||
status,
|
||||
failure_reason.to_string(),
|
||||
job.id,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
pub async fn mark_w2l_inference_job_done(pool: &MySqlPool,
|
||||
job: &W2lInferenceJobRecord,
|
||||
success: bool)
|
||||
-> AnyhowResult<()>
|
||||
{
|
||||
let status = if success { "complete_success" } else { "complete_failure" };
|
||||
|
||||
let query_result = sqlx::query!(
|
||||
r#"
|
||||
UPDATE w2l_inference_jobs
|
||||
SET
|
||||
status = ?,
|
||||
failure_reason = NULL,
|
||||
retry_at = NULL
|
||||
WHERE id = ?
|
||||
"#,
|
||||
status,
|
||||
job.id,
|
||||
)
|
||||
.execute(pool)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn insert_w2l_result(pool: &MySqlPool,
|
||||
template_type: &str, // TODO: ENUM!
|
||||
job: &W2lInferenceJobRecord,
|
||||
private_bucket_hash: &str,
|
||||
private_bucket_object_name: &str,
|
||||
private_bucket_cached_faces_object_name: &str,
|
||||
maybe_image_preview_object_name: Option<&str>,
|
||||
maybe_video_preview_object_name: Option<&str>,
|
||||
file_size_bytes: u64,
|
||||
maybe_mime_type: Option<&str>,
|
||||
frame_width: u32,
|
||||
frame_height: u32,
|
||||
frame_count: u64,
|
||||
fps: f32,
|
||||
duration_millis: u64)
|
||||
-> AnyhowResult<u64>
|
||||
{
|
||||
let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?;
|
||||
let updatable_slug = model_token.clone();
|
||||
|
||||
let query_result = sqlx::query!(
|
||||
r#"
|
||||
INSERT INTO w2l_templates
|
||||
SET
|
||||
token = ?,
|
||||
template_type = ?,
|
||||
updatable_slug = ?,
|
||||
title = ?,
|
||||
description_markdown = '',
|
||||
description_rendered_html = '',
|
||||
creator_user_token = ?,
|
||||
creator_ip_address = ?,
|
||||
original_download_url = ?,
|
||||
private_bucket_hash = ?,
|
||||
private_bucket_object_name = ?,
|
||||
private_bucket_cached_faces_object_name = ?,
|
||||
maybe_public_bucket_preview_image_object_name = ?,
|
||||
maybe_public_bucket_preview_video_object_name = ?,
|
||||
file_size_bytes = ?,
|
||||
mime_type = ?,
|
||||
frame_width = ?,
|
||||
frame_height = ?,
|
||||
frame_count = ?,
|
||||
fps = ?,
|
||||
duration_millis = ?
|
||||
"#,
|
||||
model_token,
|
||||
template_type,
|
||||
updatable_slug,
|
||||
job.title.to_string(),
|
||||
job.creator_user_token.clone(),
|
||||
job.creator_ip_address.clone(),
|
||||
job.download_url.clone(),
|
||||
private_bucket_hash.to_string(),
|
||||
private_bucket_object_name.to_string(),
|
||||
private_bucket_cached_faces_object_name.to_string(),
|
||||
maybe_image_preview_object_name,
|
||||
maybe_video_preview_object_name,
|
||||
file_size_bytes,
|
||||
maybe_mime_type.unwrap_or(""),
|
||||
frame_width,
|
||||
frame_height,
|
||||
frame_count,
|
||||
fps,
|
||||
duration_millis
|
||||
)
|
||||
.execute(pool)
|
||||
.await;
|
||||
|
||||
let record_id = match query_result {
|
||||
Ok(res) => {
|
||||
res.last_insert_id()
|
||||
},
|
||||
Err(err) => {
|
||||
// TODO: handle better
|
||||
return Err(anyhow!("Mysql error: {:?}", err));
|
||||
}
|
||||
};
|
||||
|
||||
Ok(record_id)
|
||||
}
|
||||
@@ -50,6 +50,12 @@ use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler
|
||||
use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads;
|
||||
use crate::buckets::bucket_client::BucketClient;
|
||||
|
||||
///
|
||||
/// TODO TODO TODO TODO
|
||||
/// TODO TODO TODO TODO
|
||||
/// Make a central class that configures which bucket root paths things live in.
|
||||
///
|
||||
|
||||
const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345";
|
||||
|
||||
// NB: sqlx::query is spammy and logs all queries as "info"-level
|
||||
|
||||
@@ -3,4 +3,5 @@ pub mod ffmpeg_generate_preview_video_command;
|
||||
pub mod google_drive_download_command;
|
||||
pub mod imagemagick_generate_preview_image_command;
|
||||
pub mod wav2lip_inference_command;
|
||||
pub mod wav2lip_inference_command_OLD;
|
||||
pub mod wav2lip_process_upload_command;
|
||||
@@ -3,9 +3,10 @@ use log::{info,warn};
|
||||
use std::process::{Command, Stdio};
|
||||
use subprocess::{Popen, PopenConfig, Redirection};
|
||||
use std::fs::OpenOptions;
|
||||
use crate::script_execution::wav2lip_inference_command_OLD::Wav2LipInferenceCommand;
|
||||
|
||||
/// This command is used to preprocess the face detection frames from user-submitted video.
|
||||
/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved.
|
||||
/// This command is used to run inference.
|
||||
/// It uses preprocessed face files so that it's much faster.
|
||||
#[derive(Clone)]
|
||||
pub struct Wav2LipInferenceCommand {
|
||||
w2l_directory: String,
|
||||
@@ -28,10 +29,9 @@ impl Wav2LipInferenceCommand {
|
||||
|
||||
pub fn execute(&self,
|
||||
audio_filename: &str,
|
||||
image_or_video_filename: &str,
|
||||
output_video_filename: &str,
|
||||
output_cached_faces_filename: &str,
|
||||
output_metadata_filename: &str,
|
||||
is_image: bool,
|
||||
disable_end_bump: bool,
|
||||
spawn_process: bool) -> AnyhowResult<()>
|
||||
{
|
||||
let mut command = String::new();
|
||||
@@ -46,12 +46,12 @@ impl Wav2LipInferenceCommand {
|
||||
command.push_str(&self.script_name);
|
||||
command.push_str(" --checkpoint_path ");
|
||||
command.push_str(&self.checkpoint_path);
|
||||
command.push_str(" --face ");
|
||||
command.push_str(image_or_video_filename);
|
||||
command.push_str(" --audio ");
|
||||
command.push_str(" --audio_filename");
|
||||
command.push_str(audio_filename);
|
||||
command.push_str(" --outfile ");
|
||||
command.push_str(output_video_filename);
|
||||
command.push_str(" --cached_faces_filename ");
|
||||
command.push_str(output_cached_faces_filename);
|
||||
command.push_str(" --output_metadata_filename ");
|
||||
command.push_str(output_metadata_filename);
|
||||
|
||||
if is_image {
|
||||
command.push_str(" --is_image ");
|
||||
@@ -74,14 +74,14 @@ impl Wav2LipInferenceCommand {
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open("/tmp/stdout.txt")?;
|
||||
.open("/tmp/wav2lip_upload_stdout.txt")?;
|
||||
|
||||
let stderr_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open("/tmp/stderr.txt")?;
|
||||
.open("/tmp/wav2lip_upload_stderr.txt")?;
|
||||
|
||||
let mut p = Popen::create(&command_parts, PopenConfig {
|
||||
//stdout: Redirection::Pipe,
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
use crate::AnyhowResult;
|
||||
use log::{info,warn};
|
||||
use std::process::{Command, Stdio};
|
||||
use subprocess::{Popen, PopenConfig, Redirection};
|
||||
use std::fs::OpenOptions;
|
||||
|
||||
/// This command is used to preprocess the face detection frames from user-submitted video.
|
||||
/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved.
|
||||
#[derive(Clone)]
|
||||
pub struct Wav2LipInferenceCommandOLD {
|
||||
w2l_directory: String,
|
||||
script_name: String,
|
||||
checkpoint_path: String,
|
||||
}
|
||||
|
||||
impl Wav2LipInferenceCommandOLD {
|
||||
pub fn new(
|
||||
w2l_directory: &str,
|
||||
script_name: &str,
|
||||
checkpoint_path: &str,
|
||||
) -> Self {
|
||||
Self {
|
||||
w2l_directory: w2l_directory.to_string(),
|
||||
script_name: script_name.to_string(),
|
||||
checkpoint_path: checkpoint_path.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn execute(&self,
|
||||
audio_filename: &str,
|
||||
image_or_video_filename: &str,
|
||||
output_video_filename: &str,
|
||||
is_image: bool,
|
||||
disable_end_bump: bool,
|
||||
spawn_process: bool) -> AnyhowResult<()>
|
||||
{
|
||||
let mut command = String::new();
|
||||
|
||||
command.push_str("echo 'test'");
|
||||
command.push_str(" && ");
|
||||
command.push_str(&format!("cd {}", self.w2l_directory));
|
||||
command.push_str(" && ");
|
||||
command.push_str("source python/bin/activate");
|
||||
command.push_str(" && ");
|
||||
command.push_str("python ");
|
||||
command.push_str(&self.script_name);
|
||||
command.push_str(" --checkpoint_path ");
|
||||
command.push_str(&self.checkpoint_path);
|
||||
command.push_str(" --face ");
|
||||
command.push_str(image_or_video_filename);
|
||||
command.push_str(" --audio ");
|
||||
command.push_str(audio_filename);
|
||||
command.push_str(" --outfile ");
|
||||
command.push_str(output_video_filename);
|
||||
|
||||
if is_image {
|
||||
command.push_str(" --is_image ");
|
||||
}
|
||||
|
||||
info!("Command: {:?}", command);
|
||||
|
||||
let command_parts = [
|
||||
"bash",
|
||||
"-c",
|
||||
&command
|
||||
];
|
||||
|
||||
if spawn_process {
|
||||
// NB: This forks and returns immediately.
|
||||
//let _child_pid = command_builder.spawn()?;
|
||||
|
||||
let stdout_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open("/tmp/stdout.txt")?;
|
||||
|
||||
let stderr_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open("/tmp/stderr.txt")?;
|
||||
|
||||
let mut p = Popen::create(&command_parts, PopenConfig {
|
||||
//stdout: Redirection::Pipe,
|
||||
//stderr: Redirection::Pipe,
|
||||
stdout: Redirection::File(stdout_file),
|
||||
stderr: Redirection::File(stderr_file),
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
info!("Pid : {:?}", p.pid());
|
||||
|
||||
p.detach();
|
||||
|
||||
} else {
|
||||
// NB: This is a blocking call.
|
||||
/*let output = command_builder.output()?;
|
||||
|
||||
info!("Output status: {}", output.status);
|
||||
info!("Stdout: {:?}", String::from_utf8(output.stdout));
|
||||
error!("Stderr: {:?}", String::from_utf8(output.stderr));
|
||||
|
||||
if !output.status.success() {
|
||||
bail!("Bad error code: {:?}", output.status);
|
||||
}*/
|
||||
|
||||
let mut p = Popen::create(&command_parts, PopenConfig {
|
||||
//stdout: Redirection::Pipe,
|
||||
//stderr: Redirection::Pipe,
|
||||
..Default::default()
|
||||
})?;
|
||||
|
||||
info!("Pid : {:?}", p.pid());
|
||||
|
||||
let exit_status = p.wait()?;
|
||||
|
||||
info!("Exit status: {:?}", exit_status);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -47,17 +47,19 @@ use crate::script_execution::imagemagick_generate_preview_image_command::Imagema
|
||||
const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
|
||||
const ENV_SECRET_KEY : &'static str = "SECRET_KEY";
|
||||
const ENV_REGION_NAME : &'static str = "REGION_NAME";
|
||||
const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT";
|
||||
|
||||
// Buckets (private data)
|
||||
const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME";
|
||||
// Buckets (public data)
|
||||
const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME";
|
||||
|
||||
// Various bucket roots
|
||||
const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT";
|
||||
|
||||
// Python code
|
||||
const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY";
|
||||
const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT";
|
||||
const ENV_SCRIPT_NAME : &'static str = "W2L_SCRIPT_NAME";
|
||||
const ENV_DOWNLOAD_SCRIPT_NAME : &'static str = "W2L_DOWNLOAD_SCRIPT_NAME";
|
||||
|
||||
// NB: sqlx::query is spammy and logs all queries as "info"-level
|
||||
const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn";
|
||||
@@ -104,12 +106,14 @@ async fn main() -> AnyhowResult<()> {
|
||||
let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
|
||||
let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
|
||||
let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
|
||||
let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?;
|
||||
|
||||
// Private and Public Buckets
|
||||
let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
|
||||
let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
|
||||
|
||||
// Bucket roots
|
||||
let bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?;
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
@@ -127,7 +131,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
)?;
|
||||
|
||||
let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?;
|
||||
let py_script_name = easyenv::get_env_string_required(ENV_SCRIPT_NAME)?;
|
||||
let py_script_name = easyenv::get_env_string_required(ENV_DOWNLOAD_SCRIPT_NAME)?;
|
||||
let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?;
|
||||
|
||||
let w2l_preprecess_command = Wav2LipPreprocessClient::new(
|
||||
|
||||
+492
-1
@@ -1,7 +1,498 @@
|
||||
#![deny(unused_must_use)]
|
||||
#![allow(dead_code)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_imports)]
|
||||
#![warn(unused_must_use)]
|
||||
//#![allow(warnings)]
|
||||
|
||||
#[macro_use] extern crate serde_derive;
|
||||
|
||||
pub mod buckets;
|
||||
pub mod job_queries;
|
||||
pub mod script_execution;
|
||||
pub mod util;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use chrono::Utc;
|
||||
use crate::buckets::bucket_client::BucketClient;
|
||||
use crate::buckets::bucket_paths::hash_to_bucket_path;
|
||||
use crate::buckets::file_hashing::get_file_hash;
|
||||
use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord;
|
||||
use crate::job_queries::w2l_inference_job_queries::insert_w2l_result;
|
||||
use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done;
|
||||
use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure;
|
||||
use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records;
|
||||
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use crate::util::anyhow_result::AnyhowResult;
|
||||
use crate::util::filesystem::check_directory_exists;
|
||||
use crate::util::filesystem::check_file_exists;
|
||||
use crate::util::random_crockford_token::random_crockford_token;
|
||||
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
|
||||
use log::{warn, info};
|
||||
use ring::digest::{Context, Digest, SHA256};
|
||||
use sqlx::MySqlPool;
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
use std::fs::{File, metadata};
|
||||
use std::io::{BufReader, Read};
|
||||
use std::path::{PathBuf, Path};
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
use tempdir::TempDir;
|
||||
use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand;
|
||||
use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
|
||||
use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
|
||||
use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
|
||||
|
||||
// Buckets (shared config)
|
||||
const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
|
||||
const ENV_SECRET_KEY : &'static str = "SECRET_KEY";
|
||||
const ENV_REGION_NAME : &'static str = "REGION_NAME";
|
||||
|
||||
// Buckets (private data)
|
||||
const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME";
|
||||
// Buckets (public data)
|
||||
const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME";
|
||||
|
||||
// Various bucket roots
|
||||
const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT";
|
||||
const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT";
|
||||
const ENV_INFERENCE_OUTPUT_BUCKET_ROOT : &'static str = "W2L_INFERENCE_OUTPUT_BUCKET_ROOT";
|
||||
|
||||
// Python code
|
||||
const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY";
|
||||
const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT";
|
||||
const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME";
|
||||
|
||||
// NB: sqlx::query is spammy and logs all queries as "info"-level
|
||||
const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn";
|
||||
const DEFAULT_TEMP_DIR: &'static str = "/tmp";
|
||||
|
||||
struct Inferencer {
|
||||
pub download_temp_directory: PathBuf,
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
pub private_bucket_client: BucketClient,
|
||||
pub public_bucket_client: BucketClient,
|
||||
|
||||
pub google_drive_downloader: GoogleDriveDownloadCommand,
|
||||
pub w2l_inference: Wav2LipInferenceCommand,
|
||||
//pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand,
|
||||
//pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand,
|
||||
//pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand,
|
||||
|
||||
// Command to run
|
||||
pub inference_script: String,
|
||||
|
||||
// Root to retrieve W2L templates (public and private)
|
||||
pub w2l_template_uploads_bucket_root: String,
|
||||
// Root to retrieve uploaded audio (public and private)
|
||||
pub audio_uploads_bucket_root: String,
|
||||
// Root to upload inference output
|
||||
pub inference_output_bucket_root: String,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> AnyhowResult<()> {
|
||||
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
|
||||
|
||||
// NB: Do not check this secrets-containing dotenv file into VCS.
|
||||
// This file should only contain *development* secrets, never production.
|
||||
let _ = dotenv::from_filename(".env-secrets").ok();
|
||||
|
||||
info!("Obtaining hostname...");
|
||||
|
||||
let server_hostname = hostname::get()
|
||||
.ok()
|
||||
.and_then(|h| h.into_string().ok())
|
||||
.unwrap_or("w2l-inference-job".to_string());
|
||||
|
||||
info!("Hostname: {}", &server_hostname);
|
||||
|
||||
// Bucket stuff (shared)
|
||||
let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
|
||||
let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
|
||||
let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
|
||||
|
||||
// Private and Public Buckets
|
||||
let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
|
||||
let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
|
||||
|
||||
// Bucket roots
|
||||
let w2l_template_bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?;
|
||||
let audio_bucket_root = easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?;
|
||||
let inference_output_bucket_root = easyenv::get_env_string_required(ENV_INFERENCE_OUTPUT_BUCKET_ROOT)?;
|
||||
|
||||
let private_bucket_client = BucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
&private_bucket_name,
|
||||
None,
|
||||
)?;
|
||||
|
||||
let public_bucket_client = BucketClient::create(
|
||||
&access_key,
|
||||
&secret_key,
|
||||
®ion_name,
|
||||
&public_bucket_name,
|
||||
None,
|
||||
)?;
|
||||
|
||||
let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?;
|
||||
let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?;
|
||||
let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?;
|
||||
|
||||
let w2l_inference_command = Wav2LipInferenceCommand::new(
|
||||
&py_code_directory,
|
||||
&py_script_name,
|
||||
&py_model_checkpoint,
|
||||
);
|
||||
|
||||
let temp_directory = easyenv::get_env_string_or_default(
|
||||
"DOWNLOAD_TEMP_DIR",
|
||||
DEFAULT_TEMP_DIR);
|
||||
|
||||
// TODO: In the future, we may want to enable downloading images or audio files.
|
||||
//let download_script = easyenv::get_env_string_or_default(
|
||||
// "DOWNLOAD_SCRIPT",
|
||||
// "./scripts/download_gdrive.py");
|
||||
//
|
||||
//let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
|
||||
|
||||
let temp_directory = PathBuf::from(temp_directory);
|
||||
|
||||
check_directory_exists(&temp_directory)?;
|
||||
|
||||
let db_connection_string =
|
||||
easyenv::get_env_string_or_default(
|
||||
"MYSQL_URL",
|
||||
"mysql://root:root@localhost/storyteller");
|
||||
|
||||
info!("Connecting to database...");
|
||||
|
||||
let mysql_pool = MySqlPoolOptions::new()
|
||||
.max_connections(5)
|
||||
.connect(&db_connection_string)
|
||||
.await?;
|
||||
|
||||
let inferencer = Inferencer {
|
||||
download_temp_directory: temp_directory,
|
||||
mysql_pool,
|
||||
public_bucket_client,
|
||||
private_bucket_client,
|
||||
inference_script,
|
||||
google_drive_downloader,
|
||||
//ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {},
|
||||
//ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {},
|
||||
//imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {},
|
||||
w2l_inference: w2l_inference_command,
|
||||
w2l_template_uploads_bucket_root: w2l_template_bucket_root.to_string(),
|
||||
audio_uploads_bucket_root: audio_bucket_root.to_string(),
|
||||
inference_output_bucket_root: inference_output_bucket_root.to_string(),
|
||||
};
|
||||
|
||||
main_loop(inferencer).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const START_TIMEOUT_MILLIS : u64 = 500;
|
||||
const INCREASE_TIMEOUT_MILLIS : u64 = 1000;
|
||||
|
||||
async fn main_loop(inferencer: Inferencer) {
|
||||
let mut timeout_millis = START_TIMEOUT_MILLIS;
|
||||
|
||||
loop {
|
||||
let num_records = 1;
|
||||
|
||||
let query_result = query_w2l_template_upload_job_records(
|
||||
&inferencer.mysql_pool,
|
||||
num_records)
|
||||
.await;
|
||||
|
||||
let jobs = match query_result {
|
||||
Ok(jobs) => jobs,
|
||||
Err(e) => {
|
||||
warn!("Error querying jobs: {:?}", e);
|
||||
std::thread::sleep(Duration::from_millis(timeout_millis));
|
||||
timeout_millis += INCREASE_TIMEOUT_MILLIS;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if jobs.is_empty() {
|
||||
info!("No jobs!");
|
||||
std::thread::sleep(Duration::from_millis(1500));
|
||||
continue;
|
||||
}
|
||||
|
||||
let result = process_jobs(&inferencer, jobs).await;
|
||||
|
||||
match result {
|
||||
Ok(_) => {},
|
||||
Err(e) => {
|
||||
warn!("Error querying jobs: {:?}", e);
|
||||
std::thread::sleep(Duration::from_millis(timeout_millis));
|
||||
timeout_millis += INCREASE_TIMEOUT_MILLIS;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
timeout_millis = START_TIMEOUT_MILLIS; // reset
|
||||
|
||||
std::thread::sleep(Duration::from_millis(500));
|
||||
}
|
||||
}
|
||||
|
||||
async fn process_jobs(inferencer: &Inferencer, jobs: Vec<W2lInferenceJobRecord>) -> AnyhowResult<()> {
|
||||
for job in jobs.into_iter() {
|
||||
let result = process_job(inferencer, &job).await;
|
||||
match result {
|
||||
Ok(_) => {},
|
||||
Err(e) => {
|
||||
warn!("Failure to process job: {:?}", e);
|
||||
let failure_reason = "";
|
||||
let _r = mark_w2l_inference_job_failure(
|
||||
&inferencer.mysql_pool,
|
||||
&job,
|
||||
failure_reason)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct FileMetadata {
|
||||
pub is_video: bool,
|
||||
pub width: u32,
|
||||
pub height: u32,
|
||||
pub num_frames: u64,
|
||||
pub fps: Option<f32>,
|
||||
pub duration_millis: Option<u64>,
|
||||
pub mimetype: Option<String>,
|
||||
pub file_size_bytes: u64,
|
||||
}
|
||||
|
||||
fn read_metadata_file(filename: &str) -> AnyhowResult<FileMetadata> {
|
||||
let mut file = File::open(filename)?;
|
||||
let mut buffer = String::new();
|
||||
file.read_to_string(&mut buffer)?;
|
||||
Ok(serde_json::from_str(&buffer)?)
|
||||
}
|
||||
|
||||
async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> {
|
||||
|
||||
// TODO: 1. Check if w2l model is downloaded and download it
|
||||
// TODO: 2. Check if w2l template faces are downloaded and download it
|
||||
// TODO: 3. Download audio
|
||||
|
||||
|
||||
// TODO: 1. Mark processing.
|
||||
// TODO: 2. Download. (DONE)
|
||||
// TODO: 3. Process template with face detection (DONE)
|
||||
// TODO: 4. Determine if picture or video
|
||||
// TODO: 5. Take a screenshot/gif
|
||||
// TODO: 6. Upload all (partially done).
|
||||
// TODO: 7. Save record. (DONE)
|
||||
// TODO: 8. Mark job done. (DONE)
|
||||
|
||||
let temp_dir = format!("temp_{}", job.id);
|
||||
let temp_dir = TempDir::new(&temp_dir)?;
|
||||
|
||||
let download_url = job.download_url.as_ref()
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or("".to_string());
|
||||
|
||||
// ==================== DOWNLOAD FILE ==================== //
|
||||
|
||||
info!("Calling downloader...");
|
||||
let download_filename = inferencer.google_drive_downloader
|
||||
.download_file(&download_url, &temp_dir).await?;
|
||||
|
||||
// ==================== PROCESS FACES ==================== //
|
||||
|
||||
// This is the Python Pickle file with all the face frames.
|
||||
// We'll retain it forever since it's expensive to compute.
|
||||
let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename);
|
||||
|
||||
// This is a file that we'll use to determine if the file is an image or video.
|
||||
let output_metadata_filename = format!("{}_metadata.json", &download_filename);
|
||||
|
||||
let is_image = false; // TODO: Don't always treat as video.
|
||||
let spawn_process = false;
|
||||
|
||||
inferencer.w2l_processor.execute(
|
||||
&download_filename,
|
||||
&cached_faces_filename,
|
||||
&output_metadata_filename,
|
||||
is_image,
|
||||
spawn_process)?;
|
||||
|
||||
// ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
|
||||
|
||||
let video_or_image_path = PathBuf::from(&download_filename);
|
||||
let cached_faces_path = &PathBuf::from(&cached_faces_filename);
|
||||
let output_metadata_path = &PathBuf::from(&output_metadata_filename);
|
||||
|
||||
info!("Checking that both files exist (original source + cached faces) ...");
|
||||
|
||||
check_file_exists(&video_or_image_path)?;
|
||||
check_file_exists(&cached_faces_path)?;
|
||||
check_file_exists(&output_metadata_path)?;
|
||||
|
||||
let file_metadata = read_metadata_file(&output_metadata_filename)?;
|
||||
|
||||
// ==================== BASE OBJECT NAMES BASED ON HASH ==================== //
|
||||
|
||||
let private_bucket_hash = get_file_hash(&download_filename)?;
|
||||
|
||||
info!("File hash: {}", private_bucket_hash);
|
||||
|
||||
// Full path to video/image
|
||||
let full_object_path = hash_to_bucket_path(
|
||||
&private_bucket_hash,
|
||||
Some(&inferencer.w2l_template_uploads_bucket_root))?;
|
||||
|
||||
// ==================== GENERATE VIDEO PREVIEWS ==================== //
|
||||
|
||||
let mut maybe_image_preview_filename : Option<PathBuf> = None;
|
||||
let mut maybe_image_preview_object_name : Option<String> = None;
|
||||
|
||||
let mut maybe_video_preview_filename : Option<PathBuf> = None;
|
||||
let mut maybe_video_preview_object_name : Option<String> = None;
|
||||
|
||||
if file_metadata.is_video {
|
||||
let preview_filename = format!("{}_preview.webp", &download_filename);
|
||||
|
||||
inferencer.ffmpeg_video_preview_generator.execute(
|
||||
&download_filename,
|
||||
&preview_filename,
|
||||
500,
|
||||
500,
|
||||
true,
|
||||
false
|
||||
)?;
|
||||
|
||||
let video_preview_path = PathBuf::from(&preview_filename);
|
||||
check_file_exists(&video_preview_path)?;
|
||||
|
||||
let preview_object_path = format!("{}_preview.webp", full_object_path);
|
||||
maybe_video_preview_object_name = Some(preview_object_path);
|
||||
maybe_video_preview_filename = Some(video_preview_path);
|
||||
|
||||
} else {
|
||||
let preview_filename = format!("{}_preview.webp", &download_filename);
|
||||
|
||||
inferencer.imagemagick_image_preview_generator.execute(
|
||||
&download_filename,
|
||||
&preview_filename,
|
||||
500,
|
||||
500,
|
||||
false
|
||||
)?;
|
||||
|
||||
let image_preview_path = PathBuf::from(&preview_filename);
|
||||
check_file_exists(&image_preview_path)?;
|
||||
|
||||
let preview_object_path = format!("{}_preview.webp", full_object_path);
|
||||
maybe_image_preview_object_name = Some(preview_object_path);
|
||||
maybe_image_preview_filename = Some(image_preview_path);
|
||||
}
|
||||
|
||||
// ==================== UPLOAD TO BUCKETS ==================== //
|
||||
|
||||
info!("Image/video destination bucket path: {}", full_object_path);
|
||||
|
||||
// Full path to cached faces
|
||||
let full_object_path_cached_faces = format!("{}_detected_faces.pickle", full_object_path);
|
||||
|
||||
info!("Cached faces destination bucket path: {}", full_object_path_cached_faces);
|
||||
|
||||
info!("Uploading image/video...");
|
||||
|
||||
let original_mime_type = file_metadata.mimetype
|
||||
.as_deref()
|
||||
.unwrap_or("application/octet-stream");
|
||||
|
||||
inferencer.private_bucket_client.upload_filename_with_content_type(
|
||||
&full_object_path,
|
||||
&video_or_image_path,
|
||||
original_mime_type).await?;
|
||||
|
||||
inferencer.public_bucket_client.upload_filename_with_content_type(
|
||||
&full_object_path,
|
||||
&video_or_image_path,
|
||||
original_mime_type).await?;
|
||||
|
||||
info!("Uploading cached faces...");
|
||||
inferencer.private_bucket_client.upload_filename(
|
||||
&full_object_path_cached_faces,
|
||||
&cached_faces_path).await?;
|
||||
|
||||
// TODO: Fix this ugh.
|
||||
if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() {
|
||||
if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() {
|
||||
info!("Uploading image preview...");
|
||||
inferencer.private_bucket_client.upload_filename_with_content_type(
|
||||
&image_preview_object_name,
|
||||
image_preview_filename,
|
||||
"image/webp").await?;
|
||||
|
||||
info!("Uploading image preview...");
|
||||
inferencer.public_bucket_client.upload_filename_with_content_type(
|
||||
&image_preview_object_name,
|
||||
image_preview_filename,
|
||||
"image/webp").await?;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Fix this ugh.
|
||||
if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() {
|
||||
if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() {
|
||||
info!("Uploading video preview...");
|
||||
inferencer.private_bucket_client.upload_filename_with_content_type(
|
||||
&video_preview_object_name,
|
||||
video_preview_filename,
|
||||
"image/webp").await?;
|
||||
|
||||
info!("Uploading video preview...");
|
||||
inferencer.public_bucket_client.upload_filename_with_content_type(
|
||||
&video_preview_object_name,
|
||||
video_preview_filename,
|
||||
"image/webp").await?;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== SAVE RECORDS ==================== //
|
||||
|
||||
let template_type = if file_metadata.is_video { "video" } else { "image" };
|
||||
|
||||
info!("Saving w2l inference record...");
|
||||
let id = insert_w2l_result(
|
||||
&inferencer.mysql_pool,
|
||||
template_type,
|
||||
job,
|
||||
&private_bucket_hash,
|
||||
&full_object_path,
|
||||
&full_object_path_cached_faces,
|
||||
maybe_image_preview_object_name.as_deref(),
|
||||
maybe_video_preview_object_name.as_deref(),
|
||||
file_metadata.file_size_bytes,
|
||||
file_metadata.mimetype.as_deref(),
|
||||
file_metadata.width,
|
||||
file_metadata.height,
|
||||
file_metadata.num_frames,
|
||||
file_metadata.fps.unwrap_or(0.0f32),
|
||||
file_metadata.duration_millis.unwrap_or(0))
|
||||
.await?;
|
||||
|
||||
info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}",
|
||||
job.id, id);
|
||||
|
||||
mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?;
|
||||
|
||||
fn main() -> AnyhowResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
Reference in new issue
Block a user