WIP on w2l inference job (doesn't compile)

This commit is contained in:
Brandon Thomas committed 2021-05-28 20:32:58 -04:00
1 parent 226019b09f
commit c11ccc36d7
11 files changed
+880 -19

No files matched your search

+4 -1
View File
@@ -21,8 +21,11 @@ W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public
# Bucket roots
AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio
W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates
W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output
W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip
#W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth
W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth
W2L_SCRIPT_NAME=vocodes_process_upload.py
W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py
W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py
+25
View File
@@ -0,0 +1,25 @@
/// This is designed to make it centrally configurable where
/// different types of objects are stored.
pub struct BucketPathUnifier {
}
impl BucketPathUnifier {
pub fn user_audio_for_w2l_inference_path(audio_token: &str) -> String {
"".to_string()
}
pub fn end_bump_video_for_w2l_path(end_bump_filename: &str) -> String {
"".to_string()
}
pub fn precomputed_faces_for_w2l_path(token: &str) -> String {
"".to_string()
}
pub fn pretrained_w2l_model_path(model_name: &str) -> String {
"".to_string()
}
}
+1
View File
@@ -1,3 +1,4 @@
pub mod bucket_client;
pub mod bucket_path_unifier;
pub mod bucket_paths;
pub mod file_hashing;
+2 -1
View File
@@ -1,2 +1,3 @@
pub mod tts_download_job_queries;
pub mod w2l_download_job_queries;
pub mod w2l_download_job_queries;
pub mod w2l_inference_job_queries;
@@ -0,0 +1,204 @@
//! NB: This seems required for sqlx to generate the cached queries.
//! Sqlx's prepare needs a *single* binary to work against, so we need to
//! include these in the main binary to generate all the queries.
use anyhow::anyhow;
use chrono::Utc;
use crate::util::anyhow_result::AnyhowResult;
use crate::util::random_crockford_token::random_crockford_token;
use sqlx::MySqlPool;
use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
/// table: w2l_template_upload_jobs
#[derive(Debug)]
pub struct W2lInferenceJobRecord {
pub id: i64,
pub inference_job_token: String,
pub uuid_idempotency_token: String,
// ===== FACE TEMPLATE OPTIONS =====
pub maybe_w2l_template_token: Option<String>,
pub maybe_public_image_bucket_location: Option<String>,
// ===== AUDIO SOURCE OPTIONS =====
pub maybe_tts_inference_result_token: Option<String>,
pub maybe_public_audio_bucket_location: Option<String>,
pub creator_ip_address: String,
pub maybe_creator_user_token: Option<String>,
pub creator_set_visibility: String, // TODO
pub disable_end_bump: bool,
pub disable_watermark: bool,
//pub maybe_subject_token: Option<String>,
//pub maybe_actor_subject_token: Option<String>,
pub status: String, // TODO
pub attempt_count: i32,
pub failure_reason: Option<String>,
pub created_at: chrono::DateTime<Utc>,
pub updated_at: chrono::DateTime<Utc>,
pub retry_at: Option<chrono::DateTime<Utc>>,
}
pub async fn query_w2l_inference_job_records(pool: &MySqlPool, num_records: u32)
-> AnyhowResult<Vec<W2lInferenceJobRecord>>
{
let job_records = sqlx::query_as!(
W2lInferenceJobRecord,
r#"
SELECT
token AS inference_job_token,
*
FROM w2l_inference_jobs
WHERE
(
status IN ("pending", "attempt_failed")
)
AND
(
retry_at IS NULL
OR
retry_at < CURRENT_TIMESTAMP
)
LIMIT ?
"#,
num_records,
)
.fetch_all(pool)
.await?;
Ok(job_records)
}
pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool,
job: &W2lInferenceJobRecord,
failure_reason: &str)
-> AnyhowResult<()>
{
// statuses: "attempt_failed", "complete_failure", "dead"
let status = "attempt_failed";
let query_result = sqlx::query!(
r#"
UPDATE w2l_inference_jobs
SET
status = ?,
failure_reason = ?,
retry_at = NOW() + interval 2 minute
WHERE id = ?
"#,
status,
failure_reason.to_string(),
job.id,
)
.execute(pool)
.await?;
Ok(())
}
pub async fn mark_w2l_inference_job_done(pool: &MySqlPool,
job: &W2lInferenceJobRecord,
success: bool)
-> AnyhowResult<()>
{
let status = if success { "complete_success" } else { "complete_failure" };
let query_result = sqlx::query!(
r#"
UPDATE w2l_inference_jobs
SET
status = ?,
failure_reason = NULL,
retry_at = NULL
WHERE id = ?
"#,
status,
job.id,
)
.execute(pool)
.await?;
Ok(())
}
pub async fn insert_w2l_result(pool: &MySqlPool,
template_type: &str, // TODO: ENUM!
job: &W2lInferenceJobRecord,
private_bucket_hash: &str,
private_bucket_object_name: &str,
private_bucket_cached_faces_object_name: &str,
maybe_image_preview_object_name: Option<&str>,
maybe_video_preview_object_name: Option<&str>,
file_size_bytes: u64,
maybe_mime_type: Option<&str>,
frame_width: u32,
frame_height: u32,
frame_count: u64,
fps: f32,
duration_millis: u64)
-> AnyhowResult<u64>
{
let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?;
let updatable_slug = model_token.clone();
let query_result = sqlx::query!(
r#"
INSERT INTO w2l_templates
SET
token = ?,
template_type = ?,
updatable_slug = ?,
title = ?,
description_markdown = '',
description_rendered_html = '',
creator_user_token = ?,
creator_ip_address = ?,
original_download_url = ?,
private_bucket_hash = ?,
private_bucket_object_name = ?,
private_bucket_cached_faces_object_name = ?,
maybe_public_bucket_preview_image_object_name = ?,
maybe_public_bucket_preview_video_object_name = ?,
file_size_bytes = ?,
mime_type = ?,
frame_width = ?,
frame_height = ?,
frame_count = ?,
fps = ?,
duration_millis = ?
"#,
model_token,
template_type,
updatable_slug,
job.title.to_string(),
job.creator_user_token.clone(),
job.creator_ip_address.clone(),
job.download_url.clone(),
private_bucket_hash.to_string(),
private_bucket_object_name.to_string(),
private_bucket_cached_faces_object_name.to_string(),
maybe_image_preview_object_name,
maybe_video_preview_object_name,
file_size_bytes,
maybe_mime_type.unwrap_or(""),
frame_width,
frame_height,
frame_count,
fps,
duration_millis
)
.execute(pool)
.await;
let record_id = match query_result {
Ok(res) => {
res.last_insert_id()
},
Err(err) => {
// TODO: handle better
return Err(anyhow!("Mysql error: {:?}", err));
}
};
Ok(record_id)
}
+6
View File
@@ -50,6 +50,12 @@ use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler
use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads;
use crate::buckets::bucket_client::BucketClient;
///
/// TODO TODO TODO TODO
/// TODO TODO TODO TODO
/// Make a central class that configures which bucket root paths things live in.
///
const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345";
// NB: sqlx::query is spammy and logs all queries as "info"-level
+1
View File
@@ -3,4 +3,5 @@ pub mod ffmpeg_generate_preview_video_command;
pub mod google_drive_download_command;
pub mod imagemagick_generate_preview_image_command;
pub mod wav2lip_inference_command;
pub mod wav2lip_inference_command_OLD;
pub mod wav2lip_process_upload_command;
@@ -3,9 +3,10 @@ use log::{info,warn};
use std::process::{Command, Stdio};
use subprocess::{Popen, PopenConfig, Redirection};
use std::fs::OpenOptions;
use crate::script_execution::wav2lip_inference_command_OLD::Wav2LipInferenceCommand;
/// This command is used to preprocess the face detection frames from user-submitted video.
/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved.
/// This command is used to run inference.
/// It uses preprocessed face files so that it's much faster.
#[derive(Clone)]
pub struct Wav2LipInferenceCommand {
w2l_directory: String,
@@ -28,10 +29,9 @@ impl Wav2LipInferenceCommand {
pub fn execute(&self,
audio_filename: &str,
image_or_video_filename: &str,
output_video_filename: &str,
output_cached_faces_filename: &str,
output_metadata_filename: &str,
is_image: bool,
disable_end_bump: bool,
spawn_process: bool) -> AnyhowResult<()>
{
let mut command = String::new();
@@ -46,12 +46,12 @@ impl Wav2LipInferenceCommand {
command.push_str(&self.script_name);
command.push_str(" --checkpoint_path ");
command.push_str(&self.checkpoint_path);
command.push_str(" --face ");
command.push_str(image_or_video_filename);
command.push_str(" --audio ");
command.push_str(" --audio_filename");
command.push_str(audio_filename);
command.push_str(" --outfile ");
command.push_str(output_video_filename);
command.push_str(" --cached_faces_filename ");
command.push_str(output_cached_faces_filename);
command.push_str(" --output_metadata_filename ");
command.push_str(output_metadata_filename);
if is_image {
command.push_str(" --is_image ");
@@ -74,14 +74,14 @@ impl Wav2LipInferenceCommand {
.write(true)
.create(true)
.truncate(true)
.open("/tmp/stdout.txt")?;
.open("/tmp/wav2lip_upload_stdout.txt")?;
let stderr_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open("/tmp/stderr.txt")?;
.open("/tmp/wav2lip_upload_stderr.txt")?;
let mut p = Popen::create(&command_parts, PopenConfig {
//stdout: Redirection::Pipe,
@@ -0,0 +1,125 @@
use crate::AnyhowResult;
use log::{info,warn};
use std::process::{Command, Stdio};
use subprocess::{Popen, PopenConfig, Redirection};
use std::fs::OpenOptions;
/// This command is used to preprocess the face detection frames from user-submitted video.
/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved.
#[derive(Clone)]
pub struct Wav2LipInferenceCommandOLD {
w2l_directory: String,
script_name: String,
checkpoint_path: String,
}
impl Wav2LipInferenceCommandOLD {
pub fn new(
w2l_directory: &str,
script_name: &str,
checkpoint_path: &str,
) -> Self {
Self {
w2l_directory: w2l_directory.to_string(),
script_name: script_name.to_string(),
checkpoint_path: checkpoint_path.to_string(),
}
}
pub fn execute(&self,
audio_filename: &str,
image_or_video_filename: &str,
output_video_filename: &str,
is_image: bool,
disable_end_bump: bool,
spawn_process: bool) -> AnyhowResult<()>
{
let mut command = String::new();
command.push_str("echo 'test'");
command.push_str(" && ");
command.push_str(&format!("cd {}", self.w2l_directory));
command.push_str(" && ");
command.push_str("source python/bin/activate");
command.push_str(" && ");
command.push_str("python ");
command.push_str(&self.script_name);
command.push_str(" --checkpoint_path ");
command.push_str(&self.checkpoint_path);
command.push_str(" --face ");
command.push_str(image_or_video_filename);
command.push_str(" --audio ");
command.push_str(audio_filename);
command.push_str(" --outfile ");
command.push_str(output_video_filename);
if is_image {
command.push_str(" --is_image ");
}
info!("Command: {:?}", command);
let command_parts = [
"bash",
"-c",
&command
];
if spawn_process {
// NB: This forks and returns immediately.
//let _child_pid = command_builder.spawn()?;
let stdout_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open("/tmp/stdout.txt")?;
let stderr_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open("/tmp/stderr.txt")?;
let mut p = Popen::create(&command_parts, PopenConfig {
//stdout: Redirection::Pipe,
//stderr: Redirection::Pipe,
stdout: Redirection::File(stdout_file),
stderr: Redirection::File(stderr_file),
..Default::default()
})?;
info!("Pid : {:?}", p.pid());
p.detach();
} else {
// NB: This is a blocking call.
/*let output = command_builder.output()?;
info!("Output status: {}", output.status);
info!("Stdout: {:?}", String::from_utf8(output.stdout));
error!("Stderr: {:?}", String::from_utf8(output.stderr));
if !output.status.success() {
bail!("Bad error code: {:?}", output.status);
}*/
let mut p = Popen::create(&command_parts, PopenConfig {
//stdout: Redirection::Pipe,
//stderr: Redirection::Pipe,
..Default::default()
})?;
info!("Pid : {:?}", p.pid());
let exit_status = p.wait()?;
info!("Exit status: {:?}", exit_status);
}
Ok(())
}
}
+8 -4
View File
@@ -47,17 +47,19 @@ use crate::script_execution::imagemagick_generate_preview_image_command::Imagema
const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
const ENV_SECRET_KEY : &'static str = "SECRET_KEY";
const ENV_REGION_NAME : &'static str = "REGION_NAME";
const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT";
// Buckets (private data)
const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME";
// Buckets (public data)
const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME";
// Various bucket roots
const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT";
// Python code
const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY";
const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT";
const ENV_SCRIPT_NAME : &'static str = "W2L_SCRIPT_NAME";
const ENV_DOWNLOAD_SCRIPT_NAME : &'static str = "W2L_DOWNLOAD_SCRIPT_NAME";
// NB: sqlx::query is spammy and logs all queries as "info"-level
const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn";
@@ -104,12 +106,14 @@ async fn main() -> AnyhowResult<()> {
let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?;
// Private and Public Buckets
let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
// Bucket roots
let bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?;
let private_bucket_client = BucketClient::create(
&access_key,
&secret_key,
@@ -127,7 +131,7 @@ async fn main() -> AnyhowResult<()> {
)?;
let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?;
let py_script_name = easyenv::get_env_string_required(ENV_SCRIPT_NAME)?;
let py_script_name = easyenv::get_env_string_required(ENV_DOWNLOAD_SCRIPT_NAME)?;
let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?;
let w2l_preprecess_command = Wav2LipPreprocessClient::new(
+492 -1
View File
@@ -1,7 +1,498 @@
#![deny(unused_must_use)]
#![allow(dead_code)]
#![allow(unused_mut)]
#![allow(unused_imports)]
#![warn(unused_must_use)]
//#![allow(warnings)]
#[macro_use] extern crate serde_derive;
pub mod buckets;
pub mod job_queries;
pub mod script_execution;
pub mod util;
use anyhow::anyhow;
use chrono::Utc;
use crate::buckets::bucket_client::BucketClient;
use crate::buckets::bucket_paths::hash_to_bucket_path;
use crate::buckets::file_hashing::get_file_hash;
use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord;
use crate::job_queries::w2l_inference_job_queries::insert_w2l_result;
use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done;
use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure;
use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records;
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
use crate::util::anyhow_result::AnyhowResult;
use crate::util::filesystem::check_directory_exists;
use crate::util::filesystem::check_file_exists;
use crate::util::random_crockford_token::random_crockford_token;
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
use log::{warn, info};
use ring::digest::{Context, Digest, SHA256};
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use std::fs::{File, metadata};
use std::io::{BufReader, Read};
use std::path::{PathBuf, Path};
use std::process::Command;
use std::time::Duration;
use tempdir::TempDir;
use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand;
use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
// Buckets (shared config)
const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
const ENV_SECRET_KEY : &'static str = "SECRET_KEY";
const ENV_REGION_NAME : &'static str = "REGION_NAME";
// Buckets (private data)
const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME";
// Buckets (public data)
const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME";
// Various bucket roots
const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT";
const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT";
const ENV_INFERENCE_OUTPUT_BUCKET_ROOT : &'static str = "W2L_INFERENCE_OUTPUT_BUCKET_ROOT";
// Python code
const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY";
const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT";
const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME";
// NB: sqlx::query is spammy and logs all queries as "info"-level
const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn";
const DEFAULT_TEMP_DIR: &'static str = "/tmp";
struct Inferencer {
pub download_temp_directory: PathBuf,
pub mysql_pool: MySqlPool,
pub private_bucket_client: BucketClient,
pub public_bucket_client: BucketClient,
pub google_drive_downloader: GoogleDriveDownloadCommand,
pub w2l_inference: Wav2LipInferenceCommand,
//pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand,
//pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand,
//pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand,
// Command to run
pub inference_script: String,
// Root to retrieve W2L templates (public and private)
pub w2l_template_uploads_bucket_root: String,
// Root to retrieve uploaded audio (public and private)
pub audio_uploads_bucket_root: String,
// Root to upload inference output
pub inference_output_bucket_root: String,
}
#[tokio::main]
async fn main() -> AnyhowResult<()> {
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
// NB: Do not check this secrets-containing dotenv file into VCS.
// This file should only contain *development* secrets, never production.
let _ = dotenv::from_filename(".env-secrets").ok();
info!("Obtaining hostname...");
let server_hostname = hostname::get()
.ok()
.and_then(|h| h.into_string().ok())
.unwrap_or("w2l-inference-job".to_string());
info!("Hostname: {}", &server_hostname);
// Bucket stuff (shared)
let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?;
let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?;
let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?;
// Private and Public Buckets
let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?;
let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?;
// Bucket roots
let w2l_template_bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?;
let audio_bucket_root = easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?;
let inference_output_bucket_root = easyenv::get_env_string_required(ENV_INFERENCE_OUTPUT_BUCKET_ROOT)?;
let private_bucket_client = BucketClient::create(
&access_key,
&secret_key,
&region_name,
&private_bucket_name,
None,
)?;
let public_bucket_client = BucketClient::create(
&access_key,
&secret_key,
&region_name,
&public_bucket_name,
None,
)?;
let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?;
let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?;
let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?;
let w2l_inference_command = Wav2LipInferenceCommand::new(
&py_code_directory,
&py_script_name,
&py_model_checkpoint,
);
let temp_directory = easyenv::get_env_string_or_default(
"DOWNLOAD_TEMP_DIR",
DEFAULT_TEMP_DIR);
// TODO: In the future, we may want to enable downloading images or audio files.
//let download_script = easyenv::get_env_string_or_default(
// "DOWNLOAD_SCRIPT",
// "./scripts/download_gdrive.py");
//
//let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
let temp_directory = PathBuf::from(temp_directory);
check_directory_exists(&temp_directory)?;
let db_connection_string =
easyenv::get_env_string_or_default(
"MYSQL_URL",
"mysql://root:root@localhost/storyteller");
info!("Connecting to database...");
let mysql_pool = MySqlPoolOptions::new()
.max_connections(5)
.connect(&db_connection_string)
.await?;
let inferencer = Inferencer {
download_temp_directory: temp_directory,
mysql_pool,
public_bucket_client,
private_bucket_client,
inference_script,
google_drive_downloader,
//ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {},
//ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {},
//imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {},
w2l_inference: w2l_inference_command,
w2l_template_uploads_bucket_root: w2l_template_bucket_root.to_string(),
audio_uploads_bucket_root: audio_bucket_root.to_string(),
inference_output_bucket_root: inference_output_bucket_root.to_string(),
};
main_loop(inferencer).await;
Ok(())
}
const START_TIMEOUT_MILLIS : u64 = 500;
const INCREASE_TIMEOUT_MILLIS : u64 = 1000;
async fn main_loop(inferencer: Inferencer) {
let mut timeout_millis = START_TIMEOUT_MILLIS;
loop {
let num_records = 1;
let query_result = query_w2l_template_upload_job_records(
&inferencer.mysql_pool,
num_records)
.await;
let jobs = match query_result {
Ok(jobs) => jobs,
Err(e) => {
warn!("Error querying jobs: {:?}", e);
std::thread::sleep(Duration::from_millis(timeout_millis));
timeout_millis += INCREASE_TIMEOUT_MILLIS;
continue;
}
};
if jobs.is_empty() {
info!("No jobs!");
std::thread::sleep(Duration::from_millis(1500));
continue;
}
let result = process_jobs(&inferencer, jobs).await;
match result {
Ok(_) => {},
Err(e) => {
warn!("Error querying jobs: {:?}", e);
std::thread::sleep(Duration::from_millis(timeout_millis));
timeout_millis += INCREASE_TIMEOUT_MILLIS;
continue;
}
}
timeout_millis = START_TIMEOUT_MILLIS; // reset
std::thread::sleep(Duration::from_millis(500));
}
}
async fn process_jobs(inferencer: &Inferencer, jobs: Vec<W2lInferenceJobRecord>) -> AnyhowResult<()> {
for job in jobs.into_iter() {
let result = process_job(inferencer, &job).await;
match result {
Ok(_) => {},
Err(e) => {
warn!("Failure to process job: {:?}", e);
let failure_reason = "";
let _r = mark_w2l_inference_job_failure(
&inferencer.mysql_pool,
&job,
failure_reason)
.await;
}
}
}
Ok(())
}
#[derive(Deserialize)]
struct FileMetadata {
pub is_video: bool,
pub width: u32,
pub height: u32,
pub num_frames: u64,
pub fps: Option<f32>,
pub duration_millis: Option<u64>,
pub mimetype: Option<String>,
pub file_size_bytes: u64,
}
fn read_metadata_file(filename: &str) -> AnyhowResult<FileMetadata> {
let mut file = File::open(filename)?;
let mut buffer = String::new();
file.read_to_string(&mut buffer)?;
Ok(serde_json::from_str(&buffer)?)
}
async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> {
// TODO: 1. Check if w2l model is downloaded and download it
// TODO: 2. Check if w2l template faces are downloaded and download it
// TODO: 3. Download audio
// TODO: 1. Mark processing.
// TODO: 2. Download. (DONE)
// TODO: 3. Process template with face detection (DONE)
// TODO: 4. Determine if picture or video
// TODO: 5. Take a screenshot/gif
// TODO: 6. Upload all (partially done).
// TODO: 7. Save record. (DONE)
// TODO: 8. Mark job done. (DONE)
let temp_dir = format!("temp_{}", job.id);
let temp_dir = TempDir::new(&temp_dir)?;
let download_url = job.download_url.as_ref()
.map(|c| c.to_string())
.unwrap_or("".to_string());
// ==================== DOWNLOAD FILE ==================== //
info!("Calling downloader...");
let download_filename = inferencer.google_drive_downloader
.download_file(&download_url, &temp_dir).await?;
// ==================== PROCESS FACES ==================== //
// This is the Python Pickle file with all the face frames.
// We'll retain it forever since it's expensive to compute.
let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename);
// This is a file that we'll use to determine if the file is an image or video.
let output_metadata_filename = format!("{}_metadata.json", &download_filename);
let is_image = false; // TODO: Don't always treat as video.
let spawn_process = false;
inferencer.w2l_processor.execute(
&download_filename,
&cached_faces_filename,
&output_metadata_filename,
is_image,
spawn_process)?;
// ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== //
let video_or_image_path = PathBuf::from(&download_filename);
let cached_faces_path = &PathBuf::from(&cached_faces_filename);
let output_metadata_path = &PathBuf::from(&output_metadata_filename);
info!("Checking that both files exist (original source + cached faces) ...");
check_file_exists(&video_or_image_path)?;
check_file_exists(&cached_faces_path)?;
check_file_exists(&output_metadata_path)?;
let file_metadata = read_metadata_file(&output_metadata_filename)?;
// ==================== BASE OBJECT NAMES BASED ON HASH ==================== //
let private_bucket_hash = get_file_hash(&download_filename)?;
info!("File hash: {}", private_bucket_hash);
// Full path to video/image
let full_object_path = hash_to_bucket_path(
&private_bucket_hash,
Some(&inferencer.w2l_template_uploads_bucket_root))?;
// ==================== GENERATE VIDEO PREVIEWS ==================== //
let mut maybe_image_preview_filename : Option<PathBuf> = None;
let mut maybe_image_preview_object_name : Option<String> = None;
let mut maybe_video_preview_filename : Option<PathBuf> = None;
let mut maybe_video_preview_object_name : Option<String> = None;
if file_metadata.is_video {
let preview_filename = format!("{}_preview.webp", &download_filename);
inferencer.ffmpeg_video_preview_generator.execute(
&download_filename,
&preview_filename,
500,
500,
true,
false
)?;
let video_preview_path = PathBuf::from(&preview_filename);
check_file_exists(&video_preview_path)?;
let preview_object_path = format!("{}_preview.webp", full_object_path);
maybe_video_preview_object_name = Some(preview_object_path);
maybe_video_preview_filename = Some(video_preview_path);
} else {
let preview_filename = format!("{}_preview.webp", &download_filename);
inferencer.imagemagick_image_preview_generator.execute(
&download_filename,
&preview_filename,
500,
500,
false
)?;
let image_preview_path = PathBuf::from(&preview_filename);
check_file_exists(&image_preview_path)?;
let preview_object_path = format!("{}_preview.webp", full_object_path);
maybe_image_preview_object_name = Some(preview_object_path);
maybe_image_preview_filename = Some(image_preview_path);
}
// ==================== UPLOAD TO BUCKETS ==================== //
info!("Image/video destination bucket path: {}", full_object_path);
// Full path to cached faces
let full_object_path_cached_faces = format!("{}_detected_faces.pickle", full_object_path);
info!("Cached faces destination bucket path: {}", full_object_path_cached_faces);
info!("Uploading image/video...");
let original_mime_type = file_metadata.mimetype
.as_deref()
.unwrap_or("application/octet-stream");
inferencer.private_bucket_client.upload_filename_with_content_type(
&full_object_path,
&video_or_image_path,
original_mime_type).await?;
inferencer.public_bucket_client.upload_filename_with_content_type(
&full_object_path,
&video_or_image_path,
original_mime_type).await?;
info!("Uploading cached faces...");
inferencer.private_bucket_client.upload_filename(
&full_object_path_cached_faces,
&cached_faces_path).await?;
// TODO: Fix this ugh.
if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() {
if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() {
info!("Uploading image preview...");
inferencer.private_bucket_client.upload_filename_with_content_type(
&image_preview_object_name,
image_preview_filename,
"image/webp").await?;
info!("Uploading image preview...");
inferencer.public_bucket_client.upload_filename_with_content_type(
&image_preview_object_name,
image_preview_filename,
"image/webp").await?;
}
}
// TODO: Fix this ugh.
if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() {
if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() {
info!("Uploading video preview...");
inferencer.private_bucket_client.upload_filename_with_content_type(
&video_preview_object_name,
video_preview_filename,
"image/webp").await?;
info!("Uploading video preview...");
inferencer.public_bucket_client.upload_filename_with_content_type(
&video_preview_object_name,
video_preview_filename,
"image/webp").await?;
}
}
// ==================== SAVE RECORDS ==================== //
let template_type = if file_metadata.is_video { "video" } else { "image" };
info!("Saving w2l inference record...");
let id = insert_w2l_result(
&inferencer.mysql_pool,
template_type,
job,
&private_bucket_hash,
&full_object_path,
&full_object_path_cached_faces,
maybe_image_preview_object_name.as_deref(),
maybe_video_preview_object_name.as_deref(),
file_metadata.file_size_bytes,
file_metadata.mimetype.as_deref(),
file_metadata.width,
file_metadata.height,
file_metadata.num_frames,
file_metadata.fps.unwrap_or(0.0f32),
file_metadata.duration_millis.unwrap_or(0))
.await?;
info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}",
job.id, id);
mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?;
fn main() -> AnyhowResult<()> {
Ok(())
}