diff --git a/.env b/.env index 86389b51fb..78fe012901 100644 --- a/.env +++ b/.env @@ -21,8 +21,11 @@ W2L_PUBLIC_DOWNLOAD_BUCKET_NAME=dev-vocodes-public # Bucket roots AUDIO_UPLOADS_BUCKET_ROOT=/user_uploaded_w2l_audio W2L_DOWNLOAD_BUCKET_ROOT=/user_uploaded_w2l_templates +W2L_INFERENCE_OUTPUT_BUCKET_ROOT=/w2l_output W2L_CODE_DIRECTORY=/home/bt/dev/storyteller/storyteller-web/models/Wav2Lip #W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/s3fd-619a316812.pth W2L_MODEL_CHECKPOINT=/home/bt/models/wav2lip/wav2lip_gan.pth -W2L_SCRIPT_NAME=vocodes_process_upload.py \ No newline at end of file + +W2L_DOWNLOAD_SCRIPT_NAME=vocodes_process_upload.py +W2L_INFERENCE_SCRIPT_NAME=vocodes_process_inference.py diff --git a/src/buckets/bucket_path_unifier.rs b/src/buckets/bucket_path_unifier.rs new file mode 100644 index 0000000000..e7ce5d28cb --- /dev/null +++ b/src/buckets/bucket_path_unifier.rs @@ -0,0 +1,25 @@ + +/// This is designed to make it centrally configurable where +/// different types of objects are stored. +pub struct BucketPathUnifier { + +} + +impl BucketPathUnifier { + + pub fn user_audio_for_w2l_inference_path(audio_token: &str) -> String { + "".to_string() + } + + pub fn end_bump_video_for_w2l_path(end_bump_filename: &str) -> String { + "".to_string() + } + + pub fn precomputed_faces_for_w2l_path(token: &str) -> String { + "".to_string() + } + + pub fn pretrained_w2l_model_path(model_name: &str) -> String { + "".to_string() + } +} \ No newline at end of file diff --git a/src/buckets/mod.rs b/src/buckets/mod.rs index 477fc55a78..1860d6bdad 100644 --- a/src/buckets/mod.rs +++ b/src/buckets/mod.rs @@ -1,3 +1,4 @@ pub mod bucket_client; +pub mod bucket_path_unifier; pub mod bucket_paths; pub mod file_hashing; diff --git a/src/job_queries/mod.rs b/src/job_queries/mod.rs index de78fe1474..e2b567b9df 100644 --- a/src/job_queries/mod.rs +++ b/src/job_queries/mod.rs @@ -1,2 +1,3 @@ pub mod tts_download_job_queries; -pub mod w2l_download_job_queries; \ No newline at end of file +pub mod w2l_download_job_queries; +pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/src/job_queries/w2l_inference_job_queries.rs b/src/job_queries/w2l_inference_job_queries.rs new file mode 100644 index 0000000000..7ee79feb68 --- /dev/null +++ b/src/job_queries/w2l_inference_job_queries.rs @@ -0,0 +1,204 @@ +//! NB: This seems required for sqlx to generate the cached queries. +//! Sqlx's prepare needs a *single* binary to work against, so we need to +//! include these in the main binary to generate all the queries. + +use anyhow::anyhow; +use chrono::Utc; +use crate::util::anyhow_result::AnyhowResult; +use crate::util::random_crockford_token::random_crockford_token; +use sqlx::MySqlPool; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; + +/// table: w2l_template_upload_jobs +#[derive(Debug)] +pub struct W2lInferenceJobRecord { + pub id: i64, + pub inference_job_token: String, + pub uuid_idempotency_token: String, + + // ===== FACE TEMPLATE OPTIONS ===== + pub maybe_w2l_template_token: Option, + pub maybe_public_image_bucket_location: Option, + + // ===== AUDIO SOURCE OPTIONS ===== + pub maybe_tts_inference_result_token: Option, + pub maybe_public_audio_bucket_location: Option, + + pub creator_ip_address: String, + pub maybe_creator_user_token: Option, + pub creator_set_visibility: String, // TODO + + pub disable_end_bump: bool, + pub disable_watermark: bool, + //pub maybe_subject_token: Option, + //pub maybe_actor_subject_token: Option, + pub status: String, // TODO + pub attempt_count: i32, + pub failure_reason: Option, + pub created_at: chrono::DateTime, + pub updated_at: chrono::DateTime, + pub retry_at: Option>, +} + +pub async fn query_w2l_inference_job_records(pool: &MySqlPool, num_records: u32) + -> AnyhowResult> +{ + let job_records = sqlx::query_as!( + W2lInferenceJobRecord, + r#" +SELECT + token AS inference_job_token, + * +FROM w2l_inference_jobs +WHERE + ( + status IN ("pending", "attempt_failed") + ) + AND + ( + retry_at IS NULL + OR + retry_at < CURRENT_TIMESTAMP + ) + LIMIT ? + "#, + num_records, + ) + .fetch_all(pool) + .await?; + + Ok(job_records) +} + +pub async fn mark_w2l_inference_job_failure(pool: &MySqlPool, + job: &W2lInferenceJobRecord, + failure_reason: &str) + -> AnyhowResult<()> +{ + // statuses: "attempt_failed", "complete_failure", "dead" + let status = "attempt_failed"; + + let query_result = sqlx::query!( + r#" +UPDATE w2l_inference_jobs +SET + status = ?, + failure_reason = ?, + retry_at = NOW() + interval 2 minute +WHERE id = ? + "#, + status, + failure_reason.to_string(), + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} +pub async fn mark_w2l_inference_job_done(pool: &MySqlPool, + job: &W2lInferenceJobRecord, + success: bool) + -> AnyhowResult<()> +{ + let status = if success { "complete_success" } else { "complete_failure" }; + + let query_result = sqlx::query!( + r#" +UPDATE w2l_inference_jobs +SET + status = ?, + failure_reason = NULL, + retry_at = NULL +WHERE id = ? + "#, + status, + job.id, + ) + .execute(pool) + .await?; + + Ok(()) +} + +pub async fn insert_w2l_result(pool: &MySqlPool, + template_type: &str, // TODO: ENUM! + job: &W2lInferenceJobRecord, + private_bucket_hash: &str, + private_bucket_object_name: &str, + private_bucket_cached_faces_object_name: &str, + maybe_image_preview_object_name: Option<&str>, + maybe_video_preview_object_name: Option<&str>, + file_size_bytes: u64, + maybe_mime_type: Option<&str>, + frame_width: u32, + frame_height: u32, + frame_count: u64, + fps: f32, + duration_millis: u64) + -> AnyhowResult +{ + let model_token = random_prefix_crockford_token("W2L_TPL:", 32)?; + let updatable_slug = model_token.clone(); + + let query_result = sqlx::query!( + r#" +INSERT INTO w2l_templates +SET + token = ?, + template_type = ?, + updatable_slug = ?, + title = ?, + description_markdown = '', + description_rendered_html = '', + creator_user_token = ?, + creator_ip_address = ?, + original_download_url = ?, + private_bucket_hash = ?, + private_bucket_object_name = ?, + private_bucket_cached_faces_object_name = ?, + maybe_public_bucket_preview_image_object_name = ?, + maybe_public_bucket_preview_video_object_name = ?, + file_size_bytes = ?, + mime_type = ?, + frame_width = ?, + frame_height = ?, + frame_count = ?, + fps = ?, + duration_millis = ? + "#, + model_token, + template_type, + updatable_slug, + job.title.to_string(), + job.creator_user_token.clone(), + job.creator_ip_address.clone(), + job.download_url.clone(), + private_bucket_hash.to_string(), + private_bucket_object_name.to_string(), + private_bucket_cached_faces_object_name.to_string(), + maybe_image_preview_object_name, + maybe_video_preview_object_name, + file_size_bytes, + maybe_mime_type.unwrap_or(""), + frame_width, + frame_height, + frame_count, + fps, + duration_millis + ) + .execute(pool) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + Ok(record_id) +} diff --git a/src/main.rs b/src/main.rs index c800d58f00..891991499d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,6 +50,12 @@ use crate::http_server::endpoints::tts::list_tts_models::list_tts_models_handler use crate::http_server::endpoints::w2l::enqueue_infer_w2l_with_uploads::enqueue_infer_w2l_with_uploads; use crate::buckets::bucket_client::BucketClient; +/// +/// TODO TODO TODO TODO +/// TODO TODO TODO TODO +/// Make a central class that configures which bucket root paths things live in. +/// + const DEFAULT_BIND_ADDRESS : &'static str = "0.0.0.0:12345"; // NB: sqlx::query is spammy and logs all queries as "info"-level diff --git a/src/script_execution/mod.rs b/src/script_execution/mod.rs index fcd02f3909..36879f0f3b 100644 --- a/src/script_execution/mod.rs +++ b/src/script_execution/mod.rs @@ -3,4 +3,5 @@ pub mod ffmpeg_generate_preview_video_command; pub mod google_drive_download_command; pub mod imagemagick_generate_preview_image_command; pub mod wav2lip_inference_command; +pub mod wav2lip_inference_command_OLD; pub mod wav2lip_process_upload_command; diff --git a/src/script_execution/wav2lip_inference_command.rs b/src/script_execution/wav2lip_inference_command.rs index cb26a5be61..7ed13a717e 100644 --- a/src/script_execution/wav2lip_inference_command.rs +++ b/src/script_execution/wav2lip_inference_command.rs @@ -3,9 +3,10 @@ use log::{info,warn}; use std::process::{Command, Stdio}; use subprocess::{Popen, PopenConfig, Redirection}; use std::fs::OpenOptions; +use crate::script_execution::wav2lip_inference_command_OLD::Wav2LipInferenceCommand; -/// This command is used to preprocess the face detection frames from user-submitted video. -/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +/// This command is used to run inference. +/// It uses preprocessed face files so that it's much faster. #[derive(Clone)] pub struct Wav2LipInferenceCommand { w2l_directory: String, @@ -28,10 +29,9 @@ impl Wav2LipInferenceCommand { pub fn execute(&self, audio_filename: &str, - image_or_video_filename: &str, - output_video_filename: &str, + output_cached_faces_filename: &str, + output_metadata_filename: &str, is_image: bool, - disable_end_bump: bool, spawn_process: bool) -> AnyhowResult<()> { let mut command = String::new(); @@ -46,12 +46,12 @@ impl Wav2LipInferenceCommand { command.push_str(&self.script_name); command.push_str(" --checkpoint_path "); command.push_str(&self.checkpoint_path); - command.push_str(" --face "); - command.push_str(image_or_video_filename); - command.push_str(" --audio "); + command.push_str(" --audio_filename"); command.push_str(audio_filename); - command.push_str(" --outfile "); - command.push_str(output_video_filename); + command.push_str(" --cached_faces_filename "); + command.push_str(output_cached_faces_filename); + command.push_str(" --output_metadata_filename "); + command.push_str(output_metadata_filename); if is_image { command.push_str(" --is_image "); @@ -74,14 +74,14 @@ impl Wav2LipInferenceCommand { .write(true) .create(true) .truncate(true) - .open("/tmp/stdout.txt")?; + .open("/tmp/wav2lip_upload_stdout.txt")?; let stderr_file = OpenOptions::new() .read(true) .write(true) .create(true) .truncate(true) - .open("/tmp/stderr.txt")?; + .open("/tmp/wav2lip_upload_stderr.txt")?; let mut p = Popen::create(&command_parts, PopenConfig { //stdout: Redirection::Pipe, diff --git a/src/script_execution/wav2lip_inference_command_OLD.rs b/src/script_execution/wav2lip_inference_command_OLD.rs new file mode 100644 index 0000000000..5e91b2300b --- /dev/null +++ b/src/script_execution/wav2lip_inference_command_OLD.rs @@ -0,0 +1,125 @@ +use crate::AnyhowResult; +use log::{info,warn}; +use std::process::{Command, Stdio}; +use subprocess::{Popen, PopenConfig, Redirection}; +use std::fs::OpenOptions; + +/// This command is used to preprocess the face detection frames from user-submitted video. +/// This should only ever need to run once. The frames can then be uploaded to Buckets and saved. +#[derive(Clone)] +pub struct Wav2LipInferenceCommandOLD { + w2l_directory: String, + script_name: String, + checkpoint_path: String, +} + +impl Wav2LipInferenceCommandOLD { + pub fn new( + w2l_directory: &str, + script_name: &str, + checkpoint_path: &str, + ) -> Self { + Self { + w2l_directory: w2l_directory.to_string(), + script_name: script_name.to_string(), + checkpoint_path: checkpoint_path.to_string(), + } + } + + pub fn execute(&self, + audio_filename: &str, + image_or_video_filename: &str, + output_video_filename: &str, + is_image: bool, + disable_end_bump: bool, + spawn_process: bool) -> AnyhowResult<()> + { + let mut command = String::new(); + + command.push_str("echo 'test'"); + command.push_str(" && "); + command.push_str(&format!("cd {}", self.w2l_directory)); + command.push_str(" && "); + command.push_str("source python/bin/activate"); + command.push_str(" && "); + command.push_str("python "); + command.push_str(&self.script_name); + command.push_str(" --checkpoint_path "); + command.push_str(&self.checkpoint_path); + command.push_str(" --face "); + command.push_str(image_or_video_filename); + command.push_str(" --audio "); + command.push_str(audio_filename); + command.push_str(" --outfile "); + command.push_str(output_video_filename); + + if is_image { + command.push_str(" --is_image "); + } + + info!("Command: {:?}", command); + + let command_parts = [ + "bash", + "-c", + &command + ]; + + if spawn_process { + // NB: This forks and returns immediately. + //let _child_pid = command_builder.spawn()?; + + let stdout_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stdout.txt")?; + + let stderr_file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open("/tmp/stderr.txt")?; + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + stdout: Redirection::File(stdout_file), + stderr: Redirection::File(stderr_file), + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + p.detach(); + + } else { + // NB: This is a blocking call. + /*let output = command_builder.output()?; + + info!("Output status: {}", output.status); + info!("Stdout: {:?}", String::from_utf8(output.stdout)); + error!("Stderr: {:?}", String::from_utf8(output.stderr)); + + if !output.status.success() { + bail!("Bad error code: {:?}", output.status); + }*/ + + let mut p = Popen::create(&command_parts, PopenConfig { + //stdout: Redirection::Pipe, + //stderr: Redirection::Pipe, + ..Default::default() + })?; + + info!("Pid : {:?}", p.pid()); + + let exit_status = p.wait()?; + + info!("Exit status: {:?}", exit_status); + } + + Ok(()) + } +} diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index d438772d8f..a20c7cb5d4 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -47,17 +47,19 @@ use crate::script_execution::imagemagick_generate_preview_image_command::Imagema const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; const ENV_REGION_NAME : &'static str = "REGION_NAME"; -const ENV_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; // Buckets (private data) const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; // Buckets (public data) const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; +// Various bucket roots +const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; + // Python code const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; -const ENV_SCRIPT_NAME : &'static str = "W2L_SCRIPT_NAME"; +const ENV_DOWNLOAD_SCRIPT_NAME : &'static str = "W2L_DOWNLOAD_SCRIPT_NAME"; // NB: sqlx::query is spammy and logs all queries as "info"-level const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; @@ -104,12 +106,14 @@ async fn main() -> AnyhowResult<()> { let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; - let bucket_root = easyenv::get_env_string_required(ENV_BUCKET_ROOT)?; // Private and Public Buckets let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + // Bucket roots + let bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?; + let private_bucket_client = BucketClient::create( &access_key, &secret_key, @@ -127,7 +131,7 @@ async fn main() -> AnyhowResult<()> { )?; let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; - let py_script_name = easyenv::get_env_string_required(ENV_SCRIPT_NAME)?; + let py_script_name = easyenv::get_env_string_required(ENV_DOWNLOAD_SCRIPT_NAME)?; let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; let w2l_preprecess_command = Wav2LipPreprocessClient::new( diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 559e546bd4..7434302279 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -1,7 +1,498 @@ +#![deny(unused_must_use)] +#![allow(dead_code)] +#![allow(unused_mut)] +#![allow(unused_imports)] +#![warn(unused_must_use)] +//#![allow(warnings)] + +#[macro_use] extern crate serde_derive; + +pub mod buckets; +pub mod job_queries; +pub mod script_execution; pub mod util; +use anyhow::anyhow; +use chrono::Utc; +use crate::buckets::bucket_client::BucketClient; +use crate::buckets::bucket_paths::hash_to_bucket_path; +use crate::buckets::file_hashing::get_file_hash; +use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; +use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; +use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; +use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; +use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; +use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::util::anyhow_result::AnyhowResult; +use crate::util::filesystem::check_directory_exists; +use crate::util::filesystem::check_file_exists; +use crate::util::random_crockford_token::random_crockford_token; +use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; +use log::{warn, info}; +use ring::digest::{Context, Digest, SHA256}; +use sqlx::MySqlPool; +use sqlx::mysql::MySqlPoolOptions; +use std::fs::{File, metadata}; +use std::io::{BufReader, Read}; +use std::path::{PathBuf, Path}; +use std::process::Command; +use std::time::Duration; +use tempdir::TempDir; +use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; +use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; +use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; +use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; + +// Buckets (shared config) +const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; +const ENV_SECRET_KEY : &'static str = "SECRET_KEY"; +const ENV_REGION_NAME : &'static str = "REGION_NAME"; + +// Buckets (private data) +const ENV_PRIVATE_BUCKET_NAME : &'static str = "W2L_PRIVATE_DOWNLOAD_BUCKET_NAME"; +// Buckets (public data) +const ENV_PUBLIC_BUCKET_NAME : &'static str = "W2L_PUBLIC_DOWNLOAD_BUCKET_NAME"; + +// Various bucket roots +const ENV_DOWNLOAD_BUCKET_ROOT : &'static str = "W2L_DOWNLOAD_BUCKET_ROOT"; +const ENV_AUDIO_UPLOADS_BUCKET_ROOT : &'static str = "AUDIO_UPLOADS_BUCKET_ROOT"; +const ENV_INFERENCE_OUTPUT_BUCKET_ROOT : &'static str = "W2L_INFERENCE_OUTPUT_BUCKET_ROOT"; + +// Python code +const ENV_CODE_DIRECTORY : &'static str = "W2L_CODE_DIRECTORY"; +const ENV_MODEL_CHECKPOINT : &'static str = "W2L_MODEL_CHECKPOINT"; +const ENV_INFERENCE_SCRIPT_NAME : &'static str = "W2L_INFERENCE_SCRIPT_NAME"; + +// NB: sqlx::query is spammy and logs all queries as "info"-level +const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info,sqlx::query=warn"; +const DEFAULT_TEMP_DIR: &'static str = "/tmp"; + +struct Inferencer { + pub download_temp_directory: PathBuf, + pub mysql_pool: MySqlPool, + + pub private_bucket_client: BucketClient, + pub public_bucket_client: BucketClient, + + pub google_drive_downloader: GoogleDriveDownloadCommand, + pub w2l_inference: Wav2LipInferenceCommand, + //pub ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand, + //pub ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand, + //pub imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand, + + // Command to run + pub inference_script: String, + + // Root to retrieve W2L templates (public and private) + pub w2l_template_uploads_bucket_root: String, + // Root to retrieve uploaded audio (public and private) + pub audio_uploads_bucket_root: String, + // Root to upload inference output + pub inference_output_bucket_root: String, +} + +#[tokio::main] +async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + // NB: Do not check this secrets-containing dotenv file into VCS. + // This file should only contain *development* secrets, never production. + let _ = dotenv::from_filename(".env-secrets").ok(); + + info!("Obtaining hostname..."); + + let server_hostname = hostname::get() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or("w2l-inference-job".to_string()); + + info!("Hostname: {}", &server_hostname); + + // Bucket stuff (shared) + let access_key = easyenv::get_env_string_required(ENV_ACCESS_KEY)?; + let secret_key = easyenv::get_env_string_required(ENV_SECRET_KEY)?; + let region_name = easyenv::get_env_string_required(ENV_REGION_NAME)?; + + // Private and Public Buckets + let private_bucket_name = easyenv::get_env_string_required(ENV_PRIVATE_BUCKET_NAME)?; + let public_bucket_name = easyenv::get_env_string_required(ENV_PUBLIC_BUCKET_NAME)?; + + // Bucket roots + let w2l_template_bucket_root = easyenv::get_env_string_required(ENV_DOWNLOAD_BUCKET_ROOT)?; + let audio_bucket_root = easyenv::get_env_string_required(ENV_AUDIO_UPLOADS_BUCKET_ROOT)?; + let inference_output_bucket_root = easyenv::get_env_string_required(ENV_INFERENCE_OUTPUT_BUCKET_ROOT)?; + + let private_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + )?; + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + )?; + + let py_code_directory = easyenv::get_env_string_required(ENV_CODE_DIRECTORY)?; + let py_script_name = easyenv::get_env_string_required(ENV_INFERENCE_SCRIPT_NAME)?; + let py_model_checkpoint = easyenv::get_env_string_required(ENV_MODEL_CHECKPOINT)?; + + let w2l_inference_command = Wav2LipInferenceCommand::new( + &py_code_directory, + &py_script_name, + &py_model_checkpoint, + ); + + let temp_directory = easyenv::get_env_string_or_default( + "DOWNLOAD_TEMP_DIR", + DEFAULT_TEMP_DIR); + + // TODO: In the future, we may want to enable downloading images or audio files. + //let download_script = easyenv::get_env_string_or_default( + // "DOWNLOAD_SCRIPT", + // "./scripts/download_gdrive.py"); + // + //let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + + let temp_directory = PathBuf::from(temp_directory); + + check_directory_exists(&temp_directory)?; + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + "mysql://root:root@localhost/storyteller"); + + info!("Connecting to database..."); + + let mysql_pool = MySqlPoolOptions::new() + .max_connections(5) + .connect(&db_connection_string) + .await?; + + let inferencer = Inferencer { + download_temp_directory: temp_directory, + mysql_pool, + public_bucket_client, + private_bucket_client, + inference_script, + google_drive_downloader, + //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, + //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, + //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, + w2l_inference: w2l_inference_command, + w2l_template_uploads_bucket_root: w2l_template_bucket_root.to_string(), + audio_uploads_bucket_root: audio_bucket_root.to_string(), + inference_output_bucket_root: inference_output_bucket_root.to_string(), + }; + + main_loop(inferencer).await; + + Ok(()) +} + +const START_TIMEOUT_MILLIS : u64 = 500; +const INCREASE_TIMEOUT_MILLIS : u64 = 1000; + +async fn main_loop(inferencer: Inferencer) { + let mut timeout_millis = START_TIMEOUT_MILLIS; + + loop { + let num_records = 1; + + let query_result = query_w2l_template_upload_job_records( + &inferencer.mysql_pool, + num_records) + .await; + + let jobs = match query_result { + Ok(jobs) => jobs, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + }; + + if jobs.is_empty() { + info!("No jobs!"); + std::thread::sleep(Duration::from_millis(1500)); + continue; + } + + let result = process_jobs(&inferencer, jobs).await; + + match result { + Ok(_) => {}, + Err(e) => { + warn!("Error querying jobs: {:?}", e); + std::thread::sleep(Duration::from_millis(timeout_millis)); + timeout_millis += INCREASE_TIMEOUT_MILLIS; + continue; + } + } + + timeout_millis = START_TIMEOUT_MILLIS; // reset + + std::thread::sleep(Duration::from_millis(500)); + } +} + +async fn process_jobs(inferencer: &Inferencer, jobs: Vec) -> AnyhowResult<()> { + for job in jobs.into_iter() { + let result = process_job(inferencer, &job).await; + match result { + Ok(_) => {}, + Err(e) => { + warn!("Failure to process job: {:?}", e); + let failure_reason = ""; + let _r = mark_w2l_inference_job_failure( + &inferencer.mysql_pool, + &job, + failure_reason) + .await; + } + } + } + + Ok(()) +} + +#[derive(Deserialize)] +struct FileMetadata { + pub is_video: bool, + pub width: u32, + pub height: u32, + pub num_frames: u64, + pub fps: Option, + pub duration_millis: Option, + pub mimetype: Option, + pub file_size_bytes: u64, +} + +fn read_metadata_file(filename: &str) -> AnyhowResult { + let mut file = File::open(filename)?; + let mut buffer = String::new(); + file.read_to_string(&mut buffer)?; + Ok(serde_json::from_str(&buffer)?) +} + +async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> AnyhowResult<()> { + + // TODO: 1. Check if w2l model is downloaded and download it + // TODO: 2. Check if w2l template faces are downloaded and download it + // TODO: 3. Download audio + + + // TODO: 1. Mark processing. + // TODO: 2. Download. (DONE) + // TODO: 3. Process template with face detection (DONE) + // TODO: 4. Determine if picture or video + // TODO: 5. Take a screenshot/gif + // TODO: 6. Upload all (partially done). + // TODO: 7. Save record. (DONE) + // TODO: 8. Mark job done. (DONE) + + let temp_dir = format!("temp_{}", job.id); + let temp_dir = TempDir::new(&temp_dir)?; + + let download_url = job.download_url.as_ref() + .map(|c| c.to_string()) + .unwrap_or("".to_string()); + + // ==================== DOWNLOAD FILE ==================== // + + info!("Calling downloader..."); + let download_filename = inferencer.google_drive_downloader + .download_file(&download_url, &temp_dir).await?; + + // ==================== PROCESS FACES ==================== // + + // This is the Python Pickle file with all the face frames. + // We'll retain it forever since it's expensive to compute. + let cached_faces_filename = format!("{}_detected_faces.pickle", &download_filename); + + // This is a file that we'll use to determine if the file is an image or video. + let output_metadata_filename = format!("{}_metadata.json", &download_filename); + + let is_image = false; // TODO: Don't always treat as video. + let spawn_process = false; + + inferencer.w2l_processor.execute( + &download_filename, + &cached_faces_filename, + &output_metadata_filename, + is_image, + spawn_process)?; + + // ==================== CHECK ALL FILES EXIST AND GET METADATA ==================== // + + let video_or_image_path = PathBuf::from(&download_filename); + let cached_faces_path = &PathBuf::from(&cached_faces_filename); + let output_metadata_path = &PathBuf::from(&output_metadata_filename); + + info!("Checking that both files exist (original source + cached faces) ..."); + + check_file_exists(&video_or_image_path)?; + check_file_exists(&cached_faces_path)?; + check_file_exists(&output_metadata_path)?; + + let file_metadata = read_metadata_file(&output_metadata_filename)?; + + // ==================== BASE OBJECT NAMES BASED ON HASH ==================== // + + let private_bucket_hash = get_file_hash(&download_filename)?; + + info!("File hash: {}", private_bucket_hash); + + // Full path to video/image + let full_object_path = hash_to_bucket_path( + &private_bucket_hash, + Some(&inferencer.w2l_template_uploads_bucket_root))?; + + // ==================== GENERATE VIDEO PREVIEWS ==================== // + + let mut maybe_image_preview_filename : Option = None; + let mut maybe_image_preview_object_name : Option = None; + + let mut maybe_video_preview_filename : Option = None; + let mut maybe_video_preview_object_name : Option = None; + + if file_metadata.is_video { + let preview_filename = format!("{}_preview.webp", &download_filename); + + inferencer.ffmpeg_video_preview_generator.execute( + &download_filename, + &preview_filename, + 500, + 500, + true, + false + )?; + + let video_preview_path = PathBuf::from(&preview_filename); + check_file_exists(&video_preview_path)?; + + let preview_object_path = format!("{}_preview.webp", full_object_path); + maybe_video_preview_object_name = Some(preview_object_path); + maybe_video_preview_filename = Some(video_preview_path); + + } else { + let preview_filename = format!("{}_preview.webp", &download_filename); + + inferencer.imagemagick_image_preview_generator.execute( + &download_filename, + &preview_filename, + 500, + 500, + false + )?; + + let image_preview_path = PathBuf::from(&preview_filename); + check_file_exists(&image_preview_path)?; + + let preview_object_path = format!("{}_preview.webp", full_object_path); + maybe_image_preview_object_name = Some(preview_object_path); + maybe_image_preview_filename = Some(image_preview_path); + } + + // ==================== UPLOAD TO BUCKETS ==================== // + + info!("Image/video destination bucket path: {}", full_object_path); + + // Full path to cached faces + let full_object_path_cached_faces = format!("{}_detected_faces.pickle", full_object_path); + + info!("Cached faces destination bucket path: {}", full_object_path_cached_faces); + + info!("Uploading image/video..."); + + let original_mime_type = file_metadata.mimetype + .as_deref() + .unwrap_or("application/octet-stream"); + + inferencer.private_bucket_client.upload_filename_with_content_type( + &full_object_path, + &video_or_image_path, + original_mime_type).await?; + + inferencer.public_bucket_client.upload_filename_with_content_type( + &full_object_path, + &video_or_image_path, + original_mime_type).await?; + + info!("Uploading cached faces..."); + inferencer.private_bucket_client.upload_filename( + &full_object_path_cached_faces, + &cached_faces_path).await?; + + // TODO: Fix this ugh. + if let Some(image_preview_filename) = maybe_image_preview_filename.as_deref() { + if let Some(image_preview_object_name) = maybe_image_preview_object_name.as_deref() { + info!("Uploading image preview..."); + inferencer.private_bucket_client.upload_filename_with_content_type( + &image_preview_object_name, + image_preview_filename, + "image/webp").await?; + + info!("Uploading image preview..."); + inferencer.public_bucket_client.upload_filename_with_content_type( + &image_preview_object_name, + image_preview_filename, + "image/webp").await?; + } + } + + // TODO: Fix this ugh. + if let Some(video_preview_filename) = maybe_video_preview_filename.as_deref() { + if let Some(video_preview_object_name) = maybe_video_preview_object_name.as_deref() { + info!("Uploading video preview..."); + inferencer.private_bucket_client.upload_filename_with_content_type( + &video_preview_object_name, + video_preview_filename, + "image/webp").await?; + + info!("Uploading video preview..."); + inferencer.public_bucket_client.upload_filename_with_content_type( + &video_preview_object_name, + video_preview_filename, + "image/webp").await?; + } + } + + // ==================== SAVE RECORDS ==================== // + + let template_type = if file_metadata.is_video { "video" } else { "image" }; + + info!("Saving w2l inference record..."); + let id = insert_w2l_result( + &inferencer.mysql_pool, + template_type, + job, + &private_bucket_hash, + &full_object_path, + &full_object_path_cached_faces, + maybe_image_preview_object_name.as_deref(), + maybe_video_preview_object_name.as_deref(), + file_metadata.file_size_bytes, + file_metadata.mimetype.as_deref(), + file_metadata.width, + file_metadata.height, + file_metadata.num_frames, + file_metadata.fps.unwrap_or(0.0f32), + file_metadata.duration_millis.unwrap_or(0)) + .await?; + + info!("Job {} complete success! Downloaded, processed, and uploaded. Saved model record: {}", + job.id, id); + + mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?; -fn main() -> AnyhowResult<()> { Ok(()) }