Move last remaining jobs out of storyteller-web and into own crates

- w2l-inference-job
- w2l-download-job

These are now in their own independent crates, which should be easier to
do build caching on.

Cargo.lock

@@ -4805,6 +4805,68 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
 
+[[package]]
+name = "w2l-download-job"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "clap",
+ "config",
+ "container_common",
+ "database_queries",
+ "dotenv",
+ "easyenv",
+ "google_drive_common",
+ "hostname",
+ "hyper 0.14.14",
+ "jobs_common",
+ "lfu",
+ "log",
+ "newrelic-telemetry",
+ "r2d2_redis",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "sqlx",
+ "storage_buckets_common",
+ "subprocess",
+ "tempdir",
+ "tokio 1.13.0",
+ "tts_common",
+]
+
+[[package]]
+name = "w2l-inference-job"
+version = "0.0.1"
+dependencies = [
+ "anyhow",
+ "chrono",
+ "clap",
+ "config",
+ "container_common",
+ "database_queries",
+ "dotenv",
+ "easyenv",
+ "google_drive_common",
+ "hostname",
+ "hyper 0.14.14",
+ "jobs_common",
+ "lfu",
+ "log",
+ "newrelic-telemetry",
+ "r2d2_redis",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "sqlx",
+ "storage_buckets_common",
+ "subprocess",
+ "tempdir",
+ "tokio 1.13.0",
+ "tts_common",
+]
+
 [[package]]
 name = "want"
 version = "0.3.0"

Cargo.toml

@@ -20,6 +20,8 @@ members = [
   "crates/service/tts_download_job",
   "crates/service/tts_inference_job",
   "crates/service/twitch_pubsub_subscriber",
+  "crates/service/w2l_download_job",
+  "crates/service/w2l_inference_job",
   "crates/service/websocket_gateway",
 ]
 

crates/lib/database_queries/src/queries/w2l/mod.rs

@@ -1,4 +1,5 @@
 pub mod stats;
+pub mod w2l_download_jobs;
 pub mod w2l_inference_jobs;
 pub mod w2l_results;
 pub mod w2l_templates;

crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/mod.rs

@@ -0,0 +1,2 @@
+// TODO: The "job queries" were moved here; now break them down into several files.
+pub mod w2l_download_job_queries;

crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/w2l_download_job_queries.rs

@@ -5,11 +5,12 @@
 use anyhow::anyhow;
 use chrono::Utc;
 use container_common::anyhow_result::AnyhowResult;
-use database_queries::tokens::Tokens;
+use crate::tokens::Tokens;
 use sqlx::MySqlPool;
 
-// TODO(2022-02-10): Move these queries to the 'database_queries' crate
-//  I moved tts_inference_job queries out already.
+// TODO(2022-08-04): These were moved into the 'database_queries' crate, but they need
+//  to be split up into several modules for better maintainability. cf the already moved
+//  `tts_inference_job` queries.
 
 /// table: w2l_template_upload_jobs
 #[derive(Debug)]

crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs

@@ -1 +1,2 @@
-// TODO: Move the "job queries" here and break them down into several files.
+// TODO: The "job queries" were moved here; now break them down into several files.
+pub mod w2l_inference_job_queries;

crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/w2l_inference_job_queries.rs

@@ -5,14 +5,15 @@
 use anyhow::anyhow;
 use chrono::{Utc, DateTime};
 use container_common::anyhow_result::AnyhowResult;
-use database_queries::column_types::record_visibility::RecordVisibility;
-use database_queries::tokens::Tokens;
+use crate::column_types::record_visibility::RecordVisibility;
+use crate::tokens::Tokens;
 use log::{warn, info};
 use sqlx::MySqlPool;
 use std::path::Path;
 
-// TODO(2022-02-10): Move these queries to the 'database_queries' crate
-//  I moved tts_inference_job queries out already.
+// TODO(2022-08-04): These were moved into the 'database_queries' crate, but they need
+//  to be split up into several modules for better maintainability. cf the already moved
+//  `tts_inference_job` queries.
 
 /// table: w2l_inference_jobs
 #[derive(Debug)]
@@ -75,7 +76,7 @@ SELECT
   creator_ip_address,
   maybe_creator_user_token,
 
-  creator_set_visibility as `creator_set_visibility: database_queries::column_types::record_visibility::RecordVisibility`,
+  creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`,
   disable_end_bump,
   disable_watermark,
 

crates/service/storyteller_web/Cargo.toml

@@ -13,15 +13,6 @@ publish = false
 name = "storyteller-web"
 path = "src/main.rs"
 
-# Jobs for managing downloads and inference
-[[bin]]
-name = "w2l-download-job"
-path = "src/w2l_download_job.rs"
-
-[[bin]]
-name = "w2l-inference-job"
-path = "src/w2l_inference_job.rs"
-
 [dependencies]
 
 # Internal

crates/service/storyteller_web/src/job_queries/mod.rs

@@ -1,5 +0,0 @@
-// TODO(2022-02-10): Move these queries to the 'database_queries' crate
-//  I moved tts_inference_job and tts_download_job queries out already.
-
-pub mod w2l_download_job_queries;
-pub mod w2l_inference_job_queries;

crates/service/storyteller_web/src/main.rs

@@ -27,10 +27,6 @@ pub mod threads;
 pub mod util;
 pub mod validations;
 
-// TODO: Eventually move all of these to the `database_queries` crate and no longer write inline MySQL.
-// NB: This is included so sqlx can generate all the queries.
-mod job_queries;
-
 // TODO: Eventually move all of these to the `database_queries` crate and no longer write inline MySQL.
 // NB: Also so sqlx codegens everything.
 // Not sure if this is strictly necessary.

crates/service/w2l_download_job/Cargo.toml

@@ -0,0 +1,61 @@
+[package]
+name = "w2l-download-job"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[[bin]]
+name = "w2l-download-job"
+path = "src/main.rs"
+
+[dependencies]
+
+# Internal
+config = { path = "../../lib/config" }
+container_common = { path = "../../lib/container_common" }
+database_queries = { path = "../../lib/database_queries" }
+google_drive_common = { path = "../../lib/google_drive_common" }
+jobs_common = { path = "../../lib/jobs_common" }
+storage_buckets_common = { path = "../../lib/storage_buckets_common" }
+tts_common = { path = "../../lib/tts_common" }
+
+# Vendored code
+newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" }
+
+# External
+anyhow = "1.0.40"
+chrono = { version = "0.4.19", features = ["serde"] }
+clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+dotenv = "0.15.0"
+easyenv = "0.0.6"
+hostname = "0.3.1"
+lfu = "0.2.5"
+log = "0.4.14"
+subprocess = "0.2.6"
+tempdir = "0.3.7"
+
+# MySql Database
+# "offline" supports typesafe builds without a live database
+# "rustls" doesn't appear to work with IP addresses, per:
+# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
+sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
+
+# Redis
+r2d2_redis = "0.14.0"
+
+# Runtime
+# NB: We switched to Tokio from `async-std` for broader ecosystem support.
+tokio = { version = "1.6.0", features = ["macros"] }
+
+# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime
+# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper.
+hyper = { version = "0.14.14" }
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+serde_json = "1.0.64"

crates/service/w2l_download_job/src/main.rs

@@ -10,9 +10,7 @@
 
 #[macro_use] extern crate serde_derive;
 
-pub mod job_queries;
 pub mod script_execution;
-pub mod util;
 
 use anyhow::{anyhow, Error};
 use chrono::Utc;
@@ -25,27 +23,25 @@ use container_common::filesystem::check_file_exists::check_file_exists;
 use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
 use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
 use container_common::hashing::hash_file_sha2::hash_file_sha2;
-use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord;
-use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending;
-use crate::job_queries::w2l_download_job_queries::insert_w2l_template;
-use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done;
-use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure;
-use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_permanently_dead;
-use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records;
 use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
 use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
 use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
 use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError};
-use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
 use database_queries::mediators::badge_granter::BadgeGranter;
 use database_queries::mediators::firehose_publisher::FirehosePublisher;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::W2lTemplateUploadJobRecord;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::grab_job_lock_and_mark_pending;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::insert_w2l_template;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::mark_w2l_template_upload_job_done;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::mark_w2l_template_upload_job_failure;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::mark_w2l_template_upload_job_permanently_dead;
+use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::query_w2l_template_upload_job_records;
 use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
 use jobs_common::noop_logger::NoOpLogger;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
 use log::{warn, info};
 use r2d2_redis::RedisConnectionManager;
 use r2d2_redis::r2d2;
-use ring::digest::{Context, Digest, SHA256};
 use sqlx::MySqlPool;
 use sqlx::mysql::MySqlPoolOptions;
 use std::fs::{File, metadata};

crates/service/w2l_download_job/src/script_execution/mod.rs

@@ -1,5 +1,4 @@
 pub mod ffmpeg_generate_preview_image_command;
 pub mod ffmpeg_generate_preview_video_command;
 pub mod imagemagick_generate_preview_image_command;
-pub mod wav2lip_inference_command;
-pub mod wav2lip_process_upload_command;
+pub mod wav2lip_process_upload_command;

crates/service/w2l_inference_job/Cargo.toml

@@ -0,0 +1,61 @@
+[package]
+name = "w2l-inference-job"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[[bin]]
+name = "w2l-inference-job"
+path = "src/main.rs"
+
+[dependencies]
+
+# Internal
+config = { path = "../../lib/config" }
+container_common = { path = "../../lib/container_common" }
+database_queries = { path = "../../lib/database_queries" }
+google_drive_common = { path = "../../lib/google_drive_common" }
+jobs_common = { path = "../../lib/jobs_common" }
+storage_buckets_common = { path = "../../lib/storage_buckets_common" }
+tts_common = { path = "../../lib/tts_common" }
+
+# Vendored code
+newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" }
+
+# External
+anyhow = "1.0.40"
+chrono = { version = "0.4.19", features = ["serde"] }
+clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
+dotenv = "0.15.0"
+easyenv = "0.0.6"
+hostname = "0.3.1"
+lfu = "0.2.5"
+log = "0.4.14"
+subprocess = "0.2.6"
+tempdir = "0.3.7"
+
+# MySql Database
+# "offline" supports typesafe builds without a live database
+# "rustls" doesn't appear to work with IP addresses, per:
+# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
+sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
+
+# Redis
+r2d2_redis = "0.14.0"
+
+# Runtime
+# NB: We switched to Tokio from `async-std` for broader ecosystem support.
+tokio = { version = "1.6.0", features = ["macros"] }
+
+# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime
+# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper.
+hyper = { version = "0.14.14" }
+
+# Serialization
+serde = "1.0.125"
+serde_derive = "1.0.125"
+serde_json = "1.0.64"

crates/service/w2l_inference_job/src/main.rs

@@ -10,9 +10,7 @@
 
 #[macro_use] extern crate serde_derive;
 
-pub mod job_queries;
-pub mod script_execution;
-pub mod util;
+mod script_execution;
 
 use anyhow::anyhow;
 use chrono::Utc;
@@ -24,18 +22,15 @@ use container_common::filesystem::check_directory_exists::check_directory_exists
 use container_common::filesystem::check_file_exists::check_file_exists;
 use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
 use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
-use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord;
-use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token;
-use crate::job_queries::w2l_inference_job_queries::grab_job_lock_and_mark_pending;
-use crate::job_queries::w2l_inference_job_queries::insert_w2l_result;
-use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done;
-use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure;
-use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records;
-use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
-use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
-use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
 use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand;
 use database_queries::mediators::firehose_publisher::FirehosePublisher;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::W2lInferenceJobRecord;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::get_w2l_template_by_token;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::grab_job_lock_and_mark_pending;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::insert_w2l_result;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::mark_w2l_inference_job_done;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::mark_w2l_inference_job_failure;
+use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::query_w2l_inference_job_records;
 use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
 use jobs_common::noop_logger::NoOpLogger;
 use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
@@ -240,9 +235,6 @@ async fn main() -> AnyhowResult<()> {
     private_bucket_client,
     inference_script,
     google_drive_downloader,
-    //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {},
-    //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {},
-    //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {},
     w2l_inference: w2l_inference_command,
     bucket_path_unifier: BucketPathUnifier::default_paths(),
     semi_persistent_cache,

crates/service/w2l_inference_job/src/script_execution/mod.rs

@@ -0,0 +1 @@
+pub mod wav2lip_inference_command;