diff --git a/Cargo.lock b/Cargo.lock index 2f69b71c1c..808d284926 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4805,6 +4805,68 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "w2l-download-job" +version = "0.0.1" +dependencies = [ + "anyhow", + "chrono", + "clap", + "config", + "container_common", + "database_queries", + "dotenv", + "easyenv", + "google_drive_common", + "hostname", + "hyper 0.14.14", + "jobs_common", + "lfu", + "log", + "newrelic-telemetry", + "r2d2_redis", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "storage_buckets_common", + "subprocess", + "tempdir", + "tokio 1.13.0", + "tts_common", +] + +[[package]] +name = "w2l-inference-job" +version = "0.0.1" +dependencies = [ + "anyhow", + "chrono", + "clap", + "config", + "container_common", + "database_queries", + "dotenv", + "easyenv", + "google_drive_common", + "hostname", + "hyper 0.14.14", + "jobs_common", + "lfu", + "log", + "newrelic-telemetry", + "r2d2_redis", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "storage_buckets_common", + "subprocess", + "tempdir", + "tokio 1.13.0", + "tts_common", +] + [[package]] name = "want" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 17e7c57a3a..ba0c482263 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,8 @@ members = [ "crates/service/tts_download_job", "crates/service/tts_inference_job", "crates/service/twitch_pubsub_subscriber", + "crates/service/w2l_download_job", + "crates/service/w2l_inference_job", "crates/service/websocket_gateway", ] diff --git a/crates/lib/database_queries/src/queries/w2l/mod.rs b/crates/lib/database_queries/src/queries/w2l/mod.rs index 9cd08d628b..dfaeb47d6d 100644 --- a/crates/lib/database_queries/src/queries/w2l/mod.rs +++ b/crates/lib/database_queries/src/queries/w2l/mod.rs @@ -1,4 +1,5 @@ pub mod stats; +pub mod w2l_download_jobs; pub mod w2l_inference_jobs; pub mod w2l_results; pub mod w2l_templates; \ No newline at end of file diff --git a/crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/mod.rs b/crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/mod.rs new file mode 100644 index 0000000000..5f23a6928e --- /dev/null +++ b/crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/mod.rs @@ -0,0 +1,2 @@ +// TODO: The "job queries" were moved here; now break them down into several files. +pub mod w2l_download_job_queries; diff --git a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs b/crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/w2l_download_job_queries.rs similarity index 96% rename from crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/w2l_download_job_queries.rs index fd8c8347ad..d0c23a8739 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_download_job_queries.rs +++ b/crates/lib/database_queries/src/queries/w2l/w2l_download_jobs/w2l_download_job_queries.rs @@ -5,11 +5,12 @@ use anyhow::anyhow; use chrono::Utc; use container_common::anyhow_result::AnyhowResult; -use database_queries::tokens::Tokens; +use crate::tokens::Tokens; use sqlx::MySqlPool; -// TODO(2022-02-10): Move these queries to the 'database_queries' crate -// I moved tts_inference_job queries out already. +// TODO(2022-08-04): These were moved into the 'database_queries' crate, but they need +// to be split up into several modules for better maintainability. cf the already moved +// `tts_inference_job` queries. /// table: w2l_template_upload_jobs #[derive(Debug)] diff --git a/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs b/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs index 4e398c43ff..bcfb0fad9b 100644 --- a/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs +++ b/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/mod.rs @@ -1 +1,2 @@ -// TODO: Move the "job queries" here and break them down into several files. \ No newline at end of file +// TODO: The "job queries" were moved here; now break them down into several files. +pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs b/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/w2l_inference_job_queries.rs similarity index 96% rename from crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs rename to crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/w2l_inference_job_queries.rs index 0db4a6b258..61ac44ba89 100644 --- a/crates/service/storyteller_web/src/job_queries/w2l_inference_job_queries.rs +++ b/crates/lib/database_queries/src/queries/w2l/w2l_inference_jobs/w2l_inference_job_queries.rs @@ -5,14 +5,15 @@ use anyhow::anyhow; use chrono::{Utc, DateTime}; use container_common::anyhow_result::AnyhowResult; -use database_queries::column_types::record_visibility::RecordVisibility; -use database_queries::tokens::Tokens; +use crate::column_types::record_visibility::RecordVisibility; +use crate::tokens::Tokens; use log::{warn, info}; use sqlx::MySqlPool; use std::path::Path; -// TODO(2022-02-10): Move these queries to the 'database_queries' crate -// I moved tts_inference_job queries out already. +// TODO(2022-08-04): These were moved into the 'database_queries' crate, but they need +// to be split up into several modules for better maintainability. cf the already moved +// `tts_inference_job` queries. /// table: w2l_inference_jobs #[derive(Debug)] @@ -75,7 +76,7 @@ SELECT creator_ip_address, maybe_creator_user_token, - creator_set_visibility as `creator_set_visibility: database_queries::column_types::record_visibility::RecordVisibility`, + creator_set_visibility as `creator_set_visibility: crate::column_types::record_visibility::RecordVisibility`, disable_end_bump, disable_watermark, diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index 100f42fe2f..8db78ff6f9 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -13,15 +13,6 @@ publish = false name = "storyteller-web" path = "src/main.rs" -# Jobs for managing downloads and inference -[[bin]] -name = "w2l-download-job" -path = "src/w2l_download_job.rs" - -[[bin]] -name = "w2l-inference-job" -path = "src/w2l_inference_job.rs" - [dependencies] # Internal diff --git a/crates/service/storyteller_web/src/job_queries/mod.rs b/crates/service/storyteller_web/src/job_queries/mod.rs deleted file mode 100644 index 62587a06ed..0000000000 --- a/crates/service/storyteller_web/src/job_queries/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -// TODO(2022-02-10): Move these queries to the 'database_queries' crate -// I moved tts_inference_job and tts_download_job queries out already. - -pub mod w2l_download_job_queries; -pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index 379f929852..f9fa06d5ed 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -27,10 +27,6 @@ pub mod threads; pub mod util; pub mod validations; -// TODO: Eventually move all of these to the `database_queries` crate and no longer write inline MySQL. -// NB: This is included so sqlx can generate all the queries. -mod job_queries; - // TODO: Eventually move all of these to the `database_queries` crate and no longer write inline MySQL. // NB: Also so sqlx codegens everything. // Not sure if this is strictly necessary. diff --git a/crates/service/w2l_download_job/Cargo.toml b/crates/service/w2l_download_job/Cargo.toml new file mode 100644 index 0000000000..2b0dd5df88 --- /dev/null +++ b/crates/service/w2l_download_job/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "w2l-download-job" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "w2l-download-job" +path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } +database_queries = { path = "../../lib/database_queries" } +google_drive_common = { path = "../../lib/google_drive_common" } +jobs_common = { path = "../../lib/jobs_common" } +storage_buckets_common = { path = "../../lib/storage_buckets_common" } +tts_common = { path = "../../lib/tts_common" } + +# Vendored code +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } + +# External +anyhow = "1.0.40" +chrono = { version = "0.4.19", features = ["serde"] } +clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +dotenv = "0.15.0" +easyenv = "0.0.6" +hostname = "0.3.1" +lfu = "0.2.5" +log = "0.4.14" +subprocess = "0.2.6" +tempdir = "0.3.7" + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } + +# Redis +r2d2_redis = "0.14.0" + +# Runtime +# NB: We switched to Tokio from `async-std` for broader ecosystem support. +tokio = { version = "1.6.0", features = ["macros"] } + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.14.14" } + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" diff --git a/crates/service/storyteller_web/src/w2l_download_job.rs b/crates/service/w2l_download_job/src/main.rs similarity index 96% rename from crates/service/storyteller_web/src/w2l_download_job.rs rename to crates/service/w2l_download_job/src/main.rs index 377e385429..caa675b665 100644 --- a/crates/service/storyteller_web/src/w2l_download_job.rs +++ b/crates/service/w2l_download_job/src/main.rs @@ -10,9 +10,7 @@ #[macro_use] extern crate serde_derive; -pub mod job_queries; pub mod script_execution; -pub mod util; use anyhow::{anyhow, Error}; use chrono::Utc; @@ -25,27 +23,25 @@ use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use container_common::hashing::hash_file_sha2::hash_file_sha2; -use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; -use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; -use crate::job_queries::w2l_download_job_queries::insert_w2l_template; -use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_done; -use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_failure; -use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_permanently_dead; -use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records; use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError}; -use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::mediators::badge_granter::BadgeGranter; use database_queries::mediators::firehose_publisher::FirehosePublisher; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::W2lTemplateUploadJobRecord; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::grab_job_lock_and_mark_pending; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::insert_w2l_template; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::mark_w2l_template_upload_job_done; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::mark_w2l_template_upload_job_failure; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::mark_w2l_template_upload_job_permanently_dead; +use database_queries::queries::w2l::w2l_download_jobs::w2l_download_job_queries::query_w2l_template_upload_job_records; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; -use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; diff --git a/crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs b/crates/service/w2l_download_job/src/script_execution/ffmpeg_generate_preview_image_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_image_command.rs rename to crates/service/w2l_download_job/src/script_execution/ffmpeg_generate_preview_image_command.rs diff --git a/crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs b/crates/service/w2l_download_job/src/script_execution/ffmpeg_generate_preview_video_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/ffmpeg_generate_preview_video_command.rs rename to crates/service/w2l_download_job/src/script_execution/ffmpeg_generate_preview_video_command.rs diff --git a/crates/service/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs b/crates/service/w2l_download_job/src/script_execution/imagemagick_generate_preview_image_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/imagemagick_generate_preview_image_command.rs rename to crates/service/w2l_download_job/src/script_execution/imagemagick_generate_preview_image_command.rs diff --git a/crates/service/storyteller_web/src/script_execution/mod.rs b/crates/service/w2l_download_job/src/script_execution/mod.rs similarity index 66% rename from crates/service/storyteller_web/src/script_execution/mod.rs rename to crates/service/w2l_download_job/src/script_execution/mod.rs index c11d8a59a4..7de610fa4e 100644 --- a/crates/service/storyteller_web/src/script_execution/mod.rs +++ b/crates/service/w2l_download_job/src/script_execution/mod.rs @@ -1,5 +1,4 @@ pub mod ffmpeg_generate_preview_image_command; pub mod ffmpeg_generate_preview_video_command; pub mod imagemagick_generate_preview_image_command; -pub mod wav2lip_inference_command; -pub mod wav2lip_process_upload_command; \ No newline at end of file +pub mod wav2lip_process_upload_command; diff --git a/crates/service/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs b/crates/service/w2l_download_job/src/script_execution/wav2lip_process_upload_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/wav2lip_process_upload_command.rs rename to crates/service/w2l_download_job/src/script_execution/wav2lip_process_upload_command.rs diff --git a/crates/service/w2l_inference_job/Cargo.toml b/crates/service/w2l_inference_job/Cargo.toml new file mode 100644 index 0000000000..6b8f26ab50 --- /dev/null +++ b/crates/service/w2l_inference_job/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "w2l-inference-job" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "w2l-inference-job" +path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } +database_queries = { path = "../../lib/database_queries" } +google_drive_common = { path = "../../lib/google_drive_common" } +jobs_common = { path = "../../lib/jobs_common" } +storage_buckets_common = { path = "../../lib/storage_buckets_common" } +tts_common = { path = "../../lib/tts_common" } + +# Vendored code +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } + +# External +anyhow = "1.0.40" +chrono = { version = "0.4.19", features = ["serde"] } +clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +dotenv = "0.15.0" +easyenv = "0.0.6" +hostname = "0.3.1" +lfu = "0.2.5" +log = "0.4.14" +subprocess = "0.2.6" +tempdir = "0.3.7" + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } + +# Redis +r2d2_redis = "0.14.0" + +# Runtime +# NB: We switched to Tokio from `async-std` for broader ecosystem support. +tokio = { version = "1.6.0", features = ["macros"] } + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.14.14" } + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" diff --git a/crates/service/storyteller_web/src/w2l_inference_job.rs b/crates/service/w2l_inference_job/src/main.rs similarity index 94% rename from crates/service/storyteller_web/src/w2l_inference_job.rs rename to crates/service/w2l_inference_job/src/main.rs index 3c339c94bf..4e0c229e94 100644 --- a/crates/service/storyteller_web/src/w2l_inference_job.rs +++ b/crates/service/w2l_inference_job/src/main.rs @@ -10,9 +10,7 @@ #[macro_use] extern crate serde_derive; -pub mod job_queries; -pub mod script_execution; -pub mod util; +mod script_execution; use anyhow::anyhow; use chrono::Utc; @@ -24,18 +22,15 @@ use container_common::filesystem::check_directory_exists::check_directory_exists use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; -use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; -use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; -use crate::job_queries::w2l_inference_job_queries::grab_job_lock_and_mark_pending; -use crate::job_queries::w2l_inference_job_queries::insert_w2l_result; -use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_done; -use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failure; -use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; -use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; -use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; -use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; use database_queries::mediators::firehose_publisher::FirehosePublisher; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::W2lInferenceJobRecord; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::get_w2l_template_by_token; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::grab_job_lock_and_mark_pending; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::insert_w2l_result; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::mark_w2l_inference_job_done; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::mark_w2l_inference_job_failure; +use database_queries::queries::w2l::w2l_inference_jobs::w2l_inference_job_queries::query_w2l_inference_job_records; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; @@ -240,9 +235,6 @@ async fn main() -> AnyhowResult<()> { private_bucket_client, inference_script, google_drive_downloader, - //ffmpeg_image_preview_generator: FfmpegGeneratePreviewImageCommand {}, - //ffmpeg_video_preview_generator: FfmpegGeneratePreviewVideoCommand {}, - //imagemagick_image_preview_generator: ImagemagickGeneratePreviewImageCommand {}, w2l_inference: w2l_inference_command, bucket_path_unifier: BucketPathUnifier::default_paths(), semi_persistent_cache, diff --git a/crates/service/w2l_inference_job/src/script_execution/mod.rs b/crates/service/w2l_inference_job/src/script_execution/mod.rs new file mode 100644 index 0000000000..38aebfc0cb --- /dev/null +++ b/crates/service/w2l_inference_job/src/script_execution/mod.rs @@ -0,0 +1 @@ +pub mod wav2lip_inference_command; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/script_execution/wav2lip_inference_command.rs b/crates/service/w2l_inference_job/src/script_execution/wav2lip_inference_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/wav2lip_inference_command.rs rename to crates/service/w2l_inference_job/src/script_execution/wav2lip_inference_command.rs