diff --git a/Cargo.lock b/Cargo.lock index 7301b5665c..382973eb31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1730,6 +1730,16 @@ dependencies = [ "wasi 0.10.0+wasi-snapshot-preview1", ] +[[package]] +name = "google_drive_common" +version = "0.0.1" +dependencies = [ + "anyhow", + "container_common", + "log", + "tempdir", +] + [[package]] name = "h2" version = "0.2.7" @@ -3805,6 +3815,7 @@ dependencies = [ "futures 0.1.31", "futures 0.3.17", "futures-util", + "google_drive_common", "hex", "hmac 0.10.1", "hostname", @@ -4381,6 +4392,37 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "tts-download-job" +version = "0.0.1" +dependencies = [ + "anyhow", + "chrono", + "clap", + "config", + "container_common", + "database_queries", + "dotenv", + "easyenv", + "google_drive_common", + "hostname", + "hyper 0.14.14", + "jobs_common", + "lfu", + "log", + "newrelic-telemetry", + "r2d2_redis", + "serde", + "serde_derive", + "serde_json", + "sqlx", + "storage_buckets_common", + "subprocess", + "tempdir", + "tokio 1.13.0", + "tts_common", +] + [[package]] name = "tts-inference-job" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index cb895f70fd..8c45747a62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/lib/config", "crates/lib/container_common", "crates/lib/database_queries", + "crates/lib/google_drive_common", "crates/lib/http_server_common", "crates/lib/jobs_common", "crates/lib/redis_common", @@ -15,6 +16,7 @@ members = [ # Application "crates/service/storyteller_web", + "crates/service/tts_download_job", "crates/service/tts_inference_job", "crates/service/twitch_pubsub_subscriber", "crates/service/websocket_gateway", diff --git a/codegen_database.sh b/codegen_database.sh index 1dd0c51fbe..c605e3eed1 100755 --- a/codegen_database.sh +++ b/codegen_database.sh @@ -25,6 +25,14 @@ build_storyteller_web_app() { popd } +build_tts_download_job() { + # NB: This imports the inference/upload job queries + # It should also import the shared database lib queries, but something(???) broke. + pushd crates/service/tts_download_job + SQLX_OFFLINE=true cargo sqlx prepare tts_download_job + popd +} + build_tts_inference_job() { # NB: This imports the inference/upload job queries # It should also import the shared database lib queries, but something(???) broke. @@ -39,6 +47,7 @@ combine_sqlx_queries() { jq -s '.[0] * .[1]' \ crates/lib/database_queries/sqlx-data.json \ crates/service/storyteller_web/sqlx-data.json \ + crates/service/tts_download_job/sqlx-data.json \ crates/service/tts_inference_job/sqlx-data.json \ > sqlx-data.json } @@ -46,11 +55,13 @@ combine_sqlx_queries() { cleanup_temp_files() { rm crates/lib/database_queries/sqlx-data.json \ crates/service/storyteller_web/sqlx-data.json \ + crates/service/tts_download_job/sqlx-data.json \ crates/service/tts_inference_job/sqlx-data.json } build_shared_database_library build_storyteller_web_app +build_tts_download_job build_tts_inference_job combine_sqlx_queries cleanup_temp_files diff --git a/crates/lib/database_queries/src/queries/tts/mod.rs b/crates/lib/database_queries/src/queries/tts/mod.rs index 9b179450a1..794b176cbc 100644 --- a/crates/lib/database_queries/src/queries/tts/mod.rs +++ b/crates/lib/database_queries/src/queries/tts/mod.rs @@ -1,5 +1,6 @@ pub mod stats; pub mod tts_category_assignments; +pub mod tts_download_jobs; pub mod tts_inference_jobs; pub mod tts_models; pub mod tts_results; \ No newline at end of file diff --git a/crates/lib/database_queries/src/queries/tts/tts_download_jobs/mod.rs b/crates/lib/database_queries/src/queries/tts/tts_download_jobs/mod.rs new file mode 100644 index 0000000000..63ef05af24 --- /dev/null +++ b/crates/lib/database_queries/src/queries/tts/tts_download_jobs/mod.rs @@ -0,0 +1 @@ +pub mod tts_download_job_queries; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs b/crates/lib/database_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs similarity index 97% rename from crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs rename to crates/lib/database_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs index 3108000ded..558f800cfd 100644 --- a/crates/service/storyteller_web/src/job_queries/tts_download_job_queries.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_download_jobs/tts_download_job_queries.rs @@ -5,13 +5,10 @@ use anyhow::anyhow; use chrono::Utc; use container_common::anyhow_result::AnyhowResult; -use database_queries::tokens::Tokens; +use crate::tokens::Tokens; use sqlx::MySqlPool; use std::path::Path; -// TODO(2022-02-10): Move these queries to the 'database_queries' crate -// I moved tts_inference_job queries out already. - /// table: tts_model_upload_jobs #[derive(Debug)] pub struct TtsUploadJobRecord { diff --git a/crates/lib/google_drive_common/Cargo.toml b/crates/lib/google_drive_common/Cargo.toml new file mode 100644 index 0000000000..734978a1e2 --- /dev/null +++ b/crates/lib/google_drive_common/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "google_drive_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "google_drive_common" +path = "src/lib.rs" + +[dependencies] + +# Internal +container_common = { path = "../container_common" } + +# External +anyhow = "1.0.40" +log = "0.4.14" +tempdir = "0.3.7" + diff --git a/crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs b/crates/lib/google_drive_common/src/google_drive_download_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/google_drive_download_command.rs rename to crates/lib/google_drive_common/src/google_drive_download_command.rs diff --git a/crates/lib/google_drive_common/src/lib.rs b/crates/lib/google_drive_common/src/lib.rs new file mode 100644 index 0000000000..500e829a60 --- /dev/null +++ b/crates/lib/google_drive_common/src/lib.rs @@ -0,0 +1 @@ +pub mod google_drive_download_command; \ No newline at end of file diff --git a/crates/service/storyteller_web/Cargo.toml b/crates/service/storyteller_web/Cargo.toml index 5fcbd82e21..103754eed1 100644 --- a/crates/service/storyteller_web/Cargo.toml +++ b/crates/service/storyteller_web/Cargo.toml @@ -13,16 +13,7 @@ publish = false name = "storyteller-web" path = "src/main.rs" -## OBS websocket server -#[[bin]] -#name = "obs-gateway-server" -#path = "src/obs_gateway_server.rs" - # Jobs for managing downloads and inference -[[bin]] -name = "tts-download-job" -path = "src/tts_download_job.rs" - [[bin]] name = "w2l-download-job" path = "src/w2l_download_job.rs" @@ -37,6 +28,7 @@ path = "src/w2l_inference_job.rs" config = { path = "../../lib/config" } container_common = { path = "../../lib/container_common" } database_queries = { path = "../../lib/database_queries" } +google_drive_common = { path = "../../lib/google_drive_common" } http_server_common = { path = "../../lib/http_server_common" } jobs_common = { path = "../../lib/jobs_common" } redis_common = { path = "../../lib/redis_common" } diff --git a/crates/service/storyteller_web/src/job_queries/mod.rs b/crates/service/storyteller_web/src/job_queries/mod.rs index c25ac9cc81..62587a06ed 100644 --- a/crates/service/storyteller_web/src/job_queries/mod.rs +++ b/crates/service/storyteller_web/src/job_queries/mod.rs @@ -1,6 +1,5 @@ // TODO(2022-02-10): Move these queries to the 'database_queries' crate -// I moved tts_inference_job queries out already. +// I moved tts_inference_job and tts_download_job queries out already. -pub mod tts_download_job_queries; pub mod w2l_download_job_queries; pub mod w2l_inference_job_queries; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/script_execution/mod.rs b/crates/service/storyteller_web/src/script_execution/mod.rs index a8889bac0f..c11d8a59a4 100644 --- a/crates/service/storyteller_web/src/script_execution/mod.rs +++ b/crates/service/storyteller_web/src/script_execution/mod.rs @@ -1,8 +1,5 @@ pub mod ffmpeg_generate_preview_image_command; pub mod ffmpeg_generate_preview_video_command; -pub mod google_drive_download_command; pub mod imagemagick_generate_preview_image_command; -pub mod tacotron_model_check_command; -pub mod talknet_model_check_command; pub mod wav2lip_inference_command; pub mod wav2lip_process_upload_command; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/w2l_download_job.rs b/crates/service/storyteller_web/src/w2l_download_job.rs index 4f127f1b0f..377e385429 100644 --- a/crates/service/storyteller_web/src/w2l_download_job.rs +++ b/crates/service/storyteller_web/src/w2l_download_job.rs @@ -34,12 +34,12 @@ use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_p use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records; use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; -use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError}; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::mediators::badge_granter::BadgeGranter; use database_queries::mediators::firehose_publisher::FirehosePublisher; +use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use log::{warn, info}; diff --git a/crates/service/storyteller_web/src/w2l_inference_job.rs b/crates/service/storyteller_web/src/w2l_inference_job.rs index 7cd27cf8f3..3c339c94bf 100644 --- a/crates/service/storyteller_web/src/w2l_inference_job.rs +++ b/crates/service/storyteller_web/src/w2l_inference_job.rs @@ -33,11 +33,10 @@ use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failur use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records; use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand; use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand; -use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; -use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::mediators::firehose_publisher::FirehosePublisher; +use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; @@ -46,7 +45,6 @@ use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; -use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; diff --git a/crates/service/tts_download_job/Cargo.toml b/crates/service/tts_download_job/Cargo.toml new file mode 100644 index 0000000000..c0acba04e4 --- /dev/null +++ b/crates/service/tts_download_job/Cargo.toml @@ -0,0 +1,61 @@ +[package] +name = "tts-download-job" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "tts-download-job" +path = "src/main.rs" + +[dependencies] + +# Internal +config = { path = "../../lib/config" } +container_common = { path = "../../lib/container_common" } +database_queries = { path = "../../lib/database_queries" } +google_drive_common = { path = "../../lib/google_drive_common" } +jobs_common = { path = "../../lib/jobs_common" } +storage_buckets_common = { path = "../../lib/storage_buckets_common" } +tts_common = { path = "../../lib/tts_common" } + +# Vendored code +newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" } + +# External +anyhow = "1.0.40" +chrono = { version = "0.4.19", features = ["serde"] } +clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +dotenv = "0.15.0" +easyenv = "0.0.6" +hostname = "0.3.1" +lfu = "0.2.5" +log = "0.4.14" +subprocess = "0.2.6" +tempdir = "0.3.7" + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } + +# Redis +r2d2_redis = "0.14.0" + +# Runtime +# NB: We switched to Tokio from `async-std` for broader ecosystem support. +tokio = { version = "1.6.0", features = ["macros"] } + +# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime +# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper. +hyper = { version = "0.14.14" } + +# Serialization +serde = "1.0.125" +serde_derive = "1.0.125" +serde_json = "1.0.64" diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/tts_download_job/src/main.rs similarity index 95% rename from crates/service/storyteller_web/src/tts_download_job.rs rename to crates/service/tts_download_job/src/main.rs index afd8ba7c75..a5a31a390c 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/tts_download_job/src/main.rs @@ -10,9 +10,7 @@ #[macro_use] extern crate serde_derive; -pub mod job_queries; pub mod script_execution; -pub mod util; use anyhow::anyhow; use chrono::Utc; @@ -25,25 +23,23 @@ use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use container_common::hashing::hash_file_sha2::hash_file_sha2; -use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; -use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; -use crate::job_queries::tts_download_job_queries::insert_tts_model; -use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done; -use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; -use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; -use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand; -use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::column_types::tts_model_type::TtsModelType; use database_queries::mediators::badge_granter::BadgeGranter; use database_queries::mediators::firehose_publisher::FirehosePublisher; +use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::TtsUploadJobRecord; +use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::grab_job_lock_and_mark_pending; +use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::insert_tts_model; +use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::mark_tts_upload_job_done; +use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::mark_tts_upload_job_failure; +use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::query_tts_upload_job_records; +use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; use jobs_common::noop_logger::NoOpLogger; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; use r2d2_redis::r2d2; -use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; diff --git a/crates/service/tts_download_job/src/script_execution/mod.rs b/crates/service/tts_download_job/src/script_execution/mod.rs new file mode 100644 index 0000000000..c0447db609 --- /dev/null +++ b/crates/service/tts_download_job/src/script_execution/mod.rs @@ -0,0 +1,2 @@ +pub mod tacotron_model_check_command; +pub mod talknet_model_check_command; \ No newline at end of file diff --git a/crates/service/storyteller_web/src/script_execution/tacotron_model_check_command.rs b/crates/service/tts_download_job/src/script_execution/tacotron_model_check_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/tacotron_model_check_command.rs rename to crates/service/tts_download_job/src/script_execution/tacotron_model_check_command.rs diff --git a/crates/service/storyteller_web/src/script_execution/talknet_model_check_command.rs b/crates/service/tts_download_job/src/script_execution/talknet_model_check_command.rs similarity index 100% rename from crates/service/storyteller_web/src/script_execution/talknet_model_check_command.rs rename to crates/service/tts_download_job/src/script_execution/talknet_model_check_command.rs