Move tts-download-job into its own crate independent of storyteller-web

This commit is contained in:
Brandon Thomas committed 2022-07-03 20:07:43 -04:00
1 parent a622e7a9e8
commit 7a883cdebb
19 files changed
+157 -33

No files matched your search

Generated
+42
View File
@@ -1730,6 +1730,16 @@ dependencies = [
"wasi 0.10.0+wasi-snapshot-preview1",
]
[[package]]
name = "google_drive_common"
version = "0.0.1"
dependencies = [
"anyhow",
"container_common",
"log",
"tempdir",
]
[[package]]
name = "h2"
version = "0.2.7"
@@ -3805,6 +3815,7 @@ dependencies = [
"futures 0.1.31",
"futures 0.3.17",
"futures-util",
"google_drive_common",
"hex",
"hmac 0.10.1",
"hostname",
@@ -4381,6 +4392,37 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "tts-download-job"
version = "0.0.1"
dependencies = [
"anyhow",
"chrono",
"clap",
"config",
"container_common",
"database_queries",
"dotenv",
"easyenv",
"google_drive_common",
"hostname",
"hyper 0.14.14",
"jobs_common",
"lfu",
"log",
"newrelic-telemetry",
"r2d2_redis",
"serde",
"serde_derive",
"serde_json",
"sqlx",
"storage_buckets_common",
"subprocess",
"tempdir",
"tokio 1.13.0",
"tts_common",
]
[[package]]
name = "tts-inference-job"
version = "0.0.1"
+2
View File
@@ -5,6 +5,7 @@ members = [
"crates/lib/config",
"crates/lib/container_common",
"crates/lib/database_queries",
"crates/lib/google_drive_common",
"crates/lib/http_server_common",
"crates/lib/jobs_common",
"crates/lib/redis_common",
@@ -15,6 +16,7 @@ members = [
# Application
"crates/service/storyteller_web",
"crates/service/tts_download_job",
"crates/service/tts_inference_job",
"crates/service/twitch_pubsub_subscriber",
"crates/service/websocket_gateway",
+11
View File
@@ -25,6 +25,14 @@ build_storyteller_web_app() {
popd
}
build_tts_download_job() {
# NB: This imports the inference/upload job queries
# It should also import the shared database lib queries, but something(???) broke.
pushd crates/service/tts_download_job
SQLX_OFFLINE=true cargo sqlx prepare tts_download_job
popd
}
build_tts_inference_job() {
# NB: This imports the inference/upload job queries
# It should also import the shared database lib queries, but something(???) broke.
@@ -39,6 +47,7 @@ combine_sqlx_queries() {
jq -s '.[0] * .[1]' \
crates/lib/database_queries/sqlx-data.json \
crates/service/storyteller_web/sqlx-data.json \
crates/service/tts_download_job/sqlx-data.json \
crates/service/tts_inference_job/sqlx-data.json \
> sqlx-data.json
}
@@ -46,11 +55,13 @@ combine_sqlx_queries() {
cleanup_temp_files() {
rm crates/lib/database_queries/sqlx-data.json \
crates/service/storyteller_web/sqlx-data.json \
crates/service/tts_download_job/sqlx-data.json \
crates/service/tts_inference_job/sqlx-data.json
}
build_shared_database_library
build_storyteller_web_app
build_tts_download_job
build_tts_inference_job
combine_sqlx_queries
cleanup_temp_files
@@ -1,5 +1,6 @@
pub mod stats;
pub mod tts_category_assignments;
pub mod tts_download_jobs;
pub mod tts_inference_jobs;
pub mod tts_models;
pub mod tts_results;
@@ -0,0 +1 @@
pub mod tts_download_job_queries;
@@ -5,13 +5,10 @@
use anyhow::anyhow;
use chrono::Utc;
use container_common::anyhow_result::AnyhowResult;
use database_queries::tokens::Tokens;
use crate::tokens::Tokens;
use sqlx::MySqlPool;
use std::path::Path;
// TODO(2022-02-10): Move these queries to the 'database_queries' crate
// I moved tts_inference_job queries out already.
/// table: tts_model_upload_jobs
#[derive(Debug)]
pub struct TtsUploadJobRecord {
+24
View File
@@ -0,0 +1,24 @@
[package]
name = "google_drive_common"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
name = "google_drive_common"
path = "src/lib.rs"
[dependencies]
# Internal
container_common = { path = "../container_common" }
# External
anyhow = "1.0.40"
log = "0.4.14"
tempdir = "0.3.7"
@@ -0,0 +1 @@
pub mod google_drive_download_command;
+1 -9
View File
@@ -13,16 +13,7 @@ publish = false
name = "storyteller-web"
path = "src/main.rs"
## OBS websocket server
#[[bin]]
#name = "obs-gateway-server"
#path = "src/obs_gateway_server.rs"
# Jobs for managing downloads and inference
[[bin]]
name = "tts-download-job"
path = "src/tts_download_job.rs"
[[bin]]
name = "w2l-download-job"
path = "src/w2l_download_job.rs"
@@ -37,6 +28,7 @@ path = "src/w2l_inference_job.rs"
config = { path = "../../lib/config" }
container_common = { path = "../../lib/container_common" }
database_queries = { path = "../../lib/database_queries" }
google_drive_common = { path = "../../lib/google_drive_common" }
http_server_common = { path = "../../lib/http_server_common" }
jobs_common = { path = "../../lib/jobs_common" }
redis_common = { path = "../../lib/redis_common" }
@@ -1,6 +1,5 @@
// TODO(2022-02-10): Move these queries to the 'database_queries' crate
// I moved tts_inference_job queries out already.
// I moved tts_inference_job and tts_download_job queries out already.
pub mod tts_download_job_queries;
pub mod w2l_download_job_queries;
pub mod w2l_inference_job_queries;
@@ -1,8 +1,5 @@
pub mod ffmpeg_generate_preview_image_command;
pub mod ffmpeg_generate_preview_video_command;
pub mod google_drive_download_command;
pub mod imagemagick_generate_preview_image_command;
pub mod tacotron_model_check_command;
pub mod talknet_model_check_command;
pub mod wav2lip_inference_command;
pub mod wav2lip_process_upload_command;
@@ -34,12 +34,12 @@ use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_p
use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records;
use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError};
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
use database_queries::mediators::badge_granter::BadgeGranter;
use database_queries::mediators::firehose_publisher::FirehosePublisher;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
use jobs_common::noop_logger::NoOpLogger;
use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
use log::{warn, info};
@@ -33,11 +33,10 @@ use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failur
use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records;
use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand;
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
use database_queries::mediators::firehose_publisher::FirehosePublisher;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
use jobs_common::noop_logger::NoOpLogger;
use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
@@ -46,7 +45,6 @@ use r2d2_redis::RedisConnectionManager;
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::r2d2;
use r2d2_redis::redis::Commands;
use ring::digest::{Context, Digest, SHA256};
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use std::fs::{File, metadata};
@@ -0,0 +1,61 @@
[package]
name = "tts-download-job"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[[bin]]
name = "tts-download-job"
path = "src/main.rs"
[dependencies]
# Internal
config = { path = "../../lib/config" }
container_common = { path = "../../lib/container_common" }
database_queries = { path = "../../lib/database_queries" }
google_drive_common = { path = "../../lib/google_drive_common" }
jobs_common = { path = "../../lib/jobs_common" }
storage_buckets_common = { path = "../../lib/storage_buckets_common" }
tts_common = { path = "../../lib/tts_common" }
# Vendored code
newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" }
# External
anyhow = "1.0.40"
chrono = { version = "0.4.19", features = ["serde"] }
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
dotenv = "0.15.0"
easyenv = "0.0.6"
hostname = "0.3.1"
lfu = "0.2.5"
log = "0.4.14"
subprocess = "0.2.6"
tempdir = "0.3.7"
# MySql Database
# "offline" supports typesafe builds without a live database
# "rustls" doesn't appear to work with IP addresses, per:
# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
# Redis
r2d2_redis = "0.14.0"
# Runtime
# NB: We switched to Tokio from `async-std` for broader ecosystem support.
tokio = { version = "1.6.0", features = ["macros"] }
# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime
# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper.
hyper = { version = "0.14.14" }
# Serialization
serde = "1.0.125"
serde_derive = "1.0.125"
serde_json = "1.0.64"
@@ -10,9 +10,7 @@
#[macro_use] extern crate serde_derive;
pub mod job_queries;
pub mod script_execution;
pub mod util;
use anyhow::anyhow;
use chrono::Utc;
@@ -25,25 +23,23 @@ use container_common::filesystem::check_file_exists::check_file_exists;
use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
use container_common::hashing::hash_file_sha2::hash_file_sha2;
use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord;
use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending;
use crate::job_queries::tts_download_job_queries::insert_tts_model;
use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done;
use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure;
use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records;
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand;
use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand;
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
use database_queries::column_types::tts_model_type::TtsModelType;
use database_queries::mediators::badge_granter::BadgeGranter;
use database_queries::mediators::firehose_publisher::FirehosePublisher;
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::TtsUploadJobRecord;
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::grab_job_lock_and_mark_pending;
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::insert_tts_model;
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::mark_tts_upload_job_done;
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::mark_tts_upload_job_failure;
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::query_tts_upload_job_records;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
use jobs_common::noop_logger::NoOpLogger;
use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
use log::{warn, info};
use r2d2_redis::RedisConnectionManager;
use r2d2_redis::r2d2;
use ring::digest::{Context, Digest, SHA256};
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use std::fs::File;
@@ -0,0 +1,2 @@
pub mod tacotron_model_check_command;
pub mod talknet_model_check_command;