mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-16 06:42:14 +00:00
Move tts-download-job into its own crate independent of storyteller-web
This commit is contained in:
1 parent
a622e7a9e8
commit
7a883cdebb
19 files changed
+157
-33
No files matched your search
Generated
+42
@@ -1730,6 +1730,16 @@ dependencies = [
|
||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google_drive_common"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"container_common",
|
||||
"log",
|
||||
"tempdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.2.7"
|
||||
@@ -3805,6 +3815,7 @@ dependencies = [
|
||||
"futures 0.1.31",
|
||||
"futures 0.3.17",
|
||||
"futures-util",
|
||||
"google_drive_common",
|
||||
"hex",
|
||||
"hmac 0.10.1",
|
||||
"hostname",
|
||||
@@ -4381,6 +4392,37 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
|
||||
|
||||
[[package]]
|
||||
name = "tts-download-job"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"clap",
|
||||
"config",
|
||||
"container_common",
|
||||
"database_queries",
|
||||
"dotenv",
|
||||
"easyenv",
|
||||
"google_drive_common",
|
||||
"hostname",
|
||||
"hyper 0.14.14",
|
||||
"jobs_common",
|
||||
"lfu",
|
||||
"log",
|
||||
"newrelic-telemetry",
|
||||
"r2d2_redis",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
"sqlx",
|
||||
"storage_buckets_common",
|
||||
"subprocess",
|
||||
"tempdir",
|
||||
"tokio 1.13.0",
|
||||
"tts_common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tts-inference-job"
|
||||
version = "0.0.1"
|
||||
|
||||
@@ -5,6 +5,7 @@ members = [
|
||||
"crates/lib/config",
|
||||
"crates/lib/container_common",
|
||||
"crates/lib/database_queries",
|
||||
"crates/lib/google_drive_common",
|
||||
"crates/lib/http_server_common",
|
||||
"crates/lib/jobs_common",
|
||||
"crates/lib/redis_common",
|
||||
@@ -15,6 +16,7 @@ members = [
|
||||
|
||||
# Application
|
||||
"crates/service/storyteller_web",
|
||||
"crates/service/tts_download_job",
|
||||
"crates/service/tts_inference_job",
|
||||
"crates/service/twitch_pubsub_subscriber",
|
||||
"crates/service/websocket_gateway",
|
||||
|
||||
@@ -25,6 +25,14 @@ build_storyteller_web_app() {
|
||||
popd
|
||||
}
|
||||
|
||||
build_tts_download_job() {
|
||||
# NB: This imports the inference/upload job queries
|
||||
# It should also import the shared database lib queries, but something(???) broke.
|
||||
pushd crates/service/tts_download_job
|
||||
SQLX_OFFLINE=true cargo sqlx prepare tts_download_job
|
||||
popd
|
||||
}
|
||||
|
||||
build_tts_inference_job() {
|
||||
# NB: This imports the inference/upload job queries
|
||||
# It should also import the shared database lib queries, but something(???) broke.
|
||||
@@ -39,6 +47,7 @@ combine_sqlx_queries() {
|
||||
jq -s '.[0] * .[1]' \
|
||||
crates/lib/database_queries/sqlx-data.json \
|
||||
crates/service/storyteller_web/sqlx-data.json \
|
||||
crates/service/tts_download_job/sqlx-data.json \
|
||||
crates/service/tts_inference_job/sqlx-data.json \
|
||||
> sqlx-data.json
|
||||
}
|
||||
@@ -46,11 +55,13 @@ combine_sqlx_queries() {
|
||||
cleanup_temp_files() {
|
||||
rm crates/lib/database_queries/sqlx-data.json \
|
||||
crates/service/storyteller_web/sqlx-data.json \
|
||||
crates/service/tts_download_job/sqlx-data.json \
|
||||
crates/service/tts_inference_job/sqlx-data.json
|
||||
}
|
||||
|
||||
build_shared_database_library
|
||||
build_storyteller_web_app
|
||||
build_tts_download_job
|
||||
build_tts_inference_job
|
||||
combine_sqlx_queries
|
||||
cleanup_temp_files
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
pub mod stats;
|
||||
pub mod tts_category_assignments;
|
||||
pub mod tts_download_jobs;
|
||||
pub mod tts_inference_jobs;
|
||||
pub mod tts_models;
|
||||
pub mod tts_results;
|
||||
@@ -0,0 +1 @@
|
||||
pub mod tts_download_job_queries;
|
||||
+1
-4
@@ -5,13 +5,10 @@
|
||||
use anyhow::anyhow;
|
||||
use chrono::Utc;
|
||||
use container_common::anyhow_result::AnyhowResult;
|
||||
use database_queries::tokens::Tokens;
|
||||
use crate::tokens::Tokens;
|
||||
use sqlx::MySqlPool;
|
||||
use std::path::Path;
|
||||
|
||||
// TODO(2022-02-10): Move these queries to the 'database_queries' crate
|
||||
// I moved tts_inference_job queries out already.
|
||||
|
||||
/// table: tts_model_upload_jobs
|
||||
#[derive(Debug)]
|
||||
pub struct TtsUploadJobRecord {
|
||||
@@ -0,0 +1,24 @@
|
||||
[package]
|
||||
name = "google_drive_common"
|
||||
edition = "2018"
|
||||
version = "0.0.1"
|
||||
authors = [
|
||||
"Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>",
|
||||
]
|
||||
publish = false
|
||||
|
||||
[lib]
|
||||
name = "google_drive_common"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
container_common = { path = "../container_common" }
|
||||
|
||||
# External
|
||||
anyhow = "1.0.40"
|
||||
log = "0.4.14"
|
||||
tempdir = "0.3.7"
|
||||
|
||||
File renamed without changes.
@@ -0,0 +1 @@
|
||||
pub mod google_drive_download_command;
|
||||
@@ -13,16 +13,7 @@ publish = false
|
||||
name = "storyteller-web"
|
||||
path = "src/main.rs"
|
||||
|
||||
## OBS websocket server
|
||||
#[[bin]]
|
||||
#name = "obs-gateway-server"
|
||||
#path = "src/obs_gateway_server.rs"
|
||||
|
||||
# Jobs for managing downloads and inference
|
||||
[[bin]]
|
||||
name = "tts-download-job"
|
||||
path = "src/tts_download_job.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "w2l-download-job"
|
||||
path = "src/w2l_download_job.rs"
|
||||
@@ -37,6 +28,7 @@ path = "src/w2l_inference_job.rs"
|
||||
config = { path = "../../lib/config" }
|
||||
container_common = { path = "../../lib/container_common" }
|
||||
database_queries = { path = "../../lib/database_queries" }
|
||||
google_drive_common = { path = "../../lib/google_drive_common" }
|
||||
http_server_common = { path = "../../lib/http_server_common" }
|
||||
jobs_common = { path = "../../lib/jobs_common" }
|
||||
redis_common = { path = "../../lib/redis_common" }
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
// TODO(2022-02-10): Move these queries to the 'database_queries' crate
|
||||
// I moved tts_inference_job queries out already.
|
||||
// I moved tts_inference_job and tts_download_job queries out already.
|
||||
|
||||
pub mod tts_download_job_queries;
|
||||
pub mod w2l_download_job_queries;
|
||||
pub mod w2l_inference_job_queries;
|
||||
@@ -1,8 +1,5 @@
|
||||
pub mod ffmpeg_generate_preview_image_command;
|
||||
pub mod ffmpeg_generate_preview_video_command;
|
||||
pub mod google_drive_download_command;
|
||||
pub mod imagemagick_generate_preview_image_command;
|
||||
pub mod tacotron_model_check_command;
|
||||
pub mod talknet_model_check_command;
|
||||
pub mod wav2lip_inference_command;
|
||||
pub mod wav2lip_process_upload_command;
|
||||
@@ -34,12 +34,12 @@ use crate::job_queries::w2l_download_job_queries::mark_w2l_template_upload_job_p
|
||||
use crate::job_queries::w2l_download_job_queries::query_w2l_template_upload_job_records;
|
||||
use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
|
||||
use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
|
||||
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
|
||||
use crate::script_execution::wav2lip_process_upload_command::{Wav2LipPreprocessClient, Wav2LipPreprocessError};
|
||||
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
|
||||
use database_queries::mediators::badge_granter::BadgeGranter;
|
||||
use database_queries::mediators::firehose_publisher::FirehosePublisher;
|
||||
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use jobs_common::noop_logger::NoOpLogger;
|
||||
use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
|
||||
use log::{warn, info};
|
||||
|
||||
@@ -33,11 +33,10 @@ use crate::job_queries::w2l_inference_job_queries::mark_w2l_inference_job_failur
|
||||
use crate::job_queries::w2l_inference_job_queries::query_w2l_inference_job_records;
|
||||
use crate::script_execution::ffmpeg_generate_preview_image_command::FfmpegGeneratePreviewImageCommand;
|
||||
use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGeneratePreviewVideoCommand;
|
||||
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
|
||||
use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand;
|
||||
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
|
||||
use database_queries::mediators::firehose_publisher::FirehosePublisher;
|
||||
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use jobs_common::noop_logger::NoOpLogger;
|
||||
use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
|
||||
use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
|
||||
@@ -46,7 +45,6 @@ use r2d2_redis::RedisConnectionManager;
|
||||
use r2d2_redis::r2d2::PooledConnection;
|
||||
use r2d2_redis::r2d2;
|
||||
use r2d2_redis::redis::Commands;
|
||||
use ring::digest::{Context, Digest, SHA256};
|
||||
use sqlx::MySqlPool;
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
use std::fs::{File, metadata};
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
[package]
|
||||
name = "tts-download-job"
|
||||
edition = "2018"
|
||||
version = "0.0.1"
|
||||
authors = [
|
||||
"Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>",
|
||||
]
|
||||
publish = false
|
||||
|
||||
[[bin]]
|
||||
name = "tts-download-job"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
config = { path = "../../lib/config" }
|
||||
container_common = { path = "../../lib/container_common" }
|
||||
database_queries = { path = "../../lib/database_queries" }
|
||||
google_drive_common = { path = "../../lib/google_drive_common" }
|
||||
jobs_common = { path = "../../lib/jobs_common" }
|
||||
storage_buckets_common = { path = "../../lib/storage_buckets_common" }
|
||||
tts_common = { path = "../../lib/tts_common" }
|
||||
|
||||
# Vendored code
|
||||
newrelic-telemetry = { path = "../../vendor/newrelic-telemetry-sdk-rust" }
|
||||
|
||||
# External
|
||||
anyhow = "1.0.40"
|
||||
chrono = { version = "0.4.19", features = ["serde"] }
|
||||
clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated
|
||||
dotenv = "0.15.0"
|
||||
easyenv = "0.0.6"
|
||||
hostname = "0.3.1"
|
||||
lfu = "0.2.5"
|
||||
log = "0.4.14"
|
||||
subprocess = "0.2.6"
|
||||
tempdir = "0.3.7"
|
||||
|
||||
# MySql Database
|
||||
# "offline" supports typesafe builds without a live database
|
||||
# "rustls" doesn't appear to work with IP addresses, per:
|
||||
# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828
|
||||
sqlx = { version = "0.5", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] }
|
||||
|
||||
# Redis
|
||||
r2d2_redis = "0.14.0"
|
||||
|
||||
# Runtime
|
||||
# NB: We switched to Tokio from `async-std` for broader ecosystem support.
|
||||
tokio = { version = "1.6.0", features = ["macros"] }
|
||||
|
||||
# NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime
|
||||
# NB: Furthermore, the Tokio runtime is old, so I have to use an older Hyper.
|
||||
hyper = { version = "0.14.14" }
|
||||
|
||||
# Serialization
|
||||
serde = "1.0.125"
|
||||
serde_derive = "1.0.125"
|
||||
serde_json = "1.0.64"
|
||||
+7
-11
@@ -10,9 +10,7 @@
|
||||
|
||||
#[macro_use] extern crate serde_derive;
|
||||
|
||||
pub mod job_queries;
|
||||
pub mod script_execution;
|
||||
pub mod util;
|
||||
|
||||
use anyhow::anyhow;
|
||||
use chrono::Utc;
|
||||
@@ -25,25 +23,23 @@ use container_common::filesystem::check_file_exists::check_file_exists;
|
||||
use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory;
|
||||
use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file;
|
||||
use container_common::hashing::hash_file_sha2::hash_file_sha2;
|
||||
use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord;
|
||||
use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending;
|
||||
use crate::job_queries::tts_download_job_queries::insert_tts_model;
|
||||
use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_done;
|
||||
use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure;
|
||||
use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records;
|
||||
use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand;
|
||||
use crate::script_execution::talknet_model_check_command::TalknetModelCheckCommand;
|
||||
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
|
||||
use database_queries::column_types::tts_model_type::TtsModelType;
|
||||
use database_queries::mediators::badge_granter::BadgeGranter;
|
||||
use database_queries::mediators::firehose_publisher::FirehosePublisher;
|
||||
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::TtsUploadJobRecord;
|
||||
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::grab_job_lock_and_mark_pending;
|
||||
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::insert_tts_model;
|
||||
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::mark_tts_upload_job_done;
|
||||
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::mark_tts_upload_job_failure;
|
||||
use database_queries::queries::tts::tts_download_jobs::tts_download_job_queries::query_tts_upload_job_records;
|
||||
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
|
||||
use jobs_common::noop_logger::NoOpLogger;
|
||||
use jobs_common::redis_job_status_logger::RedisJobStatusLogger;
|
||||
use log::{warn, info};
|
||||
use r2d2_redis::RedisConnectionManager;
|
||||
use r2d2_redis::r2d2;
|
||||
use ring::digest::{Context, Digest, SHA256};
|
||||
use sqlx::MySqlPool;
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
use std::fs::File;
|
||||
@@ -0,0 +1,2 @@
|
||||
pub mod tacotron_model_check_command;
|
||||
pub mod talknet_model_check_command;
|
||||
File renamed without changes.
Reference in new issue
Block a user