From 8e8a87ba513bb9a9dbc521ed32bb685ed623dd2e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 21 Jun 2022 03:43:21 -0400 Subject: [PATCH] Refactor CommonEnv into config crate --- Cargo.lock | 4 ++++ crates/lib/config/Cargo.toml | 6 ++++++ .../config}/src/common_env.rs | 7 +++---- crates/lib/config/src/lib.rs | 1 + crates/service/storyteller_web/src/main.rs | 11 +++++------ .../storyteller_web/src/tts_download_job.rs | 3 +-- .../storyteller_web/src/tts_inference_job.rs | 14 ++++++++++---- .../storyteller_web/src/w2l_download_job.rs | 3 +-- .../storyteller_web/src/w2l_inference_job.rs | 3 +-- 9 files changed, 32 insertions(+), 20 deletions(-) rename crates/{service/storyteller_web => lib/config}/src/common_env.rs (89%) diff --git a/Cargo.lock b/Cargo.lock index 2eb90c0e09..0a56c95cb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1034,6 +1034,10 @@ dependencies = [ [[package]] name = "config" version = "0.0.1" +dependencies = [ + "anyhow", + "easyenv", +] [[package]] name = "const_fn" diff --git a/crates/lib/config/Cargo.toml b/crates/lib/config/Cargo.toml index fc10d38795..b101e3461f 100644 --- a/crates/lib/config/Cargo.toml +++ b/crates/lib/config/Cargo.toml @@ -11,3 +11,9 @@ publish = false [lib] name = "config" path = "src/lib.rs" + +# TODO: Remove these dependencies. +# These were introduced in refactoring. Config should have few, if any, dependencies. +[dependencies] +anyhow = "1.0.40" +easyenv = "0.0.6" diff --git a/crates/service/storyteller_web/src/common_env.rs b/crates/lib/config/src/common_env.rs similarity index 89% rename from crates/service/storyteller_web/src/common_env.rs rename to crates/lib/config/src/common_env.rs index a2e272762a..53ac666749 100644 --- a/crates/service/storyteller_web/src/common_env.rs +++ b/crates/lib/config/src/common_env.rs @@ -1,6 +1,5 @@ -use config::shared_constants::DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING; -use config::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING; -use container_common::anyhow_result::AnyhowResult; +use crate::shared_constants::DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_REDIS_DATABASE_1_CONNECTION_STRING; // TODO: Move more shared configs here. @@ -40,7 +39,7 @@ pub struct CommonEnv { impl CommonEnv { - pub fn read_from_env() -> AnyhowResult { + pub fn read_from_env() -> anyhow::Result { Ok(Self { redis_0_connection_string: easyenv::get_env_string_or_default("REDIS_0_URL", DEFAULT_REDIS_DATABASE_0_CONNECTION_STRING), diff --git a/crates/lib/config/src/lib.rs b/crates/lib/config/src/lib.rs index f78455f59a..8e782653a4 100644 --- a/crates/lib/config/src/lib.rs +++ b/crates/lib/config/src/lib.rs @@ -1 +1,2 @@ +pub mod common_env; pub mod shared_constants; diff --git a/crates/service/storyteller_web/src/main.rs b/crates/service/storyteller_web/src/main.rs index efceb7d4b0..efb6b4f8b4 100644 --- a/crates/service/storyteller_web/src/main.rs +++ b/crates/service/storyteller_web/src/main.rs @@ -15,7 +15,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../../../../db/reserved_usernames.txt"); pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../../../../db/reserved_usernames_including.txt"); -pub mod common_env; pub mod configs; pub mod http_clients; pub mod http_server; @@ -41,10 +40,12 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; +use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; -use crate::common_env::CommonEnv; +use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; +use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; use crate::http_server::endpoints::events::list_events::list_events_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; use crate::http_server::endpoints::misc::default_route_404::default_route_404; @@ -107,6 +108,8 @@ use crate::http_server::web_utils::redis_rate_limiter::RedisRateLimiter; use crate::http_server::web_utils::session_checker::SessionChecker; use crate::routes::add_routes; use crate::server_state::{ServerState, EnvConfig, TwitchOauthSecrets, TwitchOauth, RedisRateLimiters, InMemoryCaches}; +use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; +use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; use crate::threads::ip_banlist_set::IpBanlistSet; use crate::threads::poll_ip_banlist_thread::poll_ip_bans; use crate::util::buckets::bucket_client::BucketClient; @@ -126,11 +129,7 @@ use sqlx::mysql::MySqlPoolOptions; use std::sync::Arc; use std::time::Duration; use tokio::runtime::Runtime; -use container_common::files::read_toml_file_to_struct::read_toml_file_to_struct; use twitch_common::twitch_secrets::TwitchSecrets; -use crate::configs::static_api_tokens::{StaticApiTokenConfig, StaticApiTokens, StaticApiTokenSet}; -use crate::threads::db_health_checker_thread::db_health_checker_thread::db_health_checker_thread; -use crate::threads::db_health_checker_thread::db_health_check_status::HealthCheckStatus; // TODO TODO TODO TODO // TODO TODO TODO TODO diff --git a/crates/service/storyteller_web/src/tts_download_job.rs b/crates/service/storyteller_web/src/tts_download_job.rs index 19a2b4fa33..4e217d671b 100644 --- a/crates/service/storyteller_web/src/tts_download_job.rs +++ b/crates/service/storyteller_web/src/tts_download_job.rs @@ -10,13 +10,13 @@ #[macro_use] extern crate serde_derive; -pub mod common_env; pub mod job_queries; pub mod script_execution; pub mod util; use anyhow::anyhow; use chrono::Utc; +use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; @@ -24,7 +24,6 @@ use container_common::filesystem::check_directory_exists::check_directory_exists use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; -use crate::common_env::CommonEnv; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; use crate::job_queries::tts_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::tts_download_job_queries::insert_tts_model; diff --git a/crates/service/storyteller_web/src/tts_inference_job.rs b/crates/service/storyteller_web/src/tts_inference_job.rs index cc66b181ed..7a5e81484b 100644 --- a/crates/service/storyteller_web/src/tts_inference_job.rs +++ b/crates/service/storyteller_web/src/tts_inference_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod common_env; pub mod http_clients; pub mod job_queries; pub mod script_execution; @@ -18,6 +17,7 @@ pub mod util; use anyhow::{anyhow, Error}; use chrono::{Utc, DateTime, TimeZone}; +use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; @@ -27,21 +27,27 @@ use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; use container_common::token::random_uuid::generate_random_uuid; -use crate::common_env::CommonEnv; -use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; -use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; + +// buckets_common use crate::util::buckets::bucket_client::BucketClient; use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::hashing::hash_string_sha2::hash_string_sha2; + +// tts job +use crate::http_clients::tts_inference_sidecar_client::TtsInferenceSidecarClient; +use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; use crate::util::jobs::cache_miss_strategizer::CacheMissStrategizer; use crate::util::jobs::cache_miss_strategizer::CacheMissStrategy; use crate::util::jobs::cache_miss_strategizer_multi::SyncMultiCacheMissStrategizer; use crate::util::jobs::virtual_lfu_cache::SyncVirtualLfuCache; + +// jobs_common use crate::util::noop_logger::NoOpLogger; use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; + use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use database_queries::column_types::vocoder_type::VocoderType; use database_queries::mediators::firehose_publisher::FirehosePublisher; diff --git a/crates/service/storyteller_web/src/w2l_download_job.rs b/crates/service/storyteller_web/src/w2l_download_job.rs index 2e0aabc75e..19e4b88345 100644 --- a/crates/service/storyteller_web/src/w2l_download_job.rs +++ b/crates/service/storyteller_web/src/w2l_download_job.rs @@ -10,13 +10,13 @@ #[macro_use] extern crate serde_derive; -pub mod common_env; pub mod job_queries; pub mod script_execution; pub mod util; use anyhow::{anyhow, Error}; use chrono::Utc; +use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; @@ -24,7 +24,6 @@ use container_common::filesystem::check_directory_exists::check_directory_exists use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; -use crate::common_env::CommonEnv; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; use crate::job_queries::w2l_download_job_queries::grab_job_lock_and_mark_pending; use crate::job_queries::w2l_download_job_queries::insert_w2l_template; diff --git a/crates/service/storyteller_web/src/w2l_inference_job.rs b/crates/service/storyteller_web/src/w2l_inference_job.rs index 21e082395a..327ffe3ca8 100644 --- a/crates/service/storyteller_web/src/w2l_inference_job.rs +++ b/crates/service/storyteller_web/src/w2l_inference_job.rs @@ -10,13 +10,13 @@ #[macro_use] extern crate serde_derive; -pub mod common_env; pub mod job_queries; pub mod script_execution; pub mod util; use anyhow::anyhow; use chrono::Utc; +use config::common_env::CommonEnv; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; @@ -24,7 +24,6 @@ use container_common::filesystem::check_directory_exists::check_directory_exists use container_common::filesystem::check_file_exists::check_file_exists; use container_common::filesystem::safe_delete_temp_directory::safe_delete_temp_directory; use container_common::filesystem::safe_delete_temp_file::safe_delete_temp_file; -use crate::common_env::CommonEnv; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; use crate::job_queries::w2l_inference_job_queries::grab_job_lock_and_mark_pending;