From 001ff86ed3dfb53d38831f360fbf14747187c6d9 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 25 Apr 2023 20:51:51 -0700 Subject: [PATCH] robust configuration for download-job --- Cargo.lock | 2 + Dockerfile | 2 +- crates/service/job/download_job/Cargo.toml | 1 + .../config/download-job.common.env | 1 + ...d-job.env => download-job.development.env} | 0 .../config/download-job.production.env | 1 + crates/service/job/download_job/src/main.rs | 40 +++++++++++-------- crates/service/job/inference_job/Cargo.toml | 1 + .../service/web/storyteller_web/src/main.rs | 10 ----- 9 files changed, 31 insertions(+), 27 deletions(-) create mode 100644 crates/service/job/download_job/config/download-job.common.env rename crates/service/job/download_job/config/{download-job.env => download-job.development.env} (100%) create mode 100644 crates/service/job/download_job/config/download-job.production.env diff --git a/Cargo.lock b/Cargo.lock index 0e8494e2bb..1f6a4afa2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2317,6 +2317,7 @@ name = "download-job" version = "0.0.1" dependencies = [ "anyhow", + "bootstrap", "chrono", "clap", "cloud_storage", @@ -3647,6 +3648,7 @@ name = "inference-job" version = "0.0.1" dependencies = [ "anyhow", + "bootstrap", "buckets", "chrono", "clap", diff --git a/Dockerfile b/Dockerfile index baa34d57e1..927d0113aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -225,7 +225,7 @@ RUN touch .env-secrets # Some services have default env files that live under their code directories # These should also be readable from the relative current path -COPY crates/service/job/download_job/config/download-job.env . +COPY crates/service/job/download_job/config/download-job.development.env . COPY crates/service/job/inference_job/config/inference-job.env . COPY crates/service/web/storyteller_web/config/storyteller-web.common.env . diff --git a/crates/service/job/download_job/Cargo.toml b/crates/service/job/download_job/Cargo.toml index f8a96483be..3bf90b6a9d 100644 --- a/crates/service/job/download_job/Cargo.toml +++ b/crates/service/job/download_job/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] # Internal +bootstrap = { path = "../../../lib/bootstrap" } cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } config = { path = "../../../lib/deprecated/config" } container_common = { path = "../../../lib/deprecated/container_common" } diff --git a/crates/service/job/download_job/config/download-job.common.env b/crates/service/job/download_job/config/download-job.common.env new file mode 100644 index 0000000000..6ab538a1bd --- /dev/null +++ b/crates/service/job/download_job/config/download-job.common.env @@ -0,0 +1 @@ +# download-job default .env configurations (these can be overridden with k8s) diff --git a/crates/service/job/download_job/config/download-job.env b/crates/service/job/download_job/config/download-job.development.env similarity index 100% rename from crates/service/job/download_job/config/download-job.env rename to crates/service/job/download_job/config/download-job.development.env diff --git a/crates/service/job/download_job/config/download-job.production.env b/crates/service/job/download_job/config/download-job.production.env new file mode 100644 index 0000000000..6ab538a1bd --- /dev/null +++ b/crates/service/job/download_job/config/download-job.production.env @@ -0,0 +1 @@ +# download-job default .env configurations (these can be overridden with k8s) diff --git a/crates/service/job/download_job/src/main.rs b/crates/service/job/download_job/src/main.rs index baaab87e52..991f843563 100644 --- a/crates/service/job/download_job/src/main.rs +++ b/crates/service/job/download_job/src/main.rs @@ -21,6 +21,7 @@ pub mod job_loop; pub mod job_state; pub mod job_types; +use bootstrap::bootstrap::{bootstrap, BootstrapArgs}; use cloud_storage::bucket_client::BucketClient; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use config::common_env::CommonEnv; @@ -33,6 +34,7 @@ use crate::job_types::tts::tacotron::tacotron_model_check_command::TacotronModel use crate::job_types::tts::vits::vits_model_check_command::VitsModelCheckCommand; use crate::job_types::vocoder::hifigan_softvc::hifigan_softvc_model_check_command::HifiGanSoftVcModelCheckCommand; use crate::job_types::vocoder::hifigan_tacotron::hifigan_model_check_command::HifiGanModelCheckCommand; +use crate::job_types::voice_conversion::so_vits_svc::so_vits_svc_model_check_command::SoVitsSvcModelCheckCommand; use crate::job_types::voice_conversion::softvc::softvc_model_check_command::SoftVcModelCheckCommand; use errors::AnyhowResult; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; @@ -46,7 +48,6 @@ use std::ops::Deref; use std::path::PathBuf; use std::time::Duration; use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; -use crate::job_types::voice_conversion::so_vits_svc::so_vits_svc_model_check_command::SoVitsSvcModelCheckCommand; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -59,23 +60,30 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; #[tokio::main] async fn main() -> AnyhowResult<()> { - easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); - // TODO: Deprecate pulling secrets from these two files and use the app-named env+secrets files (below) instead. - let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app. - let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control. + let container_environment = bootstrap(BootstrapArgs { + app_name: "download-job", + default_logging_override: Some(DEFAULT_RUST_LOG), + config_search_directories: &[".", "./config", "crates/service/job/download_job/config"], + })?; - let _ = envvar::read_from_filename_and_paths( - "download-job.env", - &[".", "./config", "crates/service/job/download_job/config"])?; - - let _ = envvar::read_from_filename_and_paths( - "download-job-secrets.env", - &[".", "./config", "crates/service/job/download_job/config"] - ).map_err(|err| { - // NB: Fail open. - warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err); - }); +// easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); +// +// // TODO: Deprecate pulling secrets from these two files and use the app-named env+secrets files (below) instead. +// let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app. +// let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control. +// +// let _ = envvar::read_from_filename_and_paths( +// "download-job.env", +// &[".", "./config", "crates/service/job/download_job/config"])?; +// +// let _ = envvar::read_from_filename_and_paths( +// "download-job-secrets.env", +// &[".", "./config", "crates/service/job/download_job/config"] +// ).map_err(|err| { +// // NB: Fail open. +// warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err); +// }); info!("Obtaining hostname..."); diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index 88cf45cabf..e2a056c79d 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] # Internal +bootstrap = { path = "../../../lib/bootstrap" } buckets = { path = "../../../_types/buckets" } cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } config = { path = "../../../lib/deprecated/config" } diff --git a/crates/service/web/storyteller_web/src/main.rs b/crates/service/web/storyteller_web/src/main.rs index 75a41e4221..9cf855edc2 100644 --- a/crates/service/web/storyteller_web/src/main.rs +++ b/crates/service/web/storyteller_web/src/main.rs @@ -117,16 +117,6 @@ async fn main() -> AnyhowResult<()> { config_search_directories: &[".", "./config", "crates/service/web/storyteller_web/config"], })?; -// easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); -// -// // NB: Do not check this secrets-containing dotenv file into VCS. -// // This file should only contain *development* secrets, never production. -// let _ = dotenv::from_filename(".env-secrets").ok(); -// -// let _ = envvar::read_from_filename_and_paths( -// "storyteller-web.env", -// &[".", "./config", "crates/service/web/storyteller_web/config"])?; - let common_env = CommonEnv::read_from_env()?; info!("Obtaining hostname...");