robust configuration for download-job

This commit is contained in:
Brandon Thomas committed 2023-04-25 20:51:51 -07:00
1 parent 94b406eca2
commit 001ff86ed3
9 files changed
+31 -27

No files matched your search

Generated
+2
View File
@@ -2317,6 +2317,7 @@ name = "download-job"
version = "0.0.1"
dependencies = [
"anyhow",
"bootstrap",
"chrono",
"clap",
"cloud_storage",
@@ -3647,6 +3648,7 @@ name = "inference-job"
version = "0.0.1"
dependencies = [
"anyhow",
"bootstrap",
"buckets",
"chrono",
"clap",
+1 -1
View File
@@ -225,7 +225,7 @@ RUN touch .env-secrets
# Some services have default env files that live under their code directories
# These should also be readable from the relative current path
COPY crates/service/job/download_job/config/download-job.env .
COPY crates/service/job/download_job/config/download-job.development.env .
COPY crates/service/job/inference_job/config/inference-job.env .
COPY crates/service/web/storyteller_web/config/storyteller-web.common.env .
@@ -15,6 +15,7 @@ path = "src/main.rs"
[dependencies]
# Internal
bootstrap = { path = "../../../lib/bootstrap" }
cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
config = { path = "../../../lib/deprecated/config" }
container_common = { path = "../../../lib/deprecated/container_common" }
@@ -0,0 +1 @@
# download-job default .env configurations (these can be overridden with k8s)
@@ -0,0 +1 @@
# download-job default .env configurations (these can be overridden with k8s)
+24 -16
View File
@@ -21,6 +21,7 @@ pub mod job_loop;
pub mod job_state;
pub mod job_types;
use bootstrap::bootstrap::{bootstrap, BootstrapArgs};
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use config::common_env::CommonEnv;
@@ -33,6 +34,7 @@ use crate::job_types::tts::tacotron::tacotron_model_check_command::TacotronModel
use crate::job_types::tts::vits::vits_model_check_command::VitsModelCheckCommand;
use crate::job_types::vocoder::hifigan_softvc::hifigan_softvc_model_check_command::HifiGanSoftVcModelCheckCommand;
use crate::job_types::vocoder::hifigan_tacotron::hifigan_model_check_command::HifiGanModelCheckCommand;
use crate::job_types::voice_conversion::so_vits_svc::so_vits_svc_model_check_command::SoVitsSvcModelCheckCommand;
use crate::job_types::voice_conversion::softvc::softvc_model_check_command::SoftVcModelCheckCommand;
use errors::AnyhowResult;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
@@ -46,7 +48,6 @@ use std::ops::Deref;
use std::path::PathBuf;
use std::time::Duration;
use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
use crate::job_types::voice_conversion::so_vits_svc::so_vits_svc_model_check_command::SoVitsSvcModelCheckCommand;
// Buckets
const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
@@ -59,23 +60,30 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp";
#[tokio::main]
async fn main() -> AnyhowResult<()> {
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
// TODO: Deprecate pulling secrets from these two files and use the app-named env+secrets files (below) instead.
let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app.
let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control.
let container_environment = bootstrap(BootstrapArgs {
app_name: "download-job",
default_logging_override: Some(DEFAULT_RUST_LOG),
config_search_directories: &[".", "./config", "crates/service/job/download_job/config"],
})?;
let _ = envvar::read_from_filename_and_paths(
"download-job.env",
&[".", "./config", "crates/service/job/download_job/config"])?;
let _ = envvar::read_from_filename_and_paths(
"download-job-secrets.env",
&[".", "./config", "crates/service/job/download_job/config"]
).map_err(|err| {
// NB: Fail open.
warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err);
});
// easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
//
// // TODO: Deprecate pulling secrets from these two files and use the app-named env+secrets files (below) instead.
// let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app.
// let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control.
//
// let _ = envvar::read_from_filename_and_paths(
// "download-job.env",
// &[".", "./config", "crates/service/job/download_job/config"])?;
//
// let _ = envvar::read_from_filename_and_paths(
// "download-job-secrets.env",
// &[".", "./config", "crates/service/job/download_job/config"]
// ).map_err(|err| {
// // NB: Fail open.
// warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err);
// });
info!("Obtaining hostname...");
@@ -15,6 +15,7 @@ path = "src/main.rs"
[dependencies]
# Internal
bootstrap = { path = "../../../lib/bootstrap" }
buckets = { path = "../../../_types/buckets" }
cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
config = { path = "../../../lib/deprecated/config" }
@@ -117,16 +117,6 @@ async fn main() -> AnyhowResult<()> {
config_search_directories: &[".", "./config", "crates/service/web/storyteller_web/config"],
})?;
// easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
//
// // NB: Do not check this secrets-containing dotenv file into VCS.
// // This file should only contain *development* secrets, never production.
// let _ = dotenv::from_filename(".env-secrets").ok();
//
// let _ = envvar::read_from_filename_and_paths(
// "storyteller-web.env",
// &[".", "./config", "crates/service/web/storyteller_web/config"])?;
let common_env = CommonEnv::read_from_env()?;
info!("Obtaining hostname...");