From e305928b45f72b08a8db65c555f130efb6ecbb75 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 2 Apr 2023 20:01:45 -0700 Subject: [PATCH] flexible docker image sha --- .gitignore | 3 +++ .idea/dictionaries/bt.xml | 1 + .../tacotron2_v2_early_fakeyou/process_job.rs | 23 +++++++++++-------- .../job/inference_job/src/job_dependencies.rs | 19 ++++++++++++++- crates/service/job/inference_job/src/main.rs | 18 +++++++++++++-- 5 files changed, 52 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 3a588e7f72..95d008957c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ _develop/tools/.secrets _develop/tools/db_secrets.toml twitch_secrets.toml +# New env secrets (per-app) +*-secrets.env + # Rust build directory target/ crates/lib/database_queries/target/ diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index b92131dc77..fb12863abc 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -26,6 +26,7 @@ transcoded transcodings upserting + vits vocoded vocoder vocoders diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs index ad2d013de0..f5041feaf3 100644 --- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs @@ -232,14 +232,19 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<(), ProcessSingleJo // roughly 12 seconds max. Here we map seconds to decoder steps. let max_decoder_steps = seconds_to_decoder_steps(job.max_duration_seconds); - let docker_options = DockerOptions { - image_name: "c1336c8f9970".to_string(), - maybe_bind_mount: Some(DockerFilesystemMount { - local_filesystem: "/tmp".to_string(), - container_filesystem: "/tmp".to_string(), - }), - maybe_gpu: Some(DockerGpu::All), - }; + // NB: Docker options are for development, not production. + let mut maybe_docker_options = None; + + if let Some(docker_sha) = args.job_dependencies.job_type_details.tacotron2_old_vocodes.maybe_docker_image_sha.as_deref() { + maybe_docker_options = Some(DockerOptions { + image_name: docker_sha.to_string(), + maybe_bind_mount: Some(DockerFilesystemMount { + local_filesystem: "/tmp".to_string(), + container_filesystem: "/tmp".to_string(), + }), + maybe_gpu: Some(DockerGpu::All), + }) + } // ==================== RUN INFERENCE SCRIPT ==================== // @@ -253,7 +258,7 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<(), ProcessSingleJo python, Some("source python/bin/activate"), "vocodes_inference_updated.py", - Some(docker_options), + maybe_docker_options, ); let mut maybe_mel_multiply_factor = None; diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs index 03fcf3fdfc..35f3a8de4f 100644 --- a/crates/service/job/inference_job/src/job_dependencies.rs +++ b/crates/service/job/inference_job/src/job_dependencies.rs @@ -82,6 +82,9 @@ pub struct JobDependencies { // A worker can be configured to only run jobs of a certain priority. // This finds jobs of equal or greater priority. pub maybe_minimum_priority: Option, + + // Details for each job type (grouped by the job type) + pub job_type_details: JobTypeDetails, } pub struct JobWorkerDetails { @@ -115,4 +118,18 @@ impl JobDependencies { .map(|name| name.to_string()) .unwrap_or_else(|| self.worker_details.worker_hostname.clone()) } -} \ No newline at end of file +} + +/// Per-job type details +pub struct JobTypeDetails { + pub tacotron2_old_vocodes: Tacotron2VocodesDetails, + //pub tacotron2_modern: ..., + //pub vits: ..., + //pub softvc: ..., + //pub so_vits_svc: ..., +} + +/// "Old" TT2 (vocodes-era) +pub struct Tacotron2VocodesDetails { + pub maybe_docker_image_sha: Option, +} diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs index 1b6d6b7ebb..3287128e03 100644 --- a/crates/service/job/inference_job/src/main.rs +++ b/crates/service/job/inference_job/src/main.rs @@ -30,13 +30,13 @@ use container_common::filesystem::check_directory_exists::check_directory_exists use crate::job::job_steps::job_stats::JobStats; use crate::job::job_steps::main_loop::main_loop; use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron_inference_command::TacotronInferenceCommand; -use crate::job_dependencies::{JobCaches, JobDependencies, JobWorkerDetails}; +use crate::job_dependencies::{JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, Tacotron2VocodesDetails}; use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator; use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder; use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder; use jobs_common::job_progress_reporter::redis_job_progress_reporter::RedisJobProgressReporterBuilder; use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; -use log::{info, warn}; +use log::{error, info, warn}; use memory_caching::multi_item_ttl_cache::MultiItemTtlCache; use mysql_queries::mediators::firehose_publisher::FirehosePublisher; use newrelic_telemetry::ClientBuilder; @@ -71,6 +71,7 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + // TODO: Deprecate // NB: Do not check this secrets-containing dotenv file into VCS. // This file should only contain *development* secrets, never production. let _ = dotenv::from_filename(".env-secrets").ok(); @@ -79,6 +80,14 @@ async fn main() -> AnyhowResult<()> { "inference-job.env", &[".", "crates/service/job/inference_job"])?; + let _ = envvar::read_from_filename_and_paths( + "inference-job-secrets.env", + &[".", "crates/service/job/inference_job"] + ).map_err(|err| { + // NB: Fail open. + warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err); + }); + info!("Obtaining worker hostname..."); let server_hostname = hostname::get() @@ -264,6 +273,11 @@ async fn main() -> AnyhowResult<()> { sidecar_max_synthesizer_models, low_priority_starvation_prevention_every_nth, maybe_minimum_priority, + job_type_details: JobTypeDetails { + tacotron2_old_vocodes: Tacotron2VocodesDetails { + maybe_docker_image_sha: easyenv::get_env_string_optional("TACOTRON2_VOCODES_DOCKER_IMAGE_SHA"), + }, + }, }; main_loop(job_dependencies).await;