flexible docker image sha

.gitignore

@@ -14,6 +14,9 @@ _develop/tools/.secrets
 _develop/tools/db_secrets.toml
 twitch_secrets.toml
 
+# New env secrets (per-app)
+*-secrets.env
+
 # Rust build directory
 target/
 crates/lib/database_queries/target/

.idea/dictionaries/bt.xml

@@ -26,6 +26,7 @@
       <w>transcoded</w>
       <w>transcodings</w>
       <w>upserting</w>
+      <w>vits</w>
       <w>vocoded</w>
       <w>vocoder</w>
       <w>vocoders</w>

crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs

@@ -232,14 +232,19 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<(), ProcessSingleJo
   //  roughly 12 seconds max. Here we map seconds to decoder steps.
   let max_decoder_steps = seconds_to_decoder_steps(job.max_duration_seconds);
 
-  let docker_options = DockerOptions {
-    image_name: "c1336c8f9970".to_string(),
-    maybe_bind_mount: Some(DockerFilesystemMount {
-      local_filesystem: "/tmp".to_string(),
-      container_filesystem: "/tmp".to_string(),
-    }),
-    maybe_gpu: Some(DockerGpu::All),
-  };
+  // NB: Docker options are for development, not production.
+  let mut maybe_docker_options = None;
+
+  if let Some(docker_sha) = args.job_dependencies.job_type_details.tacotron2_old_vocodes.maybe_docker_image_sha.as_deref() {
+    maybe_docker_options = Some(DockerOptions {
+      image_name: docker_sha.to_string(),
+      maybe_bind_mount: Some(DockerFilesystemMount {
+        local_filesystem: "/tmp".to_string(),
+        container_filesystem: "/tmp".to_string(),
+      }),
+      maybe_gpu: Some(DockerGpu::All),
+    })
+  }
 
   // ==================== RUN INFERENCE SCRIPT ==================== //
 
@@ -253,7 +258,7 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<(), ProcessSingleJo
     python,
     Some("source python/bin/activate"),
     "vocodes_inference_updated.py",
-    Some(docker_options),
+    maybe_docker_options,
   );
 
   let mut maybe_mel_multiply_factor = None;

crates/service/job/inference_job/src/job_dependencies.rs

@@ -82,6 +82,9 @@ pub struct JobDependencies {
   // A worker can be configured to only run jobs of a certain priority.
   // This finds jobs of equal or greater priority.
   pub maybe_minimum_priority: Option<u8>,
+
+  // Details for each job type (grouped by the job type)
+  pub job_type_details: JobTypeDetails,
 }
 
 pub struct JobWorkerDetails {
@@ -115,4 +118,18 @@ impl JobDependencies {
         .map(|name| name.to_string())
         .unwrap_or_else(|| self.worker_details.worker_hostname.clone())
   }
-}
+}
+
+/// Per-job type details
+pub struct JobTypeDetails {
+  pub tacotron2_old_vocodes: Tacotron2VocodesDetails,
+  //pub tacotron2_modern: ...,
+  //pub vits: ...,
+  //pub softvc: ...,
+  //pub so_vits_svc: ...,
+}
+
+/// "Old" TT2 (vocodes-era)
+pub struct Tacotron2VocodesDetails {
+  pub maybe_docker_image_sha: Option<String>,
+}

crates/service/job/inference_job/src/main.rs

@@ -30,13 +30,13 @@ use container_common::filesystem::check_directory_exists::check_directory_exists
 use crate::job::job_steps::job_stats::JobStats;
 use crate::job::job_steps::main_loop::main_loop;
 use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron_inference_command::TacotronInferenceCommand;
-use crate::job_dependencies::{JobCaches, JobDependencies, JobWorkerDetails};
+use crate::job_dependencies::{JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, Tacotron2VocodesDetails};
 use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
 use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
 use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder;
 use jobs_common::job_progress_reporter::redis_job_progress_reporter::RedisJobProgressReporterBuilder;
 use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
-use log::{info, warn};
+use log::{error, info, warn};
 use memory_caching::multi_item_ttl_cache::MultiItemTtlCache;
 use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
 use newrelic_telemetry::ClientBuilder;
@@ -71,6 +71,7 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp";
 async fn main() -> AnyhowResult<()> {
   easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
 
+  // TODO: Deprecate
   // NB: Do not check this secrets-containing dotenv file into VCS.
   // This file should only contain *development* secrets, never production.
   let _ = dotenv::from_filename(".env-secrets").ok();
@@ -79,6 +80,14 @@ async fn main() -> AnyhowResult<()> {
     "inference-job.env",
     &[".", "crates/service/job/inference_job"])?;
 
+  let _ = envvar::read_from_filename_and_paths(
+    "inference-job-secrets.env",
+    &[".", "crates/service/job/inference_job"]
+  ).map_err(|err| {
+    // NB: Fail open.
+    warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err);
+  });
+
   info!("Obtaining worker hostname...");
 
   let server_hostname = hostname::get()
@@ -264,6 +273,11 @@ async fn main() -> AnyhowResult<()> {
     sidecar_max_synthesizer_models,
     low_priority_starvation_prevention_every_nth,
     maybe_minimum_priority,
+    job_type_details: JobTypeDetails {
+      tacotron2_old_vocodes: Tacotron2VocodesDetails {
+        maybe_docker_image_sha: easyenv::get_env_string_optional("TACOTRON2_VOCODES_DOCKER_IMAGE_SHA"),
+      },
+    },
   };
 
   main_loop(job_dependencies).await;