diff --git a/.gitignore b/.gitignore
index 3a588e7f72..95d008957c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,6 +14,9 @@ _develop/tools/.secrets
_develop/tools/db_secrets.toml
twitch_secrets.toml
+# New env secrets (per-app)
+*-secrets.env
+
# Rust build directory
target/
crates/lib/database_queries/target/
diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml
index b92131dc77..fb12863abc 100644
--- a/.idea/dictionaries/bt.xml
+++ b/.idea/dictionaries/bt.xml
@@ -26,6 +26,7 @@
transcoded
transcodings
upserting
+ vits
vocoded
vocoder
vocoders
diff --git a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs
index ad2d013de0..f5041feaf3 100644
--- a/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs
+++ b/crates/service/job/inference_job/src/job/job_types/tts/tacotron2_v2_early_fakeyou/process_job.rs
@@ -232,14 +232,19 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<(), ProcessSingleJo
// roughly 12 seconds max. Here we map seconds to decoder steps.
let max_decoder_steps = seconds_to_decoder_steps(job.max_duration_seconds);
- let docker_options = DockerOptions {
- image_name: "c1336c8f9970".to_string(),
- maybe_bind_mount: Some(DockerFilesystemMount {
- local_filesystem: "/tmp".to_string(),
- container_filesystem: "/tmp".to_string(),
- }),
- maybe_gpu: Some(DockerGpu::All),
- };
+ // NB: Docker options are for development, not production.
+ let mut maybe_docker_options = None;
+
+ if let Some(docker_sha) = args.job_dependencies.job_type_details.tacotron2_old_vocodes.maybe_docker_image_sha.as_deref() {
+ maybe_docker_options = Some(DockerOptions {
+ image_name: docker_sha.to_string(),
+ maybe_bind_mount: Some(DockerFilesystemMount {
+ local_filesystem: "/tmp".to_string(),
+ container_filesystem: "/tmp".to_string(),
+ }),
+ maybe_gpu: Some(DockerGpu::All),
+ })
+ }
// ==================== RUN INFERENCE SCRIPT ==================== //
@@ -253,7 +258,7 @@ pub async fn process_job(args: ProcessJobArgs<'_>) -> Result<(), ProcessSingleJo
python,
Some("source python/bin/activate"),
"vocodes_inference_updated.py",
- Some(docker_options),
+ maybe_docker_options,
);
let mut maybe_mel_multiply_factor = None;
diff --git a/crates/service/job/inference_job/src/job_dependencies.rs b/crates/service/job/inference_job/src/job_dependencies.rs
index 03fcf3fdfc..35f3a8de4f 100644
--- a/crates/service/job/inference_job/src/job_dependencies.rs
+++ b/crates/service/job/inference_job/src/job_dependencies.rs
@@ -82,6 +82,9 @@ pub struct JobDependencies {
// A worker can be configured to only run jobs of a certain priority.
// This finds jobs of equal or greater priority.
pub maybe_minimum_priority: Option,
+
+ // Details for each job type (grouped by the job type)
+ pub job_type_details: JobTypeDetails,
}
pub struct JobWorkerDetails {
@@ -115,4 +118,18 @@ impl JobDependencies {
.map(|name| name.to_string())
.unwrap_or_else(|| self.worker_details.worker_hostname.clone())
}
-}
\ No newline at end of file
+}
+
+/// Per-job type details
+pub struct JobTypeDetails {
+ pub tacotron2_old_vocodes: Tacotron2VocodesDetails,
+ //pub tacotron2_modern: ...,
+ //pub vits: ...,
+ //pub softvc: ...,
+ //pub so_vits_svc: ...,
+}
+
+/// "Old" TT2 (vocodes-era)
+pub struct Tacotron2VocodesDetails {
+ pub maybe_docker_image_sha: Option,
+}
diff --git a/crates/service/job/inference_job/src/main.rs b/crates/service/job/inference_job/src/main.rs
index 1b6d6b7ebb..3287128e03 100644
--- a/crates/service/job/inference_job/src/main.rs
+++ b/crates/service/job/inference_job/src/main.rs
@@ -30,13 +30,13 @@ use container_common::filesystem::check_directory_exists::check_directory_exists
use crate::job::job_steps::job_stats::JobStats;
use crate::job::job_steps::main_loop::main_loop;
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron_inference_command::TacotronInferenceCommand;
-use crate::job_dependencies::{JobCaches, JobDependencies, JobWorkerDetails};
+use crate::job_dependencies::{JobCaches, JobDependencies, JobTypeDetails, JobWorkerDetails, Tacotron2VocodesDetails};
use crate::util::scoped_temp_dir_creator::ScopedTempDirCreator;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporterBuilder;
use jobs_common::job_progress_reporter::noop_job_progress_reporter::NoOpJobProgressReporterBuilder;
use jobs_common::job_progress_reporter::redis_job_progress_reporter::RedisJobProgressReporterBuilder;
use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
-use log::{info, warn};
+use log::{error, info, warn};
use memory_caching::multi_item_ttl_cache::MultiItemTtlCache;
use mysql_queries::mediators::firehose_publisher::FirehosePublisher;
use newrelic_telemetry::ClientBuilder;
@@ -71,6 +71,7 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp";
async fn main() -> AnyhowResult<()> {
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
+ // TODO: Deprecate
// NB: Do not check this secrets-containing dotenv file into VCS.
// This file should only contain *development* secrets, never production.
let _ = dotenv::from_filename(".env-secrets").ok();
@@ -79,6 +80,14 @@ async fn main() -> AnyhowResult<()> {
"inference-job.env",
&[".", "crates/service/job/inference_job"])?;
+ let _ = envvar::read_from_filename_and_paths(
+ "inference-job-secrets.env",
+ &[".", "crates/service/job/inference_job"]
+ ).map_err(|err| {
+ // NB: Fail open.
+ warn!("Could not load app-specific secrets from env file (this might be fine, eg. provided by k8s): {:?}", err);
+ });
+
info!("Obtaining worker hostname...");
let server_hostname = hostname::get()
@@ -264,6 +273,11 @@ async fn main() -> AnyhowResult<()> {
sidecar_max_synthesizer_models,
low_priority_starvation_prevention_every_nth,
maybe_minimum_priority,
+ job_type_details: JobTypeDetails {
+ tacotron2_old_vocodes: Tacotron2VocodesDetails {
+ maybe_docker_image_sha: easyenv::get_env_string_optional("TACOTRON2_VOCODES_DOCKER_IMAGE_SHA"),
+ },
+ },
};
main_loop(job_dependencies).await;