diff --git a/.env-download-job b/.env-download-job
new file mode 100644
index 0000000000..def03d6262
--- /dev/null
+++ b/.env-download-job
@@ -0,0 +1,19 @@
+# Environmental configurations specific to `download-job`.
+# These are defaulted for dev and are easy to overwrite in production with K8s environment variables.
+# !! DOES *NOT* CONTAIN SECRETS !!
+
+WEB_DOWNLOADER_COMMAND=./download_internet_file.py
+WEB_DOWNLOADER_MAYBE_VENV_SCRIPT=./python/bin/activate
+WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d73f28ce3ff6
+
+TACOTRON_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
+TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+TACOTRON_MODEL_CHECK_COMMAND=./vocodes_model_check_tacotron.py
+TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
+
+HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
+HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+HIFIGAN_MODEL_CHECK_COMMAND=./vocodes_model_check_hifigan.py
+HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
+
+
diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml
index f507d3e88f..9f6dca8dc5 100644
--- a/.idea/dictionaries/bt.xml
+++ b/.idea/dictionaries/bt.xml
@@ -12,6 +12,7 @@
upserting
vocoder
vocoders
+ vocodes
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 8ffd82111f..dd52eb70d6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -169,6 +169,7 @@ RUN ldd storyteller-web
# Without a .env file, Rust crashes "mysteriously" (ugh)
RUN touch .env
+RUN touch .env-download-job
RUN touch .env-secrets
EXPOSE 8080
diff --git a/crates/lib/google_drive_common/src/google_drive_download_command.rs b/crates/lib/google_drive_common/src/google_drive_download_command.rs
index f95b59debc..cc7a05dd84 100644
--- a/crates/lib/google_drive_common/src/google_drive_download_command.rs
+++ b/crates/lib/google_drive_common/src/google_drive_download_command.rs
@@ -28,7 +28,19 @@ pub struct GoogleDriveDownloadCommand {
}
impl GoogleDriveDownloadCommand {
- pub fn new(download_script: &str) -> Self {
+ pub fn new(
+ download_script: &str,
+ maybe_venv_activation_script: Option<&str>,
+ maybe_docker_options: Option
+ ) -> Self {
+ Self {
+ download_script: download_script.to_string(),
+ maybe_venv_activation_script: maybe_venv_activation_script.map(|s| s.to_string()),
+ maybe_docker_options,
+ }
+ }
+
+ pub fn new_production(download_script: &str) -> Self {
Self {
download_script: download_script.to_string(),
maybe_venv_activation_script: None,
diff --git a/crates/service/download_job/src/job_state.rs b/crates/service/download_job/src/job_state.rs
index 5f46f10346..18169265e3 100644
--- a/crates/service/download_job/src/job_state.rs
+++ b/crates/service/download_job/src/job_state.rs
@@ -19,15 +19,11 @@ pub struct JobState {
pub bucket_client: BucketClient,
pub firehose_publisher: FirehosePublisher,
pub badge_granter: BadgeGranter,
- pub google_drive_downloader: GoogleDriveDownloadCommand,
pub bucket_path_unifier: BucketPathUnifier,
- pub tacotron_model_check_command: TacotronModelCheckCommand,
- pub hifigan_model_check_command: HifiGanModelCheckCommand,
+ pub sidecar_configs: SidecarConfigs,
- // Command to run
- pub download_script: String,
// Root to store TTS results
pub bucket_root_tts_model_uploads: String,
@@ -41,3 +37,10 @@ pub struct JobState {
// NB: This is an i32 so we don't need to convert to db column type.
pub job_max_attempts: i32,
}
+
+/// Configurations and interfaces to code deployed as sidecars or container mounts.
+pub struct SidecarConfigs {
+ pub google_drive_downloader: GoogleDriveDownloadCommand,
+ pub tacotron_model_check_command: TacotronModelCheckCommand,
+ pub hifigan_model_check_command: HifiGanModelCheckCommand,
+}
\ No newline at end of file
diff --git a/crates/service/download_job/src/job_steps/process_single_job.rs b/crates/service/download_job/src/job_steps/process_single_job.rs
index d49329047e..1442aaf69b 100644
--- a/crates/service/download_job/src/job_steps/process_single_job.rs
+++ b/crates/service/download_job/src/job_steps/process_single_job.rs
@@ -42,7 +42,7 @@ pub async fn process_single_job(job_state: &JobState, job: &AvailableDownloadJob
return Err(anyhow!("Bad download URL: `{}`", &job.download_url));
}
- let download_filename = match job_state.google_drive_downloader.download_file(&job.download_url, &temp_dir).await {
+ let download_filename = match job_state.sidecar_configs.google_drive_downloader.download_file(&job.download_url, &temp_dir).await {
Ok(filename) => filename,
Err(e) => {
safe_delete_temp_directory(&temp_dir);
diff --git a/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs b/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs
index 73a2dea8e0..37a8cc5d62 100644
--- a/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs
+++ b/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs
@@ -11,7 +11,7 @@ pub struct HifiGanModelCheckCommand {
hifigan_root_code_directory: String,
/// eg. `source python/bin/activate`
- virtual_env_activation_command: String,
+ maybe_virtual_env_activation_command: Option,
hifigan_model_check_script_name: String,
@@ -22,13 +22,13 @@ pub struct HifiGanModelCheckCommand {
impl HifiGanModelCheckCommand {
pub fn new(
hifigan_root_code_directory: &str,
- virtual_env_activation_command: &str,
+ maybe_virtual_env_activation_command: Option<&str>,
hifigan_model_check_script_name: &str,
maybe_docker_options: Option,
) -> Self {
Self {
hifigan_root_code_directory: hifigan_root_code_directory.to_string(),
- virtual_env_activation_command: virtual_env_activation_command.to_string(),
+ maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()),
hifigan_model_check_script_name: hifigan_model_check_script_name.to_string(),
maybe_docker_options,
}
@@ -42,8 +42,13 @@ impl HifiGanModelCheckCommand {
let mut command = String::new();
command.push_str(&format!("cd {}", self.hifigan_root_code_directory));
- command.push_str(" && ");
- command.push_str(&self.virtual_env_activation_command);
+
+ if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() {
+ command.push_str(" && ");
+ command.push_str(venv_command);
+ command.push_str(" ");
+ }
+
command.push_str(" && ");
command.push_str("python ");
command.push_str(&self.hifigan_model_check_script_name);
diff --git a/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs b/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs
index c1973efb8f..12665dafbb 100644
--- a/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs
+++ b/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs
@@ -35,7 +35,7 @@ pub async fn process_hifigan_vocoder<'a, 'b>(
let output_metadata_fs_path = temp_dir.path().join("metadata.json");
- let model_check_result = job_state.hifigan_model_check_command.execute(
+ let model_check_result = job_state.sidecar_configs.hifigan_model_check_command.execute(
&file_path,
&output_metadata_fs_path
);
diff --git a/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs b/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs
index b6e1e0c8e6..95c2c9516e 100644
--- a/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs
+++ b/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs
@@ -36,7 +36,7 @@ pub async fn process_tacotron_model<'a, 'b>(
let output_metadata_fs_path = temp_dir.path().join("metadata.json");
- let model_check_result = job_state.tacotron_model_check_command.execute(
+ let model_check_result = job_state.sidecar_configs.tacotron_model_check_command.execute(
&file_path,
&output_metadata_fs_path,
false
diff --git a/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs b/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs
index b64e5edd41..35d829aef4 100644
--- a/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs
+++ b/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs
@@ -12,7 +12,7 @@ pub struct TacotronModelCheckCommand {
tacotron_root_code_directory: String,
/// eg. `source python/bin/activate`
- virtual_env_activation_command: String,
+ maybe_virtual_env_activation_command: Option,
tacotron_model_check_script_name: String,
@@ -23,13 +23,13 @@ pub struct TacotronModelCheckCommand {
impl TacotronModelCheckCommand {
pub fn new(
tacotron_root_code_directory: &str,
- virtual_env_activation_command: &str,
+ maybe_virtual_env_activation_command: Option<&str>,
tacotron_model_check_script_name: &str,
maybe_docker_options: Option,
) -> Self {
Self {
tacotron_root_code_directory: tacotron_root_code_directory.to_string(),
- virtual_env_activation_command: virtual_env_activation_command.to_string(),
+ maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()),
tacotron_model_check_script_name: tacotron_model_check_script_name.to_string(),
maybe_docker_options,
}
@@ -46,8 +46,13 @@ impl TacotronModelCheckCommand {
command.push_str("echo 'test'");
command.push_str(" && ");
command.push_str(&format!("cd {}", self.tacotron_root_code_directory));
- command.push_str(" && ");
- command.push_str(&self.virtual_env_activation_command);
+
+ if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() {
+ command.push_str(" && ");
+ command.push_str(venv_command);
+ command.push_str(" ");
+ }
+
command.push_str(" && ");
command.push_str("python ");
command.push_str(&self.tacotron_model_check_script_name);
diff --git a/crates/service/download_job/src/main.rs b/crates/service/download_job/src/main.rs
index 82f2d58144..69e93aa06b 100644
--- a/crates/service/download_job/src/main.rs
+++ b/crates/service/download_job/src/main.rs
@@ -26,9 +26,10 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
use config::shared_constants::DEFAULT_RUST_LOG;
use container_common::anyhow_result::AnyhowResult;
use container_common::filesystem::check_directory_exists::check_directory_exists;
-use crate::job_state::JobState;
+use crate::job_state::{JobState, SidecarConfigs};
use crate::job_steps::main_loop::main_loop;
use crate::job_types::hifigan::hifigan_model_check_command::HifiGanModelCheckCommand;
+use crate::job_types::tacotron::tacotron_model_check_command::TacotronModelCheckCommand;
use database_queries::mediators::badge_granter::BadgeGranter;
use database_queries::mediators::firehose_publisher::FirehosePublisher;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
@@ -42,7 +43,6 @@ use std::time::Duration;
use storage_buckets_common::bucket_client::BucketClient;
use storage_buckets_common::bucket_path_unifier::BucketPathUnifier;
use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
-use crate::job_types::tacotron::tacotron_model_check_command::TacotronModelCheckCommand;
// Buckets
const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
@@ -57,7 +57,8 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp";
async fn main() -> AnyhowResult<()> {
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
- let _ = dotenv::from_filename(".env-secrets").ok();
+ let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app.
+ let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control.
info!("Obtaining hostname...");
@@ -91,24 +92,92 @@ async fn main() -> AnyhowResult<()> {
"DOWNLOAD_TEMP_DIR",
DEFAULT_TEMP_DIR);
- let download_script = easyenv::get_env_string_or_default(
- "DOWNLOAD_SCRIPT",
- "./scripts/download_internet_file.py");
+ // =============== Configure Python "Sidecars" ===============
- // TODO/FIXME: Cannot be deployed as currently written.
- //let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
- let google_drive_downloader = GoogleDriveDownloadCommand::new_local_dev_docker(
- "./download_internet_file.py",
- "./python/bin/activate",
- DockerOptions {
- image_name: "d73f28ce3ff6".to_string(), // web-downloader
- maybe_bind_mount: Some(DockerFilesystemMount {
- local_filesystem: "/tmp".to_string(),
- container_filesystem: "/tmp".to_string()
- }),
- maybe_gpu: None,
- }
- );
+ let google_drive_downloader = {
+ let downloader_command= easyenv::get_env_string_or_default(
+ "WEB_DOWNLOADER_COMMAND", // TODO: Was "DOWNLOAD_SCRIPT" in old apps
+ "./download_internet_file.py");
+
+ let maybe_downloader_venv_script = easyenv::get_env_string_optional(
+ "WEB_DOWNLOADER_MAYBE_VENV_SCRIPT");
+
+ let docker_options = easyenv::get_env_string_optional(
+ "WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE")
+ .map(|image_name| {
+ DockerOptions {
+ image_name,
+ maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+ maybe_gpu: None,
+ }
+ });
+
+ GoogleDriveDownloadCommand::new(
+ &downloader_command,
+ maybe_downloader_venv_script.as_deref(),
+ docker_options,
+ )
+ };
+
+ let tacotron_model_check_command = {
+ let root_directory = easyenv::get_env_string_required(
+ "TACOTRON_MODEL_CHECK_ROOT_DIRECTORY")?;
+
+ let python_command = easyenv::get_env_string_or_default(
+ "TACOTRON_MODEL_CHECK_COMMAND",
+ "./vocodes_model_check_tacotron.py");
+
+ let maybe_venv_command = easyenv::get_env_string_optional(
+ "TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND");
+
+ let maybe_docker_options = easyenv::get_env_string_optional(
+ "TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE")
+ .map(|image_name| {
+ DockerOptions {
+ image_name,
+ maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+ maybe_gpu: Some(DockerGpu::All),
+ }
+ });
+
+ TacotronModelCheckCommand::new(
+ &root_directory,
+ maybe_venv_command.as_deref(),
+ &python_command,
+ maybe_docker_options,
+ )
+ };
+
+ let hifigan_model_check_command= {
+ let root_directory = easyenv::get_env_string_required(
+ "HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY")?;
+
+ let python_command = easyenv::get_env_string_or_default(
+ "HIFIGAN_MODEL_CHECK_COMMAND",
+ "./vocodes_model_check_tacotron.py");
+
+ let maybe_venv_command = easyenv::get_env_string_optional(
+ "HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND");
+
+ let maybe_docker_options = easyenv::get_env_string_optional(
+ "HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE")
+ .map(|image_name| {
+ DockerOptions {
+ image_name,
+ maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+ maybe_gpu: Some(DockerGpu::All),
+ }
+ });
+
+ HifiGanModelCheckCommand::new(
+ &root_directory,
+ maybe_venv_command.as_deref(),
+ &python_command,
+ maybe_docker_options,
+ )
+ };
+
+ // =============== End Configure Python "Sidecars" ===============
let temp_directory = PathBuf::from(temp_directory);
@@ -145,50 +214,21 @@ async fn main() -> AnyhowResult<()> {
firehose_publisher: firehose_publisher.clone(), // NB: Also safe
};
- // TODO/FIXME: Cannot be deployed as currently written.
- let tacotron_model_check_command = TacotronModelCheckCommand::new(
- "/models/tts",
- "source python/bin/activate",
- "./vocodes_model_check_tacotron.py",
- Some(DockerOptions {
- image_name: "5642d0fd7fc1".to_string(), // storyteller-ml
- maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
- maybe_gpu: Some(DockerGpu::All),
- })
- );
-
- // TODO/FIXME: Cannot be deployed as currently written.
- let hifigan_model_check_command= HifiGanModelCheckCommand::new(
- //&easyenv::get_env_string_required("HIFIGAN_ROOT_CODE_DIRECTORY")?,
- "/models/tts",
- //&easyenv::get_env_string_or_default(
- //"HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND",
- //"source python-tacotron/bin/activate"),
- "source python/bin/activate",
- //&easyenv::get_env_string_or_default(
- //"HIFIGAN_MODEL_CHECK_SCRIPT_NAME",
- //"vocodes_model_check_hifigan.py"),
- "./vocodes_model_check_hifigan.py",
- Some(DockerOptions {
- image_name: "5642d0fd7fc1".to_string(), // storyteller-ml
- maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
- maybe_gpu: Some(DockerGpu::All),
- }),
- );
let job_state = JobState {
download_temp_directory: temp_directory,
mysql_pool,
redis_pool,
bucket_client,
- download_script,
- google_drive_downloader,
bucket_path_unifier: BucketPathUnifier::default_paths(),
bucket_root_tts_model_uploads: bucket_root.to_string(),
firehose_publisher,
badge_granter,
- tacotron_model_check_command,
- hifigan_model_check_command,
+ sidecar_configs: SidecarConfigs {
+ google_drive_downloader,
+ tacotron_model_check_command,
+ hifigan_model_check_command,
+ },
job_batch_wait_millis: common_env.job_batch_wait_millis,
job_max_attempts: common_env.job_max_attempts as i32,
no_op_logger_millis: common_env.no_op_logger_millis,
diff --git a/crates/service/w2l_download_job/src/main.rs b/crates/service/w2l_download_job/src/main.rs
index cfa1c34252..ab773216ec 100644
--- a/crates/service/w2l_download_job/src/main.rs
+++ b/crates/service/w2l_download_job/src/main.rs
@@ -185,7 +185,8 @@ async fn main() -> AnyhowResult<()> {
"DOWNLOAD_SCRIPT",
"./scripts/download_internet_file.py");
- let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
+ let google_drive_downloader =
+ GoogleDriveDownloadCommand::new_production(&download_script);
let temp_directory = PathBuf::from(temp_directory);
diff --git a/crates/service/w2l_inference_job/src/main.rs b/crates/service/w2l_inference_job/src/main.rs
index a6126ba98e..0bb6f95ef5 100644
--- a/crates/service/w2l_inference_job/src/main.rs
+++ b/crates/service/w2l_inference_job/src/main.rs
@@ -179,7 +179,8 @@ async fn main() -> AnyhowResult<()> {
let download_script = easyenv::get_env_string_or_default(
"DOWNLOAD_SCRIPT",
"./scripts/download_internet_file.py");
- let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
+ let google_drive_downloader =
+ GoogleDriveDownloadCommand::new_production(&download_script);
let temp_directory = PathBuf::from(temp_directory);