Clean up new consolidated downloader so it can be deployed

.env-download-job

@@ -0,0 +1,19 @@
+# Environmental configurations specific to `download-job`.
+# These are defaulted for dev and are easy to overwrite in production with K8s environment variables.
+# !! DOES *NOT* CONTAIN SECRETS !!
+
+WEB_DOWNLOADER_COMMAND=./download_internet_file.py
+WEB_DOWNLOADER_MAYBE_VENV_SCRIPT=./python/bin/activate
+WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d73f28ce3ff6
+
+TACOTRON_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
+TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+TACOTRON_MODEL_CHECK_COMMAND=./vocodes_model_check_tacotron.py
+TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
+
+HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY=/models/tts
+HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate"
+HIFIGAN_MODEL_CHECK_COMMAND=./vocodes_model_check_hifigan.py
+HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1
+
+

.idea/dictionaries/bt.xml

@@ -12,6 +12,7 @@
       <w>upserting</w>
       <w>vocoder</w>
       <w>vocoders</w>
+      <w>vocodes</w>
     </words>
   </dictionary>
 </component>

Dockerfile

@@ -169,6 +169,7 @@ RUN ldd storyteller-web
 
 # Without a .env file, Rust crashes "mysteriously" (ugh)
 RUN touch .env
+RUN touch .env-download-job
 RUN touch .env-secrets
 
 EXPOSE 8080

crates/lib/google_drive_common/src/google_drive_download_command.rs

@@ -28,7 +28,19 @@ pub struct GoogleDriveDownloadCommand {
 }
 
 impl GoogleDriveDownloadCommand {
-  pub fn new(download_script: &str) -> Self {
+  pub fn new(
+    download_script: &str,
+    maybe_venv_activation_script: Option<&str>,
+    maybe_docker_options: Option<DockerOptions>
+  ) -> Self {
+    Self {
+      download_script: download_script.to_string(),
+      maybe_venv_activation_script: maybe_venv_activation_script.map(|s| s.to_string()),
+      maybe_docker_options,
+    }
+  }
+
+  pub fn new_production(download_script: &str) -> Self {
     Self {
       download_script: download_script.to_string(),
       maybe_venv_activation_script: None,

crates/service/download_job/src/job_state.rs

@@ -19,15 +19,11 @@ pub struct JobState {
   pub bucket_client: BucketClient,
   pub firehose_publisher: FirehosePublisher,
   pub badge_granter: BadgeGranter,
-  pub google_drive_downloader: GoogleDriveDownloadCommand,
 
   pub bucket_path_unifier: BucketPathUnifier,
 
-  pub tacotron_model_check_command: TacotronModelCheckCommand,
-  pub hifigan_model_check_command: HifiGanModelCheckCommand,
+  pub sidecar_configs: SidecarConfigs,
 
-  // Command to run
-  pub download_script: String,
   // Root to store TTS results
   pub bucket_root_tts_model_uploads: String,
 
@@ -41,3 +37,10 @@ pub struct JobState {
   // NB: This is an i32 so we don't need to convert to db column type.
   pub job_max_attempts: i32,
 }
+
+/// Configurations and interfaces to code deployed as sidecars or container mounts.
+pub struct SidecarConfigs {
+  pub google_drive_downloader: GoogleDriveDownloadCommand,
+  pub tacotron_model_check_command: TacotronModelCheckCommand,
+  pub hifigan_model_check_command: HifiGanModelCheckCommand,
+}

crates/service/download_job/src/job_steps/process_single_job.rs

@@ -42,7 +42,7 @@ pub async fn process_single_job(job_state: &JobState, job: &AvailableDownloadJob
     return Err(anyhow!("Bad download URL: `{}`", &job.download_url));
   }
 
-  let download_filename = match job_state.google_drive_downloader.download_file(&job.download_url, &temp_dir).await {
+  let download_filename = match job_state.sidecar_configs.google_drive_downloader.download_file(&job.download_url, &temp_dir).await {
     Ok(filename) => filename,
     Err(e) => {
       safe_delete_temp_directory(&temp_dir);

crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs

@@ -11,7 +11,7 @@ pub struct HifiGanModelCheckCommand {
   hifigan_root_code_directory: String,
   
   /// eg. `source python/bin/activate`
-  virtual_env_activation_command: String,
+  maybe_virtual_env_activation_command: Option<String>,
   
   hifigan_model_check_script_name: String,
 
@@ -22,13 +22,13 @@ pub struct HifiGanModelCheckCommand {
 impl HifiGanModelCheckCommand {
   pub fn new(
     hifigan_root_code_directory: &str,
-    virtual_env_activation_command: &str,
+    maybe_virtual_env_activation_command: Option<&str>,
     hifigan_model_check_script_name: &str,
     maybe_docker_options: Option<DockerOptions>,
   ) -> Self {
     Self {
       hifigan_root_code_directory: hifigan_root_code_directory.to_string(),
-      virtual_env_activation_command: virtual_env_activation_command.to_string(),
+      maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()),
       hifigan_model_check_script_name: hifigan_model_check_script_name.to_string(),
       maybe_docker_options,
     }
@@ -42,8 +42,13 @@ impl HifiGanModelCheckCommand {
 
     let mut command = String::new();
     command.push_str(&format!("cd {}", self.hifigan_root_code_directory));
-    command.push_str(" && ");
-    command.push_str(&self.virtual_env_activation_command);
+
+    if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() {
+      command.push_str(" && ");
+      command.push_str(venv_command);
+      command.push_str(" ");
+    }
+
     command.push_str(" && ");
     command.push_str("python ");
     command.push_str(&self.hifigan_model_check_script_name);

crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs

@@ -35,7 +35,7 @@ pub async fn process_hifigan_vocoder<'a, 'b>(
 
   let output_metadata_fs_path = temp_dir.path().join("metadata.json");
 
-  let model_check_result = job_state.hifigan_model_check_command.execute(
+  let model_check_result = job_state.sidecar_configs.hifigan_model_check_command.execute(
     &file_path,
     &output_metadata_fs_path
   );

crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs

@@ -36,7 +36,7 @@ pub async fn process_tacotron_model<'a, 'b>(
 
   let output_metadata_fs_path = temp_dir.path().join("metadata.json");
 
-  let model_check_result = job_state.tacotron_model_check_command.execute(
+  let model_check_result = job_state.sidecar_configs.tacotron_model_check_command.execute(
     &file_path,
     &output_metadata_fs_path,
     false

crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs

@@ -12,7 +12,7 @@ pub struct TacotronModelCheckCommand {
   tacotron_root_code_directory: String,
   
   /// eg. `source python/bin/activate`
-  virtual_env_activation_command: String,
+  maybe_virtual_env_activation_command: Option<String>,
   
   tacotron_model_check_script_name: String,
 
@@ -23,13 +23,13 @@ pub struct TacotronModelCheckCommand {
 impl TacotronModelCheckCommand {
   pub fn new(
     tacotron_root_code_directory: &str,
-    virtual_env_activation_command: &str,
+    maybe_virtual_env_activation_command: Option<&str>,
     tacotron_model_check_script_name: &str,
     maybe_docker_options: Option<DockerOptions>,
   ) -> Self {
     Self {
       tacotron_root_code_directory: tacotron_root_code_directory.to_string(),
-      virtual_env_activation_command: virtual_env_activation_command.to_string(),
+      maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()),
       tacotron_model_check_script_name: tacotron_model_check_script_name.to_string(),
       maybe_docker_options,
     }
@@ -46,8 +46,13 @@ impl TacotronModelCheckCommand {
     command.push_str("echo 'test'");
     command.push_str(" && ");
     command.push_str(&format!("cd {}", self.tacotron_root_code_directory));
-    command.push_str(" && ");
-    command.push_str(&self.virtual_env_activation_command);
+
+    if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() {
+      command.push_str(" && ");
+      command.push_str(venv_command);
+      command.push_str(" ");
+    }
+
     command.push_str(" && ");
     command.push_str("python ");
     command.push_str(&self.tacotron_model_check_script_name);

crates/service/download_job/src/main.rs

@@ -26,9 +26,10 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
 use config::shared_constants::DEFAULT_RUST_LOG;
 use container_common::anyhow_result::AnyhowResult;
 use container_common::filesystem::check_directory_exists::check_directory_exists;
-use crate::job_state::JobState;
+use crate::job_state::{JobState, SidecarConfigs};
 use crate::job_steps::main_loop::main_loop;
 use crate::job_types::hifigan::hifigan_model_check_command::HifiGanModelCheckCommand;
+use crate::job_types::tacotron::tacotron_model_check_command::TacotronModelCheckCommand;
 use database_queries::mediators::badge_granter::BadgeGranter;
 use database_queries::mediators::firehose_publisher::FirehosePublisher;
 use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
@@ -42,7 +43,6 @@ use std::time::Duration;
 use storage_buckets_common::bucket_client::BucketClient;
 use storage_buckets_common::bucket_path_unifier::BucketPathUnifier;
 use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions};
-use crate::job_types::tacotron::tacotron_model_check_command::TacotronModelCheckCommand;
 
 // Buckets
 const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";
@@ -57,7 +57,8 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp";
 async fn main() -> AnyhowResult<()> {
   easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
 
-  let _ = dotenv::from_filename(".env-secrets").ok();
+  let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app.
+  let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control.
 
   info!("Obtaining hostname...");
 
@@ -91,24 +92,92 @@ async fn main() -> AnyhowResult<()> {
     "DOWNLOAD_TEMP_DIR",
     DEFAULT_TEMP_DIR);
 
-  let download_script = easyenv::get_env_string_or_default(
-    "DOWNLOAD_SCRIPT",
-    "./scripts/download_internet_file.py");
+  // =============== Configure Python "Sidecars" ===============
 
-  // TODO/FIXME: Cannot be deployed as currently written.
-  //let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
-  let google_drive_downloader = GoogleDriveDownloadCommand::new_local_dev_docker(
-    "./download_internet_file.py",
-    "./python/bin/activate",
-    DockerOptions {
-      image_name: "d73f28ce3ff6".to_string(), // web-downloader
-      maybe_bind_mount: Some(DockerFilesystemMount {
-        local_filesystem: "/tmp".to_string(),
-        container_filesystem: "/tmp".to_string()
-      }),
-      maybe_gpu: None,
-    }
-  );
+  let google_drive_downloader = {
+    let downloader_command= easyenv::get_env_string_or_default(
+      "WEB_DOWNLOADER_COMMAND", // TODO: Was "DOWNLOAD_SCRIPT" in old apps
+      "./download_internet_file.py");
+
+    let maybe_downloader_venv_script = easyenv::get_env_string_optional(
+      "WEB_DOWNLOADER_MAYBE_VENV_SCRIPT");
+
+    let docker_options = easyenv::get_env_string_optional(
+      "WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE")
+        .map(|image_name| {
+            DockerOptions {
+              image_name,
+              maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+              maybe_gpu: None,
+            }
+          });
+
+    GoogleDriveDownloadCommand::new(
+      &downloader_command,
+      maybe_downloader_venv_script.as_deref(),
+      docker_options,
+    )
+  };
+
+  let tacotron_model_check_command = {
+    let root_directory = easyenv::get_env_string_required(
+      "TACOTRON_MODEL_CHECK_ROOT_DIRECTORY")?;
+
+    let python_command = easyenv::get_env_string_or_default(
+      "TACOTRON_MODEL_CHECK_COMMAND",
+      "./vocodes_model_check_tacotron.py");
+
+    let maybe_venv_command = easyenv::get_env_string_optional(
+      "TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND");
+
+    let maybe_docker_options = easyenv::get_env_string_optional(
+      "TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE")
+        .map(|image_name| {
+          DockerOptions {
+            image_name,
+            maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+            maybe_gpu: Some(DockerGpu::All),
+          }
+        });
+
+    TacotronModelCheckCommand::new(
+      &root_directory,
+      maybe_venv_command.as_deref(),
+      &python_command,
+      maybe_docker_options,
+    )
+  };
+
+  let hifigan_model_check_command= {
+    let root_directory = easyenv::get_env_string_required(
+      "HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY")?;
+
+    let python_command = easyenv::get_env_string_or_default(
+      "HIFIGAN_MODEL_CHECK_COMMAND",
+      "./vocodes_model_check_tacotron.py");
+
+    let maybe_venv_command = easyenv::get_env_string_optional(
+      "HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND");
+
+    let maybe_docker_options = easyenv::get_env_string_optional(
+      "HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE")
+        .map(|image_name| {
+          DockerOptions {
+            image_name,
+            maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
+            maybe_gpu: Some(DockerGpu::All),
+          }
+        });
+
+    HifiGanModelCheckCommand::new(
+      &root_directory,
+      maybe_venv_command.as_deref(),
+      &python_command,
+      maybe_docker_options,
+    )
+  };
+
+  // =============== End Configure Python "Sidecars" ===============
 
   let temp_directory = PathBuf::from(temp_directory);
 
@@ -145,50 +214,21 @@ async fn main() -> AnyhowResult<()> {
     firehose_publisher: firehose_publisher.clone(), // NB: Also safe
   };
 
-  // TODO/FIXME: Cannot be deployed as currently written.
-  let tacotron_model_check_command = TacotronModelCheckCommand::new(
-    "/models/tts",
-    "source python/bin/activate",
-    "./vocodes_model_check_tacotron.py",
-    Some(DockerOptions {
-      image_name: "5642d0fd7fc1".to_string(), // storyteller-ml
-      maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
-      maybe_gpu: Some(DockerGpu::All),
-    })
-  );
-
-  // TODO/FIXME: Cannot be deployed as currently written.
-  let hifigan_model_check_command= HifiGanModelCheckCommand::new(
-    //&easyenv::get_env_string_required("HIFIGAN_ROOT_CODE_DIRECTORY")?,
-    "/models/tts",
-    //&easyenv::get_env_string_or_default(
-    //"HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND",
-    //"source python-tacotron/bin/activate"),
-    "source python/bin/activate",
-    //&easyenv::get_env_string_or_default(
-    //"HIFIGAN_MODEL_CHECK_SCRIPT_NAME",
-    //"vocodes_model_check_hifigan.py"),
-    "./vocodes_model_check_hifigan.py",
-    Some(DockerOptions {
-      image_name: "5642d0fd7fc1".to_string(), // storyteller-ml
-      maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()),
-      maybe_gpu: Some(DockerGpu::All),
-    }),
-  );
 
   let job_state = JobState {
     download_temp_directory: temp_directory,
     mysql_pool,
     redis_pool,
     bucket_client,
-    download_script,
-    google_drive_downloader,
     bucket_path_unifier: BucketPathUnifier::default_paths(),
     bucket_root_tts_model_uploads: bucket_root.to_string(),
     firehose_publisher,
     badge_granter,
-    tacotron_model_check_command,
-    hifigan_model_check_command,
+    sidecar_configs: SidecarConfigs {
+      google_drive_downloader,
+      tacotron_model_check_command,
+      hifigan_model_check_command,
+    },
     job_batch_wait_millis: common_env.job_batch_wait_millis,
     job_max_attempts: common_env.job_max_attempts as i32,
     no_op_logger_millis: common_env.no_op_logger_millis,

crates/service/w2l_download_job/src/main.rs

@@ -185,7 +185,8 @@ async fn main() -> AnyhowResult<()> {
     "DOWNLOAD_SCRIPT",
     "./scripts/download_internet_file.py");
 
-  let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
+  let google_drive_downloader =
+      GoogleDriveDownloadCommand::new_production(&download_script);
 
   let temp_directory = PathBuf::from(temp_directory);
 

crates/service/w2l_inference_job/src/main.rs

@@ -179,7 +179,8 @@ async fn main() -> AnyhowResult<()> {
   let download_script = easyenv::get_env_string_or_default(
     "DOWNLOAD_SCRIPT",
     "./scripts/download_internet_file.py");
-  let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script);
+  let google_drive_downloader =
+      GoogleDriveDownloadCommand::new_production(&download_script);
 
   let temp_directory = PathBuf::from(temp_directory);