From d28fb07d6d21aa322b3f11fa36ef623ac1394bdb Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Dec 2022 21:44:32 -0500 Subject: [PATCH] Clean up new consolidated downloader so it can be deployed --- .env-download-job | 19 +++ .idea/dictionaries/bt.xml | 1 + Dockerfile | 1 + .../src/google_drive_download_command.rs | 14 +- crates/service/download_job/src/job_state.rs | 13 +- .../src/job_steps/process_single_job.rs | 2 +- .../hifigan/hifigan_model_check_command.rs | 15 +- .../hifigan/process_hifigan_vocoder.rs | 2 +- .../tacotron/process_tacotron_model.rs | 2 +- .../tacotron/tacotron_model_check_command.rs | 15 +- crates/service/download_job/src/main.rs | 148 +++++++++++------- crates/service/w2l_download_job/src/main.rs | 3 +- crates/service/w2l_inference_job/src/main.rs | 3 +- 13 files changed, 163 insertions(+), 75 deletions(-) create mode 100644 .env-download-job diff --git a/.env-download-job b/.env-download-job new file mode 100644 index 0000000000..def03d6262 --- /dev/null +++ b/.env-download-job @@ -0,0 +1,19 @@ +# Environmental configurations specific to `download-job`. +# These are defaulted for dev and are easy to overwrite in production with K8s environment variables. +# !! DOES *NOT* CONTAIN SECRETS !! + +WEB_DOWNLOADER_COMMAND=./download_internet_file.py +WEB_DOWNLOADER_MAYBE_VENV_SCRIPT=./python/bin/activate +WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE=d73f28ce3ff6 + +TACOTRON_MODEL_CHECK_ROOT_DIRECTORY=/models/tts +TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate" +TACOTRON_MODEL_CHECK_COMMAND=./vocodes_model_check_tacotron.py +TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1 + +HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY=/models/tts +HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND="source python/bin/activate" +HIFIGAN_MODEL_CHECK_COMMAND=./vocodes_model_check_hifigan.py +HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE=5642d0fd7fc1 + + diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index f507d3e88f..9f6dca8dc5 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -12,6 +12,7 @@ upserting vocoder vocoders + vocodes \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 8ffd82111f..dd52eb70d6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -169,6 +169,7 @@ RUN ldd storyteller-web # Without a .env file, Rust crashes "mysteriously" (ugh) RUN touch .env +RUN touch .env-download-job RUN touch .env-secrets EXPOSE 8080 diff --git a/crates/lib/google_drive_common/src/google_drive_download_command.rs b/crates/lib/google_drive_common/src/google_drive_download_command.rs index f95b59debc..cc7a05dd84 100644 --- a/crates/lib/google_drive_common/src/google_drive_download_command.rs +++ b/crates/lib/google_drive_common/src/google_drive_download_command.rs @@ -28,7 +28,19 @@ pub struct GoogleDriveDownloadCommand { } impl GoogleDriveDownloadCommand { - pub fn new(download_script: &str) -> Self { + pub fn new( + download_script: &str, + maybe_venv_activation_script: Option<&str>, + maybe_docker_options: Option + ) -> Self { + Self { + download_script: download_script.to_string(), + maybe_venv_activation_script: maybe_venv_activation_script.map(|s| s.to_string()), + maybe_docker_options, + } + } + + pub fn new_production(download_script: &str) -> Self { Self { download_script: download_script.to_string(), maybe_venv_activation_script: None, diff --git a/crates/service/download_job/src/job_state.rs b/crates/service/download_job/src/job_state.rs index 5f46f10346..18169265e3 100644 --- a/crates/service/download_job/src/job_state.rs +++ b/crates/service/download_job/src/job_state.rs @@ -19,15 +19,11 @@ pub struct JobState { pub bucket_client: BucketClient, pub firehose_publisher: FirehosePublisher, pub badge_granter: BadgeGranter, - pub google_drive_downloader: GoogleDriveDownloadCommand, pub bucket_path_unifier: BucketPathUnifier, - pub tacotron_model_check_command: TacotronModelCheckCommand, - pub hifigan_model_check_command: HifiGanModelCheckCommand, + pub sidecar_configs: SidecarConfigs, - // Command to run - pub download_script: String, // Root to store TTS results pub bucket_root_tts_model_uploads: String, @@ -41,3 +37,10 @@ pub struct JobState { // NB: This is an i32 so we don't need to convert to db column type. pub job_max_attempts: i32, } + +/// Configurations and interfaces to code deployed as sidecars or container mounts. +pub struct SidecarConfigs { + pub google_drive_downloader: GoogleDriveDownloadCommand, + pub tacotron_model_check_command: TacotronModelCheckCommand, + pub hifigan_model_check_command: HifiGanModelCheckCommand, +} \ No newline at end of file diff --git a/crates/service/download_job/src/job_steps/process_single_job.rs b/crates/service/download_job/src/job_steps/process_single_job.rs index d49329047e..1442aaf69b 100644 --- a/crates/service/download_job/src/job_steps/process_single_job.rs +++ b/crates/service/download_job/src/job_steps/process_single_job.rs @@ -42,7 +42,7 @@ pub async fn process_single_job(job_state: &JobState, job: &AvailableDownloadJob return Err(anyhow!("Bad download URL: `{}`", &job.download_url)); } - let download_filename = match job_state.google_drive_downloader.download_file(&job.download_url, &temp_dir).await { + let download_filename = match job_state.sidecar_configs.google_drive_downloader.download_file(&job.download_url, &temp_dir).await { Ok(filename) => filename, Err(e) => { safe_delete_temp_directory(&temp_dir); diff --git a/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs b/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs index 73a2dea8e0..37a8cc5d62 100644 --- a/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs +++ b/crates/service/download_job/src/job_types/hifigan/hifigan_model_check_command.rs @@ -11,7 +11,7 @@ pub struct HifiGanModelCheckCommand { hifigan_root_code_directory: String, /// eg. `source python/bin/activate` - virtual_env_activation_command: String, + maybe_virtual_env_activation_command: Option, hifigan_model_check_script_name: String, @@ -22,13 +22,13 @@ pub struct HifiGanModelCheckCommand { impl HifiGanModelCheckCommand { pub fn new( hifigan_root_code_directory: &str, - virtual_env_activation_command: &str, + maybe_virtual_env_activation_command: Option<&str>, hifigan_model_check_script_name: &str, maybe_docker_options: Option, ) -> Self { Self { hifigan_root_code_directory: hifigan_root_code_directory.to_string(), - virtual_env_activation_command: virtual_env_activation_command.to_string(), + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), hifigan_model_check_script_name: hifigan_model_check_script_name.to_string(), maybe_docker_options, } @@ -42,8 +42,13 @@ impl HifiGanModelCheckCommand { let mut command = String::new(); command.push_str(&format!("cd {}", self.hifigan_root_code_directory)); - command.push_str(" && "); - command.push_str(&self.virtual_env_activation_command); + + if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() { + command.push_str(" && "); + command.push_str(venv_command); + command.push_str(" "); + } + command.push_str(" && "); command.push_str("python "); command.push_str(&self.hifigan_model_check_script_name); diff --git a/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs b/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs index c1973efb8f..12665dafbb 100644 --- a/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs +++ b/crates/service/download_job/src/job_types/hifigan/process_hifigan_vocoder.rs @@ -35,7 +35,7 @@ pub async fn process_hifigan_vocoder<'a, 'b>( let output_metadata_fs_path = temp_dir.path().join("metadata.json"); - let model_check_result = job_state.hifigan_model_check_command.execute( + let model_check_result = job_state.sidecar_configs.hifigan_model_check_command.execute( &file_path, &output_metadata_fs_path ); diff --git a/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs b/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs index b6e1e0c8e6..95c2c9516e 100644 --- a/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs +++ b/crates/service/download_job/src/job_types/tacotron/process_tacotron_model.rs @@ -36,7 +36,7 @@ pub async fn process_tacotron_model<'a, 'b>( let output_metadata_fs_path = temp_dir.path().join("metadata.json"); - let model_check_result = job_state.tacotron_model_check_command.execute( + let model_check_result = job_state.sidecar_configs.tacotron_model_check_command.execute( &file_path, &output_metadata_fs_path, false diff --git a/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs b/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs index b64e5edd41..35d829aef4 100644 --- a/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs +++ b/crates/service/download_job/src/job_types/tacotron/tacotron_model_check_command.rs @@ -12,7 +12,7 @@ pub struct TacotronModelCheckCommand { tacotron_root_code_directory: String, /// eg. `source python/bin/activate` - virtual_env_activation_command: String, + maybe_virtual_env_activation_command: Option, tacotron_model_check_script_name: String, @@ -23,13 +23,13 @@ pub struct TacotronModelCheckCommand { impl TacotronModelCheckCommand { pub fn new( tacotron_root_code_directory: &str, - virtual_env_activation_command: &str, + maybe_virtual_env_activation_command: Option<&str>, tacotron_model_check_script_name: &str, maybe_docker_options: Option, ) -> Self { Self { tacotron_root_code_directory: tacotron_root_code_directory.to_string(), - virtual_env_activation_command: virtual_env_activation_command.to_string(), + maybe_virtual_env_activation_command: maybe_virtual_env_activation_command.map(|s| s.to_string()), tacotron_model_check_script_name: tacotron_model_check_script_name.to_string(), maybe_docker_options, } @@ -46,8 +46,13 @@ impl TacotronModelCheckCommand { command.push_str("echo 'test'"); command.push_str(" && "); command.push_str(&format!("cd {}", self.tacotron_root_code_directory)); - command.push_str(" && "); - command.push_str(&self.virtual_env_activation_command); + + if let Some(venv_command) = self.maybe_virtual_env_activation_command.as_deref() { + command.push_str(" && "); + command.push_str(venv_command); + command.push_str(" "); + } + command.push_str(" && "); command.push_str("python "); command.push_str(&self.tacotron_model_check_script_name); diff --git a/crates/service/download_job/src/main.rs b/crates/service/download_job/src/main.rs index 82f2d58144..69e93aa06b 100644 --- a/crates/service/download_job/src/main.rs +++ b/crates/service/download_job/src/main.rs @@ -26,9 +26,10 @@ use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use config::shared_constants::DEFAULT_RUST_LOG; use container_common::anyhow_result::AnyhowResult; use container_common::filesystem::check_directory_exists::check_directory_exists; -use crate::job_state::JobState; +use crate::job_state::{JobState, SidecarConfigs}; use crate::job_steps::main_loop::main_loop; use crate::job_types::hifigan::hifigan_model_check_command::HifiGanModelCheckCommand; +use crate::job_types::tacotron::tacotron_model_check_command::TacotronModelCheckCommand; use database_queries::mediators::badge_granter::BadgeGranter; use database_queries::mediators::firehose_publisher::FirehosePublisher; use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand; @@ -42,7 +43,6 @@ use std::time::Duration; use storage_buckets_common::bucket_client::BucketClient; use storage_buckets_common::bucket_path_unifier::BucketPathUnifier; use subprocess_common::docker_options::{DockerFilesystemMount, DockerGpu, DockerOptions}; -use crate::job_types::tacotron::tacotron_model_check_command::TacotronModelCheckCommand; // Buckets const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -57,7 +57,8 @@ const DEFAULT_TEMP_DIR: &'static str = "/tmp"; async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); - let _ = dotenv::from_filename(".env-secrets").ok(); + let _ = dotenv::from_filename(".env-download-job").ok(); // NB: Specific to `download-job` app. + let _ = dotenv::from_filename(".env-secrets").ok(); // NB: Secrets not to live in source control. info!("Obtaining hostname..."); @@ -91,24 +92,92 @@ async fn main() -> AnyhowResult<()> { "DOWNLOAD_TEMP_DIR", DEFAULT_TEMP_DIR); - let download_script = easyenv::get_env_string_or_default( - "DOWNLOAD_SCRIPT", - "./scripts/download_internet_file.py"); + // =============== Configure Python "Sidecars" =============== - // TODO/FIXME: Cannot be deployed as currently written. - //let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); - let google_drive_downloader = GoogleDriveDownloadCommand::new_local_dev_docker( - "./download_internet_file.py", - "./python/bin/activate", - DockerOptions { - image_name: "d73f28ce3ff6".to_string(), // web-downloader - maybe_bind_mount: Some(DockerFilesystemMount { - local_filesystem: "/tmp".to_string(), - container_filesystem: "/tmp".to_string() - }), - maybe_gpu: None, - } - ); + let google_drive_downloader = { + let downloader_command= easyenv::get_env_string_or_default( + "WEB_DOWNLOADER_COMMAND", // TODO: Was "DOWNLOAD_SCRIPT" in old apps + "./download_internet_file.py"); + + let maybe_downloader_venv_script = easyenv::get_env_string_optional( + "WEB_DOWNLOADER_MAYBE_VENV_SCRIPT"); + + let docker_options = easyenv::get_env_string_optional( + "WEB_DOWNLOADER_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_gpu: None, + } + }); + + GoogleDriveDownloadCommand::new( + &downloader_command, + maybe_downloader_venv_script.as_deref(), + docker_options, + ) + }; + + let tacotron_model_check_command = { + let root_directory = easyenv::get_env_string_required( + "TACOTRON_MODEL_CHECK_ROOT_DIRECTORY")?; + + let python_command = easyenv::get_env_string_or_default( + "TACOTRON_MODEL_CHECK_COMMAND", + "./vocodes_model_check_tacotron.py"); + + let maybe_venv_command = easyenv::get_env_string_optional( + "TACOTRON_MODEL_CHECK_MAYBE_VENV_COMMAND"); + + let maybe_docker_options = easyenv::get_env_string_optional( + "TACOTRON_MODEL_CHECK_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_gpu: Some(DockerGpu::All), + } + }); + + TacotronModelCheckCommand::new( + &root_directory, + maybe_venv_command.as_deref(), + &python_command, + maybe_docker_options, + ) + }; + + let hifigan_model_check_command= { + let root_directory = easyenv::get_env_string_required( + "HIFIGAN_MODEL_CHECK_ROOT_DIRECTORY")?; + + let python_command = easyenv::get_env_string_or_default( + "HIFIGAN_MODEL_CHECK_COMMAND", + "./vocodes_model_check_tacotron.py"); + + let maybe_venv_command = easyenv::get_env_string_optional( + "HIFIGAN_MODEL_CHECK_MAYBE_VENV_COMMAND"); + + let maybe_docker_options = easyenv::get_env_string_optional( + "HIFIGAN_MODEL_CHECK_MAYBE_DOCKER_IMAGE") + .map(|image_name| { + DockerOptions { + image_name, + maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), + maybe_gpu: Some(DockerGpu::All), + } + }); + + HifiGanModelCheckCommand::new( + &root_directory, + maybe_venv_command.as_deref(), + &python_command, + maybe_docker_options, + ) + }; + + // =============== End Configure Python "Sidecars" =============== let temp_directory = PathBuf::from(temp_directory); @@ -145,50 +214,21 @@ async fn main() -> AnyhowResult<()> { firehose_publisher: firehose_publisher.clone(), // NB: Also safe }; - // TODO/FIXME: Cannot be deployed as currently written. - let tacotron_model_check_command = TacotronModelCheckCommand::new( - "/models/tts", - "source python/bin/activate", - "./vocodes_model_check_tacotron.py", - Some(DockerOptions { - image_name: "5642d0fd7fc1".to_string(), // storyteller-ml - maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), - maybe_gpu: Some(DockerGpu::All), - }) - ); - - // TODO/FIXME: Cannot be deployed as currently written. - let hifigan_model_check_command= HifiGanModelCheckCommand::new( - //&easyenv::get_env_string_required("HIFIGAN_ROOT_CODE_DIRECTORY")?, - "/models/tts", - //&easyenv::get_env_string_or_default( - //"HIFIGAN_VIRTUAL_ENV_ACTIVATION_COMMAND", - //"source python-tacotron/bin/activate"), - "source python/bin/activate", - //&easyenv::get_env_string_or_default( - //"HIFIGAN_MODEL_CHECK_SCRIPT_NAME", - //"vocodes_model_check_hifigan.py"), - "./vocodes_model_check_hifigan.py", - Some(DockerOptions { - image_name: "5642d0fd7fc1".to_string(), // storyteller-ml - maybe_bind_mount: Some(DockerFilesystemMount::tmp_to_tmp()), - maybe_gpu: Some(DockerGpu::All), - }), - ); let job_state = JobState { download_temp_directory: temp_directory, mysql_pool, redis_pool, bucket_client, - download_script, - google_drive_downloader, bucket_path_unifier: BucketPathUnifier::default_paths(), bucket_root_tts_model_uploads: bucket_root.to_string(), firehose_publisher, badge_granter, - tacotron_model_check_command, - hifigan_model_check_command, + sidecar_configs: SidecarConfigs { + google_drive_downloader, + tacotron_model_check_command, + hifigan_model_check_command, + }, job_batch_wait_millis: common_env.job_batch_wait_millis, job_max_attempts: common_env.job_max_attempts as i32, no_op_logger_millis: common_env.no_op_logger_millis, diff --git a/crates/service/w2l_download_job/src/main.rs b/crates/service/w2l_download_job/src/main.rs index cfa1c34252..ab773216ec 100644 --- a/crates/service/w2l_download_job/src/main.rs +++ b/crates/service/w2l_download_job/src/main.rs @@ -185,7 +185,8 @@ async fn main() -> AnyhowResult<()> { "DOWNLOAD_SCRIPT", "./scripts/download_internet_file.py"); - let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + let google_drive_downloader = + GoogleDriveDownloadCommand::new_production(&download_script); let temp_directory = PathBuf::from(temp_directory); diff --git a/crates/service/w2l_inference_job/src/main.rs b/crates/service/w2l_inference_job/src/main.rs index a6126ba98e..0bb6f95ef5 100644 --- a/crates/service/w2l_inference_job/src/main.rs +++ b/crates/service/w2l_inference_job/src/main.rs @@ -179,7 +179,8 @@ async fn main() -> AnyhowResult<()> { let download_script = easyenv::get_env_string_or_default( "DOWNLOAD_SCRIPT", "./scripts/download_internet_file.py"); - let google_drive_downloader = GoogleDriveDownloadCommand::new(&download_script); + let google_drive_downloader = + GoogleDriveDownloadCommand::new_production(&download_script); let temp_directory = PathBuf::from(temp_directory);