From 5cc2ee3bdad627547e52c4afae7ae8582b0d7355 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 28 Aug 2021 04:06:38 -0400 Subject: [PATCH] report status of tts downloads --- .../tts/get_tts_upload_model_job_status.rs | 23 +++++++++++- src/tts_download_job.rs | 36 +++++++++++++++++-- src/util/redis/redis_job_status_logger.rs | 5 +++ src/util/redis/redis_keys.rs | 6 ++++ 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs index af1c7191ba..956fe3da5b 100644 --- a/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs +++ b/src/http_server/endpoints/tts/get_tts_upload_model_job_status.rs @@ -15,10 +15,12 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::http_server::web_utils::session_checker::SessionRecord; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_keys::RedisKeys; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; use derive_more::{Display, Error}; use log::{info, warn, log}; +use r2d2_redis::redis::Commands; use regex::Regex; use sqlx::MySqlPool; use sqlx::error::DatabaseError; @@ -134,10 +136,29 @@ WHERE jobs.token = ? } }; + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + GetTtsUploadModelStatusError::ServerError + })?; + + let extra_status_key = RedisKeys::tts_download_extra_status_info(&path.token); + let maybe_extra_status_description : Option = match redis.get(&extra_status_key) { + Ok(Some(status)) => { + Some(status) + }, + Ok(None) => None, + Err(e) => { + warn!("redis error: {:?}", e); + None // Fail open + }, + }; + let model_for_response = TtsUploadModelJobStatusForResponse { job_token: record.job_token.clone(), status: record.status.clone(), - maybe_extra_status_description: None, + maybe_extra_status_description, attempt_count: record.attempt_count as u8, maybe_model_token: record.maybe_model_token.clone(), created_at: record.created_at.clone(), diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index f5c7594071..95f30b27d4 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -40,13 +40,17 @@ use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2; use ring::digest::{Context, Digest, SHA256}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; use std::io::{BufReader, Read}; +use std::ops::Deref; use std::path::{PathBuf, Path}; use std::process::Command; use std::time::Duration; @@ -68,6 +72,9 @@ const ENV_MODEL_CHECK_SCRIPT_NAME : &'static str = "TTS_MODEL_CHECK_SCRIPT_NAME" struct Downloader { pub download_temp_directory: PathBuf, pub mysql_pool: MySqlPool, + + pub redis_pool: r2d2::Pool, + pub bucket_client: BucketClient, pub firehose_publisher: FirehosePublisher, pub badge_granter: BadgeGranter, @@ -149,6 +156,16 @@ async fn main() -> AnyhowResult<()> { .connect(&db_connection_string) .await?; + let common_env = CommonEnv::read_from_env()?; + + info!("Connecting to redis..."); + + let redis_manager = + RedisConnectionManager::new(common_env.redis_connection_string.deref())?; + + let redis_pool = r2d2::Pool::builder() + .build(redis_manager)?; + let firehose_publisher = FirehosePublisher { mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe }; @@ -166,11 +183,10 @@ async fn main() -> AnyhowResult<()> { &py_script_name, ); - let common_env = CommonEnv::read_from_env()?; - let downloader = Downloader { download_temp_directory: temp_directory, mysql_pool, + redis_pool, bucket_client, download_script, google_drive_downloader, @@ -276,6 +292,11 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) + let mut redis = downloader.redis_pool.get()?; + let mut redis_logger = RedisJobStatusLogger::new_tts_download( + &mut redis, + &job.token); + // ==================== ATTEMPT TO GRAB JOB LOCK ==================== // let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?; @@ -292,11 +313,14 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho // ==================== DOWNLOAD MODEL FILE ==================== // + info!("Calling downloader..."); + + redis_logger.log_status("downloading model")?; + let download_url = job.download_url.as_ref() .map(|c| c.to_string()) .unwrap_or("".to_string()); - info!("Calling downloader..."); let download_filename = downloader.google_drive_downloader .download_file(&download_url, &temp_dir).await?; @@ -314,6 +338,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Checking that model is valid..."); + redis_logger.log_status("checking model")?; + let file_path = PathBuf::from(download_filename); let output_metadata_fs_path = temp_dir.path().join("metadata.json"); @@ -336,6 +362,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho info!("Uploading model to GCS..."); + redis_logger.log_status("uploading model")?; + downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?; // ==================== SAVE RECORDS ==================== // @@ -373,6 +401,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho anyhow!("error maybe awarding badge") })?; + redis_logger.log_status("done")?; + info!("Job done: {}", job.id); Ok(()) diff --git a/src/util/redis/redis_job_status_logger.rs b/src/util/redis/redis_job_status_logger.rs index a407e80339..42d2dec919 100644 --- a/src/util/redis/redis_job_status_logger.rs +++ b/src/util/redis/redis_job_status_logger.rs @@ -14,6 +14,11 @@ pub struct RedisJobStatusLogger <'a> { impl <'a> RedisJobStatusLogger <'a> { pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60; + pub fn new_tts_download(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self { + let status_key = RedisKeys::tts_download_extra_status_info(w2l_job_token); + Self { redis, status_key } + } + pub fn new_tts_inference(redis: &'a mut RedisPool, tts_job_token: &str) -> Self { let status_key = RedisKeys::tts_inference_extra_status_info(tts_job_token); Self { redis, status_key } diff --git a/src/util/redis/redis_keys.rs b/src/util/redis/redis_keys.rs index 3455c60183..260c2ba8dc 100644 --- a/src/util/redis/redis_keys.rs +++ b/src/util/redis/redis_keys.rs @@ -16,6 +16,12 @@ impl RedisKeys { format!("w2lUseCount:{}", template_token) } + /// We write extra in-progress status information to keys. + /// These keys should have a TTL. + pub fn tts_download_extra_status_info(inference_job_token: &str) -> String { + format!("ttsDownloadExtraStatus:{}", inference_job_token) + } + /// We write extra in-progress status information to keys. /// These keys should have a TTL. pub fn tts_inference_extra_status_info(inference_job_token: &str) -> String {