report status of tts downloads

This commit is contained in:
Brandon Thomas committed 2021-08-28 04:06:38 -04:00
1 parent cca6d9cdfc
commit 5cc2ee3bda
4 files changed
+66 -4

No files matched your search

@@ -15,10 +15,12 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::http_server::web_utils::session_checker::SessionRecord;
use crate::server_state::ServerState;
use crate::util::random_crockford_token::random_crockford_token;
use crate::util::redis::redis_keys::RedisKeys;
use crate::validations::passwords::validate_passwords;
use crate::validations::username::validate_username;
use derive_more::{Display, Error};
use log::{info, warn, log};
use r2d2_redis::redis::Commands;
use regex::Regex;
use sqlx::MySqlPool;
use sqlx::error::DatabaseError;
@@ -134,10 +136,29 @@ WHERE jobs.token = ?
}
};
let mut redis = server_state.redis_pool
.get()
.map_err(|e| {
warn!("redis error: {:?}", e);
GetTtsUploadModelStatusError::ServerError
})?;
let extra_status_key = RedisKeys::tts_download_extra_status_info(&path.token);
let maybe_extra_status_description : Option<String> = match redis.get(&extra_status_key) {
Ok(Some(status)) => {
Some(status)
},
Ok(None) => None,
Err(e) => {
warn!("redis error: {:?}", e);
None // Fail open
},
};
let model_for_response = TtsUploadModelJobStatusForResponse {
job_token: record.job_token.clone(),
status: record.status.clone(),
maybe_extra_status_description: None,
maybe_extra_status_description,
attempt_count: record.attempt_count as u8,
maybe_model_token: record.maybe_model_token.clone(),
created_at: record.created_at.clone(),
+33 -3
View File
@@ -40,13 +40,17 @@ use crate::util::filesystem::{check_directory_exists, check_file_exists};
use crate::util::hashing::hash_file_sha2::hash_file_sha2;
use crate::util::noop_logger::NoOpLogger;
use crate::util::random_crockford_token::random_crockford_token;
use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger;
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
use log::{warn, info};
use r2d2_redis::RedisConnectionManager;
use r2d2_redis::r2d2;
use ring::digest::{Context, Digest, SHA256};
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use std::fs::File;
use std::io::{BufReader, Read};
use std::ops::Deref;
use std::path::{PathBuf, Path};
use std::process::Command;
use std::time::Duration;
@@ -68,6 +72,9 @@ const ENV_MODEL_CHECK_SCRIPT_NAME : &'static str = "TTS_MODEL_CHECK_SCRIPT_NAME"
struct Downloader {
pub download_temp_directory: PathBuf,
pub mysql_pool: MySqlPool,
pub redis_pool: r2d2::Pool<RedisConnectionManager>,
pub bucket_client: BucketClient,
pub firehose_publisher: FirehosePublisher,
pub badge_granter: BadgeGranter,
@@ -149,6 +156,16 @@ async fn main() -> AnyhowResult<()> {
.connect(&db_connection_string)
.await?;
let common_env = CommonEnv::read_from_env()?;
info!("Connecting to redis...");
let redis_manager =
RedisConnectionManager::new(common_env.redis_connection_string.deref())?;
let redis_pool = r2d2::Pool::builder()
.build(redis_manager)?;
let firehose_publisher = FirehosePublisher {
mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe
};
@@ -166,11 +183,10 @@ async fn main() -> AnyhowResult<()> {
&py_script_name,
);
let common_env = CommonEnv::read_from_env()?;
let downloader = Downloader {
download_temp_directory: temp_directory,
mysql_pool,
redis_pool,
bucket_client,
download_script,
google_drive_downloader,
@@ -276,6 +292,11 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
// TODO: 4. Save record. (DONE)
// TODO: 5. Mark job done. (DONE)
let mut redis = downloader.redis_pool.get()?;
let mut redis_logger = RedisJobStatusLogger::new_tts_download(
&mut redis,
&job.token);
// ==================== ATTEMPT TO GRAB JOB LOCK ==================== //
let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?;
@@ -292,11 +313,14 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
// ==================== DOWNLOAD MODEL FILE ==================== //
info!("Calling downloader...");
redis_logger.log_status("downloading model")?;
let download_url = job.download_url.as_ref()
.map(|c| c.to_string())
.unwrap_or("".to_string());
info!("Calling downloader...");
let download_filename = downloader.google_drive_downloader
.download_file(&download_url, &temp_dir).await?;
@@ -314,6 +338,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
info!("Checking that model is valid...");
redis_logger.log_status("checking model")?;
let file_path = PathBuf::from(download_filename);
let output_metadata_fs_path = temp_dir.path().join("metadata.json");
@@ -336,6 +362,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
info!("Uploading model to GCS...");
redis_logger.log_status("uploading model")?;
downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?;
// ==================== SAVE RECORDS ==================== //
@@ -373,6 +401,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
anyhow!("error maybe awarding badge")
})?;
redis_logger.log_status("done")?;
info!("Job done: {}", job.id);
Ok(())
@@ -14,6 +14,11 @@ pub struct RedisJobStatusLogger <'a> {
impl <'a> RedisJobStatusLogger <'a> {
pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60;
pub fn new_tts_download(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self {
let status_key = RedisKeys::tts_download_extra_status_info(w2l_job_token);
Self { redis, status_key }
}
pub fn new_tts_inference(redis: &'a mut RedisPool, tts_job_token: &str) -> Self {
let status_key = RedisKeys::tts_inference_extra_status_info(tts_job_token);
Self { redis, status_key }
+6
View File
@@ -16,6 +16,12 @@ impl RedisKeys {
format!("w2lUseCount:{}", template_token)
}
/// We write extra in-progress status information to keys.
/// These keys should have a TTL.
pub fn tts_download_extra_status_info(inference_job_token: &str) -> String {
format!("ttsDownloadExtraStatus:{}", inference_job_token)
}
/// We write extra in-progress status information to keys.
/// These keys should have a TTL.
pub fn tts_inference_extra_status_info(inference_job_token: &str) -> String {