mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-15 22:32:13 +00:00
report status of tts downloads
This commit is contained in:
1 parent
cca6d9cdfc
commit
5cc2ee3bda
4 files changed
+66
-4
No files matched your search
@@ -15,10 +15,12 @@ use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
|
||||
use crate::http_server::web_utils::session_checker::SessionRecord;
|
||||
use crate::server_state::ServerState;
|
||||
use crate::util::random_crockford_token::random_crockford_token;
|
||||
use crate::util::redis::redis_keys::RedisKeys;
|
||||
use crate::validations::passwords::validate_passwords;
|
||||
use crate::validations::username::validate_username;
|
||||
use derive_more::{Display, Error};
|
||||
use log::{info, warn, log};
|
||||
use r2d2_redis::redis::Commands;
|
||||
use regex::Regex;
|
||||
use sqlx::MySqlPool;
|
||||
use sqlx::error::DatabaseError;
|
||||
@@ -134,10 +136,29 @@ WHERE jobs.token = ?
|
||||
}
|
||||
};
|
||||
|
||||
let mut redis = server_state.redis_pool
|
||||
.get()
|
||||
.map_err(|e| {
|
||||
warn!("redis error: {:?}", e);
|
||||
GetTtsUploadModelStatusError::ServerError
|
||||
})?;
|
||||
|
||||
let extra_status_key = RedisKeys::tts_download_extra_status_info(&path.token);
|
||||
let maybe_extra_status_description : Option<String> = match redis.get(&extra_status_key) {
|
||||
Ok(Some(status)) => {
|
||||
Some(status)
|
||||
},
|
||||
Ok(None) => None,
|
||||
Err(e) => {
|
||||
warn!("redis error: {:?}", e);
|
||||
None // Fail open
|
||||
},
|
||||
};
|
||||
|
||||
let model_for_response = TtsUploadModelJobStatusForResponse {
|
||||
job_token: record.job_token.clone(),
|
||||
status: record.status.clone(),
|
||||
maybe_extra_status_description: None,
|
||||
maybe_extra_status_description,
|
||||
attempt_count: record.attempt_count as u8,
|
||||
maybe_model_token: record.maybe_model_token.clone(),
|
||||
created_at: record.created_at.clone(),
|
||||
|
||||
+33
-3
@@ -40,13 +40,17 @@ use crate::util::filesystem::{check_directory_exists, check_file_exists};
|
||||
use crate::util::hashing::hash_file_sha2::hash_file_sha2;
|
||||
use crate::util::noop_logger::NoOpLogger;
|
||||
use crate::util::random_crockford_token::random_crockford_token;
|
||||
use crate::util::redis::redis_job_status_logger::RedisJobStatusLogger;
|
||||
use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
|
||||
use log::{warn, info};
|
||||
use r2d2_redis::RedisConnectionManager;
|
||||
use r2d2_redis::r2d2;
|
||||
use ring::digest::{Context, Digest, SHA256};
|
||||
use sqlx::MySqlPool;
|
||||
use sqlx::mysql::MySqlPoolOptions;
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read};
|
||||
use std::ops::Deref;
|
||||
use std::path::{PathBuf, Path};
|
||||
use std::process::Command;
|
||||
use std::time::Duration;
|
||||
@@ -68,6 +72,9 @@ const ENV_MODEL_CHECK_SCRIPT_NAME : &'static str = "TTS_MODEL_CHECK_SCRIPT_NAME"
|
||||
struct Downloader {
|
||||
pub download_temp_directory: PathBuf,
|
||||
pub mysql_pool: MySqlPool,
|
||||
|
||||
pub redis_pool: r2d2::Pool<RedisConnectionManager>,
|
||||
|
||||
pub bucket_client: BucketClient,
|
||||
pub firehose_publisher: FirehosePublisher,
|
||||
pub badge_granter: BadgeGranter,
|
||||
@@ -149,6 +156,16 @@ async fn main() -> AnyhowResult<()> {
|
||||
.connect(&db_connection_string)
|
||||
.await?;
|
||||
|
||||
let common_env = CommonEnv::read_from_env()?;
|
||||
|
||||
info!("Connecting to redis...");
|
||||
|
||||
let redis_manager =
|
||||
RedisConnectionManager::new(common_env.redis_connection_string.deref())?;
|
||||
|
||||
let redis_pool = r2d2::Pool::builder()
|
||||
.build(redis_manager)?;
|
||||
|
||||
let firehose_publisher = FirehosePublisher {
|
||||
mysql_pool: mysql_pool.clone(), // NB: Pool is sync/send/clone-safe
|
||||
};
|
||||
@@ -166,11 +183,10 @@ async fn main() -> AnyhowResult<()> {
|
||||
&py_script_name,
|
||||
);
|
||||
|
||||
let common_env = CommonEnv::read_from_env()?;
|
||||
|
||||
let downloader = Downloader {
|
||||
download_temp_directory: temp_directory,
|
||||
mysql_pool,
|
||||
redis_pool,
|
||||
bucket_client,
|
||||
download_script,
|
||||
google_drive_downloader,
|
||||
@@ -276,6 +292,11 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
|
||||
// TODO: 4. Save record. (DONE)
|
||||
// TODO: 5. Mark job done. (DONE)
|
||||
|
||||
let mut redis = downloader.redis_pool.get()?;
|
||||
let mut redis_logger = RedisJobStatusLogger::new_tts_download(
|
||||
&mut redis,
|
||||
&job.token);
|
||||
|
||||
// ==================== ATTEMPT TO GRAB JOB LOCK ==================== //
|
||||
|
||||
let lock_acquired = grab_job_lock_and_mark_pending(&downloader.mysql_pool, job).await?;
|
||||
@@ -292,11 +313,14 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
|
||||
|
||||
// ==================== DOWNLOAD MODEL FILE ==================== //
|
||||
|
||||
info!("Calling downloader...");
|
||||
|
||||
redis_logger.log_status("downloading model")?;
|
||||
|
||||
let download_url = job.download_url.as_ref()
|
||||
.map(|c| c.to_string())
|
||||
.unwrap_or("".to_string());
|
||||
|
||||
info!("Calling downloader...");
|
||||
let download_filename = downloader.google_drive_downloader
|
||||
.download_file(&download_url, &temp_dir).await?;
|
||||
|
||||
@@ -314,6 +338,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
|
||||
|
||||
info!("Checking that model is valid...");
|
||||
|
||||
redis_logger.log_status("checking model")?;
|
||||
|
||||
let file_path = PathBuf::from(download_filename);
|
||||
|
||||
let output_metadata_fs_path = temp_dir.path().join("metadata.json");
|
||||
@@ -336,6 +362,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
|
||||
|
||||
info!("Uploading model to GCS...");
|
||||
|
||||
redis_logger.log_status("uploading model")?;
|
||||
|
||||
downloader.bucket_client.upload_filename(&synthesizer_model_bucket_path, &file_path).await?;
|
||||
|
||||
// ==================== SAVE RECORDS ==================== //
|
||||
@@ -373,6 +401,8 @@ async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> Anyho
|
||||
anyhow!("error maybe awarding badge")
|
||||
})?;
|
||||
|
||||
redis_logger.log_status("done")?;
|
||||
|
||||
info!("Job done: {}", job.id);
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -14,6 +14,11 @@ pub struct RedisJobStatusLogger <'a> {
|
||||
impl <'a> RedisJobStatusLogger <'a> {
|
||||
pub const STATUS_KEY_TTL_SECONDS : usize = 60 * 60;
|
||||
|
||||
pub fn new_tts_download(redis: &'a mut RedisPool, w2l_job_token: &str) -> Self {
|
||||
let status_key = RedisKeys::tts_download_extra_status_info(w2l_job_token);
|
||||
Self { redis, status_key }
|
||||
}
|
||||
|
||||
pub fn new_tts_inference(redis: &'a mut RedisPool, tts_job_token: &str) -> Self {
|
||||
let status_key = RedisKeys::tts_inference_extra_status_info(tts_job_token);
|
||||
Self { redis, status_key }
|
||||
|
||||
@@ -16,6 +16,12 @@ impl RedisKeys {
|
||||
format!("w2lUseCount:{}", template_token)
|
||||
}
|
||||
|
||||
/// We write extra in-progress status information to keys.
|
||||
/// These keys should have a TTL.
|
||||
pub fn tts_download_extra_status_info(inference_job_token: &str) -> String {
|
||||
format!("ttsDownloadExtraStatus:{}", inference_job_token)
|
||||
}
|
||||
|
||||
/// We write extra in-progress status information to keys.
|
||||
/// These keys should have a TTL.
|
||||
pub fn tts_inference_extra_status_info(inference_job_token: &str) -> String {
|
||||
|
||||
Reference in new issue
Block a user