From 09a6aa286d1e3389300de388b790bd27fb557ed7 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 22 May 2021 04:28:29 -0400 Subject: [PATCH] filesystem tempdir --- src/tts_download_job.rs | 44 ++++++++++++++++++++++++++++++----------- src/util/filesystem.rs | 13 ++++++++++++ src/util/mod.rs | 1 + 3 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 src/util/filesystem.rs diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index d3c0f8fb2d..df6a40c9ce 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -15,14 +15,21 @@ use crate::job::job_queries::mark_tts_upload_job_done; use crate::job::job_queries::mark_tts_upload_job_failure; use crate::job::job_queries::query_tts_upload_job_records; use crate::util::anyhow_result::AnyhowResult; +use crate::util::filesystem::check_directory_exists; use crate::util::random_token::random_token; use log::{warn, info}; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; +use std::path::PathBuf; use std::time::Duration; const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info"; +struct Downloader { + pub download_temp_directory: PathBuf, + pub mysql_pool: MySqlPool, +} + #[async_std::main] async fn main() -> AnyhowResult<()> { easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); @@ -36,6 +43,14 @@ async fn main() -> AnyhowResult<()> { info!("Hostname: {}", &server_hostname); + let temp_directory = easyenv::get_env_string_or_default( + "DOWNLOAD_TEMP_DIR", + "/tmp"); + + let temp_directory = PathBuf::from(temp_directory); + + check_directory_exists(&temp_directory)?; + let db_connection_string = easyenv::get_env_string_or_default( "MYSQL_URL", @@ -43,12 +58,17 @@ async fn main() -> AnyhowResult<()> { info!("Connecting to database..."); - let pool = MySqlPoolOptions::new() + let mysql_pool = MySqlPoolOptions::new() .max_connections(5) .connect(&db_connection_string) .await?; - main_loop(pool).await; + let downloader = Downloader { + download_temp_directory: temp_directory, + mysql_pool, + }; + + main_loop(downloader).await; Ok(()) } @@ -56,12 +76,12 @@ async fn main() -> AnyhowResult<()> { const START_TIMEOUT_MILLIS : u64 = 500; const INCREASE_TIMEOUT_MILLIS : u64 = 1000; -async fn main_loop(pool: MySqlPool) { +async fn main_loop(downloader: Downloader) { let mut timeout_millis = START_TIMEOUT_MILLIS; loop { let num_records = 1; - let query_result = query_tts_upload_job_records(&pool, num_records).await; + let query_result = query_tts_upload_job_records(&downloader.mysql_pool, num_records).await; let jobs = match query_result { Ok(jobs) => jobs, @@ -79,7 +99,7 @@ async fn main_loop(pool: MySqlPool) { continue; } - let result = process_jobs(&pool, jobs).await; + let result = process_jobs(&downloader, jobs).await; match result { Ok(_) => {}, @@ -97,15 +117,15 @@ async fn main_loop(pool: MySqlPool) { } } -async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> AnyhowResult<()> { +async fn process_jobs(downloader: &Downloader, jobs: Vec) -> AnyhowResult<()> { for job in jobs.into_iter() { - let result = process_job(pool, &job).await; + let result = process_job(downloader, &job).await; match result { Ok(_) => {}, Err(e) => { warn!("Failure to process job: {:?}", e); let failure_reason = ""; - let _r = mark_tts_upload_job_failure(pool, &job, failure_reason).await; + let _r = mark_tts_upload_job_failure(&downloader.mysql_pool, &job, failure_reason).await; } } } @@ -113,22 +133,22 @@ async fn process_jobs(pool: &MySqlPool, jobs: Vec) -> Anyhow Ok(()) } -async fn process_job(pool: &MySqlPool, job: &TtsUploadJobRecord) -> AnyhowResult<()> { +async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> { // TODO: 1. Mark processing. // TODO: 2. Download. // TODO: 3. Upload. - // TODO: 4. Save record. + // TODO: 4. Save record. (DONE) // TODO: 5. Mark job done. (DONE) let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead. info!("Saving model record..."); - let id = insert_tts_model(pool, job, &private_bucket_hash).await?; + let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?; info!("Saved model record: {}", id); info!("Job done: {}", job.id); - mark_tts_upload_job_done(pool, job, true).await?; + mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?; Ok(()) } diff --git a/src/util/filesystem.rs b/src/util/filesystem.rs new file mode 100644 index 0000000000..5a1c8e3d1c --- /dev/null +++ b/src/util/filesystem.rs @@ -0,0 +1,13 @@ +use anyhow::bail; +use crate::util::anyhow_result::AnyhowResult; +use std::path::PathBuf; + +pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> { + if !path.exists() { + bail!("Path doesn't exist: {:?}", path); + } + if !path.is_dir() { + bail!("Path isn't a directory: {:?}", path); + } + Ok(()) +} diff --git a/src/util/mod.rs b/src/util/mod.rs index 2b2f0d0bd0..3dda00a986 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,5 +1,6 @@ pub mod anyhow_result; pub mod cookie_manager; +pub mod filesystem; pub mod ip_address; pub mod random_token; pub mod session_checker; \ No newline at end of file