filesystem tempdir

This commit is contained in:
Brandon Thomas committed 2021-05-22 04:28:29 -04:00
1 parent a235aa9f88
commit 09a6aa286d
3 files changed
+46 -12

No files matched your search

+32 -12
View File
@@ -15,14 +15,21 @@ use crate::job::job_queries::mark_tts_upload_job_done;
use crate::job::job_queries::mark_tts_upload_job_failure;
use crate::job::job_queries::query_tts_upload_job_records;
use crate::util::anyhow_result::AnyhowResult;
use crate::util::filesystem::check_directory_exists;
use crate::util::random_token::random_token;
use log::{warn, info};
use sqlx::MySqlPool;
use sqlx::mysql::MySqlPoolOptions;
use std::path::PathBuf;
use std::time::Duration;
const DEFAULT_RUST_LOG: &'static str = "debug,actix_web=info";
struct Downloader {
pub download_temp_directory: PathBuf,
pub mysql_pool: MySqlPool,
}
#[async_std::main]
async fn main() -> AnyhowResult<()> {
easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG));
@@ -36,6 +43,14 @@ async fn main() -> AnyhowResult<()> {
info!("Hostname: {}", &server_hostname);
let temp_directory = easyenv::get_env_string_or_default(
"DOWNLOAD_TEMP_DIR",
"/tmp");
let temp_directory = PathBuf::from(temp_directory);
check_directory_exists(&temp_directory)?;
let db_connection_string =
easyenv::get_env_string_or_default(
"MYSQL_URL",
@@ -43,12 +58,17 @@ async fn main() -> AnyhowResult<()> {
info!("Connecting to database...");
let pool = MySqlPoolOptions::new()
let mysql_pool = MySqlPoolOptions::new()
.max_connections(5)
.connect(&db_connection_string)
.await?;
main_loop(pool).await;
let downloader = Downloader {
download_temp_directory: temp_directory,
mysql_pool,
};
main_loop(downloader).await;
Ok(())
}
@@ -56,12 +76,12 @@ async fn main() -> AnyhowResult<()> {
const START_TIMEOUT_MILLIS : u64 = 500;
const INCREASE_TIMEOUT_MILLIS : u64 = 1000;
async fn main_loop(pool: MySqlPool) {
async fn main_loop(downloader: Downloader) {
let mut timeout_millis = START_TIMEOUT_MILLIS;
loop {
let num_records = 1;
let query_result = query_tts_upload_job_records(&pool, num_records).await;
let query_result = query_tts_upload_job_records(&downloader.mysql_pool, num_records).await;
let jobs = match query_result {
Ok(jobs) => jobs,
@@ -79,7 +99,7 @@ async fn main_loop(pool: MySqlPool) {
continue;
}
let result = process_jobs(&pool, jobs).await;
let result = process_jobs(&downloader, jobs).await;
match result {
Ok(_) => {},
@@ -97,15 +117,15 @@ async fn main_loop(pool: MySqlPool) {
}
}
async fn process_jobs(pool: &MySqlPool, jobs: Vec<TtsUploadJobRecord>) -> AnyhowResult<()> {
async fn process_jobs(downloader: &Downloader, jobs: Vec<TtsUploadJobRecord>) -> AnyhowResult<()> {
for job in jobs.into_iter() {
let result = process_job(pool, &job).await;
let result = process_job(downloader, &job).await;
match result {
Ok(_) => {},
Err(e) => {
warn!("Failure to process job: {:?}", e);
let failure_reason = "";
let _r = mark_tts_upload_job_failure(pool, &job, failure_reason).await;
let _r = mark_tts_upload_job_failure(&downloader.mysql_pool, &job, failure_reason).await;
}
}
}
@@ -113,22 +133,22 @@ async fn process_jobs(pool: &MySqlPool, jobs: Vec<TtsUploadJobRecord>) -> Anyhow
Ok(())
}
async fn process_job(pool: &MySqlPool, job: &TtsUploadJobRecord) -> AnyhowResult<()> {
async fn process_job(downloader: &Downloader, job: &TtsUploadJobRecord) -> AnyhowResult<()> {
// TODO: 1. Mark processing.
// TODO: 2. Download.
// TODO: 3. Upload.
// TODO: 4. Save record.
// TODO: 4. Save record. (DONE)
// TODO: 5. Mark job done. (DONE)
let private_bucket_hash = random_token(32); // TODO: Use sha2/sha256 instead.
info!("Saving model record...");
let id = insert_tts_model(pool, job, &private_bucket_hash).await?;
let id = insert_tts_model(&downloader.mysql_pool, job, &private_bucket_hash).await?;
info!("Saved model record: {}", id);
info!("Job done: {}", job.id);
mark_tts_upload_job_done(pool, job, true).await?;
mark_tts_upload_job_done(&downloader.mysql_pool, job, true).await?;
Ok(())
}
+13
View File
@@ -0,0 +1,13 @@
use anyhow::bail;
use crate::util::anyhow_result::AnyhowResult;
use std::path::PathBuf;
pub fn check_directory_exists(path: &PathBuf) -> AnyhowResult<()> {
if !path.exists() {
bail!("Path doesn't exist: {:?}", path);
}
if !path.is_dir() {
bail!("Path isn't a directory: {:?}", path);
}
Ok(())
}
+1
View File
@@ -1,5 +1,6 @@
pub mod anyhow_result;
pub mod cookie_manager;
pub mod filesystem;
pub mod ip_address;
pub mod random_token;
pub mod session_checker;