From f26a3318c45a20da72e41b98f82d9d251bdd394c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 12 Sep 2021 17:39:56 -0400 Subject: [PATCH] put in deletion of intermediate files (but not on error) --- src/w2l_inference_job.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index f9354900cd..98bfa2d327 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -41,6 +41,8 @@ use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::filesystem::safe_delete_temp_directory; +use crate::util::filesystem::safe_delete_temp_file; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::noop_logger::NoOpLogger; use crate::util::random_crockford_token::random_crockford_token; @@ -49,6 +51,7 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::PooledConnection; use r2d2_redis::r2d2; use r2d2_redis::redis::Commands; use ring::digest::{Context, Digest, SHA256}; @@ -62,7 +65,6 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; -use r2d2_redis::r2d2::PooledConnection; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY"; @@ -572,6 +574,8 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An let file_metadata = read_metadata_file(&output_metadata_fs_path)?; + safe_delete_temp_file(&output_metadata_fs_path); + // ==================== UPLOAD TO BUCKETS ==================== // redis_logger.log_status("uploading result")?; @@ -593,6 +597,13 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An original_mime_type) .await?; + safe_delete_temp_file(&output_video_fs_path); + + // ==================== DELETE DOWNLOADED FILE ==================== // + + // NB: We should be using a tempdir, but to make absolutely certain we don't overflow the disk... + safe_delete_temp_directory(&temp_dir); + // ==================== SAVE RECORDS ==================== // info!("Saving w2l inference record...");