From 323bf9ea8d7c9e508e3fb613ce1da4190b6d5f8d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 13 Jun 2021 03:12:50 -0400 Subject: [PATCH] move bucket utils under ./utils so it's with similar stuff --- .../endpoints/tts/enqueue_upload_tts_model.rs | 4 ++-- .../w2l/enqueue_infer_w2l_with_uploads.rs | 9 ++++----- src/main.rs | 3 +-- src/server_state.rs | 2 +- src/tts_download_job.rs | 11 +++++------ src/tts_inference_job.rs | 11 +++++------ src/{ => util}/buckets/bucket_client.rs | 0 src/{ => util}/buckets/bucket_path_unifier.rs | 4 ++-- src/{ => util}/buckets/bucket_paths.rs | 0 src/{ => util}/buckets/mod.rs | 0 src/util/mod.rs | 1 + src/w2l_download_job.rs | 9 ++++----- src/w2l_inference_job.rs | 15 +++++++-------- 13 files changed, 32 insertions(+), 37 deletions(-) rename src/{ => util}/buckets/bucket_client.rs (100%) rename src/{ => util}/buckets/bucket_path_unifier.rs (99%) rename src/{ => util}/buckets/bucket_paths.rs (100%) rename src/{ => util}/buckets/mod.rs (100%) diff --git a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs index a1ee1d2265..25445fde53 100644 --- a/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs +++ b/src/http_server/endpoints/tts/enqueue_upload_tts_model.rs @@ -7,8 +7,10 @@ use actix_web::http::StatusCode; use actix_web::{Responder, web, HttpResponse, error, HttpRequest}; use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility}; use crate::http_server::web_utils::ip_address::get_request_ip; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::validations::model_uploads::validate_model_title; use crate::validations::passwords::validate_passwords; use crate::validations::username::validate_username; @@ -19,8 +21,6 @@ use sqlx::error::DatabaseError; use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::sync::Arc; -use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; #[derive(Deserialize)] pub struct UploadTtsModelRequest { diff --git a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index a396e216a3..d67c666c3f 100644 --- a/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -1,4 +1,3 @@ -//use bytes::{Bytes, BytesMut}; use actix_http::http::header; use actix_multipart::{Multipart, Field}; use actix_web::body::ResponseBody::Body; @@ -8,17 +7,18 @@ use actix_web::http::header::ContentDisposition; use actix_web::web::{Data, Json, BytesMut}; use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError}; use anyhow::anyhow; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::http_server::web_utils::ip_address::get_request_ip; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_text; use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_bytes; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::anyhow_result::AnyhowResult; -use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::random_crockford_token::random_crockford_token; +use crate::util::random_prefix_crockford_token::random_prefix_crockford_token; use crate::util::random_uuid::generate_random_uuid; use derive_more::{Display, Error}; use futures::{StreamExt, TryStreamExt}; @@ -28,7 +28,6 @@ use sqlx::error::Error::Database; use std::io::Write; use std::sync::Arc; use uuid::Uuid; -use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file"; const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file"; diff --git a/src/main.rs b/src/main.rs index 964e449103..282bcf4292 100644 --- a/src/main.rs +++ b/src/main.rs @@ -16,7 +16,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usern pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt"); pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt"); -pub mod buckets; pub mod common_queries; pub mod database_helpers; pub mod http_server; @@ -32,7 +31,6 @@ use actix_cors::Cors; use actix_http::http; use actix_web::middleware::{Logger, DefaultHeaders}; use actix_web::{HttpServer, web, HttpResponse, App}; -use crate::buckets::bucket_client::BucketClient; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::endpoints::default_route_404::default_route_404; use crate::http_server::endpoints::events::list_events::list_events_handler; @@ -81,6 +79,7 @@ use crate::server_state::{ServerState, EnvConfig}; use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING; use crate::shared_constants::DEFAULT_RUST_LOG; +use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; use log::{info}; use r2d2_redis::RedisConnectionManager; diff --git a/src/server_state.rs b/src/server_state.rs index 73b6658904..d62cf9a7c6 100644 --- a/src/server_state.rs +++ b/src/server_state.rs @@ -1,7 +1,7 @@ -use crate::buckets::bucket_client::BucketClient; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::http_server::web_utils::cookie_manager::CookieManager; use crate::http_server::web_utils::session_checker::SessionChecker; +use crate::util::buckets::bucket_client::BucketClient; use crate::util::encrypted_sort_id::SortKeyCrypto; use r2d2_redis::{r2d2, RedisConnectionManager}; use sqlx::MySqlPool; diff --git a/src/tts_download_job.rs b/src/tts_download_job.rs index 9de093df57..5013688834 100644 --- a/src/tts_download_job.rs +++ b/src/tts_download_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,9 +20,6 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_path_unifier::BucketPathUnifier; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord; @@ -34,15 +30,18 @@ use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure; use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records; use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::{check_directory_exists, check_file_exists}; use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::File; diff --git a/src/tts_inference_job.rs b/src/tts_inference_job.rs index 9d848ed5a2..00d5a9b8f5 100644 --- a/src/tts_inference_job.rs +++ b/src/tts_inference_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,9 +20,6 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_path_unifier::BucketPathUnifier; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord; @@ -34,7 +30,12 @@ use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done; use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure; use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records; use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; @@ -44,8 +45,6 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; diff --git a/src/buckets/bucket_client.rs b/src/util/buckets/bucket_client.rs similarity index 100% rename from src/buckets/bucket_client.rs rename to src/util/buckets/bucket_client.rs diff --git a/src/buckets/bucket_path_unifier.rs b/src/util/buckets/bucket_path_unifier.rs similarity index 99% rename from src/buckets/bucket_path_unifier.rs rename to src/util/buckets/bucket_path_unifier.rs index 3fb2c9ec29..0e8fa3b161 100644 --- a/src/buckets/bucket_path_unifier.rs +++ b/src/util/buckets/bucket_path_unifier.rs @@ -1,6 +1,6 @@ -use crate::buckets::bucket_paths::hash_to_bucket_path; -use std::path::PathBuf; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; +use std::path::PathBuf; /// This is designed to make it centrally configurable where /// different types of objects are stored. diff --git a/src/buckets/bucket_paths.rs b/src/util/buckets/bucket_paths.rs similarity index 100% rename from src/buckets/bucket_paths.rs rename to src/util/buckets/bucket_paths.rs diff --git a/src/buckets/mod.rs b/src/util/buckets/mod.rs similarity index 100% rename from src/buckets/mod.rs rename to src/util/buckets/mod.rs diff --git a/src/util/mod.rs b/src/util/mod.rs index 46340aa50f..0617f941a8 100644 --- a/src/util/mod.rs +++ b/src/util/mod.rs @@ -1,4 +1,5 @@ pub mod anyhow_result; +pub mod buckets; pub mod email_to_gravatar; pub mod encrypted_sort_id; pub mod filesystem; diff --git a/src/w2l_download_job.rs b/src/w2l_download_job.rs index fe56a03218..68af39d48b 100644 --- a/src/w2l_download_job.rs +++ b/src/w2l_download_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,8 +20,6 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_paths::hash_to_bucket_path; use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord; @@ -36,7 +33,11 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; use crate::util::hashing::hash_file_sha2::hash_file_sha2; @@ -44,8 +45,6 @@ use crate::util::random_crockford_token::random_crockford_token; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; diff --git a/src/w2l_inference_job.rs b/src/w2l_inference_job.rs index 082b237b26..c67e250471 100644 --- a/src/w2l_inference_job.rs +++ b/src/w2l_inference_job.rs @@ -10,7 +10,6 @@ #[macro_use] extern crate serde_derive; -pub mod buckets; pub mod common_env; pub mod common_queries; pub mod database_helpers; @@ -21,10 +20,7 @@ pub mod util; use anyhow::anyhow; use chrono::Utc; -use crate::buckets::bucket_client::BucketClient; -use crate::buckets::bucket_path_unifier::BucketPathUnifier; -use crate::buckets::bucket_paths::hash_to_bucket_path; -use util::hashing::hash_file_sha2::hash_file_sha2; +use crate::common_env::CommonEnv; use crate::common_queries::firehose_publisher::FirehosePublisher; use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord; use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token; @@ -38,16 +34,20 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand; use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand; use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand; +use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; +use crate::shared_constants::DEFAULT_RUST_LOG; use crate::util::anyhow_result::AnyhowResult; +use crate::util::buckets::bucket_client::BucketClient; +use crate::util::buckets::bucket_path_unifier::BucketPathUnifier; +use crate::util::buckets::bucket_paths::hash_to_bucket_path; use crate::util::filesystem::check_directory_exists; use crate::util::filesystem::check_file_exists; +use crate::util::hashing::hash_file_sha2::hash_file_sha2; use crate::util::random_crockford_token::random_crockford_token; use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir; use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE}; use log::{warn, info}; use ring::digest::{Context, Digest, SHA256}; -use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; -use shared_constants::DEFAULT_RUST_LOG; use sqlx::MySqlPool; use sqlx::mysql::MySqlPoolOptions; use std::fs::{File, metadata}; @@ -57,7 +57,6 @@ use std::process::Command; use std::thread; use std::time::Duration; use tempdir::TempDir; -use crate::common_env::CommonEnv; // Buckets (shared config) const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";