move bucket utils under ./utils so it's with similar stuff

src/http_server/endpoints/tts/enqueue_upload_tts_model.rs

@@ -7,8 +7,10 @@ use actix_web::http::StatusCode;
 use actix_web::{Responder, web, HttpResponse, error, HttpRequest};
 use crate::database_helpers::enums::{DownloadUrlType, TtsModelType, CreatorSetVisibility};
 use crate::http_server::web_utils::ip_address::get_request_ip;
+use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
 use crate::server_state::ServerState;
 use crate::util::random_crockford_token::random_crockford_token;
+use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
 use crate::validations::model_uploads::validate_model_title;
 use crate::validations::passwords::validate_passwords;
 use crate::validations::username::validate_username;
@@ -19,8 +21,6 @@ use sqlx::error::DatabaseError;
 use sqlx::error::Error::Database;
 use sqlx::mysql::MySqlDatabaseError;
 use std::sync::Arc;
-use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
-use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
 
 #[derive(Deserialize)]
 pub struct UploadTtsModelRequest {

src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs

@@ -1,4 +1,3 @@
-//use bytes::{Bytes, BytesMut};
 use actix_http::http::header;
 use actix_multipart::{Multipart, Field};
 use actix_web::body::ResponseBody::Body;
@@ -8,17 +7,18 @@ use actix_web::http::header::ContentDisposition;
 use actix_web::web::{Data, Json, BytesMut};
 use actix_web::{middleware, web, App, Error, HttpResponse, HttpServer, HttpRequest, Either, ResponseError};
 use anyhow::anyhow;
-use crate::buckets::bucket_client::BucketClient;
-use crate::buckets::bucket_paths::hash_to_bucket_path;
 use crate::http_server::web_utils::ip_address::get_request_ip;
 use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes;
 use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_boolean;
 use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_as_text;
 use crate::http_server::web_utils::read_multipart_field_bytes::read_multipart_field_bytes;
+use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
 use crate::server_state::ServerState;
 use crate::util::anyhow_result::AnyhowResult;
-use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
+use crate::util::buckets::bucket_client::BucketClient;
+use crate::util::buckets::bucket_paths::hash_to_bucket_path;
 use crate::util::random_crockford_token::random_crockford_token;
+use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
 use crate::util::random_uuid::generate_random_uuid;
 use derive_more::{Display, Error};
 use futures::{StreamExt, TryStreamExt};
@@ -28,7 +28,6 @@ use sqlx::error::Error::Database;
 use std::io::Write;
 use std::sync::Arc;
 use uuid::Uuid;
-use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
 
 const BUCKET_AUDIO_FILE_NAME : &'static str = "input_audio_file";
 const BUCKET_IMAGE_FILE_NAME: &'static str = "input_image_file";

src/main.rs

@@ -16,7 +16,6 @@ pub const RESERVED_USERNAMES : &'static str = include_str!("../db/reserved_usern
 pub const RESERVED_SUBSTRINGS : &'static str = include_str!("../db/reserved_usernames_including.txt");
 pub const BANNED_SLURS : &'static str = include_str!("../db/banned_slurs.txt");
 
-pub mod buckets;
 pub mod common_queries;
 pub mod database_helpers;
 pub mod http_server;
@@ -32,7 +31,6 @@ use actix_cors::Cors;
 use actix_http::http;
 use actix_web::middleware::{Logger, DefaultHeaders};
 use actix_web::{HttpServer, web, HttpResponse, App};
-use crate::buckets::bucket_client::BucketClient;
 use crate::common_queries::firehose_publisher::FirehosePublisher;
 use crate::http_server::endpoints::default_route_404::default_route_404;
 use crate::http_server::endpoints::events::list_events::list_events_handler;
@@ -81,6 +79,7 @@ use crate::server_state::{ServerState, EnvConfig};
 use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
 use crate::shared_constants::DEFAULT_REDIS_CONNECTION_STRING;
 use crate::shared_constants::DEFAULT_RUST_LOG;
+use crate::util::buckets::bucket_client::BucketClient;
 use crate::util::encrypted_sort_id::SortKeyCrypto;
 use log::{info};
 use r2d2_redis::RedisConnectionManager;

src/server_state.rs

@@ -1,7 +1,7 @@
-use crate::buckets::bucket_client::BucketClient;
 use crate::common_queries::firehose_publisher::FirehosePublisher;
 use crate::http_server::web_utils::cookie_manager::CookieManager;
 use crate::http_server::web_utils::session_checker::SessionChecker;
+use crate::util::buckets::bucket_client::BucketClient;
 use crate::util::encrypted_sort_id::SortKeyCrypto;
 use r2d2_redis::{r2d2, RedisConnectionManager};
 use sqlx::MySqlPool;

src/tts_download_job.rs

@@ -10,7 +10,6 @@
 
 #[macro_use] extern crate serde_derive;
 
-pub mod buckets;
 pub mod common_env;
 pub mod common_queries;
 pub mod database_helpers;
@@ -21,9 +20,6 @@ pub mod util;
 
 use anyhow::anyhow;
 use chrono::Utc;
-use crate::buckets::bucket_client::BucketClient;
-use crate::buckets::bucket_path_unifier::BucketPathUnifier;
-use crate::buckets::bucket_paths::hash_to_bucket_path;
 use crate::common_env::CommonEnv;
 use crate::common_queries::firehose_publisher::FirehosePublisher;
 use crate::job_queries::tts_download_job_queries::TtsUploadJobRecord;
@@ -34,15 +30,18 @@ use crate::job_queries::tts_download_job_queries::mark_tts_upload_job_failure;
 use crate::job_queries::tts_download_job_queries::query_tts_upload_job_records;
 use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
 use crate::script_execution::tacotron_model_check_command::TacotronModelCheckCommand;
+use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
+use crate::shared_constants::DEFAULT_RUST_LOG;
 use crate::util::anyhow_result::AnyhowResult;
+use crate::util::buckets::bucket_client::BucketClient;
+use crate::util::buckets::bucket_path_unifier::BucketPathUnifier;
+use crate::util::buckets::bucket_paths::hash_to_bucket_path;
 use crate::util::filesystem::{check_directory_exists, check_file_exists};
 use crate::util::hashing::hash_file_sha2::hash_file_sha2;
 use crate::util::random_crockford_token::random_crockford_token;
 use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
 use log::{warn, info};
 use ring::digest::{Context, Digest, SHA256};
-use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
-use shared_constants::DEFAULT_RUST_LOG;
 use sqlx::MySqlPool;
 use sqlx::mysql::MySqlPoolOptions;
 use std::fs::File;

src/tts_inference_job.rs

@@ -10,7 +10,6 @@
 
 #[macro_use] extern crate serde_derive;
 
-pub mod buckets;
 pub mod common_env;
 pub mod common_queries;
 pub mod database_helpers;
@@ -21,9 +20,6 @@ pub mod util;
 
 use anyhow::anyhow;
 use chrono::Utc;
-use crate::buckets::bucket_client::BucketClient;
-use crate::buckets::bucket_path_unifier::BucketPathUnifier;
-use crate::buckets::bucket_paths::hash_to_bucket_path;
 use crate::common_env::CommonEnv;
 use crate::common_queries::firehose_publisher::FirehosePublisher;
 use crate::job_queries::tts_inference_job_queries::TtsInferenceJobRecord;
@@ -34,7 +30,12 @@ use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_done;
 use crate::job_queries::tts_inference_job_queries::mark_tts_inference_job_failure;
 use crate::job_queries::tts_inference_job_queries::query_tts_inference_job_records;
 use crate::script_execution::tacotron_inference_command::TacotronInferenceCommand;
+use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
+use crate::shared_constants::DEFAULT_RUST_LOG;
 use crate::util::anyhow_result::AnyhowResult;
+use crate::util::buckets::bucket_client::BucketClient;
+use crate::util::buckets::bucket_path_unifier::BucketPathUnifier;
+use crate::util::buckets::bucket_paths::hash_to_bucket_path;
 use crate::util::filesystem::check_directory_exists;
 use crate::util::filesystem::check_file_exists;
 use crate::util::hashing::hash_file_sha2::hash_file_sha2;
@@ -44,8 +45,6 @@ use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir;
 use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
 use log::{warn, info};
 use ring::digest::{Context, Digest, SHA256};
-use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
-use shared_constants::DEFAULT_RUST_LOG;
 use sqlx::MySqlPool;
 use sqlx::mysql::MySqlPoolOptions;
 use std::fs::{File, metadata};

src/util/buckets/bucket_path_unifier.rs

@@ -1,6 +1,6 @@
-use crate::buckets::bucket_paths::hash_to_bucket_path;
-use std::path::PathBuf;
 use crate::util::anyhow_result::AnyhowResult;
+use crate::util::buckets::bucket_paths::hash_to_bucket_path;
+use std::path::PathBuf;
 
 /// This is designed to make it centrally configurable where
 /// different types of objects are stored.

src/util/mod.rs

@@ -1,4 +1,5 @@
 pub mod anyhow_result;
+pub mod buckets;
 pub mod email_to_gravatar;
 pub mod encrypted_sort_id;
 pub mod filesystem;

src/w2l_download_job.rs

@@ -10,7 +10,6 @@
 
 #[macro_use] extern crate serde_derive;
 
-pub mod buckets;
 pub mod common_env;
 pub mod common_queries;
 pub mod database_helpers;
@@ -21,8 +20,6 @@ pub mod util;
 
 use anyhow::anyhow;
 use chrono::Utc;
-use crate::buckets::bucket_client::BucketClient;
-use crate::buckets::bucket_paths::hash_to_bucket_path;
 use crate::common_env::CommonEnv;
 use crate::common_queries::firehose_publisher::FirehosePublisher;
 use crate::job_queries::w2l_download_job_queries::W2lTemplateUploadJobRecord;
@@ -36,7 +33,11 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera
 use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
 use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
 use crate::script_execution::wav2lip_process_upload_command::Wav2LipPreprocessClient;
+use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
+use crate::shared_constants::DEFAULT_RUST_LOG;
 use crate::util::anyhow_result::AnyhowResult;
+use crate::util::buckets::bucket_client::BucketClient;
+use crate::util::buckets::bucket_paths::hash_to_bucket_path;
 use crate::util::filesystem::check_directory_exists;
 use crate::util::filesystem::check_file_exists;
 use crate::util::hashing::hash_file_sha2::hash_file_sha2;
@@ -44,8 +45,6 @@ use crate::util::random_crockford_token::random_crockford_token;
 use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
 use log::{warn, info};
 use ring::digest::{Context, Digest, SHA256};
-use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
-use shared_constants::DEFAULT_RUST_LOG;
 use sqlx::MySqlPool;
 use sqlx::mysql::MySqlPoolOptions;
 use std::fs::{File, metadata};

src/w2l_inference_job.rs

@@ -10,7 +10,6 @@
 
 #[macro_use] extern crate serde_derive;
 
-pub mod buckets;
 pub mod common_env;
 pub mod common_queries;
 pub mod database_helpers;
@@ -21,10 +20,7 @@ pub mod util;
 
 use anyhow::anyhow;
 use chrono::Utc;
-use crate::buckets::bucket_client::BucketClient;
-use crate::buckets::bucket_path_unifier::BucketPathUnifier;
-use crate::buckets::bucket_paths::hash_to_bucket_path;
-use util::hashing::hash_file_sha2::hash_file_sha2;
+use crate::common_env::CommonEnv;
 use crate::common_queries::firehose_publisher::FirehosePublisher;
 use crate::job_queries::w2l_inference_job_queries::W2lInferenceJobRecord;
 use crate::job_queries::w2l_inference_job_queries::get_w2l_template_by_token;
@@ -38,16 +34,20 @@ use crate::script_execution::ffmpeg_generate_preview_video_command::FfmpegGenera
 use crate::script_execution::google_drive_download_command::GoogleDriveDownloadCommand;
 use crate::script_execution::imagemagick_generate_preview_image_command::ImagemagickGeneratePreviewImageCommand;
 use crate::script_execution::wav2lip_inference_command::Wav2LipInferenceCommand;
+use crate::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
+use crate::shared_constants::DEFAULT_RUST_LOG;
 use crate::util::anyhow_result::AnyhowResult;
+use crate::util::buckets::bucket_client::BucketClient;
+use crate::util::buckets::bucket_path_unifier::BucketPathUnifier;
+use crate::util::buckets::bucket_paths::hash_to_bucket_path;
 use crate::util::filesystem::check_directory_exists;
 use crate::util::filesystem::check_file_exists;
+use crate::util::hashing::hash_file_sha2::hash_file_sha2;
 use crate::util::random_crockford_token::random_crockford_token;
 use crate::util::semi_persistent_cache_dir::SemiPersistentCacheDir;
 use data_encoding::{HEXUPPER, HEXLOWER, HEXLOWER_PERMISSIVE};
 use log::{warn, info};
 use ring::digest::{Context, Digest, SHA256};
-use shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
-use shared_constants::DEFAULT_RUST_LOG;
 use sqlx::MySqlPool;
 use sqlx::mysql::MySqlPoolOptions;
 use std::fs::{File, metadata};
@@ -57,7 +57,6 @@ use std::process::Command;
 use std::thread;
 use std::time::Duration;
 use tempdir::TempDir;
-use crate::common_env::CommonEnv;
 
 // Buckets (shared config)
 const ENV_ACCESS_KEY : &'static str = "ACCESS_KEY";