diff --git a/Cargo.lock b/Cargo.lock index 1d05a91655..269452c5bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -981,6 +981,13 @@ dependencies = [ "libc", ] +[[package]] +name = "buckets" +version = "0.0.1" +dependencies = [ + "errors", +] + [[package]] name = "bumpalo" version = "3.6.1" @@ -1143,6 +1150,7 @@ name = "cloud_storage" version = "0.0.1" dependencies = [ "anyhow", + "buckets", "container_common", "log", "rust-s3", @@ -4577,6 +4585,7 @@ dependencies = [ "base64 0.13.0", "bcrypt", "billing_component", + "buckets", "chrono", "cloud_storage", "config", @@ -5873,6 +5882,7 @@ name = "w2l-download-job" version = "0.0.1" dependencies = [ "anyhow", + "buckets", "chrono", "clap 2.34.0", "cloud_storage", diff --git a/Cargo.toml b/Cargo.toml index 0aaff7a825..40aef4976f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,8 @@ members = [ # Types: common strongly-typed tokens, ids, payloads, etc. + # Also common cloud buckets and other "strongly typed" configs. + "crates/_types/buckets", "crates/_types/enums", "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate "crates/_types/tokens", diff --git a/crates/_types/buckets/Cargo.toml b/crates/_types/buckets/Cargo.toml new file mode 100644 index 0000000000..c5ca7bfea9 --- /dev/null +++ b/crates/_types/buckets/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "buckets" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "buckets" +path = "src/lib.rs" + +[dependencies] + +# Internal +errors = { path = "../../lib/errors" } + +# External +# (none) + +[dev-dependencies] diff --git a/crates/_types/buckets/src/lib.rs b/crates/_types/buckets/src/lib.rs new file mode 100644 index 0000000000..3929dc74b2 --- /dev/null +++ b/crates/_types/buckets/src/lib.rs @@ -0,0 +1,24 @@ +//! buckets +//! +//! The purpose of this library is to maintain strongly typed cloud storage buckets +//! that can be shared between projects. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod util; + +pub mod private; +pub mod public; diff --git a/crates/_types/buckets/src/private/mod.rs b/crates/_types/buckets/src/private/mod.rs new file mode 100644 index 0000000000..27089e4df1 --- /dev/null +++ b/crates/_types/buckets/src/private/mod.rs @@ -0,0 +1 @@ +pub mod private_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/private/private_path.rs b/crates/_types/buckets/src/private/private_path.rs new file mode 100644 index 0000000000..d16290a2ca --- /dev/null +++ b/crates/_types/buckets/src/private/private_path.rs @@ -0,0 +1,7 @@ + +/// A trait implemented by all private paths. +pub trait PrivatePath { + fn is_public(&self) -> bool { + false + } +} diff --git a/crates/_types/buckets/src/public/media_uploads/directory.rs b/crates/_types/buckets/src/public/media_uploads/directory.rs new file mode 100644 index 0000000000..405172a135 --- /dev/null +++ b/crates/_types/buckets/src/public/media_uploads/directory.rs @@ -0,0 +1,50 @@ +use crate::public::public_path::PublicPath; +use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + +// TODO: Use a central path registry for quick reference +const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media"; + +/// Directory for user media uploads. +/// Each uploaded file gets its own directory so that we can store the original +/// file alongside re-processed derivatives. +#[derive(Clone)] +pub struct MediaUploadDirectory { + object_hash: String, + directory: String, +} + +impl PublicPath for MediaUploadDirectory {} + +impl MediaUploadDirectory { + + pub fn from_object_hash(object_hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let middle = hashed_directory_path_long_string(object_hash); + let directory = format!("{}/{}{}", MEDIA_UPLOAD_DIRECTORY, middle, object_hash); + Self { + object_hash: object_hash.to_string(), + directory, + } + } + + pub fn get_directory_path_str(&self) -> &str { + &self.directory + } +} + +#[cfg(test)] +mod tests { + use crate::public::media_uploads::directory::MediaUploadDirectory; + + #[test] + pub fn get_directory_path_str() { + let directory = MediaUploadDirectory::from_object_hash("abcdefghijk"); + assert_eq!(directory.get_directory_path_str(), "/media/a/b/c/d/e/abcdefghijk"); + } + + #[test] + pub fn get_directory_path_str_short_name() { + let directory = MediaUploadDirectory::from_object_hash("foo"); + assert_eq!(directory.get_directory_path_str(), "/media/f/o/foo"); + } +} diff --git a/crates/_types/buckets/src/public/media_uploads/mod.rs b/crates/_types/buckets/src/public/media_uploads/mod.rs new file mode 100644 index 0000000000..42ede393cf --- /dev/null +++ b/crates/_types/buckets/src/public/media_uploads/mod.rs @@ -0,0 +1,2 @@ +pub mod directory; +pub mod original_file; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/media_uploads/original_file.rs b/crates/_types/buckets/src/public/media_uploads/original_file.rs new file mode 100644 index 0000000000..3ec71fac29 --- /dev/null +++ b/crates/_types/buckets/src/public/media_uploads/original_file.rs @@ -0,0 +1,52 @@ +use crate::public::media_uploads::directory::MediaUploadDirectory; +use crate::public::public_path::PublicPath; + +const ORIGINAL_FILE_BASENAME : &'static str = "original_contents.bin"; + +/// The original user upload file. +/// It may have derivative files (down samples, crops, etc.) that live alongside it. +#[derive(Clone)] +pub struct MediaUploadOriginalFilePath { + directory: MediaUploadDirectory, + full_object_path: String, +} + +impl PublicPath for MediaUploadOriginalFilePath {} + +impl MediaUploadOriginalFilePath { + + pub fn from_object_hash(hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let directory = MediaUploadDirectory::from_object_hash(hash); + let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME); + Self { + directory, + full_object_path, + } + } + + pub fn get_basename() -> &'static str { + ORIGINAL_FILE_BASENAME + } + + pub fn get_full_object_path_str(&self) -> &str { + &self.full_object_path + } +} + +#[cfg(test)] +mod tests { + use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath; + + #[test] + pub fn get_full_object_path_str() { + let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk"); + assert_eq!(file.get_full_object_path_str(), "/media/a/b/c/d/e/abcdefghijk/original_contents.bin"); + } + + #[test] + pub fn get_full_object_path_str_short_name() { + let file = MediaUploadOriginalFilePath::from_object_hash("foo"); + assert_eq!(file.get_full_object_path_str(), "/media/f/o/foo/original_contents.bin"); + } +} diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs new file mode 100644 index 0000000000..38f2952846 --- /dev/null +++ b/crates/_types/buckets/src/public/mod.rs @@ -0,0 +1,2 @@ +pub mod media_uploads; +pub mod public_path; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/public_path.rs b/crates/_types/buckets/src/public/public_path.rs new file mode 100644 index 0000000000..9c6d74d1c4 --- /dev/null +++ b/crates/_types/buckets/src/public/public_path.rs @@ -0,0 +1,7 @@ + +/// A trait implemented by all public paths. +pub trait PublicPath { + fn is_public(&self) -> bool { + true + } +} diff --git a/crates/lib/cloud_storage/src/bucket_paths.rs b/crates/_types/buckets/src/util/hash_to_bucket_path.rs similarity index 61% rename from crates/lib/cloud_storage/src/bucket_paths.rs rename to crates/_types/buckets/src/util/hash_to_bucket_path.rs index 81347b4d3a..770deede67 100644 --- a/crates/lib/cloud_storage/src/bucket_paths.rs +++ b/crates/_types/buckets/src/util/hash_to_bucket_path.rs @@ -1,11 +1,11 @@ -use anyhow::anyhow; -use anyhow::bail; -use container_common::anyhow_result::AnyhowResult; +use errors::AnyhowResult; +use errors::bail; use std::path::PathBuf; -pub fn hash_to_bucket_path(file_hash: &str, +#[deprecated(note = "this is a messy function and should be avoided")] +pub (crate) fn hash_to_bucket_path(file_hash: &str, optional_root_directory: Option<&str>) - -> AnyhowResult + -> AnyhowResult { if file_hash.len() < 4 { bail!("File length is too short"); @@ -23,29 +23,25 @@ pub fn hash_to_bucket_path(file_hash: &str, &file_hash, ); - let maybe_path = root_directory - .join(file_path) - .to_str() - .map(|s| s.to_string()); - - maybe_path.ok_or(anyhow!("Path could not be converted")) + Ok(root_directory.join(file_path)) } #[cfg(test)] mod tests { - use crate::bucket_paths::hash_to_bucket_path; + use crate::util::hash_to_bucket_path::hash_to_bucket_path; + use std::path::PathBuf; #[test] fn test_bucket_without_root() { assert_eq!( hash_to_bucket_path("abcdefg", None).unwrap(), - "/a/b/c/abcdefg".to_string()); + PathBuf::from("/a/b/c/abcdefg")); } #[test] fn test_bucket_with_root() { assert_eq!( hash_to_bucket_path("abcdefg", Some("/root")).unwrap(), - "/root/a/b/c/abcdefg".to_string()); + PathBuf::from("/root/a/b/c/abcdefg")); } } diff --git a/crates/_types/buckets/src/util/hash_to_bucket_path_string.rs b/crates/_types/buckets/src/util/hash_to_bucket_path_string.rs new file mode 100644 index 0000000000..af2e3cbb0b --- /dev/null +++ b/crates/_types/buckets/src/util/hash_to_bucket_path_string.rs @@ -0,0 +1,34 @@ +use crate::util::hash_to_bucket_path::hash_to_bucket_path; +use errors::AnyhowResult; +use errors::anyhow; + +#[deprecated(note = "this is a messy function and should be avoided")] +pub fn hash_to_bucket_path_string(file_hash: &str, + optional_root_directory: Option<&str>) + -> AnyhowResult +{ + let path = hash_to_bucket_path(file_hash, optional_root_directory)?; + + let maybe_path = path.to_str().map(|s| s.to_string()); + + maybe_path.ok_or(anyhow!("Path could not be converted")) +} + +#[cfg(test)] +mod tests { + use crate::util::hash_to_bucket_path_string::hash_to_bucket_path_string; + + #[test] + fn test_bucket_without_root() { + assert_eq!( + hash_to_bucket_path_string("abcdefg", None).unwrap(), + "/a/b/c/abcdefg".to_string()); + } + + #[test] + fn test_bucket_with_root() { + assert_eq!( + hash_to_bucket_path_string("abcdefg", Some("/root")).unwrap(), + "/root/a/b/c/abcdefg".to_string()); + } +} diff --git a/crates/_types/buckets/src/util/hashed_directory_path_long_string.rs b/crates/_types/buckets/src/util/hashed_directory_path_long_string.rs new file mode 100644 index 0000000000..6d79980d17 --- /dev/null +++ b/crates/_types/buckets/src/util/hashed_directory_path_long_string.rs @@ -0,0 +1,67 @@ + +/// Create a cloud object store directory structure that can be easily traversed without +/// too many items living in a single directory. +/// +/// With a million items, each directory will have ~1 item. +/// With a billion items, each directory will have ~953 items (totally tractable). +/// +pub fn hashed_directory_path_long_string(file_hash: &str) -> String { + match file_hash.len() { + 0 | 1 => "".to_string(), + 2 => format!("{}/", &file_hash[0..1]), + 3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]), + 4 => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]), + 5 => format!("{}/{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3], &file_hash[3..4]), + _ => format!("{}/{}/{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3], &file_hash[3..4], &file_hash[4..5]), + } +} + +#[cfg(test)] +mod tests { + use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + + #[test] + fn test_length_zero() { + assert_eq!(hashed_directory_path_long_string(""), "".to_string()); + } + + #[test] + fn test_length_one() { + assert_eq!(hashed_directory_path_long_string("a"), "".to_string()); + } + + #[test] + fn test_length_two() { + assert_eq!(hashed_directory_path_long_string("ab"), "a/".to_string()); + } + + #[test] + fn test_length_three() { + assert_eq!(hashed_directory_path_long_string("abc"), "a/b/".to_string()); + } + + #[test] + fn test_length_four() { + assert_eq!(hashed_directory_path_long_string("abcd"), "a/b/c/".to_string()); + } + + #[test] + fn test_length_five() { + assert_eq!(hashed_directory_path_long_string("abcde"), "a/b/c/d/".to_string()); + } + + #[test] + fn test_length_six() { + assert_eq!(hashed_directory_path_long_string("abcdef"), "a/b/c/d/e/".to_string()); + } + + #[test] + fn test_length_seven() { + assert_eq!(hashed_directory_path_long_string("abcdefg"), "a/b/c/d/e/".to_string()); + } + + #[test] + fn test_length_ten() { + assert_eq!(hashed_directory_path_long_string("abcdefghij"), "a/b/c/d/e/".to_string()); + } +} diff --git a/crates/_types/buckets/src/util/hashed_directory_path_short_string.rs b/crates/_types/buckets/src/util/hashed_directory_path_short_string.rs new file mode 100644 index 0000000000..c4fe2ca05a --- /dev/null +++ b/crates/_types/buckets/src/util/hashed_directory_path_short_string.rs @@ -0,0 +1,55 @@ + +/// Create a cloud object store directory structure that can be easily traversed without +/// too many items living in a single directory. +/// +/// With a million items, each directory will have ~244 items. +/// With a billion items, each directory will have ~244,140 items (impractical at this scale). +/// +pub fn hashed_directory_path_short_string(file_hash: &str) -> String { + match file_hash.len() { + 0 | 1 => "".to_string(), + 2 => format!("{}/", &file_hash[0..1]), + 3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]), + _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]), + } +} + +#[cfg(test)] +mod tests { + use crate::util::hashed_directory_path_short_string::hashed_directory_path_short_string; + + #[test] + fn test_length_zero() { + assert_eq!(hashed_directory_path_short_string(""), "".to_string()); + } + + #[test] + fn test_length_one() { + assert_eq!(hashed_directory_path_short_string("a"), "".to_string()); + } + + #[test] + fn test_length_two() { + assert_eq!(hashed_directory_path_short_string("ab"), "a/".to_string()); + } + + #[test] + fn test_length_three() { + assert_eq!(hashed_directory_path_short_string("abc"), "a/b/".to_string()); + } + + #[test] + fn test_length_four() { + assert_eq!(hashed_directory_path_short_string("abcd"), "a/b/c/".to_string()); + } + + #[test] + fn test_length_five() { + assert_eq!(hashed_directory_path_short_string("abcde"), "a/b/c/".to_string()); + } + + #[test] + fn test_length_ten() { + assert_eq!(hashed_directory_path_short_string("abcdefghij"), "a/b/c/".to_string()); + } +} diff --git a/crates/_types/buckets/src/util/mod.rs b/crates/_types/buckets/src/util/mod.rs new file mode 100644 index 0000000000..ed783efb2c --- /dev/null +++ b/crates/_types/buckets/src/util/mod.rs @@ -0,0 +1,4 @@ +pub (crate) mod hash_to_bucket_path; +pub mod hash_to_bucket_path_string; +pub mod hashed_directory_path_long_string; +pub mod hashed_directory_path_short_string; diff --git a/crates/lib/cloud_storage/Cargo.toml b/crates/lib/cloud_storage/Cargo.toml index 439559061e..bb85202245 100644 --- a/crates/lib/cloud_storage/Cargo.toml +++ b/crates/lib/cloud_storage/Cargo.toml @@ -15,6 +15,7 @@ path = "src/lib.rs" [dependencies] # Internal +buckets = { path = "../../_types/buckets" } container_common = { path = "../container_common" } # External diff --git a/crates/lib/cloud_storage/src/bucket_path_unifier.rs b/crates/lib/cloud_storage/src/bucket_path_unifier.rs index 7ad5b8c9c3..bb72214d16 100644 --- a/crates/lib/cloud_storage/src/bucket_path_unifier.rs +++ b/crates/lib/cloud_storage/src/bucket_path_unifier.rs @@ -1,9 +1,10 @@ +use buckets::util::hashed_directory_path_short_string::hashed_directory_path_short_string; use container_common::anyhow_result::AnyhowResult; -use crate::bucket_paths::hash_to_bucket_path; use std::path::PathBuf; /// This is designed to make it centrally configurable where /// different types of objects are stored. +#[deprecated(note = "see the 'buckets' crate for a better approach")] pub struct BucketPathUnifier { // TTS pub tts_synthesizer_model_root: PathBuf, @@ -187,12 +188,7 @@ impl BucketPathUnifier { // ==================== UTILITY ==================== // pub fn hashed_directory_path(file_hash: &str) -> String { - match file_hash.len() { - 0 | 1=> "".to_string(), - 2 => format!("{}/", &file_hash[0..1]), - 3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]), - _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]), - } + hashed_directory_path_short_string(file_hash) } } diff --git a/crates/lib/cloud_storage/src/lib.rs b/crates/lib/cloud_storage/src/lib.rs index 31a7cd49b6..6c74cb023f 100644 --- a/crates/lib/cloud_storage/src/lib.rs +++ b/crates/lib/cloud_storage/src/lib.rs @@ -1,3 +1,2 @@ pub mod bucket_client; pub mod bucket_path_unifier; -pub mod bucket_paths; \ No newline at end of file diff --git a/crates/lib/errors/src/lib.rs b/crates/lib/errors/src/lib.rs index ec5be1b6a3..dcaa0af2fe 100644 --- a/crates/lib/errors/src/lib.rs +++ b/crates/lib/errors/src/lib.rs @@ -19,7 +19,9 @@ #![allow(non_snake_case)] /// Easier to import than anyhow::Result. +/// (Naming things "Result" pollutes the import scope or requires nasty renames. Gross.) pub type AnyhowResult = anyhow::Result; /// Useful re-export. pub use anyhow::anyhow; +pub use anyhow::bail; diff --git a/crates/service/job/w2l_download_job/Cargo.toml b/crates/service/job/w2l_download_job/Cargo.toml index 8d18bf01e0..3d212d5ea4 100644 --- a/crates/service/job/w2l_download_job/Cargo.toml +++ b/crates/service/job/w2l_download_job/Cargo.toml @@ -15,6 +15,7 @@ path = "src/main.rs" [dependencies] # Internal +buckets = { path = "../../../_types/buckets" } cloud_storage = { path = "../../../lib/cloud_storage" } config = { path = "../../../lib/config" } container_common = { path = "../../../lib/container_common" } diff --git a/crates/service/job/w2l_download_job/src/main.rs b/crates/service/job/w2l_download_job/src/main.rs index 7daf1b978a..c2d5da4e45 100644 --- a/crates/service/job/w2l_download_job/src/main.rs +++ b/crates/service/job/w2l_download_job/src/main.rs @@ -14,9 +14,9 @@ pub mod script_execution; use anyhow::{anyhow, Error}; +use buckets::util::hash_to_bucket_path_string::hash_to_bucket_path_string; use chrono::Utc; use cloud_storage::bucket_client::BucketClient; -use cloud_storage::bucket_paths::hash_to_bucket_path; use config::common_env::CommonEnv; use config::is_bad_video_download_url::is_bad_video_download_url; use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING; @@ -471,7 +471,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord) info!("File hash: {}", private_bucket_hash); // Full path to video/image - let full_object_path = hash_to_bucket_path( + let full_object_path = hash_to_bucket_path_string( &private_bucket_hash, Some(&downloader.bucket_root_w2l_template_uploads))?; diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 96fb5849b8..4142564277 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -16,6 +16,7 @@ path = "src/main.rs" [dependencies] # Internal +buckets = { path = "../../../_types/buckets" } cloud_storage = { path = "../../../lib/cloud_storage" } config = { path = "../../../lib/config" } container_common = { path = "../../../lib/container_common" } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs index d54e58ce60..489fe50ac0 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs @@ -8,7 +8,7 @@ use actix_web::http::StatusCode; use actix_web::web::BytesMut; use actix_web::{web, HttpResponse, HttpRequest, ResponseError}; use anyhow::anyhow; -use cloud_storage::bucket_paths::hash_to_bucket_path; +use buckets::util::hash_to_bucket_path_string::hash_to_bucket_path_string; use container_common::anyhow_result::AnyhowResult; use container_common::token::random_uuid::generate_random_uuid; use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes; @@ -254,7 +254,7 @@ pub async fn enqueue_infer_w2l_with_uploads( let audio_upload_bucket_hash = upload_uuid.clone(); - let audio_upload_bucket_path = hash_to_bucket_path( + let audio_upload_bucket_path = hash_to_bucket_path_string( &upload_uuid, Some(&server_state.audio_uploads_bucket_root) ).map_err(|e| {