New 'buckets' crate for cloud storage uploads

Cargo.lock

@@ -981,6 +981,13 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "buckets"
+version = "0.0.1"
+dependencies = [
+ "errors",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.6.1"
@@ -1143,6 +1150,7 @@ name = "cloud_storage"
 version = "0.0.1"
 dependencies = [
  "anyhow",
+ "buckets",
  "container_common",
  "log",
  "rust-s3",
@@ -4577,6 +4585,7 @@ dependencies = [
  "base64 0.13.0",
  "bcrypt",
  "billing_component",
+ "buckets",
  "chrono",
  "cloud_storage",
  "config",
@@ -5873,6 +5882,7 @@ name = "w2l-download-job"
 version = "0.0.1"
 dependencies = [
  "anyhow",
+ "buckets",
  "chrono",
  "clap 2.34.0",
  "cloud_storage",

Cargo.toml

@@ -2,6 +2,8 @@
 
 members = [
   # Types: common strongly-typed tokens, ids, payloads, etc.
+  # Also common cloud buckets and other "strongly typed" configs.
+  "crates/_types/buckets",
   "crates/_types/enums",
   "crates/_types/reusable_types", # TODO(bt, 2022-12-21): Deprecate
   "crates/_types/tokens",

crates/_types/buckets/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "buckets"
+edition = "2018"
+version = "0.0.1"
+authors = [
+    "Brandon Thomas <bt@brand.io>",
+    "Brandon Thomas <echelon@gmail.com>",
+]
+publish = false
+
+[lib]
+name = "buckets"
+path = "src/lib.rs"
+
+[dependencies]
+
+# Internal
+errors = { path = "../../lib/errors" }
+
+# External
+# (none)
+
+[dev-dependencies]

crates/_types/buckets/src/lib.rs

@@ -0,0 +1,24 @@
+//! buckets
+//!
+//! The purpose of this library is to maintain strongly typed cloud storage buckets
+//! that can be shared between projects.
+//!
+
+// Never allow these
+#![forbid(private_in_public)]
+#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
+
+// Okay to toggle
+#![forbid(unreachable_patterns)]
+#![forbid(unused_imports)]
+#![forbid(unused_mut)]
+#![forbid(unused_variables)]
+
+// Always allow
+#![allow(dead_code)]
+#![allow(non_snake_case)]
+
+pub mod util;
+
+pub mod private;
+pub mod public;

crates/_types/buckets/src/private/mod.rs

@@ -0,0 +1 @@
+pub mod private_path;

crates/_types/buckets/src/private/private_path.rs

@@ -0,0 +1,7 @@
+
+/// A trait implemented by all private paths.
+pub trait PrivatePath {
+  fn is_public(&self) -> bool {
+    false
+  }
+}

crates/_types/buckets/src/public/media_uploads/directory.rs

@@ -0,0 +1,50 @@
+use crate::public::public_path::PublicPath;
+use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
+
+// TODO: Use a central path registry for quick reference
+const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media";
+
+/// Directory for user media uploads.
+/// Each uploaded file gets its own directory so that we can store the original
+/// file alongside re-processed derivatives.
+#[derive(Clone)]
+pub struct MediaUploadDirectory {
+  object_hash: String,
+  directory: String,
+}
+
+impl PublicPath for MediaUploadDirectory {}
+
+impl MediaUploadDirectory {
+
+  pub fn from_object_hash(object_hash: &str) -> Self {
+    // TODO: Path construction could be cleaner.
+    let middle = hashed_directory_path_long_string(object_hash);
+    let directory = format!("{}/{}{}", MEDIA_UPLOAD_DIRECTORY, middle, object_hash);
+    Self {
+      object_hash: object_hash.to_string(),
+      directory,
+    }
+  }
+
+  pub fn get_directory_path_str(&self) -> &str {
+    &self.directory
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::public::media_uploads::directory::MediaUploadDirectory;
+
+  #[test]
+  pub fn get_directory_path_str() {
+    let directory = MediaUploadDirectory::from_object_hash("abcdefghijk");
+    assert_eq!(directory.get_directory_path_str(), "/media/a/b/c/d/e/abcdefghijk");
+  }
+
+  #[test]
+  pub fn get_directory_path_str_short_name() {
+    let directory = MediaUploadDirectory::from_object_hash("foo");
+    assert_eq!(directory.get_directory_path_str(), "/media/f/o/foo");
+  }
+}

crates/_types/buckets/src/public/media_uploads/mod.rs

@@ -0,0 +1,2 @@
+pub mod directory;
+pub mod original_file;

crates/_types/buckets/src/public/media_uploads/original_file.rs

@@ -0,0 +1,52 @@
+use crate::public::media_uploads::directory::MediaUploadDirectory;
+use crate::public::public_path::PublicPath;
+
+const ORIGINAL_FILE_BASENAME : &'static str = "original_contents.bin";
+
+/// The original user upload file.
+/// It may have derivative files (down samples, crops, etc.) that live alongside it.
+#[derive(Clone)]
+pub struct MediaUploadOriginalFilePath {
+  directory: MediaUploadDirectory,
+  full_object_path: String,
+}
+
+impl PublicPath for MediaUploadOriginalFilePath {}
+
+impl MediaUploadOriginalFilePath {
+
+  pub fn from_object_hash(hash: &str) -> Self {
+    // TODO: Path construction could be cleaner.
+    let directory = MediaUploadDirectory::from_object_hash(hash);
+    let full_object_path = format!("{}/{}", directory.get_directory_path_str(), ORIGINAL_FILE_BASENAME);
+    Self {
+      directory,
+      full_object_path,
+    }
+  }
+
+  pub fn get_basename() -> &'static str {
+    ORIGINAL_FILE_BASENAME
+  }
+
+  pub fn get_full_object_path_str(&self) -> &str {
+    &self.full_object_path
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath;
+
+  #[test]
+  pub fn get_full_object_path_str() {
+    let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk");
+    assert_eq!(file.get_full_object_path_str(), "/media/a/b/c/d/e/abcdefghijk/original_contents.bin");
+  }
+
+  #[test]
+  pub fn get_full_object_path_str_short_name() {
+    let file = MediaUploadOriginalFilePath::from_object_hash("foo");
+    assert_eq!(file.get_full_object_path_str(), "/media/f/o/foo/original_contents.bin");
+  }
+}

crates/_types/buckets/src/public/mod.rs

@@ -0,0 +1,2 @@
+pub mod media_uploads;
+pub mod public_path;

crates/_types/buckets/src/public/public_path.rs

@@ -0,0 +1,7 @@
+
+/// A trait implemented by all public paths.
+pub trait PublicPath {
+  fn is_public(&self) -> bool {
+    true
+  }
+}

crates/_types/buckets/src/util/hash_to_bucket_path.rs

@@ -1,11 +1,11 @@
-use anyhow::anyhow;
-use anyhow::bail;
-use container_common::anyhow_result::AnyhowResult;
+use errors::AnyhowResult;
+use errors::bail;
 use std::path::PathBuf;
 
-pub fn hash_to_bucket_path(file_hash: &str,
+#[deprecated(note = "this is a messy function and should be avoided")]
+pub (crate) fn hash_to_bucket_path(file_hash: &str,
                            optional_root_directory: Option<&str>)
-  -> AnyhowResult<String>
+  -> AnyhowResult<PathBuf>
 {
   if file_hash.len() < 4 {
     bail!("File length is too short");
@@ -23,29 +23,25 @@ pub fn hash_to_bucket_path(file_hash: &str,
     &file_hash,
   );
 
-  let maybe_path = root_directory
-    .join(file_path)
-    .to_str()
-    .map(|s| s.to_string());
-
-  maybe_path.ok_or(anyhow!("Path could not be converted"))
+  Ok(root_directory.join(file_path))
 }
 
 #[cfg(test)]
 mod tests {
-  use crate::bucket_paths::hash_to_bucket_path;
+  use crate::util::hash_to_bucket_path::hash_to_bucket_path;
+  use std::path::PathBuf;
 
   #[test]
   fn test_bucket_without_root() {
     assert_eq!(
       hash_to_bucket_path("abcdefg", None).unwrap(),
-      "/a/b/c/abcdefg".to_string());
+      PathBuf::from("/a/b/c/abcdefg"));
   }
 
   #[test]
   fn test_bucket_with_root() {
     assert_eq!(
       hash_to_bucket_path("abcdefg", Some("/root")).unwrap(),
-      "/root/a/b/c/abcdefg".to_string());
+      PathBuf::from("/root/a/b/c/abcdefg"));
   }
 }

crates/_types/buckets/src/util/hash_to_bucket_path_string.rs

@@ -0,0 +1,34 @@
+use crate::util::hash_to_bucket_path::hash_to_bucket_path;
+use errors::AnyhowResult;
+use errors::anyhow;
+
+#[deprecated(note = "this is a messy function and should be avoided")]
+pub fn hash_to_bucket_path_string(file_hash: &str,
+                           optional_root_directory: Option<&str>)
+  -> AnyhowResult<String>
+{
+  let path = hash_to_bucket_path(file_hash, optional_root_directory)?;
+
+  let maybe_path = path.to_str().map(|s| s.to_string());
+
+  maybe_path.ok_or(anyhow!("Path could not be converted"))
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
+
+  #[test]
+  fn test_bucket_without_root() {
+    assert_eq!(
+      hash_to_bucket_path_string("abcdefg", None).unwrap(),
+      "/a/b/c/abcdefg".to_string());
+  }
+
+  #[test]
+  fn test_bucket_with_root() {
+    assert_eq!(
+      hash_to_bucket_path_string("abcdefg", Some("/root")).unwrap(),
+      "/root/a/b/c/abcdefg".to_string());
+  }
+}

crates/_types/buckets/src/util/hashed_directory_path_long_string.rs

@@ -0,0 +1,67 @@
+
+/// Create a cloud object store directory structure that can be easily traversed without
+/// too many items living in a single directory.
+///
+/// With a million items, each directory will have ~1 item.
+/// With a billion items, each directory will have ~953 items (totally tractable).
+///
+pub fn hashed_directory_path_long_string(file_hash: &str) -> String {
+  match file_hash.len() {
+    0 | 1 => "".to_string(),
+    2 => format!("{}/", &file_hash[0..1]),
+    3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
+    4 => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
+    5 => format!("{}/{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3], &file_hash[3..4]),
+    _ => format!("{}/{}/{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3], &file_hash[3..4], &file_hash[4..5]),
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
+
+  #[test]
+  fn test_length_zero() {
+    assert_eq!(hashed_directory_path_long_string(""), "".to_string());
+  }
+
+  #[test]
+  fn test_length_one() {
+    assert_eq!(hashed_directory_path_long_string("a"), "".to_string());
+  }
+
+  #[test]
+  fn test_length_two() {
+    assert_eq!(hashed_directory_path_long_string("ab"), "a/".to_string());
+  }
+
+  #[test]
+  fn test_length_three() {
+    assert_eq!(hashed_directory_path_long_string("abc"), "a/b/".to_string());
+  }
+
+  #[test]
+  fn test_length_four() {
+    assert_eq!(hashed_directory_path_long_string("abcd"), "a/b/c/".to_string());
+  }
+
+  #[test]
+  fn test_length_five() {
+    assert_eq!(hashed_directory_path_long_string("abcde"), "a/b/c/d/".to_string());
+  }
+
+  #[test]
+  fn test_length_six() {
+    assert_eq!(hashed_directory_path_long_string("abcdef"), "a/b/c/d/e/".to_string());
+  }
+
+  #[test]
+  fn test_length_seven() {
+    assert_eq!(hashed_directory_path_long_string("abcdefg"), "a/b/c/d/e/".to_string());
+  }
+
+  #[test]
+  fn test_length_ten() {
+    assert_eq!(hashed_directory_path_long_string("abcdefghij"), "a/b/c/d/e/".to_string());
+  }
+}

crates/_types/buckets/src/util/hashed_directory_path_short_string.rs

@@ -0,0 +1,55 @@
+
+/// Create a cloud object store directory structure that can be easily traversed without
+/// too many items living in a single directory.
+///
+/// With a million items, each directory will have ~244 items.
+/// With a billion items, each directory will have ~244,140 items (impractical at this scale).
+///
+pub fn hashed_directory_path_short_string(file_hash: &str) -> String {
+  match file_hash.len() {
+    0 | 1 => "".to_string(),
+    2 => format!("{}/", &file_hash[0..1]),
+    3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
+    _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::util::hashed_directory_path_short_string::hashed_directory_path_short_string;
+
+  #[test]
+  fn test_length_zero() {
+    assert_eq!(hashed_directory_path_short_string(""), "".to_string());
+  }
+
+  #[test]
+  fn test_length_one() {
+    assert_eq!(hashed_directory_path_short_string("a"), "".to_string());
+  }
+
+  #[test]
+  fn test_length_two() {
+    assert_eq!(hashed_directory_path_short_string("ab"), "a/".to_string());
+  }
+
+  #[test]
+  fn test_length_three() {
+    assert_eq!(hashed_directory_path_short_string("abc"), "a/b/".to_string());
+  }
+
+  #[test]
+  fn test_length_four() {
+    assert_eq!(hashed_directory_path_short_string("abcd"), "a/b/c/".to_string());
+  }
+
+  #[test]
+  fn test_length_five() {
+    assert_eq!(hashed_directory_path_short_string("abcde"), "a/b/c/".to_string());
+  }
+
+  #[test]
+  fn test_length_ten() {
+    assert_eq!(hashed_directory_path_short_string("abcdefghij"), "a/b/c/".to_string());
+  }
+}

crates/_types/buckets/src/util/mod.rs

@@ -0,0 +1,4 @@
+pub (crate) mod hash_to_bucket_path;
+pub mod hash_to_bucket_path_string;
+pub mod hashed_directory_path_long_string;
+pub mod hashed_directory_path_short_string;

crates/lib/cloud_storage/Cargo.toml

@@ -15,6 +15,7 @@ path = "src/lib.rs"
 [dependencies]
 
 # Internal
+buckets = { path = "../../_types/buckets" }
 container_common = { path = "../container_common" }
 
 # External

crates/lib/cloud_storage/src/bucket_path_unifier.rs

@@ -1,9 +1,10 @@
+use buckets::util::hashed_directory_path_short_string::hashed_directory_path_short_string;
 use container_common::anyhow_result::AnyhowResult;
-use crate::bucket_paths::hash_to_bucket_path;
 use std::path::PathBuf;
 
 /// This is designed to make it centrally configurable where
 /// different types of objects are stored.
+#[deprecated(note = "see the 'buckets' crate for a better approach")]
 pub struct BucketPathUnifier {
   // TTS
   pub tts_synthesizer_model_root: PathBuf,
@@ -187,12 +188,7 @@ impl BucketPathUnifier {
   // ==================== UTILITY ==================== //
 
   pub fn hashed_directory_path(file_hash: &str) -> String {
-    match file_hash.len() {
-      0 | 1=> "".to_string(),
-      2 => format!("{}/", &file_hash[0..1]),
-      3 => format!("{}/{}/", &file_hash[0..1], &file_hash[1..2]),
-      _ => format!("{}/{}/{}/", &file_hash[0..1], &file_hash[1..2], &file_hash[2..3]),
-    }
+    hashed_directory_path_short_string(file_hash)
   }
 }
 

crates/lib/cloud_storage/src/lib.rs

@@ -1,3 +1,2 @@
 pub mod bucket_client;
 pub mod bucket_path_unifier;
-pub mod bucket_paths;

crates/lib/errors/src/lib.rs

@@ -19,7 +19,9 @@
 #![allow(non_snake_case)]
 
 /// Easier to import than anyhow::Result.
+/// (Naming things "Result" pollutes the import scope or requires nasty renames. Gross.)
 pub type AnyhowResult<T> = anyhow::Result<T>;
 
 /// Useful re-export.
 pub use anyhow::anyhow;
+pub use anyhow::bail;

crates/service/job/w2l_download_job/Cargo.toml

@@ -15,6 +15,7 @@ path = "src/main.rs"
 [dependencies]
 
 # Internal
+buckets = { path = "../../../_types/buckets" }
 cloud_storage = { path = "../../../lib/cloud_storage" }
 config = { path = "../../../lib/config" }
 container_common = { path = "../../../lib/container_common" }

crates/service/job/w2l_download_job/src/main.rs

@@ -14,9 +14,9 @@
 pub mod script_execution;
 
 use anyhow::{anyhow, Error};
+use buckets::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
 use chrono::Utc;
 use cloud_storage::bucket_client::BucketClient;
-use cloud_storage::bucket_paths::hash_to_bucket_path;
 use config::common_env::CommonEnv;
 use config::is_bad_video_download_url::is_bad_video_download_url;
 use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
@@ -471,7 +471,7 @@ async fn process_job(downloader: &Downloader, job: &W2lTemplateUploadJobRecord)
   info!("File hash: {}", private_bucket_hash);
 
   // Full path to video/image
-  let full_object_path = hash_to_bucket_path(
+  let full_object_path = hash_to_bucket_path_string(
     &private_bucket_hash,
     Some(&downloader.bucket_root_w2l_template_uploads))?;
 

crates/service/web/storyteller_web/Cargo.toml

@@ -16,6 +16,7 @@ path = "src/main.rs"
 [dependencies]
 
 # Internal
+buckets = { path = "../../../_types/buckets" }
 cloud_storage = { path = "../../../lib/cloud_storage" }
 config = { path = "../../../lib/config" }
 container_common = { path = "../../../lib/container_common" }

crates/service/web/storyteller_web/src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs

@@ -8,7 +8,7 @@ use actix_web::http::StatusCode;
 use actix_web::web::BytesMut;
 use actix_web::{web, HttpResponse, HttpRequest, ResponseError};
 use anyhow::anyhow;
-use cloud_storage::bucket_paths::hash_to_bucket_path;
+use buckets::util::hash_to_bucket_path_string::hash_to_bucket_path_string;
 use container_common::anyhow_result::AnyhowResult;
 use container_common::token::random_uuid::generate_random_uuid;
 use crate::http_server::web_utils::read_multipart_field_bytes::checked_read_multipart_bytes;
@@ -254,7 +254,7 @@ pub async fn enqueue_infer_w2l_with_uploads(
 
   let audio_upload_bucket_hash = upload_uuid.clone();
 
-  let audio_upload_bucket_path = hash_to_bucket_path(
+  let audio_upload_bucket_path = hash_to_bucket_path_string(
     &upload_uuid,
     Some(&server_state.audio_uploads_bucket_root)
   ).map_err(|e| {