diff --git a/crates/_types/buckets/src/public/files/descriptor.rs b/crates/_types/buckets/src/public/files/descriptor.rs new file mode 100644 index 0000000000..4b642780f7 --- /dev/null +++ b/crates/_types/buckets/src/public/files/descriptor.rs @@ -0,0 +1,3 @@ +pub trait descriptor { + fn my_method(&self); +} diff --git a/crates/_types/buckets/src/public/files/media_descriptor.rs b/crates/_types/buckets/src/public/files/media_descriptor.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/_types/buckets/src/public/files/mod.rs b/crates/_types/buckets/src/public/files/mod.rs new file mode 100644 index 0000000000..b69a1e9b06 --- /dev/null +++ b/crates/_types/buckets/src/public/files/mod.rs @@ -0,0 +1,4 @@ +pub mod bucket_directory; +pub mod bucket_file_path; +pub mod web_file_manager; + diff --git a/crates/_types/buckets/src/public/files/web_file_manager.rs b/crates/_types/buckets/src/public/files/web_file_manager.rs new file mode 100644 index 0000000000..8ec339622c --- /dev/null +++ b/crates/_types/buckets/src/public/files/web_file_manager.rs @@ -0,0 +1,32 @@ + +pub struct FileBucketPath { + pub bucket_clients: BucketClients +} + +impl web_file_manager { + pub fn new(my_field: i32) -> Self { + web_file_manager { my_field } + } + + pub fn increment(&mut self) { + self.my_field += 1; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_new() { + let my_class = web_file_manager::new(10); + assert_eq!(my_class.my_field, 10); + } + + #[test] + fn test_increment() { + let mut my_class = web_file_manager::new(10); + my_class.increment(); + assert_eq!(my_class.my_field, 11); + } +} \ No newline at end of file diff --git a/crates/_types/buckets/src/public/files/weights_descriptor.rs b/crates/_types/buckets/src/public/files/weights_descriptor.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index fcceb68d9b..97aca68159 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -2,4 +2,5 @@ pub mod media_files; pub mod media_uploads; pub mod public_path; pub mod voice_conversion_results; -pub mod weight_uploads; \ No newline at end of file +pub mod weight_uploads; +pub mod weight_files; \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_files/bucket_directory.rs b/crates/_types/buckets/src/public/weight_files/bucket_directory.rs new file mode 100644 index 0000000000..5c1bb2445c --- /dev/null +++ b/crates/_types/buckets/src/public/weight_files/bucket_directory.rs @@ -0,0 +1,75 @@ +use crockford::crockford_entropy_lower; + +use crate::public::public_path::PublicPath; +use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string; + +// TODO: Generate these from a macro. +// TODO: Use a central path registry for quick reference +const WEIGHT_FILE_DIRECTORY: &str = "/weights"; + +/// Directory for user weight uploads. +/// Each uploaded file gets its own directory so that we can store the original +/// file alongside re-processed derivatives. +#[derive(Clone)] +pub struct WeightFileBucketDirectory { + object_hash: String, + directory: String, +} + +impl PublicPath for WeightFileBucketDirectory {} + +impl WeightFileBucketDirectory { + + pub fn generate_new() -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy) + } + + pub fn from_object_hash(object_hash: &str) -> Self { + // TODO: Path construction could be cleaner. + let middle = hashed_directory_path_long_string(object_hash); + let directory = format!("{}/{}{}", WEIGHT_FILE_DIRECTORY, middle, object_hash); + Self { + object_hash: object_hash.to_string(), + directory, + } + } + + pub fn get_directory_path_str(&self) -> &str { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.object_hash + } +} + +#[cfg(test)] +mod tests { + use crate::public::weight_files::bucket_directory::WeightFileBucketDirectory; + + #[test] + pub fn generate_new_entropy() { + let directory = WeightFileBucketDirectory::generate_new(); + assert_eq!(directory.get_object_hash().len(), 32); + } + + #[test] + pub fn get_directory_path_str() { + let directory = WeightFileBucketDirectory::from_object_hash("abcdefghijk"); + assert_eq!(directory.get_directory_path_str(), "/weights/a/b/c/d/e/abcdefghijk"); + } + + #[test] + pub fn get_directory_path_str_short_name() { + let directory = WeightFileBucketDirectory::from_object_hash("foo"); + assert_eq!(directory.get_directory_path_str(), "/weights/f/o/foo"); + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let directory = WeightFileBucketDirectory::from_object_hash(hash); + assert_eq!(directory.get_object_hash(), hash); + } +} \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_files/bucket_file_path.rs b/crates/_types/buckets/src/public/weight_files/bucket_file_path.rs new file mode 100644 index 0000000000..e1048a5494 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_files/bucket_file_path.rs @@ -0,0 +1,142 @@ +use std::path::PathBuf; + +use crockford::crockford_entropy_lower; +use crate::public::public_path::PublicPath; + +const ORIGINAL_FILE_BASENAME : &str = "file.bin"; + +// TODO: Generate these from a macro. +use crate::public::weight_files::bucket_directory::WeightFileBucketDirectory; +/// The original user upload file. +/// It may have derivative files (down samples, crops, etc.) that live alongside it. +#[derive(Clone)] +pub struct WeightFileBucketPath { + directory: WeightFileBucketDirectory, + + filename: String, + full_object_path: String, + + optional_prefix: Option, + /// NB: Extension contains the leading period, eg. ".mp4" + optional_extension: Option, +} + +impl PublicPath for WeightFileBucketPath {} + +impl WeightFileBucketPath { + + pub fn generate_new(optional_prefix: Option<&str>, optional_extension: Option<&str>) -> Self { + let entropy = crockford_entropy_lower(32); + Self::from_object_hash(&entropy, optional_prefix, optional_extension) + } + + pub fn from_object_hash(hash: &str, optional_prefix: Option<&str>, optional_extension: Option<&str>) -> Self { + // TODO: Path construction could be cleaner. + let directory = WeightFileBucketDirectory::from_object_hash(hash); + + let filename = match (optional_prefix, optional_extension) { + (None, None) => ORIGINAL_FILE_BASENAME.to_string(), + (None, Some(ext)) => format!("{}{}", hash, ext), + (Some(pre), None) => format!("{}{}", pre, hash), + (Some(pre), Some(ext)) => format!("{}{}{}", pre, hash, ext), + }; + + let full_object_path = format!("{}/{}", directory.get_directory_path_str(), filename); + + Self { + directory, + filename, + full_object_path, + optional_prefix: optional_prefix.map(|p| p.to_string()), + optional_extension: optional_extension.map(|e| e.to_string()), + } + } + + pub fn get_full_object_path_str(&self) -> &str { + &self.full_object_path + } + + pub fn to_full_object_pathbuf(&self) -> PathBuf { + PathBuf::from(&self.full_object_path) + } + + pub fn get_directory(&self) -> &WeightFileBucketDirectory { + &self.directory + } + + pub fn get_object_hash(&self) -> &str { + &self.directory.get_object_hash() + } + + pub fn get_basename(&self) -> &str { + &self.filename + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::public::weight_files::bucket_file_path::WeightFileBucketPath; + mod with_prefix_and_extension { + use super::*; + + #[test] + pub fn generate_new_entropy() { + let file = WeightFileBucketPath::generate_new(Some("pre_"), Some(".mp4")); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_directory().get_object_hash().len(), 32); + } + + #[test] + pub fn get_full_object_path_str() { + let file = WeightFileBucketPath::from_object_hash("abcdefghijk", Some("pre_"), Some(".mp4")); + assert_eq!(file.get_full_object_path_str(), "/weights/a/b/c/d/e/abcdefghijk/pre_abcdefghijk.mp4"); + assert_eq!(file.get_basename(), "pre_abcdefghijk.mp4"); + } + } + + mod without_prefix_and_extension { + use super::*; + + #[test] + pub fn generate_new_entropy() { + let file = WeightFileBucketPath::generate_new(None, None); + assert_eq!(file.get_object_hash().len(), 32); + assert_eq!(file.get_directory().get_object_hash().len(), 32); + } + + #[test] + pub fn get_full_object_path_str() { + let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None); + assert_eq!(file.get_full_object_path_str(), "/weights/a/b/c/d/e/abcdefghijk/file.bin"); + } + + #[test] + pub fn to_full_object_pathbuf() { + let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None); + assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/weights/a/b/c/d/e/abcdefghijk/file.bin")); + } + + #[test] + pub fn get_full_object_path_str_short_name() { + let file = WeightFileBucketPath::from_object_hash("foo", None, None); + assert_eq!(file.get_full_object_path_str(), "/weights/f/o/foo/file.bin"); + } + + #[test] + pub fn get_full_object_path_str_starts_with_directory() { + let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None); + let full_path = file.get_full_object_path_str(); + assert!(full_path.starts_with(file.get_directory().get_directory_path_str())) + } + + #[test] + pub fn get_object_hash() { + let hash = "abcdefghijk"; + let file = WeightFileBucketPath::from_object_hash(hash, None, None); + assert_eq!(file.get_object_hash(), hash); + } + } +} \ No newline at end of file diff --git a/crates/_types/buckets/src/public/weight_files/mod.rs b/crates/_types/buckets/src/public/weight_files/mod.rs new file mode 100644 index 0000000000..29c183a5c6 --- /dev/null +++ b/crates/_types/buckets/src/public/weight_files/mod.rs @@ -0,0 +1,2 @@ +pub mod bucket_directory; +pub mod bucket_file_path; \ No newline at end of file diff --git a/crates/cli/dev_database_seed/src/seeding/pythonex.py b/crates/cli/dev_database_seed/src/seeding/pythonex.py new file mode 100644 index 0000000000..19c31779f1 --- /dev/null +++ b/crates/cli/dev_database_seed/src/seeding/pythonex.py @@ -0,0 +1,33 @@ +from types import Optional + +class FileInfo: + size_in_bytes: int + mime_type: str + file_name: str + +class BucketDetails: + bucket_hash: str + maybe_bucket_name: Optional[str] + maybe_bucket_extension: Optional[str] + + +class MediaFileBucketDirectory: + object_hash: str + directory: str + optional_prefix: Optional[str] + optional_prefix_extension: Optional[str] + original_file_base_name: str + +class WeightsType: + directory: float + +class FileOperationManager: + def __init__(self, bucket,bucket_info:BucketDetails): + self.bucket = bucket + self.bucket_info = bucket_info + + def check_if_file_exists_in_db(self, file_token: str, callback) -> bool: + pass + + + diff --git a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs index e3e025f855..66f222c408 100644 --- a/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs +++ b/crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs @@ -1,8 +1,10 @@ use std::path::Path; -use log::{info, warn}; -use rand::Rng; -use sqlx::{MySql, Pool}; +use log::{ info, warn }; +use sqlx::{ MySql, Pool }; + +use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self, weight_files::bucket_file_path::WeightFileBucketPath }, private }; + use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::{ @@ -211,7 +213,7 @@ async fn seed_file_to_bucket( .map(|extension| extension.to_str()) .flatten(); // we should just have this be file bucket path and then we can use a file descriptor to turn it into a specific type of path - let bucket_location = MediaFileBucketPath::generate_new( + let bucket_location = WeightFileBucketPath::generate_new( maybe_bucket_prefix, maybe_bucket_extension ); @@ -223,7 +225,7 @@ async fn seed_file_to_bucket( let bytes = file_read_bytes(weight_file_path)?; let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream"); - info!("Uploading weights file to bucket path: {:?}", bucket_path); + info!("Copy this line here! Uploading weights file to bucket path: {:?}", bucket_path); let _r = bucket_clients.public.upload_file_with_content_type( &bucket_path,