Merge pull request #218 from storytold/feature/weights_files

Feature/weights files

crates/_types/buckets/src/public/files/descriptor.rs

@@ -0,0 +1,3 @@
+pub trait descriptor {
+    fn my_method(&self);
+}

crates/_types/buckets/src/public/files/mod.rs

@@ -0,0 +1,4 @@
+pub mod bucket_directory;
+pub mod bucket_file_path;
+pub mod web_file_manager;
+

crates/_types/buckets/src/public/files/web_file_manager.rs

@@ -0,0 +1,32 @@
+
+pub struct FileBucketPath {
+    pub bucket_clients: BucketClients
+}
+
+impl web_file_manager {
+    pub fn new(my_field: i32) -> Self {
+        web_file_manager { my_field }
+    }
+
+    pub fn increment(&mut self) {
+        self.my_field += 1;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_new() {
+        let my_class = web_file_manager::new(10);
+        assert_eq!(my_class.my_field, 10);
+    }
+
+    #[test]
+    fn test_increment() {
+        let mut my_class = web_file_manager::new(10);
+        my_class.increment();
+        assert_eq!(my_class.my_field, 11);
+    }
+}

crates/_types/buckets/src/public/mod.rs

@@ -2,4 +2,5 @@ pub mod media_files;
 pub mod media_uploads;
 pub mod public_path;
 pub mod voice_conversion_results;
-pub mod weight_uploads;
+pub mod weight_uploads;
+pub mod weight_files;

crates/_types/buckets/src/public/weight_files/bucket_directory.rs

@@ -0,0 +1,75 @@
+use crockford::crockford_entropy_lower;
+
+use crate::public::public_path::PublicPath;
+use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
+
+// TODO: Generate these from a macro.
+// TODO: Use a central path registry for quick reference
+const WEIGHT_FILE_DIRECTORY: &str = "/weights";
+
+/// Directory for user weight uploads.
+/// Each uploaded file gets its own directory so that we can store the original
+/// file alongside re-processed derivatives.
+#[derive(Clone)]
+pub struct WeightFileBucketDirectory {
+  object_hash: String,
+  directory: String,
+}
+
+impl PublicPath for WeightFileBucketDirectory {}
+
+impl WeightFileBucketDirectory {
+
+  pub fn generate_new() -> Self {
+    let entropy = crockford_entropy_lower(32);
+    Self::from_object_hash(&entropy)
+  }
+
+  pub fn from_object_hash(object_hash: &str) -> Self {
+    // TODO: Path construction could be cleaner.
+    let middle = hashed_directory_path_long_string(object_hash);
+    let directory = format!("{}/{}{}", WEIGHT_FILE_DIRECTORY, middle, object_hash);
+    Self {
+      object_hash: object_hash.to_string(),
+      directory,
+    }
+  }
+
+  pub fn get_directory_path_str(&self) -> &str {
+    &self.directory
+  }
+
+  pub fn get_object_hash(&self) -> &str {
+    &self.object_hash
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+
+  #[test]
+  pub fn generate_new_entropy() {
+    let directory = WeightFileBucketDirectory::generate_new();
+    assert_eq!(directory.get_object_hash().len(), 32);
+  }
+
+  #[test]
+  pub fn get_directory_path_str() {
+    let directory = WeightFileBucketDirectory::from_object_hash("abcdefghijk");
+    assert_eq!(directory.get_directory_path_str(), "/weights/a/b/c/d/e/abcdefghijk");
+  }
+
+  #[test]
+  pub fn get_directory_path_str_short_name() {
+    let directory = WeightFileBucketDirectory::from_object_hash("foo");
+    assert_eq!(directory.get_directory_path_str(), "/weights/f/o/foo");
+  }
+
+  #[test]
+  pub fn get_object_hash() {
+    let hash = "abcdefghijk";
+    let directory = WeightFileBucketDirectory::from_object_hash(hash);
+    assert_eq!(directory.get_object_hash(), hash);
+  }
+}

crates/_types/buckets/src/public/weight_files/bucket_file_path.rs

@@ -0,0 +1,142 @@
+use std::path::PathBuf;
+
+use crockford::crockford_entropy_lower;
+use crate::public::public_path::PublicPath;
+
+const ORIGINAL_FILE_BASENAME : &str = "file.bin";
+
+// TODO: Generate these from a macro.
+use crate::public::weight_files::bucket_directory::WeightFileBucketDirectory;
+/// The original user upload file.
+/// It may have derivative files (down samples, crops, etc.) that live alongside it.
+#[derive(Clone)]
+pub struct WeightFileBucketPath {
+  directory: WeightFileBucketDirectory,
+
+  filename: String,
+  full_object_path: String,
+
+  optional_prefix: Option<String>,
+  /// NB: Extension contains the leading period, eg. ".mp4"
+  optional_extension: Option<String>,
+}
+
+impl PublicPath for WeightFileBucketPath {}
+
+impl WeightFileBucketPath {
+
+  pub fn generate_new(optional_prefix: Option<&str>, optional_extension: Option<&str>) -> Self {
+    let entropy = crockford_entropy_lower(32);
+    Self::from_object_hash(&entropy, optional_prefix, optional_extension)
+  }
+
+  pub fn from_object_hash(hash: &str, optional_prefix: Option<&str>, optional_extension: Option<&str>) -> Self {
+    // TODO: Path construction could be cleaner.
+    let directory = WeightFileBucketDirectory::from_object_hash(hash);
+
+    let filename = match (optional_prefix, optional_extension) {
+      (None, None) => ORIGINAL_FILE_BASENAME.to_string(),
+      (None, Some(ext)) => format!("{}{}", hash, ext),
+      (Some(pre), None) => format!("{}{}", pre, hash),
+      (Some(pre), Some(ext)) => format!("{}{}{}", pre, hash, ext),
+    };
+
+    let full_object_path = format!("{}/{}", directory.get_directory_path_str(), filename);
+
+    Self {
+      directory,
+      filename,
+      full_object_path,
+      optional_prefix: optional_prefix.map(|p| p.to_string()),
+      optional_extension: optional_extension.map(|e| e.to_string()),
+    }
+  }
+
+  pub fn get_full_object_path_str(&self) -> &str {
+    &self.full_object_path
+  }
+
+  pub fn to_full_object_pathbuf(&self) -> PathBuf {
+    PathBuf::from(&self.full_object_path)
+  }
+
+  pub fn get_directory(&self) -> &WeightFileBucketDirectory {
+    &self.directory
+  }
+
+  pub fn get_object_hash(&self) -> &str {
+    &self.directory.get_object_hash()
+  }
+
+  pub fn get_basename(&self) -> &str {
+    &self.filename
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use std::path::PathBuf;
+
+  use crate::public::weight_files::bucket_file_path::WeightFileBucketPath;
+  mod with_prefix_and_extension {
+    use super::*;
+
+    #[test]
+    pub fn generate_new_entropy() {
+      let file = WeightFileBucketPath::generate_new(Some("pre_"), Some(".mp4"));
+      assert_eq!(file.get_object_hash().len(), 32);
+      assert_eq!(file.get_object_hash().len(), 32);
+      assert_eq!(file.get_directory().get_object_hash().len(), 32);
+    }
+
+    #[test]
+    pub fn get_full_object_path_str() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", Some("pre_"), Some(".mp4"));
+      assert_eq!(file.get_full_object_path_str(), "/weights/a/b/c/d/e/abcdefghijk/pre_abcdefghijk.mp4");
+      assert_eq!(file.get_basename(), "pre_abcdefghijk.mp4");
+    }
+  }
+
+  mod without_prefix_and_extension {
+    use super::*;
+
+    #[test]
+    pub fn generate_new_entropy() {
+      let file = WeightFileBucketPath::generate_new(None, None);
+      assert_eq!(file.get_object_hash().len(), 32);
+      assert_eq!(file.get_directory().get_object_hash().len(), 32);
+    }
+
+    #[test]
+    pub fn get_full_object_path_str() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None);
+      assert_eq!(file.get_full_object_path_str(), "/weights/a/b/c/d/e/abcdefghijk/file.bin");
+    }
+
+    #[test]
+    pub fn to_full_object_pathbuf() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None);
+      assert_eq!(file.to_full_object_pathbuf(), PathBuf::from("/weights/a/b/c/d/e/abcdefghijk/file.bin"));
+    }
+
+    #[test]
+    pub fn get_full_object_path_str_short_name() {
+      let file = WeightFileBucketPath::from_object_hash("foo", None, None);
+      assert_eq!(file.get_full_object_path_str(), "/weights/f/o/foo/file.bin");
+    }
+
+    #[test]
+    pub fn get_full_object_path_str_starts_with_directory() {
+      let file = WeightFileBucketPath::from_object_hash("abcdefghijk", None, None);
+      let full_path = file.get_full_object_path_str();
+      assert!(full_path.starts_with(file.get_directory().get_directory_path_str()))
+    }
+
+    #[test]
+    pub fn get_object_hash() {
+      let hash = "abcdefghijk";
+      let file = WeightFileBucketPath::from_object_hash(hash, None, None);
+      assert_eq!(file.get_object_hash(), hash);
+    }
+  }
+}

crates/_types/buckets/src/public/weight_files/mod.rs

@@ -0,0 +1,2 @@
+pub mod bucket_directory;
+pub mod bucket_file_path;

crates/cli/dev_database_seed/src/seeding/pythonex.py

@@ -0,0 +1,33 @@
+from types import Optional
+
+class FileInfo:
+    size_in_bytes: int
+    mime_type: str
+    file_name: str
+    
+class BucketDetails:
+    bucket_hash: str
+    maybe_bucket_name: Optional[str]
+    maybe_bucket_extension: Optional[str]
+    
+    
+class MediaFileBucketDirectory: 
+  object_hash: str
+  directory: str
+  optional_prefix: Optional[str]
+  optional_prefix_extension: Optional[str]
+  original_file_base_name: str
+  
+class WeightsType:
+    directory: float
+    
+class FileOperationManager:
+    def __init__(self, bucket,bucket_info:BucketDetails):
+        self.bucket = bucket
+        self.bucket_info = bucket_info
+
+    def check_if_file_exists_in_db(self, file_token: str, callback) -> bool:
+        pass
+    
+    
+    

crates/cli/dev_database_seed/src/seeding/seed_model_weights.rs

@@ -1,8 +1,10 @@
 use std::path::Path;
 
-use log::{info, warn};
-use rand::Rng;
-use sqlx::{MySql, Pool};
+use log::{ info, warn };
+use sqlx::{ MySql, Pool };
+
+use buckets::{ public::{ media_files::bucket_file_path::MediaFileBucketPath, self, weight_files::bucket_file_path::WeightFileBucketPath }, private };
+
 
 use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
 use enums::{
@@ -211,7 +213,7 @@ async fn seed_file_to_bucket(
         .map(|extension| extension.to_str())
         .flatten();
     // we should just have this be file bucket path and then we can use a file descriptor to turn it into a specific type of path
-    let bucket_location = MediaFileBucketPath::generate_new(
+    let bucket_location = WeightFileBucketPath::generate_new(
         maybe_bucket_prefix,
         maybe_bucket_extension
     );
@@ -223,7 +225,7 @@ async fn seed_file_to_bucket(
     let bytes = file_read_bytes(weight_file_path)?;
     let mimetype = get_mimetype_for_bytes(&bytes).unwrap_or("application/octet-stream");
 
-    info!("Uploading weights file to bucket path: {:?}", bucket_path);
+    info!("Copy this line here! Uploading weights file to bucket path: {:?}", bucket_path);
 
     let _r = bucket_clients.public.upload_file_with_content_type(
         &bucket_path,