builds properly now working on the download aspect.

Cargo.lock

@@ -912,9 +912,6 @@ version = "0.0.1"
 dependencies = [
  "crockford",
  "errors",
- "filesys",
- "hashing",
- "mimetypes",
  "my-workspace-hack",
 ]
 
@@ -1128,7 +1125,13 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "buckets",
+ "crockford",
+ "easyenv",
+ "errors",
+ "filesys",
+ "hashing",
  "log",
+ "mimetypes",
  "my-workspace-hack",
  "rust-s3",
  "tokio",

crates/_types/buckets/Cargo.toml

@@ -18,13 +18,6 @@ path = "src/lib.rs"
 crockford = { path = "../../lib/crockford" }
 errors = { path = "../../lib/errors" }
 
-# probably need to have it's own module.
-filesys = { path = "../../lib/files/filesys" }
-hashing = { path = "../../lib/files/hashing" }
-mimetypes = { path = "../../lib/files/mimetypes" }
-# have to add this to it's own module.....
-# buckets = { path = "../../_types/buckets" }
-# cloud_storage = { path = "../../lib/api_clients/cloud_storage" }
 
 # Workspace hack (faster compile times)
 my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" }

crates/_types/buckets/src/public/mod.rs

@@ -4,4 +4,3 @@ pub mod public_path;
 pub mod voice_conversion_results;
 pub mod weight_uploads;
 pub mod weight_files;
-pub mod files;

crates/cli/dev-database-seed/src/seeding/seed_media_files.rs

@@ -5,8 +5,10 @@ use sqlx::{MySql, Pool};
 
 use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
 use enums::by_table::media_files::media_file_type::MediaFileType;
+
 use enums::common::visibility::Visibility;
 use errors::AnyhowResult;
+
 use filesys::file_read_bytes::file_read_bytes;
 use filesys::file_size::file_size;
 use filesys::path_to_string::path_to_string;

crates/lib/api_clients/cloud_storage/Cargo.toml

@@ -16,6 +16,12 @@ path = "src/lib.rs"
 
 # Internal
 buckets = { path = "../../../_types/buckets" }
+crockford = { path = "../../../lib/crockford" }
+errors = { path = "../../../lib/errors" }
+easyenv = { path = "../../../lib/easyenv" }
+filesys = { path = "../../../lib/files/filesys" }
+hashing = { path = "../../../lib/files/hashing" }
+mimetypes = { path = "../../../lib/files/mimetypes" }
 
 # Workspace hack (faster compile times)
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }

crates/lib/api_clients/cloud_storage/src/bucket_client.rs

@@ -14,11 +14,26 @@ use tokio::io::AsyncReadExt;
 #[derive(Clone)]
 pub struct BucketClient {
   bucket: Bucket,
-  
   /// If set, put all files under this root path.
   optional_bucket_root: Option<String>,
 }
 
+
+#[derive(Debug)]
+pub enum BucketClientError {
+    ErrorWithCodeAndMessage { code: u16, message: String },
+}
+
+impl std::fmt::Display for BucketClientError {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match self {
+            BucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message),
+        }
+    }
+}
+impl std::error::Error for BucketClientError {}
+
+
 impl BucketClient {
 
   pub fn create(
@@ -87,6 +102,25 @@ impl BucketClient {
     Ok(())
   }
 
+  pub async fn upload_file_with_content_type_process(&self, object_name: &str, bytes: &[u8], content_type: &str) -> anyhow::AnyhowResult<()> {
+    info!("Filename for bucket: {}", object_name);
+    let object_name = self.get_rooted_object_name(object_name);
+    info!("Rooted filename for bucket: {}", object_name);
+    let response = self.bucket.put_object_with_content_type(&object_name, bytes, content_type).await?;
+    let body_bytes = response.bytes();
+    let code = response.status_code();
+    info!("upload code: {}", code);
+    if code != 200 {
+      let body = String::from_utf8_lossy(body_bytes);
+      warn!("upload body: {}", body);
+      BucketClientError::ErrorWithCodeAndMessage { code, message: body.to_string() };
+    } else {
+      info!("upload success: {}", code);
+      Ok(())
+    }
+  }
+
+  #[deprecated = "Use upload_file instead above it returns an error we can surface and act on."]
   pub async fn upload_file_with_content_type(&self, object_name: &str, bytes: &[u8], content_type: &str) -> anyhow::Result<()> {
     info!("Filename for bucket: {}", object_name);
 

crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs

@@ -1,9 +1,8 @@
 // Describes the file and how it will be saved to GCBucket
 // where the file should be stored based off the descriptor 
-
 use std::path::PathBuf;
 use crockford::crockford_entropy_lower;
-use crate::util::hashed_directory_path_long_string;
+use buckets::util::hashed_directory_path_long_string;
 
 // DEFAULT IMPLEMENTATION
 const REMOTE_FILE_DIRECTORY: &str = "/implement_google_cloud_storage_bucket_name";
@@ -31,76 +30,6 @@ pub trait FileDescriptor {
     }
 }
 
-pub struct FileBucketDirectory {
-  file_object_hash: String,
-  cloud_directory_hash: String, 
-  remote_cloud_base_directory: String,
-  full_remote_cloud_file_path: String,
-  file_name: String,
-  file_descriptor: Box<dyn FileDescriptor>
-}
-
-impl FileBucketDirectory {
-  pub fn generate_new(file_descriptor: Box<dyn FileDescriptor>) -> Self {
-    Self::from_object_hash(file_descriptor)
-  }
-
-  fn from_object_hash(file_descriptor: Box<dyn FileDescriptor>) -> Self {
-    let cloud_path_entropy = crockford_entropy_lower(32);
-    let file_name_entropy  = crockford_entropy_lower(32);
-
-    // gets you wiki /a/b/c/d folder structure
-    let middle = hashed_directory_path_long_string::hashed_directory_path_long_string(cloud_path_entropy.as_ref());
-
-    // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
-    let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, cloud_path_entropy);
-
-    // gets you name of the file with suffix and prefix and entropy in the centre
-    let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), file_name_entropy, file_descriptor.get_suffix());
-    // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
-    
-    // This is the full path you upload to.
-    let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name);
-    Self {
-      file_object_hash: file_name_entropy,
-      cloud_directory_hash: cloud_path_entropy,
-      remote_cloud_base_directory: remote_cloud_base_directory,
-      full_remote_cloud_file_path: full_remote_cloud_file_path,
-      file_name:file_name,
-      file_descriptor:file_descriptor
-    }
-  }
-  
-  pub fn get_file_object_hash(&self) -> &str {
-      &self.file_object_hash
-  }
-
-  pub fn get_cloud_directory_hash(&self) -> &str {
-      &self.cloud_directory_hash
-  }
-
-  pub fn get_remote_cloud_base_directory(&self) -> &str {
-      &self.remote_cloud_base_directory
-  }
-
-  pub fn get_full_remote_cloud_file_path(&self) -> &str {
-      &self.full_remote_cloud_file_path
-  }
-
-  pub fn to_full_remote_cloud_file_path_pathbuf(&self) -> PathBuf {
-    PathBuf::from(&self.full_remote_cloud_file_path)
-  }
-
-  pub fn get_file_name(&self) -> &str {
-      &self.file_name
-  }
-
-  // pub fn get_file_descriptor(&self) -> Box<dyn FileDescriptor> {
-  //   self.file_descriptor
-  // }
-
-}
-
 #[cfg(test)]
 mod tests {
 
@@ -108,8 +37,8 @@ mod tests {
   pub fn test() {}
 
   pub fn generate_new_entropy() {
-    let directory = FileDescriptorBucketDirectory::generate_new();
-    assert_eq!(directory.get_object_hash().len(), 32);
+    // let directory = FileDescriptorBucketDirectory::generate_new();
+    // assert_eq!(directory.get_object_hash().len(), 32);
   }
 
   // #[test]

crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs

@@ -0,0 +1,66 @@
+
+use std::path::PathBuf;
+use crockford::crockford_entropy_lower;
+use buckets::util::hashed_directory_path_long_string;
+use crate::remote_file_manager::file_descriptor::FileDescriptor;
+
+pub struct FileBucketDirectory {
+    file_object_hash: String, 
+    remote_cloud_base_directory: String,
+    full_remote_cloud_file_path: String,
+    file_name: String,
+    file_descriptor: Box<dyn FileDescriptor>
+  }
+  
+  impl FileBucketDirectory {
+    pub fn generate_new(file_descriptor: Box<dyn FileDescriptor>) -> Self {
+      Self::from_object_hash(file_descriptor)
+    }
+  
+    fn from_object_hash(file_descriptor: Box<dyn FileDescriptor>) -> Self {
+      let entropy = crockford_entropy_lower(32);
+      // gets you wiki /a/b/c/d folder structure
+      let middle = hashed_directory_path_long_string::hashed_directory_path_long_string(entropy.as_ref());
+      // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy
+      let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy);
+      // gets you name of the file with suffix and prefix and entropy in the centre
+      let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix());
+      // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit.
+    
+      // This is the full path you upload to.
+      let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name);
+      Self {
+        file_object_hash: entropy,
+        remote_cloud_base_directory: remote_cloud_base_directory,
+        full_remote_cloud_file_path: full_remote_cloud_file_path,
+        file_name:file_name,
+        file_descriptor:file_descriptor
+      }
+    }
+    
+    pub fn get_file_object_hash(&self) -> &str {
+        &self.file_object_hash
+    }
+
+    pub fn get_remote_cloud_base_directory(&self) -> &str {
+        &self.remote_cloud_base_directory
+    }
+  
+    pub fn get_full_remote_cloud_file_path(&self) -> &str {
+        &self.full_remote_cloud_file_path
+    }
+  
+    pub fn to_full_remote_cloud_file_path_pathbuf(&self) -> PathBuf {
+      PathBuf::from(&self.full_remote_cloud_file_path)
+    }
+  
+    pub fn get_file_name(&self) -> &str {
+        &self.file_name
+    }
+  
+    // pub fn get_file_descriptor(&self) -> Box<dyn FileDescriptor> {
+    //   self.file_descriptor
+    // }
+  
+  }
+  

crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs

@@ -1,7 +1,7 @@
 pub mod remote_cloud_file_manager;
-
+pub mod file_directory;
 pub mod file_meta_data;
-
 pub mod file_descriptor;
+
 pub mod media_descriptor;
 pub mod weights_descriptor;

crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs

@@ -1,49 +1,94 @@
 
-use crate::public::files::file_descriptor::FileDescriptor;
-use crate::public::files::file_meta_data::FileMetaData;
+use crate::bucket_client::BucketClient;
+use super::file_descriptor::FileDescriptor;
+use super::file_meta_data::FileMetaData;
 
-use super::file_descriptor::FileBucketDirectory;
+use std::time::Duration;
+use errors::{AnyhowError, AnyhowResult};
 
-use errors::AnyhowError;
 use filesys::file_read_bytes::file_read_bytes;
 use filesys::file_size::file_size;
 use filesys::path_to_string::path_to_string;
 use hashing::sha256::sha256_hash_file::sha256_hash_file;
-
 use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes;
 use mimetypes::mimetype_for_file::get_mimetype_for_file;
+use s3::bucket;
+use log::info;
 
-struct WebFileManager {
-    // bucket_client: BucketClients,
+// Takes the definition from database and is an input to enable us to download the file.
+struct RemoteCloudBucketDetails {
+    bucket_hash: String,
+    // Might require a migration to add this field to the database?
+    maybe_bucket_prefix: Option<String>,
+    maybe_bucket_extension: Option<String>,
+}
+
+struct RemoteCloudFileClient {
+    bucket_client: BucketClient,
     file_descriptor: Box<dyn FileDescriptor>
 }
 
-impl WebFileManager {
+impl RemoteCloudFileClient {
     // bucket_client:BucketClients
-    fn new(file_descriptor: Box<dyn FileDescriptor>) -> WebFileManager {
-        WebFileManager {
-            // bucket_client: bucket_client,
-            file_descriptor: file_descriptor
-        }
+    fn bucket_client_for_object(&self) -> &BucketClient {
+        &self.bucket_client
     }
 
-    
-    // also include bucket details here
-    pub fn download_file(&self, system_file_path:String) -> Result<(),AnyhowError> {
+    fn new(file_descriptor: Box<dyn FileDescriptor>) -> AnyhowResult<RemoteCloudFileClient> {
+        // Please verify that this is the correct way to get the bucket clients public and private in production and dev?
+        let access_key = easyenv::get_env_string_required("ACCESS_KEY")?;
+        let secret_key = easyenv::get_env_string_required("SECRET_KEY")?;
+        let region_name = easyenv::get_env_string_required("REGION_NAME")?;
         
-    }
-    // return error or success with meta data.
-    pub fn upload_file(&self, system_file_path:String) -> Result<FileMetaData,AnyhowError> {
-        // let bucket_client = self.bucket_client.clone();
-        if self.file_descriptor.is_public() {
-           
-        }else {
-            // use private bucket client
+        let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?;
+        let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?;
+        // NB: Long timeout for dev rust builds to upload to cloud buckets.
+        // Unoptimized binaries sometimes take a lot of time to upload, presumably due to unoptimized code.
+        let bucket_timeout = easyenv::get_env_duration_seconds_or_default(
+          "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10));
+        let mut bucket_client:BucketClient;
+        if file_descriptor.is_public() {
+            // use public bucket client
+            info!("Configuring public GCS bucket...");
+            bucket_client = BucketClient::create(
+              &access_key,
+              &secret_key,
+              &region_name,
+              &public_bucket_name,
+              None,
+              Some(bucket_timeout),
+            )?;
+        } else {    
+            // use private  bucket client
+            info!("Configuring private GCS bucket...");
+            bucket_client = BucketClient::create(
+            &access_key,
+            &secret_key,
+            &region_name,
+            &private_bucket_name,
+            None,
+            Some(bucket_timeout),
+            )?;
         }
-        let result = Self::get_file_meta_data(system_file_path.clone())?;
-        Ok(result)
+
+        Ok(RemoteCloudFileClient {
+            bucket_client: bucket_client,
+            file_descriptor: file_descriptor
+        })
     }
 
+    // // also include bucket details here
+    // pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyHowResult<> {
+    //     self.bucket_client.download_file_to_disk(object_path, filesystem_path)
+    // }
+
+    // // return error or success with meta data.
+    // pub async fn upload_file(&self, from_system_file_path:String) -> Result<FileMetaData,AnyhowError> {
+    //     // get file meta data
+    //     let result = Self::get_file_meta_data(system_file_path.clone())?;
+    //     Ok(result)
+    // }
+
     // Retrieve the metadata from the file
     pub fn get_file_meta_data(system_file_path:String) -> Result<FileMetaData,AnyhowError> {
         let file_size_bytes = file_size(system_file_path.clone())?;
@@ -60,16 +105,10 @@ impl WebFileManager {
     }
 }
 
-
-// take in system file path to upload.
-// take in system file path to download to.
-
-// should be able to take prefix suffix and entrpy to generate a file descriptor    
-
 #[cfg(test)]
 mod tests {
     //use crate::public::files::file_descriptor::{FileDescriptor};
-    //use super::WebFileManager;
+    //use super::RemoteCloudFileClient;
     #[test]
     fn test_web_file_manager() {
         println!("test")