diff --git a/Cargo.lock b/Cargo.lock index e3511bf717..1e791d92eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -912,9 +912,6 @@ version = "0.0.1" dependencies = [ "crockford", "errors", - "filesys", - "hashing", - "mimetypes", "my-workspace-hack", ] @@ -1128,7 +1125,13 @@ version = "0.0.1" dependencies = [ "anyhow", "buckets", + "crockford", + "easyenv", + "errors", + "filesys", + "hashing", "log", + "mimetypes", "my-workspace-hack", "rust-s3", "tokio", diff --git a/crates/_types/buckets/Cargo.toml b/crates/_types/buckets/Cargo.toml index 218cd6be37..75e4b6e2be 100644 --- a/crates/_types/buckets/Cargo.toml +++ b/crates/_types/buckets/Cargo.toml @@ -18,13 +18,6 @@ path = "src/lib.rs" crockford = { path = "../../lib/crockford" } errors = { path = "../../lib/errors" } -# probably need to have it's own module. -filesys = { path = "../../lib/files/filesys" } -hashing = { path = "../../lib/files/hashing" } -mimetypes = { path = "../../lib/files/mimetypes" } -# have to add this to it's own module..... -# buckets = { path = "../../_types/buckets" } -# cloud_storage = { path = "../../lib/api_clients/cloud_storage" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } diff --git a/crates/_types/buckets/src/public/mod.rs b/crates/_types/buckets/src/public/mod.rs index c357e13f4a..a0a03b9ad8 100644 --- a/crates/_types/buckets/src/public/mod.rs +++ b/crates/_types/buckets/src/public/mod.rs @@ -4,4 +4,3 @@ pub mod public_path; pub mod voice_conversion_results; pub mod weight_uploads; pub mod weight_files; -pub mod files; diff --git a/crates/cli/dev-database-seed/src/seeding/seed_media_files.rs b/crates/cli/dev-database-seed/src/seeding/seed_media_files.rs index 8036018f83..34956f17cd 100644 --- a/crates/cli/dev-database-seed/src/seeding/seed_media_files.rs +++ b/crates/cli/dev-database-seed/src/seeding/seed_media_files.rs @@ -5,8 +5,10 @@ use sqlx::{MySql, Pool}; use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use enums::by_table::media_files::media_file_type::MediaFileType; + use enums::common::visibility::Visibility; use errors::AnyhowResult; + use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; use filesys::path_to_string::path_to_string; diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index 2cd5d20198..150adaaf58 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -16,6 +16,12 @@ path = "src/lib.rs" # Internal buckets = { path = "../../../_types/buckets" } +crockford = { path = "../../../lib/crockford" } +errors = { path = "../../../lib/errors" } +easyenv = { path = "../../../lib/easyenv" } +filesys = { path = "../../../lib/files/filesys" } +hashing = { path = "../../../lib/files/hashing" } +mimetypes = { path = "../../../lib/files/mimetypes" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } diff --git a/crates/lib/api_clients/cloud_storage/src/bucket_client.rs b/crates/lib/api_clients/cloud_storage/src/bucket_client.rs index 677507c110..a1b1e28dd8 100644 --- a/crates/lib/api_clients/cloud_storage/src/bucket_client.rs +++ b/crates/lib/api_clients/cloud_storage/src/bucket_client.rs @@ -14,11 +14,26 @@ use tokio::io::AsyncReadExt; #[derive(Clone)] pub struct BucketClient { bucket: Bucket, - /// If set, put all files under this root path. optional_bucket_root: Option, } + +#[derive(Debug)] +pub enum BucketClientError { + ErrorWithCodeAndMessage { code: u16, message: String }, +} + +impl std::fmt::Display for BucketClientError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + BucketClientError::ErrorWithCodeAndMessage { code, message } => write!(f, "Error {}: {}", code, message), + } + } +} +impl std::error::Error for BucketClientError {} + + impl BucketClient { pub fn create( @@ -87,6 +102,25 @@ impl BucketClient { Ok(()) } + pub async fn upload_file_with_content_type_process(&self, object_name: &str, bytes: &[u8], content_type: &str) -> anyhow::AnyhowResult<()> { + info!("Filename for bucket: {}", object_name); + let object_name = self.get_rooted_object_name(object_name); + info!("Rooted filename for bucket: {}", object_name); + let response = self.bucket.put_object_with_content_type(&object_name, bytes, content_type).await?; + let body_bytes = response.bytes(); + let code = response.status_code(); + info!("upload code: {}", code); + if code != 200 { + let body = String::from_utf8_lossy(body_bytes); + warn!("upload body: {}", body); + BucketClientError::ErrorWithCodeAndMessage { code, message: body.to_string() }; + } else { + info!("upload success: {}", code); + Ok(()) + } + } + + #[deprecated = "Use upload_file instead above it returns an error we can surface and act on."] pub async fn upload_file_with_content_type(&self, object_name: &str, bytes: &[u8], content_type: &str) -> anyhow::Result<()> { info!("Filename for bucket: {}", object_name); diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs index b9b186a4d2..4f26dd6bf8 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs @@ -1,9 +1,8 @@ // Describes the file and how it will be saved to GCBucket // where the file should be stored based off the descriptor - use std::path::PathBuf; use crockford::crockford_entropy_lower; -use crate::util::hashed_directory_path_long_string; +use buckets::util::hashed_directory_path_long_string; // DEFAULT IMPLEMENTATION const REMOTE_FILE_DIRECTORY: &str = "/implement_google_cloud_storage_bucket_name"; @@ -31,76 +30,6 @@ pub trait FileDescriptor { } } -pub struct FileBucketDirectory { - file_object_hash: String, - cloud_directory_hash: String, - remote_cloud_base_directory: String, - full_remote_cloud_file_path: String, - file_name: String, - file_descriptor: Box -} - -impl FileBucketDirectory { - pub fn generate_new(file_descriptor: Box) -> Self { - Self::from_object_hash(file_descriptor) - } - - fn from_object_hash(file_descriptor: Box) -> Self { - let cloud_path_entropy = crockford_entropy_lower(32); - let file_name_entropy = crockford_entropy_lower(32); - - // gets you wiki /a/b/c/d folder structure - let middle = hashed_directory_path_long_string::hashed_directory_path_long_string(cloud_path_entropy.as_ref()); - - // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy - let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, cloud_path_entropy); - - // gets you name of the file with suffix and prefix and entropy in the centre - let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), file_name_entropy, file_descriptor.get_suffix()); - // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit. - - // This is the full path you upload to. - let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name); - Self { - file_object_hash: file_name_entropy, - cloud_directory_hash: cloud_path_entropy, - remote_cloud_base_directory: remote_cloud_base_directory, - full_remote_cloud_file_path: full_remote_cloud_file_path, - file_name:file_name, - file_descriptor:file_descriptor - } - } - - pub fn get_file_object_hash(&self) -> &str { - &self.file_object_hash - } - - pub fn get_cloud_directory_hash(&self) -> &str { - &self.cloud_directory_hash - } - - pub fn get_remote_cloud_base_directory(&self) -> &str { - &self.remote_cloud_base_directory - } - - pub fn get_full_remote_cloud_file_path(&self) -> &str { - &self.full_remote_cloud_file_path - } - - pub fn to_full_remote_cloud_file_path_pathbuf(&self) -> PathBuf { - PathBuf::from(&self.full_remote_cloud_file_path) - } - - pub fn get_file_name(&self) -> &str { - &self.file_name - } - - // pub fn get_file_descriptor(&self) -> Box { - // self.file_descriptor - // } - -} - #[cfg(test)] mod tests { @@ -108,8 +37,8 @@ mod tests { pub fn test() {} pub fn generate_new_entropy() { - let directory = FileDescriptorBucketDirectory::generate_new(); - assert_eq!(directory.get_object_hash().len(), 32); + // let directory = FileDescriptorBucketDirectory::generate_new(); + // assert_eq!(directory.get_object_hash().len(), 32); } // #[test] diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs new file mode 100644 index 0000000000..4bb30ac005 --- /dev/null +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs @@ -0,0 +1,66 @@ + +use std::path::PathBuf; +use crockford::crockford_entropy_lower; +use buckets::util::hashed_directory_path_long_string; +use crate::remote_file_manager::file_descriptor::FileDescriptor; + +pub struct FileBucketDirectory { + file_object_hash: String, + remote_cloud_base_directory: String, + full_remote_cloud_file_path: String, + file_name: String, + file_descriptor: Box + } + + impl FileBucketDirectory { + pub fn generate_new(file_descriptor: Box) -> Self { + Self::from_object_hash(file_descriptor) + } + + fn from_object_hash(file_descriptor: Box) -> Self { + let entropy = crockford_entropy_lower(32); + // gets you wiki /a/b/c/d folder structure + let middle = hashed_directory_path_long_string::hashed_directory_path_long_string(entropy.as_ref()); + // gets you cloud bucket path e.g weights/a/b/c/d/clould_path_entropy + let remote_cloud_base_directory = format!("{}/{}{}", file_descriptor.remote_directory_path(), middle, entropy); + // gets you name of the file with suffix and prefix and entropy in the centre + let file_name = format!("{}_{}.{}", file_descriptor.get_prefix(), entropy, file_descriptor.get_suffix()); + // note: no longer optional because it's easy to know what it would be in the db explicit is better than implcit. + + // This is the full path you upload to. + let full_remote_cloud_file_path = format!("{}/{}", remote_cloud_base_directory , file_name); + Self { + file_object_hash: entropy, + remote_cloud_base_directory: remote_cloud_base_directory, + full_remote_cloud_file_path: full_remote_cloud_file_path, + file_name:file_name, + file_descriptor:file_descriptor + } + } + + pub fn get_file_object_hash(&self) -> &str { + &self.file_object_hash + } + + pub fn get_remote_cloud_base_directory(&self) -> &str { + &self.remote_cloud_base_directory + } + + pub fn get_full_remote_cloud_file_path(&self) -> &str { + &self.full_remote_cloud_file_path + } + + pub fn to_full_remote_cloud_file_path_pathbuf(&self) -> PathBuf { + PathBuf::from(&self.full_remote_cloud_file_path) + } + + pub fn get_file_name(&self) -> &str { + &self.file_name + } + + // pub fn get_file_descriptor(&self) -> Box { + // self.file_descriptor + // } + + } + \ No newline at end of file diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs index d818e2bc52..4bb18414b2 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs @@ -1,7 +1,7 @@ pub mod remote_cloud_file_manager; - +pub mod file_directory; pub mod file_meta_data; - pub mod file_descriptor; + pub mod media_descriptor; pub mod weights_descriptor; \ No newline at end of file diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs index 992968af46..41f95bf1f2 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs @@ -1,49 +1,94 @@ -use crate::public::files::file_descriptor::FileDescriptor; -use crate::public::files::file_meta_data::FileMetaData; +use crate::bucket_client::BucketClient; +use super::file_descriptor::FileDescriptor; +use super::file_meta_data::FileMetaData; -use super::file_descriptor::FileBucketDirectory; +use std::time::Duration; +use errors::{AnyhowError, AnyhowResult}; -use errors::AnyhowError; use filesys::file_read_bytes::file_read_bytes; use filesys::file_size::file_size; use filesys::path_to_string::path_to_string; use hashing::sha256::sha256_hash_file::sha256_hash_file; - use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mimetypes::mimetype_for_file::get_mimetype_for_file; +use s3::bucket; +use log::info; -struct WebFileManager { - // bucket_client: BucketClients, +// Takes the definition from database and is an input to enable us to download the file. +struct RemoteCloudBucketDetails { + bucket_hash: String, + // Might require a migration to add this field to the database? + maybe_bucket_prefix: Option, + maybe_bucket_extension: Option, +} + +struct RemoteCloudFileClient { + bucket_client: BucketClient, file_descriptor: Box } -impl WebFileManager { +impl RemoteCloudFileClient { // bucket_client:BucketClients - fn new(file_descriptor: Box) -> WebFileManager { - WebFileManager { - // bucket_client: bucket_client, - file_descriptor: file_descriptor - } + fn bucket_client_for_object(&self) -> &BucketClient { + &self.bucket_client } - - // also include bucket details here - pub fn download_file(&self, system_file_path:String) -> Result<(),AnyhowError> { + fn new(file_descriptor: Box) -> AnyhowResult { + // Please verify that this is the correct way to get the bucket clients public and private in production and dev? + let access_key = easyenv::get_env_string_required("ACCESS_KEY")?; + let secret_key = easyenv::get_env_string_required("SECRET_KEY")?; + let region_name = easyenv::get_env_string_required("REGION_NAME")?; - } - // return error or success with meta data. - pub fn upload_file(&self, system_file_path:String) -> Result { - // let bucket_client = self.bucket_client.clone(); - if self.file_descriptor.is_public() { - - }else { - // use private bucket client + let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?; + let private_bucket_name = easyenv::get_env_string_required("PRIVATE_BUCKET_NAME")?; + // NB: Long timeout for dev rust builds to upload to cloud buckets. + // Unoptimized binaries sometimes take a lot of time to upload, presumably due to unoptimized code. + let bucket_timeout = easyenv::get_env_duration_seconds_or_default( + "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 10)); + let mut bucket_client:BucketClient; + if file_descriptor.is_public() { + // use public bucket client + info!("Configuring public GCS bucket..."); + bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + Some(bucket_timeout), + )?; + } else { + // use private bucket client + info!("Configuring private GCS bucket..."); + bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &private_bucket_name, + None, + Some(bucket_timeout), + )?; } - let result = Self::get_file_meta_data(system_file_path.clone())?; - Ok(result) + + Ok(RemoteCloudFileClient { + bucket_client: bucket_client, + file_descriptor: file_descriptor + }) } + // // also include bucket details here + // pub async fn download_file(&self, remote_cloud_bucket_details:RemoteCloudBucketDetails, to_system_file_path:String) -> AnyHowResult<> { + // self.bucket_client.download_file_to_disk(object_path, filesystem_path) + // } + + // // return error or success with meta data. + // pub async fn upload_file(&self, from_system_file_path:String) -> Result { + // // get file meta data + // let result = Self::get_file_meta_data(system_file_path.clone())?; + // Ok(result) + // } + // Retrieve the metadata from the file pub fn get_file_meta_data(system_file_path:String) -> Result { let file_size_bytes = file_size(system_file_path.clone())?; @@ -60,16 +105,10 @@ impl WebFileManager { } } - -// take in system file path to upload. -// take in system file path to download to. - -// should be able to take prefix suffix and entrpy to generate a file descriptor - #[cfg(test)] mod tests { //use crate::public::files::file_descriptor::{FileDescriptor}; - //use super::WebFileManager; + //use super::RemoteCloudFileClient; #[test] fn test_web_file_manager() { println!("test")