start refactoring / consolidating bucket paths

Cargo.lock

@@ -1345,6 +1345,7 @@ version = "0.0.1"
 dependencies = [
  "anyhow",
  "async-trait",
+ "bucket_paths",
  "buckets",
  "crockford",
  "easyenv",
@@ -2050,6 +2051,7 @@ dependencies = [
 name = "dev-database-seed"
 version = "0.0.1"
 dependencies = [
+ "bucket_paths",
  "buckets",
  "clap",
  "cloud_storage",
@@ -3646,6 +3648,7 @@ dependencies = [
  "anyhow",
  "async-trait",
  "bootstrap",
+ "bucket_paths",
  "buckets",
  "chrono",
  "cloud_storage",

crates/cli/devenv/dev-database-seed/Cargo.toml

@@ -17,6 +17,7 @@ normal = ["my-workspace-hack"]
 
 [dependencies]
 # Internal
+bucket_paths = { path = "../../../lib/bucket_paths" }
 buckets = { path = "../../../_types/buckets" }
 cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
 config = { path = "../../../lib/deprecated/config" }

crates/cli/devenv/dev-database-seed/src/seeding/media_files.rs

@@ -1,7 +1,7 @@
 use log::info;
 use sqlx::{MySql, Pool};
 
-use cloud_storage::remote_file_manager::media_descriptor::MediaVideoMp4Descriptor;
+use bucket_paths::legacy::remote_file_manager_paths::media_descriptor::MediaVideoMp4Descriptor;
 use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
 use enums::by_table::media_files::media_file_type::MediaFileType;
 use enums::common::visibility::Visibility;

crates/cli/devenv/dev-database-seed/src/seeding/model_weights.rs

@@ -4,12 +4,12 @@ use std::vec;
 use log::info;
 use sqlx::{MySql, Pool};
 
+use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor, WeightsWorkflowDescriptor};
 use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
 use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
-use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor, WeightsWorkflowDescriptor};
 use enums::by_table::model_weights::{
-  weights_category::WeightsCategory,
-  weights_types::WeightsType,
+    weights_category::WeightsCategory,
+    weights_types::WeightsType,
 };
 use enums::common::visibility::Visibility;
 use errors::{anyhow, AnyhowResult};

crates/lib/api_clients/cloud_storage/Cargo.toml

@@ -16,18 +16,19 @@ path = "src/lib.rs"
 normal = ["my-workspace-hack"]
 
 [dependencies]
+# Workspace hack (faster compile times)
+my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
 
 # Internal
+bucket_paths = { path = "../../../lib/bucket_paths" }
 buckets = { path = "../../../_types/buckets" }
 crockford = { path = "../../../lib/data/crockford" }
-errors = { path = "../../../lib/errors" }
 easyenv = { path = "../../../lib/easyenv" }
+errors = { path = "../../../lib/errors" }
 filesys = { path = "../../../lib/files/filesys" }
 hashing = { path = "../../../lib/files/hashing" }
 mimetypes = { path = "../../../lib/files/mimetypes" }
 storyteller_root = { path = "../../../lib/storyteller_root" }
-# Workspace hack (faster compile times)
-my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
 
 # External
 anyhow = "1.0.86"

crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs

@@ -1,9 +1,8 @@
-use std::path::PathBuf;
-
+use bucket_paths::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor;
 use buckets::util::hashed_directory_path_long_string;
 use crockford::crockford_entropy_lower;
+use std::path::PathBuf;
 
-use crate::remote_file_manager::file_descriptor::FileDescriptor;
 use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
 
 pub struct FileBucketDirectory {

crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs

@@ -6,7 +6,3 @@ pub mod bucket_orchestration;
 
 pub mod file_directory;
 pub mod file_meta_data;
-pub mod file_descriptor;
-
-pub mod media_descriptor;
-pub mod weights_descriptor;

crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_bucket_details.rs

@@ -1,7 +1,6 @@
-use crate::remote_file_manager::media_descriptor;
-use crate::remote_file_manager::weights_descriptor::{self};
-
-use super::file_descriptor::FileDescriptor;
+use bucket_paths::legacy::remote_file_manager_paths::media_descriptor;
+use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::{self};
+use bucket_paths::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor;
 
 #[derive(Debug, Clone)]
 pub struct RemoteCloudBucketDetails {

crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs

@@ -1,5 +1,5 @@
 use anyhow::anyhow;
-
+use bucket_paths::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor;
 use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
 use errors::AnyhowResult;
 use filesys::file_read_bytes::file_read_bytes;
@@ -11,7 +11,6 @@ use crate::remote_file_manager::bucket_orchestration::{BucketOrchestration, Buck
 use crate::remote_file_manager::file_directory::FileBucketDirectory;
 use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
 
-use super::file_descriptor::FileDescriptor;
 use super::file_meta_data::FileMetaData;
 
 pub struct RemoteCloudFileClient {
@@ -112,15 +111,14 @@ impl RemoteCloudFileClient {
 
 #[cfg(test)]
 mod tests {
-  use async_trait::async_trait;
-  use env_logger;
+    use async_trait::async_trait;
+    use env_logger;
 
-  use errors::AnyhowResult;
+    use crate::remote_file_manager::bucket_orchestration::BucketOrchestrationCore;
+    use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsLoRADescriptor;
+    use errors::AnyhowResult;
 
-  use crate::remote_file_manager::bucket_orchestration::BucketOrchestrationCore;
-  use crate::remote_file_manager::weights_descriptor::WeightsLoRADescriptor;
-
-  struct BucketOrchestrationMock {}
+    struct BucketOrchestrationMock {}
 
     #[async_trait]
     impl BucketOrchestrationCore for BucketOrchestrationMock {

crates/lib/bucket_paths/src/legacy/mod.rs

@@ -0,0 +1 @@
+pub mod remote_file_manager_paths;

crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/media_descriptor.rs

@@ -1,4 +1,4 @@
-use crate::remote_file_manager::file_descriptor::FileDescriptor;
+use crate::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor;
 
 // TODO ASK BRANDON WHAT path should this be?
 const MEDIA_FILE_DIRECTORY: &str = "/media";

crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/mod.rs

@@ -0,0 +1,3 @@
+pub mod file_descriptor;
+pub mod media_descriptor;
+pub mod weights_descriptor;

crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/weights_descriptor.rs

@@ -101,7 +101,7 @@ impl FileDescriptor for WeightsSDXLDescriptor {
 
 pub struct WeightsSVCDescriptor;
 
-impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsSVCDescriptor {
+impl FileDescriptor for WeightsSVCDescriptor {
     fn remote_directory_path(&self) -> &str {
         REMOTE_FILE_DIRECTORY
     }
@@ -125,7 +125,7 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsS
 
 pub struct WeightsRVCDescriptor;
 
-impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsRVCDescriptor {
+impl FileDescriptor for WeightsRVCDescriptor {
     fn remote_directory_path(&self) -> &str {
         REMOTE_FILE_DIRECTORY
     }
@@ -149,7 +149,7 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR
 
 pub struct WeightsRVCIndexDescriptor;
 
-impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsRVCIndexDescriptor {
+impl FileDescriptor for WeightsRVCIndexDescriptor {
     fn remote_directory_path(&self) -> &str {
         REMOTE_FILE_DIRECTORY
     }
@@ -173,7 +173,7 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR
 
 pub struct WeightsVallePromptDescriptor;
 
-impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsVallePromptDescriptor {
+impl FileDescriptor for WeightsVallePromptDescriptor {
     fn remote_directory_path(&self) -> &str {
         REMOTE_FILE_DIRECTORY
     }

crates/service/job/inference_job/Cargo.toml

@@ -20,6 +20,7 @@ normal = ["my-workspace-hack"]
 # Internal
 actix_helpers = { path = "../../../lib/actix_helpers" }
 bootstrap = { path = "../../../lib/bootstrap" }
+bucket_paths = { path = "../../../lib/bucket_paths" }
 buckets = { path = "../../../_types/buckets" }
 cloud_storage = { path = "../../../lib/api_clients/cloud_storage" }
 collections = { path = "../../../lib/data/collections" }

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_inference.rs

@@ -5,7 +5,7 @@ use anyhow::anyhow;
 use log::{error, info, warn};
 use serde_json;
 
-use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
+use bucket_paths::legacy::remote_file_manager_paths::media_descriptor::MediaImagePngDescriptor;
 use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
 use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_lora_upload.rs

@@ -5,8 +5,8 @@ use actix_web::dev::ResourcePath;
 use anyhow::anyhow;
 use log::{error, info, warn};
 
+use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsLoRADescriptor;
 use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
-use cloud_storage::remote_file_manager::weights_descriptor::WeightsLoRADescriptor;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
 use enums::by_table::model_weights::weights_category::WeightsCategory;
 use enums::by_table::model_weights::weights_types::WeightsType;

crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_sd_upload.rs

@@ -6,8 +6,8 @@ use actix_web::dev::ResourcePath;
 use anyhow::anyhow;
 use log::{error, info, warn};
 
+use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsSD15Descriptor;
 use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
-use cloud_storage::remote_file_manager::weights_descriptor::WeightsSD15Descriptor;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
 use enums::by_table::model_weights::weights_category::WeightsCategory;
 use enums::by_table::model_weights::weights_types::WeightsType;

crates/service/job/inference_job/src/job/job_types/workflow/upload_workflow/process_upload_workflow_job.rs

@@ -3,8 +3,8 @@ use std::time::Duration;
 use anyhow::{anyhow, Result};
 use log::info;
 
+use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsWorkflowDescriptor;
 use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
-use cloud_storage::remote_file_manager::weights_descriptor::WeightsWorkflowDescriptor;
 use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
 use enums::by_table::model_weights::weights_category::WeightsCategory;
 use enums::by_table::model_weights::weights_types::WeightsType;