From cea2a3d0f11675e70b4872921239a1caa7719ea2 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 26 Sep 2024 20:32:01 -0400 Subject: [PATCH] start refactoring / consolidating bucket paths --- Cargo.lock | 3 +++ crates/cli/devenv/dev-database-seed/Cargo.toml | 1 + .../dev-database-seed/src/seeding/media_files.rs | 2 +- .../src/seeding/model_weights.rs | 6 +++--- crates/lib/api_clients/cloud_storage/Cargo.toml | 7 ++++--- .../src/remote_file_manager/file_directory.rs | 5 ++--- .../cloud_storage/src/remote_file_manager/mod.rs | 4 ---- .../remote_cloud_bucket_details.rs | 7 +++---- .../remote_cloud_file_manager.rs | 16 +++++++--------- crates/lib/bucket_paths/src/legacy/mod.rs | 1 + .../file_descriptor.rs | 0 .../media_descriptor.rs | 2 +- .../src/legacy/remote_file_manager_paths/mod.rs | 3 +++ .../weights_descriptor.rs | 8 ++++---- crates/service/job/inference_job/Cargo.toml | 1 + .../image_generation/sd/process_job_inference.rs | 2 +- .../sd/process_job_lora_upload.rs | 2 +- .../image_generation/sd/process_job_sd_upload.rs | 2 +- .../process_upload_workflow_job.rs | 2 +- 19 files changed, 38 insertions(+), 36 deletions(-) rename crates/lib/{api_clients/cloud_storage/src/remote_file_manager => bucket_paths/src/legacy/remote_file_manager_paths}/file_descriptor.rs (100%) rename crates/lib/{api_clients/cloud_storage/src/remote_file_manager => bucket_paths/src/legacy/remote_file_manager_paths}/media_descriptor.rs (93%) create mode 100644 crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/mod.rs rename crates/lib/{api_clients/cloud_storage/src/remote_file_manager => bucket_paths/src/legacy/remote_file_manager_paths}/weights_descriptor.rs (93%) diff --git a/Cargo.lock b/Cargo.lock index 66ae20a554..cb620197c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1345,6 +1345,7 @@ version = "0.0.1" dependencies = [ "anyhow", "async-trait", + "bucket_paths", "buckets", "crockford", "easyenv", @@ -2050,6 +2051,7 @@ dependencies = [ name = "dev-database-seed" version = "0.0.1" dependencies = [ + "bucket_paths", "buckets", "clap", "cloud_storage", @@ -3646,6 +3648,7 @@ dependencies = [ "anyhow", "async-trait", "bootstrap", + "bucket_paths", "buckets", "chrono", "cloud_storage", diff --git a/crates/cli/devenv/dev-database-seed/Cargo.toml b/crates/cli/devenv/dev-database-seed/Cargo.toml index d50d577a61..058a20e800 100644 --- a/crates/cli/devenv/dev-database-seed/Cargo.toml +++ b/crates/cli/devenv/dev-database-seed/Cargo.toml @@ -17,6 +17,7 @@ normal = ["my-workspace-hack"] [dependencies] # Internal +bucket_paths = { path = "../../../lib/bucket_paths" } buckets = { path = "../../../_types/buckets" } cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } config = { path = "../../../lib/deprecated/config" } diff --git a/crates/cli/devenv/dev-database-seed/src/seeding/media_files.rs b/crates/cli/devenv/dev-database-seed/src/seeding/media_files.rs index b62ad4774c..6ac85b024c 100644 --- a/crates/cli/devenv/dev-database-seed/src/seeding/media_files.rs +++ b/crates/cli/devenv/dev-database-seed/src/seeding/media_files.rs @@ -1,7 +1,7 @@ use log::info; use sqlx::{MySql, Pool}; -use cloud_storage::remote_file_manager::media_descriptor::MediaVideoMp4Descriptor; +use bucket_paths::legacy::remote_file_manager_paths::media_descriptor::MediaVideoMp4Descriptor; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; use enums::by_table::media_files::media_file_type::MediaFileType; use enums::common::visibility::Visibility; diff --git a/crates/cli/devenv/dev-database-seed/src/seeding/model_weights.rs b/crates/cli/devenv/dev-database-seed/src/seeding/model_weights.rs index 1870450bcf..1c983319cf 100644 --- a/crates/cli/devenv/dev-database-seed/src/seeding/model_weights.rs +++ b/crates/cli/devenv/dev-database-seed/src/seeding/model_weights.rs @@ -4,12 +4,12 @@ use std::vec; use log::info; use sqlx::{MySql, Pool}; +use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor, WeightsWorkflowDescriptor}; use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; -use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor, WeightsWorkflowDescriptor}; use enums::by_table::model_weights::{ - weights_category::WeightsCategory, - weights_types::WeightsType, + weights_category::WeightsCategory, + weights_types::WeightsType, }; use enums::common::visibility::Visibility; use errors::{anyhow, AnyhowResult}; diff --git a/crates/lib/api_clients/cloud_storage/Cargo.toml b/crates/lib/api_clients/cloud_storage/Cargo.toml index 67bd2c9262..56af04a7e2 100644 --- a/crates/lib/api_clients/cloud_storage/Cargo.toml +++ b/crates/lib/api_clients/cloud_storage/Cargo.toml @@ -16,18 +16,19 @@ path = "src/lib.rs" normal = ["my-workspace-hack"] [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # Internal +bucket_paths = { path = "../../../lib/bucket_paths" } buckets = { path = "../../../_types/buckets" } crockford = { path = "../../../lib/data/crockford" } -errors = { path = "../../../lib/errors" } easyenv = { path = "../../../lib/easyenv" } +errors = { path = "../../../lib/errors" } filesys = { path = "../../../lib/files/filesys" } hashing = { path = "../../../lib/files/hashing" } mimetypes = { path = "../../../lib/files/mimetypes" } storyteller_root = { path = "../../../lib/storyteller_root" } -# Workspace hack (faster compile times) -my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External anyhow = "1.0.86" diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs index 6f04e8a82e..928bb47009 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_directory.rs @@ -1,9 +1,8 @@ -use std::path::PathBuf; - +use bucket_paths::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor; use buckets::util::hashed_directory_path_long_string; use crockford::crockford_entropy_lower; +use std::path::PathBuf; -use crate::remote_file_manager::file_descriptor::FileDescriptor; use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails; pub struct FileBucketDirectory { diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs index 1bb4eac41f..69fa9c0188 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/mod.rs @@ -6,7 +6,3 @@ pub mod bucket_orchestration; pub mod file_directory; pub mod file_meta_data; -pub mod file_descriptor; - -pub mod media_descriptor; -pub mod weights_descriptor; diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_bucket_details.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_bucket_details.rs index c5a5a1f639..353bf4aac9 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_bucket_details.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_bucket_details.rs @@ -1,7 +1,6 @@ -use crate::remote_file_manager::media_descriptor; -use crate::remote_file_manager::weights_descriptor::{self}; - -use super::file_descriptor::FileDescriptor; +use bucket_paths::legacy::remote_file_manager_paths::media_descriptor; +use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::{self}; +use bucket_paths::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor; #[derive(Debug, Clone)] pub struct RemoteCloudBucketDetails { diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs index e96664ed2f..aabde4c195 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs +++ b/crates/lib/api_clients/cloud_storage/src/remote_file_manager/remote_cloud_file_manager.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; - +use bucket_paths::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor; use buckets::public::media_files::bucket_file_path::MediaFileBucketPath; use errors::AnyhowResult; use filesys::file_read_bytes::file_read_bytes; @@ -11,7 +11,6 @@ use crate::remote_file_manager::bucket_orchestration::{BucketOrchestration, Buck use crate::remote_file_manager::file_directory::FileBucketDirectory; use crate::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails; -use super::file_descriptor::FileDescriptor; use super::file_meta_data::FileMetaData; pub struct RemoteCloudFileClient { @@ -112,15 +111,14 @@ impl RemoteCloudFileClient { #[cfg(test)] mod tests { - use async_trait::async_trait; - use env_logger; + use async_trait::async_trait; + use env_logger; - use errors::AnyhowResult; + use crate::remote_file_manager::bucket_orchestration::BucketOrchestrationCore; + use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsLoRADescriptor; + use errors::AnyhowResult; - use crate::remote_file_manager::bucket_orchestration::BucketOrchestrationCore; - use crate::remote_file_manager::weights_descriptor::WeightsLoRADescriptor; - - struct BucketOrchestrationMock {} + struct BucketOrchestrationMock {} #[async_trait] impl BucketOrchestrationCore for BucketOrchestrationMock { diff --git a/crates/lib/bucket_paths/src/legacy/mod.rs b/crates/lib/bucket_paths/src/legacy/mod.rs index e69de29bb2..67d8bedca8 100644 --- a/crates/lib/bucket_paths/src/legacy/mod.rs +++ b/crates/lib/bucket_paths/src/legacy/mod.rs @@ -0,0 +1 @@ +pub mod remote_file_manager_paths; \ No newline at end of file diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/file_descriptor.rs similarity index 100% rename from crates/lib/api_clients/cloud_storage/src/remote_file_manager/file_descriptor.rs rename to crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/file_descriptor.rs diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/media_descriptor.rs b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/media_descriptor.rs similarity index 93% rename from crates/lib/api_clients/cloud_storage/src/remote_file_manager/media_descriptor.rs rename to crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/media_descriptor.rs index 614cf92bb2..d99720540c 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/media_descriptor.rs +++ b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/media_descriptor.rs @@ -1,4 +1,4 @@ -use crate::remote_file_manager::file_descriptor::FileDescriptor; +use crate::legacy::remote_file_manager_paths::file_descriptor::FileDescriptor; // TODO ASK BRANDON WHAT path should this be? const MEDIA_FILE_DIRECTORY: &str = "/media"; diff --git a/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/mod.rs b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/mod.rs new file mode 100644 index 0000000000..d5b66c42fb --- /dev/null +++ b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/mod.rs @@ -0,0 +1,3 @@ +pub mod file_descriptor; +pub mod media_descriptor; +pub mod weights_descriptor; \ No newline at end of file diff --git a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/weights_descriptor.rs b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/weights_descriptor.rs similarity index 93% rename from crates/lib/api_clients/cloud_storage/src/remote_file_manager/weights_descriptor.rs rename to crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/weights_descriptor.rs index 4d509aa55c..e92d204284 100644 --- a/crates/lib/api_clients/cloud_storage/src/remote_file_manager/weights_descriptor.rs +++ b/crates/lib/bucket_paths/src/legacy/remote_file_manager_paths/weights_descriptor.rs @@ -101,7 +101,7 @@ impl FileDescriptor for WeightsSDXLDescriptor { pub struct WeightsSVCDescriptor; -impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsSVCDescriptor { +impl FileDescriptor for WeightsSVCDescriptor { fn remote_directory_path(&self) -> &str { REMOTE_FILE_DIRECTORY } @@ -125,7 +125,7 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsS pub struct WeightsRVCDescriptor; -impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsRVCDescriptor { +impl FileDescriptor for WeightsRVCDescriptor { fn remote_directory_path(&self) -> &str { REMOTE_FILE_DIRECTORY } @@ -149,7 +149,7 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR pub struct WeightsRVCIndexDescriptor; -impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsRVCIndexDescriptor { +impl FileDescriptor for WeightsRVCIndexDescriptor { fn remote_directory_path(&self) -> &str { REMOTE_FILE_DIRECTORY } @@ -173,7 +173,7 @@ impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsR pub struct WeightsVallePromptDescriptor; -impl FileDescriptor for crate::remote_file_manager::weights_descriptor::WeightsVallePromptDescriptor { +impl FileDescriptor for WeightsVallePromptDescriptor { fn remote_directory_path(&self) -> &str { REMOTE_FILE_DIRECTORY } diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index cb2ffab5f0..db47e311cf 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -20,6 +20,7 @@ normal = ["my-workspace-hack"] # Internal actix_helpers = { path = "../../../lib/actix_helpers" } bootstrap = { path = "../../../lib/bootstrap" } +bucket_paths = { path = "../../../lib/bucket_paths" } buckets = { path = "../../../_types/buckets" } cloud_storage = { path = "../../../lib/api_clients/cloud_storage" } collections = { path = "../../../lib/data/collections" } diff --git a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_inference.rs b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_inference.rs index cda576fea7..0312afcefe 100644 --- a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_inference.rs +++ b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_inference.rs @@ -5,7 +5,7 @@ use anyhow::anyhow; use log::{error, info, warn}; use serde_json; -use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor; +use bucket_paths::legacy::remote_file_manager_paths::media_descriptor::MediaImagePngDescriptor; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; diff --git a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_lora_upload.rs b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_lora_upload.rs index 7ae7b56f84..415cd60bd4 100644 --- a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_lora_upload.rs +++ b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_lora_upload.rs @@ -5,8 +5,8 @@ use actix_web::dev::ResourcePath; use anyhow::anyhow; use log::{error, info, warn}; +use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsLoRADescriptor; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; -use cloud_storage::remote_file_manager::weights_descriptor::WeightsLoRADescriptor; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::by_table::model_weights::weights_types::WeightsType; diff --git a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_sd_upload.rs b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_sd_upload.rs index 869262f7ca..8da004bbfc 100644 --- a/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_sd_upload.rs +++ b/crates/service/job/inference_job/src/job/job_types/image_generation/sd/process_job_sd_upload.rs @@ -6,8 +6,8 @@ use actix_web::dev::ResourcePath; use anyhow::anyhow; use log::{error, info, warn}; +use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsSD15Descriptor; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; -use cloud_storage::remote_file_manager::weights_descriptor::WeightsSD15Descriptor; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::by_table::model_weights::weights_types::WeightsType; diff --git a/crates/service/job/inference_job/src/job/job_types/workflow/upload_workflow/process_upload_workflow_job.rs b/crates/service/job/inference_job/src/job/job_types/workflow/upload_workflow/process_upload_workflow_job.rs index c4faaf0563..78b450c5d0 100644 --- a/crates/service/job/inference_job/src/job/job_types/workflow/upload_workflow/process_upload_workflow_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/workflow/upload_workflow/process_upload_workflow_job.rs @@ -3,8 +3,8 @@ use std::time::Duration; use anyhow::{anyhow, Result}; use log::info; +use bucket_paths::legacy::remote_file_manager_paths::weights_descriptor::WeightsWorkflowDescriptor; use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient; -use cloud_storage::remote_file_manager::weights_descriptor::WeightsWorkflowDescriptor; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::by_table::model_weights::weights_types::WeightsType;