refactor bucket modules

This commit is contained in:
Brandon Thomas committed 2023-12-12 02:55:23 -05:00
1 parent 795eb7a141
commit 07750fa122
38 files changed
+65 -71

No files matched your search

+2 -1
View File
@@ -1 +1,2 @@
pub mod private_path;
pub mod private_path;
pub mod zs_voices;
@@ -1,6 +1,6 @@
use crockford::crockford_entropy_lower;
use crate::private::private_path::PrivatePath;
use crate::private::private_path::PrivatePath;
use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
// TODO: Generate these from a macro.
@@ -79,7 +79,7 @@ impl ZeroShotVoiceEmbeddingBucketDirectory {
#[cfg(test)]
mod tests {
use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory};
use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory};
#[test]
pub fn generate_new_entropy() {
@@ -3,11 +3,10 @@ use std::path::PathBuf;
use crockford::crockford_entropy_lower;
use crate::private::private_path::PrivatePath;
use crate::public::zs_voices::directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory};
use crate::private::zs_voices::bucket_directory::{ModelCategory, ModelType, ZeroShotVoiceEmbeddingBucketDirectory};
// TODO: Generate these from a macro.
/// The original user upload file.
/// It may have derivative files (down samples, crops, etc.) that live alongside it.
#[derive(Clone)]
@@ -74,11 +73,9 @@ impl ZeroShotVoiceEmbeddingBucketPath {
mod tests {
use std::path::PathBuf;
use crate::public::media_files::original_file::MediaFileBucketPath;
mod with_prefix_and_extension {
use crate::public::zs_voices::directory::ModelCategory;
use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath};
use crate::private::zs_voices::bucket_directory::ModelCategory;
use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath};
#[test]
pub fn generate_new_entropy() {
@@ -97,8 +94,8 @@ mod tests {
}
mod without_prefix_and_extension {
use crate::public::zs_voices::directory::ModelCategory;
use crate::public::zs_voices::file::{ModelType, ZeroShotVoiceEmbeddingBucketPath};
use crate::private::zs_voices::bucket_directory::ModelCategory;
use crate::private::zs_voices::bucket_file_path::{ModelType, ZeroShotVoiceEmbeddingBucketPath};
use super::*;
@@ -135,7 +132,7 @@ mod tests {
#[test]
pub fn get_object_hash() {
let hash = "abcdefghijk";
let file = MediaFileBucketPath::from_object_hash(hash, None, None);
let file = ZeroShotVoiceEmbeddingBucketPath::from_object_hash(hash, ModelCategory::Tts, ModelType::VallEx, 1);
assert_eq!(file.get_object_hash(), hash);
}
}
@@ -0,0 +1,2 @@
pub mod bucket_directory;
pub mod bucket_file_path;
@@ -4,7 +4,6 @@ use crate::public::public_path::PublicPath;
use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
// TODO: Generate these from a macro.
// TODO: Use a central path registry for quick reference
const MEDIA_FILE_DIRECTORY: &str = "/media";
@@ -47,7 +46,7 @@ impl MediaFileBucketDirectory {
#[cfg(test)]
mod tests {
use crate::public::media_files::directory::MediaFileBucketDirectory;
use crate::public::media_files::bucket_directory::MediaFileBucketDirectory;
#[test]
pub fn generate_new_entropy() {
@@ -2,7 +2,7 @@ use std::path::PathBuf;
use crockford::crockford_entropy_lower;
use crate::public::media_files::directory::MediaFileBucketDirectory;
use crate::public::media_files::bucket_directory::MediaFileBucketDirectory;
use crate::public::public_path::PublicPath;
const ORIGINAL_FILE_BASENAME : &str = "file.bin";
@@ -79,7 +79,7 @@ impl MediaFileBucketPath {
mod tests {
use std::path::PathBuf;
use crate::public::media_files::original_file::MediaFileBucketPath;
use crate::public::media_files::bucket_file_path::MediaFileBucketPath;
mod with_prefix_and_extension {
use super::*;
@@ -1,2 +1,2 @@
pub mod directory;
pub mod original_file;
pub mod bucket_directory;
pub mod bucket_file_path;
@@ -47,7 +47,7 @@ impl MediaUploadDirectory {
#[cfg(test)]
mod tests {
use crate::public::media_uploads::directory::MediaUploadDirectory;
use crate::public::media_uploads::bucket_directory::MediaUploadDirectory;
#[test]
pub fn generate_new_entropy() {
@@ -2,7 +2,7 @@ use std::path::PathBuf;
use crockford::crockford_entropy_lower;
use crate::public::media_uploads::directory::MediaUploadDirectory;
use crate::public::media_uploads::bucket_directory::MediaUploadDirectory;
use crate::public::public_path::PublicPath;
const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin";
@@ -61,7 +61,7 @@ impl MediaUploadOriginalFilePath {
mod tests {
use std::path::PathBuf;
use crate::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use crate::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
#[test]
pub fn generate_new_entropy() {
@@ -1,2 +1,2 @@
pub mod directory;
pub mod original_file;
pub mod bucket_directory;
pub mod bucket_file_path;
-1
View File
@@ -2,5 +2,4 @@ pub mod media_files;
pub mod media_uploads;
pub mod public_path;
pub mod voice_conversion_results;
pub mod zs_voices;
pub mod weight_uploads;
@@ -40,7 +40,7 @@ impl VoiceConversionResultDirectory {
#[cfg(test)]
mod tests {
use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory;
use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory;
#[test]
pub fn get_directory_path_str() {
@@ -3,7 +3,7 @@ use std::path::PathBuf;
use crockford::crockford_entropy_lower;
use crate::public::public_path::PublicPath;
use crate::public::voice_conversion_results::directory::VoiceConversionResultDirectory;
use crate::public::voice_conversion_results::bucket_directory::VoiceConversionResultDirectory;
const ORIGINAL_FILE_PREFIX : &str = "fakeyou_";
const ORIGINAL_FILE_SUFFIX : &str = ".wav";
@@ -65,7 +65,7 @@ impl VoiceConversionResultOriginalFilePath {
mod tests {
use std::path::PathBuf;
use crate::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath;
use crate::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath;
#[test]
pub fn generate_new_entropy() {
@@ -1,2 +1,2 @@
pub mod directory;
pub mod original_file;
pub mod bucket_directory;
pub mod bucket_file_path;
@@ -48,7 +48,7 @@ impl WeightUploadDirectory {
#[cfg(test)]
mod tests {
use crate::public::weight_uploads::directory::WeightUploadDirectory;
use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory;
#[test]
pub fn generate_new_entropy() {
@@ -2,8 +2,8 @@ use std::path::PathBuf;
use crockford::crockford_entropy_lower;
use crate::public::weight_uploads::directory::WeightUploadDirectory;
use crate::public::public_path::PublicPath;
use crate::public::weight_uploads::bucket_directory::WeightUploadDirectory;
const ORIGINAL_FILE_BASENAME : &str = "original_upload.bin";
@@ -56,7 +56,8 @@ impl WeightUploadOriginalFilePath {
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use crate::public::weight_uploads::original_file::WeightUploadOriginalFilePath;
use crate::public::weight_uploads::bucket_file_path::WeightUploadOriginalFilePath;
#[test]
pub fn generate_new_entropy() {
@@ -1,2 +1,2 @@
pub mod directory;
pub mod original_file;
pub mod bucket_directory;
pub mod bucket_file_path;
@@ -1,2 +0,0 @@
pub mod directory;
pub mod file;
@@ -16,7 +16,7 @@ use tokens::tokens::users::UserToken;
use crate::seeding::users::HANASHI_USERNAME;
pub async fn seed_weights_for_paging( mysql_pool: &Pool<MySql>,user_token: UserToken) -> AnyhowResult<()> {
pub async fn seed_weights_for_paging(mysql_pool: &Pool<MySql>, user_token: UserToken) -> AnyhowResult<()> {
let sd1_5_markdown_description = r#"
# Dragonfruit AI Models Update and Workflow
@@ -81,9 +81,7 @@ pub async fn seed_weights_for_paging( mysql_pool: &Pool<MySql>,user_token: User
*Thank you for your continued support and feedback!*
"#;
let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg";
let user_token = user_token;
let sd1_5_image_token = "https://image.civitai.com/xG1nkqKTMzGDvpLrqFT7WA/be706282-2978-42a0-aaa2-73881aad94e9/width=1024/00049-2287632957-1girl,face,curly%20hair,red%20hair,white%20background,.jpeg";
// create a loop that loops from 1 to 100
for i in 1..=100 {
@@ -3,7 +3,7 @@ use std::path::Path;
use log::{info, warn};
use sqlx::{MySql, Pool};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
@@ -1,7 +1,7 @@
use log::info;
use sqlx::{MySql, Pool};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
+1 -1
View File
@@ -3,7 +3,7 @@ use std::time::Duration;
use log::info;
use sqlx::mysql::MySqlPoolOptions;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG};
use enums::by_table::media_files::media_file_type::MediaFileType;
@@ -2,7 +2,7 @@ use anyhow::anyhow;
use log::warn;
use sqlx::MySqlPool;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use enums::by_table::media_uploads::media_upload_source::MediaUploadSource;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
use enums::common::visibility::Visibility;
@@ -4,10 +4,9 @@ use sqlx::MySqlPool;
use enums::by_table::user_bookmarks::user_bookmark_entity_type::UserBookmarkEntityType;
use errors::AnyhowResult;
use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark;
use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord;
use tokens::tokens::user_bookmarks::UserBookmarkToken;
use crate::queries::user_bookmarks::list_user_bookmarks_result::RawUserBookmarkRecord;
use crate::queries::user_bookmarks::list_user_bookmarks_result::UserBookmark;
pub async fn list_user_bookmarks(
username: &str,
@@ -1,7 +1,7 @@
use anyhow::anyhow;
use sqlx::MySqlPool;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
@@ -5,9 +5,9 @@ use log::{error, warn};
use sqlx::MySqlPool;
use tempdir::TempDir;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationAudioSource;
@@ -5,8 +5,8 @@ use log::{error, warn};
use sqlx::MySqlPool;
use tempdir::TempDir;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use cloud_storage::bucket_client::BucketClient;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mysql_queries::payloads::generic_inference_args::lipsync_payload::LipsyncAnimationImageSource;
@@ -4,7 +4,7 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::file_size::file_size;
@@ -4,7 +4,7 @@ use anyhow::anyhow;
use log::info;
use tempdir::TempDir;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
use filesys::safe_delete_temp_file::safe_delete_temp_file;
@@ -5,9 +5,9 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::zs_voices::directory::{ModelCategory, ModelType};
use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath;
use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
@@ -5,9 +5,9 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::zs_voices::directory::{ModelCategory, ModelType};
use buckets::public::zs_voices::file::ZeroShotVoiceEmbeddingBucketPath;
use buckets::private::zs_voices::bucket_directory::{ModelCategory, ModelType};
use buckets::private::zs_voices::bucket_file_path::ZeroShotVoiceEmbeddingBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use cloud_storage::bucket_path_unifier::BucketPathUnifier;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
@@ -5,8 +5,8 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
@@ -4,8 +4,8 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
@@ -9,8 +9,8 @@ use chrono::{DateTime, Utc};
use log::error;
use r2d2_redis::redis::{Commands, RedisResult};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::voice_conversion_results::original_file::VoiceConversionResultOriginalFilePath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use buckets::public::voice_conversion_results::bucket_file_path::VoiceConversionResultOriginalFilePath;
use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::common::job_status_plus::JobStatusPlus;
@@ -8,7 +8,7 @@ use actix_web::web::Path;
use chrono::{DateTime, Utc};
use log::warn;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use mysql_queries::queries::media_files::get_media_file::get_media_file;
@@ -5,7 +5,7 @@ use actix_multipart::Multipart;
use actix_web::{HttpRequest, web};
use log::{error, info, warn};
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use buckets::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use enums::by_table::media_uploads::media_upload_source::MediaUploadSource;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
use enums::common::visibility::Visibility;
@@ -8,7 +8,7 @@ use actix_web::web::Path;
use chrono::{DateTime, Utc};
use log::{error, log};
use r2d2_redis::redis::{Commands, RedisResult};
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status;
use mysql_queries::queries::tts::tts_inference_jobs::get_tts_inference_job_status::get_tts_inference_job_status;
@@ -6,7 +6,7 @@ use actix_web::{HttpRequest, HttpResponse, ResponseError, web};
use hyper::StatusCode;
use log::{error, info, warn};
use once_cell::sync::Lazy;
use buckets::public::media_files::original_file::MediaFileBucketPath;
use buckets::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType::MediaUpload;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;