fix lints

This commit is contained in:
Brandon Thomas committed 2024-01-17 20:34:13 -05:00
1 parent 8dc7c81a78
commit f6da4f3247
15 files changed
+74 -111

No files matched your search

Generated
+1
View File
@@ -6090,6 +6090,7 @@ dependencies = [
"sha2 0.9.9",
"speculoos",
"sqlx",
"strum",
"symphonia",
"time 0.3.30",
"tokens",
+1 -3
View File
@@ -7,10 +7,8 @@ use errors::AnyhowResult;
use crate::bucket_clients::get_bucket_clients;
use crate::cli_args::parse_cli_args;
use crate::seeding::model_weights::seed_weights; // V1
use crate::seeding::model_weights::seed_weights;
use crate::seeding::seed_media_files::seed_media_files;
use crate::seeding::seed_model_weights::seed_weights_files; // V2 v3 will come later on when everything is more conslidated and clear.
use crate::seeding::model_weights::test_seed_weights_files;
use crate::seeding::tts_tacotron2::seed_tts_tacotron2;
use crate::seeding::users::seed_user_accounts;
use crate::seeding::voice_conversion::seed_voice_conversion;
@@ -3,13 +3,10 @@ use std::vec;
use log::info;
use sqlx::{MySql, Pool};
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use cloud_storage::remote_file_manager::weights_descriptor;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use cloud_storage::remote_file_manager::weights_descriptor::{WeightsLoRADescriptor, WeightsSD15Descriptor};
use easyenv::from_filename;
use enums::by_table::model_weights::{
weights_category::WeightsCategory,
weights_types::WeightsType,
@@ -17,7 +14,6 @@ use enums::by_table::model_weights::{
use enums::common::visibility::Visibility;
use errors::{anyhow, AnyhowResult};
use mysql_queries::queries::model_weights::create::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::users::user::create_account::create_account;
use mysql_queries::queries::users::user::get_user_token_by_username::get_user_token_by_username;
use storyteller_root::get_seed_tool_data_root;
use tokens::tokens::model_weights::ModelWeightToken;
@@ -631,10 +627,6 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
public_bucket_hash: metadata1.bucket_details.clone().unwrap().object_hash,
maybe_public_bucket_prefix: Some(metadata1.bucket_details.clone().unwrap().prefix),
maybe_public_bucket_extension: Some(metadata1.bucket_details.clone().unwrap().suffix),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
maybe_cached_user_ratings_ratio: Some(0.9),
version: 1,
mysql_pool: &mysql_pool, // replace with actual MySqlPool
};
@@ -657,10 +649,6 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
public_bucket_hash: metadata2.bucket_details.clone().unwrap().object_hash.clone(),
maybe_public_bucket_prefix: Some(metadata2.bucket_details.clone().unwrap().prefix),
maybe_public_bucket_extension: Some(metadata2.bucket_details.clone().unwrap().suffix),
cached_user_ratings_total_count: 20,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 2,
maybe_cached_user_ratings_ratio: Some(0.9),
version: 2,
mysql_pool: &mysql_pool, // replace with actual MySqlPool
};
@@ -683,10 +671,6 @@ Noosphere by skumerz + dalcefoPainting + 饭特稀V08 by zhazhahui345 + GhostMix
public_bucket_hash: metadata3.bucket_details.clone().unwrap().object_hash,
maybe_public_bucket_prefix: Some(metadata3.bucket_details.clone().unwrap().prefix),
maybe_public_bucket_extension: Some(metadata3.bucket_details.clone().unwrap().suffix),
cached_user_ratings_total_count: 10,
cached_user_ratings_positive_count: 9,
cached_user_ratings_negative_count: 1,
maybe_cached_user_ratings_ratio: Some(0.9),
version: 1,
mysql_pool: &mysql_pool, // replace with actual MySqlPool
};
@@ -2,7 +2,8 @@ use crate::remote_file_manager::file_descriptor::FileDescriptor;
// TODO ASK BRANDON WHAT path should this be?
const MEDIA_FILE_DIRECTORY: &str = "/media";
pub struct MediaImagePngDescriptor {}
pub struct MediaImagePngDescriptor;
impl FileDescriptor for MediaImagePngDescriptor {
fn remote_directory_path(&self) -> &str {
MEDIA_FILE_DIRECTORY
@@ -68,8 +68,8 @@ impl RemoteCloudFileClient {
result.bucket_details = Some(RemoteCloudBucketDetails {
object_hash: directory.get_file_object_hash().to_string(),
suffix:suffix,
prefix:prefix,
suffix,
prefix,
});
println!("Uploading media file to bucket path: {:?}",directory.get_full_remote_cloud_file_path());
@@ -67,7 +67,6 @@ impl GoogleDriveDownloadCommand {
.unwrap_or("/tmp")
.to_string();
let temp_filename = random_crockford_token(10);
let temp_filename = format!("{}/{}", temp_dir_path, file_name);
info!("Downloading {} to: {}", download_url, temp_filename);
@@ -1,11 +1,11 @@
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use errors::AnyhowResult;
use crate::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs;
use crate::payloads::generic_inference_args::mocap_payload::MocapArgs;
use crate::payloads::generic_inference_args::tts_payload::TTSArgs;
use crate::payloads::generic_inference_args::videofilter_payload::RerenderArgs;
use crate::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
/// Used to encode extra state for the `generic_inference_jobs` table.
/// This should act somewhat like a serialized protobuf stored inside a record.
@@ -146,11 +146,12 @@ impl InferenceCategoryAbbreviated {
#[cfg(test)]
mod tests {
use tokens::tokens::{media_files::MediaFileToken, model_weights::ModelWeightToken};
use crate::payloads::generic_inference_args::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
use crate::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
use crate::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource, LipsyncArgs};
use crate::payloads::generic_inference_args::tts_payload::TTSArgs;
use crate::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
use tokens::tokens::{model_weights::ModelWeightToken, media_files::MediaFileToken};
#[test]
fn typical_lipsync_animation_args_serialize() {
@@ -218,7 +219,7 @@ mod tests {
args: Some(PolymorphicInferenceArgs::Ig(StableDiffusionArgs {
maybe_sd_model_token: Some(sd_model_token),
maybe_lora_model_token: Some(lora_model_token),
maybe_sampler: Some("sampler".as_ref()),
maybe_sampler: Some("sampler".to_string()),
maybe_height: Some(512),
maybe_width: Some(512),
maybe_cfg_scale: Some(7),
@@ -229,9 +230,9 @@ mod tests {
maybe_seed: Some(seed),
maybe_upload_path: Some(upload_path),
maybe_lora_upload_path: Some(lora_upload_path),
type_of_inference: type_of_inference,
description: Some("Option".as_ref()),
title: Some("Model Name".as_ref())
type_of_inference,
description: Some("Option".to_string()),
name: Some("Model Name".to_string())
})),
};
@@ -1,4 +1,4 @@
use tokens::tokens::{model_weights::ModelWeightToken, media_files::MediaFileToken};
use tokens::tokens::model_weights::ModelWeightToken;
/// Video sources can be one of several:
/// - F: media_files (todo)
@@ -1,6 +1,6 @@
use anyhow::anyhow;
use log::error;
use sqlx::{Executor, MySql, QueryBuilder, Transaction};
use sqlx::{MySql, QueryBuilder, Transaction};
use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
use errors::AnyhowResult;
@@ -61,11 +61,12 @@ pub async fn insert_batch_generation_records<'a, 'b>(args: InsertBatchArgs<'a, '
#[ignore]
#[cfg(test)]
mod tests {
use sqlx::mysql::MySqlPoolOptions;
use composite_identifiers::by_table::batch_generations::batch_generation_entity::BatchGenerationEntity;
use config::shared_constants::DEFAULT_MYSQL_CONNECTION_STRING;
use tokens::tokens::media_files::MediaFileToken;
use crate::queries::batch_generations::insert_batch_generation_records::{insert_batch_generation_records, InsertBatchArgs};
async fn setup() -> sqlx::Pool<sqlx::MySql> {
@@ -1,14 +1,14 @@
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use errors::anyhow;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use crate::job::job_loop::job_success_result::JobSuccessResult;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::image_generation::sd::process_job::process_job_selection;
use crate::job_dependencies::JobDependencies;
use crate::job::job_types::image_generation::sd::process_job::StableDiffusionProcessArgs;
use errors::anyhow;
use crate::job_dependencies::JobDependencies;
pub async fn process_single_ig_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
let job = job;
match job.maybe_model_type {
Some(InferenceModelType::StableDiffusion) => {
dispatch_sd_job(
@@ -1,46 +1,30 @@
use std::env::temp_dir;
use std::ops::Deref;
use std::path::PathBuf;
use std::thread;
use std::thread::Thread;
use std::time::{Duration, Instant};
use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use crate::job::job_loop::job_success_result::{ JobSuccessResult, ResultEntity };
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job_dependencies::JobDependencies;
use anyhow::anyhow;
use log::info;
use sqlx::MySqlPool;
use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use cloud_storage::remote_file_manager::media_descriptor::MediaImagePngDescriptor;
use cloud_storage::remote_file_manager::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use cloud_storage::remote_file_manager::weights_descriptor::WeightsLoRADescriptor;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType::UploadModel;
use enums::by_table::generic_synthetic_ids::id_category::IdCategory::MediaFile;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::media_uploads::media_upload_source::MediaUploadSource;
use enums::by_table::media_uploads::media_upload_type::MediaUploadType;
use enums::by_table::model_weights::weights_category::WeightsCategory;
use enums::by_table::model_weights::weights_types::WeightsType;
use enums::common::visibility::Visibility;
use filesys::path_to_string::path_to_string;
use filesys::safe_delete_temp_directory::safe_delete_temp_directory;
use mysql_queries::payloads::media_upload_modification_details::MediaUploadModificationDetails;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::create::insert_media_file_from_file_upload::{insert_media_file_from_file_upload, InsertMediaFileFromUploadArgs, UploadType};
use mysql_queries::queries::media_uploads::insert_media_upload::{Args, insert_media_upload};
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use mysql_queries::queries::model_weights::create::create_weight::{create_weight, CreateModelWeightsArgs};
use mysql_queries::queries::model_weights::get_weight::get_weight_by_token;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
use crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies;
use crate::job::job_types::image_generation::sd::validate_inputs::validate_inputs;
use google_drive_common::google_drive_download_command::GoogleDriveDownloadCommand;
use mysql_queries::queries::model_weights::create::create_weight::{create_weight, CreateModelWeightsArgs};
use crate::job::job_loop::job_success_result::JobSuccessResult;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::image_generation::sd::sd_inference_command::InferenceArgs;
use crate::job_dependencies::JobDependencies;
pub struct StableDiffusionProcessArgs<'a> {
pub job_dependencies: &'a JobDependencies,
@@ -82,7 +66,6 @@ pub async fn sd_args_from_job(
pub async fn process_job_selection (args: StableDiffusionProcessArgs<'_>
) -> Result<JobSuccessResult, ProcessSingleJobError> {
let job = args.job;
let sd_args = sd_args_from_job(&args).await?;
if sd_args.type_of_inference == "inference" {
process_job_inference(&args).await
@@ -98,7 +81,7 @@ pub async fn process_job_selection (args: StableDiffusionProcessArgs<'_>
}
}
pub async fn process_job_checkpoint(args: &StableDiffusionProcessArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
pub async fn process_job_checkpoint(_args: &StableDiffusionProcessArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
// insert model record in create model record
@@ -111,7 +94,7 @@ pub async fn process_job_checkpoint(args: &StableDiffusionProcessArgs<'_>) -> Re
inference_duration: Duration::from_secs(0),
})
}
pub async fn process_job_lora(args: &StableDiffusionProcessArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
pub async fn process_job_lora(_args: &StableDiffusionProcessArgs<'_>) -> Result<JobSuccessResult, ProcessSingleJobError> {
Ok(JobSuccessResult {
maybe_result_entity: None,
inference_duration: Duration::from_secs(0),
@@ -147,9 +130,10 @@ pub async fn process_job_inference(
}
let mut job_progress_reporter = args.job_dependencies.clients.job_progress_reporter
let _job_progress_reporter = args.job_dependencies.clients.job_progress_reporter
.new_generic_inference(job.inference_job_token.as_str())
.map_err(|e| ProcessSingleJobError::Other(anyhow!(e)))?;
//==================== TEMP DIR ==================== //
let work_temp_dir = format!("temp_stable_diffusion_inference_{}", job.id.0);
@@ -192,9 +176,10 @@ pub async fn process_job_inference(
}
}
if download_url.len() <= 0 {
if download_url.len() == 0 {
return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download URL Missing")));
}
let file_name = "download.safetensors";
let download_script = easyenv::get_env_string_or_default(
"DOWNLOAD_SCRIPT",
@@ -202,9 +187,10 @@ pub async fn process_job_inference(
let google_drive_downloader =
GoogleDriveDownloadCommand::new_production(&download_script);
info!("Downloading {}",download_url);
let download_filename = match google_drive_downloader.download_file_with_file_name(&download_url, &work_temp_dir,file_name).await {
let _download_filename = match google_drive_downloader.download_file_with_file_name(&download_url, &work_temp_dir,file_name).await {
Ok(filename) => filename,
Err(e) => {
Err(_e) => {
return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("Failed to Download")));
}
};
@@ -300,9 +286,7 @@ pub async fn process_job_inference(
).await?;
let vae_weight_record = match vae_weight_record {
Some(val) => {
val
},
Some(val) => val,
None => {
return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no VAE? thats a problem.")))
}
@@ -325,31 +309,26 @@ pub async fn process_job_inference(
).await?;
let prompt = match sd_args.maybe_prompt {
Some(val) => {
val
}
Some(val) => val,
None => return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("No Prompt provided!")))
};
let stderr_output_file = work_temp_dir.path().join("sd_err.txt");
let number_of_samples = match sd_args.maybe_number_of_samples {
Some(val) => {
val
},
None => {
20
}
Some(val) => val,
None => 20,
};
let inference_start_time = Instant::now();
sd_deps.inference_command.execute_inference(InferenceArgs {
work_dir: work_temp_dir.path().to_path_buf(),
output_file: output_path.clone(),
stderr_output_file: stderr_output_file,
prompt: prompt,
stderr_output_file,
prompt,
negative_prompt:sd_args.maybe_n_prompt.unwrap_or_default(),
number_of_samples: number_of_samples,
number_of_samples,
samplers: sd_args.maybe_sampler.unwrap_or(String::from("Euler a")),
width: sd_args.maybe_width.unwrap_or(512),
height: sd_args.maybe_height.unwrap_or(512),
@@ -387,7 +366,7 @@ pub async fn process_job_inference(
pool: mysql_pool,
maybe_creator_user_token: Some(&creator_user_token),
maybe_creator_anonymous_visitor_token: None,
creator_ip_address: creator_ip_address,
creator_ip_address,
creator_set_visibility: Default::default(),
upload_type: UploadType::Filesystem,
media_file_type: MediaFileType::Image,
@@ -424,7 +403,7 @@ pub async fn process_job_inference(
maybe_description_markdown: None,
maybe_description_rendered_html: None,
creator_user_token: Some(&creator_user_token),
creator_ip_address: creator_ip_address,
creator_ip_address,
creator_set_visibility: Default::default(),
maybe_last_update_user_token: None,
original_download_url: None,
@@ -450,14 +429,17 @@ pub async fn process_job_inference(
#[cfg(test)]
mod tests {
use anyhow::anyhow;
use std::path::PathBuf;
use cloud_storage::remote_file_manager::{
remote_cloud_file_manager::{RemoteCloudFileClient, self},
remote_cloud_bucket_details::RemoteCloudBucketDetails,
};
use errors::{AnyhowError, AnyhowResult};
#[tokio::test]
use std::path::PathBuf;
use anyhow::anyhow;
use cloud_storage::remote_file_manager::{
remote_cloud_bucket_details::RemoteCloudBucketDetails,
remote_cloud_file_manager::RemoteCloudFileClient,
};
use errors::AnyhowResult;
#[tokio::test]
async fn test_seed_weights_files() -> AnyhowResult<()> {
let seed_path = PathBuf::from("/storyteller/root/custom-seed-tool-data");
let remote_cloud_file_client = RemoteCloudFileClient::get_remote_cloud_file_client().await;
@@ -1,8 +1,9 @@
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use errors::AnyhowResult;
use sqlx::pool::maybe;
use crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies;
use crate::job::job_types::lipsync::sad_talker::sad_talker_dependencies::SadTalkerDependencies;
use crate::job::job_types::mocap::mocap_net::mocapnet_dependencies::MocapNetDependencies;
use crate::job::job_types::tts::styletts2::styletts2_dependencies::StyleTTS2Dependencies;
use crate::job::job_types::tts::tacotron2_v2_early_fakeyou::tacotron2_dependencies::Tacotron2Dependencies;
use crate::job::job_types::tts::vall_e_x::vall_e_x_dependencies::VallExDependencies;
@@ -10,9 +11,6 @@ use crate::job::job_types::tts::vits::vits_dependencies::VitsDependencies;
use crate::job::job_types::vc::rvc_v2::rvc_v2_dependencies::RvcV2Dependencies;
use crate::job::job_types::vc::so_vits_svc::svc_dependencies::SvcDependencies;
use crate::job::job_types::videofilter::rerender_a_video::rerender_dependencies::RerenderDependencies;
use crate::job::job_types::image_generation::sd::stable_diffusion_dependencies::StableDiffusionDependencies;
use crate::job::job_types::mocap::mocap_net::mocapnet_dependencies::MocapNetDependencies;
use crate::util::scoped_execution::ScopedExecution;
pub struct JobSpecificDependencies {
@@ -177,3 +177,5 @@ diesel = { version = "2.1.3", features = ["mysql"] }
[dev-dependencies]
speculoos = "0.9.0" # Fluent assertions
strum = { version = "0.25", features = ["derive"] }
@@ -8,7 +8,7 @@ use utoipa::ToSchema;
#[cfg_attr(test, derive(EnumIter, EnumCount))]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Deserialize, Serialize, ToSchema)]
#[derive(Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd, Debug, Deserialize, Serialize, ToSchema)]
pub enum EnqueueImageGenType {
#[serde(rename = "upload_lora")]
UploadLoRA,
@@ -2,6 +2,7 @@
#![forbid(unused_mut)]
#![forbid(unused_variables)]
use std::fmt::{Display, Formatter};
use std::fmt::Debug;
use std::sync::Arc;
@@ -11,8 +12,6 @@ use actix_web::http::StatusCode;
use log::warn;
use serde::Deserialize;
use serde::Serialize;
use tokens::tokens::model_weights::ModelWeightToken;
use utoipa::ToSchema;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
@@ -24,22 +23,19 @@ use mysql_queries::payloads::generic_inference_args::generic_inference_args::{
InferenceCategoryAbbreviated,
PolymorphicInferenceArgs,
};
use mysql_queries::payloads::generic_inference_args::image_generation_payload::StableDiffusionArgs;
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{
insert_generic_inference_job,
InsertGenericInferenceArgs,
};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
use std::fmt::{Display, Formatter};
/// Debug requests can get routed to special "debug-only" workers, which can
/// be used to trial new code, run debugging, etc.
const DEBUG_HEADER_NAME: &str = "enable-debug-mode";
@@ -344,7 +340,7 @@ pub async fn enqueue_image_generation_request(
maybe_seed: Some(seed),
maybe_upload_path: Some(upload_path),
maybe_lora_upload_path: Some(lora_upload_path),
type_of_inference: type_of_inference,
type_of_inference,
maybe_cfg_scale: Some(cfg_scale),
maybe_number_of_samples: Some(number_of_samples),
maybe_batch_count: Some(batch_count),