fix lots of unused imports

This commit is contained in:
Kasisnu committed 2024-10-23 13:41:03 +05:30
1 parent b6eb0e9394
commit 806f70d7ba
39 files changed
+28 -249

No files matched your search

@@ -4,7 +4,7 @@ use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::header::ContentType;
use actix_web::http::StatusCode;
use chrono::{DateTime, Duration, Utc};
use chrono::{DateTime, Utc};
use log::error;
use actix_helpers::response_serializers::error_to_json_http_response::error_to_json_http_response;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use filesys::path_to_string::path_to_string;
@@ -1,6 +1,5 @@
use std::fs::read_to_string;
use std::thread;
use std::time::{Duration, Instant};
use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
@@ -11,15 +10,11 @@ use crate::job::job_types::f5_tts::f5_tts_inference_command::InferenceArgs;
use crate::job::job_types::f5_tts::tts_inference::check_and_validate_job::check_and_validate_job;
use crate::job::job_types::vc::media_for_inference::MediaForInference;
use crate::state::job_dependencies::JobDependencies;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_truncate_args::FfmpegAudioTruncateArgs;
use crate::util::downloaders::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_type::MediaFileType;
use filesys::check_file_exists::check_file_exists;
use filesys::file_exists::file_exists;
use filesys::file_size::file_size;
@@ -29,7 +24,6 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_f5_tts::{insert_media_file_from_f5_tts, InsertF5TTSMediaFileArgs};
use mysql_queries::queries::media_files::get::get_media_file_for_inference::get_media_file_for_inference;
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::media_uploads::MediaUploadToken;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use filesys::path_to_string::path_to_string;
@@ -1,9 +1,7 @@
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::path::Path;
use anyhow::anyhow;
use log::{debug, error, info, warn};
use once_cell::sync::Lazy;
use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_file_path::WeightFileBucketPath;
@@ -1,6 +1,5 @@
use std::fs::read_to_string;
use std::thread;
use std::time::{Duration, Instant};
use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
@@ -8,7 +7,6 @@ use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_type::MediaFileType;
use filesys::check_file_exists::check_file_exists;
use filesys::file_exists::file_exists;
@@ -27,7 +25,6 @@ use crate::job::job_types::gpt_sovits::model_package::download_package::download
use crate::job::job_types::gpt_sovits::model_package::model_package::GptSovitsPackageFileType;
use crate::job::job_types::gpt_sovits::tts_inference::check_and_validate_job::check_and_validate_job;
use crate::state::job_dependencies::JobDependencies;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_truncate_args::FfmpegAudioTruncateArgs;
const BUCKET_FILE_PREFIX: &str = "fakeyou_";
@@ -1,5 +1,4 @@
use std::io::{BufReader, Cursor, Read, Seek};
use std::path::PathBuf;
use std::io::{BufReader, Cursor, Read};
use log::{error, info, warn};
use zip::ZipArchive;
@@ -18,11 +18,9 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer
use mysql_queries::queries::model_weights::create::create_weight;
use mysql_queries::queries::model_weights::create::create_weight::CreateModelWeightsArgs;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::gpt_sovits::model_package::model_package::SUFFIX;
use crate::job::job_types::gpt_sovits::upload_model::extract_and_upload_gpt_sovits_package::extract_and_upload_gpt_sovits_package_files;
use crate::job::job_types::gpt_sovits::upload_model::extract_gpt_sovits_payload_from_job::extract_gpt_sovits_payload_from_job;
use crate::state::job_dependencies::JobDependencies;
@@ -3,7 +3,6 @@ use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
use serde_json;
use bucket_paths::legacy::remote_file_manager_paths::media_descriptor::MediaImagePngDescriptor;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -271,7 +271,7 @@ pub async fn process_create_voice(
model_version: CURRENT_STYLE_TTS2_MODEL_VERSION,
model_encoding_type: ZsVoiceEncodingType::Encodec,
voice_title: &voice_name,
bucket_hash: embedding_bucket_location.get_object_hash().clone(),
bucket_hash: embedding_bucket_location.get_object_hash(),
maybe_creator_user_token: Some(&creator_user_token),
creator_ip_address: &creator_ip_address,
creator_set_visibility: Visibility::Public,
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -292,7 +292,7 @@ pub async fn process_create_voice(
model_version: CURRENT_VALLE_MODEL_VERSION,
model_encoding_type: ZsVoiceEncodingType::Encodec,
voice_title: &voice_name,
bucket_hash: embedding_bucket_location.get_object_hash().clone(),
bucket_hash: embedding_bucket_location.get_object_hash(),
maybe_creator_user_token: Some(&creator_user_token),
creator_ip_address: &creator_ip_address,
creator_set_visibility: Visibility::Public,
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -6,7 +6,6 @@ use anyhow::anyhow;
use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
@@ -20,8 +19,6 @@ use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
@@ -11,10 +11,9 @@ use tokens::tokens::media_uploads::MediaUploadToken;
use crate::job::job_loop::job_success_result::JobSuccessResult;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::vc::{rvc_v2, so_vits_svc};
use crate::job::job_types::vc::rvc_v2;
use crate::job::job_types::vc::media_for_inference::MediaForInference;
use crate::job::job_types::vc::rvc_v2::process_rvc_job::RvcV2ProcessJobArgs;
use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs;
use crate::state::job_dependencies::JobDependencies;
/// This is meant to take over from the `process_single_vc_job` function for a short bit
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use filesys::path_to_string::path_to_string;
@@ -1,6 +1,5 @@
use std::fs::read_to_string;
use std::thread;
use std::time::{Duration, Instant};
use std::time::Instant;
use anyhow::anyhow;
use log::{error, info, warn};
@@ -11,15 +10,11 @@ use crate::job::job_types::vc::media_for_inference::MediaForInference;
use crate::job::job_types::vc::seed_vc::seed_vc_inference_command::InferenceArgs;
use crate::job::job_types::vc::seed_vc::vc_inference::check_and_validate_job::check_and_validate_job;
use crate::state::job_dependencies::JobDependencies;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_truncate_args::FfmpegAudioTruncateArgs;
use crate::util::downloaders::maybe_download_file_from_bucket::{maybe_download_file_from_bucket, MaybeDownloadArgs};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::weight_files::bucket_directory::WeightFileBucketDirectory;
use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_type::MediaFileType;
use filesys::check_file_exists::check_file_exists;
use filesys::file_exists::file_exists;
use filesys::file_size::file_size;
@@ -29,7 +24,6 @@ use mysql_queries::queries::generic_inference::job::list_available_generic_infer
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_seed_vc::{insert_media_file_from_seed_vc, InsertSeedVcMediaFileArgs};
use mysql_queries::queries::media_files::get::get_media_file_for_inference::get_media_file_for_inference;
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::media_uploads::MediaUploadToken;
@@ -5,7 +5,6 @@ use anyhow::anyhow;
use log::{error, info, warn};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::typified_paths::public::media_uploads::bucket_file_path::MediaUploadOriginalFilePath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use filesys::check_file_exists::check_file_exists;
use filesys::create_dir_all_if_missing::create_dir_all_if_missing;
@@ -19,8 +18,6 @@ use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_voice_conversion::{insert_media_file_from_voice_conversion, InsertMediaFileArgs, VoiceConversionModelType};
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::MediaUploadRecordForInference;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::model_weights::ModelWeightToken;
use tokens::tokens::users::UserToken;
@@ -1,5 +1,3 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
@@ -7,7 +5,6 @@ use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -1,13 +1,9 @@
use std::collections::HashSet;
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::info;
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig, Redirection};
use errors::AnyhowResult;
@@ -1,7 +1,3 @@
use std::collections::{HashMap, HashSet};
use std::env;
use std::ffi::OsString;
use std::fs::File;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::Stdio;
@@ -10,20 +6,13 @@ use std::time::Duration;
use anyhow::anyhow;
// These environment vars are not copied over to the subprocess
use log::{debug, info, warn};
use once_cell::sync::Lazy;
// TODO/FIXME(bt, 2023-05-28): This is horrific security!
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::redis::Commands;
use r2d2_redis::RedisConnectionManager;
use subprocess::{Popen, PopenConfig};
use tokio::fs::OpenOptions;
use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader, Stdout};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::process::Command;
use tokio_util::codec::{FramedRead, FramedWrite, LinesCodec};
use crate::job::job_types::workflow::face_fusion::command_args::FaceFusionCommandArgs;
use crate::job::job_types::workflow::live_portrait::command_args::LivePortraitCommandArgs;
use crate::util::get_filtered_env_vars::{get_filtered_env_vars, get_filtered_env_vars_hashmap};
use crate::util::get_filtered_env_vars::get_filtered_env_vars_hashmap;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use errors::AnyhowResult;
use filesys::path_to_string::path_to_string;
@@ -1,73 +1,37 @@
use std::collections::HashMap;
use std::fs::{File, read_to_string};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::fs::read_to_string;
use std::path::Path;
use std::thread;
use std::time::{Duration, Instant};
use anyhow::{anyhow, Result};
use log::{debug, error, info, warn};
use serde_json::Value;
use walkdir::WalkDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::remote_file_manager_paths::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::prompts::prompt_type::PromptType;
use errors::AnyhowResult;
use filesys::check_file_exists::check_file_exists;
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
use filesys::file_deletion::safe_delete_file::safe_delete_file;
use filesys::file_deletion::safe_recursively_delete_files::safe_recursively_delete_files;
use filesys::file_exists::file_exists;
use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::common::watermark_type::WatermarkType;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Cu;
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::NewValue;
use mysql_queries::payloads::media_file_extra_info::inner_payloads::face_fusion_video_extra_info::FaceFusionVideoExtraInfo;
use mysql_queries::payloads::prompt_args::encoded_style_transfer_name::EncodedStyleTransferName;
use mysql_queries::payloads::prompt_args::prompt_inner_payload::{PromptInnerPayload, PromptInnerPayloadBuilder};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::create::insert_media_file_from_face_fusion::{insert_media_file_from_face_fusion, InsertFaceFusionArgs};
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_comfy_ui::{insert_media_file_from_comfy_ui, InsertArgs};
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use primitives::trim_or_empty::trim_or_empty;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::prompts::PromptToken;
use videos::ffprobe_get_info::ffprobe_get_info;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error;
use crate::job::job_types::workflow::comfy_ui_inference_command::{InferenceArgs, InferenceDetails};
use crate::job::job_types::workflow::face_fusion::categorize_face_fusion_error::categorize_face_fusion_error;
use crate::job::job_types::workflow::face_fusion::command_args::FaceFusionCommandArgs;
use crate::job::job_types::workflow::face_fusion::extract_face_fusion_payload_from_job::extract_face_fusion_payload_from_job;
use crate::job::job_types::workflow::face_fusion::face_fusion_title::face_fusion_title;
use crate::job::job_types::workflow::video_style_transfer::extract_vst_workflow_payload_from_job::extract_vst_workflow_payload_from_job;
use crate::job::job_types::workflow::video_style_transfer::steps::check_and_validate_job::check_and_validate_job;
use crate::job::job_types::workflow::video_style_transfer::steps::download_input_videos::{download_input_videos, DownloadInputVideoArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::post_process_add_watermark::{post_process_add_watermark, PostProcessAddWatermarkArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::post_process_restore_audio::{post_process_restore_audio, PostProcessRestoreVideoArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::preprocess_save_audio::{preprocess_save_audio, ProcessSaveAudioArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::preprocess_trim_and_resample_videos::{preprocess_trim_and_resample_videos, ProcessTrimAndResampleVideoArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::validate_and_save_results::{SaveResultsArgs, validate_and_save_results};
use crate::job::job_types::workflow::video_style_transfer::util::comfy_dirs::ComfyDirs;
use crate::job::job_types::workflow::video_style_transfer::util::video_pathing::{PrimaryInputVideoAndPaths, SecondaryInputVideoAndPaths, VideoPathing};
use crate::job::job_types::workflow::video_style_transfer::util::write_workflow_prompt::{WorkflowPromptArgs, write_workflow_prompt};
use crate::state::job_dependencies::JobDependencies;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
use crate::util::common_commands::ffmpeg::old::ffmpeg_logo_watermark_command::WatermarkArgs;
use crate::util::downloaders::download_media_file::{download_media_file, DownloadMediaFileArgs};
@@ -1,26 +1,16 @@
use std::collections::HashMap;
use std::fs::{File, read_to_string};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::fs::read_to_string;
use std::path::Path;
use std::thread;
use std::time::{Duration, Instant};
use anyhow::{anyhow, Result};
use log::{debug, error, info, warn};
use serde_json::Value;
use walkdir::WalkDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use bucket_paths::legacy::remote_file_manager_paths::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::prompts::prompt_type::PromptType;
use errors::AnyhowResult;
use filesys::check_file_exists::check_file_exists;
use filesys::file_exists::file_exists;
use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
@@ -29,45 +19,19 @@ use filesys::file_deletion::safe_recursively_delete_files::safe_recursively_dele
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::common::watermark_type::WatermarkType;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Cu;
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::NewValue;
use mysql_queries::payloads::media_file_extra_info::inner_payloads::live_portrait_video_extra_info::LivePortraitVideoExtraInfo;
use mysql_queries::payloads::prompt_args::encoded_style_transfer_name::EncodedStyleTransferName;
use mysql_queries::payloads::prompt_args::prompt_inner_payload::{PromptInnerPayload, PromptInnerPayloadBuilder};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_files::create::specialized_insert::insert_media_file_from_comfy_ui::{insert_media_file_from_comfy_ui, InsertArgs};
use mysql_queries::queries::media_files::create::insert_media_file_from_live_portrait::{insert_media_file_from_live_portrait, InsertLivePortraitArgs};
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token;
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use primitives::trim_or_empty::trim_or_empty;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, ThumbnailTaskInputMimeType};
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::prompts::PromptToken;
use videos::ffprobe_get_info::ffprobe_get_info;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::lipsync::sad_talker::categorize_error::categorize_error;
use crate::job::job_types::workflow::comfy_ui_inference_command::{InferenceArgs, InferenceDetails};
use crate::job::job_types::workflow::live_portrait::categorize_live_portrait_error::categorize_live_portrait_error;
use crate::job::job_types::workflow::live_portrait::command_args::LivePortraitCommandArgs;
use crate::job::job_types::workflow::live_portrait::extract_live_portrait_payload_from_job::extract_live_portrait_payload_from_job;
use crate::job::job_types::workflow::live_portrait::live_portrait_title::live_portrait_title;
use crate::job::job_types::workflow::video_style_transfer::extract_vst_workflow_payload_from_job::extract_vst_workflow_payload_from_job;
use crate::job::job_types::workflow::video_style_transfer::steps::check_and_validate_job::check_and_validate_job;
use crate::job::job_types::workflow::video_style_transfer::steps::download_input_videos::{download_input_videos, DownloadInputVideoArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::post_process_add_watermark::{post_process_add_watermark, PostProcessAddWatermarkArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::post_process_restore_audio::{post_process_restore_audio, PostProcessRestoreVideoArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::preprocess_save_audio::{preprocess_save_audio, ProcessSaveAudioArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::preprocess_trim_and_resample_videos::{preprocess_trim_and_resample_videos, ProcessTrimAndResampleVideoArgs};
use crate::job::job_types::workflow::video_style_transfer::steps::validate_and_save_results::{SaveResultsArgs, validate_and_save_results};
use crate::job::job_types::workflow::video_style_transfer::util::comfy_dirs::ComfyDirs;
use crate::job::job_types::workflow::video_style_transfer::util::video_pathing::{PrimaryInputVideoAndPaths, SecondaryInputVideoAndPaths, VideoPathing};
use crate::job::job_types::workflow::video_style_transfer::util::write_workflow_prompt::{WorkflowPromptArgs, write_workflow_prompt};
use crate::state::job_dependencies::JobDependencies;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
use crate::util::common_commands::ffmpeg::old::ffmpeg_logo_watermark_command::WatermarkArgs;
use crate::util::downloaders::download_media_file::{download_media_file, DownloadMediaFileArgs};
@@ -1,55 +1,26 @@
use std::collections::HashMap;
use std::fs::{read_to_string, File};
use std::fs::read_to_string;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::thread;
use std::time::{Duration, Instant};
use anyhow::{anyhow, Result};
use log::{debug, error, info, warn};
use log::{error, info, warn};
use oneshot::channel;
use r2d2_redis::redis::Commands;
use serde_json::Value;
use tokio::sync::mpsc;
use tokio::sync::oneshot;
use walkdir::WalkDir;
use bucket_paths::legacy::remote_file_manager_paths::remote_cloud_bucket_details::RemoteCloudBucketDetails;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_directory::MediaFileBucketDirectory;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::job_status_plus::JobStatusPlus;
use errors::AnyhowResult;
use filesys::check_file_exists::check_file_exists;
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
use filesys::file_deletion::safe_delete_file::safe_delete_file;
use filesys::file_deletion::safe_delete_files_and_directories::safe_delete_files_and_directories;
use filesys::file_deletion::safe_delete_possible_files_and_directories::safe_delete_possible_files_and_directories;
use filesys::file_deletion::safe_recursively_delete_files::safe_recursively_delete_files;
use filesys::file_exists::file_exists;
use filesys::file_read_bytes::file_read_bytes;
use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::PolymorphicInferenceArgs::Cu;
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::NewValue;
use mysql_queries::payloads::prompt_args::encoded_style_transfer_name::EncodedStyleTransferName;
use mysql_queries::payloads::prompt_args::prompt_inner_payload::{PromptInnerPayload, PromptInnerPayloadBuilder};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::generic_inference::web::get_inference_job_status::get_inference_job_status;
use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use mysql_queries::queries::model_weights::get::get_weight::{get_weight_by_token, get_weight_by_token_with_transactor};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use mysql_queries::queries::model_weights::get::get_weight::get_weight_by_token_with_transactor;
use mysql_queries::utils::transactor::Transactor;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use thumbnail_generator::task_client::thumbnail_task::ThumbnailTaskBuilder;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::prompts::PromptToken;
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
@@ -2,7 +2,6 @@ use std::path::{Path, PathBuf};
use anyhow::anyhow;
use log::{error, info};
use sqlx::MySqlPool;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
@@ -10,10 +9,8 @@ use filesys::path_to_string::path_to_string;
use mysql_queries::queries::media_files::get::get_media_file::{get_media_file_with_transactor, MediaFile};
use mysql_queries::utils::transactor::Transactor;
use tokens::tokens::media_files::MediaFileToken;
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::workflow::video_style_transfer::steps::check_and_validate_job::JobArgs;
const DEFAULT_SUFFIX : &str = ".jpg";
@@ -1,21 +1,17 @@
use std::path::{Path, PathBuf};
use anyhow::anyhow;
use log::{error, info};
use sqlx::{MySqlConnection, MySqlPool};
use sqlx::MySqlConnection;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use errors::AnyhowResult;
use filesys::path_to_string::path_to_string;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_transactor, MediaFilesByTokensRecord};
use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, get_media_file_with_transactor, MediaFile};
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens_with_transactor, MediaFilesByTokensRecord};
use mysql_queries::queries::media_files::get::get_media_file::get_media_file_with_transactor;
use mysql_queries::utils::transactor::Transactor;
use tokens::tokens::media_files::MediaFileToken;
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::workflow::comfy_ui_dependencies::ComfyDependencies;
use crate::job::job_types::workflow::video_style_transfer::steps::check_and_validate_job::JobArgs;
use crate::job::job_types::workflow::video_style_transfer::util::comfy_dirs::ComfyDirs;
use crate::job::job_types::workflow::video_style_transfer::util::video_pathing::{PrimaryInputVideoAndPaths, SecondaryInputVideoAndPaths, VideoPathing};
@@ -1,10 +1,8 @@
use log::{error, info};
use filesys::check_file_exists::check_file_exists;
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::WorkflowArgs;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::workflow::comfy_ui_dependencies::ComfyDependencies;
use crate::job::job_types::workflow::video_style_transfer::util::video_pathing::VideoPathing;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
@@ -1,14 +1,11 @@
use anyhow::anyhow;
use log::{error, info};
use filesys::check_file_exists::check_file_exists;
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::WorkflowArgs;
use subprocess_common::command_runner::command_runner_args::{RunAsSubprocessArgs, StreamRedirection};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::workflow::comfy_ui_dependencies::ComfyDependencies;
use crate::job::job_types::workflow::video_style_transfer::util::video_pathing::VideoPathing;
use crate::util::common_commands::ffmpeg::ffmpeg_audio_replace_args::FfmpegAudioReplaceArgs;
use crate::util::common_commands::ffmpeg::ffmpeg_extract_audio_args::FfmpegExtractAudioArgs;
pub struct ProcessSaveAudioArgs<'a> {
@@ -1,14 +1,11 @@
use std::path::{Path, PathBuf};
use std::path::PathBuf;
use std::process::{Command, Stdio};
use anyhow::anyhow;
use log::{error, info};
use sqlx::MySqlPool;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use filesys::path_to_string::path_to_string;
use mysql_queries::payloads::generic_inference_args::inner_payloads::workflow_payload::WorkflowArgs;
use tokens::tokens::media_files::MediaFileToken;
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
@@ -1,23 +1,15 @@
use std::path::Path;
use std::thread;
use std::time::Duration;
use anyhow::anyhow;
use log::{debug, error, info};
use tempdir::TempDir;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::no_table::style_transfer::style_transfer_name::StyleTransferName;
use errors::AnyhowResult;
use filesys::file_size::file_size;
use filesys::path_to_string::path_to_string;
use filesys::file_deletion::safe_delete_directory::safe_delete_directory;
use filesys::file_deletion::safe_delete_file::safe_delete_file;
use filesys::file_deletion::safe_recursively_delete_files::safe_recursively_delete_files;
use hashing::sha256::sha256_hash_file::sha256_hash_file;
use jobs_common::job_progress_reporter::job_progress_reporter::JobProgressReporter;
use mimetypes::mimetype_for_file::get_mimetype_for_file;
@@ -30,7 +22,6 @@ use thumbnail_generator::task_client::thumbnail_task::{ThumbnailTaskBuilder, Thu
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::prompts::PromptToken;
use crate::job::job_loop::job_success_result::{JobSuccessResult, ResultEntity};
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::workflow::comfy_ui_dependencies::ComfyDependencies;
use crate::job::job_types::workflow::video_style_transfer::steps::check_and_validate_job::JobArgs;
@@ -3,7 +3,6 @@ use cloud_storage::bucket_client::BucketClient;
use filesys::file_read_bytes::file_read_bytes;
use log::{debug, warn};
use std::path::PathBuf;
use std::time::Duration;
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinSet;
@@ -1,7 +1,7 @@
use chrono::Utc;
use log::{debug, info, warn};
use log::debug;
use once_cell::sync::Lazy;
use r2d2_redis::r2d2::{Pool, PooledConnection};
use r2d2_redis::r2d2::Pool;
use r2d2_redis::redis::Commands;
use r2d2_redis::RedisConnectionManager;
use std::cmp::PartialEq;
@@ -16,14 +16,10 @@ use bucket_paths::legacy::typified_paths::public::media_files::bucket_directory:
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::bucket_client::BucketClient;
use crockford::crockford_entropy_lower;
use filesys::file_read_bytes::file_read_bytes;
use jobs_common::redis_job_status_logger::RedisPool;
use redis_schema::keys::inference_job::style_transfer_progress_key::StyleTransferProgressKey;
use redis_schema::payloads::inference_job::style_transfer_progress_state::{InferenceProgressDetailsResponse, InferenceStageDetails};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokio::sync::oneshot::Sender;
use tokio::sync::{mpsc, oneshot};
use tokio::task::JoinSet;
use tokio::time::MissedTickBehavior;
static ALLOWED_TYPES_FRAMES : Lazy<HashSet<&'static str>> = Lazy::new(|| {
HashSet::from([
@@ -389,7 +385,7 @@ impl StageDirectoryState {
}
async fn upload_multiple_frames_from_disk(&self, bucket_client: &BucketClient, frames: Vec<(u32, PathBuf)>) -> (Vec<PreviewFrameUpdate>) {
async fn upload_multiple_frames_from_disk(&self, bucket_client: &BucketClient, frames: Vec<(u32, PathBuf)>) -> Vec<PreviewFrameUpdate> {
let mut bucket_upload_requests = Vec::with_capacity(frames.len());
let mut updates = Vec::with_capacity(frames.len());
@@ -753,7 +749,7 @@ impl PreviewProcessor {
}
}
async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &BucketClient, disk_path: PathBuf) -> (PreviewFrameUploadResult) {
async fn upload_frame_from_disk(&self, stage: PreviewStage, frame_number: u32, bucket_client: &BucketClient, disk_path: PathBuf) -> PreviewFrameUploadResult {
let stage_state = self.stages.get(&stage).unwrap();
if stage_state.state == PreviewStageState::UploadComplete {
log::debug!("Skipped re-uploading frame: {:?}", disk_path);
@@ -1,10 +1,8 @@
use std::collections::HashSet;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::anyhow;
use log::{info, warn};
use once_cell::sync::Lazy;
use subprocess::{Popen, PopenConfig};
use errors::AnyhowResult;
@@ -6,18 +6,11 @@ use sqlx::MySqlPool;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use cloud_storage::remote_file_manager::remote_cloud_file_manager::RemoteCloudFileClient;
use errors::AnyhowResult;
use filesys::path_to_string::path_to_string;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, MediaFilesByTokensRecord};
use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, MediaFile};
use tokens::tokens::media_files::MediaFileToken;
use videos::ffprobe_get_dimensions::ffprobe_get_dimensions;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::workflow::comfy_ui_dependencies::ComfyDependencies;
use crate::job::job_types::workflow::video_style_transfer::steps::check_and_validate_job::JobArgs;
use crate::job::job_types::workflow::video_style_transfer::util::comfy_dirs::ComfyDirs;
use crate::job::job_types::workflow::video_style_transfer::util::video_pathing::{PrimaryInputVideoAndPaths, SecondaryInputVideoAndPaths, VideoPathing};
pub struct DownloadMediaFileArgs<'a> {
pub mysql_pool: &'a MySqlPool,
@@ -44,7 +37,7 @@ pub async fn download_media_file(
info!("Querying media file by token: {:?} ...", &args.media_file_token);
let mut media_file = get_media_file(
let media_file = get_media_file(
&args.media_file_token,
args.can_see_deleted,
args.mysql_pool
@@ -305,7 +305,7 @@ pub async fn list_user_bookmarks_for_user_handler(
},
maybe_weight_data: match user_bookmark.entity_type {
UserBookmarkEntityType::ModelWeight =>
match (maybe_model_weight_cover) {
match maybe_model_weight_cover {
Some(cover) => Some(WeightsData {
// TODO(bt,2023-12-28): Proper default, optional, or "unknown" values would be better.
title: user_bookmark.maybe_entity_descriptive_text.clone().unwrap_or_else(|| "weight".to_string()),