From c116ad90d08e335cc2407f471c43e4299865f472 Mon Sep 17 00:00:00 2001 From: Kasisnu <5044120+kasisnu@users.noreply.github.com> Date: Tue, 19 Dec 2023 16:45:38 +0530 Subject: [PATCH] fail voice conversion jobs after model deleted --- .../query_vc_model_for_migration.rs | 38 ++++++++++++++++--- .../job_loop/determine_dependency_status.rs | 9 ++++- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/crates/lib/migration/src/voice_conversion/query_vc_model_for_migration.rs b/crates/lib/migration/src/voice_conversion/query_vc_model_for_migration.rs index ad30dc50f3..91028c5f26 100644 --- a/crates/lib/migration/src/voice_conversion/query_vc_model_for_migration.rs +++ b/crates/lib/migration/src/voice_conversion/query_vc_model_for_migration.rs @@ -6,28 +6,54 @@ use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath; use cloud_storage::bucket_path_unifier::BucketPathUnifier; use enums::by_table::model_weights::weights_types::WeightsType; use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType; -use errors::AnyhowResult; use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir; -use mysql_queries::queries::model_weights::inference::get_model_weight_for_voice_conversion_inference::{get_model_weight_for_voice_conversion_inference, ModelWeightForVoiceConversionInference}; -use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference}; +use mysql_queries::queries::model_weights::inference::get_model_weight_for_voice_conversion_inference::{get_model_weight_for_voice_conversion_inference, ModelWeightError, ModelWeightForVoiceConversionInference}; +use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference, VoiceConversionModelForInferenceError}; use tokens::tokens::model_weights::ModelWeightToken; +#[derive(Clone, Debug)] +pub enum VcModelError { + ModelDeleted, + DatabaseError { reason: String }, +} + +impl std::fmt::Display for VcModelError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + VcModelError::ModelDeleted => write!(f, "ModelDeleted"), + VcModelError::DatabaseError { reason} => + write!(f, "Database error: {:?}", reason), + } + } +} + +impl std::error::Error for VcModelError {} + /// Query model info /// Depending on the token prefix, this does either a modern or legacy lookup. -pub async fn query_vc_model_for_migration(model_token: &str, mysql_pool: &MySqlPool) -> AnyhowResult> { +pub async fn query_vc_model_for_migration(model_token: &str, mysql_pool: &MySqlPool) -> Result, VcModelError> { // NB: This is temporary migration code as we switch from the `voice_conversion_models` table to the `model_weights` table. if model_token.starts_with(ModelWeightToken::token_prefix()) { let model_weights_token = ModelWeightToken::new_from_str(model_token); let maybe_model = get_model_weight_for_voice_conversion_inference( - &mysql_pool, &model_weights_token).await?; + &mysql_pool, &model_weights_token).await.map_err(|err| + match err { + ModelWeightError::ModelDeleted => VcModelError::ModelDeleted, + ModelWeightError::DatabaseError { reason } => VcModelError::DatabaseError { reason }, + } + )?; Ok(maybe_model .map(|model| VcModel::ModelWeight(model))) } else { let maybe_vc_model = get_voice_conversion_model_for_inference( - &mysql_pool, model_token).await?; + &mysql_pool, model_token).await.map_err(|err| + match err { + VoiceConversionModelForInferenceError::ModelDeleted => VcModelError::ModelDeleted, + VoiceConversionModelForInferenceError::DatabaseError { reason } => VcModelError::DatabaseError { reason }, + })?; Ok(maybe_vc_model .map(|model| VcModel::LegacyVoiceConversion(model))) diff --git a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs index a6490adfe4..5af55d57fa 100644 --- a/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs +++ b/crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs @@ -6,9 +6,10 @@ use log::{info, warn}; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::tts_models::tts_model_type::TtsModelType; use filesys::file_exists::file_exists; -use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelType}; +use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelError, VcModelType}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord}; +use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInferenceError; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job_dependencies::JobDependencies; @@ -163,7 +164,11 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie let maybe_vc_model = query_vc_model_for_migration(token, &job_dependencies.db.mysql_pool) .await - .map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?; + .map_err(|err| + match err { + VcModelError::ModelDeleted => ProcessSingleJobError::ModelDeleted, + _ => ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)) + })?; match maybe_vc_model { None => MaybeInferenceModel::None,