Merge pull request #228 from storytold/dec-19-mithoo

fail voice conversion jobs after model deleted

crates/lib/migration/src/voice_conversion/query_vc_model_for_migration.rs

@@ -6,28 +6,54 @@ use buckets::public::weight_files::bucket_file_path::WeightFileBucketPath;
 use cloud_storage::bucket_path_unifier::BucketPathUnifier;
 use enums::by_table::model_weights::weights_types::WeightsType;
 use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
-use errors::AnyhowResult;
 use jobs_common::semi_persistent_cache_dir::SemiPersistentCacheDir;
-use mysql_queries::queries::model_weights::inference::get_model_weight_for_voice_conversion_inference::{get_model_weight_for_voice_conversion_inference, ModelWeightForVoiceConversionInference};
-use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference};
+use mysql_queries::queries::model_weights::inference::get_model_weight_for_voice_conversion_inference::{get_model_weight_for_voice_conversion_inference, ModelWeightError, ModelWeightForVoiceConversionInference};
+use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::{get_voice_conversion_model_for_inference, VoiceConversionModelForInference, VoiceConversionModelForInferenceError};
 use tokens::tokens::model_weights::ModelWeightToken;
 
+#[derive(Clone, Debug)]
+pub enum VcModelError {
+  ModelDeleted,
+  DatabaseError { reason: String },
+}
+
+impl std::fmt::Display for VcModelError {
+  fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+    match self {
+      VcModelError::ModelDeleted => write!(f, "ModelDeleted"),
+      VcModelError::DatabaseError { reason} =>
+        write!(f, "Database error: {:?}", reason),
+    }
+  }
+}
+
+impl std::error::Error for VcModelError {}
+
 /// Query model info
 /// Depending on the token prefix, this does either a modern or legacy lookup.
-pub async fn query_vc_model_for_migration(model_token: &str, mysql_pool: &MySqlPool) -> AnyhowResult<Option<VcModel>> {
+pub async fn query_vc_model_for_migration(model_token: &str, mysql_pool: &MySqlPool) -> Result<Option<VcModel>, VcModelError> {
   // NB: This is temporary migration code as we switch from the `voice_conversion_models` table to the `model_weights` table.
   if model_token.starts_with(ModelWeightToken::token_prefix()) {
     let model_weights_token = ModelWeightToken::new_from_str(model_token);
 
     let maybe_model = get_model_weight_for_voice_conversion_inference(
-      &mysql_pool, &model_weights_token).await?;
+      &mysql_pool, &model_weights_token).await.map_err(|err|
+      match err {
+        ModelWeightError::ModelDeleted =>  VcModelError::ModelDeleted,
+        ModelWeightError::DatabaseError { reason } => VcModelError::DatabaseError { reason },
+      }
+    )?;
 
     Ok(maybe_model
         .map(|model| VcModel::ModelWeight(model)))
 
   } else {
     let maybe_vc_model = get_voice_conversion_model_for_inference(
-      &mysql_pool, model_token).await?;
+      &mysql_pool, model_token).await.map_err(|err|
+      match err {
+        VoiceConversionModelForInferenceError::ModelDeleted =>  VcModelError::ModelDeleted,
+        VoiceConversionModelForInferenceError::DatabaseError { reason } => VcModelError::DatabaseError { reason },
+      })?;
 
     Ok(maybe_vc_model
         .map(|model| VcModel::LegacyVoiceConversion(model)))

crates/service/job/inference_job/src/job/job_loop/determine_dependency_status.rs

@@ -6,9 +6,10 @@ use log::{info, warn};
 use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
 use enums::by_table::tts_models::tts_model_type::TtsModelType;
 use filesys::file_exists::file_exists;
-use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelType};
+use migration::voice_conversion::query_vc_model_for_migration::{query_vc_model_for_migration, VcModel, VcModelError, VcModelType};
 use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
 use mysql_queries::queries::tts::tts_models::get_tts_model_for_inference_improved::{get_tts_model_for_inference_improved, TtsModelForInferenceError, TtsModelForInferenceRecord};
+use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::VoiceConversionModelForInferenceError;
 
 use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
 use crate::job_dependencies::JobDependencies;
@@ -163,7 +164,11 @@ async fn get_model_record_from_cacheable_query(job_dependencies: &JobDependencie
           let maybe_vc_model = query_vc_model_for_migration(token,
             &job_dependencies.db.mysql_pool)
               .await
-              .map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?;
+              .map_err(|err|
+                  match err {
+                    VcModelError::ModelDeleted => ProcessSingleJobError::ModelDeleted,
+                    _ => ProcessSingleJobError::Other(anyhow!("database error: {:?}", err))
+                  })?;
 
           match maybe_vc_model {
             None => MaybeInferenceModel::None,