diff --git a/_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/down.sql b/_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/down.sql new file mode 100644 index 0000000000..c0e2b83487 --- /dev/null +++ b/_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/down.sql @@ -0,0 +1,13 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +ALTER TABLE generic_inference_jobs + MODIFY status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead' + ) NOT NULL DEFAULT 'pending'; diff --git a/_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/up.sql b/_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/up.sql new file mode 100644 index 0000000000..22074064a2 --- /dev/null +++ b/_sql/migrations/2023-10-24-044500_alter_generic_inference_jobs_add_status_values/up.sql @@ -0,0 +1,16 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- Add two states to allow us to distinguish user-cancelled jobs and system-cancelled jobs. +ALTER TABLE generic_inference_jobs + MODIFY status ENUM( + 'pending', + 'started', + 'complete_success', + 'complete_failure', + 'attempt_failed', + 'dead', + 'cancelled_by_user', + 'cancelled_by_system' +) NOT NULL DEFAULT 'pending'; diff --git a/_sql/migrations_squashed/generic_inference_jobs.sql b/_sql/migrations_squashed/generic_inference_jobs.sql index 304cf70dea..61535e1e36 100644 --- a/_sql/migrations_squashed/generic_inference_jobs.sql +++ b/_sql/migrations_squashed/generic_inference_jobs.sql @@ -145,9 +145,11 @@ CREATE TABLE generic_inference_jobs ( -- * Pending = job is ready to go -- * Started = job is running -- * Complete_Success = job is done (success) - -- * Complete_Failure = job is done (failure) + -- * Complete_Failure = job is done (permanent failure, eg. a Face Animator job without a face) -- * Attempt_Failed = job failed but may retry. - -- * Dead = job failed permanently. + -- * Dead = job failed permanently (ie. exhausted retries) + -- * Cancelled_By_User = the user canceled their own job + -- * Cancelled_By_System = the system terminated the job (ie. load shedding, user account deletion, etc.) -- -- Pending -> Started -> Complete_Success -- |-> Complete_Failure @@ -158,7 +160,10 @@ CREATE TABLE generic_inference_jobs ( 'complete_success', 'complete_failure', 'attempt_failed', - 'dead') NOT NULL DEFAULT 'pending', + 'dead', + 'cancelled_by_user', + 'cancelled_by_system' + ) NOT NULL DEFAULT 'pending', -- We can track this against a "max_attempt_count" attempt_count SMALLINT UNSIGNED NOT NULL DEFAULT 0, diff --git a/crates/_types/enums/src/common/job_status.rs b/crates/_types/enums/src/common/job_status.rs new file mode 100644 index 0000000000..8637c968ed --- /dev/null +++ b/crates/_types/enums/src/common/job_status.rs @@ -0,0 +1,147 @@ +use std::collections::BTreeSet; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// This is used in nearly every job system as an actual MySQL ENUM value: +/// +/// - tts_download_job +/// - tts_inference_job +/// - w2l_download_job +/// - w2l_inference_job +/// - generic_download_job +/// - (NOT generic_inference_job, which uses JobStatusPlus) +/// +/// See the documentation on the table for usage. +/// +/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. + +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "snake_case")] +#[sqlx(rename_all = "snake_case")] +pub enum JobStatus { + Pending, + Started, + CompleteSuccess, + CompleteFailure, + AttemptFailed, + Dead, +} + +// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(JobStatus); + +// NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders` + +impl JobStatus { + pub fn to_str(&self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Started => "started", + Self::CompleteSuccess => "complete_success", + Self::CompleteFailure => "complete_failure", + Self::AttemptFailed => "attempt_failed", + Self::Dead => "dead", + } + } + + pub fn from_str(job_status: &str) -> Result { + match job_status { + "pending" => Ok(Self::Pending), + "started" => Ok(Self::Started), + "complete_success" => Ok(Self::CompleteSuccess), + "complete_failure" => Ok(Self::CompleteFailure), + "attempt_failed" => Ok(Self::AttemptFailed), + "dead" => Ok(Self::Dead), + _ => Err(format!("invalid job_status: {:?}", job_status)), + } + } + + pub fn all_variants() -> BTreeSet { + // NB: BTreeSet is sorted + // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. + BTreeSet::from([ + Self::Pending, + Self::Started, + Self::CompleteSuccess, + Self::CompleteFailure, + Self::AttemptFailed, + Self::Dead, + ]) + } +} + +#[cfg(test)] +mod tests { + use crate::common::job_status::JobStatus; + use crate::test_helpers::assert_serialization; + + mod explicit_checks { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(JobStatus::Pending, "pending"); + assert_serialization(JobStatus::Started, "started"); + assert_serialization(JobStatus::CompleteSuccess, "complete_success"); + assert_serialization(JobStatus::CompleteFailure, "complete_failure"); + assert_serialization(JobStatus::AttemptFailed, "attempt_failed"); + assert_serialization(JobStatus::Dead, "dead"); + } + + #[test] + fn to_str() { + assert_eq!(JobStatus::Pending.to_str(), "pending"); + assert_eq!(JobStatus::Started.to_str(), "started"); + assert_eq!(JobStatus::CompleteSuccess.to_str(), "complete_success"); + assert_eq!(JobStatus::CompleteFailure.to_str(), "complete_failure"); + assert_eq!(JobStatus::AttemptFailed.to_str(), "attempt_failed"); + assert_eq!(JobStatus::Dead.to_str(), "dead"); + } + + #[test] + fn from_str() { + assert_eq!(JobStatus::from_str("pending").unwrap(), JobStatus::Pending); + assert_eq!(JobStatus::from_str("started").unwrap(), JobStatus::Started); + assert_eq!(JobStatus::from_str("complete_success").unwrap(), JobStatus::CompleteSuccess); + assert_eq!(JobStatus::from_str("complete_failure").unwrap(), JobStatus::CompleteFailure); + assert_eq!(JobStatus::from_str("attempt_failed").unwrap(), JobStatus::AttemptFailed); + assert_eq!(JobStatus::from_str("dead").unwrap(), JobStatus::Dead); + } + + #[test] + fn all_variants() { + let mut variants = JobStatus::all_variants(); + assert_eq!(variants.len(), 6); + assert_eq!(variants.pop_first(), Some(JobStatus::Pending)); + assert_eq!(variants.pop_first(), Some(JobStatus::Started)); + assert_eq!(variants.pop_first(), Some(JobStatus::CompleteSuccess)); + assert_eq!(variants.pop_first(), Some(JobStatus::CompleteFailure)); + assert_eq!(variants.pop_first(), Some(JobStatus::AttemptFailed)); + assert_eq!(variants.pop_first(), Some(JobStatus::Dead)); + assert_eq!(variants.pop_first(), None); + } + } + + mod mechanical_checks { + use super::*; + + #[test] + fn variant_length() { + use strum::IntoEnumIterator; + assert_eq!(JobStatus::all_variants().len(), JobStatus::iter().len()); + } + + #[test] + fn round_trip() { + for variant in JobStatus::all_variants() { + assert_eq!(variant, JobStatus::from_str(variant.to_str()).unwrap()); + assert_eq!(variant, JobStatus::from_str(&format!("{}", variant)).unwrap()); + assert_eq!(variant, JobStatus::from_str(&format!("{:?}", variant)).unwrap()); + } + } + } +} diff --git a/crates/_types/enums/src/common/job_status_plus.rs b/crates/_types/enums/src/common/job_status_plus.rs new file mode 100644 index 0000000000..9d55939b75 --- /dev/null +++ b/crates/_types/enums/src/common/job_status_plus.rs @@ -0,0 +1,160 @@ +use std::collections::BTreeSet; + +#[cfg(test)] +use strum::EnumCount; +#[cfg(test)] +use strum::EnumIter; + +/// This is used in newer jobs (that add additional enum states) +/// +/// - generic_inference_job +/// - (no other jobs yet) +/// +/// See the documentation on the table for usage. +/// +/// DO NOT CHANGE VALUES WITHOUT A MIGRATION STRATEGY. + +#[cfg_attr(test, derive(EnumIter, EnumCount))] +#[derive(Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd, Serialize, Deserialize, sqlx::Type)] +#[serde(rename_all = "snake_case")] +#[sqlx(rename_all = "snake_case")] +pub enum JobStatusPlus { + Pending, + Started, + CompleteSuccess, + CompleteFailure, + AttemptFailed, + Dead, + CancelledByUser, + CancelledBySystem, +} + +// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! +impl_enum_display_and_debug_using_to_str!(JobStatusPlus); + +// NB: We can derive `sqlx::Type` instead of using `impl_mysql_enum_coders` + +impl JobStatusPlus { + pub fn to_str(&self) -> &'static str { + match self { + Self::Pending => "pending", + Self::Started => "started", + Self::CompleteSuccess => "complete_success", + Self::CompleteFailure => "complete_failure", + Self::AttemptFailed => "attempt_failed", + Self::Dead => "dead", + Self::CancelledByUser => "cancelled_by_user", + Self::CancelledBySystem => "cancelled_by_system", + } + } + + pub fn from_str(job_status: &str) -> Result { + match job_status { + "pending" => Ok(Self::Pending), + "started" => Ok(Self::Started), + "complete_success" => Ok(Self::CompleteSuccess), + "complete_failure" => Ok(Self::CompleteFailure), + "attempt_failed" => Ok(Self::AttemptFailed), + "dead" => Ok(Self::Dead), + "cancelled_by_user" => Ok(Self::CancelledByUser), + "cancelled_by_system" => Ok(Self::CancelledBySystem), + _ => Err(format!("invalid job_status: {:?}", job_status)), + } + } + + pub fn all_variants() -> BTreeSet { + // NB: BTreeSet is sorted + // NB: BTreeSet::from() isn't const, but not worth using LazyStatic, etc. + BTreeSet::from([ + Self::Pending, + Self::Started, + Self::CompleteSuccess, + Self::CompleteFailure, + Self::AttemptFailed, + Self::Dead, + Self::CancelledByUser, + Self::CancelledBySystem, + ]) + } +} + +#[cfg(test)] +mod tests { + use crate::common::job_status_plus::JobStatusPlus; + use crate::test_helpers::assert_serialization; + + mod explicit_checks { + use super::*; + + #[test] + fn test_serialization() { + assert_serialization(JobStatusPlus::Pending, "pending"); + assert_serialization(JobStatusPlus::Started, "started"); + assert_serialization(JobStatusPlus::CompleteSuccess, "complete_success"); + assert_serialization(JobStatusPlus::CompleteFailure, "complete_failure"); + assert_serialization(JobStatusPlus::AttemptFailed, "attempt_failed"); + assert_serialization(JobStatusPlus::Dead, "dead"); + assert_serialization(JobStatusPlus::CancelledByUser, "cancelled_by_user"); + assert_serialization(JobStatusPlus::CancelledBySystem, "cancelled_by_system"); + } + + #[test] + fn to_str() { + assert_eq!(JobStatusPlus::Pending.to_str(), "pending"); + assert_eq!(JobStatusPlus::Started.to_str(), "started"); + assert_eq!(JobStatusPlus::CompleteSuccess.to_str(), "complete_success"); + assert_eq!(JobStatusPlus::CompleteFailure.to_str(), "complete_failure"); + assert_eq!(JobStatusPlus::AttemptFailed.to_str(), "attempt_failed"); + assert_eq!(JobStatusPlus::Dead.to_str(), "dead"); + assert_eq!(JobStatusPlus::CancelledByUser.to_str(), "cancelled_by_user"); + assert_eq!(JobStatusPlus::CancelledBySystem.to_str(), "cancelled_by_system"); + } + + #[test] + fn from_str() { + assert_eq!(JobStatusPlus::from_str("pending").unwrap(), JobStatusPlus::Pending); + assert_eq!(JobStatusPlus::from_str("started").unwrap(), JobStatusPlus::Started); + assert_eq!(JobStatusPlus::from_str("complete_success").unwrap(), JobStatusPlus::CompleteSuccess); + assert_eq!(JobStatusPlus::from_str("complete_failure").unwrap(), JobStatusPlus::CompleteFailure); + assert_eq!(JobStatusPlus::from_str("attempt_failed").unwrap(), JobStatusPlus::AttemptFailed); + assert_eq!(JobStatusPlus::from_str("dead").unwrap(), JobStatusPlus::Dead); + assert_eq!(JobStatusPlus::from_str("cancelled_by_user").unwrap(), JobStatusPlus::CancelledByUser); + assert_eq!(JobStatusPlus::from_str("cancelled_by_system").unwrap(), JobStatusPlus::CancelledBySystem); + } + + #[test] + fn all_variants() { + let mut variants = JobStatusPlus::all_variants(); + assert_eq!(variants.len(), 8); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::Pending)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::Started)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::CompleteSuccess)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::CompleteFailure)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::AttemptFailed)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::Dead)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::CancelledByUser)); + assert_eq!(variants.pop_first(), Some(JobStatusPlus::CancelledBySystem)); + assert_eq!(variants.pop_first(), None); + } + } + + mod mechanical_checks { + use super::*; + + #[test] + fn variant_length() { + use strum::IntoEnumIterator; + assert_eq!(JobStatusPlus::all_variants().len(), JobStatusPlus::iter().len()); + } + + #[test] + fn round_trip() { + for variant in JobStatusPlus::all_variants() { + // Test to_str(), from_str(), Display, and Debug. + assert_eq!(variant, JobStatusPlus::from_str(variant.to_str()).unwrap()); + assert_eq!(variant, JobStatusPlus::from_str(&format!("{}", variant)).unwrap()); + assert_eq!(variant, JobStatusPlus::from_str(&format!("{:?}", variant)).unwrap()); + } + } + } +} diff --git a/crates/_types/enums/src/common/mod.rs b/crates/_types/enums/src/common/mod.rs index 492371802d..c56c4971d0 100644 --- a/crates/_types/enums/src/common/mod.rs +++ b/crates/_types/enums/src/common/mod.rs @@ -1,3 +1,5 @@ +pub mod job_status; +pub mod job_status_plus; pub mod sqlite; pub mod visibility; pub mod vocoder_type; diff --git a/crates/lib/mysql_queries/src/column_types/job_status.rs b/crates/lib/mysql_queries/src/column_types/job_status.rs deleted file mode 100644 index 8db063d433..0000000000 --- a/crates/lib/mysql_queries/src/column_types/job_status.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! These are columns where users can control the visibility of their data. - -use anyhow::anyhow; - -use errors::AnyhowResult; - -/// To use this in a query, the query must have type annotations. -/// See: https://www.gitmemory.com/issue/launchbadge/sqlx/1241/847154375 -/// eg. job_status as `job_status: crate::column_types::job_status::JobStatus` -#[derive(Clone, Copy, PartialEq, Eq, Debug, Serialize, Deserialize, sqlx::Type)] -#[serde(rename_all = "snake_case")] -#[sqlx(rename_all = "snake_case")] -pub enum JobStatus { - Pending, - Started, - CompleteSuccess, - CompleteFailure, - AttemptFailed, - Dead, -} - -impl JobStatus { - pub fn to_str(&self) -> &'static str { - match self { - Self::Pending => "pending", - Self::Started => "started", - Self::CompleteSuccess => "complete_success", - Self::CompleteFailure => "complete_failure", - Self::AttemptFailed => "attempt_failed", - Self::Dead => "dead", - } - } - - pub fn from_str(job_status: &str) -> AnyhowResult { - match job_status { - "pending" => Ok(Self::Pending), - "started" => Ok(Self::Started), - "complete_success" => Ok(Self::CompleteSuccess), - "complete_failure" => Ok(Self::CompleteFailure), - "attempt_failed" => Ok(Self::AttemptFailed), - "dead" => Ok(Self::Dead), - _ => Err(anyhow!("invalid job_status: {:?}", job_status)), - } - } -} diff --git a/crates/lib/mysql_queries/src/column_types/mod.rs b/crates/lib/mysql_queries/src/column_types/mod.rs index 4a218e379e..6ee0a1832b 100644 --- a/crates/lib/mysql_queries/src/column_types/mod.rs +++ b/crates/lib/mysql_queries/src/column_types/mod.rs @@ -1,3 +1,2 @@ -pub mod job_status; pub mod twitch_event_category; pub mod vocoder_type; diff --git a/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs index f8fa4c275a..23d3c9df77 100644 --- a/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_download/job/list_available_generic_download_jobs.rs @@ -4,11 +4,11 @@ use chrono::Utc; use sqlx::MySqlPool; use enums::by_table::generic_download_jobs::generic_download_type::GenericDownloadType; +use enums::common::job_status::JobStatus; use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::generic_download_jobs::DownloadJobToken; -use crate::column_types::job_status::JobStatus; use crate::queries::generic_download::job::_keys::GenericDownloadJobId; /// table: generic_download_jobs diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index dd5a14c45c..f895de8265 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -6,11 +6,11 @@ use sqlx::MySqlPool; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType; +use enums::common::job_status_plus::JobStatusPlus; use enums::common::visibility::Visibility; use errors::AnyhowResult; use tokens::tokens::generic_inference_jobs::InferenceJobToken; -use crate::column_types::job_status::JobStatus; use crate::helpers::boolean_converters::i8_to_bool; use crate::payloads::generic_inference_args::generic_inference_args::GenericInferenceArgs; use crate::queries::generic_inference::job::_keys::GenericInferenceJobId; @@ -42,7 +42,7 @@ pub struct AvailableInferenceJob { pub creator_set_visibility: Visibility, // Job information - pub status: JobStatus, + pub status: JobStatusPlus, pub attempt_count: u16, pub priority_level: u16, @@ -128,7 +128,8 @@ pub async fn list_available_generic_inference_jobs( .map(|args| GenericInferenceArgs::from_json(args)) .transpose()?, maybe_raw_inference_text: record.maybe_raw_inference_text, - status: JobStatus::from_str(&record.status)?, + status: JobStatusPlus::from_str(&record.status) + .map_err(|err| anyhow!("JobStatus failure to parse: {:?}", err))?, attempt_count: record.attempt_count, priority_level: record.priority_level, is_keepalive_required: i8_to_bool(record.is_keepalive_required), diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_pending_and_grab_lock.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_pending_and_grab_lock.rs index b5553d79ac..2c3e379d9e 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_pending_and_grab_lock.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_pending_and_grab_lock.rs @@ -62,6 +62,8 @@ FOR UPDATE "complete_success" => false, // Job already complete "complete_failure" => false, // Job already complete (permanently dead; no need to retry) "dead" => false, // Job already complete (permanently dead; retries exhausted) + "cancelled_by_user" => false, // Job already complete (permanently dead; killed by user) + "cancelled_by_system" => false, // Job already complete (permanently dead; killed by system) _ => false, // Future-proof }; diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs index ebefdf0804..71a2786c0b 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/mod.rs @@ -1,4 +1,3 @@ pub mod get_inference_job_status; pub mod get_pending_inference_job_count; pub mod insert_generic_inference_job; -pub mod result_details; diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/result_details/mod.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/result_details/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/mysql_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs index b098d2b59f..bc2eaab9cf 100644 --- a/crates/lib/mysql_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/tts/tts_inference_jobs/list_available_tts_inference_jobs.rs @@ -6,10 +6,10 @@ use chrono::{DateTime, Utc}; use sqlx::MySqlPool; +use enums::common::job_status::JobStatus; use enums::common::visibility::Visibility; use errors::AnyhowResult; -use crate::column_types::job_status::JobStatus; use crate::helpers::boolean_converters::i8_to_bool; use crate::queries::tts::tts_inference_jobs::_keys::TtsInferenceJobId; @@ -83,7 +83,7 @@ SELECT is_for_twitch, is_debug_request, - status as `status: crate::column_types::job_status::JobStatus`, + status as `status: enums::common::job_status::JobStatus`, priority_level, attempt_count, failure_reason, @@ -132,7 +132,7 @@ SELECT is_for_twitch, is_debug_request, - status as `status: crate::column_types::job_status::JobStatus`, + status as `status: enums::common::job_status::JobStatus`, priority_level, attempt_count, failure_reason, @@ -224,7 +224,7 @@ SELECT is_for_twitch, is_debug_request, - status as `status: crate::column_types::job_status::JobStatus`, + status as `status: enums::common::job_status::JobStatus`, priority_level, attempt_count, failure_reason,