diff --git a/crates/lib/mysql_queries/src/queries/voice_conversion/results/get_voice_conversion_result.rs b/crates/lib/mysql_queries/src/queries/voice_conversion/results/get_voice_conversion_result.rs new file mode 100644 index 0000000000..a50624700e --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/voice_conversion/results/get_voice_conversion_result.rs @@ -0,0 +1,317 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use crate::column_types::vocoder_type::VocoderType; +use crate::helpers::boolean_converters::{nullable_i8_to_bool, i8_to_bool}; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use log::warn; +use sqlx::MySqlPool; +use tokens::tokens::voice_conversion_results::VoiceConversionResultToken; +use tokens::users::user::UserToken; +use tokens::voice_conversion::model::VoiceConversionModelToken; + +pub struct VoiceConversionResult { + pub voice_conversion_result_token: VoiceConversionResultToken, + + pub voice_conversion_model_token: VoiceConversionModelToken, + pub voice_conversion_model_title: Option, // TODO: Shouldn't be Option. + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_model_creator_user_token: Option, + pub maybe_model_creator_username: Option, + pub maybe_model_creator_display_name: Option, + pub maybe_model_creator_gravatar_hash: Option, + + pub public_bucket_hash: String, + + pub creator_set_visibility: Visibility, + + // Worker hostname that generated the audio. Has the value "unknown" if null. + pub generated_by_worker: String, + + // If the request was originally targeted to a special "debug" worker. + pub is_debug_request: bool, + + pub file_size_bytes: u32, + pub duration_millis: u32, + + //pub model_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + pub maybe_moderator_fields: Option, +} + +/// "Moderator-only fields" that we wouldn't want to expose to ordinary users. +/// It's the web endpoint controller's responsibility to clear these for non-mods. +#[derive(Serialize)] +pub struct VoiceConversionResultModeratorFields { + pub model_creator_is_banned: bool, + pub result_creator_is_banned_if_user: bool, + pub result_creator_ip_address: String, + pub result_creator_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +#[derive(Serialize)] +pub struct VoiceConversionResultRecordRaw { + pub voice_conversion_result_token: VoiceConversionResultToken, + + pub voice_conversion_model_token: VoiceConversionModelToken, + pub voice_conversion_model_title: Option, + + pub maybe_creator_is_banned: Option, + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub maybe_model_creator_is_banned: Option, + pub maybe_model_creator_user_token: Option, + pub maybe_model_creator_username: Option, + pub maybe_model_creator_display_name: Option, + pub maybe_model_creator_gravatar_hash: Option, + + pub public_bucket_hash: String, + + pub creator_set_visibility: String, + + pub generated_by_worker: Option, + + pub is_debug_request: i8, + + pub file_size_bytes: i32, + pub duration_millis: i32, + + //pub model_is_mod_approved: i8, // needs convert + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + // Moderator fields + pub creator_ip_address: String, + pub user_deleted_at: Option>, + pub mod_deleted_at: Option>, +} + +pub async fn get_voice_conversion_result( + voice_conversion_result_token: &str, + can_see_deleted: bool, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + + let maybe_record = if can_see_deleted { + select_including_deleted(tts_result_token, mysql_pool).await + } else { + select_without_deleted(tts_result_token, mysql_pool).await + }; + + let ir : VoiceConversionResultRecordRaw = match maybe_record { + Ok(inference_result) => inference_result, + Err(ref err) => { + match err { + sqlx::Error::RowNotFound => { + warn!("tts result not found: {:?}", &err); + return Ok(None); + }, + _ => { + warn!("tts result query error: {:?}", &err); + return Err(anyhow!("database error")); + } + } + } + }; + + let mut pretrained_vocoder = None; + if let Some(vocoder) = ir.maybe_pretrained_vocoder_used.as_deref() { + pretrained_vocoder = Some(VocoderType::from_str(vocoder)?); + } + + let ir_for_response = VoiceConversionResult { + tts_result_token: ir.tts_result_token, + + raw_inference_text: ir.raw_inference_text, + + tts_model_token: ir.tts_model_token, + tts_model_title: ir.tts_model_title, + + maybe_creator_user_token: ir.maybe_creator_user_token, + maybe_creator_username: ir.maybe_creator_username, + maybe_creator_display_name: ir.maybe_creator_display_name, + maybe_creator_gravatar_hash: ir.maybe_creator_gravatar_hash, + + maybe_model_creator_user_token: ir.maybe_model_creator_user_token, + maybe_model_creator_username: ir.maybe_model_creator_username, + maybe_model_creator_display_name: ir.maybe_model_creator_display_name, + maybe_model_creator_gravatar_hash: ir.maybe_model_creator_gravatar_hash, + + //model_is_mod_approved: if ir.model_is_mod_approved == 0 { false } else { true }, + + public_bucket_wav_audio_path: ir.public_bucket_wav_audio_path, + public_bucket_spectrogram_path: ir.public_bucket_spectrogram_path, + + // NB: Fail open/public since we're already looking at it + creator_set_visibility: Visibility::from_str(&ir.creator_set_visibility) + .unwrap_or(Visibility::Public), + + generated_by_worker: ir.generated_by_worker.unwrap_or("unknown".to_string()), + + is_debug_request: i8_to_bool(ir.is_debug_request), + + file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, + duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + + created_at: ir.created_at, + updated_at: ir.updated_at, + + maybe_moderator_fields: Some(VoiceConversionResultModeratorFields { + model_creator_is_banned: + nullable_i8_to_bool(ir.maybe_model_creator_is_banned, false), + result_creator_is_banned_if_user: + nullable_i8_to_bool(ir.maybe_creator_is_banned, false), + result_creator_ip_address: ir.creator_ip_address, + result_creator_deleted_at: ir.user_deleted_at, + mod_deleted_at: ir.mod_deleted_at, + }), + }; + + Ok(Some(ir_for_response)) +} + +async fn select_including_deleted( + tts_result_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + VoiceConversionResultRecordRaw, + r#" +SELECT + tts_results.token as tts_result_token, + + tts_results.raw_inference_text, + + tts_results.model_token as tts_model_token, + tts_models.title as tts_model_title, + + tts_results.maybe_pretrained_vocoder_used, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + users.is_banned as maybe_creator_is_banned, + + model_users.token as maybe_model_creator_user_token, + model_users.username as maybe_model_creator_username, + model_users.display_name as maybe_model_creator_display_name, + model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash, + model_users.is_banned as maybe_model_creator_is_banned, + + tts_results.public_bucket_wav_audio_path, + tts_results.public_bucket_spectrogram_path, + + tts_results.creator_set_visibility, + + tts_results.generated_by_worker, + tts_results.is_debug_request, + + tts_results.file_size_bytes, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at, + + tts_results.creator_ip_address, + tts_results.user_deleted_at, + tts_results.mod_deleted_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as model_users + ON tts_models.creator_user_token = model_users.token +WHERE + tts_results.token = ? + "#, + tts_result_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} + +async fn select_without_deleted( + tts_result_token: &str, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + VoiceConversionResultRecordRaw, + r#" +SELECT + tts_results.token as tts_result_token, + + tts_results.raw_inference_text, + + tts_results.model_token as tts_model_token, + tts_models.title as tts_model_title, + + tts_results.maybe_pretrained_vocoder_used, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + users.is_banned as maybe_creator_is_banned, + + model_users.token as maybe_model_creator_user_token, + model_users.username as maybe_model_creator_username, + model_users.display_name as maybe_model_creator_display_name, + model_users.email_gravatar_hash as maybe_model_creator_gravatar_hash, + model_users.is_banned as maybe_model_creator_is_banned, + + tts_results.public_bucket_wav_audio_path, + tts_results.public_bucket_spectrogram_path, + + tts_results.creator_set_visibility, + + tts_results.generated_by_worker, + tts_results.is_debug_request, + + tts_results.file_size_bytes, + tts_results.duration_millis, + tts_results.created_at, + tts_results.updated_at, + + tts_results.creator_ip_address, + tts_results.user_deleted_at, + tts_results.mod_deleted_at + +FROM tts_results +LEFT OUTER JOIN tts_models + ON tts_results.model_token = tts_models.token +LEFT OUTER JOIN users + ON tts_results.maybe_creator_user_token = users.token +LEFT OUTER JOIN users as model_users + ON tts_models.creator_user_token = model_users.token +WHERE + tts_results.token = ? + AND tts_results.user_deleted_at IS NULL + AND tts_results.mod_deleted_at IS NULL + "#, + tts_result_token + ) + .fetch_one(mysql_pool) + .await // TODO: This will return error if it doesn't exist +} diff --git a/crates/lib/mysql_queries/src/queries/voice_conversion/results/mod.rs b/crates/lib/mysql_queries/src/queries/voice_conversion/results/mod.rs index 016cbb72af..34ed0e88be 100644 --- a/crates/lib/mysql_queries/src/queries/voice_conversion/results/mod.rs +++ b/crates/lib/mysql_queries/src/queries/voice_conversion/results/mod.rs @@ -1 +1,2 @@ +pub mod get_voice_conversion_result; pub mod insert_voice_conversion_result; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs index ed35c6669b..2c7f98ce9b 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/process_job.rs @@ -22,6 +22,7 @@ use std::path::PathBuf; use std::time::Instant; use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use mysql_queries::payloads::generic_inference_args::lipsync_payload::{LipsyncAnimationAudioSource, LipsyncAnimationImageSource}; +use mysql_queries::queries::voice_conversion::results::get_voice_conversion_result::get_voice_conversion_result; use tokens::users::user::UserToken; use crate::job::job_types::lipsync::sad_talker::validate_job::validate_job; @@ -77,7 +78,15 @@ pub async fn process_job(args: SadTalkerProcessJobArgs<'_>) -> Result {} LipsyncAnimationAudioSource::U(media_upload_token) => {} LipsyncAnimationAudioSource::T(tts_result_token) => {} - LipsyncAnimationAudioSource::V(voice_conversion_result_token) => {} + LipsyncAnimationAudioSource::V(voice_conversion_result_token) => { + let voice_conversion_result = get_voice_conversion_result( + voice_conversion_result_token, + false, + &args.job_dependencies.mysql_pool, + ).await; + + // TODO ... + } } // ==================== DECIDE WHAT TYPE OF IMAGE TO DOWNLOAD ==================== //