From ebf172945941438f0c6b9e7efebeec658de70596 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 27 Apr 2024 09:07:47 -0400 Subject: [PATCH] voice conversion: accept media file token, document --- .../web/storyteller_web/src/api_doc.rs | 7 ++ .../edit/rename_media_file_handler.rs | 2 +- .../enqueue_voice_conversion_inference.rs | 74 ++++++++++++++++--- 3 files changed, 73 insertions(+), 10 deletions(-) diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 7eac693d0c..9db8e1de9a 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -46,6 +46,7 @@ use crate::http_server::endpoints::inference_job::list_session_jobs_handler::*; use crate::http_server::endpoints::inference_job::terminate_inference_job_handler::*; use crate::http_server::endpoints::media_files::delete::delete_media_file_handler::*; use crate::http_server::endpoints::media_files::edit::change_media_file_visibility_handler::*; +use crate::http_server::endpoints::voice_conversion::inference::enqueue_voice_conversion_inference::*; use crate::http_server::endpoints::media_files::edit::rename_media_file_handler::*; use crate::http_server::endpoints::media_files::edit::set_media_file_cover_image_handler::*; use crate::http_server::endpoints::media_files::get::batch_get_media_files_handler::*; @@ -122,6 +123,7 @@ use crate::http_server::web_utils::response_success_helpers::*; crate::http_server::endpoints::user_ratings::batch_get_user_rating_handler::batch_get_user_rating_handler, crate::http_server::endpoints::user_ratings::get_user_rating_handler::get_user_rating_handler, crate::http_server::endpoints::user_ratings::set_user_rating_handler::set_user_rating_handler, + crate::http_server::endpoints::voice_conversion::inference::enqueue_voice_conversion_inference::enqueue_voice_conversion_inference_handler, crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, crate::http_server::endpoints::voice_designer::voice_datasets::list_datasets_by_user::list_datasets_by_user_handler, crate::http_server::endpoints::weights::delete_weight_handler::delete_weight_handler, @@ -208,6 +210,11 @@ use crate::http_server::web_utils::response_success_helpers::*; ChangeMediaFileVisibilityPathInfo, ChangeMediaFileVisibilityRequest, CreateSceneError, + EnqueueVoiceConversionInferenceError, + EnqueueVoiceConversionInferenceSuccessResponse, + + EnqueueVoiceConversionInferenceRequest, + FundamentalFrequencyMethod, CreateSceneSuccessResponse, CreateUserBookmarkError, CreateUserBookmarkRequest, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/edit/rename_media_file_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/edit/rename_media_file_handler.rs index cbbb6289f4..ca6a6c4306 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/edit/rename_media_file_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/edit/rename_media_file_handler.rs @@ -70,7 +70,7 @@ impl fmt::Display for RenameMediaFileError { tag = "Media Files", path = "/v1/media_files/rename/{token}", responses( - (status = 200, description = "Success Delete", body = SimpleGenericJsonSuccess), + (status = 200, description = "Success", body = SimpleGenericJsonSuccess), (status = 400, description = "Bad input", body = RenameMediaFileError), (status = 401, description = "Not authorized", body = RenameMediaFileError), (status = 500, description = "Server error", body = RenameMediaFileError), diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index edf3222e25..5fdded84dc 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -13,6 +13,7 @@ use log::{error, info, warn}; use r2d2_redis::redis::Commands; use sqlx::MySql; use sqlx::pool::PoolConnection; +use utoipa::ToSchema; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_input_source_token_type::InferenceInputSourceTokenType; @@ -26,6 +27,7 @@ use mysql_queries::payloads::generic_inference_args::generic_inference_args::{Fu use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs}; use redis_common::redis_keys::RedisKeys; use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; use tokens::tokens::media_uploads::MediaUploadToken; use tokens::tokens::users::UserToken; use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL; @@ -44,13 +46,20 @@ const DEBUG_HEADER_NAME : &str = "enable-debug-mode"; /// This is useful for catching the live logs or intercepting the job. const ROUTING_TAG_HEADER_NAME : &str = "routing-tag"; -#[derive(Deserialize)] +#[derive(Deserialize, ToSchema)] pub struct EnqueueVoiceConversionInferenceRequest { uuid_idempotency_token: String, voice_conversion_model_token: String, - // TODO: Make media upload token optional and allow result audio to be re-used as well - source_media_upload_token: MediaUploadToken, + /// NB: This can be a `MediaUploadToken` or a `MediaFileToken` token. + /// We will eventually migrate everything to media files. + source_media_upload_token: String, + + // /// Media upload tokens are the old, deprecated format + // source_media_upload_token: Option, + + // /// Media file tokens are the new, preferred format + // source_media_file_token: Option, creator_set_visibility: Option, is_storyteller_demo: Option, @@ -69,7 +78,7 @@ pub struct EnqueueVoiceConversionInferenceRequest { transpose: Option, } -#[derive(Deserialize, Clone, Copy)] +#[derive(Deserialize, Clone, Copy, ToSchema)] pub enum FundamentalFrequencyMethod { /// RMVPE is the best algorithm as of 2023-10-10. #[serde(rename = "rmvpe")] @@ -82,13 +91,13 @@ pub enum FundamentalFrequencyMethod { Harvest, } -#[derive(Serialize)] +#[derive(Serialize, ToSchema)] pub struct EnqueueVoiceConversionInferenceSuccessResponse { pub success: bool, pub inference_job_token: InferenceJobToken, } -#[derive(Debug)] +#[derive(Debug, ToSchema)] pub enum EnqueueVoiceConversionInferenceError { BadInput(String), NotAuthorized, @@ -125,6 +134,21 @@ impl fmt::Display for EnqueueVoiceConversionInferenceError { } } +#[utoipa::path( + post, + tag = "Voice Conversion", + path = "/v1/voice_conversion/inference", + responses( + (status = 200, description = "Success", body = EnqueueVoiceConversionInferenceSuccessResponse), + (status = 400, description = "Bad input", body = EnqueueVoiceConversionInferenceError), + (status = 401, description = "Not authorized", body = EnqueueVoiceConversionInferenceError), + (status = 429, description = "Rate limited", body = EnqueueVoiceConversionInferenceError), + (status = 500, description = "Server error", body = EnqueueVoiceConversionInferenceError), + ), + params( + ("request" = EnqueueVoiceConversionInferenceRequest, description = "Payload for Request"), + ) +)] pub async fn enqueue_voice_conversion_inference_handler( http_request: HttpRequest, request: web::Json, @@ -234,7 +258,6 @@ pub async fn enqueue_voice_conversion_inference_handler( // TODO(bt): CHECK DATABASE! let model_token = request.voice_conversion_model_token.clone(); - let media_token = request.source_media_upload_token.clone(); let model_inference_info = lookup_model_info( &model_token, @@ -242,6 +265,39 @@ pub async fn enqueue_voice_conversion_inference_handler( &mut mysql_connection ).await?; + // ==================== HANDLE AUDIO INPUT ==================== // + + let media_token = request.source_media_upload_token.clone(); + + let media_token_type = + if media_token.starts_with(MediaUploadToken::token_prefix()) { + InferenceInputSourceTokenType::MediaUpload + } else if media_token.starts_with(MediaFileToken::token_prefix()) { + InferenceInputSourceTokenType::MediaFile + } else { + return Err(EnqueueVoiceConversionInferenceError::BadInput( + "input token is not a media_upload or media_file token".to_string())); + }; + + // NB: This is better, but we'd need to change the API and migrate callers. + // let (input_token, input_token_type) = + // match (&request.source_media_upload_token, &request.source_media_file_token) { + // (Some(media_upload), None) => { + // (media_upload.to_string(), InferenceInputSourceTokenType::MediaUpload) + // }, + // (None, Some(media_file)) => { + // (media_file.to_string(), InferenceInputSourceTokenType::MediaFile) + // }, + // (Some(_media_upload), Some(_media_file)) => { + // return Err(EnqueueVoiceConversionInferenceError::BadInput( + // "both source_media_upload_token and source_media_file_token provided".to_string())); + // }, + // (None, None) => { + // return Err(EnqueueVoiceConversionInferenceError::BadInput( + // "neither source_media_upload_token nor source_media_file_token provided".to_string())); + // }, + // }; + // ==================== CHECK AND ENQUEUE VOICE CONVERSION ==================== // let mut redis = server_state.redis_pool @@ -306,8 +362,8 @@ pub async fn enqueue_voice_conversion_inference_handler( inference_category: InferenceCategory::VoiceConversion, maybe_model_type: Some(model_inference_info.job_model_type), maybe_model_token: Some(&model_token), - maybe_input_source_token: Some(media_token.as_str()), - maybe_input_source_token_type: Some(InferenceInputSourceTokenType::MediaUpload), + maybe_input_source_token: Some(&media_token), + maybe_input_source_token_type: Some(media_token_type), maybe_download_url: None, maybe_cover_image_media_file_token: None, maybe_raw_inference_text: None, // NB: Voice conversion isn't TTS, so there's no text.