This commit is contained in:
salt committed 2023-12-12 14:50:50 -08:00
1 parent 2d59c5c573
commit 43030402ca
20 files changed
+351 -8

No files matched your search

Generated
+1
View File
@@ -6025,6 +6025,7 @@ dependencies = [
"hex",
"hmac 0.10.1",
"hostname",
"http",
"http_server_common",
"hyper",
"infer 0.5.0",
@@ -48,7 +48,8 @@ impl InferenceModelType {
Self::SoVitsSvc => "so_vits_svc",
Self::Tacotron2 => "tacotron2",
Self::Vits => "vits",
Self::VallEX => "vall_e_x"
Self::VallEX => "vall_e_x",
Self::RerenderAVideo => "rerender_a_video",
}
}
@@ -3,7 +3,7 @@ use errors::AnyhowResult;
use crate::payloads::generic_inference_args::lipsync_payload::LipsyncArgs;
use crate::payloads::generic_inference_args::tts_payload::TTSArgs;
use crate::payloads::generic_inference_args::videofilter_payload::VideofilterVideoSource;
use crate::payloads::generic_inference_args::videofilter_payload::{RerenderArgs};
/// Used to encode extra state for the `generic_inference_jobs` table.
@@ -32,5 +32,5 @@ pub struct RerenderArgs {
#[serde(rename = "sd")]
#[serde(skip_serializing_if = "Option::is_none")]
pub sd_model: String,
pub maybe_sd_model: Option<String>,
}
@@ -177,6 +177,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
InferenceCategory::LipsyncAnimation => Some("lipsync animation"),
InferenceCategory::TextToSpeech => record.maybe_tts_model_title.as_deref(),
InferenceCategory::VoiceConversion => record.maybe_voice_conversion_model_title.as_deref(),
InferenceCategory::RerenderAVideo => Some("rerender animation"),
};
// NB: A bit of a hack. We store TTS results with a full path.
@@ -185,6 +186,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
InferenceCategory::LipsyncAnimation => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
InferenceCategory::TextToSpeech => (false, record.maybe_tts_public_bucket_path.as_deref()),
InferenceCategory::VoiceConversion => (true, record.maybe_voice_conversion_public_bucket_hash.as_deref()),
InferenceCategory::RerenderAVideo => (true, record.maybe_media_file_public_bucket_directory_hash.as_deref()),
};
// NB: We've moved voice conversion out of their own table and into media_files
@@ -182,6 +182,9 @@ async fn do_process_single_job(
InferenceCategory::VoiceConversion => {
process_single_vc_job(job_dependencies, job).await?
}
InferenceCategory::RerenderAVideo => {
process_single_rerender_job(job_dependencies, job).await?
}
};
let maybe_entity_type = job_success_result.maybe_result_entity
@@ -239,6 +239,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
PolymorphicInferenceArgs::Tts { .. } => None,
PolymorphicInferenceArgs::La(_) => None,
PolymorphicInferenceArgs::Vc { override_f0_method, .. } => *override_f0_method,
PolymorphicInferenceArgs::Rr(_) => None,
})
.flatten();
@@ -247,6 +248,7 @@ pub async fn process_job(args: RvcV2ProcessJobArgs<'_>) -> Result<JobSuccessResu
PolymorphicInferenceArgs::Tts { .. } => None,
PolymorphicInferenceArgs::La(_) => None,
PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose,
PolymorphicInferenceArgs::Rr(_) => None,
})
.flatten();
@@ -168,6 +168,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
PolymorphicInferenceArgs::Tts { .. } => None,
PolymorphicInferenceArgs::La(_) => None,
PolymorphicInferenceArgs::Vc { auto_predict_f0, .. } => *auto_predict_f0,
PolymorphicInferenceArgs::Rr(_) => None,
})
.flatten()
.unwrap_or(false);
@@ -177,6 +178,7 @@ pub async fn process_job(args: SoVitsSvcProcessJobArgs<'_>) -> Result<JobSuccess
PolymorphicInferenceArgs::Tts { .. } => None,
PolymorphicInferenceArgs::La(_) => None,
PolymorphicInferenceArgs::Vc { transpose, .. } => *transpose,
PolymorphicInferenceArgs::Rr(_) => None,
})
.flatten();
@@ -1 +1,2 @@
pub(crate) mod rerender_a_video;
pub(crate) mod rerender_a_video;
mod process_single_vf_job;
@@ -0,0 +1,63 @@
use std::time::Duration;
use anyhow::anyhow;
use log::{error, info};
use enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType;
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::media_uploads::get_media_upload_for_inference::get_media_upload_for_inference;
use mysql_queries::queries::voice_conversion::inference::get_voice_conversion_model_for_inference::get_voice_conversion_model_for_inference;
use tokens::tokens::media_uploads::MediaUploadToken;
use crate::job::job_loop::job_success_result::JobSuccessResult;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job::job_types::vc::{rvc_v2, so_vits_svc};
use crate::job::job_types::vc::rvc_v2::process_job::RvcV2ProcessJobArgs;
use crate::job::job_types::vc::so_vits_svc::process_job::SoVitsSvcProcessJobArgs;
use crate::job_dependencies::JobDependencies;
pub async fn process_single_rr_job(job_dependencies: &JobDependencies, job: &AvailableInferenceJob) -> Result<JobSuccessResult, ProcessSingleJobError> {
let maybe_media_upload_token = job.maybe_input_source_token
.as_deref()
.map(|token| MediaUploadToken::new_from_str(token));
let media_upload_token = match maybe_media_upload_token {
None => return Err(ProcessSingleJobError::Other(anyhow!("no associated media upload for vc job: {:?}", job.inference_job_token))),
Some(token) => token,
};
let maybe_media_upload_result =
get_media_upload_for_inference(&media_upload_token, &job_dependencies.db.mysql_pool).await;
let media_upload = match maybe_media_upload_result {
Ok(Some(media_upload)) => media_upload,
Ok(None) => {
error!("no media upload record found for token: {:?}", media_upload_token);
return Err(ProcessSingleJobError::Other(anyhow!("no media upload record found for token: {:?}", media_upload_token)));
},
Err(err) => {
error!("error fetching media upload record from db: {:?}", err);
return Err(ProcessSingleJobError::Other(err));
},
};
info!("Source media upload file size (bytes): {}", &media_upload.original_file_size_bytes);
info!("Source media upload duration (millis): {}", &media_upload.original_duration_millis);
info!("Source media upload duration (seconds): {}", (media_upload.original_duration_millis as f32 / 1000.0));
let job_success_result = match vc_model.model_type {
VoiceConversionModelType::RvcV2 => {
rvc_v2::process_job::process_job(RvcV2ProcessJobArgs {
job_dependencies,
job,
vc_model: &vc_model,
media_upload_token: &media_upload_token,
media_upload: &media_upload,
}).await?
}
};
Ok(job_success_result)
}
@@ -3,4 +3,4 @@ mod validate_job;
mod download_video_file;
mod rerender_inference_command;
pub(crate) mod rerender_dependencies;
mod categorize_error;
mod categorize_error;
@@ -83,7 +83,7 @@ pub async fn process_job(args: RerenderProcessJobArgs<'_>) -> Result<JobSuccessR
&deps.db.mysql_pool
).await?;
info!("Downloaded video file: {:?}", image_path.filesystem_path);
info!("Downloaded video file: {:?}", video_path.filesystem_path);
// ==================== TRANSCODE MEDIA (IF NECESSARY) ==================== //
@@ -53,7 +53,7 @@ pub fn validate_job(job: &AvailableInferenceJob) -> Result<JobArgs, ProcessSingl
}
};
let sd_model = match &inference_args.sd_model {
let sd_model = match &inference_args.maybe_sd_model {
Some(args) => args,
None => {
return Err(ProcessSingleJobError::from_anyhow_error(anyhow!("no sd model!")));
@@ -1,4 +1,5 @@
use actix_web::HttpRequest;
use mockall::automock;
/// Allows us to purge session caches that might keep stale premium plan/billing data around
#[cfg_attr(test, automock)]
@@ -164,6 +164,7 @@ pulldown-cmark = "0.9.3"
async-trait = "0.1.58"
utoipa = { version = "4", features = ["actix_extras"] }
utoipa-swagger-ui = { version = "5", features = ["actix-web"] }
http = "0.2.11"
[build-dependencies]
# NB: Only using diesel for migrations support
@@ -49,6 +49,9 @@ pub struct Plan {
can_remove_visual_watermarks: bool,
// ========== Features for Videofilters ==========
rerender_requires_frontend_keepalive: bool,
// ========== Features for TTS ==========
tts_base_priority_level: u8,
@@ -91,6 +94,7 @@ impl Plan {
is_development_plan: builder.is_development_plan,
is_synthetic_plan: builder.is_synthetic_plan,
lipsync_requires_frontend_keepalive: builder.lipsync_requires_frontend_keepalive,
rerender_requires_frontend_keepalive: builder.rerender_requires_frontend_keepalive,
can_remove_visual_watermarks: builder.can_remove_visual_watermarks,
tts_base_priority_level: builder.tts_base_priority_level,
tts_max_duration: builder.tts_max_duration,
@@ -146,6 +150,10 @@ impl Plan {
self.lipsync_requires_frontend_keepalive
}
pub fn rerender_requires_frontent_keepalive(&self) -> bool {
self.rerender_requires_frontend_keepalive
}
pub fn can_remove_visual_watermarks(&self) -> bool {
self.can_remove_visual_watermarks
}
@@ -192,6 +200,9 @@ pub struct PlanBuilder {
can_remove_visual_watermarks: bool,
// ========== Features for Videofilters ==========
rerender_requires_frontend_keepalive: bool,
// ========== Features for TTS ==========
tts_base_priority_level: u8,
@@ -239,6 +250,9 @@ impl PlanBuilder {
lipsync_requires_frontend_keepalive: true,
can_remove_visual_watermarks: false,
// Videofilters
rerender_requires_frontend_keepalive: true,
// TTS
tts_base_priority_level : TTS_DEFAULT_PRIORITY_LEVEL,
tts_max_duration: Duration::seconds(TTS_DEFAULT_DURATION_SECONDS),
@@ -304,6 +318,11 @@ impl PlanBuilder {
self
}
pub fn rerender_requires_frontend_keepalive(mut self, value: bool) -> Self {
self.rerender_requires_frontend_keepalive = value;
self
}
pub fn can_remove_visual_watermarks(mut self, value: bool) -> Self {
self.can_remove_visual_watermarks = value;
self
@@ -0,0 +1,237 @@
// NB: Incrementally getting rid of build warnings...
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
use std::sync::Arc;
use actix_web::{HttpRequest, HttpResponse, web};
use actix_web::error::ResponseError;
use actix_web::http::StatusCode;
use log::{info, warn};
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_model_type::InferenceModelType;
use enums::common::visibility::Visibility;
use http_server_common::request::get_request_header_optional::get_request_header_optional;
use http_server_common::request::get_request_ip::get_request_ip;
use mysql_queries::payloads::generic_inference_args::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs};
use mysql_queries::payloads::generic_inference_args::videofilter_payload::{RerenderArgs, VideofilterVideoSource};
use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{insert_generic_inference_job, InsertGenericInferenceArgs};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use tokens::tokens::media_uploads::MediaUploadToken;
use tokens::tokens::users::UserToken;
use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session;
use crate::http_server::web_utils::response_error_helpers::to_simple_json_error;
use crate::server_state::ServerState;
/// Debug requests can get routed to special "debug-only" workers, which can
/// be used to trial new code, run debugging, etc.
const DEBUG_HEADER_NAME : &str = "enable-debug-mode";
/// The routing tag header can send workloads to particular k8s hosts.
/// This is useful for catching the live logs or intercepting the job.
const ROUTING_TAG_HEADER_NAME : &str = "routing-tag";
#[derive(Deserialize)]
pub struct EnqueueRerenderAnimationRequest {
uuid_idempotency_token: String,
video_source: VideoSource,
sd_model: String,
creator_set_visibility: Option<Visibility>,
}
/// Treated as an enum. Only one of these may be set.
#[derive(Deserialize)]
pub struct VideoSource {
maybe_media_file_token: Option<MediaFileToken>,
maybe_media_upload_token: Option<MediaUploadToken>,
}
#[derive(Serialize)]
pub struct EnqueueRerenderAnimationSuccessResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
#[derive(Debug)]
pub enum EnqueueRerenderAnimationError {
BadInput(String),
NotAuthorized,
ServerError,
RateLimited,
}
impl ResponseError for EnqueueRerenderAnimationError {
fn status_code(&self) -> StatusCode {
match *self {
EnqueueRerenderAnimationError::BadInput(_) => StatusCode::BAD_REQUEST,
EnqueueRerenderAnimationError::NotAuthorized => StatusCode::UNAUTHORIZED,
EnqueueRerenderAnimationError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
EnqueueRerenderAnimationError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
}
}
fn error_response(&self) -> HttpResponse {
let error_reason = match self {
EnqueueRerenderAnimationError::BadInput(reason) => reason.to_string(),
EnqueueRerenderAnimationError::NotAuthorized => "unauthorized".to_string(),
EnqueueRerenderAnimationError::ServerError => "server error".to_string(),
EnqueueRerenderAnimationError::RateLimited => "rate limited".to_string(),
};
to_simple_json_error(&error_reason, self.status_code())
}
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl std::fmt::Display for EnqueueRerenderAnimationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{:?}", self)
}
}
pub async fn enqueue_rerender_animation_handler(
http_request: HttpRequest,
request: web::Json<EnqueueRerenderAnimationRequest>,
server_state: web::Data<Arc<ServerState>>) -> Result<HttpResponse, EnqueueRerenderAnimationError>
{
let mut maybe_user_token : Option<UserToken> = None;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await
.map_err(|err| {
warn!("MySql pool error: {:?}", err);
EnqueueRerenderAnimationError::ServerError
})?;
let maybe_avt_token = server_state.avt_cookie_manager.get_avt_token_from_request(&http_request);
// ==================== USER SESSION ==================== //
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
EnqueueRerenderAnimationError::ServerError
})?;
if let Some(user_session) = maybe_user_session.as_ref() {
maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token));
}
// TODO: Plan should handle "first anonymous use" and "investor" cases.
let plan = get_correct_plan_for_session(
server_state.server_environment,
maybe_user_session.as_ref());
// TODO: Separate priority for animation.
let priority_level = plan.web_vc_base_priority_level();
// ==================== DEBUG MODE + ROUTING TAG ==================== //
let is_debug_request =
get_request_header_optional(&http_request, DEBUG_HEADER_NAME)
.is_some();
let maybe_routing_tag=
get_request_header_optional(&http_request, ROUTING_TAG_HEADER_NAME)
.map(|routing_tag| routing_tag.trim().to_string());
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.logged_out,
Some(ref user) => {
if user.role.is_banned {
return Err(EnqueueRerenderAnimationError::NotAuthorized);
}
&server_state.redis_rate_limiters.logged_in
},
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
return Err(EnqueueRerenderAnimationError::RateLimited);
}
// ==================== LOOK UP MODEL INFO ==================== //
// TODO(bt): CHECK DATABASE FOR TOKENS!
let video_source: VideofilterVideoSource = {
if let Some(ref token) = request.video_source.maybe_media_file_token {
VideofilterVideoSource::media_file_token(token.as_str())
} else if let Some(ref token) = request.video_source.maybe_media_upload_token {
VideofilterVideoSource::media_upload_token(token.as_str())
} else {
return Err(EnqueueRerenderAnimationError::BadInput("video source not fully specified".to_string()));
}
};
let ip_address = get_request_ip(&http_request);
let maybe_user_preferred_visibility : Option<Visibility> = maybe_user_session
.as_ref()
.map(|user_session: &users_component::utils::user_session_extended::UserSessionExtended| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc
let set_visibility = request.creator_set_visibility
.or(maybe_user_preferred_visibility)
.unwrap_or(Visibility::Public);
let inference_args = RerenderArgs {
maybe_video_source: Some(video_source),
maybe_sd_model: Some(request.sd_model.clone()),
};
info!("Creating rerender animation job record...");
let query_result = insert_generic_inference_job(InsertGenericInferenceArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
inference_category: InferenceCategory::RerenderAVideo,
maybe_model_type: Some(InferenceModelType::RerenderAVideo), // NB: Model is static during inference
maybe_model_token: None, // NB: Model is static during inference
maybe_input_source_token: None, // TODO: Introduce a second foreign key ?
maybe_input_source_token_type: None, // TODO: Introduce a second foreign key ?
maybe_raw_inference_text: None, // No text
maybe_max_duration_seconds: None,
maybe_inference_args: Some(GenericInferenceArgs {
inference_category: Some(InferenceCategoryAbbreviated::RerenderAVideo),
args: Some(PolymorphicInferenceArgs::Rr(inference_args)),
}),
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: set_visibility,
priority_level,
requires_keepalive: plan.rerender_requires_frontent_keepalive(),
is_debug_request,
maybe_routing_tag: maybe_routing_tag.as_deref(),
mysql_pool: &server_state.mysql_pool,
}).await;
let job_token = match query_result {
Ok((job_token, _id)) => job_token,
Err(err) => {
warn!("New generic inference job creation DB error: {:?}", err);
return Err(EnqueueRerenderAnimationError::ServerError);
}
};
let response: EnqueueRerenderAnimationSuccessResponse = EnqueueRerenderAnimationSuccessResponse {
success: true,
inference_job_token: job_token,
};
let body = serde_json::to_string(&response)
.map_err(|_e| EnqueueRerenderAnimationError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body))
}
@@ -1 +1,2 @@
pub mod enqueue_lipsync_animation;
pub mod enqueue_lipsync_animation;
pub mod enqueue_rerender_animation;
@@ -256,6 +256,14 @@ fn record_to_payload(
.get_full_object_path_str()
.to_string()
}
InferenceCategory::RerenderAVideo => {
MediaFileBucketPath::from_object_hash(
&result_details.public_bucket_location_or_hash,
result_details.maybe_media_file_public_bucket_prefix.as_deref(),
result_details.maybe_media_file_public_bucket_extension.as_deref())
.get_full_object_path_str()
.to_string()
}
InferenceCategory::TextToSpeech => {
match result_details.entity_type.as_str() {
"media_file" => {
@@ -12,6 +12,7 @@ use users_component::endpoints::edit_profile_handler::edit_profile_handler;
use users_component::endpoints::get_profile_handler::get_profile_handler;
use crate::http_server::endpoints::animation::enqueue_lipsync_animation::enqueue_lipsync_animation_handler;
use crate::http_server::endpoints::animation::enqueue_rerender_animation::enqueue_rerender_animation_handler;
use crate::http_server::endpoints::api_tokens::create_api_token::create_api_token_handler;
use crate::http_server::endpoints::api_tokens::delete_api_token::delete_api_token_handler;
use crate::http_server::endpoints::api_tokens::edit_api_token::edit_api_token_handler;