From 138057e105c2b60c8aeb4c157b1fff16e90c795f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 13 Oct 2023 02:38:34 -0400 Subject: [PATCH] v1 of media_file GET endpoint. Media files are used for inference outputs, file uploads, Zero Shot files, etc. We don't have any of the basic CRUD operations around them yet, so this is a start. This endpoint will need more metadata - moderation, file provenance, file details, etc. But this will do for now. --- .../generic_inference_jobs.sql | 5 +- .../src/queries/media_files/get_media_file.rs | 196 ++++++++++++++++++ .../src/queries/media_files/mod.rs | 1 + .../common_responses/user_details_lite.rs | 70 +++++++ .../endpoints/media_files/get_media_file.rs | 186 +++++++++++++++++ .../http_server/endpoints/media_files/mod.rs | 1 + .../src/http_server/endpoints/mod.rs | 3 +- .../service/web/storyteller_web/src/routes.rs | 23 ++ 8 files changed, 481 insertions(+), 4 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs diff --git a/_sql/analytics/generic_inference/generic_inference_jobs.sql b/_sql/analytics/generic_inference/generic_inference_jobs.sql index f4fbef7b51..aa6ace1752 100644 --- a/_sql/analytics/generic_inference/generic_inference_jobs.sql +++ b/_sql/analytics/generic_inference/generic_inference_jobs.sql @@ -37,7 +37,6 @@ and status IN ('pending', 'started', 'complete_failure', 'attempt_failed') group by maybe_creator_user_token, maybe_creator_username, creator_ip_address order by attempts desc; - -- Detailed report on most recent jobs, ordered by worst performing. -- TODO: Determine if the problem is in downloading models. Make sure the job timer doesn't include -- sections where models / files get downloaded. @@ -71,8 +70,8 @@ from ( m.token = jobs.maybe_input_source_token where jobs.status != 'pending' - AND jobs.created_at > NOW() - INTERVAL 2000 MINUTE - AND jobs.maybe_model_type IN ('sad_talker') + AND jobs.created_at > NOW() - INTERVAL 20 MINUTE + AND jobs.maybe_model_type IN ('so_vits_svc') order by id desc limit 200 ) as t diff --git a/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs new file mode 100644 index 0000000000..ec94000555 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/get_media_file.rs @@ -0,0 +1,196 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use anyhow::anyhow; +use chrono::{DateTime, Utc}; +use sqlx::MySqlPool; + +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +#[derive(Serialize)] +pub struct MediaFile { + pub token: MediaFileToken, + + pub media_type: MediaFileType, + + // TODO: Bucket hash bits. + + // TODO: Other media details (file size, mime type, dimensions, duration, etc.) + // TODO: Provenance data (product, upload vs inference, model details and foreign keys) + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub creator_set_visibility: Visibility, + + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, + + // pub maybe_moderator_fields: Option, +} + +// /// "Moderator-only fields" that we wouldn't want to expose to ordinary users. +// /// It's the web endpoint controller's responsibility to clear these for non-mods. +// #[derive(Serialize)] +// pub struct MediaFileModeratorFields { +// pub model_creator_is_banned: bool, +// pub result_creator_is_banned_if_user: bool, +// pub result_creator_ip_address: String, +// pub result_creator_deleted_at: Option>, +// pub mod_deleted_at: Option>, +// pub maybe_mod_user_token: Option, +// } + +#[derive(Serialize)] +pub struct MediaFileRaw { + pub token: MediaFileToken, + + pub media_type: MediaFileType, + + // TODO: Bucket hash bits. + + pub maybe_creator_user_token: Option, + pub maybe_creator_username: Option, + pub maybe_creator_display_name: Option, + pub maybe_creator_gravatar_hash: Option, + + pub creator_set_visibility: Visibility, + + pub public_bucket_directory_hash: String, + pub maybe_public_bucket_prefix: Option, + pub maybe_public_bucket_extension: Option, + + //pub model_is_mod_approved: bool, // converted + //pub maybe_mod_user_token: Option, + + pub created_at: DateTime, + pub updated_at: DateTime, +} + +pub async fn get_media_file( + media_file_token: &MediaFileToken, + can_see_deleted: bool, + mysql_pool: &MySqlPool +) -> AnyhowResult> { + + let record = if can_see_deleted { + select_including_deleted(media_file_token, mysql_pool).await + } else { + select_without_deleted(media_file_token, mysql_pool).await + }; + + let record = match record { + Ok(record) => record, + Err(ref err) => { + return match err { + sqlx::Error::RowNotFound => Ok(None), + _ => Err(anyhow!("database error: {:?}", err)), + } + } + }; + + Ok(Some(MediaFile { + token: record.token, + media_type: record.media_type, + maybe_creator_user_token: record.maybe_creator_user_token, + maybe_creator_username: record.maybe_creator_username, + maybe_creator_display_name: record.maybe_creator_display_name, + maybe_creator_gravatar_hash: record.maybe_creator_gravatar_hash, + creator_set_visibility: record.creator_set_visibility, + public_bucket_directory_hash: record.public_bucket_directory_hash, + maybe_public_bucket_prefix: record.maybe_public_bucket_prefix, + maybe_public_bucket_extension: record.maybe_public_bucket_extension, + created_at: record.created_at, + updated_at: record.updated_at, + })) +} + +async fn select_including_deleted( + media_file_token: &MediaFileToken, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + MediaFileRaw, + r#" +SELECT + m.token as `token: tokens::tokens::media_files::MediaFileToken`, + + m.media_type as `media_type: enums::by_table::media_files::media_file_type::MediaFileType`, + + users.token as `maybe_creator_user_token: tokens::tokens::users::UserToken`, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.created_at, + m.updated_at + +FROM media_files AS m +LEFT OUTER JOIN users + ON m.maybe_creator_user_token = users.token +WHERE + m.token = ? + "#, + media_file_token + ) + .fetch_one(mysql_pool) + .await +} + +async fn select_without_deleted( + media_file_token: &MediaFileToken, + mysql_pool: &MySqlPool +) -> Result { + sqlx::query_as!( + MediaFileRaw, + r#" +SELECT + m.token as `token: tokens::tokens::media_files::MediaFileToken`, + + m.media_type as `media_type: enums::by_table::media_files::media_file_type::MediaFileType`, + + users.token as `maybe_creator_user_token: tokens::tokens::users::UserToken`, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + users.email_gravatar_hash as maybe_creator_gravatar_hash, + + m.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + + m.public_bucket_directory_hash, + m.maybe_public_bucket_prefix, + m.maybe_public_bucket_extension, + + m.created_at, + m.updated_at + +FROM media_files AS m +LEFT OUTER JOIN users + ON m.maybe_creator_user_token = users.token +WHERE + m.token = ? + AND m.user_deleted_at IS NULL + AND m.mod_deleted_at IS NULL + "#, + media_file_token + ) + .fetch_one(mysql_pool) + .await +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index a6f05122a6..632f5cd244 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -1 +1,2 @@ +pub mod get_media_file; pub mod insert_media_file_from_face_animation; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs index 80d0a04741..9e83bdf178 100644 --- a/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs +++ b/crates/service/web/storyteller_web/src/http_server/common_responses/user_details_lite.rs @@ -28,6 +28,45 @@ pub struct UserDetailsLight { // In the future, we'll also support user-uploaded avatars that we store on our servers. } +impl UserDetailsLight { + + pub fn from_optional_db_fields( + maybe_user_token: Option<&UserToken>, + maybe_username: Option<&str>, + maybe_display_name: Option<&str>, + maybe_gravatar_hash: Option<&str>, + ) -> Option { + Self::from_optional_db_fields_owned( + maybe_user_token.map(|u| u.clone()), + maybe_username.map(|s|s.to_string()), + maybe_display_name.map(|s|s.to_string()), + maybe_gravatar_hash.map(|s|s.to_string()), + ) + } + + pub fn from_optional_db_fields_owned( + maybe_user_token: Option, + maybe_username: Option, + maybe_display_name: Option, + maybe_gravatar_hash: Option, + ) -> Option { + match (maybe_user_token, maybe_username, maybe_display_name, maybe_gravatar_hash) { + (Some(user_token), Some(username), Some(display_name), Some(gravatar_hash)) => { + Some(Self { + default_avatar: DefaultAvatarInfo::from_username(&username), + user_token, + username, + display_name, + gravatar_hash, + }) + } + _ => { + None + } + } + } +} + #[derive(Clone, Serialize)] pub struct DefaultAvatarInfo { pub image_index: u8, @@ -45,3 +84,34 @@ impl DefaultAvatarInfo { } } } + +#[cfg(test)] +mod tests { + use tokens::tokens::users::UserToken; + + use crate::http_server::common_responses::user_details_lite::UserDetailsLight; + + #[test] + fn test_from_optional_db_fields() { + let user_token = UserToken::new_from_str("token"); + let username = "username"; + let display_name = "display_name"; + let gravatar_hash= "adsf"; + + let user_details = UserDetailsLight::from_optional_db_fields( + Some(&user_token), + Some(username), + Some(display_name), + Some(gravatar_hash) + ); + + let user_details = user_details.expect("Should not be optional."); + + assert_eq!(user_details.user_token, user_token); + assert_eq!(user_details.username, username); + assert_eq!(user_details.display_name, display_name); + assert_eq!(user_details.gravatar_hash, gravatar_hash); + assert_eq!(user_details.default_avatar.color_index, 5); + assert_eq!(user_details.default_avatar.image_index, 16); + } +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs new file mode 100644 index 0000000000..350294d6f0 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/get_media_file.rs @@ -0,0 +1,186 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpMessage, HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::web::Path; +use chrono::{DateTime, Utc}; +use log::warn; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use mysql_queries::queries::media_files::get_media_file::get_media_file; +use tokens::tokens::media_files::MediaFileToken; + +use crate::http_server::common_responses::user_details_lite::UserDetailsLight; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +/// For the URL PathInfo +#[derive(Deserialize)] +pub struct GetMediaFilePathInfo { + token: MediaFileToken, +} + +#[derive(Serialize)] +pub struct GetMediaFileSuccessResponse { + pub success: bool, + pub media_file: MediaFile, +} + +#[derive(Serialize)] +pub struct MediaFile { + pub token: MediaFileToken, + + /// Type of media will dictate which fields are populated and what + /// the frontend should display (eg. video player vs audio player). + pub media_type: MediaFileType, + + /// URL to the media file + pub public_bucket_path: String, + + //// Provenance Data + //pub origin_product: MediaFileOriginProductCategory, + //pub origin_category: MediaFileOriginCategory, + //pub maybe_origin_model_type: Option, + //pub maybe_origin_model_token: Option, + //pub maybe_origin_filename: Option, + + // User + pub maybe_creator_user: Option, + + pub creator_set_visibility: Visibility, + + pub created_at: DateTime, + pub updated_at: DateTime, + + //pub maybe_moderator_fields: Option, +} + +//#[derive(Serialize)] +//pub struct ModeratorFields { +// pub creator_ip_address: String, +// pub creator_is_banned_if_user: bool, +// pub maybe_creator_deleted_at: Option>, +// pub maybe_mod_deleted_at: Option>, +//} + +#[derive(Debug)] +pub enum GetMediaFileError { + ServerError, + NotFound, +} + +impl ResponseError for GetMediaFileError { + fn status_code(&self) -> StatusCode { + match *self { + GetMediaFileError::ServerError=> StatusCode::INTERNAL_SERVER_ERROR, + GetMediaFileError::NotFound=> StatusCode::NOT_FOUND, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + GetMediaFileError::ServerError => "server error".to_string(), + GetMediaFileError::NotFound => "not found".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for GetMediaFileError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn get_media_file_handler( + http_request: HttpRequest, + path: Path, + server_state: web::Data>) -> Result +{ + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session(&http_request, &server_state.mysql_pool) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + GetMediaFileError::ServerError + })?; + + let mut show_deleted_results = false; + let mut is_moderator = false; + + if let Some(user_session) = maybe_user_session { + // NB: Moderators can see deleted results. + // Original creators cannot see them (unless they're moderators!) + show_deleted_results = user_session.can_delete_other_users_tts_results; + // Moderators get to see all the fields. + is_moderator = user_session.can_delete_other_users_tts_results + || user_session.can_edit_other_users_tts_models; + } + + let result = get_media_file( + &path.token, + show_deleted_results, + &server_state.mysql_pool + ).await; + + let result = match result { + Err(e) => { + warn!("query error: {:?}", e); + return Err(GetMediaFileError::ServerError); + } + Ok(None) => return Err(GetMediaFileError::NotFound), + Ok(Some(result)) => result, + }; + + //if let Some(moderator_fields) = result.maybe_moderator_fields.as_ref() { + // // NB: The moderator fields will always be present before removal + // // We don't want non-mods seeing stuff made by banned users. + // if (moderator_fields.model_creator_is_banned || moderator_fields.result_creator_is_banned_if_user) + // && !is_moderator{ + // return Err(GetMediaFileError::NotFound); + // } + //} + + //if !is_moderator { + // result.maybe_moderator_fields = None; + //} + + let public_bucket_path = MediaFileBucketPath::from_object_hash( + &result.public_bucket_directory_hash, + result.maybe_public_bucket_prefix.as_deref(), + result.maybe_public_bucket_extension.as_deref()) + .get_full_object_path_str() + .to_string(); + + let response = GetMediaFileSuccessResponse { + success: true, + media_file: MediaFile { + token: result.token, + media_type: result.media_type, + public_bucket_path, + maybe_creator_user: UserDetailsLight::from_optional_db_fields_owned( + result.maybe_creator_user_token, + result.maybe_creator_username, + result.maybe_creator_display_name, + result.maybe_creator_gravatar_hash, + ), + creator_set_visibility: result.creator_set_visibility, + created_at: result.created_at, + updated_at: result.updated_at, + }, + }; + + let body = serde_json::to_string(&response) + .map_err(|e| GetMediaFileError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs new file mode 100644 index 0000000000..e320d83853 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_files/mod.rs @@ -0,0 +1 @@ +pub mod get_media_file; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 69a12beb9f..88a72ab810 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -8,6 +8,7 @@ pub mod flags; pub mod inference_job; pub mod investor_demo; pub mod leaderboard; +pub mod media_files; pub mod media_uploads; pub mod misc; pub mod moderation; @@ -23,5 +24,5 @@ pub mod user_ratings; pub mod vocoders; pub mod voice_clone_requests; pub mod voice_conversion; +pub mod voice_designer; pub mod w2l; -pub mod voice_designer; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index ca3392bb3d..bbd27aa871 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -34,6 +34,7 @@ use crate::http_server::endpoints::inference_job::get_pending_inference_job_coun use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler; use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler; use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler; +use crate::http_server::endpoints::media_files::get_media_file::get_media_file_handler; use crate::http_server::endpoints::media_uploads::list_user_media_uploads_of_type::list_user_media_uploads_of_type_handler; use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler; use crate::http_server::endpoints::media_uploads::upload_image::upload_image_handler; @@ -163,6 +164,7 @@ pub fn add_routes (app: App) -> App app = add_investor_demo_routes(app); /* /demo_mode */ // TODO: DEFINITELY TEMPORARY app = add_flag_routes(app); /* /flag */ app = add_desktop_app_routes(app); /* /v1/vc/... */ + app = add_media_file_routes(app); /* /v1/media_files/... */ app = add_media_upload_routes(app); /* /v1/media_upload/... */ app = add_trending_routes(app); /* /v1/trending/... */ app = add_user_rating_routes(app); /* /v1/user_rating/... */ @@ -956,6 +958,27 @@ fn add_desktop_app_routes (app: App) -> App ) } +// ==================== MEDIA FILE ROUTES ==================== + +fn add_media_file_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service(web::scope("/v1/media_files") + .service(web::resource("/file/{token}") + .route(web::get().to(get_media_file_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +} + // ==================== MEDIA UPLOAD ROUTES ==================== fn add_media_upload_routes (app: App) -> App