From 2e8cf5a8762b327c50dfcb0f6af06109dbcb2708 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 23 May 2024 05:44:39 -0400 Subject: [PATCH] new session recording endpoint --- ...ed295e87c46ffd5268b83a258c74f4ffc2e18.json | 12 ++ crates/_types/tokens/src/prefixes.rs | 2 + .../tokens/src/tokens/browser_session_logs.rs | 16 ++ crates/_types/tokens/src/tokens/mod.rs | 3 +- .../src/queries/browser_session_logs/mod.rs | 1 + .../upsert_browser_session_log.rs | 66 +++++++ crates/lib/mysql_queries/src/queries/mod.rs | 1 + .../web/storyteller_web/src/api_doc.rs | 13 +- .../analytics/log_browser_session_handler.rs | 161 ++++++++++++++++++ .../http_server/endpoints/analytics/mod.rs | 1 + .../src/http_server/endpoints/mod.rs | 1 + 11 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 .sqlx/query-b1ee86b547130219e2a5958ff3eed295e87c46ffd5268b83a258c74f4ffc2e18.json create mode 100644 crates/_types/tokens/src/tokens/browser_session_logs.rs create mode 100644 crates/lib/mysql_queries/src/queries/browser_session_logs/mod.rs create mode 100644 crates/lib/mysql_queries/src/queries/browser_session_logs/upsert_browser_session_log.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/analytics/log_browser_session_handler.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/analytics/mod.rs diff --git a/.sqlx/query-b1ee86b547130219e2a5958ff3eed295e87c46ffd5268b83a258c74f4ffc2e18.json b/.sqlx/query-b1ee86b547130219e2a5958ff3eed295e87c46ffd5268b83a258c74f4ffc2e18.json new file mode 100644 index 0000000000..671e0d27ac --- /dev/null +++ b/.sqlx/query-b1ee86b547130219e2a5958ff3eed295e87c46ffd5268b83a258c74f4ffc2e18.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\nINSERT INTO browser_session_logs\nSET\n token = ?,\n ip_address = ?,\n maybe_user_token = ?,\n maybe_anonymous_visitor_token = ?,\n maybe_last_action = ?\nON DUPLICATE KEY UPDATE\n maybe_last_action = ?,\n update_count = update_count + 1,\n maybe_last_updated_at = NOW()\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "b1ee86b547130219e2a5958ff3eed295e87c46ffd5268b83a258c74f4ffc2e18" +} diff --git a/crates/_types/tokens/src/prefixes.rs b/crates/_types/tokens/src/prefixes.rs index f671088e08..4a3d8621ab 100644 --- a/crates/_types/tokens/src/prefixes.rs +++ b/crates/_types/tokens/src/prefixes.rs @@ -26,6 +26,7 @@ pub(crate) enum TokenPrefix { AuditLog, BatchGeneration, BetaKey, + BrowserSessionLog, Comment, DownloadJob, EmailSenderJob, @@ -96,6 +97,7 @@ impl PrefixGenerator for TokenPrefix { Self::AuditLog => "audit_", Self::BatchGeneration => "batch_g_", Self::BetaKey => "beta_key_", + Self::BrowserSessionLog => "bsl_", Self::Comment => "comment_", Self::DownloadJob => "jdown_", // NB: Previously "JGUP:" Self::EmailSenderJob => "email_job_", diff --git a/crates/_types/tokens/src/tokens/browser_session_logs.rs b/crates/_types/tokens/src/tokens/browser_session_logs.rs new file mode 100644 index 0000000000..87e9d31fd8 --- /dev/null +++ b/crates/_types/tokens/src/tokens/browser_session_logs.rs @@ -0,0 +1,16 @@ +use std::fmt::Debug; + +use serde::Deserialize; +use serde::Serialize; +use utoipa::ToSchema; + +use crate::prefixes::TokenPrefix; + +/// The primary key for Audit Logs. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, sqlx::Type, Debug, Serialize, Deserialize, ToSchema)] +#[sqlx(transparent)] +pub struct BrowserSessionLogToken(pub String); + +impl_crockford_generator!(BrowserSessionLogToken, 32usize, TokenPrefix::BrowserSessionLog, CrockfordLower); +impl_mysql_token_from_row!(BrowserSessionLogToken); +impl_string_token!(BrowserSessionLogToken); diff --git a/crates/_types/tokens/src/tokens/mod.rs b/crates/_types/tokens/src/tokens/mod.rs index 9daed64a6e..373f02d525 100644 --- a/crates/_types/tokens/src/tokens/mod.rs +++ b/crates/_types/tokens/src/tokens/mod.rs @@ -4,6 +4,7 @@ pub mod api_tokens_internal; pub mod audit_logs; pub mod batch_generations; pub mod beta_keys; +pub mod browser_session_logs; pub mod comments; pub mod email_sender_job_token; pub mod firehose_entries; @@ -16,6 +17,7 @@ pub mod model_weights; pub mod password_reset; pub mod prompts; pub mod sqlite; +pub mod tags; pub mod tts_inference_jobs; pub mod tts_model_upload_jobs; pub mod tts_models; @@ -38,4 +40,3 @@ pub mod w2l_templates; pub mod zs_voice_dataset_samples; pub mod zs_voice_datasets; pub mod zs_voices; -mod tags; diff --git a/crates/lib/mysql_queries/src/queries/browser_session_logs/mod.rs b/crates/lib/mysql_queries/src/queries/browser_session_logs/mod.rs new file mode 100644 index 0000000000..d257d76ac1 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/browser_session_logs/mod.rs @@ -0,0 +1 @@ +pub mod upsert_browser_session_log; \ No newline at end of file diff --git a/crates/lib/mysql_queries/src/queries/browser_session_logs/upsert_browser_session_log.rs b/crates/lib/mysql_queries/src/queries/browser_session_logs/upsert_browser_session_log.rs new file mode 100644 index 0000000000..5580326c0b --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/browser_session_logs/upsert_browser_session_log.rs @@ -0,0 +1,66 @@ +use anyhow::anyhow; +use log::error; +use sqlx::{MySqlPool, QueryBuilder}; + +use errors::AnyhowResult; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; +use tokens::tokens::browser_session_logs::BrowserSessionLogToken; +use tokens::tokens::users::UserToken; + +pub struct UpsertBrowserSessionLogArgs<'a> { + pub maybe_log_token: Option<&'a BrowserSessionLogToken>, + + pub ip_address: &'a str, + pub maybe_user_token: Option<&'a UserToken>, + pub maybe_visitor_token: Option<&'a AnonymousVisitorTrackingToken>, + + pub maybe_last_action: Option<&'a str>, + + pub mysql_pool: &'a MySqlPool, +} + +pub async fn upsert_browser_session_log<'a>(args: UpsertBrowserSessionLogArgs<'a>) -> AnyhowResult { + + let token = args.maybe_log_token + .map(|token| token.clone()) + .unwrap_or_else(|| BrowserSessionLogToken::generate()); + + let maybe_last_action = args.maybe_last_action + .map(|action| { + let mut action = action.trim().to_string(); + action.truncate(32); // NB: Field is 32 wide. + action + }); + + let query_result = sqlx::query!( + r#" +INSERT INTO browser_session_logs +SET + token = ?, + ip_address = ?, + maybe_user_token = ?, + maybe_anonymous_visitor_token = ?, + maybe_last_action = ? +ON DUPLICATE KEY UPDATE + maybe_last_action = ?, + update_count = update_count + 1, + maybe_last_updated_at = NOW() + "#, + token.as_str(), + args.ip_address, + args.maybe_user_token.map(|t| t.as_str()), + args.maybe_visitor_token.map(|t| t.as_str()), + maybe_last_action.as_deref(), + maybe_last_action.as_deref(), + ) + .execute(args.mysql_pool) + .await; + + match query_result { + Ok(_) => Ok(token), + Err(err) => { + error!("Error with query: {:?}", &err); + Err(anyhow!("query error: {:?}", &err)) + }, + } +} diff --git a/crates/lib/mysql_queries/src/queries/mod.rs b/crates/lib/mysql_queries/src/queries/mod.rs index 39f968345e..46e996096c 100644 --- a/crates/lib/mysql_queries/src/queries/mod.rs +++ b/crates/lib/mysql_queries/src/queries/mod.rs @@ -3,6 +3,7 @@ pub mod audit_logs; pub mod batch_generations; pub mod beta_keys; pub mod billing; +pub mod browser_session_logs; pub mod comments; pub mod email_sender_jobs; pub mod entity_stats; diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 867ad798cd..33100afbbc 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -26,6 +26,7 @@ use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use tokens::tokens::batch_generations::*; use tokens::tokens::beta_keys::*; +use tokens::tokens::browser_session_logs::*; use tokens::tokens::comments::*; use tokens::tokens::generic_inference_jobs::*; use tokens::tokens::media_files::*; @@ -48,6 +49,7 @@ use crate::http_server::common_responses::pagination_cursors::PaginationCursors; use crate::http_server::common_responses::pagination_page::PaginationPage; use crate::http_server::common_responses::simple_entity_stats::SimpleEntityStats; use crate::http_server::common_responses::weights_cover_image_details::*; +use crate::http_server::endpoints::analytics::log_browser_session_handler::*; use crate::http_server::endpoints::beta_keys::create_beta_keys_handler::*; use crate::http_server::endpoints::beta_keys::list_beta_keys_handler::*; use crate::http_server::endpoints::beta_keys::redeem_beta_key_handler::*; @@ -118,6 +120,7 @@ use crate::http_server::web_utils::response_success_helpers::*; #[openapi( paths( billing_component::stripe::http_endpoints::checkout::create::stripe_create_checkout_session_json_handler::stripe_create_checkout_session_json_handler, + crate::http_server::endpoints::analytics::log_browser_session_handler::log_browser_session_handler, crate::http_server::endpoints::beta_keys::create_beta_keys_handler::create_beta_keys_handler, crate::http_server::endpoints::beta_keys::list_beta_keys_handler::list_beta_keys_handler, crate::http_server::endpoints::beta_keys::redeem_beta_key_handler::redeem_beta_key_handler, @@ -189,6 +192,7 @@ use crate::http_server::web_utils::response_success_helpers::*; // Tokens BatchGenerationToken, BetaKeyToken, + BrowserSessionLogToken, CommentToken, InferenceJobToken, MediaFileToken, @@ -334,11 +338,11 @@ use crate::http_server::web_utils::response_success_helpers::*; GetWeightError, GetWeightPathInfo, GetWeightResponse, - InferenceJobStatusResponsePayload, - InferenceJobTokenType, InferTtsError, InferTtsRequest, InferTtsSuccessResponse, + InferenceJobStatusResponsePayload, + InferenceJobTokenType, ListAvailableWeightsQuery, ListAvailableWeightsSuccessResponse, ListBetaKeysError, @@ -387,6 +391,9 @@ use crate::http_server::web_utils::response_success_helpers::*; ListWeightsByUserError, ListWeightsByUserPathInfo, ListWeightsByUserSuccessResponse, + LogBrowserSessionError, + LogBrowserSessionRequest, + LogBrowserSessionSuccessResponse, LoginErrorResponse, LoginErrorType, LoginRequest, @@ -397,8 +404,8 @@ use crate::http_server::web_utils::response_success_helpers::*; MediaFileForUserListItem, MediaFileInfo, MediaFileListItem, - MediaFilesByBatchListItem, MediaFileUploadError, + MediaFilesByBatchListItem, ModelWeightForList, ModelWeightSearchResult, PinnedMediaFile, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/analytics/log_browser_session_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/analytics/log_browser_session_handler.rs new file mode 100644 index 0000000000..1d257e4304 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/analytics/log_browser_session_handler.rs @@ -0,0 +1,161 @@ +use std::fmt; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, web}; +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use log::warn; +use utoipa::ToSchema; + +use crockford::crockford_entropy_lower; +use enums::by_table::beta_keys::beta_key_product::BetaKeyProduct; +use enums::by_table::comments::comment_entity_type::CommentEntityType; +use http_server_common::request::get_request_ip::get_request_ip; +use mysql_queries::queries::beta_keys::insert_batch_beta_keys::{insert_batch_beta_keys, InsertBatchArgs}; +use mysql_queries::queries::browser_session_logs::upsert_browser_session_log::{upsert_browser_session_log, UpsertBrowserSessionLogArgs}; +use mysql_queries::queries::comments::comment_entity_token::CommentEntityToken; +use mysql_queries::queries::comments::insert_comment::{insert_comment, InsertCommentArgs}; +use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::get_user_profile_by_username; +use tokens::tokens::browser_session_logs::BrowserSessionLogToken; +use tokens::tokens::comments::CommentToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::tts_models::TtsModelToken; +use tokens::tokens::tts_results::TtsResultToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::w2l_results::W2lResultToken; +use tokens::tokens::w2l_templates::W2lTemplateToken; +use user_input_common::check_for_slurs::contains_slurs; +use user_input_common::markdown_to_html::markdown_to_html; + +use crate::http_server::endpoints::moderation::user_feature_flags::edit_user_feature_flags_handler::EditUserFeatureFlagsError; +use crate::http_server::web_utils::require_moderator::{require_moderator, RequireModeratorError}; +use crate::http_server::web_utils::require_user_session::RequireUserSessionError; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; + +#[derive(Deserialize, ToSchema)] +pub struct LogBrowserSessionRequest { + /// Make the first request to this endpoint without a token. + /// The server will give you a token to use for every subsequent call. + /// Using this, we can calculate how sticky product usage is. + maybe_log_token: Option, + + /// OPTIONAL: Optionally include an up to 32-character identifier of what + /// action the user was last performing. + maybe_last_action: Option, +} + +#[derive(Serialize, ToSchema)] +pub struct LogBrowserSessionSuccessResponse { + pub success: bool, + + /// Store and use this token in subsequent requests. + /// If the user closes the browser, do not save it in session storage. + /// You can get a new token with each browser session. + pub log_token: BrowserSessionLogToken, +} + +#[derive(Debug, ToSchema)] +pub enum LogBrowserSessionError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for LogBrowserSessionError { + fn status_code(&self) -> StatusCode { + match *self { + LogBrowserSessionError::BadInput(_) => StatusCode::BAD_REQUEST, + LogBrowserSessionError::NotAuthorized => StatusCode::UNAUTHORIZED, + LogBrowserSessionError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + LogBrowserSessionError::BadInput(reason) => reason.to_string(), + LogBrowserSessionError::NotAuthorized => "unauthorized".to_string(), + LogBrowserSessionError::ServerError => "server error".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for LogBrowserSessionError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +/// Log a browser session (so we can calculate duration, etc.) +#[utoipa::path( + post, + tag = "Analytics", + path = "/v1/analytics/log_session", + responses( + (status = 200, description = "Success", body = LogBrowserSessionSuccessResponse), + (status = 400, description = "Bad input", body = LogBrowserSessionError), + (status = 401, description = "Not authorized", body = LogBrowserSessionError), + (status = 500, description = "Server error", body = LogBrowserSessionError), + ), + params( + ("request" = LogBrowserSessionRequest, description = "Payload for Request"), + ) +)] +pub async fn log_browser_session_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>, +) -> Result +{ + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + LogBrowserSessionError::ServerError + })?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + LogBrowserSessionError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + let ip_address = get_request_ip(&http_request); + + let token = upsert_browser_session_log(UpsertBrowserSessionLogArgs { + maybe_log_token: request.maybe_log_token.as_ref(), + ip_address: &ip_address, + maybe_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_visitor_token: maybe_avt_token.as_ref(), + maybe_last_action: request.maybe_last_action.as_deref(), + mysql_pool: &server_state.mysql_pool, + }).await.map_err(|err| { + warn!("Error inserting beta keys: {:?}", err); + LogBrowserSessionError::ServerError + })?; + + let response = LogBrowserSessionSuccessResponse { + success: true, + log_token: token, + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| LogBrowserSessionError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/analytics/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/analytics/mod.rs new file mode 100644 index 0000000000..cdaa741f37 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/analytics/mod.rs @@ -0,0 +1 @@ +pub mod log_browser_session_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index b4c05864a4..776fb479ad 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -35,3 +35,4 @@ pub mod voice_designer; pub mod w2l; pub mod weights; pub mod workflows; +pub mod analytics;