From 2a101487b03fb9df6e50c97136cb9ddcb3cfa9b5 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 5 Mar 2026 21:22:02 -0500 Subject: [PATCH] find out if marketing is effective (#1107) NB: Ip address is to prevent spam/abuse --- ...8af73d37fd7118914d0ba12a02f53c1b84c78.json | 12 ++ .../artcraft/artcraft_api_defs/src/lib.rs | 3 +- .../src/web_referrals/log_web_referral.rs | 17 ++ .../src/web_referrals/mod.rs | 1 + .../database/mysql_queries/src/queries/mod.rs | 3 +- .../web_referrals/insert_web_referral.rs | 41 +++++ .../src/queries/web_referrals/mod.rs | 1 + .../web/storyteller_web/src/docs/api_doc.rs | 8 + .../avt_cookie_manager.rs | 7 +- .../avt_cookie_payload.rs | 7 + .../src/http_server/endpoints/mod.rs | 1 + .../web_referrals/log_web_referral_handler.rs | 152 ++++++++++++++++++ .../endpoints/web_referrals/mod.rs | 1 + .../add_application_routes.rs | 2 + .../routes/application_routes/mod.rs | 1 + .../web_referrals_routes.rs | 26 +++ 16 files changed, 280 insertions(+), 3 deletions(-) create mode 100644 .sqlx/query-80f73702374593c141d183d86bd8af73d37fd7118914d0ba12a02f53c1b84c78.json create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/log_web_referral.rs create mode 100644 crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/mod.rs create mode 100644 crates/schema/database/mysql_queries/src/queries/web_referrals/insert_web_referral.rs create mode 100644 crates/schema/database/mysql_queries/src/queries/web_referrals/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/log_web_referral_handler.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/routes/application_routes/web_referrals_routes.rs diff --git a/.sqlx/query-80f73702374593c141d183d86bd8af73d37fd7118914d0ba12a02f53c1b84c78.json b/.sqlx/query-80f73702374593c141d183d86bd8af73d37fd7118914d0ba12a02f53c1b84c78.json new file mode 100644 index 0000000000..322a19fbea --- /dev/null +++ b/.sqlx/query-80f73702374593c141d183d86bd8af73d37fd7118914d0ba12a02f53c1b84c78.json @@ -0,0 +1,12 @@ +{ + "db_name": "MySQL", + "query": "\n INSERT INTO web_referrals (\n url,\n domain,\n utm_source,\n utm_medium,\n utm_campaign,\n ip_address,\n anonymous_visitor_token\n ) VALUES (?, ?, ?, ?, ?, ?, ?)\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + }, + "hash": "80f73702374593c141d183d86bd8af73d37fd7118914d0ba12a02f53c1b84c78" +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs index 5330fe68c8..efc073248f 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/lib.rs @@ -9,4 +9,5 @@ pub mod prompts; pub mod stripe_artcraft; pub mod subscriptions; pub mod users; -pub mod utils; \ No newline at end of file +pub mod utils; +pub mod web_referrals; \ No newline at end of file diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/log_web_referral.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/log_web_referral.rs new file mode 100644 index 0000000000..c7d2e4616f --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/log_web_referral.rs @@ -0,0 +1,17 @@ +use serde_derive::{Deserialize, Serialize}; +use utoipa::ToSchema; + +pub const LOG_WEB_REFERRAL_URL_PATH: &str = "/v1/web_referrals/record"; + +#[derive(Deserialize, ToSchema)] +pub struct LogWebReferralRequest { + /// Optional: The referral URL the user arrived from when signing up. + /// The browser can send `document.referrer` to the backend so we know how people are finding us. + /// If the browser doesn't send this parameter, we'll try the `referer` header. + pub maybe_referral_url: Option, +} + +#[derive(Serialize, ToSchema)] +pub struct LogWebReferralResponse { + pub success: bool, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/mod.rs new file mode 100644 index 0000000000..ea486b9a8e --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/web_referrals/mod.rs @@ -0,0 +1 @@ +pub mod log_web_referral; diff --git a/crates/schema/database/mysql_queries/src/queries/mod.rs b/crates/schema/database/mysql_queries/src/queries/mod.rs index 06aa67391b..a51375e819 100644 --- a/crates/schema/database/mysql_queries/src/queries/mod.rs +++ b/crates/schema/database/mysql_queries/src/queries/mod.rs @@ -39,4 +39,5 @@ pub mod voice_conversion; pub mod voice_designer; pub mod w2l; pub mod wallet_ledger_entries; -pub mod wallets; \ No newline at end of file +pub mod wallets; +pub mod web_referrals; \ No newline at end of file diff --git a/crates/schema/database/mysql_queries/src/queries/web_referrals/insert_web_referral.rs b/crates/schema/database/mysql_queries/src/queries/web_referrals/insert_web_referral.rs new file mode 100644 index 0000000000..097483a1c0 --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/web_referrals/insert_web_referral.rs @@ -0,0 +1,41 @@ +use sqlx::{MySql, Pool}; + +pub struct InsertWebReferralArgs<'a> { + pub url: &'a str, + pub maybe_domain: Option<&'a str>, + pub utm_source: Option<&'a str>, + pub utm_medium: Option<&'a str>, + pub utm_campaign: Option<&'a str>, + pub ip_address: &'a str, + pub maybe_anonymous_visitor_token: Option<&'a str>, +} + +pub async fn insert_web_referral( + mysql_pool: &Pool, + args: InsertWebReferralArgs<'_>, +) -> Result<(), sqlx::Error> { + sqlx::query!( + r#" + INSERT INTO web_referrals ( + url, + domain, + utm_source, + utm_medium, + utm_campaign, + ip_address, + anonymous_visitor_token + ) VALUES (?, ?, ?, ?, ?, ?, ?) + "#, + args.url, + args.maybe_domain, + args.utm_source, + args.utm_medium, + args.utm_campaign, + args.ip_address, + args.maybe_anonymous_visitor_token, + ) + .execute(mysql_pool) + .await?; + + Ok(()) +} diff --git a/crates/schema/database/mysql_queries/src/queries/web_referrals/mod.rs b/crates/schema/database/mysql_queries/src/queries/web_referrals/mod.rs new file mode 100644 index 0000000000..ec511bb538 --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/web_referrals/mod.rs @@ -0,0 +1 @@ +pub mod insert_web_referral; diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index 402a40d6e1..4d9ac3e2d3 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -273,6 +273,8 @@ use artcraft_api_defs::moderation::wallet_ledger_entries::moderator_get_wallet_l use artcraft_api_defs::moderation::wallets::list_user_wallets::*; use artcraft_api_defs::moderation::wallets::moderator_add_banked_balance_to_wallet::*; use artcraft_api_defs::moderation::wallets::moderator_get_wallet::*; +use artcraft_api_defs::web_referrals::log_web_referral::*; +use crate::http_server::endpoints::web_referrals::log_web_referral_handler::*; use crate::http_server::endpoints::webhooks::fal_webhook_handler::*; use crate::http_server::endpoints::image_studio::update_gpt_image_job_status_handler::*; use crate::http_server::endpoints::voice_conversion::enqueue_seed_vc_inference_handler::*; @@ -454,6 +456,8 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux crate::http_server::endpoints::credits::get_session_credits_handler::get_session_credits_handler, // Subscriptions crate::http_server::endpoints::subscriptions::get_session_subscription_handler::get_session_subscription_handler, + // Web Referrals + crate::http_server::endpoints::web_referrals::log_web_referral_handler::log_web_referral_handler, // Webhooks crate::http_server::endpoints::webhooks::fal_webhook_handler::fal_webhook_handler, // Image Studio @@ -1043,6 +1047,10 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux GetSessionSubscriptionResponse, ListBatchGeneratedReduxMediaFilesPathInfo, ListBatchGeneratedReduxMediaFilesSuccessResponse, + + // Web Referrals + LogWebReferralRequest, + LogWebReferralResponse, )) )] pub struct ApiDoc; diff --git a/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_manager.rs b/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_manager.rs index 93b1f906a2..a81f24ae36 100644 --- a/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_manager.rs +++ b/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_manager.rs @@ -31,7 +31,12 @@ impl AvtCookieManager { } pub fn make_new_cookie(&self) -> AnyhowResult { - let payload = AvtCookiePayload::new(); + let token = AnonymousVisitorTrackingToken::generate(); + self.make_new_cookie_with_apriori_token(&token) + } + + pub fn make_new_cookie_with_apriori_token(&self, token: &AnonymousVisitorTrackingToken) -> AnyhowResult { + let payload = AvtCookiePayload::from_token(token.clone()); let claims = payload.to_map(); let jwt_string = self.jwt_signer.claims_to_jwt(&claims)?; diff --git a/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_payload.rs b/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_payload.rs index 0706d90590..0ed768f511 100644 --- a/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_payload.rs +++ b/crates/service/web/storyteller_web/src/http_server/cookies/anonymous_visitor_tracking/avt_cookie_payload.rs @@ -36,6 +36,13 @@ impl AvtCookiePayload { } } + pub fn from_token(avt_token: AnonymousVisitorTrackingToken) -> Self { + AvtCookiePayload { + avt_token, + cookie_version: COOKIE_VERSION, + } + } + pub fn from_map(map: BTreeMap) -> Result { let avt_token = map.get("avt_token") .ok_or_else(|| InternalError::VisitorCookieMissingField("avt_token"))?; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index 6a864a1cbd..f4d0df6a53 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -27,5 +27,6 @@ pub mod users; pub mod voice_clone_requests; pub mod voice_conversion; pub mod voice_designer; +pub mod web_referrals; pub mod webhooks; pub mod weights; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/log_web_referral_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/log_web_referral_handler.rs new file mode 100644 index 0000000000..f34b160ec6 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/log_web_referral_handler.rs @@ -0,0 +1,152 @@ +use std::sync::Arc; + +use actix_web::{web, HttpRequest, HttpResponse}; +use log::{info, warn}; +use url::Url; + +use artcraft_api_defs::web_referrals::log_web_referral::{LogWebReferralRequest, LogWebReferralResponse}; +use http_server_common::request::get_request_ip::get_request_ip; +use mysql_queries::queries::web_referrals::insert_web_referral::{insert_web_referral, InsertWebReferralArgs}; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::state::server_state::ServerState; + +/// Record a web referral for analytics tracking. +#[utoipa::path( + post, + tag = "Web Referrals", + path = "/v1/web_referrals/record", + responses( + (status = 200, description = "Success", body = LogWebReferralResponse), + (status = 500, description = "Server error"), + ), +)] +pub async fn log_web_referral_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>, +) -> Result { + + // 1. Try to get the referral URL from the request body, falling back to the Referer header. + let referral_url = request.maybe_referral_url.as_deref() + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()) + .or_else(|| { + http_request.headers().get("referer") + .and_then(|v| v.to_str().ok()) + .map(|s| s.to_string()) + }) + .filter(|s| !s.is_empty()); + + // 2. If no URL from either source, early exit 200 OK. + let referral_url = match referral_url { + Some(url) => url, + None => { + return ok_response(); + } + }; + + // 3. Parse the URL (infallible — we don't fail on bad URLs). + // NB: Parse before truncating so we don't lose UTM params. + let parsed = Url::parse(&referral_url).ok(); + + // Truncate the URL to 255 characters for storage. + let referral_url: String = referral_url.chars().take(255).collect(); + + // 4. Extract domain if URL parsed. + let maybe_domain = parsed.as_ref() + .and_then(|u| u.host_str()) + .map(|s| s.to_string()); + + // 5. Extract UTM parameters if URL parsed. + let utm_source = extract_query_param(parsed.as_ref(), "utm_source"); + let utm_medium = extract_query_param(parsed.as_ref(), "utm_medium"); + let utm_campaign = extract_query_param(parsed.as_ref(), "utm_campaign"); + + // 6. Grab the request IP address. + let ip_address = get_request_ip(&http_request); + + // 7. Check inbound cookies for AVT. If not present, mint a new one. + let existing_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + let (avt_token, maybe_new_avt_cookie) = match existing_avt_token { + Some(token) => (token, None), + None => { + let token = AnonymousVisitorTrackingToken::generate(); + let cookie = server_state + .avt_cookie_manager + .make_new_cookie_with_apriori_token(&token) + .map_err(|e| { + warn!("AVT cookie creation error: {:?}", e); + CommonWebError::ServerError + })?; + (token, Some(cookie)) + } + }; + + let avt_token_str = avt_token.to_string(); + + // Truncate UTM fields to fit the schema. + let utm_source = utm_source.map(|s| s.chars().take(32).collect::()); + let utm_medium = utm_medium.map(|s| s.chars().take(32).collect::()); + let utm_campaign = utm_campaign.map(|s| s.chars().take(255).collect::()); + + // Insert into the database. + insert_web_referral(&server_state.mysql_pool, InsertWebReferralArgs { + url: &referral_url, + maybe_domain: maybe_domain.as_deref(), + utm_source: utm_source.as_deref(), + utm_medium: utm_medium.as_deref(), + utm_campaign: utm_campaign.as_deref(), + ip_address: &ip_address, + maybe_anonymous_visitor_token: Some(&avt_token_str), + }).await.map_err(|e| { + warn!("Error inserting web referral: {:?}", e); + CommonWebError::ServerError + })?; + + info!("Recorded web referral: {}", referral_url); + + // Build response, setting the AVT cookie if we minted a new one. + let response = LogWebReferralResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|_| CommonWebError::ServerError)?; + + let mut response_builder = HttpResponse::Ok(); + + if let Some(cookie) = maybe_new_avt_cookie { + response_builder.cookie(cookie); + } + + Ok(response_builder + .content_type("application/json") + .body(body)) +} + +fn ok_response() -> Result { + let response = LogWebReferralResponse { + success: true, + }; + + let body = serde_json::to_string(&response) + .map_err(|_| CommonWebError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} + +fn extract_query_param(parsed: Option<&Url>, key: &str) -> Option { + parsed.and_then(|u| { + u.query_pairs() + .find(|(k, _)| k == key) + .map(|(_, v)| v.to_string()) + }) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/mod.rs new file mode 100644 index 0000000000..3d02c7f868 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/web_referrals/mod.rs @@ -0,0 +1 @@ +pub mod log_web_referral_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/add_application_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/add_application_routes.rs index 6fd2482086..865738356e 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/add_application_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/add_application_routes.rs @@ -14,6 +14,7 @@ use crate::http_server::routes::application_routes::user_bookmarks_routes::add_u use crate::http_server::routes::application_routes::user_rating_routes::add_user_rating_routes; use crate::http_server::routes::application_routes::user_routes::add_user_routes; use crate::http_server::routes::application_routes::wallet_routes::add_wallet_routes; +use crate::http_server::routes::application_routes::web_referrals_routes::add_web_referrals_routes; use crate::http_server::routes::application_routes::webhook_routes::add_webhook_routes; use crate::http_server::routes::application_routes::weights_routes::add_weights_routes; use actix_http::body::MessageBody; @@ -55,6 +56,7 @@ where // Other useful tools app = add_analytics_routes(app); // /v1/analytics/... + app = add_web_referrals_routes(app); // /v1/web_referrals/... // User and user-adjacent routes app = add_comments_routes(app); // /v1/comments/... diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/mod.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/mod.rs index 161119bb56..622f26c535 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/mod.rs @@ -14,6 +14,7 @@ mod user_bookmarks_routes; mod user_rating_routes; mod user_routes; mod wallet_routes; +mod web_referrals_routes; mod webhook_routes; mod weights_routes; pub (super) mod add_application_routes; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/web_referrals_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/web_referrals_routes.rs new file mode 100644 index 0000000000..814ffc4d11 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/web_referrals_routes.rs @@ -0,0 +1,26 @@ +use actix_http::body::MessageBody; +use actix_service::ServiceFactory; +use actix_web::dev::{ServiceRequest, ServiceResponse}; +use actix_web::error::Error; +use actix_web::{web, App, HttpResponse}; + +use crate::http_server::endpoints::web_referrals::log_web_referral_handler::log_web_referral_handler; + +pub fn add_web_referrals_routes (app: App) -> App +where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service(web::scope("/v1/web_referrals") + .service(web::resource("/record") + .route(web::post().to(log_web_referral_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) +}