daniel543x/artcraft
1
0 Fork 0
mirror of https://github.com/storytold/artcraft.git synced 2026-11-08 19:11:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove legacy endpoint

Browse Source
This commit is contained in:
Brandon Thomas committed 2026-07-09 17:20:42 -04:00
1 parent ff5cac25b8
commit 724e59c35f
7 files changed
+1 -628
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

_metrics/datadog/dashboards/per_endpoint/POST__v1_media_files_write_scene_file.json
-267
View File
@@ -1,267 +0,0 @@
{
"title": "[POST] /v1/media_files/write/scene_file",
"description": "Auto-generated per-endpoint dashboard for `POST /v1/media_files/write/scene_file`.\n\nFilters all metrics on `service:storyteller-web,route:/v1/media_files/write/scene_file,method:post`.\n\nSource: `script/metrics/generate_endpoint_dashboards.py`. Do not hand-edit; changes will be overwritten on the next regeneration.",
"layout_type": "ordered",
"notify_list": [],
"reflow_type": "fixed",
"widgets": [
{
"definition": {
"type": "group",
"layout_type": "ordered",
"title": "Top-line health",
"show_title": true,
"widgets": [
{
"definition": {
"type": "query_value",
"title": "Requests/sec",
"title_size": "16",
"title_align": "left",
"precision": 2,
"autoscale": true,
"requests": [
{
"q": "sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}.as_rate()",
"aggregator": "avg"
}
]
},
"layout": {
"x": 0,
"y": 0,
"width": 3,
"height": 2
}
},
{
"definition": {
"type": "query_value",
"title": "5xx error rate (%)",
"title_size": "16",
"title_align": "left",
"precision": 2,
"autoscale": false,
"custom_unit": "%",
"requests": [
{
"q": "(sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post,status_class:5xx}.as_count() / sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}.as_count()) * 100",
"aggregator": "avg",
"conditional_formats": [
{
"comparator": ">",
"value": 5.0,
"palette": "white_on_red"
},
{
"comparator": ">",
"value": 1.0,
"palette": "white_on_yellow"
},
{
"comparator": "<=",
"value": 1.0,
"palette": "white_on_green"
}
]
}
]
},
"layout": {
"x": 3,
"y": 0,
"width": 3,
"height": 2
}
},
{
"definition": {
"type": "query_value",
"title": "Latency p95 (ms)",
"title_size": "16",
"title_align": "left",
"precision": 0,
"autoscale": true,
"custom_unit": "ms",
"requests": [
{
"q": "p95:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"aggregator": "avg",
"conditional_formats": [
{
"comparator": ">",
"value": 1000,
"palette": "white_on_red"
},
{
"comparator": ">",
"value": 250,
"palette": "white_on_yellow"
},
{
"comparator": "<=",
"value": 250,
"palette": "white_on_green"
}
]
}
]
},
"layout": {
"x": 6,
"y": 0,
"width": 3,
"height": 2
}
},
{
"definition": {
"type": "query_value",
"title": "Latency p99 (ms)",
"title_size": "16",
"title_align": "left",
"precision": 0,
"autoscale": true,
"custom_unit": "ms",
"requests": [
{
"q": "p99:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"aggregator": "avg"
}
]
},
"layout": {
"x": 9,
"y": 0,
"width": 3,
"height": 2
}
}
]
},
"layout": {
"x": 0,
"y": 0,
"width": 12,
"height": 3
}
},
{
"definition": {
"type": "group",
"layout_type": "ordered",
"title": "Trends",
"show_title": true,
"widgets": [
{
"definition": {
"type": "timeseries",
"title": "Requests/sec by status class",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_layout": "horizontal",
"requests": [
{
"q": "sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post} by {status_class}.as_rate()",
"display_type": "bars",
"style": {
"palette": "semantic",
"line_type": "solid",
"line_width": "normal"
}
}
],
"yaxis": {
"scale": "linear",
"include_zero": true
}
},
"layout": {
"x": 0,
"y": 0,
"width": 6,
"height": 4
}
},
{
"definition": {
"type": "timeseries",
"title": "Latency percentiles",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_layout": "horizontal",
"requests": [
{
"q": "p50:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"display_type": "line",
"style": {
"palette": "cool",
"line_type": "solid",
"line_width": "normal"
}
},
{
"q": "p95:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"display_type": "line",
"style": {
"palette": "warm",
"line_type": "solid",
"line_width": "normal"
}
},
{
"q": "p99:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"display_type": "line",
"style": {
"palette": "warm",
"line_type": "dashed",
"line_width": "normal"
}
}
],
"yaxis": {
"scale": "linear",
"include_zero": true,
"label": "ms"
}
},
"layout": {
"x": 6,
"y": 0,
"width": 6,
"height": 4
}
}
]
},
"layout": {
"x": 0,
"y": 3,
"width": 12,
"height": 5
}
},
{
"definition": {
"type": "toplist",
"title": "Status code distribution",
"title_size": "16",
"title_align": "left",
"requests": [
{
"q": "top(sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post} by {status_code}.as_count(), 25, 'sum', 'desc')"
}
]
},
"layout": {
"x": 0,
"y": 8,
"width": 12,
"height": 4
}
}
],
"id": "c8z-t2z-nk4"
}
crates/service/web/storyteller_web/src/docs/api_doc.rs
-3
View File
@@ -84,7 +84,6 @@ use crate::http_server::endpoints::omni_api::upload::omni_upload_image_media_fil
use crate::http_server::endpoints::omni_api::upload::omni_upload_video_media_file_handler::*;
use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::*;
use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*;
use crate::http_server::endpoints::model_download::enqueue_gptsovits_model_download_handler::*;
use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::*;
use artcraft_api_defs::moderation::debug_logs::debug_log_entry::*;
@@ -483,7 +482,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::media_files::upload::upload_video_new::upload_new_video_media_file_handler::upload_new_video_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_video_old::upload_video_media_file_handler::upload_video_media_file_handler,
crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler,
crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler,
crate::http_server::endpoints::model_download::enqueue_gptsovits_model_download_handler::enqueue_gptsovits_model_download_handler,
crate::http_server::endpoints::moderation::user_feature_flags::moderator_edit_user_feature_flags_handler::moderator_edit_user_feature_flags_handler,
crate::http_server::endpoints::moderation::user_feature_flags::moderator_list_all_available_user_feature_flags_handler::moderator_list_all_available_user_feature_flags_handler,
@@ -1211,7 +1209,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
Weight,
WeightsData,
WriteEngineAssetMediaSuccessResponse,
WriteSceneFileMediaSuccessResponse,
// Cost Estimate types
EstimateImageCostRequest,
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/mod.rs
+1 -2
View File
@@ -1,3 +1,2 @@
pub mod write_engine_asset;
pub mod write_error;
pub mod write_scene_file;
pub mod write_error;
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/write_scene_file/drain_multipart_request.rs
-94
View File
@@ -1,94 +0,0 @@
use actix_multipart::Multipart;
use actix_web::web::BytesMut;
use futures::TryStreamExt;
use log::warn;
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text};
pub struct MediaFileUploadData {
pub uuid_idempotency_token: Option<String>,
pub file_name: Option<String>,
pub file_bytes: Option<BytesMut>,
pub media_file_token: Option<MediaFileToken>,
// Optional: title of the scene (media_files' maybe_title)
pub maybe_title: Option<String>,
// Optional: visibility
pub maybe_visibility: Option<String>,
}
/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads.
pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> Result<MediaFileUploadData, MediaFileWriteError> {
let mut uuid_idempotency_token = None;
let mut file_bytes = None;
let mut file_name = None;
let mut media_file_token = None;
let mut title = None;
let mut visibility = None;
while let Ok(Some(mut field)) = multipart_payload.try_next().await {
let mut field_name = None;
let mut field_filename = None;
if let Some(content_disposition) = field.content_disposition() {
field_name = content_disposition.get_name()
.map(|s| s.to_string());
field_filename = content_disposition.get_filename() // NB: Only used for the file bytes.
.map(|s| s.to_string());
}
match field_name.as_deref() {
Some("uuid_idempotency_token") => {
uuid_idempotency_token = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading idempotency token: {:?}", &err);
MediaFileWriteError::BadInput("Error reading idempotency token".to_string())
})?;
},
Some("file") => {
file_name = field_filename.clone();
file_bytes = checked_read_multipart_bytes(&mut field).await
.map_err(|err| {
warn!("Error reading audio upload: {:?}", &err);
MediaFileWriteError::BadInput("Error reading file bytes".to_string())
})?;
},
Some("media_file_token") => {
media_file_token = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading source: {:?}", &err);
MediaFileWriteError::BadInput("Error reading media_file_token".to_string())
})?
.map(|field| MediaFileToken::new_from_str(&field));
},
Some("title") => {
title = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading title: {:}", &err);
MediaFileWriteError::BadInput("Error reading title".to_string())
})?;
},
Some("visibility") => {
visibility = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading visibility: {:}", &err);
MediaFileWriteError::BadInput("Error reading visibility".to_string())
})?;
},
_ => continue,
}
}
Ok(MediaFileUploadData {
uuid_idempotency_token,
file_name,
file_bytes,
media_file_token,
maybe_title: title,
maybe_visibility: visibility,
})
}
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/write_scene_file/mod.rs
-2
View File
@@ -1,2 +0,0 @@
pub mod drain_multipart_request;
pub mod write_scene_file_media_file_handler;
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/write_scene_file/write_scene_file_media_file_handler.rs
-255
View File
@@ -1,255 +0,0 @@
use std::path::PathBuf;
use std::sync::Arc;
use actix_multipart::Multipart;
use actix_web::{web, HttpRequest, HttpResponse};
use log::{error, info, warn};
use utoipa::ToSchema;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use mysql_queries::queries::media_files::upsert::upsert_media_file_from_file_upload::{upsert_media_file_from_file_upload, UploadType, UpsertMediaFileFromUploadArgs};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::drain_multipart_request::drain_multipart_request;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult};
// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*.
#[derive(Serialize, ToSchema)]
pub struct WriteSceneFileMediaSuccessResponse {
pub success: bool,
pub media_file_token: MediaFileToken,
}
/// DEPRECATED: Use the "new scene" and "saved scene" endpoints instead.
#[deprecated]
#[utoipa::path(
post,
tag = "Media Files [Deprecated]",
path = "/v1/media_files/write/scene_file",
responses(
(status = 200, description = "Success Update", body = WriteSceneFileMediaSuccessResponse),
(status = 400, description = "Bad input", body = MediaFileWriteError),
(status = 401, description = "Not authorized", body = MediaFileWriteError),
(status = 429, description = "Too many requests", body = MediaFileWriteError),
(status = 500, description = "Server error", body = MediaFileWriteError),
),
params(
("request" = (), description = "Ask Brandon. This is form-multipart."),
)
)]
pub async fn write_scene_file_media_file_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
mut multipart_payload: Multipart,
) -> Result<HttpResponse, MediaFileWriteError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await
.map_err(|err| {
error!("MySql pool error: {:?}", err);
MediaFileWriteError::ServerError
})?;
// ==================== READ SESSION ==================== //
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
error!("Session checker error: {:?}", e);
MediaFileWriteError::ServerError
})?;
let maybe_user_token = maybe_user_session
.as_ref()
.map(|session| session.get_user_token());
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.is_banned {
return Err(MediaFileWriteError::NotAuthorizedVerbose("user is banned".to_string()));
}
}
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.file_upload_logged_out,
Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in,
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request).await {
return Err(MediaFileWriteError::RateLimited);
}
// ==================== READ MULTIPART REQUEST ==================== //
let upload_media_request = drain_multipart_request(multipart_payload).await?;
// ==================== MAKE SURE USER OWNS FILE ==================== //
if let Some(media_file_token) = upload_media_request.media_file_token.as_ref() {
// TODO(bt,2024-03-26): Don't use the mysql_pool, use the mysql_connection.
let media_file =
get_media_file(media_file_token, false, &server_state.mysql_pool)
.await
.map_err(|err| {
error!("Error getting media file: {:?}", err);
MediaFileWriteError::ServerError
})?
.ok_or_else(|| MediaFileWriteError::NotFoundVerbose("media file not found with that token".to_string()))?;
let creator_check = check_creator_tokens(CheckCreatorTokenArgs {
maybe_creator_user_token: media_file.maybe_creator_user_token.as_ref(),
maybe_current_request_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: media_file.maybe_creator_anonymous_visitor_token.as_ref(),
maybe_current_request_anonymous_visitor_token: maybe_avt_token.as_ref(),
});
match creator_check {
CheckCreatorTokenResult::UserTokenMatch => {} // Allowed
CheckCreatorTokenResult::NoUserAnonymousVisitorTokenMatch => {} // Allowed
CheckCreatorTokenResult::InsufficientInformation => {} // TODO(bt,2024-03-28): Temporary fallthrough. This should be a 401.
CheckCreatorTokenResult::UserTokenMismatch => return Err(MediaFileWriteError::NotAuthorizedVerbose(
"user tokens do not match".to_string())),
CheckCreatorTokenResult::NoUserAnonymousVisitorTokenMismatch => return Err(MediaFileWriteError::NotAuthorizedVerbose(
"anonymous visitor tokens do not match".to_string())),
}
}
// ==================== HANDLE IDEMPOTENCY ==================== //
// TODO(bt, 2024-02-26): This should be a transaction.
let uuid_idempotency_token = upload_media_request.uuid_idempotency_token
.ok_or(MediaFileWriteError::BadInput("no uuid".to_string()))?;
if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) {
return Err(MediaFileWriteError::BadInput(reason));
}
insert_idempotency_token(&uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
MediaFileWriteError::BadInput("invalid idempotency token".to_string())
})?;
// ==================== UPLOAD METADATA ==================== //
let creator_set_visibility = maybe_user_session
.as_ref()
.map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control.
.unwrap_or(Visibility::default());
// ==================== USER DATA ==================== //
let ip_address = get_request_ip(&http_request);
// ==================== FILE DATA ==================== //
let file_bytes = match upload_media_request.file_bytes {
None => return Err(MediaFileWriteError::BadInput("missing file contents".to_string())),
Some(bytes) => bytes,
};
let maybe_filename = upload_media_request.file_name
.as_deref()
.map(|filename| PathBuf::from(filename));
let maybe_file_extension = maybe_filename
.as_ref()
.and_then(|filename| filename.extension())
.and_then(|ext| ext.to_str());
let file_size_bytes = file_bytes.len();
let hash = sha256_hash_bytes(&file_bytes)
.map_err(|io_error| {
error!("Problem hashing bytes: {:?}", io_error);
MediaFileWriteError::ServerError
})?;
// ==================== UPLOAD AND SAVE ==================== //
// TODO(bt,2024-03-26): At first I thought we should map these to the existing file paths on upsert,
// but now I'm thinking we can just lead cruft in the bucket and clean it later. We don't have the
// benefit of restoring old versions (if we mapped to existing paths but had a versioning scheme),
// but we can move fast.
const MIMETYPE: &str = "application/json";
const PREFIX : Option<&str> = Some("upload_");
const SUFFIX: &str = ".json";
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(SUFFIX));
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
server_state.public_bucket_client.upload_file_with_content_type(
public_upload_path.get_full_object_path_str(),
file_bytes.as_ref(),
MIMETYPE)
.await
.map_err(|e| {
warn!("Upload media bytes to bucket error: {:?}", e);
MediaFileWriteError::ServerError
})?;
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = upsert_media_file_from_file_upload(UpsertMediaFileFromUploadArgs {
maybe_media_file_token: upload_media_request.media_file_token.as_ref(),
maybe_media_class: Some(MediaFileClass::Dimensional),
media_file_type: MediaFileType::SceneJson,
maybe_engine_category: Some(MediaFileEngineCategory::Scene),
maybe_animation_type: None,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility,
upload_type: UploadType::Filesystem,
maybe_mime_type: Some(MIMETYPE),
file_size_bytes: file_size_bytes as u64,
duration_millis: 0,
sha256_checksum: &hash,
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: Some(SUFFIX),
pool: &server_state.mysql_pool,
})
.await
.map_err(|err| {
warn!("New file creation DB error: {:?}", err);
MediaFileWriteError::ServerError
})?;
info!("new media file id: {} token: {:?}", record_id, &token);
let response = WriteSceneFileMediaSuccessResponse {
success: true,
media_file_token: token,
};
let body = serde_json::to_string(&response)
.map_err(|e| MediaFileWriteError::ServerError)?;
return Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body));
}
crates/service/web/storyteller_web/src/http_server/routes/application_routes/media_files_routes.rs
-5
View File
@@ -34,7 +34,6 @@ use crate::http_server::endpoints::media_files::upload::upload_studio_shot::uplo
use crate::http_server::endpoints::media_files::upload::upload_video_new::upload_new_video_media_file_handler::upload_new_video_media_file_handler;
use crate::http_server::endpoints::media_files::upload::upload_video_old::upload_video_media_file_handler::upload_video_media_file_handler;
use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler;
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler;
pub fn add_media_file_routes<T, B> (app: App<T>) -> App<T>
where
@@ -138,10 +137,6 @@ pub fn add_media_file_routes<T, B> (app: App<T>) -> App<T>
.route(web::post().to(write_engine_asset_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/write/scene_file")
.route(web::post().to(write_scene_file_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/upload/new_engine_asset")
.route(web::post().to(upload_new_engine_asset_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))