daniel543x/artcraft-services
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add metadata to project files; remove legacy 3d editor endpoint (#1761)

Browse Source
This commit is contained in:
Brandon Thomas authored and GitHub committed 2026-07-09 17:23:09 -04:00
1 parent f50bfa9ee7
commit 98c4fad729
30 files changed
+69 -653
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.sqlx/query-7369b2f3fd9e7c1e968dc5501ef9bea7e1610fd7a044a0f9ad8b908dfada2b6a.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n media_class = ?,\n media_type = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n maybe_engine_category = ?,\n maybe_animation_type = ?,\n\n maybe_mime_type = ?,\n file_size_bytes = ?,\n\n checksum_sha2 = ?,\n\n maybe_prompt_token = ?,\n maybe_batch_token = ?,\n\n maybe_title = ?,\n maybe_duration_millis = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n\n maybe_scene_source_media_file_token = ?,\n\n is_intermediate_system_file = ?,\n is_user_upload = ?,\n\n maybe_generation_provider = ?,\n\n maybe_creator_file_synthetic_id = ?,\n maybe_creator_category_synthetic_id = ?,\n\n maybe_origin_model_type = NULL,\n maybe_origin_model_token = NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 27
},
"nullable": []
},
"hash": "7369b2f3fd9e7c1e968dc5501ef9bea7e1610fd7a044a0f9ad8b908dfada2b6a"
}
.sqlx/query-a25fe977f51329ed078e2441446ef7ff16683dcad8b81c8d0a0a4540b5e985bc.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\nINSERT INTO media_files\nSET\n token = ?,\n\n media_class = ?,\n media_type = ?,\n maybe_project_type = ?,\n\n origin_category = ?,\n origin_product_category = ?,\n\n maybe_engine_category = ?,\n maybe_animation_type = ?,\n\n maybe_mime_type = ?,\n file_size_bytes = ?,\n\n checksum_sha2 = ?,\n\n maybe_prompt_token = ?,\n maybe_batch_token = ?,\n\n maybe_title = ?,\n maybe_duration_millis = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_creator_user_token = ?,\n maybe_creator_anonymous_visitor_token = ?,\n creator_ip_address = ?,\n\n creator_set_visibility = ?,\n\n maybe_scene_source_media_file_token = ?,\n\n is_intermediate_system_file = ?,\n is_user_upload = ?,\n\n maybe_generation_provider = ?,\n\n maybe_creator_file_synthetic_id = ?,\n maybe_creator_category_synthetic_id = ?,\n\n maybe_origin_model_type = NULL,\n maybe_origin_model_token = NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 28
},
"nullable": []
},
"hash": "a25fe977f51329ed078e2441446ef7ff16683dcad8b81c8d0a0a4540b5e985bc"
}
.sqlx/query-ad311b4aa95a71b1dc37aa09c27850a0e963380c6e7780f47a9fc5a95ab70583.json
Generated
-12
View File
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "\n UPDATE media_files\n SET\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_mime_type = ?,\n file_size_bytes = ?,\n\n creator_ip_address = ?\n WHERE token = ?\n LIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 7
},
"nullable": []
},
"hash": "ad311b4aa95a71b1dc37aa09c27850a0e963380c6e7780f47a9fc5a95ab70583"
}
.sqlx/query-e33fa6280a39fff22b2e8c2731b7b6058b4705b5b557f34dba52a6440c22529b.json
Generated
+12
View File
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "\n UPDATE media_files\n SET\n media_class = ?,\n maybe_project_type = ?,\n\n public_bucket_directory_hash = ?,\n maybe_public_bucket_prefix = ?,\n maybe_public_bucket_extension = ?,\n\n maybe_mime_type = ?,\n file_size_bytes = ?,\n\n creator_ip_address = ?\n WHERE token = ?\n LIMIT 1\n ",
"describe": {
"columns": [],
"parameters": {
"Right": 9
},
"nullable": []
},
"hash": "e33fa6280a39fff22b2e8c2731b7b6058b4705b5b557f34dba52a6440c22529b"
}
_metrics/datadog/dashboards/per_endpoint/POST__v1_media_files_write_scene_file.json
-267
View File
@@ -1,267 +0,0 @@
{
"title": "[POST] /v1/media_files/write/scene_file",
"description": "Auto-generated per-endpoint dashboard for `POST /v1/media_files/write/scene_file`.\n\nFilters all metrics on `service:storyteller-web,route:/v1/media_files/write/scene_file,method:post`.\n\nSource: `script/metrics/generate_endpoint_dashboards.py`. Do not hand-edit; changes will be overwritten on the next regeneration.",
"layout_type": "ordered",
"notify_list": [],
"reflow_type": "fixed",
"widgets": [
{
"definition": {
"type": "group",
"layout_type": "ordered",
"title": "Top-line health",
"show_title": true,
"widgets": [
{
"definition": {
"type": "query_value",
"title": "Requests/sec",
"title_size": "16",
"title_align": "left",
"precision": 2,
"autoscale": true,
"requests": [
{
"q": "sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}.as_rate()",
"aggregator": "avg"
}
]
},
"layout": {
"x": 0,
"y": 0,
"width": 3,
"height": 2
}
},
{
"definition": {
"type": "query_value",
"title": "5xx error rate (%)",
"title_size": "16",
"title_align": "left",
"precision": 2,
"autoscale": false,
"custom_unit": "%",
"requests": [
{
"q": "(sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post,status_class:5xx}.as_count() / sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}.as_count()) * 100",
"aggregator": "avg",
"conditional_formats": [
{
"comparator": ">",
"value": 5.0,
"palette": "white_on_red"
},
{
"comparator": ">",
"value": 1.0,
"palette": "white_on_yellow"
},
{
"comparator": "<=",
"value": 1.0,
"palette": "white_on_green"
}
]
}
]
},
"layout": {
"x": 3,
"y": 0,
"width": 3,
"height": 2
}
},
{
"definition": {
"type": "query_value",
"title": "Latency p95 (ms)",
"title_size": "16",
"title_align": "left",
"precision": 0,
"autoscale": true,
"custom_unit": "ms",
"requests": [
{
"q": "p95:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"aggregator": "avg",
"conditional_formats": [
{
"comparator": ">",
"value": 1000,
"palette": "white_on_red"
},
{
"comparator": ">",
"value": 250,
"palette": "white_on_yellow"
},
{
"comparator": "<=",
"value": 250,
"palette": "white_on_green"
}
]
}
]
},
"layout": {
"x": 6,
"y": 0,
"width": 3,
"height": 2
}
},
{
"definition": {
"type": "query_value",
"title": "Latency p99 (ms)",
"title_size": "16",
"title_align": "left",
"precision": 0,
"autoscale": true,
"custom_unit": "ms",
"requests": [
{
"q": "p99:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"aggregator": "avg"
}
]
},
"layout": {
"x": 9,
"y": 0,
"width": 3,
"height": 2
}
}
]
},
"layout": {
"x": 0,
"y": 0,
"width": 12,
"height": 3
}
},
{
"definition": {
"type": "group",
"layout_type": "ordered",
"title": "Trends",
"show_title": true,
"widgets": [
{
"definition": {
"type": "timeseries",
"title": "Requests/sec by status class",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_layout": "horizontal",
"requests": [
{
"q": "sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post} by {status_class}.as_rate()",
"display_type": "bars",
"style": {
"palette": "semantic",
"line_type": "solid",
"line_width": "normal"
}
}
],
"yaxis": {
"scale": "linear",
"include_zero": true
}
},
"layout": {
"x": 0,
"y": 0,
"width": 6,
"height": 4
}
},
{
"definition": {
"type": "timeseries",
"title": "Latency percentiles",
"title_size": "16",
"title_align": "left",
"show_legend": true,
"legend_layout": "horizontal",
"requests": [
{
"q": "p50:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"display_type": "line",
"style": {
"palette": "cool",
"line_type": "solid",
"line_width": "normal"
}
},
{
"q": "p95:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"display_type": "line",
"style": {
"palette": "warm",
"line_type": "solid",
"line_width": "normal"
}
},
{
"q": "p99:http.server.request.duration_ms{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post}",
"display_type": "line",
"style": {
"palette": "warm",
"line_type": "dashed",
"line_width": "normal"
}
}
],
"yaxis": {
"scale": "linear",
"include_zero": true,
"label": "ms"
}
},
"layout": {
"x": 6,
"y": 0,
"width": 6,
"height": 4
}
}
]
},
"layout": {
"x": 0,
"y": 3,
"width": 12,
"height": 5
}
},
{
"definition": {
"type": "toplist",
"title": "Status code distribution",
"title_size": "16",
"title_align": "left",
"requests": [
{
"q": "top(sum:http.server.request.count{service:storyteller-web,route:/v1/media_files/write/scene_file,method:post} by {status_code}.as_count(), 25, 'sum', 'desc')"
}
]
},
"layout": {
"x": 0,
"y": 8,
"width": 12,
"height": 4
}
}
],
"id": "c8z-t2z-nk4"
}
crates/schema/database/mysql_queries/src/queries/media_files/create/specialized_insert/insert_media_file_from_file_upload.rs
+7
View File
@@ -8,6 +8,7 @@ use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_project_type::MediaFileProjectType;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::visibility::Visibility;
@@ -33,6 +34,9 @@ pub struct InsertMediaFileFromUploadArgs<'a> {
pub media_file_type: MediaFileType,
pub maybe_media_class: Option<MediaFileClass>,
/// Only set for internal Artcraft project documents (`media_class = 'project'`).
pub maybe_project_type: Option<MediaFileProjectType>,
pub upload_type: UploadType,
pub maybe_engine_category: Option<MediaFileEngineCategory>,
@@ -126,6 +130,7 @@ SET
media_class = ?,
media_type = ?,
maybe_project_type = ?,
origin_category = ?,
origin_product_category = ?,
@@ -175,6 +180,8 @@ SET
args.media_file_type.to_str(),
args.maybe_project_type.map(|project_type| project_type.to_str()),
origin_category.to_str(),
ORIGIN_PRODUCT_CATEGORY.to_str(),
crates/schema/database/mysql_queries/src/queries/media_files/edit/update_media_file_stored_cloud_contents.rs
+15
View File
@@ -1,5 +1,7 @@
use sqlx::MySqlPool;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_project_type::MediaFileProjectType;
use errors::AnyhowResult;
use tokens::tokens::media_files::MediaFileToken;
@@ -14,6 +16,13 @@ use tokens::tokens::media_files::MediaFileToken;
pub struct UpdateArgs<'a> {
pub media_file_token: &'a MediaFileToken,
/// The class the record should hold after the update. Also backfills legacy
/// records that were written before the current classification scheme.
pub media_class: MediaFileClass,
/// Only set for internal Artcraft project documents (`media_class = 'project'`).
pub maybe_project_type: Option<MediaFileProjectType>,
pub public_bucket_directory_hash: &'a str,
pub maybe_public_bucket_prefix: Option<&'a str>,
pub maybe_public_bucket_extension: Option<&'a str>,
@@ -37,6 +46,9 @@ pub async fn updated_media_file_stored_cloud_contents(args: UpdateArgs<'_>) -> A
r#"
UPDATE media_files
SET
media_class = ?,
maybe_project_type = ?,
public_bucket_directory_hash = ?,
maybe_public_bucket_prefix = ?,
maybe_public_bucket_extension = ?,
@@ -48,6 +60,9 @@ pub async fn updated_media_file_stored_cloud_contents(args: UpdateArgs<'_>) -> A
WHERE token = ?
LIMIT 1
"#,
args.media_class.to_str(),
args.maybe_project_type.map(|project_type| project_type.to_str()),
args.public_bucket_directory_hash,
args.maybe_public_bucket_prefix,
args.maybe_public_bucket_extension,
crates/service/web/storyteller_web/src/docs/api_doc.rs
-3
View File
@@ -84,7 +84,6 @@ use crate::http_server::endpoints::omni_api::upload::omni_upload_image_media_fil
use crate::http_server::endpoints::omni_api::upload::omni_upload_video_media_file_handler::*;
use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::*;
use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::*;
use crate::http_server::endpoints::model_download::enqueue_gptsovits_model_download_handler::*;
use artcraft_api_defs::generate::video::edit::beeble_switchx_edit_video::*;
use artcraft_api_defs::moderation::debug_logs::debug_log_entry::*;
@@ -483,7 +482,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
crate::http_server::endpoints::media_files::upload::upload_video_new::upload_new_video_media_file_handler::upload_new_video_media_file_handler,
crate::http_server::endpoints::media_files::upload::upload_video_old::upload_video_media_file_handler::upload_video_media_file_handler,
crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler,
crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler,
crate::http_server::endpoints::model_download::enqueue_gptsovits_model_download_handler::enqueue_gptsovits_model_download_handler,
crate::http_server::endpoints::moderation::user_feature_flags::moderator_edit_user_feature_flags_handler::moderator_edit_user_feature_flags_handler,
crate::http_server::endpoints::moderation::user_feature_flags::moderator_list_all_available_user_feature_flags_handler::moderator_list_all_available_user_feature_flags_handler,
@@ -1211,7 +1209,6 @@ use crate::http_server::endpoints::media_files::list::list_batch_generated_redux
Weight,
WeightsData,
WriteEngineAssetMediaSuccessResponse,
WriteSceneFileMediaSuccessResponse,
// Cost Estimate types
EstimateImageCostRequest,
crates/service/web/storyteller_web/src/http_server/deprecated_endpoints/engine/create_scene_handler.rs
+1
View File
@@ -166,6 +166,7 @@ pub async fn create_scene_handler(
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Dimensional),
maybe_project_type: None,
media_file_type: MediaFileType::SceneRon,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/deprecated_endpoints/engine/process_upload_media_file.rs
+1
View File
@@ -308,6 +308,7 @@ pub async fn process_upload_media_file(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: None,
maybe_project_type: None,
media_file_type,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/upload/upload_snapshot_media_file_handler.rs
+1
View File
@@ -244,6 +244,7 @@ pub async fn upload_snapshot_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Image),
maybe_project_type: None,
media_file_type,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_audio_media_file_handler.rs
+1
View File
@@ -335,6 +335,7 @@ pub async fn upload_audio_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Audio),
maybe_project_type: None,
media_file_type: MediaFileType::try_from_mime_type(&mimetype)
.or_else(|| maybe_upload_filename.and_then(MediaFileType::try_from_filename_or_extension))
.unwrap_or(MediaFileType::Audio), // Coarse fallback for unrecognized files
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_engine_asset/upload_engine_asset_media_file_handler.rs
+1
View File
@@ -205,6 +205,7 @@ pub async fn upload_engine_asset_media_file_handler(
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Dimensional),
maybe_project_type: None,
media_file_type,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_image_media_file_handler.rs
+1
View File
@@ -307,6 +307,7 @@ pub async fn upload_image_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Image),
maybe_project_type: None,
media_file_type,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_engine_asset_media_file_handler.rs
+1
View File
@@ -210,6 +210,7 @@ pub async fn upload_new_engine_asset_media_file_handler(
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(file_info.media_class),
maybe_project_type: None,
media_file_type: file_info.media_type,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_new_scene_media_file_handler.rs
+3 -1
View File
@@ -13,6 +13,7 @@ use utoipa::ToSchema;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_project_type::MediaFileProjectType;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
@@ -224,7 +225,8 @@ pub async fn upload_new_scene_media_file_handler(
})?;
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Dimensional),
maybe_media_class: Some(MediaFileClass::Project),
maybe_project_type: Some(MediaFileProjectType::Scene3d),
media_file_type: MediaFileType::SceneJson,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_pmx/upload_pmx_media_file_handler.rs
+1
View File
@@ -257,6 +257,7 @@ pub async fn upload_pmx_media_file_handler(
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Dimensional),
maybe_project_type: None,
media_file_type: MediaFileType::Pmx,
maybe_creator_user_token: Some(user_session.get_user_token()),
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_saved_scene_media_file_handler.rs
+4
View File
@@ -11,6 +11,8 @@ use log::{error, info, warn};
use utoipa::ToSchema;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_project_type::MediaFileProjectType;
use enums::common::visibility::Visibility;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
@@ -243,6 +245,8 @@ pub async fn upload_saved_scene_media_file_handler(
updated_media_file_stored_cloud_contents(UpdateArgs {
media_file_token: &path.token,
media_class: MediaFileClass::Project,
maybe_project_type: Some(MediaFileProjectType::Scene3d),
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: Some(SUFFIX),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_scene_snapshot_media_file_handler.rs
+1
View File
@@ -218,6 +218,7 @@ pub async fn upload_scene_snapshot_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Dimensional),
maybe_project_type: None,
media_file_type: MediaFileType::SceneJson,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_video_new/upload_new_video_media_file_handler.rs
+1
View File
@@ -508,6 +508,7 @@ pub async fn upload_new_video_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Video),
maybe_project_type: None,
media_file_type: MediaFileType::try_from_mime_type(&mimetype)
.or_else(|| maybe_upload_filename.and_then(MediaFileType::try_from_filename_or_extension))
.unwrap_or(MediaFileType::Video), // Coarse fallback for unrecognized files
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upload/upload_video_old/upload_video_media_file_handler.rs
+1
View File
@@ -221,6 +221,7 @@ pub async fn upload_video_media_file_handler(
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Video),
maybe_project_type: None,
media_file_type: MediaFileType::try_from_mime_type(mimetype)
.unwrap_or(MediaFileType::Video), // Coarse fallback for unrecognized mimes
maybe_creator_user_token: maybe_user_token,
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/mod.rs
+1 -2
View File
@@ -1,3 +1,2 @@
pub mod write_engine_asset;
pub mod write_error;
pub mod write_scene_file;
pub mod write_error;
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/write_scene_file/drain_multipart_request.rs
-94
View File
@@ -1,94 +0,0 @@
use actix_multipart::Multipart;
use actix_web::web::BytesMut;
use futures::TryStreamExt;
use log::warn;
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
use crate::http_server::web_utils::read_multipart_field_bytes::{checked_read_multipart_bytes, read_multipart_field_as_text};
pub struct MediaFileUploadData {
pub uuid_idempotency_token: Option<String>,
pub file_name: Option<String>,
pub file_bytes: Option<BytesMut>,
pub media_file_token: Option<MediaFileToken>,
// Optional: title of the scene (media_files' maybe_title)
pub maybe_title: Option<String>,
// Optional: visibility
pub maybe_visibility: Option<String>,
}
/// Pull common parts out of multipart media HTTP requests, typically for handling file uploads.
pub async fn drain_multipart_request(mut multipart_payload: Multipart) -> Result<MediaFileUploadData, MediaFileWriteError> {
let mut uuid_idempotency_token = None;
let mut file_bytes = None;
let mut file_name = None;
let mut media_file_token = None;
let mut title = None;
let mut visibility = None;
while let Ok(Some(mut field)) = multipart_payload.try_next().await {
let mut field_name = None;
let mut field_filename = None;
if let Some(content_disposition) = field.content_disposition() {
field_name = content_disposition.get_name()
.map(|s| s.to_string());
field_filename = content_disposition.get_filename() // NB: Only used for the file bytes.
.map(|s| s.to_string());
}
match field_name.as_deref() {
Some("uuid_idempotency_token") => {
uuid_idempotency_token = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading idempotency token: {:?}", &err);
MediaFileWriteError::BadInput("Error reading idempotency token".to_string())
})?;
},
Some("file") => {
file_name = field_filename.clone();
file_bytes = checked_read_multipart_bytes(&mut field).await
.map_err(|err| {
warn!("Error reading audio upload: {:?}", &err);
MediaFileWriteError::BadInput("Error reading file bytes".to_string())
})?;
},
Some("media_file_token") => {
media_file_token = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading source: {:?}", &err);
MediaFileWriteError::BadInput("Error reading media_file_token".to_string())
})?
.map(|field| MediaFileToken::new_from_str(&field));
},
Some("title") => {
title = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading title: {:}", &err);
MediaFileWriteError::BadInput("Error reading title".to_string())
})?;
},
Some("visibility") => {
visibility = read_multipart_field_as_text(&mut field).await
.map_err(|err| {
warn!("Error reading visibility: {:}", &err);
MediaFileWriteError::BadInput("Error reading visibility".to_string())
})?;
},
_ => continue,
}
}
Ok(MediaFileUploadData {
uuid_idempotency_token,
file_name,
file_bytes,
media_file_token,
maybe_title: title,
maybe_visibility: visibility,
})
}
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/write_scene_file/mod.rs
-2
View File
@@ -1,2 +0,0 @@
pub mod drain_multipart_request;
pub mod write_scene_file_media_file_handler;
crates/service/web/storyteller_web/src/http_server/endpoints/media_files/upsert_upload/write_scene_file/write_scene_file_media_file_handler.rs
-255
View File
@@ -1,255 +0,0 @@
use std::path::PathBuf;
use std::sync::Arc;
use actix_multipart::Multipart;
use actix_web::{web, HttpRequest, HttpResponse};
use log::{error, info, warn};
use utoipa::ToSchema;
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use enums::common::visibility::Visibility;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use http_server_common::request::get_request_ip::get_request_ip;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
use mysql_queries::queries::media_files::upsert::upsert_media_file_from_file_upload::{upsert_media_file_from_file_upload, UploadType, UpsertMediaFileFromUploadArgs};
use tokens::tokens::media_files::MediaFileToken;
use crate::http_server::endpoints::media_files::upsert_upload::write_error::MediaFileWriteError;
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::drain_multipart_request::drain_multipart_request;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::check_creator_tokens::{check_creator_tokens, CheckCreatorTokenArgs, CheckCreatorTokenResult};
// Unlike the "upload" endpoints, which are pure inserts, these endpoints are *upserts*.
#[derive(Serialize, ToSchema)]
pub struct WriteSceneFileMediaSuccessResponse {
pub success: bool,
pub media_file_token: MediaFileToken,
}
/// DEPRECATED: Use the "new scene" and "saved scene" endpoints instead.
#[deprecated]
#[utoipa::path(
post,
tag = "Media Files [Deprecated]",
path = "/v1/media_files/write/scene_file",
responses(
(status = 200, description = "Success Update", body = WriteSceneFileMediaSuccessResponse),
(status = 400, description = "Bad input", body = MediaFileWriteError),
(status = 401, description = "Not authorized", body = MediaFileWriteError),
(status = 429, description = "Too many requests", body = MediaFileWriteError),
(status = 500, description = "Server error", body = MediaFileWriteError),
),
params(
("request" = (), description = "Ask Brandon. This is form-multipart."),
)
)]
pub async fn write_scene_file_media_file_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
mut multipart_payload: Multipart,
) -> Result<HttpResponse, MediaFileWriteError> {
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await
.map_err(|err| {
error!("MySql pool error: {:?}", err);
MediaFileWriteError::ServerError
})?;
// ==================== READ SESSION ==================== //
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
error!("Session checker error: {:?}", e);
MediaFileWriteError::ServerError
})?;
let maybe_user_token = maybe_user_session
.as_ref()
.map(|session| session.get_user_token());
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// ==================== BANNED USERS ==================== //
if let Some(ref user) = maybe_user_session {
if user.is_banned {
return Err(MediaFileWriteError::NotAuthorizedVerbose("user is banned".to_string()));
}
}
// ==================== RATE LIMIT ==================== //
let rate_limiter = match maybe_user_session {
None => &server_state.redis_rate_limiters.file_upload_logged_out,
Some(ref _session) => &server_state.redis_rate_limiters.file_upload_logged_in,
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request).await {
return Err(MediaFileWriteError::RateLimited);
}
// ==================== READ MULTIPART REQUEST ==================== //
let upload_media_request = drain_multipart_request(multipart_payload).await?;
// ==================== MAKE SURE USER OWNS FILE ==================== //
if let Some(media_file_token) = upload_media_request.media_file_token.as_ref() {
// TODO(bt,2024-03-26): Don't use the mysql_pool, use the mysql_connection.
let media_file =
get_media_file(media_file_token, false, &server_state.mysql_pool)
.await
.map_err(|err| {
error!("Error getting media file: {:?}", err);
MediaFileWriteError::ServerError
})?
.ok_or_else(|| MediaFileWriteError::NotFoundVerbose("media file not found with that token".to_string()))?;
let creator_check = check_creator_tokens(CheckCreatorTokenArgs {
maybe_creator_user_token: media_file.maybe_creator_user_token.as_ref(),
maybe_current_request_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: media_file.maybe_creator_anonymous_visitor_token.as_ref(),
maybe_current_request_anonymous_visitor_token: maybe_avt_token.as_ref(),
});
match creator_check {
CheckCreatorTokenResult::UserTokenMatch => {} // Allowed
CheckCreatorTokenResult::NoUserAnonymousVisitorTokenMatch => {} // Allowed
CheckCreatorTokenResult::InsufficientInformation => {} // TODO(bt,2024-03-28): Temporary fallthrough. This should be a 401.
CheckCreatorTokenResult::UserTokenMismatch => return Err(MediaFileWriteError::NotAuthorizedVerbose(
"user tokens do not match".to_string())),
CheckCreatorTokenResult::NoUserAnonymousVisitorTokenMismatch => return Err(MediaFileWriteError::NotAuthorizedVerbose(
"anonymous visitor tokens do not match".to_string())),
}
}
// ==================== HANDLE IDEMPOTENCY ==================== //
// TODO(bt, 2024-02-26): This should be a transaction.
let uuid_idempotency_token = upload_media_request.uuid_idempotency_token
.ok_or(MediaFileWriteError::BadInput("no uuid".to_string()))?;
if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) {
return Err(MediaFileWriteError::BadInput(reason));
}
insert_idempotency_token(&uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
MediaFileWriteError::BadInput("invalid idempotency token".to_string())
})?;
// ==================== UPLOAD METADATA ==================== //
let creator_set_visibility = maybe_user_session
.as_ref()
.map(|user_session| user_session.preferred_tts_result_visibility) // TODO: We need a new type of visibility control.
.unwrap_or(Visibility::default());
// ==================== USER DATA ==================== //
let ip_address = get_request_ip(&http_request);
// ==================== FILE DATA ==================== //
let file_bytes = match upload_media_request.file_bytes {
None => return Err(MediaFileWriteError::BadInput("missing file contents".to_string())),
Some(bytes) => bytes,
};
let maybe_filename = upload_media_request.file_name
.as_deref()
.map(|filename| PathBuf::from(filename));
let maybe_file_extension = maybe_filename
.as_ref()
.and_then(|filename| filename.extension())
.and_then(|ext| ext.to_str());
let file_size_bytes = file_bytes.len();
let hash = sha256_hash_bytes(&file_bytes)
.map_err(|io_error| {
error!("Problem hashing bytes: {:?}", io_error);
MediaFileWriteError::ServerError
})?;
// ==================== UPLOAD AND SAVE ==================== //
// TODO(bt,2024-03-26): At first I thought we should map these to the existing file paths on upsert,
// but now I'm thinking we can just lead cruft in the bucket and clean it later. We don't have the
// benefit of restoring old versions (if we mapped to existing paths but had a versioning scheme),
// but we can move fast.
const MIMETYPE: &str = "application/json";
const PREFIX : Option<&str> = Some("upload_");
const SUFFIX: &str = ".json";
let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(SUFFIX));
info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str());
server_state.public_bucket_client.upload_file_with_content_type(
public_upload_path.get_full_object_path_str(),
file_bytes.as_ref(),
MIMETYPE)
.await
.map_err(|e| {
warn!("Upload media bytes to bucket error: {:?}", e);
MediaFileWriteError::ServerError
})?;
// TODO(bt, 2024-02-22): This should be a transaction.
let (token, record_id) = upsert_media_file_from_file_upload(UpsertMediaFileFromUploadArgs {
maybe_media_file_token: upload_media_request.media_file_token.as_ref(),
maybe_media_class: Some(MediaFileClass::Dimensional),
media_file_type: MediaFileType::SceneJson,
maybe_engine_category: Some(MediaFileEngineCategory::Scene),
maybe_animation_type: None,
maybe_creator_user_token: maybe_user_token,
maybe_creator_anonymous_visitor_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility,
upload_type: UploadType::Filesystem,
maybe_mime_type: Some(MIMETYPE),
file_size_bytes: file_size_bytes as u64,
duration_millis: 0,
sha256_checksum: &hash,
public_bucket_directory_hash: public_upload_path.get_object_hash(),
maybe_public_bucket_prefix: PREFIX,
maybe_public_bucket_extension: Some(SUFFIX),
pool: &server_state.mysql_pool,
})
.await
.map_err(|err| {
warn!("New file creation DB error: {:?}", err);
MediaFileWriteError::ServerError
})?;
info!("new media file id: {} token: {:?}", record_id, &token);
let response = WriteSceneFileMediaSuccessResponse {
success: true,
media_file_token: token,
};
let body = serde_json::to_string(&response)
.map_err(|e| MediaFileWriteError::ServerError)?;
return Ok(HttpResponse::Ok()
.content_type("application/json")
.body(body));
}
crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/shared_utils/media_uploads/upload_url_to_media_file.rs
+1
View File
@@ -110,6 +110,7 @@ pub async fn upload_url_to_media_file(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(kind.media_file_class()),
maybe_project_type: None,
media_file_type: kind.media_file_type(&mimetype),
maybe_creator_user_token: Some(user_token),
// NB: AVT (anonymous visitor) tokens are a web-session concept; API-key callers have none.
crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_audio_media_file_handler.rs
+1
View File
@@ -279,6 +279,7 @@ pub async fn omni_upload_audio_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Audio),
maybe_project_type: None,
media_file_type: MediaFileType::try_from_mime_type(&mimetype)
.or_else(|| maybe_upload_filename.and_then(MediaFileType::try_from_filename_or_extension))
.unwrap_or(MediaFileType::Audio), // Coarse fallback for unrecognized files
crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_image_media_file_handler.rs
+1
View File
@@ -267,6 +267,7 @@ pub async fn omni_upload_image_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Image),
maybe_project_type: None,
media_file_type,
maybe_creator_user_token: maybe_user_token,
// NB: AVT (anonymous visitor) tokens are a web-session concept; API-key callers have none.
crates/service/web/storyteller_web/src/http_server/endpoints/omni_api/upload/omni_upload_video_media_file_handler.rs
+1
View File
@@ -460,6 +460,7 @@ pub async fn omni_upload_video_media_file_handler(
let (token, record_id) = insert_media_file_from_file_upload(InsertMediaFileFromUploadArgs {
maybe_media_class: Some(MediaFileClass::Video),
maybe_project_type: None,
media_file_type: MediaFileType::try_from_mime_type(&mimetype)
.or_else(|| maybe_upload_filename.and_then(MediaFileType::try_from_filename_or_extension))
.unwrap_or(MediaFileType::Video), // Coarse fallback for unrecognized files
crates/service/web/storyteller_web/src/http_server/routes/application_routes/media_files_routes.rs
-5
View File
@@ -34,7 +34,6 @@ use crate::http_server::endpoints::media_files::upload::upload_studio_shot::uplo
use crate::http_server::endpoints::media_files::upload::upload_video_new::upload_new_video_media_file_handler::upload_new_video_media_file_handler;
use crate::http_server::endpoints::media_files::upload::upload_video_old::upload_video_media_file_handler::upload_video_media_file_handler;
use crate::http_server::endpoints::media_files::upsert_upload::write_engine_asset::write_engine_asset_media_file_handler::write_engine_asset_media_file_handler;
use crate::http_server::endpoints::media_files::upsert_upload::write_scene_file::write_scene_file_media_file_handler::write_scene_file_media_file_handler;
pub fn add_media_file_routes<T, B> (app: App<T>) -> App<T>
where
@@ -138,10 +137,6 @@ pub fn add_media_file_routes<T, B> (app: App<T>) -> App<T>
.route(web::post().to(write_engine_asset_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/write/scene_file")
.route(web::post().to(write_scene_file_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/upload/new_engine_asset")
.route(web::post().to(upload_new_engine_asset_media_file_handler))
.route(web::head().to(|| HttpResponse::Ok()))