From 45b7af9bd25082fb05bf287cb9b7a3e33809bc85 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 21 Dec 2022 02:32:56 -0500 Subject: [PATCH] detect upload mimetype --- Cargo.lock | 52 +++++++++++++++---- Cargo.toml | 3 +- .../media_uploads/insert_media_upload.rs | 39 ++++---------- crates/lib/files_common/Cargo.toml | 20 +++++++ crates/lib/files_common/src/lib.rs | 22 ++++++++ crates/lib/files_common/src/mimetype.rs | 13 +++++ crates/service/web/storyteller_web/Cargo.toml | 1 + .../endpoints/media_uploads/upload_audio.rs | 8 ++- .../enums/src/files/media_upload_type.rs | 1 + 9 files changed, 119 insertions(+), 40 deletions(-) create mode 100644 crates/lib/files_common/Cargo.toml create mode 100644 crates/lib/files_common/src/lib.rs create mode 100644 crates/lib/files_common/src/mimetype.rs diff --git a/Cargo.lock b/Cargo.lock index 1996857525..bf5b198dad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,7 @@ dependencies = [ "smol_str", "thiserror", "tokio 1.16.1", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -1040,7 +1040,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca453e8624711b2f0f4eb47076a318feda166252a827ee25d067b43de83dcba0" dependencies = [ "byteorder", - "uuid", + "uuid 0.8.2", +] + +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid 1.2.2", ] [[package]] @@ -1210,7 +1221,7 @@ dependencies = [ "serde_json", "tempfile", "toml", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -1734,6 +1745,13 @@ dependencies = [ "instant", ] +[[package]] +name = "files_common" +version = "0.0.1" +dependencies = [ + "infer 0.11.0", +] + [[package]] name = "firestorm" version = "0.4.6" @@ -2423,7 +2441,7 @@ dependencies = [ "tts_common", "twitch_common", "user_input_common", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -2448,7 +2466,16 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea70330449622910e0edebab230734569516269fb32342fb0a8956340fa48c6c" dependencies = [ - "cfb", + "cfb 0.4.0", +] + +[[package]] +name = "infer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6c16b11a665b26aeeb9b1d7f954cdeb034be38dd00adab4f2ae921a8fee804" +dependencies = [ + "cfb 0.7.3", ] [[package]] @@ -3022,7 +3049,7 @@ dependencies = [ "serde", "serde_json", "tokio 1.16.1", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -4530,6 +4557,7 @@ dependencies = [ "dotenv", "easyenv", "enums", + "files_common", "futures 0.1.31", "futures 0.3.17", "futures-util", @@ -4584,7 +4612,7 @@ dependencies = [ "url_config", "user_input_common", "users_component", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -5344,7 +5372,7 @@ dependencies = [ "twitch_api2", "twitch_common", "twitch_oauth2", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -5606,6 +5634,12 @@ dependencies = [ "getrandom 0.2.2", ] +[[package]] +name = "uuid" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c" + [[package]] name = "vcpkg" version = "0.2.12" @@ -5878,7 +5912,7 @@ dependencies = [ "twitch_api2", "twitch_common", "twitch_oauth2", - "uuid", + "uuid 0.8.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4c70aa854a..eb0241d11b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ members = [ "crates/lib/config", "crates/lib/container_common", "crates/lib/database_queries", + "crates/lib/files_common", "crates/lib/google_drive_common", "crates/lib/http_server_common", "crates/lib/jobs_common", @@ -28,7 +29,7 @@ members = [ # Types: common strongly-typed tokens, ids, payloads, etc. "crates/types/enums", - "crates/types/reusable_types", + "crates/types/reusable_types", # TODO(bt, 2022-12-21): Deprecate "crates/types/tokens", # Services: API servers, jobs, workers, etc. diff --git a/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs b/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs index 3fbd1ba57c..6119a93059 100644 --- a/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs +++ b/crates/lib/database_queries/src/queries/media_uploads/insert_media_upload.rs @@ -36,33 +36,6 @@ pub struct Args <'a> { } pub async fn insert_media_upload(args: Args<'_>) -> AnyhowResult { - - /* - maybe_original_filename = ?, - original_file_size_bytes = ?, - original_duration_millis = ?, - maybe_original_mime_type = ?, - maybe_original_audio_encoding = ?, - maybe_original_video_encoding = ?, - maybe_original_frame_width = ?, - maybe_original_frame_height = ?, - - - - INSERT INTO - media_uploads - SET - token = 'foo', - uuid_idempotency_token = 'bar', - public_bucket_directory_full_path = 'asdf', - extra_file_modification_info = 'asdf', - maybe_creator_user_token = 'asdf', - maybe_creator_anonymous_visitor_token = 'asdf', - creator_ip_address = 'asdf', - creator_set_visibility = 'asdf', - maybe_creator_synthetic_id = 'asdf' - - */ let query = sqlx::query!( r#" INSERT INTO media_uploads @@ -71,6 +44,14 @@ SET uuid_idempotency_token = ?, media_type = ?, + maybe_original_filename = ?, + original_file_size_bytes = ?, + original_duration_millis = ?, + maybe_original_mime_type = ?, + maybe_original_audio_encoding = ?, + maybe_original_video_encoding = ?, + maybe_original_frame_width = ?, + maybe_original_frame_height = ?, checksum_sha2 = ?, public_bucket_directory_full_path = ?, @@ -87,14 +68,14 @@ SET args.uuid_idempotency_token, args.media_type, - /*args.maybe_original_filename, + args.maybe_original_filename, args.original_file_size_bytes, args.original_duration_millis, args.maybe_original_mime_type, args.maybe_original_audio_encoding, args.maybe_original_video_encoding, args.maybe_original_frame_width, - args.maybe_original_frame_height,*/ + args.maybe_original_frame_height, args.checksum_sha2, args.public_bucket_directory_full_path, diff --git a/crates/lib/files_common/Cargo.toml b/crates/lib/files_common/Cargo.toml new file mode 100644 index 0000000000..07311ba06a --- /dev/null +++ b/crates/lib/files_common/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "files_common" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +name = "files_common" +path = "src/lib.rs" + +[dependencies] +# Mimetype detection +infer = "0.11" + +[dev-dependencies] +# None yet diff --git a/crates/lib/files_common/src/lib.rs b/crates/lib/files_common/src/lib.rs new file mode 100644 index 0000000000..a66b0885cb --- /dev/null +++ b/crates/lib/files_common/src/lib.rs @@ -0,0 +1,22 @@ +//! files +//! +//! The purpose of this library is to have a strongly-typed MySQL enum-type wrapper. +//! This should also work for CHAR/VARCHAR fields that work similarly to enums (typically +//! as part of a composite key) +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod mimetype; diff --git a/crates/lib/files_common/src/mimetype.rs b/crates/lib/files_common/src/mimetype.rs new file mode 100644 index 0000000000..0a11467925 --- /dev/null +++ b/crates/lib/files_common/src/mimetype.rs @@ -0,0 +1,13 @@ + +const DEFAULT_BINARY_MIMETYPE : &'static str = "application/octet-stream"; + +pub fn get_mimetype_for_bytes(bytes: &[u8]) -> Option<&'static str> { + infer::get(bytes) + .map(|typ| typ.mime_type()) +} + +pub fn get_mimetype_for_bytes_or_default(bytes: &[u8]) -> &'static str { + infer::get(bytes) + .map(|typ| typ.mime_type()) + .unwrap_or(DEFAULT_BINARY_MIMETYPE) +} diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index b591342d1b..da0f5abf30 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -20,6 +20,7 @@ config = { path = "../../../lib/config" } container_common = { path = "../../../lib/container_common" } database_queries = { path = "../../../lib/database_queries" } enums = { path = "../../../types/enums" } +files_common = { path = "../../../lib/files_common" } google_drive_common = { path = "../../../lib/google_drive_common" } http_server_common = { path = "../../../lib/http_server_common" } jobs_common = { path = "../../../lib/jobs_common" } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs index 47d2ac18f2..d2dbf95070 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs @@ -26,6 +26,7 @@ use sqlx::error::Error::Database; use sqlx::mysql::MySqlDatabaseError; use std::fmt; use std::sync::Arc; +use files_common::mimetype::{get_mimetype_for_bytes, get_mimetype_for_bytes_or_default}; use tokens::files::media_upload::MediaUploadToken; #[derive(Serialize)] @@ -132,6 +133,11 @@ pub async fn upload_audio_handler( let token = MediaUploadToken::generate(); + let maybe_mimetype = upload_media_request.file_bytes + .as_ref() + .map(|bytes| get_mimetype_for_bytes(bytes.as_ref())) + .flatten(); + let record_id = insert_media_upload(Args { token: &token, uuid_idempotency_token: &uuid_idempotency_token, @@ -139,7 +145,7 @@ pub async fn upload_audio_handler( maybe_original_filename: upload_media_request.file_name.as_deref(), original_file_size_bytes: 0, original_duration_millis: 0, - maybe_original_mime_type: None, + maybe_original_mime_type: maybe_mimetype, maybe_original_audio_encoding: None, maybe_original_video_encoding: None, maybe_original_frame_width: None, diff --git a/crates/types/enums/src/files/media_upload_type.rs b/crates/types/enums/src/files/media_upload_type.rs index 7fd56e5c9f..4e5d9d169d 100644 --- a/crates/types/enums/src/files/media_upload_type.rs +++ b/crates/types/enums/src/files/media_upload_type.rs @@ -19,6 +19,7 @@ pub enum MediaUploadType { Video, } +// TODO(bt, 2022-12-21): This desperately needs MySQL integration tests! impl_enum_display_and_debug_using_to_str!(MediaUploadType); impl_enum_sqlx_coders!(MediaUploadType);