mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-15 22:32:13 +00:00
consolidate upload handling, use new bucket paths
This commit is contained in:
1 parent
651ed42b64
commit
8061880a83
10 files changed
+89
-40
No files matched your search
Generated
+1
@@ -1504,6 +1504,7 @@ version = "0.0.1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64 0.13.0",
|
||||
"buckets",
|
||||
"chrono",
|
||||
"config",
|
||||
"container_common",
|
||||
|
||||
@@ -60,6 +60,10 @@ CREATE TABLE media_uploads (
|
||||
|
||||
-- ========== UPLOAD, TRANSCODING, AND TRUNCATION DETAILS ==========
|
||||
|
||||
-- The hash for the bucket directory that contains the original upload
|
||||
-- as well as any transcodings, downsamplings, etc.
|
||||
public_bucket_directory_hash VARCHAR(32) NOT NULL,
|
||||
|
||||
-- TODO(bt, 2022-12-20): Before landing, rename this field be better and indicate this
|
||||
-- is a *directory*. Add a second field to contain the default file name.
|
||||
-- The directory this media is uploaded to will be exclusive for this file.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use crate::public::public_path::PublicPath;
|
||||
use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
|
||||
|
||||
// TODO: Generate these from a macro.
|
||||
|
||||
// TODO: Use a central path registry for quick reference
|
||||
const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media";
|
||||
|
||||
@@ -30,6 +32,10 @@ impl MediaUploadDirectory {
|
||||
pub fn get_directory_path_str(&self) -> &str {
|
||||
&self.directory
|
||||
}
|
||||
|
||||
pub fn get_object_hash(&self) -> &str {
|
||||
&self.object_hash
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -47,4 +53,11 @@ mod tests {
|
||||
let directory = MediaUploadDirectory::from_object_hash("foo");
|
||||
assert_eq!(directory.get_directory_path_str(), "/media/f/o/foo");
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn get_object_hash() {
|
||||
let hash = "abcdefghijk";
|
||||
let directory = MediaUploadDirectory::from_object_hash(hash);
|
||||
assert_eq!(directory.get_object_hash(), hash);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ use crate::public::public_path::PublicPath;
|
||||
|
||||
const ORIGINAL_FILE_BASENAME : &'static str = "original_contents.bin";
|
||||
|
||||
// TODO: Generate these from a macro.
|
||||
|
||||
/// The original user upload file.
|
||||
/// It may have derivative files (down samples, crops, etc.) that live alongside it.
|
||||
#[derive(Clone)]
|
||||
@@ -25,13 +27,21 @@ impl MediaUploadOriginalFilePath {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_basename() -> &'static str {
|
||||
ORIGINAL_FILE_BASENAME
|
||||
}
|
||||
|
||||
pub fn get_full_object_path_str(&self) -> &str {
|
||||
&self.full_object_path
|
||||
}
|
||||
|
||||
pub fn get_directory(&self) -> &MediaUploadDirectory {
|
||||
&self.directory
|
||||
}
|
||||
|
||||
pub fn get_object_hash(&self) -> &str {
|
||||
&self.directory.get_object_hash()
|
||||
}
|
||||
|
||||
pub fn get_basename() -> &'static str {
|
||||
ORIGINAL_FILE_BASENAME
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -49,4 +59,18 @@ mod tests {
|
||||
let file = MediaUploadOriginalFilePath::from_object_hash("foo");
|
||||
assert_eq!(file.get_full_object_path_str(), "/media/f/o/foo/original_contents.bin");
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn get_full_object_path_str_starts_with_directory() {
|
||||
let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk");
|
||||
let full_path = file.get_full_object_path_str();
|
||||
assert!(full_path.starts_with(file.get_directory().get_directory_path_str()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
pub fn get_object_hash() {
|
||||
let hash = "abcdefghijk";
|
||||
let file = MediaUploadOriginalFilePath::from_object_hash(hash);
|
||||
assert_eq!(file.get_object_hash(), hash);
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ path = "src/lib.rs"
|
||||
[dependencies]
|
||||
|
||||
# Internal
|
||||
buckets = { path = "../../_types/buckets" }
|
||||
config = { path = "../config" }
|
||||
container_common = { path = "../container_common" }
|
||||
enums = { path = "../../_types/enums" }
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use anyhow::anyhow;
|
||||
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
|
||||
use container_common::anyhow_result::AnyhowResult;
|
||||
use enums::core::visibility::Visibility;
|
||||
use enums::files::media_upload_type::MediaUploadType;
|
||||
@@ -22,7 +23,7 @@ pub struct Args <'a> {
|
||||
pub maybe_original_frame_height: Option<u64>,
|
||||
pub checksum_sha2: &'a str,
|
||||
|
||||
pub public_bucket_directory_full_path: &'a str,
|
||||
pub public_upload_path: &'a MediaUploadOriginalFilePath,
|
||||
pub extra_file_modification_info: MediaUploadDetails,
|
||||
|
||||
pub maybe_creator_user_token: Option<&'a UserToken>,
|
||||
@@ -54,6 +55,7 @@ SET
|
||||
maybe_original_frame_height = ?,
|
||||
checksum_sha2 = ?,
|
||||
|
||||
public_bucket_directory_hash = ?,
|
||||
public_bucket_directory_full_path = ?,
|
||||
extra_file_modification_info = ?,
|
||||
|
||||
@@ -78,7 +80,8 @@ SET
|
||||
args.maybe_original_frame_height,
|
||||
args.checksum_sha2,
|
||||
|
||||
args.public_bucket_directory_full_path,
|
||||
args.public_upload_path.get_object_hash(),
|
||||
args.public_upload_path.get_full_object_path_str(),
|
||||
"", // TODO: args.extra_file_modification_info,
|
||||
|
||||
args.maybe_creator_user_token,
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
pub mod drain_multipart_request;
|
||||
pub mod upload_audio;
|
||||
pub mod upload_video;
|
||||
pub mod upload_media;
|
||||
+32
-28
@@ -34,17 +34,18 @@ use symphonia::core::formats::FormatOptions;
|
||||
use symphonia::core::io::{MediaSourceStream, ReadOnlySource};
|
||||
use symphonia::core::meta::MetadataOptions;
|
||||
use symphonia::core::probe::Hint;
|
||||
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
|
||||
use media::decode_basic_audio_info::decode_basic_audio_info;
|
||||
use tokens::files::media_upload::MediaUploadToken;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct UploadAudioSuccessResponse {
|
||||
pub struct UploadMediaSuccessResponse {
|
||||
pub success: bool,
|
||||
pub upload_token: MediaUploadToken,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub enum UploadAudioError {
|
||||
pub enum UploadMediaError {
|
||||
BadInput(String),
|
||||
NotAuthorized,
|
||||
MustBeLoggedIn,
|
||||
@@ -52,14 +53,14 @@ pub enum UploadAudioError {
|
||||
RateLimited,
|
||||
}
|
||||
|
||||
impl ResponseError for UploadAudioError {
|
||||
impl ResponseError for UploadMediaError {
|
||||
fn status_code(&self) -> StatusCode {
|
||||
match *self {
|
||||
UploadAudioError::BadInput(_) => StatusCode::BAD_REQUEST,
|
||||
UploadAudioError::NotAuthorized => StatusCode::UNAUTHORIZED,
|
||||
UploadAudioError::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
|
||||
UploadAudioError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
UploadAudioError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
|
||||
UploadMediaError::BadInput(_) => StatusCode::BAD_REQUEST,
|
||||
UploadMediaError::NotAuthorized => StatusCode::UNAUTHORIZED,
|
||||
UploadMediaError::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
|
||||
UploadMediaError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
UploadMediaError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,17 +70,17 @@ impl ResponseError for UploadAudioError {
|
||||
}
|
||||
|
||||
// NB: Not using derive_more::Display since Clion doesn't understand it.
|
||||
impl fmt::Display for UploadAudioError {
|
||||
impl fmt::Display for UploadMediaError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{:?}", self)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn upload_audio_handler(
|
||||
pub async fn upload_media_handler(
|
||||
http_request: HttpRequest,
|
||||
server_state: web::Data<Arc<ServerState>>,
|
||||
mut multipart_payload: Multipart,
|
||||
) -> Result<HttpResponse, UploadAudioError> {
|
||||
) -> Result<HttpResponse, UploadMediaError> {
|
||||
|
||||
// ==================== READ SESSION ==================== //
|
||||
|
||||
@@ -89,7 +90,7 @@ pub async fn upload_audio_handler(
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("Session checker error: {:?}", e);
|
||||
UploadAudioError::ServerError
|
||||
UploadMediaError::ServerError
|
||||
})?;
|
||||
|
||||
// ==================== RATE LIMIT ==================== //
|
||||
@@ -98,18 +99,18 @@ pub async fn upload_audio_handler(
|
||||
None => &server_state.redis_rate_limiters.logged_out,
|
||||
Some(ref user) => {
|
||||
if user.is_banned {
|
||||
return Err(UploadAudioError::NotAuthorized);
|
||||
return Err(UploadMediaError::NotAuthorized);
|
||||
}
|
||||
&server_state.redis_rate_limiters.logged_in
|
||||
},
|
||||
};
|
||||
|
||||
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
|
||||
return Err(UploadAudioError::RateLimited);
|
||||
return Err(UploadMediaError::RateLimited);
|
||||
}
|
||||
|
||||
if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) {
|
||||
return Err(UploadAudioError::RateLimited);
|
||||
return Err(UploadMediaError::RateLimited);
|
||||
}
|
||||
|
||||
// ==================== READ MULTIPART REQUEST ==================== //
|
||||
@@ -118,15 +119,15 @@ pub async fn upload_audio_handler(
|
||||
.await
|
||||
.map_err(|e| {
|
||||
// TODO: Error handling could be nicer.
|
||||
UploadAudioError::BadInput("bad request".to_string())
|
||||
UploadMediaError::BadInput("bad request".to_string())
|
||||
})?;
|
||||
|
||||
|
||||
let uuid_idempotency_token = upload_media_request.uuid_idempotency_token
|
||||
.ok_or(UploadAudioError::BadInput("no uuid".to_string()))?;
|
||||
.ok_or(UploadMediaError::BadInput("no uuid".to_string()))?;
|
||||
|
||||
if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) {
|
||||
return Err(UploadAudioError::BadInput(reason));
|
||||
return Err(UploadMediaError::BadInput(reason));
|
||||
}
|
||||
|
||||
let creator_set_visibility = maybe_user_session
|
||||
@@ -152,16 +153,16 @@ pub async fn upload_audio_handler(
|
||||
.transpose()
|
||||
.map_err(|io_error| {
|
||||
warn!("Problem hashing bytes: {:?}", io_error);
|
||||
return UploadAudioError::ServerError;
|
||||
return UploadMediaError::ServerError;
|
||||
})?;
|
||||
|
||||
let hash = match maybe_hash {
|
||||
None => return Err(UploadAudioError::BadInput("invalid file".to_string())),
|
||||
None => return Err(UploadMediaError::BadInput("invalid file".to_string())),
|
||||
Some(hash) => hash,
|
||||
};
|
||||
|
||||
let bytes = match upload_media_request.file_bytes {
|
||||
None => return Err(UploadAudioError::BadInput("invalid file".to_string())),
|
||||
None => return Err(UploadMediaError::BadInput("invalid file".to_string())),
|
||||
Some(bytes) => bytes,
|
||||
};
|
||||
|
||||
@@ -201,7 +202,7 @@ pub async fn upload_audio_handler(
|
||||
None
|
||||
).map_err(|e| {
|
||||
warn!("file decoding error: {:?}", e);
|
||||
UploadAudioError::BadInput("could not decode file".to_string())
|
||||
UploadMediaError::BadInput("could not decode file".to_string())
|
||||
})?;
|
||||
|
||||
maybe_duration_millis = basic_info.duration_millis;
|
||||
@@ -213,10 +214,13 @@ pub async fn upload_audio_handler(
|
||||
Some(m) => m,
|
||||
None => {
|
||||
warn!("Invalid mimetype: {:?}", maybe_mimetype);
|
||||
return Err(UploadAudioError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype)))
|
||||
return Err(UploadMediaError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype)))
|
||||
},
|
||||
};
|
||||
|
||||
let public_upload_path =
|
||||
MediaUploadOriginalFilePath::from_object_hash("todo");
|
||||
|
||||
let record_id = insert_media_upload(Args {
|
||||
token: &token,
|
||||
uuid_idempotency_token: &uuid_idempotency_token,
|
||||
@@ -230,7 +234,7 @@ pub async fn upload_audio_handler(
|
||||
maybe_original_frame_width: None,
|
||||
maybe_original_frame_height: None,
|
||||
checksum_sha2: &hash,
|
||||
public_bucket_directory_full_path: "",
|
||||
public_upload_path: &public_upload_path,
|
||||
extra_file_modification_info: MediaUploadDetails {}, // TODO
|
||||
maybe_creator_user_token: maybe_user_token.as_ref(),
|
||||
maybe_creator_anonymous_visitor_token: None,
|
||||
@@ -242,7 +246,7 @@ pub async fn upload_audio_handler(
|
||||
.await
|
||||
.map_err(|err| {
|
||||
warn!("New generic download creation DB error: {:?}", err);
|
||||
UploadAudioError::ServerError
|
||||
UploadMediaError::ServerError
|
||||
})?;
|
||||
|
||||
info!("new media upload id: {}", record_id);
|
||||
@@ -253,16 +257,16 @@ pub async fn upload_audio_handler(
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!("error publishing event: {:?}", e);
|
||||
UploadAudioError::ServerError
|
||||
UploadMediaError::ServerError
|
||||
})?;
|
||||
|
||||
let response = UploadAudioSuccessResponse {
|
||||
let response = UploadMediaSuccessResponse {
|
||||
success: true,
|
||||
upload_token: token,
|
||||
};
|
||||
|
||||
let body = serde_json::to_string(&response)
|
||||
.map_err(|e| UploadAudioError::ServerError)?;
|
||||
.map_err(|e| UploadMediaError::ServerError)?;
|
||||
|
||||
Ok(HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
Whitespace-only changes.
@@ -21,7 +21,7 @@ use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_ref
|
||||
use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler;
|
||||
use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler;
|
||||
use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler;
|
||||
use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler;
|
||||
use crate::http_server::endpoints::media_uploads::upload_media::upload_media_handler;
|
||||
use crate::http_server::endpoints::misc::default_route_404::default_route_404;
|
||||
use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler;
|
||||
use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler;
|
||||
@@ -791,9 +791,9 @@ fn add_media_upload_routes<T, B> (app: App<T, B>) -> App<T, B>
|
||||
InitError = (),
|
||||
>,
|
||||
{
|
||||
app.service(web::scope("/v1/media_upload")
|
||||
.service(web::resource("/upload_audio")
|
||||
.route(web::post().to(upload_audio_handler))
|
||||
app.service(web::scope("/v1/media_uploads")
|
||||
.service(web::resource("/upload")
|
||||
.route(web::post().to(upload_media_handler))
|
||||
.route(web::head().to(|| HttpResponse::Ok()))
|
||||
)
|
||||
)
|
||||
|
||||
Reference in new issue
Block a user