consolidate upload handling, use new bucket paths

This commit is contained in:
Brandon Thomas committed 2023-01-11 00:33:40 -05:00
1 parent 651ed42b64
commit 8061880a83
10 files changed
+89 -40

No files matched your search

Generated
+1
View File
@@ -1504,6 +1504,7 @@ version = "0.0.1"
dependencies = [
"anyhow",
"base64 0.13.0",
"buckets",
"chrono",
"config",
"container_common",
@@ -60,6 +60,10 @@ CREATE TABLE media_uploads (
-- ========== UPLOAD, TRANSCODING, AND TRUNCATION DETAILS ==========
-- The hash for the bucket directory that contains the original upload
-- as well as any transcodings, downsamplings, etc.
public_bucket_directory_hash VARCHAR(32) NOT NULL,
-- TODO(bt, 2022-12-20): Before landing, rename this field be better and indicate this
-- is a *directory*. Add a second field to contain the default file name.
-- The directory this media is uploaded to will be exclusive for this file.
@@ -1,6 +1,8 @@
use crate::public::public_path::PublicPath;
use crate::util::hashed_directory_path_long_string::hashed_directory_path_long_string;
// TODO: Generate these from a macro.
// TODO: Use a central path registry for quick reference
const MEDIA_UPLOAD_DIRECTORY : &'static str = "/media";
@@ -30,6 +32,10 @@ impl MediaUploadDirectory {
pub fn get_directory_path_str(&self) -> &str {
&self.directory
}
pub fn get_object_hash(&self) -> &str {
&self.object_hash
}
}
#[cfg(test)]
@@ -47,4 +53,11 @@ mod tests {
let directory = MediaUploadDirectory::from_object_hash("foo");
assert_eq!(directory.get_directory_path_str(), "/media/f/o/foo");
}
#[test]
pub fn get_object_hash() {
let hash = "abcdefghijk";
let directory = MediaUploadDirectory::from_object_hash(hash);
assert_eq!(directory.get_object_hash(), hash);
}
}
@@ -3,6 +3,8 @@ use crate::public::public_path::PublicPath;
const ORIGINAL_FILE_BASENAME : &'static str = "original_contents.bin";
// TODO: Generate these from a macro.
/// The original user upload file.
/// It may have derivative files (down samples, crops, etc.) that live alongside it.
#[derive(Clone)]
@@ -25,13 +27,21 @@ impl MediaUploadOriginalFilePath {
}
}
pub fn get_basename() -> &'static str {
ORIGINAL_FILE_BASENAME
}
pub fn get_full_object_path_str(&self) -> &str {
&self.full_object_path
}
pub fn get_directory(&self) -> &MediaUploadDirectory {
&self.directory
}
pub fn get_object_hash(&self) -> &str {
&self.directory.get_object_hash()
}
pub fn get_basename() -> &'static str {
ORIGINAL_FILE_BASENAME
}
}
#[cfg(test)]
@@ -49,4 +59,18 @@ mod tests {
let file = MediaUploadOriginalFilePath::from_object_hash("foo");
assert_eq!(file.get_full_object_path_str(), "/media/f/o/foo/original_contents.bin");
}
#[test]
pub fn get_full_object_path_str_starts_with_directory() {
let file = MediaUploadOriginalFilePath::from_object_hash("abcdefghijk");
let full_path = file.get_full_object_path_str();
assert!(full_path.starts_with(file.get_directory().get_directory_path_str()))
}
#[test]
pub fn get_object_hash() {
let hash = "abcdefghijk";
let file = MediaUploadOriginalFilePath::from_object_hash(hash);
assert_eq!(file.get_object_hash(), hash);
}
}
+1
View File
@@ -15,6 +15,7 @@ path = "src/lib.rs"
[dependencies]
# Internal
buckets = { path = "../../_types/buckets" }
config = { path = "../config" }
container_common = { path = "../container_common" }
enums = { path = "../../_types/enums" }
@@ -1,4 +1,5 @@
use anyhow::anyhow;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use container_common::anyhow_result::AnyhowResult;
use enums::core::visibility::Visibility;
use enums::files::media_upload_type::MediaUploadType;
@@ -22,7 +23,7 @@ pub struct Args <'a> {
pub maybe_original_frame_height: Option<u64>,
pub checksum_sha2: &'a str,
pub public_bucket_directory_full_path: &'a str,
pub public_upload_path: &'a MediaUploadOriginalFilePath,
pub extra_file_modification_info: MediaUploadDetails,
pub maybe_creator_user_token: Option<&'a UserToken>,
@@ -54,6 +55,7 @@ SET
maybe_original_frame_height = ?,
checksum_sha2 = ?,
public_bucket_directory_hash = ?,
public_bucket_directory_full_path = ?,
extra_file_modification_info = ?,
@@ -78,7 +80,8 @@ SET
args.maybe_original_frame_height,
args.checksum_sha2,
args.public_bucket_directory_full_path,
args.public_upload_path.get_object_hash(),
args.public_upload_path.get_full_object_path_str(),
"", // TODO: args.extra_file_modification_info,
args.maybe_creator_user_token,
@@ -1,3 +1,2 @@
pub mod drain_multipart_request;
pub mod upload_audio;
pub mod upload_video;
pub mod upload_media;
@@ -34,17 +34,18 @@ use symphonia::core::formats::FormatOptions;
use symphonia::core::io::{MediaSourceStream, ReadOnlySource};
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use buckets::public::media_uploads::original_file::MediaUploadOriginalFilePath;
use media::decode_basic_audio_info::decode_basic_audio_info;
use tokens::files::media_upload::MediaUploadToken;
#[derive(Serialize)]
pub struct UploadAudioSuccessResponse {
pub struct UploadMediaSuccessResponse {
pub success: bool,
pub upload_token: MediaUploadToken,
}
#[derive(Debug, Serialize)]
pub enum UploadAudioError {
pub enum UploadMediaError {
BadInput(String),
NotAuthorized,
MustBeLoggedIn,
@@ -52,14 +53,14 @@ pub enum UploadAudioError {
RateLimited,
}
impl ResponseError for UploadAudioError {
impl ResponseError for UploadMediaError {
fn status_code(&self) -> StatusCode {
match *self {
UploadAudioError::BadInput(_) => StatusCode::BAD_REQUEST,
UploadAudioError::NotAuthorized => StatusCode::UNAUTHORIZED,
UploadAudioError::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
UploadAudioError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
UploadAudioError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
UploadMediaError::BadInput(_) => StatusCode::BAD_REQUEST,
UploadMediaError::NotAuthorized => StatusCode::UNAUTHORIZED,
UploadMediaError::MustBeLoggedIn => StatusCode::UNAUTHORIZED,
UploadMediaError::ServerError => StatusCode::INTERNAL_SERVER_ERROR,
UploadMediaError::RateLimited => StatusCode::TOO_MANY_REQUESTS,
}
}
@@ -69,17 +70,17 @@ impl ResponseError for UploadAudioError {
}
// NB: Not using derive_more::Display since Clion doesn't understand it.
impl fmt::Display for UploadAudioError {
impl fmt::Display for UploadMediaError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", self)
}
}
pub async fn upload_audio_handler(
pub async fn upload_media_handler(
http_request: HttpRequest,
server_state: web::Data<Arc<ServerState>>,
mut multipart_payload: Multipart,
) -> Result<HttpResponse, UploadAudioError> {
) -> Result<HttpResponse, UploadMediaError> {
// ==================== READ SESSION ==================== //
@@ -89,7 +90,7 @@ pub async fn upload_audio_handler(
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
UploadAudioError::ServerError
UploadMediaError::ServerError
})?;
// ==================== RATE LIMIT ==================== //
@@ -98,18 +99,18 @@ pub async fn upload_audio_handler(
None => &server_state.redis_rate_limiters.logged_out,
Some(ref user) => {
if user.is_banned {
return Err(UploadAudioError::NotAuthorized);
return Err(UploadMediaError::NotAuthorized);
}
&server_state.redis_rate_limiters.logged_in
},
};
if let Err(_err) = rate_limiter.rate_limit_request(&http_request) {
return Err(UploadAudioError::RateLimited);
return Err(UploadMediaError::RateLimited);
}
if let Err(_err) = server_state.redis_rate_limiters.model_upload.rate_limit_request(&http_request) {
return Err(UploadAudioError::RateLimited);
return Err(UploadMediaError::RateLimited);
}
// ==================== READ MULTIPART REQUEST ==================== //
@@ -118,15 +119,15 @@ pub async fn upload_audio_handler(
.await
.map_err(|e| {
// TODO: Error handling could be nicer.
UploadAudioError::BadInput("bad request".to_string())
UploadMediaError::BadInput("bad request".to_string())
})?;
let uuid_idempotency_token = upload_media_request.uuid_idempotency_token
.ok_or(UploadAudioError::BadInput("no uuid".to_string()))?;
.ok_or(UploadMediaError::BadInput("no uuid".to_string()))?;
if let Err(reason) = validate_idempotency_token_format(&uuid_idempotency_token) {
return Err(UploadAudioError::BadInput(reason));
return Err(UploadMediaError::BadInput(reason));
}
let creator_set_visibility = maybe_user_session
@@ -152,16 +153,16 @@ pub async fn upload_audio_handler(
.transpose()
.map_err(|io_error| {
warn!("Problem hashing bytes: {:?}", io_error);
return UploadAudioError::ServerError;
return UploadMediaError::ServerError;
})?;
let hash = match maybe_hash {
None => return Err(UploadAudioError::BadInput("invalid file".to_string())),
None => return Err(UploadMediaError::BadInput("invalid file".to_string())),
Some(hash) => hash,
};
let bytes = match upload_media_request.file_bytes {
None => return Err(UploadAudioError::BadInput("invalid file".to_string())),
None => return Err(UploadMediaError::BadInput("invalid file".to_string())),
Some(bytes) => bytes,
};
@@ -201,7 +202,7 @@ pub async fn upload_audio_handler(
None
).map_err(|e| {
warn!("file decoding error: {:?}", e);
UploadAudioError::BadInput("could not decode file".to_string())
UploadMediaError::BadInput("could not decode file".to_string())
})?;
maybe_duration_millis = basic_info.duration_millis;
@@ -213,10 +214,13 @@ pub async fn upload_audio_handler(
Some(m) => m,
None => {
warn!("Invalid mimetype: {:?}", maybe_mimetype);
return Err(UploadAudioError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype)))
return Err(UploadMediaError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype)))
},
};
let public_upload_path =
MediaUploadOriginalFilePath::from_object_hash("todo");
let record_id = insert_media_upload(Args {
token: &token,
uuid_idempotency_token: &uuid_idempotency_token,
@@ -230,7 +234,7 @@ pub async fn upload_audio_handler(
maybe_original_frame_width: None,
maybe_original_frame_height: None,
checksum_sha2: &hash,
public_bucket_directory_full_path: "",
public_upload_path: &public_upload_path,
extra_file_modification_info: MediaUploadDetails {}, // TODO
maybe_creator_user_token: maybe_user_token.as_ref(),
maybe_creator_anonymous_visitor_token: None,
@@ -242,7 +246,7 @@ pub async fn upload_audio_handler(
.await
.map_err(|err| {
warn!("New generic download creation DB error: {:?}", err);
UploadAudioError::ServerError
UploadMediaError::ServerError
})?;
info!("new media upload id: {}", record_id);
@@ -253,16 +257,16 @@ pub async fn upload_audio_handler(
.await
.map_err(|e| {
warn!("error publishing event: {:?}", e);
UploadAudioError::ServerError
UploadMediaError::ServerError
})?;
let response = UploadAudioSuccessResponse {
let response = UploadMediaSuccessResponse {
success: true,
upload_token: token,
};
let body = serde_json::to_string(&response)
.map_err(|e| UploadAudioError::ServerError)?;
.map_err(|e| UploadMediaError::ServerError)?;
Ok(HttpResponse::Ok()
.content_type("application/json")
@@ -21,7 +21,7 @@ use crate::http_server::endpoints::flags::design_refresh_flag::enable_design_ref
use crate::http_server::endpoints::investor_demo::disable_demo_mode_handler::disable_demo_mode_handler;
use crate::http_server::endpoints::investor_demo::enable_demo_mode_handler::enable_demo_mode_handler;
use crate::http_server::endpoints::leaderboard::get_leaderboard::leaderboard_handler;
use crate::http_server::endpoints::media_uploads::upload_audio::upload_audio_handler;
use crate::http_server::endpoints::media_uploads::upload_media::upload_media_handler;
use crate::http_server::endpoints::misc::default_route_404::default_route_404;
use crate::http_server::endpoints::misc::detect_locale_handler::detect_locale_handler;
use crate::http_server::endpoints::misc::enable_alpha_easy_handler::enable_alpha_easy_handler;
@@ -791,9 +791,9 @@ fn add_media_upload_routes<T, B> (app: App<T, B>) -> App<T, B>
InitError = (),
>,
{
app.service(web::scope("/v1/media_upload")
.service(web::resource("/upload_audio")
.route(web::post().to(upload_audio_handler))
app.service(web::scope("/v1/media_uploads")
.service(web::resource("/upload")
.route(web::post().to(upload_media_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
)