mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-05 17:58:55 +00:00
upload
This commit is contained in:
1 parent
63bc5f7878
commit
d26a45f7f7
6 files changed
+113
No files matched your search
@@ -41,6 +41,7 @@ artcraft_router.workspace = true
|
||||
sha1_hash.workspace = true
|
||||
video_info.workspace = true
|
||||
bootstrap.workspace = true
|
||||
bucket_client.workspace = true
|
||||
bucket_paths.workspace = true
|
||||
cloud_storage.workspace = true
|
||||
collections.workspace = true
|
||||
|
||||
+44
@@ -17,6 +17,7 @@ use log::{error, warn};
|
||||
use utoipa::ToSchema;
|
||||
|
||||
use artcraft_api_defs::video_info::upload::VideoInfoUploadResponse;
|
||||
use bucket_client::UploadFileBytesArgs;
|
||||
use ffmpeg_utils::ffprobe::ffprobe_get_dimensions::ffprobe_get_dimensions;
|
||||
use http_server_common::request::get_request_ip::get_request_ip;
|
||||
use mysql_queries::queries::uploaded_videos::get_uploaded_video_by_sha1_checksum::{
|
||||
@@ -105,6 +106,23 @@ pub async fn video_info_upload_handler(
|
||||
|
||||
let provenance = to_provenance(&parse_result, maybe_encoder);
|
||||
|
||||
// Archive the original bytes to the Seedance video bucket, if configured.
|
||||
// Done before moving `bytes` into the ffprobe task, and before acquiring a DB
|
||||
// connection. Fail-soft: a storage error must not fail provenance detection.
|
||||
if let Some(bucket) = server_state.seedance_video_bucket.as_ref() {
|
||||
let object_name = seedance_video_object_name(&sha1_checksum);
|
||||
if let Err(err) = bucket
|
||||
.upload_file_bytes(UploadFileBytesArgs {
|
||||
object_name: object_name.as_str(),
|
||||
bytes: &bytes,
|
||||
content_type: Some("video/mp4"),
|
||||
})
|
||||
.await
|
||||
{
|
||||
warn!("seedance video bucket upload failed for {}: {:?}", object_name, err);
|
||||
}
|
||||
}
|
||||
|
||||
// ffprobe is a blocking subprocess — run it off the async executor, and BEFORE
|
||||
// acquiring a DB connection (never hold a pooled connection across this).
|
||||
//
|
||||
@@ -180,6 +198,19 @@ pub async fn video_info_upload_handler(
|
||||
}))
|
||||
}
|
||||
|
||||
/// Object key for archiving an uploaded video, sharded into four directory
|
||||
/// levels by the first four characters of the checksum:
|
||||
/// `uploads/{c0}/{c1}/{c2}/{c3}/{checksum}.mp4`
|
||||
/// (e.g. checksum `f0a2cda0…` → `uploads/f/0/a/2/f0a2cda0….mp4`).
|
||||
fn seedance_video_object_name(sha1_checksum: &str) -> String {
|
||||
let shard: String = sha1_checksum
|
||||
.chars()
|
||||
.take(4)
|
||||
.map(|character| format!("{character}/"))
|
||||
.collect();
|
||||
format!("uploads/{shard}{sha1_checksum}.mp4")
|
||||
}
|
||||
|
||||
/// Write the bytes to a temp file and ffprobe its video dimensions. Fail-soft:
|
||||
/// returns `None` on any failure (not a probeable video, ffprobe missing, I/O
|
||||
/// error) so the caller can proceed without dimensions. Never panics.
|
||||
@@ -208,3 +239,16 @@ fn probe_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::seedance_video_object_name;
|
||||
|
||||
#[test]
|
||||
fn shards_object_name_by_first_four_checksum_chars() {
|
||||
assert_eq!(
|
||||
seedance_video_object_name("f0a2cda0deadbeef"),
|
||||
"uploads/f/0/a/2/f0a2cda0deadbeef.mp4"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -34,6 +34,7 @@ use crate::startup::setup_bans::{
|
||||
load_static_container_ip_bans, load_troll_user_token_bans,
|
||||
};
|
||||
use crate::startup::setup_inference_providers::setup_inference_providers;
|
||||
use crate::startup::setup_seedance_video_bucket::setup_seedance_video_bucket;
|
||||
use crate::startup::setup_static_feature_flags::setup_static_feature_flags;
|
||||
use crate::startup::setup_stripe_artcraft::setup_stripe_artcraft;
|
||||
use crate::startup::setup_stripe_fakeyou::setup_stripe_fakeyou;
|
||||
@@ -236,6 +237,7 @@ pub async fn setup_dependencies(server_hostname: &str) -> AnyhowResult<SetupResu
|
||||
private_bucket_client,
|
||||
public_bucket_client,
|
||||
auto_gc_bucket_client,
|
||||
seedance_video_bucket: setup_seedance_video_bucket(),
|
||||
audio_uploads_bucket_root,
|
||||
sort_key_crypto,
|
||||
opaque_cursors: opaque_cursor_encoder,
|
||||
|
||||
@@ -2,6 +2,7 @@ pub mod build_dependencies;
|
||||
pub mod setup_bans;
|
||||
pub mod setup_metrics;
|
||||
pub mod setup_pager;
|
||||
pub mod setup_seedance_video_bucket;
|
||||
pub mod setup_disabled_endpoints;
|
||||
pub mod setup_inference_providers;
|
||||
pub mod setup_static_feature_flags;
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
//! Optional setup for the Seedance video archive bucket.
|
||||
//!
|
||||
//! Reads the four `SEEDANCE_VIDEO_BUCKET_*` env vars. All must be present to
|
||||
//! build a client; otherwise the client is `None` and archival uploads are
|
||||
//! simply skipped. A *partial* configuration is treated as a misconfiguration
|
||||
//! (warned, not built).
|
||||
|
||||
use std::env;
|
||||
|
||||
use bucket_client::{BucketClient, BucketClientBuilder};
|
||||
use log::{info, warn};
|
||||
|
||||
/// Build the optional Seedance video bucket client from the environment.
|
||||
pub fn setup_seedance_video_bucket() -> Option<BucketClient> {
|
||||
let bucket_name = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_NAME");
|
||||
let region = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_REGION");
|
||||
let access_key = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_ACCESS_KEY_ID");
|
||||
let secret_key = read_non_empty_env("SEEDANCE_VIDEO_BUCKET_SECRET_ACCESS_KEY");
|
||||
|
||||
match (bucket_name, region, access_key, secret_key) {
|
||||
(Some(bucket_name), Some(region), Some(access_key), Some(secret_key)) => {
|
||||
match BucketClientBuilder::new()
|
||||
.access_key(access_key)
|
||||
.secret_key(secret_key)
|
||||
.region_name(region)
|
||||
.bucket_name(&bucket_name)
|
||||
.build()
|
||||
{
|
||||
Ok(client) => {
|
||||
info!("Seedance video bucket configured: {}", bucket_name);
|
||||
Some(client)
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Seedance video bucket env present but client setup failed: {}", err);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
(None, None, None, None) => {
|
||||
info!("Seedance video bucket not configured; archival uploads disabled.");
|
||||
None
|
||||
}
|
||||
_ => {
|
||||
warn!(
|
||||
"Seedance video bucket is only partially configured \
|
||||
(need all of SEEDANCE_VIDEO_BUCKET_NAME / _REGION / _ACCESS_KEY_ID / \
|
||||
_SECRET_ACCESS_KEY); skipping setup."
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read an env var, treating unset or whitespace-only as absent.
|
||||
fn read_non_empty_env(name: &str) -> Option<String> {
|
||||
match env::var(name) {
|
||||
Ok(value) if !value.trim().is_empty() => Some(value),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,7 @@ use billing_artcraft_component::utils::artcraft_stripe_config::ArtcraftStripeCon
|
||||
use billing_component::stripe::stripe_config::StripeConfig;
|
||||
use chrono::{DateTime, Utc};
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use bucket_client::BucketClient as SeedanceVideoBucketClient;
|
||||
use elasticsearch::Elasticsearch;
|
||||
use memory_caching::arc_ttl_sieve::ArcTtlSieve;
|
||||
use memory_caching::single_item_ttl_cache::SingleItemTtlCache;
|
||||
@@ -88,6 +89,10 @@ pub struct ServerState {
|
||||
pub public_bucket_client: BucketClient,
|
||||
pub auto_gc_bucket_client: BucketClient,
|
||||
|
||||
/// Optional archive bucket for uploaded Seedance videos. `None` when the
|
||||
/// `SEEDANCE_VIDEO_BUCKET_*` env vars aren't configured.
|
||||
pub seedance_video_bucket: Option<SeedanceVideoBucketClient>,
|
||||
|
||||
pub inference_providers: InferenceProviders,
|
||||
|
||||
pub resend: ResendData,
|
||||
|
||||
Reference in new issue
Block a user