From 26794b3d2e65eb10c28c70237f3118bab10f4020 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 19 Oct 2023 21:34:26 -0400 Subject: [PATCH] CLI tool to upload media files and save them to the DB --- Cargo.lock | 21 +++ Cargo.toml | 1 + .../dev_database_seed/src/seeding/users.rs | 1 + crates/cli/dev_upload_media_file/Cargo.toml | 44 +++++ .../cli/dev_upload_media_file/src/cli_args.rs | 34 ++++ crates/cli/dev_upload_media_file/src/main.rs | 154 ++++++++++++++++++ crates/lib/easyenv/src/init.rs | 10 ++ crates/lib/easyenv/src/lib.rs | 1 + .../lib/files/filesys/src/file_read_bytes.rs | 16 ++ crates/lib/files/filesys/src/lib.rs | 1 + .../insert_media_file_from_cli_tool.rs | 111 +++++++++++++ .../src/queries/media_files/mod.rs | 3 +- 12 files changed, 396 insertions(+), 1 deletion(-) create mode 100644 crates/cli/dev_upload_media_file/Cargo.toml create mode 100644 crates/cli/dev_upload_media_file/src/cli_args.rs create mode 100644 crates/cli/dev_upload_media_file/src/main.rs create mode 100644 crates/lib/files/filesys/src/file_read_bytes.rs create mode 100644 crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs diff --git a/Cargo.lock b/Cargo.lock index 79caee9f16..3f4fd9a687 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1841,6 +1841,27 @@ dependencies = [ "tokio 1.25.0", ] +[[package]] +name = "dev-upload-media-file" +version = "0.0.1" +dependencies = [ + "buckets", + "clap", + "cloud_storage", + "config", + "easyenv", + "enums", + "errors", + "filesys", + "hashing", + "log", + "mimetypes", + "mysql_queries", + "sqlx", + "tokens", + "tokio 1.25.0", +] + [[package]] name = "diesel" version = "2.1.3" diff --git a/Cargo.toml b/Cargo.toml index 658ef97d41..9edb6162f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ members = [ "crates/cli/database_backup", "crates/cli/database_cleanup", "crates/cli/dev_database_seed", + "crates/cli/dev_upload_media_file", "crates/cli/kubectl_pod_cleanup", "crates/cli/scrape_test", diff --git a/crates/cli/dev_database_seed/src/seeding/users.rs b/crates/cli/dev_database_seed/src/seeding/users.rs index 7d6f989aea..1ab5590757 100644 --- a/crates/cli/dev_database_seed/src/seeding/users.rs +++ b/crates/cli/dev_database_seed/src/seeding/users.rs @@ -12,6 +12,7 @@ pub const HANASHI_USERNAME : &str = "hanashi"; pub async fn seed_user_accounts(mysql_pool: &Pool) -> AnyhowResult<()> { info!("Seeding user accounts..."); + // NB: This is idempotent and will only install the accounts once. let users = [ (ADMIN_USERNAME, "password"), (HANASHI_USERNAME, "password"), diff --git a/crates/cli/dev_upload_media_file/Cargo.toml b/crates/cli/dev_upload_media_file/Cargo.toml new file mode 100644 index 0000000000..8d39eb15a4 --- /dev/null +++ b/crates/cli/dev_upload_media_file/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "dev-upload-media-file" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[[bin]] +name = "dev-upload-media-file" +path = "src/main.rs" + +[dependencies] +# Internal +buckets = { path = "../../_types/buckets" } +cloud_storage = { path = "../../lib/api_clients/cloud_storage" } +config = { path = "../../lib/deprecated/config" } +easyenv = { path = "../../lib/easyenv" } +enums = { path = "../../_types/enums" } +errors = { path = "../../lib/errors" } +filesys = { path = "../../lib/files/filesys" } +hashing = { path = "../../lib/files/hashing" } +mimetypes = { path = "../../lib/files/mimetypes" } +mysql_queries = { path = "../../lib/mysql_queries" } +tokens = { path = "../../_types/tokens" } + +# External +clap = "2.34.0" # TODO: This is really out of date, but has conflicts when updated +log = "0.4.14" + +# Runtime +# NB: We switched to Tokio from `async-std` for broader ecosystem support. +tokio = { version = "1.6.0", features = ["macros"] } + +# MySql Database +# "offline" supports typesafe builds without a live database +# "rustls" doesn't appear to work with IP addresses, per: +# https://github.com/launchbadge/sqlx/issues/1095#issuecomment-798050828 +sqlx = { version = "0.6.2", features = [ "mysql", "runtime-actix-native-tls", "offline", "chrono" ] } + +[dev-dependencies] +# None yet diff --git a/crates/cli/dev_upload_media_file/src/cli_args.rs b/crates/cli/dev_upload_media_file/src/cli_args.rs new file mode 100644 index 0000000000..fae304c29b --- /dev/null +++ b/crates/cli/dev_upload_media_file/src/cli_args.rs @@ -0,0 +1,34 @@ +use std::path::PathBuf; +use clap::{App, Arg}; +use errors::{anyhow, AnyhowResult}; + +pub struct CliArgs { + pub file_path: PathBuf, +} + +pub fn parse_cli_args() -> AnyhowResult { + let matches = App::new("scrape_test") + .arg(Arg::with_name("file") + .short("f") + .long("file") + .value_name("FILE") + .help("File to upload") + .takes_value(true) + .required(true)) + //.arg(Arg::with_name("print_html") + // .long("html") + // .help("Print the HTML scraped") + // .takes_value(false) + // .required(false)) + .get_matches(); + + let file_path = matches.value_of("file") + .map(|val| val.trim().to_string()) + .ok_or(anyhow!("no url supplied"))?; + + //let print_html = matches.is_present("print_html"); + + Ok(CliArgs { + file_path: PathBuf::from(file_path), + }) +} diff --git a/crates/cli/dev_upload_media_file/src/main.rs b/crates/cli/dev_upload_media_file/src/main.rs new file mode 100644 index 0000000000..5cd409da93 --- /dev/null +++ b/crates/cli/dev_upload_media_file/src/main.rs @@ -0,0 +1,154 @@ +use std::time::Duration; + +use log::info; +use sqlx::mysql::MySqlPoolOptions; + +use buckets::public::media_files::original_file::MediaFileBucketPath; +use cloud_storage::bucket_client::BucketClient; +use config::shared_constants::{DEFAULT_MYSQL_CONNECTION_STRING, DEFAULT_RUST_LOG}; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::{AnyhowResult, bail}; +use filesys::file_read_bytes::file_read_bytes; +use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; +use mysql_queries::queries::media_files::insert_media_file_from_cli_tool::{insert_media_file_from_cli_tool, InsertArgs}; + +use crate::cli_args::parse_cli_args; + +pub mod cli_args; + +#[tokio::main] +pub async fn main() -> AnyhowResult<()> { + easyenv::init_all_with_default_logging(Some(DEFAULT_RUST_LOG)); + + info!("Media file upload"); + + // NB: Read secrets (eg. ACCESS_KEY) + easyenv::from_filename(".env-secrets")?; + + let db_connection_string = + easyenv::get_env_string_or_default( + "MYSQL_URL", + DEFAULT_MYSQL_CONNECTION_STRING); + + info!("Connecting to MySQL..."); + + let pool = MySqlPoolOptions::new() + .max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 3)?) + .connect(&db_connection_string) + .await?; + + let access_key = easyenv::get_env_string_required("ACCESS_KEY")?; + let secret_key = easyenv::get_env_string_required("SECRET_KEY")?; + let region_name = easyenv::get_env_string_required("REGION_NAME")?; + let public_bucket_name = easyenv::get_env_string_required("PUBLIC_BUCKET_NAME")?; + + let bucket_timeout = easyenv::get_env_duration_seconds_or_default( + "BUCKET_TIMEOUT_SECONDS", Duration::from_secs(60 * 5)); + + info!("Configuring GCS bucket..."); + + let public_bucket_client = BucketClient::create( + &access_key, + &secret_key, + ®ion_name, + &public_bucket_name, + None, + Some(bucket_timeout), + )?; + + let args = parse_cli_args()?; + + info!("Reading file to upload: {:?} ...", args.file_path); + + let bytes = file_read_bytes(args.file_path)?; + let mimetype = get_mimetype_for_bytes(&bytes); + + let mut media_file_type; + let mut maybe_mime_type; + let mut maybe_public_bucket_prefix; + let mut maybe_public_bucket_extension; + + match mimetype { + Some("audio/wav") | + Some("audio/x-wav") => { + media_file_type = MediaFileType::Video; + maybe_mime_type = Some("audio/wav"); + maybe_public_bucket_prefix = Some("dev_upload_"); + maybe_public_bucket_extension = Some(".wav"); + + } + Some("video/mp4") => { + media_file_type = MediaFileType::Video; + maybe_mime_type = mimetype; + maybe_public_bucket_prefix = Some("dev_upload_"); + maybe_public_bucket_extension = Some(".mp4"); + } + Some("image/jpeg") => { + media_file_type = MediaFileType::Image; + maybe_mime_type = mimetype; + maybe_public_bucket_prefix = Some("dev_upload_"); + maybe_public_bucket_extension = Some(".jpg"); + } + Some("image/png") => { + media_file_type = MediaFileType::Image; + maybe_mime_type = mimetype; + maybe_public_bucket_prefix = Some("dev_upload_"); + maybe_public_bucket_extension = Some(".png"); + } + _ => { + bail!("Invalid mime type: {:?}", mimetype); + } + } + + let mimetype = mimetype.unwrap_or("text/plain"); + + let creator_set_visibility = Visibility::Public; + + let public_upload_path = MediaFileBucketPath::generate_new( + maybe_public_bucket_prefix, + maybe_public_bucket_extension); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + let _r = public_bucket_client.upload_file_with_content_type( + public_upload_path.get_full_object_path_str(), + bytes.as_ref(), + mimetype) + .await?; + + let (media_file_token, _id) = insert_media_file_from_cli_tool(InsertArgs { + pool: &pool, + media_file_type, + maybe_mime_type, + file_size_bytes: 0, + sha256_checksum: "", + maybe_creator_user_token: None, + creator_set_visibility, + public_bucket_directory_hash: public_upload_path.get_object_hash(), + maybe_public_bucket_prefix, + maybe_public_bucket_extension, + }).await?; + + info!("New media file token: {:?}", media_file_token); + + let bucket_url = format!( + "https://storage.cloud.google.com/{}{}", + public_bucket_name, + public_upload_path.get_full_object_path_str() + ); + + let gcp_url = format!( + "https://console.cloud.google.com/storage/browser/_details/{}{}", + public_bucket_name, + public_upload_path.get_full_object_path_str() + ); + + println!("\n\n ========== FILE INFO ==========\n\n"); + println!(" token: {}", media_file_token); + println!(" URL: {}", bucket_url); + println!(" GCP Admin: {}", gcp_url); + println!("\n"); + + Ok(()) +} diff --git a/crates/lib/easyenv/src/init.rs b/crates/lib/easyenv/src/init.rs index 5c603d5c70..8bc49f897c 100644 --- a/crates/lib/easyenv/src/init.rs +++ b/crates/lib/easyenv/src/init.rs @@ -11,6 +11,16 @@ pub fn init_all_with_default_logging(default_if_absent: Option<&str>) { init_env_logger(default_if_absent) } +/// Read env configs from a filename. +pub fn from_filename>(filename: P) -> Result<(), InitError> { + let _path = dotenv::from_filename(&filename) + .map_err(|err| { + error!("Could not read env config from path {:?} : {:?}", filename.as_ref(), err); + InitError::DotEnvError + })?; + Ok(()) +} + /// Try loading in an environment file across many search paths (first found wins). /// Returns an error if no file could be read or if in attempting to read a file there was an error. pub fn read_env_config_from_filename_and_paths, Q: AsRef>(filename: P, paths: &[Q]) -> Result<(), InitError> { diff --git a/crates/lib/easyenv/src/lib.rs b/crates/lib/easyenv/src/lib.rs index 03ac3ef4e6..46a4589bcd 100644 --- a/crates/lib/easyenv/src/lib.rs +++ b/crates/lib/easyenv/src/lib.rs @@ -32,6 +32,7 @@ // It's easy to just keep them exported here to maintain compatibility. pub use error::EnvError; pub use error::InitError; +pub use init::from_filename; pub use init::init_all_with_default_logging; pub use init::maybe_read_env_config_from_filename_and_paths; pub use init::read_env_config_from_filename_and_paths; diff --git a/crates/lib/files/filesys/src/file_read_bytes.rs b/crates/lib/files/filesys/src/file_read_bytes.rs new file mode 100644 index 0000000000..00fd8d737f --- /dev/null +++ b/crates/lib/files/filesys/src/file_read_bytes.rs @@ -0,0 +1,16 @@ +use std::fs::File; +use std::io; +use std::io::{BufReader, Read}; +use std::path::Path; + +/// Read a file into a byte buffer +pub fn file_read_bytes>(file_path: P) -> io::Result> { + let f = File::open(file_path)?; + let mut reader = BufReader::new(f); + let mut buffer = Vec::new(); + + // Read file into vector. + reader.read_to_end(&mut buffer)?; + + Ok(buffer) +} diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index 77a085d899..04a482ddcc 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -21,6 +21,7 @@ pub mod create_dir_all_if_missing; pub mod directory_exists; pub mod file_exists; +pub mod file_read_bytes; pub mod file_size; pub mod filename_concat; pub mod path_to_string; diff --git a/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs new file mode 100644 index 0000000000..d951ea49b8 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/media_files/insert_media_file_from_cli_tool.rs @@ -0,0 +1,111 @@ +use anyhow::anyhow; +use log::error; +use sqlx; +use sqlx::MySqlPool; + +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_model_type::MediaFileOriginModelType; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::users::UserToken; + +pub struct InsertArgs<'a> { + pub pool: &'a MySqlPool, + + pub media_file_type: MediaFileType, + pub maybe_mime_type: Option<&'a str>, + pub file_size_bytes: u64, + pub sha256_checksum: &'a str, + + pub maybe_creator_user_token: Option, + pub creator_set_visibility: Visibility, + + pub public_bucket_directory_hash: &'a str, + pub maybe_public_bucket_prefix: Option<&'a str>, + pub maybe_public_bucket_extension: Option<&'a str>, +} + +// NB: Do not use from website / jobs. This is meant to be used from the CLI tool, +// and shoudl allow for a more free-form interface. +pub async fn insert_media_file_from_cli_tool( + args: InsertArgs<'_> +) -> AnyhowResult<(MediaFileToken, u64)> +{ + let media_file_token = MediaFileToken::generate(); + + let mut transaction = args.pool.begin().await?; + + const ORIGIN_CATEGORY : MediaFileOriginCategory = MediaFileOriginCategory::Upload; + const ORIGIN_PRODUCT_CATEGORY : MediaFileOriginProductCategory = MediaFileOriginProductCategory::Unknown; + + // TODO(bt,2023-10-19): UserToken should automatically serialize to the DB. What's going on? + let maybe_user_token = args.maybe_creator_user_token.map(|u| u.to_string()); + + let record_id = { + let query_result = sqlx::query!( + r#" +INSERT INTO media_files +SET + token = ?, + + origin_category = ?, + origin_product_category = ?, + + media_type = ?, + maybe_mime_type = ?, + file_size_bytes = ?, + checksum_sha2 = ?, + + public_bucket_directory_hash = ?, + maybe_public_bucket_prefix = ?, + maybe_public_bucket_extension = ?, + + maybe_creator_user_token = ?, + creator_ip_address = ?, + + creator_set_visibility = ? + "#, + media_file_token, + + ORIGIN_CATEGORY, + ORIGIN_PRODUCT_CATEGORY, + + args.media_file_type, + args.maybe_mime_type, + args.file_size_bytes, + args.sha256_checksum, + + args.public_bucket_directory_hash, + args.maybe_public_bucket_prefix, + args.maybe_public_bucket_extension, + + maybe_user_token, + "127.0.0.1", + + args.creator_set_visibility.to_str(), + ) + .execute(&mut transaction) + .await; + + let record_id = match query_result { + Ok(res) => { + res.last_insert_id() + }, + Err(err) => { + // TODO: handle better + //transaction.rollback().await?; + error!("Error executing query: {:?}", err); + return Err(anyhow!("Mysql error: {:?}", err)); + } + }; + + record_id + }; + + transaction.commit().await?; + + Ok((media_file_token, record_id)) +} diff --git a/crates/lib/mysql_queries/src/queries/media_files/mod.rs b/crates/lib/mysql_queries/src/queries/media_files/mod.rs index 632f5cd244..cab9cbf24c 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/mod.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/mod.rs @@ -1,2 +1,3 @@ pub mod get_media_file; -pub mod insert_media_file_from_face_animation; \ No newline at end of file +pub mod insert_media_file_from_face_animation; +pub mod insert_media_file_from_cli_tool; \ No newline at end of file