From d85fe3fb3d3095eb88c3bb2a463d908e4a94c628 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 20 Feb 2024 23:19:08 -0500 Subject: [PATCH] CLI tool to delete users' files --- Cargo.lock | 11 ++- crates/cli/production/db-cleanup/Cargo.toml | 19 +++-- .../cli/production/db-cleanup/src/cli_args.rs | 34 +++++++++ crates/cli/production/db-cleanup/src/main.rs | 40 ++++++++++- .../delete_user_files/delete_user_files.rs | 70 +++++++++++++++++++ .../src/operations/delete_user_files/mod.rs | 1 + .../db-cleanup/src/operations/mod.rs | 1 + .../production/elasticsearch-cli/Cargo.toml | 7 +- .../cli/production/migration-tool/src/main.rs | 1 - .../get_user_profile_by_username.rs | 4 +- 10 files changed, 172 insertions(+), 16 deletions(-) create mode 100644 crates/cli/production/db-cleanup/src/cli_args.rs create mode 100644 crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs create mode 100644 crates/cli/production/db-cleanup/src/operations/delete_user_files/mod.rs create mode 100644 crates/cli/production/db-cleanup/src/operations/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 2156e36f00..51c65cbc77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1765,14 +1765,23 @@ dependencies = [ name = "db-cleanup" version = "0.0.1" dependencies = [ - "anyhow", + "buckets", + "clap", "container_common", "dotenv", "easyenv", + "enums", + "errors", "log", "my-workspace-hack", + "mysql_queries", "serde", "serde_derive", + "serde_json", + "sqlx", + "storyteller_root", + "tokens", + "tokio", ] [[package]] diff --git a/crates/cli/production/db-cleanup/Cargo.toml b/crates/cli/production/db-cleanup/Cargo.toml index 6116186f4f..c1f1dc7007 100644 --- a/crates/cli/production/db-cleanup/Cargo.toml +++ b/crates/cli/production/db-cleanup/Cargo.toml @@ -13,21 +13,32 @@ name = "db-cleanup" path = "src/main.rs" [dependencies] - # Internal +buckets = { path = "../../../_types/buckets" } container_common = { path = "../../../lib/deprecated/container_common" } easyenv = { path = "../../../lib/easyenv" } +enums = { path = "../../../_types/enums" } +errors = { path = "../../../lib/errors" } +mysql_queries = { path = "../../../lib/mysql_queries" } +storyteller_root = { path = "../../../lib/storyteller_root" } +tokens = { path = "../../../_types/tokens" } # Workspace hack (faster compile times) my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External -anyhow = "1.0.75" -# TODO/NB: Commenting out clap4 as it prevents windows builds -# clap4 = { package = "clap", version = "4.0" } # TODO/NB: We're using an older version of clap elsewhere, hence the naming. +clap = "4.4.8" dotenv = "0.15.0" log = "0.4.14" +# Bigger pieces +tokio = { version = "1.34.0", features = ["macros"] } +sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } + # Serialization serde = "1.0.125" serde_derive = "1.0.125" +serde_json = "1.0.108" + +[dev-dependencies] +# None yet diff --git a/crates/cli/production/db-cleanup/src/cli_args.rs b/crates/cli/production/db-cleanup/src/cli_args.rs new file mode 100644 index 0000000000..c9ef2980a3 --- /dev/null +++ b/crates/cli/production/db-cleanup/src/cli_args.rs @@ -0,0 +1,34 @@ +use clap::{Parser, ValueEnum}; +use serde_derive::Deserialize; + +use errors::AnyhowResult; + +#[derive(Parser, Debug, Deserialize)] +#[command(name="db-cleanup")] +#[serde(rename_all = "snake_case")] +pub struct Args { + // The primary action to take + #[arg(name="action", long="action", help="the primary action to take", required=true)] + pub action: Action, + + // Optional username for some actions + #[arg(name="username", long="username", help="optional username", required=false)] + pub username: Option, +} + +#[derive(Clone, Copy, Debug, Deserialize, ValueEnum)] +#[serde(rename_all = "snake_case")] +pub enum Action { + /// Delete files for a particular user. + DeleteUserFiles, +} + +pub fn parse_cli_args() -> AnyhowResult { + let mut args = Args::parse(); + + if let Some(username) = args.username.as_deref() { + args.username = Some(username.trim().to_string()); + } + + Ok(args) +} diff --git a/crates/cli/production/db-cleanup/src/main.rs b/crates/cli/production/db-cleanup/src/main.rs index 8e0e6818a4..a5e4bafeb1 100644 --- a/crates/cli/production/db-cleanup/src/main.rs +++ b/crates/cli/production/db-cleanup/src/main.rs @@ -1,12 +1,46 @@ -use easyenv::init_all_with_default_logging; use log::info; +use sqlx::{MySql, Pool}; +use sqlx::mysql::MySqlPoolOptions; use container_common::anyhow_result::AnyhowResult; +use easyenv::init_all_with_default_logging; + +use crate::cli_args::Action; +use crate::operations::delete_user_files::delete_user_files::delete_user_files; + +mod cli_args; +mod operations; + +#[tokio::main] +async fn main() -> AnyhowResult<()> { + println!("db-cleanup: hard or soft delete database records"); -fn main() -> AnyhowResult<()> { init_all_with_default_logging(None); - info!("TODO..."); + // NB: This secrets file differs from the rest because we might want to delete from production. + // (Hopefully this isn't getting out of hand at this point.) + easyenv::from_filename(".env-db-cleanup-secrets")?; + + let args = cli_args::parse_cli_args()?; + + let mysql = get_mysql("MYSQL_PRODUCTION_URL").await?; + + match args.action { + Action::DeleteUserFiles => { + delete_user_files(&args, &mysql).await?; + } + } Ok(()) } + +async fn get_mysql(env_var_name: &str) -> errors::AnyhowResult> { + info!("Connecting to MySQL {env_var_name}..."); + + let pool = MySqlPoolOptions::new() + .max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 3)?) + .connect(&easyenv::get_env_string_required(env_var_name)?) + .await?; + + Ok(pool) +} diff --git a/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs b/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs new file mode 100644 index 0000000000..76e62bd5e1 --- /dev/null +++ b/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs @@ -0,0 +1,70 @@ +use std::thread; +use std::time::Duration; +use log::info; +use sqlx::{MySql, Pool}; +use enums::common::view_as::ViewAs; + +use errors::{anyhow, AnyhowResult}; +use mysql_queries::queries::media_files::delete_media_file::delete_media_file_as_mod; +use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs}; +use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::get_user_profile_by_username; +use tokens::tokens::users::UserToken; + +use crate::cli_args::Args; + +const ECHELON_USER_TOKEN : &str = "U:937C1VN0S5G8H"; + +pub async fn delete_user_files(args: &Args, mysql: &Pool) -> AnyhowResult<()> { + info!("delete user files for username: {:?}", args.username); + + let username = match args.username.as_deref() { + Some(username) => username, + None => return Err(anyhow!("username is required")), + }; + + let maybe_user_profile = get_user_profile_by_username(username, mysql).await?; + + let user_profile = match maybe_user_profile { + Some(user_profile) => user_profile, + None => return Err(anyhow!("user not found")), + }; + + info!("user_profile: {:?}", &user_profile); + + delete_all_files(username, mysql).await?; + + Ok(()) +} + +pub async fn delete_all_files(username: &str, mysql: &Pool) -> AnyhowResult<()> { + let mut page_index = 0; + + loop { + info!("Querying page {page_index} of files..."); + + let media_files = list_media_files_for_user(ListMediaFileForUserArgs { + username, + maybe_filter_media_type: None, + page_size: 100, + page_index, + sort_ascending: false, + view_as: ViewAs::Moderator, + mysql_pool: &mysql, + }).await?; + + if media_files.records.is_empty() { + break; + } + + for file in media_files.records { + info!("Deleting file: {:?}", &file.token); + delete_media_file_as_mod(&file.token, ECHELON_USER_TOKEN, &mysql).await?; + } + + page_index += 1; + + thread::sleep(Duration::from_millis(1000)); + } + + Ok(()) +} diff --git a/crates/cli/production/db-cleanup/src/operations/delete_user_files/mod.rs b/crates/cli/production/db-cleanup/src/operations/delete_user_files/mod.rs new file mode 100644 index 0000000000..6ddb9fc941 --- /dev/null +++ b/crates/cli/production/db-cleanup/src/operations/delete_user_files/mod.rs @@ -0,0 +1 @@ +pub mod delete_user_files; \ No newline at end of file diff --git a/crates/cli/production/db-cleanup/src/operations/mod.rs b/crates/cli/production/db-cleanup/src/operations/mod.rs new file mode 100644 index 0000000000..6ddb9fc941 --- /dev/null +++ b/crates/cli/production/db-cleanup/src/operations/mod.rs @@ -0,0 +1 @@ +pub mod delete_user_files; \ No newline at end of file diff --git a/crates/cli/production/elasticsearch-cli/Cargo.toml b/crates/cli/production/elasticsearch-cli/Cargo.toml index e1e59cd1eb..e0a5614e27 100644 --- a/crates/cli/production/elasticsearch-cli/Cargo.toml +++ b/crates/cli/production/elasticsearch-cli/Cargo.toml @@ -28,16 +28,13 @@ tokens = { path = "../../../_types/tokens" } my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" } # External -log = "0.4.14" clap = "4.4.8" +log = "0.4.14" strum = { version = "0.25", features = ["derive"] } -# Runtime -# NB: We switched to Tokio from `async-std` for broader ecosystem support. +# Bigger pieces tokio = { version = "1.34.0", features = ["macros"] } - sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] } - elasticsearch = "8.5.0-alpha.1" # Serialization diff --git a/crates/cli/production/migration-tool/src/main.rs b/crates/cli/production/migration-tool/src/main.rs index 71a4270e68..7b615d03ec 100644 --- a/crates/cli/production/migration-tool/src/main.rs +++ b/crates/cli/production/migration-tool/src/main.rs @@ -3,7 +3,6 @@ //! Migrate database records. //! - use std::time::Duration; use log::info; diff --git a/crates/lib/mysql_queries/src/queries/users/user_profiles/get_user_profile_by_username.rs b/crates/lib/mysql_queries/src/queries/users/user_profiles/get_user_profile_by_username.rs index 7982c273b4..7f0ba0e015 100644 --- a/crates/lib/mysql_queries/src/queries/users/user_profiles/get_user_profile_by_username.rs +++ b/crates/lib/mysql_queries/src/queries/users/user_profiles/get_user_profile_by_username.rs @@ -11,7 +11,7 @@ use tokens::tokens::users::UserToken; use crate::helpers::boolean_converters::i8_to_bool; // TODO: Make non-`Serialize` and make the HTTP endpoints do the work -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct UserProfileResult { pub user_token: UserToken, pub username: String, @@ -47,7 +47,7 @@ pub struct UserProfileResult { } // TODO: Make non-`Serialize` and make the HTTP endpoints do the work -#[derive(Serialize, Deserialize, Clone)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct UserProfileModeratorFields { pub is_banned: bool, pub maybe_mod_comments: Option,