CLI tool to delete users' files

Cargo.lock

@@ -1765,14 +1765,23 @@ dependencies = [
 name = "db-cleanup"
 version = "0.0.1"
 dependencies = [
- "anyhow",
+ "buckets",
+ "clap",
  "container_common",
  "dotenv",
  "easyenv",
+ "enums",
+ "errors",
  "log",
  "my-workspace-hack",
+ "mysql_queries",
  "serde",
  "serde_derive",
+ "serde_json",
+ "sqlx",
+ "storyteller_root",
+ "tokens",
+ "tokio",
 ]
 
 [[package]]

crates/cli/production/db-cleanup/Cargo.toml

@@ -13,21 +13,32 @@ name = "db-cleanup"
 path = "src/main.rs"
 
 [dependencies]
-
 # Internal
+buckets = { path = "../../../_types/buckets" }
 container_common = { path = "../../../lib/deprecated/container_common" }
 easyenv = { path = "../../../lib/easyenv" }
+enums = { path = "../../../_types/enums" }
+errors = { path = "../../../lib/errors" }
+mysql_queries = { path = "../../../lib/mysql_queries" }
+storyteller_root = { path = "../../../lib/storyteller_root" }
+tokens = { path = "../../../_types/tokens" }
 
 # Workspace hack (faster compile times)
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
 
 # External
-anyhow = "1.0.75"
-# TODO/NB: Commenting out clap4 as it prevents windows builds
-# clap4 = { package = "clap", version = "4.0" } # TODO/NB: We're using an older version of clap elsewhere, hence the naming.
+clap = "4.4.8"
 dotenv = "0.15.0"
 log = "0.4.14"
 
+# Bigger pieces
+tokio = { version = "1.34.0", features = ["macros"] }
+sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
+
 # Serialization
 serde = "1.0.125"
 serde_derive = "1.0.125"
+serde_json = "1.0.108"
+
+[dev-dependencies]
+# None yet

crates/cli/production/db-cleanup/src/cli_args.rs

@@ -0,0 +1,34 @@
+use clap::{Parser, ValueEnum};
+use serde_derive::Deserialize;
+
+use errors::AnyhowResult;
+
+#[derive(Parser, Debug, Deserialize)]
+#[command(name="db-cleanup")]
+#[serde(rename_all = "snake_case")]
+pub struct Args {
+  // The primary action to take
+  #[arg(name="action", long="action", help="the primary action to take", required=true)]
+  pub action: Action,
+
+  // Optional username for some actions
+  #[arg(name="username", long="username", help="optional username", required=false)]
+  pub username: Option<String>,
+}
+
+#[derive(Clone, Copy, Debug, Deserialize, ValueEnum)]
+#[serde(rename_all = "snake_case")]
+pub enum Action {
+  /// Delete files for a particular user.
+  DeleteUserFiles,
+}
+
+pub fn parse_cli_args() -> AnyhowResult<Args> {
+  let mut args = Args::parse();
+
+  if let Some(username) = args.username.as_deref() {
+    args.username = Some(username.trim().to_string());
+  }
+
+  Ok(args)
+}

crates/cli/production/db-cleanup/src/main.rs

@@ -1,12 +1,46 @@
-use easyenv::init_all_with_default_logging;
 use log::info;
+use sqlx::{MySql, Pool};
+use sqlx::mysql::MySqlPoolOptions;
 
 use container_common::anyhow_result::AnyhowResult;
+use easyenv::init_all_with_default_logging;
+
+use crate::cli_args::Action;
+use crate::operations::delete_user_files::delete_user_files::delete_user_files;
+
+mod cli_args;
+mod operations;
+
+#[tokio::main]
+async fn main() -> AnyhowResult<()> {
+  println!("db-cleanup: hard or soft delete database records");
 
-fn main() -> AnyhowResult<()> {
   init_all_with_default_logging(None);
 
-  info!("TODO...");
+  // NB: This secrets file differs from the rest because we might want to delete from production.
+  // (Hopefully this isn't getting out of hand at this point.)
+  easyenv::from_filename(".env-db-cleanup-secrets")?;
+
+  let args = cli_args::parse_cli_args()?;
+
+  let mysql = get_mysql("MYSQL_PRODUCTION_URL").await?;
+
+  match args.action {
+    Action::DeleteUserFiles => {
+      delete_user_files(&args, &mysql).await?;
+    }
+  }
 
   Ok(())
 }
+
+async fn get_mysql(env_var_name: &str) -> errors::AnyhowResult<Pool<MySql>> {
+  info!("Connecting to MySQL {env_var_name}...");
+
+  let pool = MySqlPoolOptions::new()
+      .max_connections(easyenv::get_env_num("MYSQL_MAX_CONNECTIONS", 3)?)
+      .connect(&easyenv::get_env_string_required(env_var_name)?)
+      .await?;
+
+  Ok(pool)
+}

crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs

@@ -0,0 +1,70 @@
+use std::thread;
+use std::time::Duration;
+use log::info;
+use sqlx::{MySql, Pool};
+use enums::common::view_as::ViewAs;
+
+use errors::{anyhow, AnyhowResult};
+use mysql_queries::queries::media_files::delete_media_file::delete_media_file_as_mod;
+use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs};
+use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::get_user_profile_by_username;
+use tokens::tokens::users::UserToken;
+
+use crate::cli_args::Args;
+
+const ECHELON_USER_TOKEN : &str = "U:937C1VN0S5G8H";
+
+pub async fn delete_user_files(args: &Args, mysql: &Pool<MySql>) -> AnyhowResult<()> {
+  info!("delete user files for username: {:?}", args.username);
+
+  let username = match args.username.as_deref() {
+    Some(username) => username,
+    None => return Err(anyhow!("username is required")),
+  };
+
+  let maybe_user_profile = get_user_profile_by_username(username, mysql).await?;
+
+  let user_profile = match maybe_user_profile {
+    Some(user_profile) => user_profile,
+    None => return Err(anyhow!("user not found")),
+  };
+
+  info!("user_profile: {:?}", &user_profile);
+
+  delete_all_files(username, mysql).await?;
+
+  Ok(())
+}
+
+pub async fn delete_all_files(username: &str, mysql: &Pool<MySql>) -> AnyhowResult<()> {
+  let mut page_index = 0;
+
+  loop {
+    info!("Querying page {page_index} of files...");
+
+    let media_files = list_media_files_for_user(ListMediaFileForUserArgs {
+      username,
+      maybe_filter_media_type: None,
+      page_size: 100,
+      page_index,
+      sort_ascending: false,
+      view_as: ViewAs::Moderator,
+      mysql_pool: &mysql,
+    }).await?;
+
+    if media_files.records.is_empty() {
+      break;
+    }
+
+    for file in media_files.records {
+      info!("Deleting file: {:?}", &file.token);
+      delete_media_file_as_mod(&file.token, ECHELON_USER_TOKEN, &mysql).await?;
+    }
+
+    page_index += 1;
+
+    thread::sleep(Duration::from_millis(1000));
+  }
+
+  Ok(())
+}

crates/cli/production/db-cleanup/src/operations/delete_user_files/mod.rs

@@ -0,0 +1 @@
+pub mod delete_user_files;

crates/cli/production/db-cleanup/src/operations/mod.rs

@@ -0,0 +1 @@
+pub mod delete_user_files;

crates/cli/production/elasticsearch-cli/Cargo.toml

@@ -28,16 +28,13 @@ tokens = { path = "../../../_types/tokens" }
 my-workspace-hack = { version = "0.1", path = "../../../my-workspace-hack" }
 
 # External
-log = "0.4.14"
 clap = "4.4.8"
+log = "0.4.14"
 strum = { version = "0.25", features = ["derive"] }
 
-# Runtime
-# NB: We switched to Tokio from `async-std` for broader ecosystem support.
+# Bigger pieces
 tokio = { version = "1.34.0", features = ["macros"] }
-
 sqlx = { version = "0.7.2", features = [ "mysql", "runtime-tokio", "tls-rustls", "chrono" ] }
-
 elasticsearch = "8.5.0-alpha.1"
 
 # Serialization

crates/cli/production/migration-tool/src/main.rs

@@ -3,7 +3,6 @@
 //! Migrate database records.
 //!
 
-
 use std::time::Duration;
 
 use log::info;

crates/lib/mysql_queries/src/queries/users/user_profiles/get_user_profile_by_username.rs

@@ -11,7 +11,7 @@ use tokens::tokens::users::UserToken;
 use crate::helpers::boolean_converters::i8_to_bool;
 
 // TODO: Make non-`Serialize` and make the HTTP endpoints do the work
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct UserProfileResult {
   pub user_token: UserToken,
   pub username: String,
@@ -47,7 +47,7 @@ pub struct UserProfileResult {
 }
 
 // TODO: Make non-`Serialize` and make the HTTP endpoints do the work
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct UserProfileModeratorFields {
   pub is_banned: bool,
   pub maybe_mod_comments: Option<String>,