From 2dd02272f941a760208b7c366252b0fdd39f2c9c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 24 Feb 2024 02:26:25 -0500 Subject: [PATCH] tool to soft delete anonymous user images - We'll need to go back and clean up the buckets for some of these - Some of these are probably just for face animator and could help with training a better face detection model --- .../cli/production/db-cleanup/src/cli_args.rs | 3 + crates/cli/production/db-cleanup/src/main.rs | 5 ++ .../delete_all_anonymous_user_images.rs | 66 +++++++++++++++++++ .../delete_all_anonymous_user_images/mod.rs | 1 + .../delete_user_files/delete_user_files.rs | 3 +- .../db-cleanup/src/operations/mod.rs | 1 + .../db-cleanup/src/util/constants.rs | 3 + .../cli/production/db-cleanup/src/util/mod.rs | 1 + .../classify_text/tests/csam_prompt_tests.rs | 3 + 9 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/delete_all_anonymous_user_images.rs create mode 100644 crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/mod.rs create mode 100644 crates/cli/production/db-cleanup/src/util/constants.rs create mode 100644 crates/cli/production/db-cleanup/src/util/mod.rs diff --git a/crates/cli/production/db-cleanup/src/cli_args.rs b/crates/cli/production/db-cleanup/src/cli_args.rs index c9ef2980a3..5efe4fdafa 100644 --- a/crates/cli/production/db-cleanup/src/cli_args.rs +++ b/crates/cli/production/db-cleanup/src/cli_args.rs @@ -19,6 +19,9 @@ pub struct Args { #[derive(Clone, Copy, Debug, Deserialize, ValueEnum)] #[serde(rename_all = "snake_case")] pub enum Action { + /// Delete all anonymous user images. + DeleteAllAnonymousUserImages, + /// Delete files for a particular user. DeleteUserFiles, } diff --git a/crates/cli/production/db-cleanup/src/main.rs b/crates/cli/production/db-cleanup/src/main.rs index 2d4c637899..f6bd1ea19c 100644 --- a/crates/cli/production/db-cleanup/src/main.rs +++ b/crates/cli/production/db-cleanup/src/main.rs @@ -6,10 +6,12 @@ use container_common::anyhow_result::AnyhowResult; use easyenv::init_all_with_default_logging; use crate::cli_args::Action; +use crate::operations::delete_all_anonymous_user_images::delete_all_anonymous_user_images::delete_all_anonymous_user_images; use crate::operations::delete_user_files::delete_user_files::delete_user_files; mod cli_args; mod operations; +mod util; /* @@ -45,6 +47,9 @@ async fn main() -> AnyhowResult<()> { let mysql = get_mysql("MYSQL_PRODUCTION_URL").await?; match args.action { + Action::DeleteAllAnonymousUserImages => { + delete_all_anonymous_user_images(&args, &mysql).await?; + } Action::DeleteUserFiles => { delete_user_files(&args, &mysql).await?; } diff --git a/crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/delete_all_anonymous_user_images.rs b/crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/delete_all_anonymous_user_images.rs new file mode 100644 index 0000000000..4fb58deda0 --- /dev/null +++ b/crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/delete_all_anonymous_user_images.rs @@ -0,0 +1,66 @@ +use std::thread; +use std::time::Duration; +use log::info; +use sqlx::{MySql, Pool}; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::common::view_as::ViewAs; + +use errors::{anyhow, AnyhowResult}; +use mysql_queries::queries::media_files::delete_media_file::delete_media_file_as_mod; +use mysql_queries::queries::media_files::list::list_media_files::{list_media_files, ListMediaFilesArgs}; +use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_media_files_for_user, ListMediaFileForUserArgs}; +use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::get_user_profile_by_username; + +use crate::cli_args::Args; +use crate::util::constants::ECHELON_USER_TOKEN; + +pub async fn delete_all_anonymous_user_images(_args: &Args, mysql: &Pool) -> AnyhowResult<()> { + info!("delete anonymous user images"); + + let mut page_index = 0; + let mut maybe_offset = None; + + loop { + info!("Querying page {page_index} of files..."); + + let media_files = list_media_files(ListMediaFilesArgs { + limit: 100, + maybe_filter_media_type: Some(MediaFileType::Image), + maybe_offset, + cursor_is_reversed: false, + sort_ascending: false, + view_as: ViewAs::Moderator, + mysql_pool: &mysql, + }).await?; + + if media_files.records.is_empty() { + break; + } + + for file in media_files.records { + if file.maybe_creator_user_token.is_some() { + info!("Skipping file: {:?} , which has author: {:?}", &file.token, &file.maybe_creator_user_token); + continue; + } + + match file.media_type { + MediaFileType::Image => {}, + _ => { + info!("Skipping file: {:?} , which is not an image", &file.token); + continue; + } + } + + info!("Deleting image file: {:?}", &file.token); + delete_media_file_as_mod(&file.token, ECHELON_USER_TOKEN, &mysql).await?; + } + + page_index += 1; + + maybe_offset = media_files.last_id.map(|id| id as usize); + + thread::sleep(Duration::from_millis(1000)); + } + + Ok(()) +} diff --git a/crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/mod.rs b/crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/mod.rs new file mode 100644 index 0000000000..24bea8206d --- /dev/null +++ b/crates/cli/production/db-cleanup/src/operations/delete_all_anonymous_user_images/mod.rs @@ -0,0 +1 @@ +pub mod delete_all_anonymous_user_images; \ No newline at end of file diff --git a/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs b/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs index 428ca22396..b761898638 100644 --- a/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs +++ b/crates/cli/production/db-cleanup/src/operations/delete_user_files/delete_user_files.rs @@ -10,8 +10,7 @@ use mysql_queries::queries::media_files::list::list_media_files_for_user::{list_ use mysql_queries::queries::users::user_profiles::get_user_profile_by_username::get_user_profile_by_username; use crate::cli_args::Args; - -const ECHELON_USER_TOKEN : &str = "U:937C1VN0S5G8H"; +use crate::util::constants::ECHELON_USER_TOKEN; pub async fn delete_user_files(args: &Args, mysql: &Pool) -> AnyhowResult<()> { info!("delete user files for username: {:?}", args.username); diff --git a/crates/cli/production/db-cleanup/src/operations/mod.rs b/crates/cli/production/db-cleanup/src/operations/mod.rs index 6ddb9fc941..528507b286 100644 --- a/crates/cli/production/db-cleanup/src/operations/mod.rs +++ b/crates/cli/production/db-cleanup/src/operations/mod.rs @@ -1 +1,2 @@ +pub mod delete_all_anonymous_user_images; pub mod delete_user_files; \ No newline at end of file diff --git a/crates/cli/production/db-cleanup/src/util/constants.rs b/crates/cli/production/db-cleanup/src/util/constants.rs new file mode 100644 index 0000000000..03b720b7df --- /dev/null +++ b/crates/cli/production/db-cleanup/src/util/constants.rs @@ -0,0 +1,3 @@ + +/// User token to use as the moderator field when deleting files. +pub (crate) const ECHELON_USER_TOKEN : &str = "U:937C1VN0S5G8H"; diff --git a/crates/cli/production/db-cleanup/src/util/mod.rs b/crates/cli/production/db-cleanup/src/util/mod.rs new file mode 100644 index 0000000000..901c346ba2 --- /dev/null +++ b/crates/cli/production/db-cleanup/src/util/mod.rs @@ -0,0 +1 @@ +pub (crate) mod constants; \ No newline at end of file diff --git a/crates/lib/classify_text/tests/csam_prompt_tests.rs b/crates/lib/classify_text/tests/csam_prompt_tests.rs index 17329833cb..aaa5bec405 100644 --- a/crates/lib/classify_text/tests/csam_prompt_tests.rs +++ b/crates/lib/classify_text/tests/csam_prompt_tests.rs @@ -23,6 +23,9 @@ fn prompts_that_generate_nsfw_possibly_ambiguous_but_could_accidentally_generate "tiny nude girl", "tiny nude girl, full body", "tiny nude girl, sexy", + "tiny tits turkish teen small ", + "turkish teen b cups freckles big nipples", + "turkish teen tiny tits big nipples freckles ", ]); }