From 8dc3d29daf7409174a26e883fb33fbf9d61d1b09 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 19 Feb 2024 02:31:07 -0500 Subject: [PATCH] Block basic absuive SD prompts --- Cargo.lock | 41 +- crates/service/web/storyteller_web/Cargo.toml | 2 +- .../image_gen/enqueue_image_generation.rs | 13 +- .../src/util/classify_prompt.rs | 168 +++++ .../web/storyteller_web/src/util/mod.rs | 3 +- .../dictionary_children_terms.txt | 24 + .../binary_includes/dictionary_sex_terms.txt | 587 ++++++++++++++++++ 7 files changed, 830 insertions(+), 8 deletions(-) create mode 100644 crates/service/web/storyteller_web/src/util/classify_prompt.rs create mode 100644 includes/binary_includes/dictionary_children_terms.txt create mode 100644 includes/binary_includes/dictionary_sex_terms.txt diff --git a/Cargo.lock b/Cargo.lock index f671d3203e..b2637f6450 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2978,7 +2978,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "speculoos", + "speculoos 0.9.0", ] [[package]] @@ -4128,13 +4128,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e" dependencies = [ "num-bigint 0.1.44", - "num-complex", + "num-complex 0.1.43", "num-integer", "num-iter", "num-rational 0.1.42", "num-traits", ] +[[package]] +name = "num" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +dependencies = [ + "num-bigint 0.4.4", + "num-complex 0.4.5", + "num-integer", + "num-iter", + "num-rational 0.4.1", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.1.44" @@ -4185,6 +4199,15 @@ dependencies = [ "rustc-serialize", ] +[[package]] +name = "num-complex" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -4225,6 +4248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" dependencies = [ "autocfg", + "num-bigint 0.4.4", "num-integer", "num-traits", ] @@ -6006,7 +6030,16 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bcff6a72e2ddfb09ce0064568857e70d3cf6f4342244ebb1f84fa6f89b16c26" dependencies = [ - "num", + "num 0.1.42", +] + +[[package]] +name = "speculoos" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65881c9270d6157f30a09233305da51bed97eef9192d0ea21e57b1c8f05c3620" +dependencies = [ + "num 0.4.1", ] [[package]] @@ -6368,7 +6401,7 @@ dependencies = [ "serde_json", "server_environment", "sha2 0.9.9", - "speculoos", + "speculoos 0.11.0", "sqlx", "strum", "symphonia", diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 0926381123..7e34a44754 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -176,6 +176,6 @@ http = "0.2.11" diesel = { version = "2.1.3", features = ["mysql"] } [dev-dependencies] -speculoos = "0.9.0" # Fluent assertions +speculoos = "0.11.0" # Fluent assertions strum = { version = "0.25", features = ["derive"] } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs index 1761ad9a7c..d5fc89ef40 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs @@ -13,6 +13,7 @@ use log::warn; use serde::Deserialize; use serde::Serialize; use utoipa::ToSchema; +use web::Data; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_job_type::InferenceJobType; @@ -39,6 +40,7 @@ use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_se use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; use crate::server_state::ServerState; use crate::util::allowed_studio_access::allowed_studio_access; +use crate::util::classify_prompt::classify_prompt; use crate::validations::validate_idempotency_token_format::validate_idempotency_token_format; /// Debug requests can get routed to special "debug-only" workers, which can @@ -189,8 +191,8 @@ impl Display for EnqueueImageGenRequestError { )] pub async fn enqueue_image_generation_request( http_request: HttpRequest, - request: web::Json, - server_state: web::Data> + request: Json, + server_state: Data> ) -> Result { let inference_mode = inference_mode_from_http_request(&http_request) @@ -488,6 +490,13 @@ fn validate_request( return Err(EnqueueImageGenRequestError::BadInput("Missing Model Upload Path".to_string())); } + if let Some(prompt) = request.maybe_prompt.as_deref() { + let classification = classify_prompt(prompt); + if classification.is_child_abuse() { + return Err(EnqueueImageGenRequestError::BadInput("Abusive prompt detected".to_string())); + } + } + Ok(()) } diff --git a/crates/service/web/storyteller_web/src/util/classify_prompt.rs b/crates/service/web/storyteller_web/src/util/classify_prompt.rs new file mode 100644 index 0000000000..f9a8be0b49 --- /dev/null +++ b/crates/service/web/storyteller_web/src/util/classify_prompt.rs @@ -0,0 +1,168 @@ +use std::collections::HashSet; +use once_cell::sync::Lazy; +use regex::Regex; + +const CHILD_TERMS : &str = include_str!("../../../../../../includes/binary_includes/dictionary_children_terms.txt"); +const SEX_TERMS : &str = include_str!("../../../../../../includes/binary_includes/dictionary_sex_terms.txt"); + +/// Classification of a text prompt +pub struct PromptClassification { + pub prompt_references_children: bool, + pub prompt_references_sex: bool, + pub prompt_references_violence: bool, + pub prompt_references_racism: bool, +} + +impl PromptClassification { + pub fn is_abusive(&self) -> bool { + self.prompt_references_racism || self.is_child_abuse() + + } + + pub fn is_child_abuse(&self) -> bool { + self.prompt_references_children && ( self.prompt_references_sex || self.prompt_references_violence ) + } +} + +pub fn classify_prompt(text_prompt: &str) -> PromptClassification { + let prompt_tokens = text_prompt.split_whitespace() + .map(|term| term.trim().to_lowercase()) + .map(|term| term.chars().filter(|c| c.is_alphanumeric()).collect::()) + .collect::>(); + + let prompt_references_children = references_children(&prompt_tokens); + let prompt_references_sex = references_sex(&prompt_tokens); + let prompt_references_violence = references_violence(&prompt_tokens); + let prompt_references_racism = references_racism(&prompt_tokens); + + PromptClassification { + prompt_references_children, + prompt_references_sex, + prompt_references_violence, + prompt_references_racism, + } +} + +const CHILD_TERM_DICTIONARY : Lazy> = Lazy::new(|| { + CHILD_TERMS.lines() + .map(|line| line.trim().to_lowercase()) + .filter(|line| !line.is_empty() && !line.starts_with("#")) + .collect::>() +}); + +const SEX_TERM_DICTIONARY : Lazy> = Lazy::new(|| { + SEX_TERMS.lines() + .map(|line| line.trim().to_lowercase()) + .filter(|line| !line.is_empty() && !line.starts_with("#")) + .collect::>() +}); + +fn references_children(prompt_tokens: &[String]) -> bool { + prompt_tokens.iter().any(|term| CHILD_TERM_DICTIONARY.contains(term)) +} + +fn references_sex(prompt_tokens: &[String]) -> bool { + prompt_tokens.iter().any(|term| SEX_TERM_DICTIONARY.contains(term)) +} + +fn references_violence(prompt_tokens: &[String]) -> bool { + // TODO(bt,2024-02-19): Need to implement + false +} + +fn references_racism(prompt_tokens: &[String]) -> bool { + // TODO(bt,2024-02-19): Need to implement + false +} + + +#[cfg(test)] +mod tests { + use crate::util::classify_prompt::classify_prompt; + use speculoos::asserting; + + #[test] + fn test_bad_user_csam_prompts() { + // These were actual prompts from several bad actors (username "ElFabro17", "markiwrench") + // These need to be blocked. + let prompts = [ + "A beautiful blonde girl of 13 years old with blue eyes.She's naked ,showing her private parts ", + "A blonde girl of 13 years old with blue eyes.She's naked ,showing her private parts. I'm fucking her now", + "A blonde girl of 13 years old with blue eyes.She's naked ,showing her tits and ass", + "Boy jerking off", + "loli butt panties", + "loli n@ked", + "loli on knees, back side view, feet, butt, pink panties on.", + "loli p00ping in toilet", + "loli sitting on toilet", + "loli skirt cute upskirt pink panties", + "loli", + "male kid in underwear, big butt", + "young boy in underwear, front view, bulge in underwear", + "young boy p3nis", + "young girl butt, pooping on bed", + "young loli butt, 13 years old", + ]; + + for prompt in prompts { + let classification = classify_prompt(prompt); + + asserting(&format!("is abusive - prompt: {}", prompt)) + .that(&classification.is_abusive()) + .is_equal_to(true); + + asserting(&format!("is child abuse - prompt: {}", prompt)) + .that(&classification.is_child_abuse()) + .is_equal_to(true); + } + } + + #[test] + fn borderline_cases() { + // These are actual user prompts that use borderline language, but are safe + let prompts = [ + "girl in a strapless dress in a feald of flower", + ]; + + for prompt in prompts { + let classification = classify_prompt(prompt); + assert!(!classification.is_abusive()); + assert!(!classification.is_child_abuse()); + } + } + + #[test] + fn permissable_nsfw() { + // We're allowing NSFW for now, but this may change + // These may not be underage or illegal + let prompts = [ + "Hot naked anime maids", + "Naked, cute, hot, nice boobs", + ]; + + for prompt in prompts { + let classification = classify_prompt(prompt); + assert!(!classification.is_abusive()); + assert!(!classification.is_child_abuse()); + } + } + + #[test] + fn okay_cases() { + // These are actual user prompts that use borderline language, but are safe + let prompts = [ + "Maid with the prince standing next to her", + "boy with black and blue hair in a red shirt and jeans happy", + "girl in a prom dress crying in the rain", + "girl teen in a dress with horns and a tail in a feald of flowers", + "girl with bloned hair wering a light blue dress whell drawing with a boy with black hair wering a red shirt siting nex to her", + "girl with horns and a tail that has durdyblond hair wereing a pink sweter and riped jeans", + ]; + + for prompt in prompts { + let classification = classify_prompt(prompt); + assert!(!classification.is_abusive()); + assert!(!classification.is_child_abuse()); + } + } +} diff --git a/crates/service/web/storyteller_web/src/util/mod.rs b/crates/service/web/storyteller_web/src/util/mod.rs index a6a0fef0d9..537c733406 100644 --- a/crates/service/web/storyteller_web/src/util/mod.rs +++ b/crates/service/web/storyteller_web/src/util/mod.rs @@ -2,4 +2,5 @@ pub mod allowed_studio_access; pub mod delete_role_disambiguation; pub mod encrypted_sort_id; pub mod placeholder_images; -pub mod troll_user_bans; \ No newline at end of file +pub mod troll_user_bans; +pub mod classify_prompt; \ No newline at end of file diff --git a/includes/binary_includes/dictionary_children_terms.txt b/includes/binary_includes/dictionary_children_terms.txt new file mode 100644 index 0000000000..9bbbe9d7d5 --- /dev/null +++ b/includes/binary_includes/dictionary_children_terms.txt @@ -0,0 +1,24 @@ +1boy +1girl +baby +boy +brat +child +daughter +girl +juvenile +kid +kiddie +loli +lolicon +minor +offspring +pedo +pubescent +son +toddler +tot +tyke +young +youngster +youth diff --git a/includes/binary_includes/dictionary_sex_terms.txt b/includes/binary_includes/dictionary_sex_terms.txt new file mode 100644 index 0000000000..8423f763f0 --- /dev/null +++ b/includes/binary_includes/dictionary_sex_terms.txt @@ -0,0 +1,587 @@ +# Other words we've uncovered from users +butt +butts +loli +n@ked +p3nis +private +underwear +upskirt + +# Adapted from: +# https://github.com/snguyenthanh/better_profanity/blob/master/better_profanity/profanity_wordlist.txt +anal +anus +areole +arrse +arse +arsehole +asanchez +ass +assbang +assbanged +asses +assfuck +assfucker +assfukka +asshole +assmunch +asswhole +autoerotic +ballsack +bdsm +beastial +beastiality +bestial +bestiality +bimbo +bimbos +bitch +bitches +bitching +blowjob +blowjobs +bondage +boner +boob +boobs +booobs +boooobs +booooobs +booooooobs +booty +bootycall +breasts +bukake +bukkake +bullshit +busty +butthole +cawk +clit +clitoris +clits +cock +cockface +cockhead +cockmunch +cockmuncher +cocks +cocksuck +cocksucked +cocksucker +cocksucking +cocksucks +cokmuncher +crotch +cum +cuming +cummer +cumming +cums +cumshot +cunilingus +cunillingus +cunnilingus +cunt +cuntlicker +cuntlicking +cunts +deepthroat +dick +dickhead +dildo +dildos +dogfucker +doggiestyle +doggin +dogging +doggy +doggystyle +dong +donkeyribber +douche +dyke +dykes +eatadick +ejaculate +ejaculated +ejaculates +ejaculating +ejaculatings +ejaculation +ejakulate +enlargement +erect +erection +erotic +erotism +facial +fag +fagg +fagged +fagging +faggit +faggitt +faggot +faggs +fagot +fagots +fags +fanny +fannybandit +fannyflaps +fannyfucker +fanyy +fcuk +fcuker +fcuking +felch +felcher +felching +fellate +fellatio +feltch +feltcher +femdom +fingerfuck +fingerfucked +fingerfucker +fingerfuckers +fingerfucking +fingerfucks +fingering +fisted +fistfuck +fistfucked +fistfucker +fistfuckers +fistfucking +fistfuckings +fistfucks +fisting +floozy +fondle +footjob +foreskin +fuck +fucka +fuckass +fuckbitch +fucked +fucker +fuckers +fuckface +fuckhead +fuckheads +fuckhole +fuckin +fucking +fuckings +fuckingshitmotherfucker +fuckme +fuckmeat +fucknugget +fucknut +fuckoff +fuckpuppet +fucks +fucktard +fucktoy +fucktrophy +fuckup +fuckwad +fuckwhit +fuckwit +fuckyomama +fudgepacker +fuk +fuker +fukker +fukkin +fukking +fuks +fukwhit +fukwit +futanari +futanary +fux +gangbang +gangbanged +gangbangs +gassyass +gaysex +gigolo +glans +goatse +goldenshower +gonad +gonads +gspot +hamflap +handjob +hardcoresex +hardon +hentai +homoerotic +hooch +hookah +hooker +hooter +hooters +horniest +horny +hotsex +hump +humped +humping +hymen +inbred +incest +jackhole +jackoff +jerked +jerking +jerkoff +jism +jiz +jizm +jizz +jizzed +kawk +kinky +kock +kondum +kondums +kooch +kooches +kootch +kum +kummer +kumming +kums +kunilingus +labia +lesbians +lesbo +lesbos +lezbian +lezbians +lezbo +lezbos +lezzie +lezzies +lezzy +loin +loins +lube +lust +lusting +lusty +m-fucking +masochist +masterb8 +masterbate +masterbating +masterbation +masterbations +masturbate +masturbating +masturbation +maxi +menses +menstruate +menstruation +milf +molest +mothafuck +mothafucka +mothafuckas +mothafuckaz +mothafucked +mothafucker +mothafuckers +mothafuckin +mothafucking +mothafuckings +mothafucks +motherfuck +motherfucka +motherfucked +motherfucker +motherfuckers +motherfuckin +motherfucking +motherfuckings +motherfuckka +motherfucks +mtherfucker +mthrfucker +mthrfucking +muff +muffdiver +muffpuff +muthafecker +muthafuckaz +muthafucker +muthafuckker +mutherfucker +mutherfucking +muthrfucking +naked +nipple +nipples +nude +nudes +nutsack +nympho +oral +orally +organ +orgasim +orgasims +orgasm +orgasmic +orgasms +orgies +orgy +ovary +ovum +ovums +pantie +panties +panty +pastie +pasty +pecker +pedo +pedophile +pedophilia +pedophiliac +pee +peepee +penetrate +penetration +penial +penile +penis +penisfucker +perversion +phalli +phallic +phonesex +phuck +phuk +phuked +phuking +phukked +phukking +phuks +phuq +pigfucker +pimp +pimpis +piss +pisser +pissers +pisses +pissflaps +pissin +pissing +pissoff +playboy +pms +poon +poontang +poop +porn +porno +pornography +pornos +potty +prick +pricks +pron +prostitute +prude +pube +pubic +pubis +pusse +pussi +pussies +pussy +pussyfart +pussypalace +pussypounder +pussys +queaf +queef +queer +quicky +quim +racy +rape +raped +raper +raping +rapist +raunch +rectal +rectum +rectus +rimjob +rimming +rump +rumprammer +sadism +sadist +sausagequeen +scantily +schlong +screw +screwed +screwing +scrote +scrotum +seamen +seduce +semen +sex +sexual +shag +shagger +shaggin +shagging +shamedame +shemale +shibari +shibary +shit +shitdick +shite +shiteater +shited +shitey +shitface +shitfuck +shitfucker +shitfull +shithead +shithole +shithouse +shiting +shitings +shits +shitt +shitted +shitter +shitters +shitting +shittings +shitty +shota +sissy +skag +skank +sleaze +sleazy +slut +slutbucket +slutdumper +slutkiss +sluts +smegma +smut +smutty +snatch +snuff +sodom +sperm +spooge +spunk +steamy +stiffy +strip +stripclub +stroke +suck +sucked +sucking +sumofabiatch +tampon +teabagging +teat +teets +teste +testee +testes +testical +testicle +testis +threesome +throating +thrust +tinkle +tit +titfuck +titi +tits +titt +tittiefucker +titties +titty +tittyfuck +tittyfucker +tittywank +titwank +tramp +transsexual +trashy +tubgirl +turd +twat +twathead +twats +twatty +twunt +twunter +undies +urinal +urine +uterus +vag +vagina +virgin +vixen +vomit +voyeur +vulgar +vulva +wang +wank +wanker +wanky +weenie +weewee +weiner +wench +whoar +whoralicious +whore +whorealicious +whored +whoreface +whorehopper +whorehouse +whores +whoring +willies +willy +womb +woody +xx +xxx +yaoi +yuri +yury \ No newline at end of file