diff --git a/crates/lib/classify_text/src/classify_prompt.rs b/crates/lib/classify_text/src/classify_prompt.rs index 6295e0c84d..80172c42c2 100644 --- a/crates/lib/classify_text/src/classify_prompt.rs +++ b/crates/lib/classify_text/src/classify_prompt.rs @@ -4,6 +4,7 @@ use once_cell::sync::Lazy; use regex::Regex; use crate::minor_regex::lowercase_mentions_underage; +use crate::potential_minor_regex::lowercase_mentions_potential_underage; const CHILD_TERMS : &str = include_str!("../../../../includes/binary_includes/dictionary_children_terms.txt"); const SEX_TERMS : &str = include_str!("../../../../includes/binary_includes/dictionary_sex_terms.txt"); @@ -12,6 +13,7 @@ const RACIST_TERMS : &str = include_str!("../../../../includes/binary_includes/d /// Classification of a text prompt pub struct PromptClassification { pub prompt_references_children: bool, + pub prompt_references_potential_minors: bool, pub prompt_references_sex: bool, pub prompt_references_violence: bool, pub prompt_references_racism: bool, @@ -46,25 +48,21 @@ pub fn classify_prompt(text_prompt: &str) -> PromptClassification { prompt_tokens.extend(prompt_tokens_alpha); prompt_tokens.extend(prompt_tokens_lower); - let mut prompt_references_children = references_children(&prompt_tokens); + let prompt_references_children = references_children(&prompt_tokens) || lowercase_mentions_underage(&text_prompt); + let prompt_references_potential_minors = lowercase_mentions_potential_underage(&text_prompt); let prompt_references_sex = references_sex(&prompt_tokens); let prompt_references_violence = references_violence(&prompt_tokens); let prompt_references_racism = references_racism(&prompt_tokens); - if lowercase_mentions_underage(&text_prompt) { - prompt_references_children = true; - } - PromptClassification { prompt_references_children, + prompt_references_potential_minors, prompt_references_sex, prompt_references_violence, prompt_references_racism, } } - - fn references_children(prompt_tokens: &[String]) -> bool { static CHILD_TERM_DICTIONARY : Lazy> = Lazy::new(|| { CHILD_TERMS.lines() diff --git a/crates/lib/classify_text/src/lib.rs b/crates/lib/classify_text/src/lib.rs index 6dae4be774..af1c9d67ef 100644 --- a/crates/lib/classify_text/src/lib.rs +++ b/crates/lib/classify_text/src/lib.rs @@ -19,5 +19,6 @@ #![allow(non_snake_case)] pub (crate) mod minor_regex; +pub (crate) mod potential_minor_regex; pub mod classify_prompt; diff --git a/crates/lib/classify_text/src/minor_regex.rs b/crates/lib/classify_text/src/minor_regex.rs index a22fda2eff..202ee08e55 100644 --- a/crates/lib/classify_text/src/minor_regex.rs +++ b/crates/lib/classify_text/src/minor_regex.rs @@ -2,6 +2,7 @@ use once_cell::sync::Lazy; use regex::RegexSet; pub (crate) fn lowercase_mentions_underage(text: &str) -> bool { + let text = text.to_lowercase(); // NB: Unfortunately Rust Regex does not support look-around, so our age matching is awkward // Matching digits "0" and "1" are tricky because they would match "19" and "0100". @@ -13,7 +14,10 @@ pub (crate) fn lowercase_mentions_underage(text: &str) -> bool { r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)([^\d]){1,3}(years?)", r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)yos?", r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)yrs?", - r"\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|tweleve|thirteen|fourteen|fifteen|sixteen|seventeen)(.{1,3}|\w+)(years?)(.{1,3}|\w+)(old)", + r"\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen)(.{0,3}|\w+)(years?)(.{0,3}|\w+)(old)", + + // English typos + r"\b(tweleve|fiveteen)(.{0,3}|\w+)(years?)(.{0,3}|\w+)(old)", // Spanish r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)([^\d])+(años?)", @@ -26,7 +30,7 @@ pub (crate) fn lowercase_mentions_underage(text: &str) -> bool { //println!("text: {:?}", text); - AGE_REGEX.matches(text).matched_any() + AGE_REGEX.matches(&text).matched_any() } #[cfg(test)] @@ -195,8 +199,10 @@ mod tests { "a blonde girl of 13 years old with blue eyes.she's naked ,showing her private parts. i'm fucking her now", "a blonde girl of 13 years old with blue eyes.she's naked ,showing her tits and ass", "a little boy have brown eyes and hair,nude body", + "Fiveteen years old, naked, Brown hair, big ass, Brest d, open mouth, close eyes, seamen face", "naked 8yo blonde girl", "seven year old girl naked legs spread", + "Twelve years old girl , sucking thirty years old cock men", "young loli butt, 13 years old", // Spanish diff --git a/crates/lib/classify_text/src/potential_minor_regex.rs b/crates/lib/classify_text/src/potential_minor_regex.rs new file mode 100644 index 0000000000..1e7311b611 --- /dev/null +++ b/crates/lib/classify_text/src/potential_minor_regex.rs @@ -0,0 +1,62 @@ +use once_cell::sync::Lazy; +use regex::RegexSet; + +pub (crate) fn lowercase_mentions_potential_underage(text: &str) -> bool { + let text = text.to_lowercase(); + + // NB: Unfortunately Rust Regex does not support look-around, so our age matching is awkward + // Matching digits "0" and "1" are tricky because they would match "19" and "0100". + static AGE_REGEX: Lazy = Lazy::new(|| { + let patterns = [ + // English + r"\b(teen(age(r)?)?s?)\b", + r"\b(young|tiny|small|petite)\b(.{0,7})\b(girl|boy)s?\b", + + ]; + RegexSet::new(&patterns).expect("regex should be valid") + }); + + //println!("text: {:?}", text); + + AGE_REGEX.matches(&text).matched_any() +} + +#[cfg(test)] +mod tests { + + mod english { + + mod positive_match { + use crate::potential_minor_regex::lowercase_mentions_potential_underage; + + #[test] + fn spot_check_matches() { + assert!(lowercase_mentions_potential_underage("teen")); + assert!(lowercase_mentions_potential_underage("teens")); + assert!(lowercase_mentions_potential_underage("teenage")); + assert!(lowercase_mentions_potential_underage("teenager")); + assert!(lowercase_mentions_potential_underage("teenagers")); + + assert!(lowercase_mentions_potential_underage("tiny girl")); + assert!(lowercase_mentions_potential_underage("tiny boy")); + assert!(lowercase_mentions_potential_underage("small boy")); + assert!(lowercase_mentions_potential_underage("small girl")); + + assert!(lowercase_mentions_potential_underage("tiny girls")); + assert!(lowercase_mentions_potential_underage("tiny boys")); + assert!(lowercase_mentions_potential_underage("small boys")); + assert!(lowercase_mentions_potential_underage("small girls")); + + assert!(lowercase_mentions_potential_underage("young girl")); + assert!(lowercase_mentions_potential_underage("young boy")); + assert!(lowercase_mentions_potential_underage("young boy")); + assert!(lowercase_mentions_potential_underage("young girl")); + + assert!(lowercase_mentions_potential_underage("petite girl")); + assert!(lowercase_mentions_potential_underage("petite boy")); + assert!(lowercase_mentions_potential_underage("petite boy")); + assert!(lowercase_mentions_potential_underage("petite girl")); + } + } + } +} diff --git a/crates/lib/classify_text/tests/csam_prompt_tests.rs b/crates/lib/classify_text/tests/csam_prompt_tests.rs index adc16c43c8..17329833cb 100644 --- a/crates/lib/classify_text/tests/csam_prompt_tests.rs +++ b/crates/lib/classify_text/tests/csam_prompt_tests.rs @@ -5,6 +5,27 @@ use speculoos::asserting; use classify_text::classify_prompt::classify_prompt; +#[test] +fn sfw_prompts_including_children() { + assert_prompts_are_sfw_but_contain_children(&[ + "one little girl wearing pink ballerina style tank top with pink long-sleeved cardigan and pink skirt lying down on pink princess tent background and watching to viewers, who smiles, going prone", + ]); +} + +#[test] +fn prompts_that_generate_nsfw_possibly_ambiguous_but_could_accidentally_generate_csam() { + // "Teen" usually means 18+. + // We should swap these words for "adult" in prompt engineering. + assert_prompts_mention_potential_minors(&[ + "Red hair, Brest d, almost naked, teenage ", + "teen nude girl, full body", + "teen nude girl, full body, puffy pussy", + "tiny nude girl", + "tiny nude girl, full body", + "tiny nude girl, sexy", + ]); +} + #[test] fn test_english_bad_user_csam_prompts() { assert_prompts_are_csam(&[ @@ -17,13 +38,17 @@ fn test_english_bad_user_csam_prompts() { "A blonde girl of 13 years old with blue eyes.She's naked ,showing her private parts (no showing her face)", "A blonde girl of 13 years old with blue eyes.She's naked ,showing her private parts. I'm fucking her now", "A blonde girl of 13 years old with blue eyes.She's naked ,showing her tits and ass", - "a little boy have brown eyes and hair,nude body", // NB: That's not a comma! "A young girl how eats her classmates with her butt-hole", "Beautiful young naked preteen girl hugging a young naked boy on a couch in front of a window. Boy Touch tits. Perfect teen’s breasts. Laneya Grace. superflat, a photorealistic 16k", "Boy jerking off", + "Fiveteen years old, naked, Brown hair, big ass, Brest d, open mouth, close eyes, seamen face", + "Little girl pussy cum dripping out", + "Twelve years old girl , sucking thirty years old cock men", + "Twelve years old girl, in bedroom pink , sucking thirty years old cock men", + "Young girls vore boys", + "a little boy have brown eyes and hair,nude body", // NB: That's not a comma! "child porn", "goth loli skirt, pink p@nties. butt", - "Little girl pussy cum dripping out", "loli butt panties", "loli n@ked", "loli on knees, back side view, feet, butt, pink panties on", @@ -37,10 +62,10 @@ fn test_english_bad_user_csam_prompts() { "naked 8yo blonde girl", "seven year old girl naked legs spread", "sex 2 people. Full body 14yo teen topless, and naked boy friends. with short and dr. martens. Perfect innocent face . At school . Details face. tan lines. slutty. puffy’s areolas. masturbate(Canon alpha7, perfect quality, HD, 8K", + "underage sex", "young boy in underwear, front view, bulge in underwear", "young boy p3nis", "young girl butt, pooping on bed", - "Young girls vore boys", "young loli butt, 13 years old", "yummy little jailbait teen girls naked", ]); @@ -74,6 +99,34 @@ fn test_italian_bad_user_csam_prompts() { ]); } +fn assert_prompts_are_sfw_but_contain_children(prompts: &[&str]) { + for prompt in prompts { + let classification = classify_prompt(prompt); + + asserting(&format!("is abusive - prompt: {}", prompt)) + .that(&classification.is_abusive()) + .is_equal_to(false); + + asserting(&format!("is child abuse - prompt: {}", prompt)) + .that(&classification.is_child_abuse()) + .is_equal_to(false); + + asserting(&format!("contains children - prompt: {}", prompt)) + .that(&classification.prompt_references_children) + .is_equal_to(true); + } +} + +fn assert_prompts_mention_potential_minors(prompts: &[&str]) { + for prompt in prompts { + let classification = classify_prompt(prompt); + + asserting(&format!("contains children - prompt: {}", prompt)) + .that(&classification.prompt_references_potential_minors) + .is_equal_to(true); + } +} + fn assert_prompts_are_csam(prompts: &[&str]) { for prompt in prompts { let classification = classify_prompt(prompt); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs index 6cc37fc0de..82dfe67dc2 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/enqueue_image_generation.rs @@ -110,7 +110,7 @@ pub fn is_valid_string(input: &str) -> bool { valid_strings.contains(&input) } -#[derive(Deserialize, ToSchema)] +#[derive(Deserialize, Default, ToSchema)] pub struct EnqueueImageGenRequest { pub uuid_idempotency_token: String, pub maybe_image_source: Option, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/prompt_enrichment.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/prompt_enrichment.rs index 21c77a120f..9bd30b9cc9 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/prompt_enrichment.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_gen/prompt_enrichment.rs @@ -10,6 +10,9 @@ pub struct EnrichedPrompts { const DEFAULT_NEGATIVE_PROMPT : &str = "nudity, nsfw, naked, sex"; +const ADD_TO_NSFW_POSITIVE_PROMPT: &str = "legal adult, legal age"; +const ADD_TO_NSFW_NEGATIVE_PROMPT: &str = "underage, child, minor, kid, children, young, youngling, younglings"; + /// Enrich the user-provided prompts. /// In the future, we can add prompt engineering to these. pub fn enrich_prompt(request: &EnqueueImageGenRequest) -> Option { @@ -37,6 +40,24 @@ pub fn enrich_prompt(request: &EnqueueImageGenRequest) -> Option { + maybe_negative_prompt = Some(ADD_TO_NSFW_NEGATIVE_PROMPT.to_string()); + } + Some(negative_prompt) => { + let new_negative_prompt = format!("{}, {}", negative_prompt, ADD_TO_NSFW_NEGATIVE_PROMPT); + maybe_negative_prompt = Some(new_negative_prompt); + } + } + } + + // These prompts get treated harshly. if classification.is_abusive() { // NB: Save the original prompt as the negative prompt so that we can study it. maybe_negative_prompt = Some(positive_prompt.clone()); @@ -48,3 +69,106 @@ pub fn enrich_prompt(request: &EnqueueImageGenRequest) -> Option, + } + + impl Prompt { + fn positive(positive: &'static str) -> Self { + Self { + positive, + maybe_negative: None, + } + } + + fn positive_negative(positive: &'static str, negative: &'static str) -> Self { + Self { + positive, + maybe_negative: Some(negative), + } + } + + fn to_request(&self) -> EnqueueImageGenRequest { + EnqueueImageGenRequest { + maybe_prompt: Some(self.positive.to_string()), + maybe_n_prompt: self.maybe_negative.as_deref().map(|s| s.to_string()), + ..Default::default() + } + } + } + + mod sfw { + use super::*; + + #[test] + fn sfw_prompt_no_negative_prompt() { + let result = enrich_prompt(&Prompt::positive("hello").to_request()).unwrap(); + + assert_eq!(result.positive_prompt, "hello".to_string()); + assert_eq!(result.maybe_negative_prompt, Some("nudity, nsfw, naked, sex".to_string())); + } + + #[test] + fn sfw_prompt_with_negative_prompt() { + let result = enrich_prompt(&Prompt::positive_negative("hello", "goodbye").to_request()).unwrap(); + + assert_eq!(result.positive_prompt, "hello".to_string()); + assert_eq!(result.maybe_negative_prompt, Some("goodbye, nudity, nsfw, naked, sex".to_string())); + } + } + + mod nsfw { + use super::*; + + #[test] + fn legal_nsfw_prompt_no_negative_prompt() { + let result = enrich_prompt(&Prompt::positive("sex").to_request()).unwrap(); + + assert_eq!(result.positive_prompt, "sex, legal adult, legal age".to_string()); + assert_eq!(result.maybe_negative_prompt, Some("underage, child, minor, kid, children, young, youngling, younglings".to_string())); + } + + #[test] + fn legal_nsfw_prompt_with_negative_prompt() { + let result = enrich_prompt(&Prompt::positive_negative("sex", "goodbye").to_request()).unwrap(); + + assert_eq!(result.positive_prompt, "sex, legal adult, legal age".to_string()); + assert_eq!(result.maybe_negative_prompt, Some("goodbye, underage, child, minor, kid, children, young, youngling, younglings".to_string())); + } + } + + mod illegal { + use super::*; + + #[test] + fn csam_prompt_no_negative_prompt() { + let result = enrich_prompt(&Prompt::positive("underage sex").to_request()).unwrap(); + + // Scrubbed from prompt + assert!(!result.positive_prompt.contains("underage")); + assert!(!result.positive_prompt.contains("sex")); + + // Bad prompt is preserved as the negative prompt + assert_eq!(result.maybe_negative_prompt, Some("underage sex".to_string())); + } + + #[test] + fn csam_prompt_with_negative_prompt() { + let result = enrich_prompt(&Prompt::positive_negative("underage sex", "goodbye").to_request()).unwrap(); + + // Scrubbed from prompt + assert!(!result.positive_prompt.contains("underage")); + assert!(!result.positive_prompt.contains("sex")); + + // Bad prompt is preserved as the negative prompt + assert_eq!(result.maybe_negative_prompt, Some("underage sex".to_string())); + } + } +} diff --git a/includes/binary_includes/dictionary_children_terms.txt b/includes/binary_includes/dictionary_children_terms.txt index 7d8a7ffbbe..6c872dda5b 100644 --- a/includes/binary_includes/dictionary_children_terms.txt +++ b/includes/binary_includes/dictionary_children_terms.txt @@ -44,6 +44,7 @@ toddler toddlers tot tyke +underage young youngster youngsters