mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-19 08:05:20 +00:00
additional safety features
This commit is contained in:
1 parent
814209949e
commit
6c6debe64b
8 files changed
+258
-13
No files matched your search
@@ -4,6 +4,7 @@ use once_cell::sync::Lazy;
|
||||
use regex::Regex;
|
||||
|
||||
use crate::minor_regex::lowercase_mentions_underage;
|
||||
use crate::potential_minor_regex::lowercase_mentions_potential_underage;
|
||||
|
||||
const CHILD_TERMS : &str = include_str!("../../../../includes/binary_includes/dictionary_children_terms.txt");
|
||||
const SEX_TERMS : &str = include_str!("../../../../includes/binary_includes/dictionary_sex_terms.txt");
|
||||
@@ -12,6 +13,7 @@ const RACIST_TERMS : &str = include_str!("../../../../includes/binary_includes/d
|
||||
/// Classification of a text prompt
|
||||
pub struct PromptClassification {
|
||||
pub prompt_references_children: bool,
|
||||
pub prompt_references_potential_minors: bool,
|
||||
pub prompt_references_sex: bool,
|
||||
pub prompt_references_violence: bool,
|
||||
pub prompt_references_racism: bool,
|
||||
@@ -46,25 +48,21 @@ pub fn classify_prompt(text_prompt: &str) -> PromptClassification {
|
||||
prompt_tokens.extend(prompt_tokens_alpha);
|
||||
prompt_tokens.extend(prompt_tokens_lower);
|
||||
|
||||
let mut prompt_references_children = references_children(&prompt_tokens);
|
||||
let prompt_references_children = references_children(&prompt_tokens) || lowercase_mentions_underage(&text_prompt);
|
||||
let prompt_references_potential_minors = lowercase_mentions_potential_underage(&text_prompt);
|
||||
let prompt_references_sex = references_sex(&prompt_tokens);
|
||||
let prompt_references_violence = references_violence(&prompt_tokens);
|
||||
let prompt_references_racism = references_racism(&prompt_tokens);
|
||||
|
||||
if lowercase_mentions_underage(&text_prompt) {
|
||||
prompt_references_children = true;
|
||||
}
|
||||
|
||||
PromptClassification {
|
||||
prompt_references_children,
|
||||
prompt_references_potential_minors,
|
||||
prompt_references_sex,
|
||||
prompt_references_violence,
|
||||
prompt_references_racism,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
fn references_children(prompt_tokens: &[String]) -> bool {
|
||||
static CHILD_TERM_DICTIONARY : Lazy<HashSet<String>> = Lazy::new(|| {
|
||||
CHILD_TERMS.lines()
|
||||
|
||||
@@ -19,5 +19,6 @@
|
||||
#![allow(non_snake_case)]
|
||||
|
||||
pub (crate) mod minor_regex;
|
||||
pub (crate) mod potential_minor_regex;
|
||||
|
||||
pub mod classify_prompt;
|
||||
@@ -2,6 +2,7 @@ use once_cell::sync::Lazy;
|
||||
use regex::RegexSet;
|
||||
|
||||
pub (crate) fn lowercase_mentions_underage(text: &str) -> bool {
|
||||
let text = text.to_lowercase();
|
||||
|
||||
// NB: Unfortunately Rust Regex does not support look-around, so our age matching is awkward
|
||||
// Matching digits "0" and "1" are tricky because they would match "19" and "0100".
|
||||
@@ -13,7 +14,10 @@ pub (crate) fn lowercase_mentions_underage(text: &str) -> bool {
|
||||
r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)([^\d]){1,3}(years?)",
|
||||
r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)yos?",
|
||||
r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)yrs?",
|
||||
r"\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|tweleve|thirteen|fourteen|fifteen|sixteen|seventeen)(.{1,3}|\w+)(years?)(.{1,3}|\w+)(old)",
|
||||
r"\b(one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve|thirteen|fourteen|fifteen|sixteen|seventeen)(.{0,3}|\w+)(years?)(.{0,3}|\w+)(old)",
|
||||
|
||||
// English typos
|
||||
r"\b(tweleve|fiveteen)(.{0,3}|\w+)(years?)(.{0,3}|\w+)(old)",
|
||||
|
||||
// Spanish
|
||||
r"\b(0|1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17)([^\d])+(años?)",
|
||||
@@ -26,7 +30,7 @@ pub (crate) fn lowercase_mentions_underage(text: &str) -> bool {
|
||||
|
||||
//println!("text: {:?}", text);
|
||||
|
||||
AGE_REGEX.matches(text).matched_any()
|
||||
AGE_REGEX.matches(&text).matched_any()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -195,8 +199,10 @@ mod tests {
|
||||
"a blonde girl of 13 years old with blue eyes.she's naked ,showing her private parts. i'm fucking her now",
|
||||
"a blonde girl of 13 years old with blue eyes.she's naked ,showing her tits and ass",
|
||||
"a little boy have brown eyes and hair,nude body",
|
||||
"Fiveteen years old, naked, Brown hair, big ass, Brest d, open mouth, close eyes, seamen face",
|
||||
"naked 8yo blonde girl",
|
||||
"seven year old girl naked legs spread",
|
||||
"Twelve years old girl , sucking thirty years old cock men",
|
||||
"young loli butt, 13 years old",
|
||||
|
||||
// Spanish
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
use once_cell::sync::Lazy;
|
||||
use regex::RegexSet;
|
||||
|
||||
pub (crate) fn lowercase_mentions_potential_underage(text: &str) -> bool {
|
||||
let text = text.to_lowercase();
|
||||
|
||||
// NB: Unfortunately Rust Regex does not support look-around, so our age matching is awkward
|
||||
// Matching digits "0" and "1" are tricky because they would match "19" and "0100".
|
||||
static AGE_REGEX: Lazy<RegexSet> = Lazy::new(|| {
|
||||
let patterns = [
|
||||
// English
|
||||
r"\b(teen(age(r)?)?s?)\b",
|
||||
r"\b(young|tiny|small|petite)\b(.{0,7})\b(girl|boy)s?\b",
|
||||
|
||||
];
|
||||
RegexSet::new(&patterns).expect("regex should be valid")
|
||||
});
|
||||
|
||||
//println!("text: {:?}", text);
|
||||
|
||||
AGE_REGEX.matches(&text).matched_any()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
mod english {
|
||||
|
||||
mod positive_match {
|
||||
use crate::potential_minor_regex::lowercase_mentions_potential_underage;
|
||||
|
||||
#[test]
|
||||
fn spot_check_matches() {
|
||||
assert!(lowercase_mentions_potential_underage("teen"));
|
||||
assert!(lowercase_mentions_potential_underage("teens"));
|
||||
assert!(lowercase_mentions_potential_underage("teenage"));
|
||||
assert!(lowercase_mentions_potential_underage("teenager"));
|
||||
assert!(lowercase_mentions_potential_underage("teenagers"));
|
||||
|
||||
assert!(lowercase_mentions_potential_underage("tiny girl"));
|
||||
assert!(lowercase_mentions_potential_underage("tiny boy"));
|
||||
assert!(lowercase_mentions_potential_underage("small boy"));
|
||||
assert!(lowercase_mentions_potential_underage("small girl"));
|
||||
|
||||
assert!(lowercase_mentions_potential_underage("tiny girls"));
|
||||
assert!(lowercase_mentions_potential_underage("tiny boys"));
|
||||
assert!(lowercase_mentions_potential_underage("small boys"));
|
||||
assert!(lowercase_mentions_potential_underage("small girls"));
|
||||
|
||||
assert!(lowercase_mentions_potential_underage("young girl"));
|
||||
assert!(lowercase_mentions_potential_underage("young boy"));
|
||||
assert!(lowercase_mentions_potential_underage("young boy"));
|
||||
assert!(lowercase_mentions_potential_underage("young girl"));
|
||||
|
||||
assert!(lowercase_mentions_potential_underage("petite girl"));
|
||||
assert!(lowercase_mentions_potential_underage("petite boy"));
|
||||
assert!(lowercase_mentions_potential_underage("petite boy"));
|
||||
assert!(lowercase_mentions_potential_underage("petite girl"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,27 @@ use speculoos::asserting;
|
||||
|
||||
use classify_text::classify_prompt::classify_prompt;
|
||||
|
||||
#[test]
|
||||
fn sfw_prompts_including_children() {
|
||||
assert_prompts_are_sfw_but_contain_children(&[
|
||||
"one little girl wearing pink ballerina style tank top with pink long-sleeved cardigan and pink skirt lying down on pink princess tent background and watching to viewers, who smiles, going prone",
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prompts_that_generate_nsfw_possibly_ambiguous_but_could_accidentally_generate_csam() {
|
||||
// "Teen" usually means 18+.
|
||||
// We should swap these words for "adult" in prompt engineering.
|
||||
assert_prompts_mention_potential_minors(&[
|
||||
"Red hair, Brest d, almost naked, teenage ",
|
||||
"teen nude girl, full body",
|
||||
"teen nude girl, full body, puffy pussy",
|
||||
"tiny nude girl",
|
||||
"tiny nude girl, full body",
|
||||
"tiny nude girl, sexy",
|
||||
]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_english_bad_user_csam_prompts() {
|
||||
assert_prompts_are_csam(&[
|
||||
@@ -17,13 +38,17 @@ fn test_english_bad_user_csam_prompts() {
|
||||
"A blonde girl of 13 years old with blue eyes.She's naked ,showing her private parts (no showing her face)",
|
||||
"A blonde girl of 13 years old with blue eyes.She's naked ,showing her private parts. I'm fucking her now",
|
||||
"A blonde girl of 13 years old with blue eyes.She's naked ,showing her tits and ass",
|
||||
"a little boy have brown eyes and hair,nude body", // NB: That's not a comma!
|
||||
"A young girl how eats her classmates with her butt-hole",
|
||||
"Beautiful young naked preteen girl hugging a young naked boy on a couch in front of a window. Boy Touch tits. Perfect teen’s breasts. Laneya Grace. superflat, a photorealistic 16k",
|
||||
"Boy jerking off",
|
||||
"Fiveteen years old, naked, Brown hair, big ass, Brest d, open mouth, close eyes, seamen face",
|
||||
"Little girl pussy cum dripping out",
|
||||
"Twelve years old girl , sucking thirty years old cock men",
|
||||
"Twelve years old girl, in bedroom pink , sucking thirty years old cock men",
|
||||
"Young girls vore boys",
|
||||
"a little boy have brown eyes and hair,nude body", // NB: That's not a comma!
|
||||
"child porn",
|
||||
"goth loli skirt, pink p@nties. butt",
|
||||
"Little girl pussy cum dripping out",
|
||||
"loli butt panties",
|
||||
"loli n@ked",
|
||||
"loli on knees, back side view, feet, butt, pink panties on",
|
||||
@@ -37,10 +62,10 @@ fn test_english_bad_user_csam_prompts() {
|
||||
"naked 8yo blonde girl",
|
||||
"seven year old girl naked legs spread",
|
||||
"sex 2 people. Full body 14yo teen topless, and naked boy friends. with short and dr. martens. Perfect innocent face . At school . Details face. tan lines. slutty. puffy’s areolas. masturbate(Canon alpha7, perfect quality, HD, 8K",
|
||||
"underage sex",
|
||||
"young boy in underwear, front view, bulge in underwear",
|
||||
"young boy p3nis",
|
||||
"young girl butt, pooping on bed",
|
||||
"Young girls vore boys",
|
||||
"young loli butt, 13 years old",
|
||||
"yummy little jailbait teen girls naked",
|
||||
]);
|
||||
@@ -74,6 +99,34 @@ fn test_italian_bad_user_csam_prompts() {
|
||||
]);
|
||||
}
|
||||
|
||||
fn assert_prompts_are_sfw_but_contain_children(prompts: &[&str]) {
|
||||
for prompt in prompts {
|
||||
let classification = classify_prompt(prompt);
|
||||
|
||||
asserting(&format!("is abusive - prompt: {}", prompt))
|
||||
.that(&classification.is_abusive())
|
||||
.is_equal_to(false);
|
||||
|
||||
asserting(&format!("is child abuse - prompt: {}", prompt))
|
||||
.that(&classification.is_child_abuse())
|
||||
.is_equal_to(false);
|
||||
|
||||
asserting(&format!("contains children - prompt: {}", prompt))
|
||||
.that(&classification.prompt_references_children)
|
||||
.is_equal_to(true);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_prompts_mention_potential_minors(prompts: &[&str]) {
|
||||
for prompt in prompts {
|
||||
let classification = classify_prompt(prompt);
|
||||
|
||||
asserting(&format!("contains children - prompt: {}", prompt))
|
||||
.that(&classification.prompt_references_potential_minors)
|
||||
.is_equal_to(true);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_prompts_are_csam(prompts: &[&str]) {
|
||||
for prompt in prompts {
|
||||
let classification = classify_prompt(prompt);
|
||||
|
||||
+1
-1
@@ -110,7 +110,7 @@ pub fn is_valid_string(input: &str) -> bool {
|
||||
valid_strings.contains(&input)
|
||||
}
|
||||
|
||||
#[derive(Deserialize, ToSchema)]
|
||||
#[derive(Deserialize, Default, ToSchema)]
|
||||
pub struct EnqueueImageGenRequest {
|
||||
pub uuid_idempotency_token: String,
|
||||
pub maybe_image_source: Option<String>,
|
||||
|
||||
+124
@@ -10,6 +10,9 @@ pub struct EnrichedPrompts {
|
||||
|
||||
const DEFAULT_NEGATIVE_PROMPT : &str = "nudity, nsfw, naked, sex";
|
||||
|
||||
const ADD_TO_NSFW_POSITIVE_PROMPT: &str = "legal adult, legal age";
|
||||
const ADD_TO_NSFW_NEGATIVE_PROMPT: &str = "underage, child, minor, kid, children, young, youngling, younglings";
|
||||
|
||||
/// Enrich the user-provided prompts.
|
||||
/// In the future, we can add prompt engineering to these.
|
||||
pub fn enrich_prompt(request: &EnqueueImageGenRequest) -> Option<EnrichedPrompts> {
|
||||
@@ -37,6 +40,24 @@ pub fn enrich_prompt(request: &EnqueueImageGenRequest) -> Option<EnrichedPrompts
|
||||
}
|
||||
}
|
||||
|
||||
if classification.prompt_references_sex {
|
||||
if !classification.is_abusive() {
|
||||
// Abusive cases get handled below
|
||||
positive_prompt = format!("{}, {}", positive_prompt, ADD_TO_NSFW_POSITIVE_PROMPT);
|
||||
}
|
||||
|
||||
match maybe_negative_prompt.as_deref() {
|
||||
None => {
|
||||
maybe_negative_prompt = Some(ADD_TO_NSFW_NEGATIVE_PROMPT.to_string());
|
||||
}
|
||||
Some(negative_prompt) => {
|
||||
let new_negative_prompt = format!("{}, {}", negative_prompt, ADD_TO_NSFW_NEGATIVE_PROMPT);
|
||||
maybe_negative_prompt = Some(new_negative_prompt);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// These prompts get treated harshly.
|
||||
if classification.is_abusive() {
|
||||
// NB: Save the original prompt as the negative prompt so that we can study it.
|
||||
maybe_negative_prompt = Some(positive_prompt.clone());
|
||||
@@ -48,3 +69,106 @@ pub fn enrich_prompt(request: &EnqueueImageGenRequest) -> Option<EnrichedPrompts
|
||||
maybe_negative_prompt,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::http_server::endpoints::image_gen::enqueue_image_generation::EnqueueImageGenRequest;
|
||||
use crate::http_server::endpoints::image_gen::prompt_enrichment::enrich_prompt;
|
||||
|
||||
struct Prompt {
|
||||
pub positive: &'static str,
|
||||
pub maybe_negative: Option<&'static str>,
|
||||
}
|
||||
|
||||
impl Prompt {
|
||||
fn positive(positive: &'static str) -> Self {
|
||||
Self {
|
||||
positive,
|
||||
maybe_negative: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn positive_negative(positive: &'static str, negative: &'static str) -> Self {
|
||||
Self {
|
||||
positive,
|
||||
maybe_negative: Some(negative),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_request(&self) -> EnqueueImageGenRequest {
|
||||
EnqueueImageGenRequest {
|
||||
maybe_prompt: Some(self.positive.to_string()),
|
||||
maybe_n_prompt: self.maybe_negative.as_deref().map(|s| s.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mod sfw {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn sfw_prompt_no_negative_prompt() {
|
||||
let result = enrich_prompt(&Prompt::positive("hello").to_request()).unwrap();
|
||||
|
||||
assert_eq!(result.positive_prompt, "hello".to_string());
|
||||
assert_eq!(result.maybe_negative_prompt, Some("nudity, nsfw, naked, sex".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sfw_prompt_with_negative_prompt() {
|
||||
let result = enrich_prompt(&Prompt::positive_negative("hello", "goodbye").to_request()).unwrap();
|
||||
|
||||
assert_eq!(result.positive_prompt, "hello".to_string());
|
||||
assert_eq!(result.maybe_negative_prompt, Some("goodbye, nudity, nsfw, naked, sex".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
mod nsfw {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn legal_nsfw_prompt_no_negative_prompt() {
|
||||
let result = enrich_prompt(&Prompt::positive("sex").to_request()).unwrap();
|
||||
|
||||
assert_eq!(result.positive_prompt, "sex, legal adult, legal age".to_string());
|
||||
assert_eq!(result.maybe_negative_prompt, Some("underage, child, minor, kid, children, young, youngling, younglings".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legal_nsfw_prompt_with_negative_prompt() {
|
||||
let result = enrich_prompt(&Prompt::positive_negative("sex", "goodbye").to_request()).unwrap();
|
||||
|
||||
assert_eq!(result.positive_prompt, "sex, legal adult, legal age".to_string());
|
||||
assert_eq!(result.maybe_negative_prompt, Some("goodbye, underage, child, minor, kid, children, young, youngling, younglings".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
mod illegal {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn csam_prompt_no_negative_prompt() {
|
||||
let result = enrich_prompt(&Prompt::positive("underage sex").to_request()).unwrap();
|
||||
|
||||
// Scrubbed from prompt
|
||||
assert!(!result.positive_prompt.contains("underage"));
|
||||
assert!(!result.positive_prompt.contains("sex"));
|
||||
|
||||
// Bad prompt is preserved as the negative prompt
|
||||
assert_eq!(result.maybe_negative_prompt, Some("underage sex".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn csam_prompt_with_negative_prompt() {
|
||||
let result = enrich_prompt(&Prompt::positive_negative("underage sex", "goodbye").to_request()).unwrap();
|
||||
|
||||
// Scrubbed from prompt
|
||||
assert!(!result.positive_prompt.contains("underage"));
|
||||
assert!(!result.positive_prompt.contains("sex"));
|
||||
|
||||
// Bad prompt is preserved as the negative prompt
|
||||
assert_eq!(result.maybe_negative_prompt, Some("underage sex".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,7 @@ toddler
|
||||
toddlers
|
||||
tot
|
||||
tyke
|
||||
underage
|
||||
young
|
||||
youngster
|
||||
youngsters
|
||||
|
||||
Reference in new issue
Block a user