From c4e591ef0c56febe0ad18a0b32d84c2423afc657 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 9 Apr 2025 18:57:53 -0400 Subject: [PATCH] refactor packages --- Cargo.lock | 1 + Cargo.toml | 4 +++- crates/lib/machine_learning/ml_models/Cargo.toml | 3 +++ .../{ => machine_learning}/prompt_engineering/Cargo.toml | 6 +++--- .../prompt_engineering/src/classify_prompt.rs | 0 .../src/keywords/contains_children_keyword.rs | 2 +- .../src/keywords/contains_potential_minor_keyword.rs | 2 +- .../src/keywords/contains_racist_keyword.rs | 2 +- .../prompt_engineering/src/keywords/contains_sex_keyword.rs | 2 +- .../prompt_engineering/src/keywords/mod.rs | 0 .../{ => machine_learning}/prompt_engineering/src/lib.rs | 0 .../prompt_engineering/src/regexes/minor_regex.rs | 0 .../prompt_engineering/src/regexes/mod.rs | 0 .../prompt_engineering/src/regexes/nsfw_regex.rs | 0 .../prompt_engineering/src/regexes/potential_minor_regex.rs | 0 .../prompt_engineering/tests/borderline_tests.rs | 0 .../prompt_engineering/tests/csam_prompt_tests.rs | 0 .../prompt_engineering/tests/nsfw_prompt_tests.rs | 0 .../prompt_engineering/tests/racist_prompt_tests.rs | 0 .../prompt_engineering/tests/sfw_prompt_tests.rs | 0 .../prompt_engineering/tests/spicy_prompt_tests.rs | 0 crates/service/web/storyteller_web/Cargo.toml | 6 +++--- 22 files changed, 17 insertions(+), 11 deletions(-) rename crates/lib/{ => machine_learning}/prompt_engineering/Cargo.toml (86%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/classify_prompt.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/keywords/contains_children_keyword.rs (80%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs (79%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/keywords/contains_racist_keyword.rs (80%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/keywords/contains_sex_keyword.rs (80%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/keywords/mod.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/lib.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/regexes/minor_regex.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/regexes/mod.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/regexes/nsfw_regex.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/src/regexes/potential_minor_regex.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/tests/borderline_tests.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/tests/csam_prompt_tests.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/tests/nsfw_prompt_tests.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/tests/racist_prompt_tests.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/tests/sfw_prompt_tests.rs (100%) rename crates/lib/{ => machine_learning}/prompt_engineering/tests/spicy_prompt_tests.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index d33b27463b..b5540c47df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7007,6 +7007,7 @@ dependencies = [ "log", "memory_caching", "ml_weights_registry", + "my-workspace-hack", "ndarray", "nvidia-checker", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 3f103df1d2..df089781a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,11 +63,13 @@ members = [ "crates/lib/image_algorithms", "crates/lib/jobs_common", "crates/lib/logging", + "crates/lib/machine_learning/ml_models", + "crates/lib/machine_learning/ml_weights_registry", + "crates/lib/machine_learning/prompt_engineering", "crates/lib/markdown", "crates/lib/migration", "crates/lib/mysql_queries", "crates/lib/password", - "crates/lib/prompt_engineering", "crates/lib/redis_common", "crates/lib/redis_schema", "crates/lib/server_environment", diff --git a/crates/lib/machine_learning/ml_models/Cargo.toml b/crates/lib/machine_learning/ml_models/Cargo.toml index d01e0a7146..d3f66cb557 100644 --- a/crates/lib/machine_learning/ml_models/Cargo.toml +++ b/crates/lib/machine_learning/ml_models/Cargo.toml @@ -18,6 +18,9 @@ normal = ["my-workspace-hack"] [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { workspace = true } + # Internal filesys = { path = "../../files/filesys" } image_algorithms = { path = "../../image_algorithms" } diff --git a/crates/lib/prompt_engineering/Cargo.toml b/crates/lib/machine_learning/prompt_engineering/Cargo.toml similarity index 86% rename from crates/lib/prompt_engineering/Cargo.toml rename to crates/lib/machine_learning/prompt_engineering/Cargo.toml index a4e01ab904..3e45f5d999 100644 --- a/crates/lib/prompt_engineering/Cargo.toml +++ b/crates/lib/machine_learning/prompt_engineering/Cargo.toml @@ -13,12 +13,12 @@ path = "src/lib.rs" normal = ["my-workspace-hack"] [dependencies] +# Workspace hack (faster compile times) +my-workspace-hack = { workspace = true } + # Internal #errors = { path = "../errors" } -# Workspace hack (faster compile times) -my-workspace-hack = { version = "0.1", path = "../../my-workspace-hack" } - # External once_cell = "1.18.0" regex = "1.5.4" diff --git a/crates/lib/prompt_engineering/src/classify_prompt.rs b/crates/lib/machine_learning/prompt_engineering/src/classify_prompt.rs similarity index 100% rename from crates/lib/prompt_engineering/src/classify_prompt.rs rename to crates/lib/machine_learning/prompt_engineering/src/classify_prompt.rs diff --git a/crates/lib/prompt_engineering/src/keywords/contains_children_keyword.rs b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs similarity index 80% rename from crates/lib/prompt_engineering/src/keywords/contains_children_keyword.rs rename to crates/lib/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs index 375b4a9436..f8d991f614 100644 --- a/crates/lib/prompt_engineering/src/keywords/contains_children_keyword.rs +++ b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS : &str = include_str!("../../../../../includes/binary_includes/dictionary_children_terms.txt"); +const TERMS : &str = include_str!("../../../../../../includes/binary_includes/dictionary_children_terms.txt"); pub (crate) fn contains_children_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY : Lazy> = Lazy::new(|| { diff --git a/crates/lib/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs similarity index 79% rename from crates/lib/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs rename to crates/lib/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs index b6cc17b8fb..a9b765af65 100644 --- a/crates/lib/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs +++ b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS: &str = include_str!("../../../../../includes/binary_includes/dictionary_potential_minor_terms.txt"); +const TERMS: &str = include_str!("../../../../../../includes/binary_includes/dictionary_potential_minor_terms.txt"); pub (crate) fn contains_potential_minor_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY: Lazy> = Lazy::new(|| { diff --git a/crates/lib/prompt_engineering/src/keywords/contains_racist_keyword.rs b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs similarity index 80% rename from crates/lib/prompt_engineering/src/keywords/contains_racist_keyword.rs rename to crates/lib/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs index 6c1609cc8d..033857355a 100644 --- a/crates/lib/prompt_engineering/src/keywords/contains_racist_keyword.rs +++ b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS : &str = include_str!("../../../../../includes/binary_includes/dictionary_racist_terms.txt"); +const TERMS : &str = include_str!("../../../../../../includes/binary_includes/dictionary_racist_terms.txt"); pub (crate) fn contains_racist_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY : Lazy> = Lazy::new(|| { diff --git a/crates/lib/prompt_engineering/src/keywords/contains_sex_keyword.rs b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs similarity index 80% rename from crates/lib/prompt_engineering/src/keywords/contains_sex_keyword.rs rename to crates/lib/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs index f83455c35c..b14ac1a603 100644 --- a/crates/lib/prompt_engineering/src/keywords/contains_sex_keyword.rs +++ b/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS : &str = include_str!("../../../../../includes/binary_includes/dictionary_sex_terms.txt"); +const TERMS : &str = include_str!("../../../../../../includes/binary_includes/dictionary_sex_terms.txt"); pub (crate) fn contains_sex_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY : Lazy> = Lazy::new(|| { TERMS.lines() diff --git a/crates/lib/prompt_engineering/src/keywords/mod.rs b/crates/lib/machine_learning/prompt_engineering/src/keywords/mod.rs similarity index 100% rename from crates/lib/prompt_engineering/src/keywords/mod.rs rename to crates/lib/machine_learning/prompt_engineering/src/keywords/mod.rs diff --git a/crates/lib/prompt_engineering/src/lib.rs b/crates/lib/machine_learning/prompt_engineering/src/lib.rs similarity index 100% rename from crates/lib/prompt_engineering/src/lib.rs rename to crates/lib/machine_learning/prompt_engineering/src/lib.rs diff --git a/crates/lib/prompt_engineering/src/regexes/minor_regex.rs b/crates/lib/machine_learning/prompt_engineering/src/regexes/minor_regex.rs similarity index 100% rename from crates/lib/prompt_engineering/src/regexes/minor_regex.rs rename to crates/lib/machine_learning/prompt_engineering/src/regexes/minor_regex.rs diff --git a/crates/lib/prompt_engineering/src/regexes/mod.rs b/crates/lib/machine_learning/prompt_engineering/src/regexes/mod.rs similarity index 100% rename from crates/lib/prompt_engineering/src/regexes/mod.rs rename to crates/lib/machine_learning/prompt_engineering/src/regexes/mod.rs diff --git a/crates/lib/prompt_engineering/src/regexes/nsfw_regex.rs b/crates/lib/machine_learning/prompt_engineering/src/regexes/nsfw_regex.rs similarity index 100% rename from crates/lib/prompt_engineering/src/regexes/nsfw_regex.rs rename to crates/lib/machine_learning/prompt_engineering/src/regexes/nsfw_regex.rs diff --git a/crates/lib/prompt_engineering/src/regexes/potential_minor_regex.rs b/crates/lib/machine_learning/prompt_engineering/src/regexes/potential_minor_regex.rs similarity index 100% rename from crates/lib/prompt_engineering/src/regexes/potential_minor_regex.rs rename to crates/lib/machine_learning/prompt_engineering/src/regexes/potential_minor_regex.rs diff --git a/crates/lib/prompt_engineering/tests/borderline_tests.rs b/crates/lib/machine_learning/prompt_engineering/tests/borderline_tests.rs similarity index 100% rename from crates/lib/prompt_engineering/tests/borderline_tests.rs rename to crates/lib/machine_learning/prompt_engineering/tests/borderline_tests.rs diff --git a/crates/lib/prompt_engineering/tests/csam_prompt_tests.rs b/crates/lib/machine_learning/prompt_engineering/tests/csam_prompt_tests.rs similarity index 100% rename from crates/lib/prompt_engineering/tests/csam_prompt_tests.rs rename to crates/lib/machine_learning/prompt_engineering/tests/csam_prompt_tests.rs diff --git a/crates/lib/prompt_engineering/tests/nsfw_prompt_tests.rs b/crates/lib/machine_learning/prompt_engineering/tests/nsfw_prompt_tests.rs similarity index 100% rename from crates/lib/prompt_engineering/tests/nsfw_prompt_tests.rs rename to crates/lib/machine_learning/prompt_engineering/tests/nsfw_prompt_tests.rs diff --git a/crates/lib/prompt_engineering/tests/racist_prompt_tests.rs b/crates/lib/machine_learning/prompt_engineering/tests/racist_prompt_tests.rs similarity index 100% rename from crates/lib/prompt_engineering/tests/racist_prompt_tests.rs rename to crates/lib/machine_learning/prompt_engineering/tests/racist_prompt_tests.rs diff --git a/crates/lib/prompt_engineering/tests/sfw_prompt_tests.rs b/crates/lib/machine_learning/prompt_engineering/tests/sfw_prompt_tests.rs similarity index 100% rename from crates/lib/prompt_engineering/tests/sfw_prompt_tests.rs rename to crates/lib/machine_learning/prompt_engineering/tests/sfw_prompt_tests.rs diff --git a/crates/lib/prompt_engineering/tests/spicy_prompt_tests.rs b/crates/lib/machine_learning/prompt_engineering/tests/spicy_prompt_tests.rs similarity index 100% rename from crates/lib/prompt_engineering/tests/spicy_prompt_tests.rs rename to crates/lib/machine_learning/prompt_engineering/tests/spicy_prompt_tests.rs diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 1fc7bb6a6f..5946be9280 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -55,21 +55,21 @@ memory_caching = { path = "../../../lib/caching/memory_caching" } migration = { path = "../../../lib/migration" } mimetypes = { path = "../../../lib/files/mimetypes" } mysql_queries = { path = "../../../lib/mysql_queries" } +openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" } password = { path = "../../../lib/password" } primitives = { path = "../../../lib/data/primitives" } -prompt_engineering = { path = "../../../lib/prompt_engineering" } +prompt_engineering = { path = "../../../lib/machine_learning/prompt_engineering" } redis_caching = { path = "../../../lib/caching/redis_caching" } redis_common = { path = "../../../lib/redis_common" } redis_schema = { path = "../../../lib/redis_schema" } reusable_types = { path = "../../../_types/reusable_types" } server_environment = { path = "../../../lib/server_environment" } -tokens = { path = "../../../_types/tokens" } thumbnail_generator = { path = "../../../lib/api_clients/thumbnail_generator" } +tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } url_config = { path = "../../../lib/url_config" } user_input_common = { path = "../../../lib/user_input_common" } videos = { path = "../../../lib/files/videos" } -openai_sora_client = { path = "../../../lib/api_clients/openai_sora_client" } # Internal Components billing_component = { path = "../../plugins/billing" }