diff --git a/Cargo.lock b/Cargo.lock index 94af3ecfce..57da835033 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -339,10 +339,10 @@ dependencies = [ "actix-web", "actix_helpers", "anyhow", - "cookies", "enums", "errors", "http_headers", + "jwt_signer", "log", "tokens", "workspace-build-acceleration", @@ -405,7 +405,7 @@ checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ "cfg-if", "cipher 0.3.0", - "cpufeatures 0.2.12", + "cpufeatures", "opaque-debug", ] @@ -417,7 +417,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher 0.4.4", - "cpufeatures 0.2.12", + "cpufeatures", ] [[package]] @@ -656,15 +656,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" -[[package]] -name = "archives" -version = "0.0.1" -dependencies = [ - "errors", - "workspace-build-acceleration", - "zip 0.6.6", -] - [[package]] name = "arcstr" version = "1.2.0" @@ -715,7 +706,6 @@ dependencies = [ "filesys", "futures 0.3.32", "grok_client", - "idempotency", "image", "images", "indexmap 2.13.0", @@ -748,6 +738,7 @@ dependencies = [ "tokio-util", "url", "url_utils", + "uuid_utils", "web_base64", "workspace-build-acceleration", "worldlabs_consumer_client", @@ -783,7 +774,6 @@ dependencies = [ "enums", "enums_public", "errors", - "idempotency", "log", "mimetypes", "reqwest 0.12.23", @@ -795,6 +785,7 @@ dependencies = [ "url", "url-builder", "uuid 1.22.0", + "uuid_utils", "workspace-build-acceleration", ] @@ -1222,19 +1213,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "atom_syndication" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "571832dcff775e26562e8e6930cd483de5587301d40d3a3b85d532b6383e15a7" -dependencies = [ - "chrono", - "derive_builder", - "diligent-date-parser", - "never", - "quick-xml 0.30.0", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -2398,13 +2376,6 @@ dependencies = [ "static_assertions", ] -[[package]] -name = "component_subsystem_inference_workloads" -version = "0.0.1" -dependencies = [ - "workspace-build-acceleration", -] - [[package]] name = "component_traits" version = "0.0.1" @@ -2597,17 +2568,6 @@ dependencies = [ "url", ] -[[package]] -name = "cookies" -version = "0.0.1" -dependencies = [ - "errors", - "hmac 0.10.1", - "jwt", - "sha2 0.9.9", - "workspace-build-acceleration", -] - [[package]] name = "core-foundation" version = "0.9.4" @@ -2711,15 +2671,6 @@ dependencies = [ "libc", ] -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - [[package]] name = "crc" version = "3.0.1" @@ -2880,19 +2831,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "cssparser" -version = "0.31.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be" -dependencies = [ - "cssparser-macros", - "dtoa-short", - "itoa 1.0.11", - "phf 0.11.2", - "smallvec", -] - [[package]] name = "cssparser" version = "0.35.0" @@ -2963,16 +2901,6 @@ dependencies = [ "darling_macro 0.13.4", ] -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - [[package]] name = "darling" version = "0.20.10" @@ -2997,20 +2925,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - [[package]] name = "darling_core" version = "0.20.10" @@ -3036,17 +2950,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", - "quote", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.20.10" @@ -3207,37 +3110,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core", - "syn 1.0.109", -] - [[package]] name = "derive_more" version = "0.99.17" @@ -3303,7 +3175,6 @@ dependencies = [ "errors", "filesys", "hashing", - "idempotency", "log", "mimetypes", "mysql_queries", @@ -3312,6 +3183,7 @@ dependencies = [ "storyteller_root", "tokens", "tokio", + "uuid_utils", "workspace-build-acceleration", ] @@ -3410,22 +3282,6 @@ dependencies = [ "crypto-common 0.2.1", ] -[[package]] -name = "diligent-date-parser" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6cf7fe294274a222363f84bcb63cdea762979a0443b4cf1f4f8fd17c86b1182" -dependencies = [ - "chrono", -] - -[[package]] -name = "dimensions" -version = "0.0.1" -dependencies = [ - "workspace-build-acceleration", -] - [[package]] name = "directories" version = "6.0.0" @@ -3604,7 +3460,7 @@ dependencies = [ "enums", "errors", "filesys", - "google_drive_common", + "google_drive_downloader", "hashing", "hostname 0.4.0", "hyper 1.6.0", @@ -3748,12 +3604,6 @@ dependencies = [ "getrandom 0.2.14", ] -[[package]] -name = "ego-tree" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" - [[package]] name = "ego-tree" version = "0.10.0" @@ -4926,7 +4776,7 @@ dependencies = [ ] [[package]] -name = "google_drive_common" +name = "google_drive_downloader" version = "0.0.1" dependencies = [ "anyhow", @@ -4998,7 +4848,7 @@ dependencies = [ "log4rs", "once_cell", "regex", - "scraper 0.24.0", + "scraper", "serde", "serde_json", "sha2 0.11.0-rc.5", @@ -5320,20 +5170,6 @@ dependencies = [ "windows 0.52.0", ] -[[package]] -name = "html5ever" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" -dependencies = [ - "log", - "mac", - "markup5ever 0.11.0", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "html5ever" version = "0.29.1" @@ -5806,14 +5642,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" -[[package]] -name = "idempotency" -version = "0.0.1" -dependencies = [ - "uuid 1.22.0", - "workspace-build-acceleration", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -5988,7 +5816,7 @@ dependencies = [ "errors", "ffmpeg_utils", "filesys", - "google_drive_common", + "google_drive_downloader", "hashing", "hostname 0.4.0", "images", @@ -6348,6 +6176,17 @@ dependencies = [ "workspace-build-acceleration", ] +[[package]] +name = "jwt_signer" +version = "0.0.1" +dependencies = [ + "errors", + "hmac 0.10.1", + "jwt", + "sha2 0.9.9", + "workspace-build-acceleration", +] + [[package]] name = "k256" version = "0.13.3" @@ -6364,12 +6203,11 @@ dependencies = [ [[package]] name = "keccak" -version = "0.2.0" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures", ] [[package]] @@ -6862,20 +6700,6 @@ dependencies = [ "workspace-build-acceleration", ] -[[package]] -name = "markup5ever" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" -dependencies = [ - "log", - "phf 0.10.1", - "phf_codegen 0.10.0", - "string_cache", - "string_cache_codegen", - "tendril", -] - [[package]] name = "markup5ever" version = "0.14.1" @@ -7328,7 +7152,6 @@ dependencies = [ "composite_identifiers", "enums", "errors", - "idempotency", "log", "num-traits", "once_cell", @@ -7342,6 +7165,7 @@ dependencies = [ "sqlx", "tokens", "tokio", + "uuid_utils", "workspace-build-acceleration", ] @@ -7452,12 +7276,6 @@ dependencies = [ "jni-sys", ] -[[package]] -name = "never" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c96aba5aa877601bb3f6dd6a63a969e1f82e60646e81e71b14496995e9853c91" - [[package]] name = "new_debug_unreachable" version = "1.0.4" @@ -8040,7 +7858,6 @@ dependencies = [ "cloudflare_errors", "cookie 0.18.1", "errors", - "idempotency", "log", "once_cell", "rand 0.9.2", @@ -8053,6 +7870,7 @@ dependencies = [ "tokio", "toml 1.0.7+spec-1.1.0", "url", + "uuid_utils", "workspace-build-acceleration", "wreq", "wreq-util", @@ -8070,7 +7888,6 @@ dependencies = [ "errors", "filesys", "hex-literal", - "idempotency", "log", "once_cell", "rand 0.9.2", @@ -8084,6 +7901,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "toml 1.0.7+spec-1.1.0", + "uuid_utils", "workspace-build-acceleration", ] @@ -8504,16 +8322,6 @@ dependencies = [ "phf_shared 0.8.0", ] -[[package]] -name = "phf_codegen" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", -] - [[package]] name = "phf_codegen" version = "0.11.3" @@ -9072,16 +8880,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" -[[package]] -name = "quick-xml" -version = "0.30.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" -dependencies = [ - "encoding_rs", - "memchr", -] - [[package]] name = "quick-xml" version = "0.32.0" @@ -9911,18 +9709,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rss" -version = "2.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6c0ea0e621c2a3aa34850ebd711526f0ac7385921f57d2430a47cecc7b9cbc" -dependencies = [ - "atom_syndication", - "derive_builder", - "never", - "quick-xml 0.30.0", -] - [[package]] name = "runwayml_client" version = "0.0.1" @@ -9935,7 +9721,6 @@ dependencies = [ "errors", "filesys", "hex-literal", - "idempotency", "jwt_light", "log", "once_cell", @@ -9950,6 +9735,7 @@ dependencies = [ "thiserror 2.0.18", "tokio", "toml 1.0.7+spec-1.1.0", + "uuid_utils", "workspace-build-acceleration", ] @@ -10371,34 +10157,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "scrape-test" -version = "0.0.1" -dependencies = [ - "clap 4.6.0", - "errors", - "tokio", - "web_scrapers", - "workspace-build-acceleration", -] - -[[package]] -name = "scraper" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c95a930e03325234c18c7071fd2b60118307e025d6fff3e12745ffbf63a3d29c" -dependencies = [ - "ahash 0.8.12", - "cssparser 0.31.2", - "ego-tree 0.6.2", - "getopts", - "html5ever 0.26.0", - "once_cell", - "selectors 0.25.0", - "smallvec", - "tendril", -] - [[package]] name = "scraper" version = "0.24.0" @@ -10406,7 +10164,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5f3a24d916e78954af99281a455168d4a9515d65eca99a18da1b813689c4ad9" dependencies = [ "cssparser 0.35.0", - "ego-tree 0.10.0", + "ego-tree", "getopts", "html5ever 0.35.0", "precomputed-hash", @@ -10557,25 +10315,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "selectors" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06" -dependencies = [ - "bitflags 2.11.0", - "cssparser 0.31.2", - "derive_more 0.99.17", - "fxhash", - "log", - "new_debug_unreachable", - "phf 0.10.1", - "phf_codegen 0.10.0", - "precomputed-hash", - "servo_arc 0.3.0", - "smallvec", -] - [[package]] name = "selectors" version = "0.30.0" @@ -10949,15 +10688,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "servo_arc" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "servo_arc" version = "0.4.1" @@ -10983,7 +10713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures 0.2.12", + "cpufeatures", "digest 0.10.7", ] @@ -11001,7 +10731,7 @@ checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", "cfg-if", - "cpufeatures 0.2.12", + "cpufeatures", "digest 0.9.0", "opaque-debug", ] @@ -11013,7 +10743,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", - "cpufeatures 0.2.12", + "cpufeatures", "digest 0.10.7", ] @@ -11024,17 +10754,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c5f3b1e2dc8aad28310d8410bd4d7e180eca65fca176c52ab00d364475d0024" dependencies = [ "cfg-if", - "cpufeatures 0.2.12", + "cpufeatures", "digest 0.11.2", ] [[package]] name = "sha3" -version = "0.11.0-rc.8" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95f78cd62cc39ece5aefbeb6caaa2ea44f70b4815d4b85f7e150ac685ada2bb5" +checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" dependencies = [ - "digest 0.11.2", + "digest 0.10.7", "keccak", ] @@ -11585,7 +11315,6 @@ dependencies = [ "component_traits", "composite_identifiers", "config", - "cookies", "crockford", "datetimes", "derive_more 0.99.17", @@ -11606,9 +11335,9 @@ dependencies = [ "hashing", "hostname 0.4.0", "http_server_common", - "idempotency", "images", "infer 0.16.0", + "jwt_signer", "language-tags", "lexical-sort", "limitation", @@ -11662,6 +11391,7 @@ dependencies = [ "users", "utoipa", "utoipa-swagger-ui", + "uuid_utils", "videos", "workspace-build-acceleration", "worldlabs_api_client", @@ -13254,7 +12984,7 @@ dependencies = [ "enums", "errors", "filesys", - "google_drive_common", + "google_drive_downloader", "hashing", "hostname 0.4.0", "hyper 1.6.0", @@ -13685,6 +13415,14 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "uuid_utils" +version = "0.0.1" +dependencies = [ + "uuid 1.22.0", + "workspace-build-acceleration", +] + [[package]] name = "v_frame" version = "0.3.8" @@ -13819,7 +13557,7 @@ dependencies = [ "easyenv", "errors", "filesys", - "google_drive_common", + "google_drive_downloader", "hashing", "hostname 0.4.0", "hyper 1.6.0", @@ -13850,7 +13588,7 @@ dependencies = [ "easyenv", "errors", "filesys", - "google_drive_common", + "google_drive_downloader", "hostname 0.4.0", "hyper 1.6.0", "jobs_common", @@ -14163,26 +13901,6 @@ dependencies = [ "workspace-build-acceleration", ] -[[package]] -name = "web_scrapers" -version = "0.0.1" -dependencies = [ - "bytes", - "chrono", - "enums", - "errors", - "log", - "once_cell", - "reqwest 0.12.23", - "rss", - "scraper 0.17.1", - "serde", - "serde_derive", - "serde_json", - "strum 0.26.2", - "workspace-build-acceleration", -] - [[package]] name = "webkit2gtk" version = "2.0.1" @@ -15215,7 +14933,7 @@ dependencies = [ "log4rs", "once_cell", "regex", - "scraper 0.24.0", + "scraper", "serde", "serde_json", "sha2 0.11.0-rc.5", @@ -15628,6 +15346,15 @@ dependencies = [ "zstd 0.13.3", ] +[[package]] +name = "zip_archives" +version = "0.0.1" +dependencies = [ + "errors", + "workspace-build-acceleration", + "zip 0.6.6", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 69ca8a1488..4b0ead81b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ "crates/api_clients/artcraft/artcraft_client", "crates/api_clients/cloud_storage", "crates/api_clients/fal_client", - "crates/api_clients/google_drive_common", + "crates/api_clients/google_drive_downloader", "crates/api_clients/google_sign_in", "crates/api_clients/grok_client", "crates/api_clients/midjourney_client", @@ -26,7 +26,6 @@ members = [ # CLI tools for operations, etc. "crates/cli/devenv/dev-database-seed", "crates/cli/devenv/dev-upload-media-file", - "crates/cli/devenv/scrape-test", "crates/cli/production/db-backfill", "crates/cli/production/db-backup", "crates/cli/production/db-cleanup", @@ -53,7 +52,7 @@ members = [ "crates/lib/cloudflare_mitigation", "crates/lib/concurrency", "crates/lib/cookie_store", - "crates/lib/cookies", + "crates/lib/jwt_signer", "crates/lib/data/collections", "crates/lib/data/crockford", "crates/lib/data/datetimes", @@ -61,12 +60,11 @@ members = [ "crates/lib/data/web_base64", "crates/lib/deprecated/config", # TODO(bt, 2023-02-13): Deprecate or repurpose "crates/lib/deprecated/http_server_common", # TODO(bt, 2023-03-20): Move useful pieces to actix_helpers - "crates/lib/dimensions", "crates/lib/easyenv", # NB: Vendored version of https://github.com/echelon/easyenv.rs that we'll improve. - "crates/lib/email_sender", + "crates/lib/deprecated/email_sender", "crates/lib/errors", "crates/lib/ffmpeg_utils", - "crates/lib/files/archives", + "crates/lib/files/zip_archives", "crates/lib/files/file_to_base64_url", "crates/lib/files/filesys", "crates/lib/files/hashing", @@ -75,31 +73,30 @@ members = [ "crates/lib/files/mimetypes", "crates/lib/files/videos", "crates/lib/http_headers", - "crates/lib/idempotency", + "crates/lib/uuid_utils", "crates/api_clients/artcraft/artcraft_router", - "crates/lib/image_algorithms", + "crates/lib/deprecated/image_algorithms", "crates/lib/jobs_common", "crates/lib/jwt_light", "crates/lib/logging", - "crates/lib/machine_learning/ml_weights_registry", - "crates/lib/machine_learning/prompt_engineering", + "crates/lib/deprecated/machine_learning/ml_weights_registry", + "crates/lib/deprecated/machine_learning/prompt_engineering", "crates/lib/markdown", - "crates/lib/migration", + "crates/schema/database/migration", "crates/lib/password", "crates/lib/server_environment", "crates/lib/sqlx_mysql_helpers", "crates/lib/storyteller_root", - "crates/lib/subprocess_common", + "crates/lib/deprecated/subprocess_common", "crates/lib/test_data", "crates/lib/testing", - "crates/lib/tts_common", + "crates/lib/deprecated/tts_common", "crates/lib/url_config", "crates/lib/url_utils", "crates/lib/user_input_common", "crates/lib/users", - "crates/lib/web_scrapers", #"crates/lib/api_clients/fakeyou_client", - #"crates/lib/machine_learning/ml_models", + #"crates/lib/deprecated/machine_learning/ml_models", # Schema (database): queries, database layouts, Redis keys, etc. "crates/schema/database/elasticsearch_schema", @@ -117,18 +114,18 @@ members = [ "crates/schema/public/tokens", # Services: API servers, jobs, workers, etc. - "crates/service/job/analytics_job", - "crates/service/job/download_job", - "crates/service/job/email_sender_job", - "crates/service/job/es-update-job", - "crates/service/job/inference_job", "crates/service/job/seedance2_pro_job", "crates/service/job/video_thumbnail_job", "crates/service/job/worldlabs_job", - "crates/service/job/tts_download_job", - "crates/service/job/tts_inference_job", - "crates/service/job/w2l_download_job", - "crates/service/job/w2l_inference_job", + "crates/service/job/deprecated/analytics_job", + "crates/service/job/deprecated/download_job", + "crates/service/job/deprecated/email_sender_job", + "crates/service/job/deprecated/es-update-job", + "crates/service/job/deprecated/inference_job", + "crates/service/job/deprecated/tts_download_job", + "crates/service/job/deprecated/tts_inference_job", + "crates/service/job/deprecated/w2l_download_job", + "crates/service/job/deprecated/w2l_inference_job", "crates/service/web/dummy_service", "crates/service/web/storyteller_web", @@ -136,7 +133,6 @@ members = [ "crates/service/plugins/billing", "crates/service/plugins/billing_artcraft", "crates/service/plugins/component_traits", - "crates/service/plugins/inference_workloads", "crates/service/plugins/user_traits_component", # Vendored code @@ -228,7 +224,7 @@ collections = { path = "crates/lib/data/collections" } component_traits = { path = "crates/service/plugins/component_traits" } concurrency = { path = "crates/lib/concurrency" } cookie_store = { path = "crates/lib/cookie_store" } -cookies = { path = "crates/lib/cookies" } +jwt_signer = { path = "crates/lib/jwt_signer" } crockford = { path = "crates/lib/data/crockford" } easyenv = { path = "crates/lib/easyenv" } enums = { path = "crates/schema/public/enums" } @@ -240,7 +236,7 @@ file_to_base64_url = { path = "crates/lib/files/file_to_base64_url" } filesys = { path = "crates/lib/files/filesys" } grok_client = { path = "crates/api_clients/grok_client" } http_headers = { path = "crates/lib/http_headers" } -idempotency = { path = "crates/lib/idempotency" } +uuid_utils = { path = "crates/lib/uuid_utils" } images = { path = "crates/lib/files/images" } jobs_common = { path = "crates/lib/jobs_common" } jwt_light = { path = "crates/lib/jwt_light" } @@ -255,7 +251,7 @@ primitives = { path = "crates/lib/data/primitives" } seedance2pro_client = { path = "crates/api_clients/seedance2pro_client" } shared_service_components = { path = "crates/lib/shared_service_components" } sqlite_tasks = { path = "crates/schema/database/sqlite_tasks" } -subprocess_common = { path = "crates/lib/subprocess_common" } +subprocess_common = { path = "crates/lib/deprecated/subprocess_common" } test_data = { path = "crates/lib/test_data" } testing = { path = "crates/lib/testing" } tokens = { path = "crates/schema/public/tokens" } @@ -304,7 +300,7 @@ serde_derive = "1.0.228" serde_json = "1.0.149" serde_yaml = "0.9.34+deprecated" sha2 = "0.11.0-rc.5" -sha3 = "0.11.0-rc.8" +sha3 = "0.10.8" strum = { version = "0.28.0", features = ["derive"] } tauri = { version = "2.8.3", features = ["unstable", "devtools"] } # NB: "devtools" prevents App Store distribution; "ubstable" is needed for window access tauri-plugin-dialog = "2.3.3" diff --git a/build/service_gpu.Dockerfile b/build/service_gpu.Dockerfile index abfe5ed430..4eaabc9574 100644 --- a/build/service_gpu.Dockerfile +++ b/build/service_gpu.Dockerfile @@ -178,11 +178,11 @@ RUN touch .env-secrets # Some services have default env files that live under their code directories # These should also be readable from the relative current path -COPY crates/service/job/download_job/config/download-job.common.env . -COPY crates/service/job/download_job/config/download-job.production.env . +COPY crates/service/job/deprecated/download_job/config/download-job.common.env . +COPY crates/service/job/deprecated/download_job/config/download-job.production.env . -COPY crates/service/job/inference_job/config/inference-job.common.env . -COPY crates/service/job/inference_job/config/inference-job.production.env . +COPY crates/service/job/deprecated/inference_job/config/inference-job.common.env . +COPY crates/service/job/deprecated/inference_job/config/inference-job.production.env . # # Need python to make use of other containers' venv # # TODO(bt,2023-04-26): This is only necessary for download-job and inference-job diff --git a/crates/api_clients/artcraft/artcraft_client/Cargo.toml b/crates/api_clients/artcraft/artcraft_client/Cargo.toml index 104baf014a..f3660a06a7 100644 --- a/crates/api_clients/artcraft/artcraft_client/Cargo.toml +++ b/crates/api_clients/artcraft/artcraft_client/Cargo.toml @@ -28,7 +28,7 @@ cloudflare_errors.workspace = true enums.workspace = true enums_public.workspace = true errors.workspace = true -idempotency.workspace = true +uuid_utils.workspace = true mimetypes.workspace = true tokens.workspace = true diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/legacy_upload_media_file_from_file.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/legacy_upload_media_file_from_file.rs index b6ec126639..272879a397 100644 --- a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/legacy_upload_media_file_from_file.rs +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/legacy_upload_media_file_from_file.rs @@ -16,7 +16,7 @@ use enums::common::generation_provider::GenerationProvider; use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::debug; use reqwest::multipart::{Form, Part}; use reqwest::Client; diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_bytes.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_bytes.rs index 7030865f47..2128e5021a 100644 --- a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_bytes.rs +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_bytes.rs @@ -15,7 +15,7 @@ use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::debug; use reqwest::multipart::{Form, Part}; use reqwest::Client; diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_file.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_file.rs index e39cc857d2..708963cf0e 100644 --- a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_file.rs +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_image_media_file_from_file.rs @@ -16,7 +16,7 @@ use enums::common::generation_provider::GenerationProvider; use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::debug; use reqwest::multipart::{Form, Part}; use reqwest::Client; diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_new_engine_asset_from_file.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_new_engine_asset_from_file.rs index 76b0b8c0e8..00c72a474a 100644 --- a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_new_engine_asset_from_file.rs +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_new_engine_asset_from_file.rs @@ -13,7 +13,7 @@ use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::debug; use reqwest::multipart::{Form, Part}; use reqwest::Client; diff --git a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_video_media_file_from_file.rs b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_video_media_file_from_file.rs index 5737bbeaaf..73800f129c 100644 --- a/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_video_media_file_from_file.rs +++ b/crates/api_clients/artcraft/artcraft_client/src/endpoints/media_files/upload_video_media_file_from_file.rs @@ -13,7 +13,7 @@ use enums::by_table::model_weights::weights_category::WeightsCategory; use enums::common::visibility::Visibility; use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::debug; use reqwest::multipart::{Form, Part}; use reqwest::Client; diff --git a/crates/api_clients/cloud_storage/Cargo.toml b/crates/api_clients/cloud_storage/Cargo.toml index 64c90c7e44..2695812cbb 100644 --- a/crates/api_clients/cloud_storage/Cargo.toml +++ b/crates/api_clients/cloud_storage/Cargo.toml @@ -16,13 +16,13 @@ path = "src/lib.rs" normal = ["workspace-build-acceleration"] [dependencies] -# Workspace hack (faster compile times) +# cargo-hakari workspace hack (faster compile times) workspace-build-acceleration.workspace = true # Internal bucket_paths.workspace = true crockford.workspace = true -easyenv = { path = "../../lib/easyenv" } +easyenv.workspace = true errors.workspace = true filesys.workspace = true hashing = { path = "../../lib/files/hashing" } @@ -31,12 +31,11 @@ storyteller_root = { path = "../../lib/storyteller_root" } # External anyhow.workspace = true -log.workspace = true -env_logger = "0.11.3" async-trait = "0.1.75" +env_logger.workspace = true +log.workspace = true +tokio.workspace = true + # The "never-encode-slash" feature seems to have been removed: # https://github.com/durch/rust-s3/pull/254 rust-s3 = { version = "0.36.0-beta.2", default-features = false, features = ["fail-on-err", "tags", "tokio-rustls-tls"] } - -# Runtime -tokio.workspace = true diff --git a/crates/api_clients/google_drive_common/Cargo.toml b/crates/api_clients/google_drive_downloader/Cargo.toml similarity index 78% rename from crates/api_clients/google_drive_common/Cargo.toml rename to crates/api_clients/google_drive_downloader/Cargo.toml index 8f575ca32a..76b6dbc8f8 100644 --- a/crates/api_clients/google_drive_common/Cargo.toml +++ b/crates/api_clients/google_drive_downloader/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "google_drive_common" +name = "google_drive_downloader" edition = "2018" version = "0.0.1" authors = [ @@ -12,7 +12,7 @@ publish = false normal = ["workspace-build-acceleration"] [lib] -name = "google_drive_common" +name = "google_drive_downloader" path = "src/lib.rs" [dependencies] @@ -22,7 +22,7 @@ workspace-build-acceleration.workspace = true # Internal crockford.workspace = true errors.workspace = true -subprocess_common = { path = "../../lib/subprocess_common" } +subprocess_common = { path = "../../lib/deprecated/subprocess_common" } # External anyhow.workspace = true diff --git a/crates/api_clients/google_drive_common/src/google_drive_download_command.rs b/crates/api_clients/google_drive_downloader/src/google_drive_download_command.rs similarity index 100% rename from crates/api_clients/google_drive_common/src/google_drive_download_command.rs rename to crates/api_clients/google_drive_downloader/src/google_drive_download_command.rs diff --git a/crates/api_clients/google_drive_common/src/lib.rs b/crates/api_clients/google_drive_downloader/src/lib.rs similarity index 100% rename from crates/api_clients/google_drive_common/src/lib.rs rename to crates/api_clients/google_drive_downloader/src/lib.rs diff --git a/crates/api_clients/openai_sora_client/Cargo.toml b/crates/api_clients/openai_sora_client/Cargo.toml index 52c0e329da..cb6cde7f1f 100644 --- a/crates/api_clients/openai_sora_client/Cargo.toml +++ b/crates/api_clients/openai_sora_client/Cargo.toml @@ -17,7 +17,7 @@ workspace-build-acceleration.workspace = true # Internal errors.workspace = true -idempotency.workspace = true +uuid_utils.workspace = true # External anyhow.workspace = true diff --git a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/generate_sentinel_token.rs b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/generate_sentinel_token.rs index 85a31926df..9491c3207b 100644 --- a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/generate_sentinel_token.rs +++ b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/generate_sentinel_token.rs @@ -4,7 +4,7 @@ use crate::error::sora_error::SoraError; use crate::error::sora_generic_api_error::SoraGenericApiError; use crate::utils_internal::classify_general_http_error::classify_general_http_error; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::error; use serde_derive::{Deserialize, Serialize}; use thiserror::Error; diff --git a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/request.rs b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/request.rs index 723e80fc33..057aab068c 100644 --- a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/request.rs +++ b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel/request.rs @@ -3,7 +3,7 @@ use base64::prelude::BASE64_STANDARD; use base64::Engine; use chrono::FixedOffset; use chrono::Utc; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use rand::Rng; use sha3::{Digest, Sha3_512}; diff --git a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/generate_sentinel_token_2.rs b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/generate_sentinel_token_2.rs index bb7d296062..e1adb119d0 100644 --- a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/generate_sentinel_token_2.rs +++ b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/generate_sentinel_token_2.rs @@ -10,7 +10,7 @@ use crate::requests::auth_sentinel_2::response::SentinelResponse; use crate::utils_internal::classify_general_http_error::classify_general_http_error; use chrono::{DateTime, NaiveDateTime, TimeDelta, TimeZone, Utc}; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{debug, error, info}; use serde_derive::{Deserialize, Serialize}; use std::io::Write; diff --git a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/request.rs b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/request.rs index 1c07587787..9beb3ddab3 100644 --- a/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/request.rs +++ b/crates/api_clients/openai_sora_client/src/requests/auth_sentinel_2/request.rs @@ -1,4 +1,4 @@ -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use serde_derive::Serialize; // TODO: Support more of these. diff --git a/crates/api_clients/openart_client/Cargo.toml b/crates/api_clients/openart_client/Cargo.toml index ffea137a7c..7af8844c5a 100644 --- a/crates/api_clients/openart_client/Cargo.toml +++ b/crates/api_clients/openart_client/Cargo.toml @@ -19,8 +19,8 @@ workspace-build-acceleration.workspace = true # Internal crockford.workspace = true errors.workspace = true -idempotency.workspace = true -subprocess_common = { path = "../../lib/subprocess_common" } +uuid_utils.workspace = true +subprocess_common = { path = "../../lib/deprecated/subprocess_common" } # External anyhow.workspace = true diff --git a/crates/api_clients/runwayml_client/Cargo.toml b/crates/api_clients/runwayml_client/Cargo.toml index 02536d192e..c50eb1ba70 100644 --- a/crates/api_clients/runwayml_client/Cargo.toml +++ b/crates/api_clients/runwayml_client/Cargo.toml @@ -19,9 +19,9 @@ workspace-build-acceleration.workspace = true # Internal crockford.workspace = true errors.workspace = true -idempotency.workspace = true +uuid_utils.workspace = true jwt_light.workspace = true -subprocess_common = { path = "../../lib/subprocess_common" } +subprocess_common = { path = "../../lib/deprecated/subprocess_common" } # External anyhow.workspace = true diff --git a/crates/cli/devenv/dev-database-seed/Cargo.toml b/crates/cli/devenv/dev-database-seed/Cargo.toml index 6a926460a7..091351c6c4 100644 --- a/crates/cli/devenv/dev-database-seed/Cargo.toml +++ b/crates/cli/devenv/dev-database-seed/Cargo.toml @@ -28,7 +28,7 @@ enums.workspace = true errors.workspace = true filesys.workspace = true hashing = { path = "../../../lib/files/hashing" } -idempotency.workspace = true +uuid_utils.workspace = true mimetypes.workspace = true mysql_queries = { path = "../../../schema/database/mysql_queries" } storyteller_root = { path = "../../../lib/storyteller_root" } diff --git a/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs b/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs index da6b74de76..c8d61de3ca 100644 --- a/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs +++ b/crates/cli/devenv/dev-database-seed/src/seeding/zero_shot_tts.rs @@ -11,7 +11,7 @@ use enums::by_table::zs_voices::model_type::ZsVoiceModelType; use enums::common::visibility::Visibility; use errors::{anyhow, AnyhowResult}; use filesys::file_read_bytes::file_read_bytes; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use mimetypes::mimetype_for_bytes::get_mimetype_for_bytes; use mysql_queries::queries::users::user::get::get_user_token_by_username::get_user_token_by_username; use mysql_queries::queries::voice_designer::datasets::create_dataset::{create_dataset, CreateDatasetArgs}; diff --git a/crates/cli/devenv/scrape-test/Cargo.toml b/crates/cli/devenv/scrape-test/Cargo.toml deleted file mode 100644 index f4a7619ab2..0000000000 --- a/crates/cli/devenv/scrape-test/Cargo.toml +++ /dev/null @@ -1,28 +0,0 @@ -[package] -name = "scrape-test" -publish = false -edition = "2021" -version = "0.0.1" -authors = [ - "Brandon Thomas ", - "Brandon Thomas ", -] - -[[bin]] -name = "scrape-test" -path = "src/main.rs" - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-build-acceleration"] - -[dependencies] -# Workspace hack (faster compile times) -workspace-build-acceleration = { workspace = true } - -# Internal -errors = { path = "../../../lib/errors" } -web_scrapers = { path = "../../../lib/web_scrapers" } - -# External -clap = "4.4.8" -tokio = { version = "1.38.0", features = ["macros"] } diff --git a/crates/cli/devenv/scrape-test/src/main.rs b/crates/cli/devenv/scrape-test/src/main.rs deleted file mode 100644 index 1f24b8d613..0000000000 --- a/crates/cli/devenv/scrape-test/src/main.rs +++ /dev/null @@ -1,32 +0,0 @@ -use clap::Parser; - -use errors::AnyhowResult; -use web_scrapers::scrape_supported_webpage::scrape_supported_webpage; - -#[derive(Parser, Debug)] -#[command(name = "scrape-test")] -pub struct Args { - #[arg(name="url", short='u', long = "url", help = "URL of the page to scrape", required = true)] - url: String, - - #[arg(name="print_html", long = "html", help = "Print the HTML scraped", required = true)] - print_html: bool, -} - -#[tokio::main] -pub async fn main() -> AnyhowResult<()> { - let args = Args::parse(); - - let url = args.url.trim().to_string(); - let print_html = args.print_html; - - let scrape_result = scrape_supported_webpage(&url).await?; - - if print_html { - println!("{}", scrape_result.original_html) - } else { - println!("{:#?}", scrape_result.result) - } - - Ok(()) -} diff --git a/crates/desktop/artcraft/Cargo.toml b/crates/desktop/artcraft/Cargo.toml index f2d8416ee7..75d79e203b 100644 --- a/crates/desktop/artcraft/Cargo.toml +++ b/crates/desktop/artcraft/Cargo.toml @@ -35,7 +35,7 @@ enums.workspace = true errors.workspace = true filesys.workspace = true grok_client.workspace = true -idempotency.workspace = true +uuid_utils.workspace = true images.workspace = true memory_store.workspace = true midjourney_client.workspace = true diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/artcraft/handle_generic_bg_removal_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/artcraft/handle_generic_bg_removal_artcraft.rs index 0c9b15622a..785bd36282 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/artcraft/handle_generic_bg_removal_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_bg_removal/artcraft/handle_generic_bg_removal_artcraft.rs @@ -16,7 +16,7 @@ use anyhow::anyhow; use artcraft_api_defs::generate::image::bg_removal::remove_image_background::RemoveImageBackgroundRequest; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::image::bg_removal::remove_image_background::remove_image_background; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_flux_kontext_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_flux_kontext_edit.rs index e4ae3a03d1..252dcbbab0 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_flux_kontext_edit.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_flux_kontext_edit.rs @@ -15,7 +15,7 @@ use artcraft_api_defs::generate::image::edit::flux_pro_kontext_max_edit_image::{ use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; use artcraft_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1_edit.rs index 4255dff854..edbf355ec1 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1_edit.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1_edit.rs @@ -14,7 +14,7 @@ use anyhow::anyhow; use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; use tauri::AppHandle; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/sora/handle_sora_gpt_image_1_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/sora/handle_sora_gpt_image_1_edit.rs index 1d12fb64b1..ed0bc273ea 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/sora/handle_sora_gpt_image_1_edit.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/sora/handle_sora_gpt_image_1_edit.rs @@ -20,7 +20,7 @@ use anyhow::anyhow; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info, warn}; use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_dev_juggernaut_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_dev_juggernaut_inpaint.rs index 4f183f4052..e8000eb97f 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_dev_juggernaut_inpaint.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_dev_juggernaut_inpaint.rs @@ -18,7 +18,7 @@ use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1 use artcraft_api_defs::generate::image::inpaint::flux_dev_juggernaut_inpaint_image::{FluxDevJuggernautInpaintImageNumImages, FluxDevJuggernautInpaintImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_1_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_1_inpaint.rs index f664213cc8..b3236e9130 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_1_inpaint.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_1_inpaint.rs @@ -18,7 +18,7 @@ use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1 use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_kontext_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_kontext_inpaint.rs index ad51bf6f5c..f149efb832 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_kontext_inpaint.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_flux_pro_kontext_inpaint.rs @@ -19,7 +19,7 @@ use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1 use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_nano_banana_inpaint.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_nano_banana_inpaint.rs index 368f92497e..db5b67f9a7 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_nano_banana_inpaint.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_inpaint/artcraft/handle_artcraft_nano_banana_inpaint.rs @@ -20,7 +20,7 @@ use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1 use artcraft_api_defs::generate::image::inpaint::flux_pro_1_inpaint_image::{FluxPro1InpaintImageNumImages, FluxPro1InpaintImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use images::encoding::image_bytes_to_png_bytes::image_bytes_to_png_bytes; use images::encoding::image_bytes_to_png_bytes_with_dimensions::image_bytes_to_png_bytes_with_dimensions; use images::mask_images::normalize_image_bytes_to_flux_mask::normalize_image_bytes_to_flux_mask; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_gaussian/worldlabs/handle_worldlabs_marble.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_gaussian/worldlabs/handle_worldlabs_marble.rs index f739278c2e..591e701893 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_gaussian/worldlabs/handle_worldlabs_marble.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_gaussian/worldlabs/handle_worldlabs_marble.rs @@ -13,7 +13,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use crate::services::worldlabs::state::worldlabs_credential_manager::WorldlabsCredentialManager; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use std::path::PathBuf; use artcraft_client::endpoints::media_files::get_media_file::get_media_file; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_object/artcraft/handle_artcraft_object.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_object/artcraft/handle_artcraft_object.rs index 971c1dce93..7944a7fb38 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_object/artcraft/handle_artcraft_object.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_object/artcraft/handle_artcraft_object.rs @@ -12,7 +12,7 @@ use artcraft_api_defs::generate::object::generate_hunyuan_2_0_image_to_3d::Gener use artcraft_api_defs::generate::object::generate_hunyuan_2_1_image_to_3d::GenerateHunyuan21ImageTo3dRequest; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::endpoints::generate::object::generate_hunyuan_3d_2_0_image_to_3d::generate_hunyuan3d_2_0_image_to_3d; use artcraft_client::endpoints::generate::object::generate_hunyuan_3d_2_1_image_to_3d::generate_hunyuan3d_2_1_image_to_3d; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_1p6_pro.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_1p6_pro.rs index 309c0f6abb..b9ee7a2679 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_1p6_pro.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_1p6_pro.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProAspectRatio, GenerateKling16ProDuration, GenerateKling16ProImageToVideoRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_kling_16_pro_image_to_video::generate_kling_16_pro_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_master.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_master.rs index 0f04db6dd1..cc66d35dff 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_master.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_master.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::generate_kling_2_1_master_image_to_video::{GenerateKling21MasterAspectRatio, GenerateKling21MasterDuration, GenerateKling21MasterImageToVideoRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_kling_21_master_image_to_video::generate_kling_21_master_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_pro.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_pro.rs index 6af631a612..5892fa1366 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_pro.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p1_pro.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProAspectRatio, GenerateKling21ProDuration, GenerateKling21ProImageToVideoRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_kling_21_pro_image_to_video::generate_kling_21_pro_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p5_turbo_pro.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p5_turbo_pro.rs index ce91d24df2..b8bf4015e9 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p5_turbo_pro.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p5_turbo_pro.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::multi_function::kling_2_5_turbo_multi_function_video_gen::{Kling2p5TurboProMultiFunctionVideoGenAspectRatio, Kling2p5TurboProMultiFunctionVideoGenDuration, Kling2p5TurboProMultiFunctionVideoGenRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::multi_function::kling_2p5_turbo_pro_multi_function_video_gen::kling_2p5_turbo_pro_multi_function_video_gen; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p6_pro.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p6_pro.rs index 553b145f3e..b2c196c885 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p6_pro.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_kling_2p6_pro.rs @@ -16,7 +16,7 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{ use artcraft_api_defs::generate::video::multi_function::kling_2_6_multi_function_video_gen::{Kling2p6ProMultiFunctionVideoGenAspectRatio, Kling2p6ProMultiFunctionVideoGenDuration, Kling2p6ProMultiFunctionVideoGenRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_kling_16_pro_image_to_video::generate_kling_16_pro_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_seedance_1_lite.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_seedance_1_lite.rs index 60d8103c1b..5e07d28a07 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_seedance_1_lite.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_seedance_1_lite.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::generate_seedance_1_0_lite_image_to_video::{GenerateSeedance10LiteDuration, GenerateSeedance10LiteImageToVideoRequest, GenerateSeedance10LiteResolution}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_kling_16_pro_image_to_video::generate_kling_16_pro_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2.rs index a7e6b3de66..c8660aa9ab 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2.rs @@ -16,7 +16,7 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{ use artcraft_api_defs::generate::video::multi_function::sora_2_multi_function_video_gen::{Sora2MultiFunctionVideoGenAspectRatio, Sora2MultiFunctionVideoGenDuration, Sora2MultiFunctionVideoGenRequest, Sora2MultiFunctionVideoGenResolution}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use openai_sora_client::recipes::generate_sora2_video::generate_sora2_video_with_session_auto_renew::generate_sora2_video_with_session_auto_renew; use openai_sora_client::recipes::list_sora2_drafts::list_sora2_drafts_with_session_auto_renew::list_sora2_drafts_with_session_auto_renew; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2_pro.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2_pro.rs index 77042fb5d4..9dec7ef6ae 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2_pro.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_sora2_pro.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::multi_function::sora_2_pro_multi_function_video_gen::{Sora2ProMultiFunctionVideoGenAspectRatio, Sora2ProMultiFunctionVideoGenDuration, Sora2ProMultiFunctionVideoGenRequest, Sora2ProMultiFunctionVideoGenResolution}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use openai_sora_client::recipes::generate_sora2_video::generate_sora2_video_with_session_auto_renew::generate_sora2_video_with_session_auto_renew; use openai_sora_client::recipes::list_sora2_drafts::list_sora2_drafts_with_session_auto_renew::list_sora2_drafts_with_session_auto_renew; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo2.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo2.rs index 2ecb5f4b26..f90128a926 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo2.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo2.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::generate_veo_2_image_to_video::{GenerateVeo2AspectRatio, GenerateVeo2ImageToVideoRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_veo_2_image_to_video::generate_veo_2_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3.rs index d50f6c80bc..9036b604e8 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3.rs @@ -16,7 +16,7 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{ use artcraft_api_defs::generate::video::generate_veo_3_image_to_video::{GenerateVeo3AspectRatio, GenerateVeo3ImageToVideoRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_veo_3_image_to_video::generate_veo_3_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3_fast.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3_fast.rs index e36e696700..05996448f1 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3_fast.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3_fast.rs @@ -16,7 +16,7 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{ use artcraft_api_defs::generate::video::generate_veo_3_fast_image_to_video::GenerateVeo3FastImageToVideoRequest; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::generate_veo_3_fast_image_to_video::generate_veo_3_fast_image_to_video; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1.rs index 9b46dbf671..fdad3af739 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1.rs @@ -16,7 +16,7 @@ use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{ use artcraft_api_defs::generate::video::multi_function::veo_3p1_multi_function_video_gen::{Veo3p1MultiFunctionVideoGenAspectRatio, Veo3p1MultiFunctionVideoGenDuration, Veo3p1MultiFunctionVideoGenRequest, Veo3p1MultiFunctionVideoGenResolution}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::multi_function::veo_3p1_multi_function_video_gen::veo_3p1_multi_function_video_gen; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1_fast.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1_fast.rs index 8dc33cc9f3..afbfb0e2f4 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1_fast.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/artcraft/handle_artcraft_veo3p1_fast.rs @@ -15,7 +15,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen::{Veo3p1FastMultiFunctionVideoGenAspectRatio, Veo3p1FastMultiFunctionVideoGenDuration, Veo3p1FastMultiFunctionVideoGenRequest, Veo3p1FastMultiFunctionVideoGenResolution}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use artcraft_client::credentials::storyteller_credential_set::StorytellerCredentialSet; use artcraft_client::endpoints::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen::veo_3p1_fast_multi_function_video_gen; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1_text_to_image.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1_text_to_image.rs index 55a943f9d6..4265016619 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1_text_to_image.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1_text_to_image.rs @@ -14,7 +14,7 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use artcraft_api_defs::generate::image::text::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageImageQuality, GenerateGptImage1TextToImageImageSize, GenerateGptImage1TextToImageNumImages, GenerateGptImage1TextToImageRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs index dee51f43cd..8318f5f6e6 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs @@ -9,7 +9,7 @@ use crate::services::grok::state::grok_image_prompt_queue::{GrokImagePromptQueue use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use grok_client::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use tauri::AppHandle; pub async fn handle_grok( diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/midjourney/handle_midjourney.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/midjourney/handle_midjourney.rs index cbede4534a..848ebe8105 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/midjourney/handle_midjourney.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/midjourney/handle_midjourney.rs @@ -17,7 +17,7 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use midjourney_client::client::midjourney_hostname::MidjourneyHostname; use midjourney_client::endpoints::submit_job::{submit_job, SubmitJobRequest}; diff --git a/crates/desktop/artcraft/src/services/grok/threads/grok_image_websocket_thread/grok_image_websocket_thread.rs b/crates/desktop/artcraft/src/services/grok/threads/grok_image_websocket_thread/grok_image_websocket_thread.rs index a1eb7f8444..ce8cfb721d 100644 --- a/crates/desktop/artcraft/src/services/grok/threads/grok_image_websocket_thread/grok_image_websocket_thread.rs +++ b/crates/desktop/artcraft/src/services/grok/threads/grok_image_websocket_thread/grok_image_websocket_thread.rs @@ -25,7 +25,7 @@ use grok_client::requests::image_websocket::grok_wrapped_websocket::GrokWrappedW use grok_client::requests::image_websocket::listen_for_websocket_images::{listen_for_websocket_images, ImageResults, ListenForWebsocketImagesArgs}; use grok_client::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio; use grok_client::requests::image_websocket::prompt_websocket_image::{prompt_websocket_image, PromptWebsocketImageArgs}; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info, warn}; use sqlite_tasks::queries::get_task_by_provider_and_provider_job_id::{get_task_by_provider_and_provider_job_id, GetTaskByProviderAndProviderJobIdArgs}; use sqlite_tasks::queries::list_tasks_by_provider_and_status::{list_tasks_by_provider_and_status, ListTasksByProviderAndStatusArgs}; diff --git a/crates/desktop/artcraft/src/services/grok/threads/grok_video_task_polling/grok_video_task_polling_thread.rs b/crates/desktop/artcraft/src/services/grok/threads/grok_video_task_polling/grok_video_task_polling_thread.rs index 6a5c0273cc..336b170e46 100644 --- a/crates/desktop/artcraft/src/services/grok/threads/grok_video_task_polling/grok_video_task_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/grok/threads/grok_video_task_polling/grok_video_task_polling_thread.rs @@ -25,7 +25,7 @@ use grok_client::error::grok_error::GrokError; use grok_client::requests::download_video_file::download_video_file::{download_video_file, DownloadVideoFileArgs}; use grok_client::requests::download_video_file::grok_download_video::GrokDownloadVideo; use grok_client::requests::media_posts::list_media_posts::grok_list_media_posts::{GrokMediaPostList, GrokMediaPostListRequest, VideoData}; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use midjourney_client::client::midjourney_hostname::MidjourneyHostname; use midjourney_client::credentials::midjourney_user_id::MidjourneyUserId; diff --git a/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs b/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs index e7a1eddf64..429e410cb2 100644 --- a/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs @@ -19,7 +19,7 @@ use enums::common::model_type::ModelType; use enums::tauri::tasks::task_media_file_class::TaskMediaFileClass; use enums::tauri::tasks::task_status::TaskStatus; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use midjourney_client::client::midjourney_hostname::MidjourneyHostname; use midjourney_client::credentials::midjourney_user_id::MidjourneyUserId; diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/handle_successful_generations.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/handle_successful_generations.rs index ab9af46b95..15b54c8b95 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/handle_successful_generations.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/handle_successful_generations.rs @@ -20,7 +20,7 @@ use enums::common::model_type::ModelType; use enums::tauri::tasks::task_media_file_class::TaskMediaFileClass; use enums::tauri::tasks::task_status; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info, warn}; use once_cell::sync::Lazy; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_classic_sora_tasks.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_classic_sora_tasks.rs index 7a08a22aca..9a85ad3f6a 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_classic_sora_tasks.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_classic_sora_tasks.rs @@ -19,7 +19,7 @@ use enums::common::generation_provider::GenerationProvider; use enums::common::model_type::ModelType; use enums::tauri::tasks::task_status; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info, warn}; use once_cell::sync::Lazy; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_sora_2_tasks.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_sora_2_tasks.rs index 95a8fc3709..8ef9d1ea38 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_sora_2_tasks.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/helpers/poll_sora_2_tasks.rs @@ -19,7 +19,7 @@ use enums::common::generation_provider::GenerationProvider; use enums::common::model_type::ModelType; use enums::tauri::tasks::task_status; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info, warn}; use once_cell::sync::Lazy; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs index 555ae421d4..e56f0c0092 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling/sora_task_polling_thread.rs @@ -20,7 +20,7 @@ use enums::common::generation_provider::GenerationProvider; use enums::common::model_type::ModelType; use enums::tauri::tasks::task_status; use errors::AnyhowResult; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info, warn}; use once_cell::sync::Lazy; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; diff --git a/crates/desktop/artcraft/src/services/worldlabs/threads/worldlabs_marble_task_polling.rs b/crates/desktop/artcraft/src/services/worldlabs/threads/worldlabs_marble_task_polling.rs index 5ae9f1ec44..81821fb460 100644 --- a/crates/desktop/artcraft/src/services/worldlabs/threads/worldlabs_marble_task_polling.rs +++ b/crates/desktop/artcraft/src/services/worldlabs/threads/worldlabs_marble_task_polling.rs @@ -28,7 +28,7 @@ use grok_client::error::grok_error::GrokError; use grok_client::requests::download_video_file::download_video_file::{download_video_file, DownloadVideoFileArgs}; use grok_client::requests::download_video_file::grok_download_video::GrokDownloadVideo; use grok_client::requests::media_posts::list_media_posts::grok_list_media_posts::{GrokMediaPostList, GrokMediaPostListRequest, VideoData}; -use idempotency::uuid::generate_random_uuid; +use uuid_utils::uuid::generate_random_uuid; use log::{error, info}; use midjourney_client::client::midjourney_hostname::MidjourneyHostname; use midjourney_client::credentials::midjourney_user_id::MidjourneyUserId; diff --git a/crates/desktop/tauri-realtime/Cargo.toml b/crates/desktop/tauri-realtime/Cargo.toml index c3184bfa7b..226fb2cf74 100644 --- a/crates/desktop/tauri-realtime/Cargo.toml +++ b/crates/desktop/tauri-realtime/Cargo.toml @@ -24,7 +24,7 @@ tauri-build = { version = "2.1.1", features = [] } crockford = { path = "../../lib/data/crockford" } filesys = { path = "../../lib/files/filesys" } memory_caching = { path = "../../lib/caching/memory_caching" } -ml_weights_registry = { path = "../../lib/machine_learning/ml_weights_registry" } +ml_weights_registry = { path = "../../lib/deprecated/machine_learning/ml_weights_registry" } primitives = { path = "../../lib/data/primitives" } # Internal ML @@ -64,7 +64,7 @@ tokio-util = { version = "0.7", features = ["codec"] } # tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] } [target.'cfg(not(target_os = "macos"))'.dependencies] -ml_models = { path = "../../lib/machine_learning/ml_models" } +ml_models = { path = "../../lib/deprecated/machine_learning/ml_models" } [dev-dependencies] libloading = "0.8.6" diff --git a/crates/lib/actix_artcraft/Cargo.toml b/crates/lib/actix_artcraft/Cargo.toml index 3f05da3ec5..a1dec467af 100644 --- a/crates/lib/actix_artcraft/Cargo.toml +++ b/crates/lib/actix_artcraft/Cargo.toml @@ -22,7 +22,7 @@ workspace-build-acceleration.workspace = true # Internal actix_helpers.workspace = true -cookies.workspace = true +jwt_signer.workspace = true enums.workspace = true errors.workspace = true http_headers.workspace = true diff --git a/crates/lib/actix_artcraft/src/sessions/payload_signer.rs b/crates/lib/actix_artcraft/src/sessions/payload_signer.rs index a16c67e80a..46009f5466 100644 --- a/crates/lib/actix_artcraft/src/sessions/payload_signer.rs +++ b/crates/lib/actix_artcraft/src/sessions/payload_signer.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use crate::sessions::http_user_session_payload::HttpUserSessionPayload; -use cookies::jwt_signer::JwtSigner; +use jwt_signer::jwt_signer::JwtSigner; use errors::AnyhowResult; use tokens::tokens::user_sessions::UserSessionToken; use tokens::tokens::users::UserToken; diff --git a/crates/lib/data/primitives/Cargo.toml b/crates/lib/data/primitives/Cargo.toml index ca45208673..315d757d01 100644 --- a/crates/lib/data/primitives/Cargo.toml +++ b/crates/lib/data/primitives/Cargo.toml @@ -17,19 +17,24 @@ path = "src/lib.rs" normal = ["workspace-build-acceleration"] [dependencies] -# Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } +# cargo-hakari workspace hack (faster compile times) +workspace-build-acceleration.workspace = true # NB: This is designed to be a central crate with few dependencies. # Consider the dependencies here wisely. # This is used in MySQL, etc., so dependencies are ill-advised. # Internal -#errors = { path = "../../errors" } +# **DO NOT ADD ANY!** +# (None) # External +# **DO NOT ADD ANY!** # (None) [dev-dependencies] -testing = { path = "../../testing" } +# Internal +testing.workspace = true + +# External assertor = "0.0.3" diff --git a/crates/lib/email_sender/Cargo.toml b/crates/lib/deprecated/email_sender/Cargo.toml similarity index 88% rename from crates/lib/email_sender/Cargo.toml rename to crates/lib/deprecated/email_sender/Cargo.toml index 7365e9a3ff..c8dc97c25b 100644 --- a/crates/lib/email_sender/Cargo.toml +++ b/crates/lib/deprecated/email_sender/Cargo.toml @@ -14,10 +14,10 @@ normal = ["workspace-build-acceleration"] [dependencies] # Internal -errors = { path = "../errors" } +errors = { path = "../../errors" } # Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" } +workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } # External lettre = { version = "0.11.2", default-features = false, features = ["smtp-transport", "pool", "hostname", "builder", "rustls-tls"] } diff --git a/crates/lib/email_sender/src/lib.rs b/crates/lib/deprecated/email_sender/src/lib.rs similarity index 100% rename from crates/lib/email_sender/src/lib.rs rename to crates/lib/deprecated/email_sender/src/lib.rs diff --git a/crates/lib/email_sender/src/smtp_email_sender.rs b/crates/lib/deprecated/email_sender/src/smtp_email_sender.rs similarity index 100% rename from crates/lib/email_sender/src/smtp_email_sender.rs rename to crates/lib/deprecated/email_sender/src/smtp_email_sender.rs diff --git a/crates/lib/image_algorithms/Cargo.toml b/crates/lib/deprecated/image_algorithms/Cargo.toml similarity index 82% rename from crates/lib/image_algorithms/Cargo.toml rename to crates/lib/deprecated/image_algorithms/Cargo.toml index afec3d192a..edf8b157b7 100644 --- a/crates/lib/image_algorithms/Cargo.toml +++ b/crates/lib/deprecated/image_algorithms/Cargo.toml @@ -18,13 +18,13 @@ normal = ["workspace-build-acceleration"] [dependencies] # Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" } +workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } # Internal -errors = { path = "../errors" } +errors = { path = "../../errors" } # External image = "0.25.6" [dev-dependencies] -testing = { path = "../testing" } +testing = { path = "../../testing" } diff --git a/crates/lib/image_algorithms/src/lib.rs b/crates/lib/deprecated/image_algorithms/src/lib.rs similarity index 100% rename from crates/lib/image_algorithms/src/lib.rs rename to crates/lib/deprecated/image_algorithms/src/lib.rs diff --git a/crates/lib/image_algorithms/src/remove_image_border.rs b/crates/lib/deprecated/image_algorithms/src/remove_image_border.rs similarity index 100% rename from crates/lib/image_algorithms/src/remove_image_border.rs rename to crates/lib/deprecated/image_algorithms/src/remove_image_border.rs diff --git a/crates/lib/machine_learning/ml_models/Cargo.toml b/crates/lib/deprecated/machine_learning/ml_models/Cargo.toml similarity index 80% rename from crates/lib/machine_learning/ml_models/Cargo.toml rename to crates/lib/deprecated/machine_learning/ml_models/Cargo.toml index 408f920145..fccc24d5d6 100644 --- a/crates/lib/machine_learning/ml_models/Cargo.toml +++ b/crates/lib/deprecated/machine_learning/ml_models/Cargo.toml @@ -22,11 +22,11 @@ normal = ["workspace-build-acceleration"] workspace-build-acceleration = { workspace = true } # Internal -filesys = { path = "../../files/filesys" } -image_algorithms = { path = "../../image_algorithms" } -memory_caching = { path = "../../caching/memory_caching" } +filesys = { path = "../../../files/filesys" } +image_algorithms = { path = "../image_algorithms" } +memory_caching = { path = "../../../caching/memory_caching" } ml_weights_registry = { path = "../ml_weights_registry" } -primitives = { path = "../../data/primitives" } +primitives = { path = "../../../data/primitives" } # Bypass cudarc issues on Windows https://github.com/coreylowman/cudarc/issues/332 cudarc = "0.13.7" @@ -35,9 +35,9 @@ cudarc = "0.13.7" anyhow = { workspace = true } base64 = { workspace = true } bytes = "1.10" -candle-core = { path = "../../../vendor/candle/candle-core", features = ["cuda"] } -candle-nn = { path = "../../../vendor/candle/candle-nn", features = ["cuda"] } -candle-transformers = { path = "../../../vendor/candle/candle-transformers", features = ["cuda"] } +candle-core = { path = "../../../../vendor/candle/candle-core", features = ["cuda"] } +candle-nn = { path = "../../../../vendor/candle/candle-nn", features = ["cuda"] } +candle-transformers = { path = "../../../../vendor/candle/candle-transformers", features = ["cuda"] } const_format = "0.2.34" directories = "6.0.0" dirs = "6.0.0" diff --git a/crates/lib/machine_learning/ml_models/src/lib.rs b/crates/lib/deprecated/machine_learning/ml_models/src/lib.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/lib.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/lib.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/background_removal/mod.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/background_removal/mod.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/background_removal/mod.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/background_removal/mod.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/background_removal/onnx_session.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/background_removal/onnx_session.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/background_removal/onnx_session.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/background_removal/onnx_session.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/background_removal/remove_image_background.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/background_removal/remove_image_background.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/background_removal/remove_image_background.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/background_removal/remove_image_background.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/flux/flux_pipeline.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/flux/flux_pipeline.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/flux/flux_pipeline.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/flux/flux_pipeline.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/flux/mod.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/flux/mod.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/flux/mod.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/flux/mod.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/image/dynamic_image_to_tensor.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/image/dynamic_image_to_tensor.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/image/dynamic_image_to_tensor.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/image/dynamic_image_to_tensor.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor_2.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor_2.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor_2.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/image/load_image_file_to_tensor_2.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/image/mod.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/image/mod.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/image/mod.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/image/mod.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/image/save_image_from_tensor.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/image/save_image_from_tensor.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/image/save_image_from_tensor.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/image/save_image_from_tensor.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/image/tensor_to_image_buffer.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/image/tensor_to_image_buffer.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/image/tensor_to_image_buffer.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/image/tensor_to_image_buffer.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/mod.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/mod.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/mod.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/mod.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/model_cache.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/model_cache.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/model_cache.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/model_cache.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/model_config.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/model_config.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/model_config.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/model_config.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/model_file.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/model_file.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/model_file.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/model_file.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/prompt_cache.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/prompt_cache.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/prompt_cache.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/prompt_cache.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/get_vae_scale.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/get_vae_scale.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/get_vae_scale.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/get_vae_scale.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/infer_clip_text_embeddings.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/infer_clip_text_embeddings.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/infer_clip_text_embeddings.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/infer_clip_text_embeddings.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/lcm_pipeline.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/lcm_pipeline.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/lcm_pipeline.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/lcm_pipeline.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/mod.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/mod.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/mod.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/mod.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/remap_lcm_strength_range.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/remap_lcm_strength_range.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/remap_lcm_strength_range.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/remap_lcm_strength_range.rs diff --git a/crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/stable_diffusion_pipeline.rs b/crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/stable_diffusion_pipeline.rs similarity index 100% rename from crates/lib/machine_learning/ml_models/src/ml/stable_diffusion/stable_diffusion_pipeline.rs rename to crates/lib/deprecated/machine_learning/ml_models/src/ml/stable_diffusion/stable_diffusion_pipeline.rs diff --git a/crates/lib/machine_learning/ml_weights_registry/Cargo.toml b/crates/lib/deprecated/machine_learning/ml_weights_registry/Cargo.toml similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/Cargo.toml rename to crates/lib/deprecated/machine_learning/ml_weights_registry/Cargo.toml diff --git a/crates/lib/machine_learning/ml_weights_registry/src/lib.rs b/crates/lib/deprecated/machine_learning/ml_weights_registry/src/lib.rs similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/src/lib.rs rename to crates/lib/deprecated/machine_learning/ml_weights_registry/src/lib.rs diff --git a/crates/lib/machine_learning/ml_weights_registry/src/weights_registry/flux_weights.rs b/crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/flux_weights.rs similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/src/weights_registry/flux_weights.rs rename to crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/flux_weights.rs diff --git a/crates/lib/machine_learning/ml_weights_registry/src/weights_registry/mod.rs b/crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/mod.rs similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/src/weights_registry/mod.rs rename to crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/mod.rs diff --git a/crates/lib/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor.rs b/crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor.rs similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor.rs rename to crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor.rs diff --git a/crates/lib/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor_builder.rs b/crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor_builder.rs similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor_builder.rs rename to crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/weight_descriptor_builder.rs diff --git a/crates/lib/machine_learning/ml_weights_registry/src/weights_registry/weights.rs b/crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/weights.rs similarity index 100% rename from crates/lib/machine_learning/ml_weights_registry/src/weights_registry/weights.rs rename to crates/lib/deprecated/machine_learning/ml_weights_registry/src/weights_registry/weights.rs diff --git a/crates/lib/machine_learning/prompt_engineering/Cargo.toml b/crates/lib/deprecated/machine_learning/prompt_engineering/Cargo.toml similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/Cargo.toml rename to crates/lib/deprecated/machine_learning/prompt_engineering/Cargo.toml diff --git a/crates/lib/machine_learning/prompt_engineering/src/classify_prompt.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/classify_prompt.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/classify_prompt.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/classify_prompt.rs diff --git a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs similarity index 78% rename from crates/lib/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs index 4ad13e84e0..1d9169c7ad 100644 --- a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs +++ b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_children_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS : &str = include_str!("../../../../../../includes/binary_includes/safety/dictionary_children_terms.txt"); +const TERMS : &str = include_str!("../../../../../../../includes/binary_includes/safety/dictionary_children_terms.txt"); pub (crate) fn contains_children_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY : Lazy> = Lazy::new(|| { diff --git a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs similarity index 78% rename from crates/lib/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs index 96d4293c50..462a80e1e2 100644 --- a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs +++ b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_potential_minor_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS: &str = include_str!("../../../../../../includes/binary_includes/safety/dictionary_potential_minor_terms.txt"); +const TERMS: &str = include_str!("../../../../../../../includes/binary_includes/safety/dictionary_potential_minor_terms.txt"); pub (crate) fn contains_potential_minor_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY: Lazy> = Lazy::new(|| { diff --git a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs similarity index 79% rename from crates/lib/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs index d04337cd66..cd5f4d5ad9 100644 --- a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs +++ b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_racist_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS : &str = include_str!("../../../../../../includes/binary_includes/safety/dictionary_racist_terms.txt"); +const TERMS : &str = include_str!("../../../../../../../includes/binary_includes/safety/dictionary_racist_terms.txt"); pub (crate) fn contains_racist_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY : Lazy> = Lazy::new(|| { diff --git a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs similarity index 79% rename from crates/lib/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs index b4825b8430..1eba7d66c7 100644 --- a/crates/lib/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs +++ b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/contains_sex_keyword.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use once_cell::sync::Lazy; -const TERMS : &str = include_str!("../../../../../../includes/binary_includes/safety/dictionary_sex_terms.txt"); +const TERMS : &str = include_str!("../../../../../../../includes/binary_includes/safety/dictionary_sex_terms.txt"); pub (crate) fn contains_sex_keyword(prompt_tokens: &[String]) -> bool { static TERM_DICTIONARY : Lazy> = Lazy::new(|| { TERMS.lines() diff --git a/crates/lib/machine_learning/prompt_engineering/src/keywords/mod.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/mod.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/keywords/mod.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/keywords/mod.rs diff --git a/crates/lib/machine_learning/prompt_engineering/src/lib.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/lib.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/lib.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/lib.rs diff --git a/crates/lib/machine_learning/prompt_engineering/src/regexes/minor_regex.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/minor_regex.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/regexes/minor_regex.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/minor_regex.rs diff --git a/crates/lib/machine_learning/prompt_engineering/src/regexes/mod.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/mod.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/regexes/mod.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/mod.rs diff --git a/crates/lib/machine_learning/prompt_engineering/src/regexes/nsfw_regex.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/nsfw_regex.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/regexes/nsfw_regex.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/nsfw_regex.rs diff --git a/crates/lib/machine_learning/prompt_engineering/src/regexes/potential_minor_regex.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/potential_minor_regex.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/src/regexes/potential_minor_regex.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/src/regexes/potential_minor_regex.rs diff --git a/crates/lib/machine_learning/prompt_engineering/tests/borderline_tests.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/tests/borderline_tests.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/tests/borderline_tests.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/tests/borderline_tests.rs diff --git a/crates/lib/machine_learning/prompt_engineering/tests/csam_prompt_tests.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/tests/csam_prompt_tests.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/tests/csam_prompt_tests.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/tests/csam_prompt_tests.rs diff --git a/crates/lib/machine_learning/prompt_engineering/tests/nsfw_prompt_tests.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/tests/nsfw_prompt_tests.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/tests/nsfw_prompt_tests.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/tests/nsfw_prompt_tests.rs diff --git a/crates/lib/machine_learning/prompt_engineering/tests/racist_prompt_tests.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/tests/racist_prompt_tests.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/tests/racist_prompt_tests.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/tests/racist_prompt_tests.rs diff --git a/crates/lib/machine_learning/prompt_engineering/tests/sfw_prompt_tests.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/tests/sfw_prompt_tests.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/tests/sfw_prompt_tests.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/tests/sfw_prompt_tests.rs diff --git a/crates/lib/machine_learning/prompt_engineering/tests/spicy_prompt_tests.rs b/crates/lib/deprecated/machine_learning/prompt_engineering/tests/spicy_prompt_tests.rs similarity index 100% rename from crates/lib/machine_learning/prompt_engineering/tests/spicy_prompt_tests.rs rename to crates/lib/deprecated/machine_learning/prompt_engineering/tests/spicy_prompt_tests.rs diff --git a/crates/lib/subprocess_common/Cargo.toml b/crates/lib/deprecated/subprocess_common/Cargo.toml similarity index 84% rename from crates/lib/subprocess_common/Cargo.toml rename to crates/lib/deprecated/subprocess_common/Cargo.toml index 69f3d6da4f..2a95352f60 100644 --- a/crates/lib/subprocess_common/Cargo.toml +++ b/crates/lib/deprecated/subprocess_common/Cargo.toml @@ -18,10 +18,10 @@ normal = ["workspace-build-acceleration"] [dependencies] # Internal -errors = { path = "../errors" } -filesys = { path = "../files/filesys" } +errors = { path = "../../errors" } +filesys = { path = "../../files/filesys" } -workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" } +workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } # External log = "0.4.14" diff --git a/crates/lib/subprocess_common/src/command_exit_status.rs b/crates/lib/deprecated/subprocess_common/src/command_exit_status.rs similarity index 100% rename from crates/lib/subprocess_common/src/command_exit_status.rs rename to crates/lib/deprecated/subprocess_common/src/command_exit_status.rs diff --git a/crates/lib/subprocess_common/src/command_runner/command_args.rs b/crates/lib/deprecated/subprocess_common/src/command_runner/command_args.rs similarity index 100% rename from crates/lib/subprocess_common/src/command_runner/command_args.rs rename to crates/lib/deprecated/subprocess_common/src/command_runner/command_args.rs diff --git a/crates/lib/subprocess_common/src/command_runner/command_runner.rs b/crates/lib/deprecated/subprocess_common/src/command_runner/command_runner.rs similarity index 100% rename from crates/lib/subprocess_common/src/command_runner/command_runner.rs rename to crates/lib/deprecated/subprocess_common/src/command_runner/command_runner.rs diff --git a/crates/lib/subprocess_common/src/command_runner/command_runner_args.rs b/crates/lib/deprecated/subprocess_common/src/command_runner/command_runner_args.rs similarity index 100% rename from crates/lib/subprocess_common/src/command_runner/command_runner_args.rs rename to crates/lib/deprecated/subprocess_common/src/command_runner/command_runner_args.rs diff --git a/crates/lib/subprocess_common/src/command_runner/env_var_policy.rs b/crates/lib/deprecated/subprocess_common/src/command_runner/env_var_policy.rs similarity index 100% rename from crates/lib/subprocess_common/src/command_runner/env_var_policy.rs rename to crates/lib/deprecated/subprocess_common/src/command_runner/env_var_policy.rs diff --git a/crates/lib/subprocess_common/src/command_runner/mod.rs b/crates/lib/deprecated/subprocess_common/src/command_runner/mod.rs similarity index 100% rename from crates/lib/subprocess_common/src/command_runner/mod.rs rename to crates/lib/deprecated/subprocess_common/src/command_runner/mod.rs diff --git a/crates/lib/subprocess_common/src/docker_options.rs b/crates/lib/deprecated/subprocess_common/src/docker_options.rs similarity index 100% rename from crates/lib/subprocess_common/src/docker_options.rs rename to crates/lib/deprecated/subprocess_common/src/docker_options.rs diff --git a/crates/lib/subprocess_common/src/executable_or_command.rs b/crates/lib/deprecated/subprocess_common/src/executable_or_command.rs similarity index 100% rename from crates/lib/subprocess_common/src/executable_or_command.rs rename to crates/lib/deprecated/subprocess_common/src/executable_or_command.rs diff --git a/crates/lib/subprocess_common/src/lib.rs b/crates/lib/deprecated/subprocess_common/src/lib.rs similarity index 100% rename from crates/lib/subprocess_common/src/lib.rs rename to crates/lib/deprecated/subprocess_common/src/lib.rs diff --git a/crates/lib/tts_common/Cargo.toml b/crates/lib/deprecated/tts_common/Cargo.toml similarity index 83% rename from crates/lib/tts_common/Cargo.toml rename to crates/lib/deprecated/tts_common/Cargo.toml index a615b69e35..205b9369d3 100644 --- a/crates/lib/tts_common/Cargo.toml +++ b/crates/lib/deprecated/tts_common/Cargo.toml @@ -18,11 +18,11 @@ normal = ["workspace-build-acceleration"] [dependencies] # Internal -errors = { path = "../errors" } -user_input_common = { path = "../user_input_common" } +errors = { path = "../../errors" } +user_input_common = { path = "../../user_input_common" } # Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" } +workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } # External anyhow = "1.0.86" diff --git a/crates/lib/tts_common/src/clean_symbols.rs b/crates/lib/deprecated/tts_common/src/clean_symbols.rs similarity index 100% rename from crates/lib/tts_common/src/clean_symbols.rs rename to crates/lib/deprecated/tts_common/src/clean_symbols.rs diff --git a/crates/lib/tts_common/src/lib.rs b/crates/lib/deprecated/tts_common/src/lib.rs similarity index 100% rename from crates/lib/tts_common/src/lib.rs rename to crates/lib/deprecated/tts_common/src/lib.rs diff --git a/crates/lib/tts_common/src/priority.rs b/crates/lib/deprecated/tts_common/src/priority.rs similarity index 100% rename from crates/lib/tts_common/src/priority.rs rename to crates/lib/deprecated/tts_common/src/priority.rs diff --git a/crates/lib/tts_common/src/text_pipelines/guess_pipeline.rs b/crates/lib/deprecated/tts_common/src/text_pipelines/guess_pipeline.rs similarity index 100% rename from crates/lib/tts_common/src/text_pipelines/guess_pipeline.rs rename to crates/lib/deprecated/tts_common/src/text_pipelines/guess_pipeline.rs diff --git a/crates/lib/tts_common/src/text_pipelines/mod.rs b/crates/lib/deprecated/tts_common/src/text_pipelines/mod.rs similarity index 100% rename from crates/lib/tts_common/src/text_pipelines/mod.rs rename to crates/lib/deprecated/tts_common/src/text_pipelines/mod.rs diff --git a/crates/lib/tts_common/src/text_pipelines/text_pipeline_type.rs b/crates/lib/deprecated/tts_common/src/text_pipelines/text_pipeline_type.rs similarity index 100% rename from crates/lib/tts_common/src/text_pipelines/text_pipeline_type.rs rename to crates/lib/deprecated/tts_common/src/text_pipelines/text_pipeline_type.rs diff --git a/crates/lib/dimensions/Cargo.toml b/crates/lib/dimensions/Cargo.toml deleted file mode 100644 index 63a527ec53..0000000000 --- a/crates/lib/dimensions/Cargo.toml +++ /dev/null @@ -1,30 +0,0 @@ -[package] -name = "dimensions" -edition = "2018" -version = "0.0.1" -authors = [ - "Brandon Thomas ", - "Brandon Thomas ", -] -publish = false - -[lib] -# NB: Public "dimensions" crate is not relevant / abandoned, so we're fine to use this internally. -name = "dimensions" -path = "src/lib.rs" - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-build-acceleration"] - -[dependencies] -# Workspace hack (faster compile times) -workspace-build-acceleration = { workspace = true } - -# Internal -# (None Yet) - -# External -# (None Yet) - -[dev-dependencies] -# None yet diff --git a/crates/lib/dimensions/src/lib.rs b/crates/lib/dimensions/src/lib.rs deleted file mode 100644 index 1b83d9f365..0000000000 --- a/crates/lib/dimensions/src/lib.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! dimensions -//! -//! The purpose of this library is to specify common dimensions for video and -//! to do aspect ratio calculations in a reusable manner. -//! - -// Never allow these -#![forbid(private_bounds)] -#![forbid(private_interfaces)] -#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things - -// Okay to toggle -#![forbid(unreachable_patterns)] -#![forbid(unused_imports)] -#![forbid(unused_mut)] -#![forbid(unused_variables)] - -// Always allow -#![allow(dead_code)] -#![allow(non_snake_case)] diff --git a/crates/lib/files/archives/Cargo.toml b/crates/lib/files/archives/Cargo.toml deleted file mode 100644 index f516dfe991..0000000000 --- a/crates/lib/files/archives/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "archives" -edition = "2018" -version = "0.0.1" -authors = [ - "Brandon Thomas ", - "Brandon Thomas ", -] -publish = false - -[lib] -# NB: Public "archives" crate does not exist, so it's safe to use for now. -# We'll use this for zip, tgz, rar, etc. file types and related utilities. -name = "archives" -path = "src/lib.rs" - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-build-acceleration"] - -[dependencies] -# Internal -errors = { path = "../../errors" } - -# Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } - -# External -zip = "0.6.6" - -[dev-dependencies] -# None yet diff --git a/crates/lib/files/file_to_base64_url/Cargo.toml b/crates/lib/files/file_to_base64_url/Cargo.toml index 1f1ae40177..4f5672a8c6 100644 --- a/crates/lib/files/file_to_base64_url/Cargo.toml +++ b/crates/lib/files/file_to_base64_url/Cargo.toml @@ -17,12 +17,12 @@ path = "src/lib.rs" workspace-build-acceleration = { workspace = true } # Internal -errors = { path = "../../errors" } +errors.workspace = true # External -anyhow = { workspace = true } -base64 = { workspace = true } -mimetypes = { workspace = true } +anyhow.workspace = true +base64.workspace = true +mimetypes.workspace = true [dev-dependencies] testing = { path = "../../testing" } diff --git a/crates/lib/files/filesys/Cargo.toml b/crates/lib/files/filesys/Cargo.toml index 6bff68871f..93d130e7bf 100644 --- a/crates/lib/files/filesys/Cargo.toml +++ b/crates/lib/files/filesys/Cargo.toml @@ -17,15 +17,15 @@ path = "src/lib.rs" normal = ["workspace-build-acceleration"] [dependencies] -# Workspace hack (faster compile times) +# cargo-hakari workspace hack (faster compile times) workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } # Internal (keep this to a minimum!) -errors = { path = "../../errors" } +errors.workspace = true # External # KEEP THIS TO A MINIMUM OR NIL. -log = "0.4.20" +log.workspace = true walkdir = "2.4.0" [dev-dependencies] diff --git a/crates/service/plugins/inference_workloads/Cargo.toml b/crates/lib/files/zip_archives/Cargo.toml similarity index 73% rename from crates/service/plugins/inference_workloads/Cargo.toml rename to crates/lib/files/zip_archives/Cargo.toml index 954c95e3f4..f742b53a14 100644 --- a/crates/service/plugins/inference_workloads/Cargo.toml +++ b/crates/lib/files/zip_archives/Cargo.toml @@ -1,21 +1,29 @@ [package] -publish = false -name = "component_subsystem_inference_workloads" +name = "zip_archives" edition = "2018" version = "0.0.1" authors = [ "Brandon Thomas ", "Brandon Thomas ", ] +publish = false [lib] -name = "component_subsystem_inference_workloads" +name = "zip_archives" path = "src/lib.rs" [package.metadata.cargo-udeps.ignore] normal = ["workspace-build-acceleration"] [dependencies] +# Internal +errors = { path = "../../errors" } # Workspace hack (faster compile times) workspace-build-acceleration = { version = "0.1", path = "../../../workspace-build-acceleration" } + +# External +zip = "0.6.6" + +[dev-dependencies] +# None yet diff --git a/crates/lib/files/archives/src/lib.rs b/crates/lib/files/zip_archives/src/lib.rs similarity index 100% rename from crates/lib/files/archives/src/lib.rs rename to crates/lib/files/zip_archives/src/lib.rs diff --git a/crates/lib/files/archives/src/list_archive_files.rs b/crates/lib/files/zip_archives/src/list_archive_files.rs similarity index 100% rename from crates/lib/files/archives/src/list_archive_files.rs rename to crates/lib/files/zip_archives/src/list_archive_files.rs diff --git a/crates/lib/jwt_light/Cargo.toml b/crates/lib/jwt_light/Cargo.toml index f1a5b275ec..ae523f83e8 100644 --- a/crates/lib/jwt_light/Cargo.toml +++ b/crates/lib/jwt_light/Cargo.toml @@ -13,16 +13,16 @@ path = "src/lib.rs" [dependencies] -# Workspace hack (faster compile times) -workspace-build-acceleration = { workspace = true } +# cargo-hakari workspace hack (faster compile times) +workspace-build-acceleration.workspace = true # Internal # (None) # External -base64 = { workspace = true } -chrono = { workspace = true} -serde_json = { workspace = true } +base64.workspace = true +chrono.workspace = true +serde_json.workspace = true [dev-dependencies] # (None) \ No newline at end of file diff --git a/crates/lib/cookies/Cargo.toml b/crates/lib/jwt_signer/Cargo.toml similarity index 80% rename from crates/lib/cookies/Cargo.toml rename to crates/lib/jwt_signer/Cargo.toml index f16b493b84..2ecc02a03a 100644 --- a/crates/lib/cookies/Cargo.toml +++ b/crates/lib/jwt_signer/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "cookies" +name = "jwt_signer" edition = "2018" version = "0.0.1" authors = [ @@ -9,20 +9,19 @@ authors = [ publish = false [lib] -# NB: Public "cookies" crate is version 0.0.1 from 10 years ago! -name = "cookies" +name = "jwt_signer" path = "src/lib.rs" [package.metadata.cargo-udeps.ignore] normal = ["workspace-build-acceleration"] [dependencies] -# Internal -errors = { path = "../errors" } - # Workspace hack (faster compile times) workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" } +# Internal +errors.workspace = true + # External hmac = "0.10.1" # jwt signing jwt = "0.13.0" diff --git a/crates/lib/cookies/src/jwt_signer.rs b/crates/lib/jwt_signer/src/jwt_signer.rs similarity index 100% rename from crates/lib/cookies/src/jwt_signer.rs rename to crates/lib/jwt_signer/src/jwt_signer.rs diff --git a/crates/lib/cookies/src/lib.rs b/crates/lib/jwt_signer/src/lib.rs similarity index 100% rename from crates/lib/cookies/src/lib.rs rename to crates/lib/jwt_signer/src/lib.rs diff --git a/crates/lib/reqwest_helpers/Cargo.toml b/crates/lib/reqwest_helpers/Cargo.toml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/reqwest_helpers/src/lib.rs b/crates/lib/reqwest_helpers/src/lib.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/idempotency/Cargo.toml b/crates/lib/uuid_utils/Cargo.toml similarity index 93% rename from crates/lib/idempotency/Cargo.toml rename to crates/lib/uuid_utils/Cargo.toml index 16978a4edb..5dcf6a781e 100644 --- a/crates/lib/idempotency/Cargo.toml +++ b/crates/lib/uuid_utils/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "idempotency" +name = "uuid_utils" edition = "2018" version = "0.0.1" authors = [ @@ -9,7 +9,7 @@ authors = [ publish = false [lib] -name = "idempotency" +name = "uuid_utils" path = "src/lib.rs" [package.metadata.cargo-udeps.ignore] diff --git a/crates/lib/idempotency/src/lib.rs b/crates/lib/uuid_utils/src/lib.rs similarity index 100% rename from crates/lib/idempotency/src/lib.rs rename to crates/lib/uuid_utils/src/lib.rs diff --git a/crates/lib/idempotency/src/uuid.rs b/crates/lib/uuid_utils/src/uuid.rs similarity index 100% rename from crates/lib/idempotency/src/uuid.rs rename to crates/lib/uuid_utils/src/uuid.rs diff --git a/crates/lib/web_scrapers/Cargo.toml b/crates/lib/web_scrapers/Cargo.toml deleted file mode 100644 index 7e593e1106..0000000000 --- a/crates/lib/web_scrapers/Cargo.toml +++ /dev/null @@ -1,46 +0,0 @@ -[package] -name = "web_scrapers" -edition = "2018" -version = "0.0.1" -authors = [ - "Brandon Thomas ", - "Brandon Thomas ", -] -publish = false - -[lib] -name = "web_scrapers" -path = "src/lib.rs" - -[package.metadata.cargo-udeps.ignore] -normal = ["workspace-build-acceleration"] - -[dependencies] -# Internal -enums = { path = "../../schema/public/enums" } -errors = { path = "../../lib/errors" } - -# Workspace hack (faster compile times) -workspace-build-acceleration = { version = "0.1", path = "../../workspace-build-acceleration" } - -# External -bytes = "1.6" -chrono = { version = "0.4.22", features = ["serde"] } -log = "0.4.14" -once_cell = "1.18.0" -strum = { version = "0.26", features = ["derive"] } # Fancy enum iteration, etc. - -# Serialization -serde = "1.0.125" -serde_derive = "1.0.125" -serde_json = "1.0.108" - -# HTTP -reqwest = { version = "0.12", default-features = false, features = ["charset", "http2", "system-proxy", "rustls-tls", "json"] } # HTTP Client - -# Scraping -rss = "2.0.2" -scraper = "0.17.1" - -[dev-dependencies] -# None yet diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs b/crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs deleted file mode 100644 index 978ea28a58..0000000000 --- a/crates/lib/web_scrapers/src/common_extractors/extract_attribute.rs +++ /dev/null @@ -1,47 +0,0 @@ -use scraper::{Html, Selector}; - -pub fn extract_attribute(document: &Html, element_selector: &Selector, attribute_name: &str) -> Option { - let element = match document.select(element_selector).next() { - None => { - println!("NOT FOUND"); - return None - }, - Some(element) => element, - }; - - println!("FOUND"); - - element.value() - .attr(attribute_name) - .map(|value| value.to_string()) -} - -//#[cfg(test)] -//mod tests { -// use crate::common_extractors::extract_text::extract_text; -// use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; -// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; -// use scraper::{Html, Selector}; -// -// #[test] -// fn test_extract_text_cnn() { -// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); -// let document = Html::parse_document(&html); -// -// let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR); -// -// assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), -// maybe_text.as_deref()); -// } -// -// #[test] -// fn test_extract_text_techcrunch() { -// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); -// let document = Html::parse_document(&html); -// -// let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR); -// -// assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), -// maybe_text.as_deref()); -// } -//} diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs b/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs deleted file mode 100644 index a9fe506e78..0000000000 --- a/crates/lib/web_scrapers/src/common_extractors/extract_featured_image.rs +++ /dev/null @@ -1,48 +0,0 @@ -use scraper::{Html, Selector}; - -pub fn extract_featured_image(document: &Html, image_selector: &Selector) -> Option { - let image = match document.select(image_selector).next() { - None => return None, - Some(title_element) => title_element, - }; - - if let Some(img_source) = image.value().attr("src") { - return Some(img_source.to_string()); - } - - // TODO: We might want more heuristics here, though there's a danger this method gets complicated. - - None -} - -//#[cfg(test)] -//mod tests { -// use scraper::{Html, Selector}; -// -// use crate::common_extractors::extract_featured_image::extract_featured_image; -// use crate::sites::cnn::cnn_article_scraper::CNN_FEATURED_IMAGE_SELECTOR; -// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_FEATURED_IMAGE_SELECTOR; -// -// #[test] -// fn test_extract_cnn_with_video() { -// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); -// let document = Html::parse_document(&html); -// -// let maybe_image = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR); -// -// assert_eq!(Some("https://media.cnn.com/api/v1/images/stellar/prod/220426143055-bill-ford-executive-chairman-ford-motor-company.jpg?c=16x9&q=w_850,c_fill"), -// maybe_image.as_deref()); -// } -// -// #[test] -// fn test_extract_techcrunch() { -// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); -// let document = Html::parse_document(&html); -// -// let selector = Selector::parse(".article__title").expect("selector should parse"); -// let maybe_image = extract_featured_image(&document, &TECHCRUNCH_FEATURED_IMAGE_SELECTOR); -// -// assert_eq!(Some("https://techcrunch.com/wp-content/uploads/2023/02/Screen-Shot-2023-02-02-at-1.46.55-PM.png?w=650"), -// maybe_image.as_deref()); -// } -//} diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs b/crates/lib/web_scrapers/src/common_extractors/extract_text.rs deleted file mode 100644 index 8ad2ce4520..0000000000 --- a/crates/lib/web_scrapers/src/common_extractors/extract_text.rs +++ /dev/null @@ -1,50 +0,0 @@ -use scraper::{Html, Selector}; - -pub fn extract_text(document: &Html, text_selector: &Selector) -> Option { - let text_element = match document.select(text_selector).next() { - None => return None, - Some(text_element) => text_element, - }; - - let mut pieces = Vec::new(); - - for mut text in text_element.text() { - text = text.trim(); - if !text.is_empty() { - pieces.push(text.to_string()); - } - } - - Some(pieces.join(" ").trim().to_string()) -} - -//#[cfg(test)] -//mod tests { -// use scraper::Html; -// -// use crate::common_extractors::extract_text::extract_text; -// use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; -// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; -// -// #[test] -// fn test_extract_text_cnn() { -// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); -// let document = Html::parse_document(&html); -// -// let maybe_text = extract_text(&document, &CNN_TITLE_SELECTOR); -// -// assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), -// maybe_text.as_deref()); -// } -// -// #[test] -// fn test_extract_text_techcrunch() { -// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); -// let document = Html::parse_document(&html); -// -// let maybe_text = extract_text(&document, &TECHCRUNCH_TITLE_SELECTOR); -// -// assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), -// maybe_text.as_deref()); -// } -//} diff --git a/crates/lib/web_scrapers/src/common_extractors/extract_title.rs b/crates/lib/web_scrapers/src/common_extractors/extract_title.rs deleted file mode 100644 index a1ef30c50a..0000000000 --- a/crates/lib/web_scrapers/src/common_extractors/extract_title.rs +++ /dev/null @@ -1,50 +0,0 @@ -use scraper::{Html, Selector}; - -pub fn extract_title(document: &Html, title_selector: &Selector) -> Option { - let title_element = match document.select(title_selector).next() { - None => return None, - Some(title_element) => title_element, - }; - - let mut pieces = Vec::new(); - - for mut text in title_element.text() { - text = text.trim(); - if !text.is_empty() { - pieces.push(text.to_string()); - } - } - - Some(pieces.join(" ").trim().to_string()) -} - -//#[cfg(test)] -//mod tests { -// use scraper::Html; -// -// use crate::common_extractors::extract_title::extract_title; -// use crate::sites::cnn::cnn_article_scraper::CNN_TITLE_SELECTOR; -// use crate::sites::techcrunch::techcrunch_article_scraper::TECHCRUNCH_TITLE_SELECTOR; -// -// #[test] -// fn test_extract_title_cnn() { -// let html = include_str!("../../../../../test_data/html_scraping/cnn_article_with_video.html"); -// let document = Html::parse_document(&html); -// -// let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR); -// -// assert_eq!(Some("Ford Mustang Mach-E has a mile of wires it doesn’t need. That’s a big deal"), -// maybe_title.as_deref()); -// } -// -// #[test] -// fn test_extract_title_techcrunch() { -// let html = include_str!("../../../../../test_data/html_scraping/techcrunch_article.html"); -// let document = Html::parse_document(&html); -// -// let maybe_title = extract_title(&document, &TECHCRUNCH_TITLE_SELECTOR); -// -// assert_eq!(Some("‘Nothing, Forever,’ an AI ‘Seinfeld’ spoof, is the next ‘Twitch Plays Pokémon’"), -// maybe_title.as_deref()); -// } -//} diff --git a/crates/lib/web_scrapers/src/common_extractors/mod.rs b/crates/lib/web_scrapers/src/common_extractors/mod.rs deleted file mode 100644 index 8d7c564017..0000000000 --- a/crates/lib/web_scrapers/src/common_extractors/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -pub mod extract_attribute; -pub mod extract_featured_image; -pub mod extract_text; -pub mod extract_title; - diff --git a/crates/lib/web_scrapers/src/lib.rs b/crates/lib/web_scrapers/src/lib.rs deleted file mode 100644 index 3d6b9e285b..0000000000 --- a/crates/lib/web_scrapers/src/lib.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! web_scrapers -//! -//! The purpose of this library is to scrape content from websites. -//! This should power our AI Chatbot and other functionalities. -//! - -// Never allow these -#![forbid(private_bounds)] -#![forbid(private_interfaces)] -#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things - -//// Okay to toggle -//#![forbid(unreachable_patterns)] -//#![forbid(unused_imports)] -//#![forbid(unused_mut)] -//#![forbid(unused_variables)] - -// Always allow -#![allow(dead_code)] -#![allow(non_snake_case)] - -#[macro_use] extern crate serde_derive; -#[macro_use] extern crate strum; - -pub mod common_extractors; -pub mod payloads; -pub mod scrape_supported_webpage; -pub mod sites; -pub (crate) mod utils; diff --git a/crates/lib/web_scrapers/src/payloads/mod.rs b/crates/lib/web_scrapers/src/payloads/mod.rs deleted file mode 100644 index 4c5bdc3ffc..0000000000 --- a/crates/lib/web_scrapers/src/payloads/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod web_scraping_target; -pub mod web_scraping_result; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs deleted file mode 100644 index 8fdc48d236..0000000000 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_result.rs +++ /dev/null @@ -1,55 +0,0 @@ -use chrono::{DateTime, Utc}; - -use enums::common::sqlite::web_content_type::WebContentType; - -// TODO: Rename this to not confuse with `Result`. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct WebScrapingResult { - /// Surface the original HTML to callers that care about it. - pub original_html: String, - - /// Output of *successful* scraping. - pub result: ScrapedWebArticle, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ScrapedWebArticle { - /// Location the article came from - pub url: String, - - /// What this is and where it came from. - pub web_content_type: WebContentType, - - /// From page title or heading - pub maybe_title: Option, - - /// Some pages have subtitles under the main title. - pub maybe_subtitle: Option, - - /// The author of the article - pub maybe_author: Option, - - /// The paragraphs - pub paragraphs: Vec, - - /// The paragraphs joined by newlines. - pub body_text: String, - - /// A heading image (at top), if present - pub maybe_heading_image_url: Option, - - /// A featured image (somewhere in the body), if present - pub maybe_featured_image_url: Option, - - /// A timestamp that has not been parsed to a proper time - pub maybe_publish_timestamp_raw: Option, - - /// A timestamp that has not been parsed to a proper time - pub maybe_publish_datetime_utc: Option>, -} - -// TODO: Other types - threaded discussions (HN, Reddit), video essays (CNN, YouTube), etc. -// then return these in an enum container. -pub struct ScrapedThreadedDiscussion; - -pub struct ScrapedVideo; diff --git a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs b/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs deleted file mode 100644 index f84fb1943d..0000000000 --- a/crates/lib/web_scrapers/src/payloads/web_scraping_target.rs +++ /dev/null @@ -1,19 +0,0 @@ -use enums::common::sqlite::skip_reason::SkipReason; -use enums::common::sqlite::web_content_type::WebContentType; - -// TODO: Include the article publish date(!) - -/// From RSS feeds and index pages, we can deduce a list of WebScrapingTargets. -/// These will be inserted into the database for later downstream processing. -#[derive(Clone, Debug)] -pub struct WebScrapingTarget { - pub canonical_url: String, - pub web_content_type: WebContentType, - pub maybe_title: Option, - pub maybe_full_image_url: Option, - pub maybe_thumbnail_image_url: Option, - - /// If a possible skip reason was detected, we can choose to bail out. - /// At this stage, only the URL, RSS metadata, etc. can inform this. - pub maybe_skip_reason: Option, -} diff --git a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs b/crates/lib/web_scrapers/src/scrape_supported_webpage.rs deleted file mode 100644 index 8893f05544..0000000000 --- a/crates/lib/web_scrapers/src/scrape_supported_webpage.rs +++ /dev/null @@ -1,51 +0,0 @@ -use std::error::Error; -use std::fmt::{Display, Formatter}; - -use reqwest::Url; - -use crate::payloads::web_scraping_result::WebScrapingResult; -use crate::scrape_supported_webpage::ScrapeUtilityError::UrlParseError; -use crate::sites::cbsnews::cbsnews_article_scraper::cbsnews_article_scraper; -use crate::sites::cnn::cnn_article_scraper::cnn_article_scraper; -use crate::sites::gizmodo::gizmodo_article_scraper::gizmodo_article_scraper; -use crate::sites::kotaku::kotaku_article_scraper::kotaku_article_scraper; -use crate::sites::techcrunch::techcrunch_article_scraper::techcrunch_article_scraper; -use crate::sites::theguardian::theguardian_article_scraper::theguardian_article_scraper; - -#[derive(Debug, Clone)] -pub enum ScrapeUtilityError { - UrlParseError, - NoHostnameError, - UnknownUrl, - // TODO: Make inner reason more strongly typed. - ScrapingError { reason: String }, -} - -impl Display for ScrapeUtilityError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", self) - } -} - -impl Error for ScrapeUtilityError {} - -pub async fn scrape_supported_webpage(url: &str) -> Result { - let parsed_url = Url::parse(url) - .map_err(|_err| UrlParseError)?; - - let hostname = parsed_url.host() - .ok_or(ScrapeUtilityError::NoHostnameError)? - .to_string(); - - let maybe_result = match hostname.as_ref() { - "cbsnews.com" | "www.cbsnews.com" => cbsnews_article_scraper(url).await, - "cnn.com" | "www.cnn.com" => cnn_article_scraper(url).await, - "gizmodo.com" | "www.gizmodo.com" => gizmodo_article_scraper(url).await, - "kotaku.com" | "www.kotaku.com" => kotaku_article_scraper(url).await, - "techcrunch.com" | "www.techcrunch.com" => techcrunch_article_scraper(url).await, - "theguardian.com" | "www.theguardian.com" => theguardian_article_scraper(url).await, - _ => return Err(ScrapeUtilityError::UnknownUrl), - }; - - maybe_result.map_err(|err| ScrapeUtilityError::ScrapingError { reason: err.to_string() }) -} diff --git a/crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs deleted file mode 100644 index 24ee7f84fd..0000000000 --- a/crates/lib/web_scrapers/src/sites/cbsnews/cbsnews_article_scraper.rs +++ /dev/null @@ -1,122 +0,0 @@ -use std::ops::Deref; - -use chrono::{DateTime, LocalResult, TimeZone, Utc}; -use once_cell::sync::Lazy; -use scraper::{Html, Selector}; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::{anyhow, AnyhowResult}; - -use crate::common_extractors::extract_attribute::extract_attribute; -use crate::common_extractors::extract_featured_image::extract_featured_image; -use crate::common_extractors::extract_title::extract_title; -use crate::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult}; - -/// The main article content container -static ARTICLE_CONTENT_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse("section.content__body").expect("this selector should parse") -}); - -/// Paragraphs within the article -static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse("p").expect("this selector should parse") -}); - -/// The title of the article -pub static TITLE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("h1.content__title").expect("this selector should parse") -}); - -///// The title of the article -//pub static SUBTITLE_SELECTOR: Lazy = Lazy::new(|| { -// Selector::parse("div[data-gu-name=\"standfirst\"] p").expect("this selector should parse") -//}); - -/// The article featured image -pub static FEATURED_IMAGE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("figure .embed__content img").expect("this selector should parse") -}); - -pub static DATETIME_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".content__meta--timestamp time").expect("this selector should parse") -}); - -pub async fn cbsnews_article_scraper(url: &str) -> AnyhowResult { - let downloaded_document= reqwest::get(url) - .await? - .bytes() - .await?; - - let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); - let document = Html::parse_document(&downloaded_document); - - let mut paragraphs = Vec::new(); - - if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { - for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() { - - let mut paragraph_assembly = Vec::new(); - - for text in paragraph.text() { - let stripped = text.trim(); - if !stripped.is_empty() { - paragraph_assembly.push(stripped.to_string()); - } - } - - let paragraph_full_text = paragraph_assembly.join(" ") - .trim() - .to_string(); - - if !paragraph_full_text.is_empty() { - paragraphs.push(paragraph_full_text); - } - } - } - - let maybe_title = extract_title(&document, &TITLE_SELECTOR); - let maybe_featured_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR); - - let maybe_timestamp_raw = extract_attribute(&document, &DATETIME_SELECTOR, "datetime"); - - // Timestamp is present as *almost* RFC3339: 2023-02-23T19:30:30-0500 - // Frustratingly, there should be a colon in the timezone offset. - let maybe_timestamp = maybe_timestamp_raw - .as_deref() - // This is a lazy af fix for the RFC3339 non-compliance. - .map(|invalid| invalid.replace("-0400", "-04:00")) - .map(|invalid| invalid.replace("-0500", "-05:00")) - .map(|invalid| invalid.replace("-0600", "-06:00")) - .map(|invalid| invalid.replace("-0700", "-07:00")) - .map(|invalid| invalid.replace("-0800", "-08:00")) - .map(|invalid| invalid.replace("-0900", "-09:00")) - .map(|ts| DateTime::parse_from_rfc3339(ts.trim())) - .transpose()? - // We need to juggle from DateTime to DateTime. - .map(|dt| dt.naive_utc()) - .map(|naive| match Utc.from_local_datetime(&naive) { - LocalResult::None => Err(anyhow!("invalid datetime conversion")), - LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")), - LocalResult::Single(time) => Ok(time), - }) - .transpose()?; - - let body_text = paragraphs.join("\n\n"); - - Ok(WebScrapingResult { - original_html: downloaded_document, - result: ScrapedWebArticle { - url: url.to_string(), - web_content_type: WebContentType::CbsNewsArticle, - maybe_title, - maybe_subtitle: None, - maybe_author: None, // TODO - paragraphs, - body_text, - maybe_heading_image_url: None, // TODO - maybe_featured_image_url, - maybe_publish_timestamp_raw: maybe_timestamp_raw, - maybe_publish_datetime_utc: maybe_timestamp, - } - }) -} diff --git a/crates/lib/web_scrapers/src/sites/cbsnews/mod.rs b/crates/lib/web_scrapers/src/sites/cbsnews/mod.rs deleted file mode 100644 index d1e25b853d..0000000000 --- a/crates/lib/web_scrapers/src/sites/cbsnews/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod cbsnews_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs deleted file mode 100644 index 5020c82200..0000000000 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_article_scraper.rs +++ /dev/null @@ -1,158 +0,0 @@ -use std::ops::{Add, Deref}; - -use chrono::{DateTime, Duration, LocalResult, NaiveDateTime, TimeZone, Utc}; -use once_cell::sync::Lazy; -use scraper::{Html, Selector}; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::{anyhow, AnyhowResult}; - -use crate::common_extractors::extract_featured_image::extract_featured_image; -use crate::common_extractors::extract_text::extract_text; -use crate::common_extractors::extract_title::extract_title; -use crate::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult}; -use crate::utils::remove_timestamp_abbreviated_timezone::remove_timestamp_abbreviated_timezone; -use crate::utils::remove_timestamp_abbreviated_weekday_name::remove_timestamp_abbreviated_weekday_name; - -/// The main article content container -static ARTICLE_CONTENT_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".article__content").expect("this selector should parse") -}); - -/// Paragraphs within the article -static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse("p.paragraph").expect("this selector should parse") -}); - -/// The title of the article -pub static CNN_TITLE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse(".headline__text").expect("this selector should parse") -}); - -/// The article featured image -pub static CNN_FEATURED_IMAGE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse(".image__lede .image__picture > img").expect("this selector should parse") -}); - -pub static DATETIME_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".timestamp").expect("this selector should parse") -}); - -pub async fn cnn_article_scraper(url: &str) -> AnyhowResult { - let downloaded_document= reqwest::get(url) - .await? - .bytes() - .await?; - - let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); - let document = Html::parse_document(&downloaded_document); - - let mut paragraphs = Vec::new(); - - if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { - for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() { - - let mut paragraph_assembly = Vec::new(); - - for text in paragraph.text() { - let stripped = text.trim(); - if !stripped.is_empty() { - paragraph_assembly.push(stripped.to_string()); - } - } - - let paragraph_full_text = paragraph_assembly.join(" ") - .trim() - .to_string(); - - if !paragraph_full_text.is_empty() { - paragraphs.push(paragraph_full_text); - } - } - } - - let maybe_title = extract_title(&document, &CNN_TITLE_SELECTOR); - let maybe_heading_image_url = extract_featured_image(&document, &CNN_FEATURED_IMAGE_SELECTOR); - - // Timestamp is present as "Published\n11:25 PM EST, Thu February 23, 2023" - let maybe_timestamp_raw = extract_text(&document, &DATETIME_SELECTOR) - .map(|raw| raw.replace("Published", "")) - .map(|raw| raw.replace("Updated", "")) - .map(|raw| raw.trim().to_string()); - - let maybe_timestamp = maybe_timestamp_raw - .as_deref() - .map(|ts| parse_timestamp(ts)) - .flatten(); - - let body_text = paragraphs.join("\n\n"); - - Ok(WebScrapingResult { - original_html: downloaded_document, - result: ScrapedWebArticle { - url: url.to_string(), - web_content_type: WebContentType::CnnArticle, - maybe_title, - maybe_subtitle: None, - maybe_author: None, // TODO - paragraphs, - body_text, - maybe_heading_image_url, - maybe_featured_image_url: None, // TODO - maybe_publish_timestamp_raw: maybe_timestamp_raw, - maybe_publish_datetime_utc: maybe_timestamp, - } - }) -} - -fn parse_timestamp(timestamp: &str) -> Option> { - // NB: CNN article timestamps are horrible. - let timestamp = timestamp - .replace("Published\n", "") - .replace("Updated\n", "") - .trim() - .to_string(); - - // NB: Remove attributes that parse_from_str() struggles with. - // TODO: This is such a flaky, unstable hack - let maybe_timestamp = Some(timestamp) - .map(|ts| remove_timestamp_abbreviated_weekday_name(&ts)) - .map(|ts| remove_timestamp_abbreviated_timezone(&ts)) - .map(|ts| ts.replace(" ", " ")) - .map(|ts| ts.replace(" , ", ", ")); - - // Parse - let maybe_timestamp = maybe_timestamp - .map(|ts| NaiveDateTime::parse_from_str(&ts, "%l:%M %P, %B %e, %Y")) - .transpose() - .ok() - .flatten() - .map(|naive| match Utc.from_local_datetime(&naive) { - LocalResult::None => Err(anyhow!("invalid datetime conversion")), - LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")), - LocalResult::Single(time) => Ok(time), - }) - .transpose() - .ok() - .flatten() - // NB: The "Utc" time above is actually EST/EDT. - .map(|utc| utc.add(Duration::hours(5))); - - maybe_timestamp -} - -#[cfg(test)] -mod tests { - use crate::sites::cnn::cnn_article_scraper::parse_timestamp; - - #[test] - fn test_published() { - // NB: Actual website timestamp - assert!(parse_timestamp("Updated\n 5:55 PM EST, Thu February 23, 2023").is_some()); - } - - fn test_updated() { - // NB: Actual website timestamp - assert!(parse_timestamp("Published\n 11:25 PM EST, Thu February 23, 2023").is_some()); - } -} \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs b/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs deleted file mode 100644 index 5347b47368..0000000000 --- a/crates/lib/web_scrapers/src/sites/cnn/cnn_indexer.rs +++ /dev/null @@ -1,93 +0,0 @@ -use log::warn; -use rss::Channel; - -use enums::common::sqlite::skip_reason::SkipReason; -use enums::common::sqlite::web_content_type::WebContentType; -use errors::AnyhowResult; - -use crate::payloads::web_scraping_target::WebScrapingTarget; - -// NB: Contains nearly 70 items -const RSS_TOP_STORIES : &str = "http://rss.cnn.com/rss/cnn_topstories.rss"; - -const RSS_WORLD : &str = "http://rss.cnn.com/rss/cnn_world.rss"; - -const RSS_US : &str = "http://rss.cnn.com/rss/cnn_us.rss"; - -const RSS_TECH : &str = "http://rss.cnn.com/rss/cnn_tech.rss"; - -const RSS_ENTERTAINMENT : &str = "http://rss.cnn.com/rss/cnn_showbiz.rss"; - -#[derive(Copy, Clone, Debug, EnumIter, EnumCount)] -pub enum CnnFeed { - TopStories, - World, - UnitedStates, - Tech, - Entertainment, -} - -impl CnnFeed { - fn url(&self) -> &'static str { - match self { - Self::TopStories => RSS_TOP_STORIES, - Self::World => RSS_WORLD, - Self::UnitedStates => RSS_US, - Self::Tech => RSS_TECH, - Self::Entertainment => RSS_ENTERTAINMENT, - } - } -} - -pub async fn cnn_indexer(feed: CnnFeed) -> AnyhowResult> { - let content = reqwest::get(feed.url()) - .await? - .bytes() - .await?; - - let channel = Channel::read_from(&content[..])?; - - let mut targets = Vec::with_capacity(channel.items.len()); - - for item in channel.items { - let canonical_url = match item.link { - Some(url) => url.clone(), - None => { - warn!("Skipping item due to not having a URL"); - continue; - } - }; - - let mut maybe_skip_reason = None; - - // Lazy heuristic to skip CNN video articles, which contain little text. - // eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn - if canonical_url.contains("/videos/") { - maybe_skip_reason = Some(SkipReason::VideoContent); - } - - let maybe_image_url = item.extensions.get("media") - .map(|media| media.get("group")) - .flatten() - .map(|group| group.get(0)) - .flatten() - .map(|extension| extension.children.get("content")) - .flatten() - .map(|extensions| extensions.get(0)) // NB: First image is biggest - .flatten() - .map(|extension| extension.attrs.get("url")) - .flatten() - .map(|url| url.to_string()); - - targets.push(WebScrapingTarget { - canonical_url, - web_content_type: WebContentType::CnnArticle, - maybe_title: item.title.clone(), - maybe_full_image_url: maybe_image_url, - maybe_thumbnail_image_url: None, - maybe_skip_reason, - }); - } - - Ok(targets) -} diff --git a/crates/lib/web_scrapers/src/sites/cnn/mod.rs b/crates/lib/web_scrapers/src/sites/cnn/mod.rs deleted file mode 100644 index 5709b91d9e..0000000000 --- a/crates/lib/web_scrapers/src/sites/cnn/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod cnn_indexer; -pub mod cnn_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_article_scraper.rs b/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_article_scraper.rs deleted file mode 100644 index af45f6c986..0000000000 --- a/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_article_scraper.rs +++ /dev/null @@ -1,108 +0,0 @@ -use std::ops::Deref; - -use chrono::{DateTime, LocalResult, TimeZone, Utc}; -use once_cell::sync::Lazy; -use scraper::{Html, Selector}; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::{anyhow, AnyhowResult}; - -use crate::common_extractors::extract_attribute::extract_attribute; -use crate::common_extractors::extract_title::extract_title; -use crate::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult}; - -/// The main article content container -static ARTICLE_CONTENT_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".js_post-content").expect("this selector should parse") -}); - -/// Paragraphs within the article -static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse("p").expect("this selector should parse") -}); - -/// The title of the article -pub static TITLE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("h1").expect("this selector should parse") -}); - -/// The title of the article -pub static SUBTITLE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("h2.js_regular-subhead").expect("this selector should parse") -}); - -pub static DATETIME_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".js_starterpost time").expect("this selector should parse") -}); - -pub async fn gizmodo_article_scraper(url: &str) -> AnyhowResult { - let downloaded_document= reqwest::get(url) - .await? - .bytes() - .await?; - - let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); - let document = Html::parse_document(&downloaded_document); - - let mut paragraphs = Vec::new(); - - if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { - for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() { - - let mut paragraph_assembly = Vec::new(); - - for text in paragraph.text() { - let stripped = text.trim(); - if !stripped.is_empty() { - paragraph_assembly.push(stripped.to_string()); - } - } - - let paragraph_full_text = paragraph_assembly.join(" ") - .trim() - .to_string(); - - if !paragraph_full_text.is_empty() { - paragraphs.push(paragraph_full_text); - } - } - } - - let maybe_title = extract_title(&document, &TITLE_SELECTOR); - let maybe_subtitle = extract_title(&document, &SUBTITLE_SELECTOR); - - let maybe_timestamp_raw = extract_attribute(&document, &DATETIME_SELECTOR, "datetime"); - - // Timestamp is present as RFC3339: 2023-02-23T19:30:30-05:00 - let maybe_timestamp = maybe_timestamp_raw - .as_deref() - .map(|ts| DateTime::parse_from_rfc3339(ts.trim())) - .transpose()? - // We need to juggle from DateTime to DateTime. - .map(|dt| dt.naive_utc()) - .map(|naive| match Utc.from_local_datetime(&naive) { - LocalResult::None => Err(anyhow!("invalid datetime conversion")), - LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")), - LocalResult::Single(time) => Ok(time), - }) - .transpose()?; - - let body_text = paragraphs.join("\n\n"); - - Ok(WebScrapingResult { - original_html: downloaded_document, - result: ScrapedWebArticle { - url: url.to_string(), - web_content_type: WebContentType::GizmodoArticle, - maybe_title, - maybe_subtitle, - maybe_author: None, // TODO - paragraphs, - body_text, - maybe_heading_image_url: None, // TODO - maybe_featured_image_url: None, // TODO - maybe_publish_timestamp_raw: maybe_timestamp_raw, - maybe_publish_datetime_utc: maybe_timestamp, - } - }) -} diff --git a/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs b/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs deleted file mode 100644 index 638b8a0251..0000000000 --- a/crates/lib/web_scrapers/src/sites/gizmodo/gizmodo_indexer.rs +++ /dev/null @@ -1,66 +0,0 @@ -use log::warn; -use rss::Channel; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::AnyhowResult; - -use crate::payloads::web_scraping_target::WebScrapingTarget; - -const GIZMODO_RSS : &str = "https://gizmodo.com/rss"; - -// TODO: Rename foo_feed() or foo_rss(), etc. -pub async fn gizmodo_indexer() -> AnyhowResult> { - let content = reqwest::get(GIZMODO_RSS) - .await? - .bytes() - .await?; - - let channel = Channel::read_from(&content[..])?; - - let mut targets = Vec::with_capacity(channel.items.len()); - - for item in channel.items { - let canonical_url = match item.link { - Some(url) => url.clone(), - None => { - warn!("Skipping item due to not having a URL"); - continue; - } - }; - - let maybe_skip_reason = None; - - //// Lazy heuristic to skip CNN video articles, which contain little text. - //// eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn - //if canonical_url.contains("/videos/") { - // maybe_skip_reason = Some(SkipReason::VideoContent); - //} - - // NB: I'm not using these as I had anticipated. - let maybe_image_url = None; - - //let maybe_image_url = item.extensions.get("media") - // .map(|media| media.get("group")) - // .flatten() - // .map(|group| group.get(0)) - // .flatten() - // .map(|extension| extension.children.get("content")) - // .flatten() - // .map(|extensions| extensions.get(0)) // NB: First image is biggest - // .flatten() - // .map(|extension| extension.attrs.get("url")) - // .flatten() - // .map(|url| url.to_string()); - - targets.push(WebScrapingTarget { - canonical_url, - web_content_type: WebContentType::GizmodoArticle, - maybe_title: item.title.clone(), - maybe_full_image_url: maybe_image_url, - maybe_thumbnail_image_url: None, - maybe_skip_reason, - }); - } - - Ok(targets) -} diff --git a/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs b/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs deleted file mode 100644 index be546ecdc0..0000000000 --- a/crates/lib/web_scrapers/src/sites/gizmodo/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod gizmodo_article_scraper; -pub mod gizmodo_indexer; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/hackernews/mod.rs b/crates/lib/web_scrapers/src/sites/hackernews/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/web_scrapers/src/sites/kotaku/kotaku_article_scraper.rs b/crates/lib/web_scrapers/src/sites/kotaku/kotaku_article_scraper.rs deleted file mode 100644 index 51ec576d3f..0000000000 --- a/crates/lib/web_scrapers/src/sites/kotaku/kotaku_article_scraper.rs +++ /dev/null @@ -1,115 +0,0 @@ -use std::ops::Deref; - -use chrono::{DateTime, LocalResult, TimeZone, Utc}; -use once_cell::sync::Lazy; -use scraper::{Html, Selector}; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::{anyhow, AnyhowResult}; - -use crate::common_extractors::extract_attribute::extract_attribute; -use crate::common_extractors::extract_featured_image::extract_featured_image; -use crate::common_extractors::extract_title::extract_title; -use crate::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult}; - -/// The main article content container -static ARTICLE_CONTENT_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".js_post-content").expect("this selector should parse") -}); - -/// Paragraphs within the article -static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse("p").expect("this selector should parse") -}); - -/// The title of the article -pub static TITLE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("header h1").expect("this selector should parse") -}); - -/// The title of the article -pub static SUBTITLE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("h2.js_regular-subhead").expect("this selector should parse") -}); - -/// The article featured image -pub static FEATURED_IMAGE_SELECTOR: Lazy = Lazy::new(|| { - Selector::parse("figure picture img").expect("this selector should parse") -}); - -pub static DATETIME_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".js_starterpost time").expect("this selector should parse") -}); - -pub async fn kotaku_article_scraper(url: &str) -> AnyhowResult { - let downloaded_document= reqwest::get(url) - .await? - .bytes() - .await?; - - let downloaded_document = String::from_utf8_lossy(downloaded_document.deref()).to_string(); - let document = Html::parse_document(&downloaded_document); - - let mut paragraphs = Vec::new(); - - if let Some(article_content_div) = document.select(&ARTICLE_CONTENT_SELECTOR).next() { - for paragraph in article_content_div.select(&PARAGRAPH_SELECTOR).into_iter() { - - let mut paragraph_assembly = Vec::new(); - - for text in paragraph.text() { - let stripped = text.trim(); - if !stripped.is_empty() { - paragraph_assembly.push(stripped.to_string()); - } - } - - let paragraph_full_text = paragraph_assembly.join(" ") - .trim() - .to_string(); - - if !paragraph_full_text.is_empty() { - paragraphs.push(paragraph_full_text); - } - } - } - - let maybe_title = extract_title(&document, &TITLE_SELECTOR); - let maybe_subtitle = extract_title(&document, &SUBTITLE_SELECTOR); - let maybe_heading_image_url = extract_featured_image(&document, &FEATURED_IMAGE_SELECTOR); - - let maybe_timestamp_raw = extract_attribute(&document, &DATETIME_SELECTOR, "datetime"); - - // Timestamp is present as RFC3339: 2023-02-23T19:30:30-05:00 - let maybe_timestamp = maybe_timestamp_raw - .as_deref() - .map(|ts| DateTime::parse_from_rfc3339(ts.trim())) - .transpose()? - // We need to juggle from DateTime to DateTime. - .map(|dt| dt.naive_utc()) - .map(|naive| match Utc.from_local_datetime(&naive) { - LocalResult::None => Err(anyhow!("invalid datetime conversion")), - LocalResult::Ambiguous(_, _) => Err(anyhow!("ambiguous datetime conversion")), - LocalResult::Single(time) => Ok(time), - }) - .transpose()?; - - let body_text = paragraphs.join("\n\n"); - - Ok(WebScrapingResult { - original_html: downloaded_document, - result: ScrapedWebArticle { - url: url.to_string(), - web_content_type: WebContentType::KotakuArticle, - maybe_title, - maybe_subtitle, - maybe_author: None, // TODO - paragraphs, - body_text, - maybe_heading_image_url, - maybe_featured_image_url: None, // TODO - maybe_publish_timestamp_raw: maybe_timestamp_raw, - maybe_publish_datetime_utc: maybe_timestamp, - } - }) -} diff --git a/crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs b/crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs deleted file mode 100644 index f7f1f6bc31..0000000000 --- a/crates/lib/web_scrapers/src/sites/kotaku/kotaku_indexer.rs +++ /dev/null @@ -1,66 +0,0 @@ -use log::warn; -use rss::Channel; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::AnyhowResult; - -use crate::payloads::web_scraping_target::WebScrapingTarget; - -const KOTAKU_RSS: &str = "https://kotaku.com/rss"; - -// TODO: Rename foo_feed() or foo_rss(), etc. -pub async fn kotaku_indexer() -> AnyhowResult> { - let content = reqwest::get(KOTAKU_RSS) - .await? - .bytes() - .await?; - - let channel = Channel::read_from(&content[..])?; - - let mut targets = Vec::with_capacity(channel.items.len()); - - for item in channel.items { - let canonical_url = match item.link { - Some(url) => url.clone(), - None => { - warn!("Skipping item due to not having a URL"); - continue; - } - }; - - let maybe_skip_reason = None; - - //// Lazy heuristic to skip CNN video articles, which contain little text. - //// eg. https://www.cnn.com/videos/politics/2023/02/08/pelosi-gop-hecklers-reaction-joe-biden-state-of-the-union-tapper-intv-vpx.cnn - //if canonical_url.contains("/videos/") { - // maybe_skip_reason = Some(SkipReason::VideoContent); - //} - - // NB: I'm not using these as I had anticipated. - let maybe_image_url = None; - - //let maybe_image_url = item.extensions.get("media") - // .map(|media| media.get("group")) - // .flatten() - // .map(|group| group.get(0)) - // .flatten() - // .map(|extension| extension.children.get("content")) - // .flatten() - // .map(|extensions| extensions.get(0)) // NB: First image is biggest - // .flatten() - // .map(|extension| extension.attrs.get("url")) - // .flatten() - // .map(|url| url.to_string()); - - targets.push(WebScrapingTarget { - canonical_url, - web_content_type: WebContentType::KotakuArticle, - maybe_title: item.title.clone(), - maybe_full_image_url: maybe_image_url, - maybe_thumbnail_image_url: None, - maybe_skip_reason, - }); - } - - Ok(targets) -} diff --git a/crates/lib/web_scrapers/src/sites/kotaku/mod.rs b/crates/lib/web_scrapers/src/sites/kotaku/mod.rs deleted file mode 100644 index 89dad7da74..0000000000 --- a/crates/lib/web_scrapers/src/sites/kotaku/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod kotaku_article_scraper; -pub mod kotaku_indexer; diff --git a/crates/lib/web_scrapers/src/sites/mod.rs b/crates/lib/web_scrapers/src/sites/mod.rs deleted file mode 100644 index 738d5500e7..0000000000 --- a/crates/lib/web_scrapers/src/sites/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -pub mod cbsnews; -pub mod cnn; -pub mod gizmodo; -pub mod hackernews; -pub mod kotaku; -pub mod reddit; -pub mod reuters; -pub mod slashdot; -pub mod techcrunch; -pub mod theguardian; -pub mod twitter; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/reddit/mod.rs b/crates/lib/web_scrapers/src/sites/reddit/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/web_scrapers/src/sites/reuters/mod.rs b/crates/lib/web_scrapers/src/sites/reuters/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/web_scrapers/src/sites/slashdot/mod.rs b/crates/lib/web_scrapers/src/sites/slashdot/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/web_scrapers/src/sites/substack/mod.rs b/crates/lib/web_scrapers/src/sites/substack/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs b/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs deleted file mode 100644 index dd95ed530f..0000000000 --- a/crates/lib/web_scrapers/src/sites/techcrunch/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod techcrunch_indexer; -pub mod techcrunch_article_scraper; \ No newline at end of file diff --git a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs b/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs deleted file mode 100644 index 3d7660f91c..0000000000 --- a/crates/lib/web_scrapers/src/sites/techcrunch/techcrunch_article_scraper.rs +++ /dev/null @@ -1,111 +0,0 @@ -use std::ops::Deref; - -use once_cell::sync::Lazy; -use scraper::{Html, Selector}; - -use enums::common::sqlite::web_content_type::WebContentType; -use errors::AnyhowResult; - -use crate::common_extractors::extract_featured_image::extract_featured_image; -use crate::common_extractors::extract_title::extract_title; -use crate::payloads::web_scraping_result::{ScrapedWebArticle, WebScrapingResult}; - -/// The main article content container -static ARTICLE_CONTENT_SELECTOR : Lazy = Lazy::new(|| { - Selector::parse(".article-content").expect("this selector should parse") -}); - -/// Paragraphs within the article -static PARAGRAPH_SELECTOR : Lazy = Lazy::new(|| { - // NB: Techcrunch content issue: - // The "div >" removes mysterious inclusion of Twitter