From ff9d2c6b344461c12ab9c67734c5d90d73ebef96 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 18 Feb 2025 01:29:56 -0500 Subject: [PATCH] realtime + speedup (#414) Co-authored-by: Kasisnu <5044120+kasisnu@users.noreply.github.com> --- .idea/dictionaries/bt.xml | 2 + Cargo.lock | 860 ++++++++++++++++-- app_config.yaml | 7 + crates/desktop/src-tauri/Cargo.toml | 33 +- .../src-tauri/capabilities/default.json | 5 +- .../flip_image.rs} | 16 +- .../src-tauri/src/endpoints/image_endpoint.rs | 92 ++ crates/desktop/src-tauri/src/endpoints/mod.rs | 4 + crates/desktop/src-tauri/src/endpoints/sd.rs | 172 ++++ .../src-tauri/src/endpoints/test_counter.rs | 24 + crates/desktop/src-tauri/src/lib.rs | 87 +- .../src/ml/create_inpainting_tensors.rs | 80 ++ .../src/ml/image/dynamic_image_to_tensor.rs | 17 + .../src/ml/image/load_image_file_to_tensor.rs | 28 + .../ml/image/load_image_file_to_tensor_2.rs | 18 + crates/desktop/src-tauri/src/ml/image/mod.rs | 5 + .../src/ml/image/save_image_from_tensor.rs | 28 + .../src/ml/image/tensor_to_image_buffer.rs | 18 + .../src/ml/infer_clip_text_embeddings.rs | 153 ++++ crates/desktop/src-tauri/src/ml/mod.rs | 8 + .../desktop/src-tauri/src/ml/model_cache.rs | 118 +++ crates/desktop/src-tauri/src/ml/model_file.rs | 83 ++ .../src/ml/models/lazy_load_vae_model.rs | 130 +++ crates/desktop/src-tauri/src/ml/models/mod.rs | 2 + .../src-tauri/src/ml/models/unet_model.rs | 35 + .../desktop/src-tauri/src/ml/prompt_cache.rs | 22 + .../src/ml/stable_diffusion/get_vae_scale.rs | 13 + .../src-tauri/src/ml/stable_diffusion/mod.rs | 1 + .../desktop/src-tauri/src/state/app_config.rs | 93 ++ crates/desktop/src-tauri/src/state/mod.rs | 2 + .../src-tauri/src/state/yaml_config.rs | 24 + crates/lib/files/filesys/src/lib.rs | 1 + .../src/recursively_find_file_by_name.rs | 56 ++ script/build_production_realtime_desktop.sh | 12 + script/launch_realtime_desktop.sh | 2 +- 35 files changed, 2121 insertions(+), 130 deletions(-) create mode 100644 app_config.yaml rename crates/desktop/src-tauri/src/{image_endpoint.rs => endpoints/flip_image.rs} (79%) create mode 100644 crates/desktop/src-tauri/src/endpoints/image_endpoint.rs create mode 100644 crates/desktop/src-tauri/src/endpoints/mod.rs create mode 100644 crates/desktop/src-tauri/src/endpoints/sd.rs create mode 100644 crates/desktop/src-tauri/src/endpoints/test_counter.rs create mode 100644 crates/desktop/src-tauri/src/ml/create_inpainting_tensors.rs create mode 100644 crates/desktop/src-tauri/src/ml/image/dynamic_image_to_tensor.rs create mode 100644 crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor.rs create mode 100644 crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor_2.rs create mode 100644 crates/desktop/src-tauri/src/ml/image/mod.rs create mode 100644 crates/desktop/src-tauri/src/ml/image/save_image_from_tensor.rs create mode 100644 crates/desktop/src-tauri/src/ml/image/tensor_to_image_buffer.rs create mode 100644 crates/desktop/src-tauri/src/ml/infer_clip_text_embeddings.rs create mode 100644 crates/desktop/src-tauri/src/ml/mod.rs create mode 100644 crates/desktop/src-tauri/src/ml/model_cache.rs create mode 100644 crates/desktop/src-tauri/src/ml/model_file.rs create mode 100644 crates/desktop/src-tauri/src/ml/models/lazy_load_vae_model.rs create mode 100644 crates/desktop/src-tauri/src/ml/models/mod.rs create mode 100644 crates/desktop/src-tauri/src/ml/models/unet_model.rs create mode 100644 crates/desktop/src-tauri/src/ml/prompt_cache.rs create mode 100644 crates/desktop/src-tauri/src/ml/stable_diffusion/get_vae_scale.rs create mode 100644 crates/desktop/src-tauri/src/ml/stable_diffusion/mod.rs create mode 100644 crates/desktop/src-tauri/src/state/app_config.rs create mode 100644 crates/desktop/src-tauri/src/state/mod.rs create mode 100644 crates/desktop/src-tauri/src/state/yaml_config.rs create mode 100644 crates/lib/files/filesys/src/recursively_find_file_by_name.rs create mode 100755 script/build_production_realtime_desktop.sh diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index d253914687..489e55b875 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -31,6 +31,7 @@ hifigan hparams huggingface + kasisnu lipsync mocap mochi @@ -47,6 +48,7 @@ techcrunch transcoded transcodings + unet unpermitted upserting upserts diff --git a/Cargo.lock b/Cargo.lock index 7932ece0cf..c80bd569e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,7 +12,7 @@ dependencies = [ "actix-rt", "actix_derive", "bitflags 2.5.0", - "bytes 1.6.0", + "bytes 1.10.0", "crossbeam-channel", "futures-core", "futures-sink", @@ -34,7 +34,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ "bitflags 2.5.0", - "bytes 1.6.0", + "bytes 1.10.0", "futures-core", "futures-sink", "memchr", @@ -73,7 +73,7 @@ dependencies = [ "base64 0.22.1", "bitflags 2.5.0", "brotli 6.0.0", - "bytes 1.6.0", + "bytes 1.10.0", "bytestring", "derive_more", "encoding_rs", @@ -117,7 +117,7 @@ dependencies = [ "actix-multipart-derive", "actix-utils", "actix-web", - "bytes 1.6.0", + "bytes 1.10.0", "derive_more", "futures-core", "futures-util", @@ -227,7 +227,7 @@ dependencies = [ "actix-utils", "actix-web-codegen", "ahash 0.8.11", - "bytes 1.6.0", + "bytes 1.10.0", "bytestring", "cfg-if 1.0.0", "cookie 0.16.2", @@ -262,7 +262,7 @@ dependencies = [ "actix-codec", "actix-http", "actix-web", - "bytes 1.6.0", + "bytes 1.10.0", "bytestring", "futures-core", "pin-project-lite", @@ -297,7 +297,7 @@ dependencies = [ "ahash 0.8.11", "arc-swap", "async-trait", - "bytes 1.6.0", + "bytes 1.10.0", "bytestring", "csv", "derive_more", @@ -390,6 +390,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aes" version = "0.7.5" @@ -597,15 +603,26 @@ checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" name = "app" version = "0.1.0" dependencies = [ + "anyhow", "base64 0.22.1", + "bytes 1.10.0", + "candle-core", + "candle-nn", + "candle-transformers", + "filesys", + "hf-hub", "image", "log", + "memory_caching", "once_cell", + "rand 0.8.5", "serde", "serde_json", + "serde_yaml", "tauri", "tauri-build", "tauri-plugin-log", + "tokenizers", ] [[package]] @@ -884,7 +901,7 @@ dependencies = [ "async-trait", "axum-core", "bitflags 1.3.2", - "bytes 1.6.0", + "bytes 1.10.0", "futures-util", "http 0.2.11", "http-body 0.4.6", @@ -910,7 +927,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.6.0", + "bytes 1.10.0", "futures-util", "http 0.2.11", "http-body 0.4.6", @@ -930,7 +947,7 @@ dependencies = [ "cc", "cfg-if 1.0.0", "libc", - "miniz_oxide", + "miniz_oxide 0.7.2", "object", "rustc-demangle", ] @@ -1027,12 +1044,38 @@ dependencies = [ "utoipa", ] +[[package]] +name = "bindgen_cuda" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8489af5b7d17a81bffe37e0f4d6e1e4de87c87329d05447f22c35d95a1227d" +dependencies = [ + "glob", + "num_cpus", + "rayon", +] + [[package]] name = "binstring" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e0d60973d9320722cb1206f412740e162a33b8547ea8d6be75d7cff237c7a85" +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bit_field" version = "0.10.2" @@ -1269,6 +1312,20 @@ name = "bytemuck" version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] [[package]] name = "byteorder" @@ -1294,9 +1351,9 @@ dependencies = [ [[package]] name = "bytes" -version = "1.6.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" dependencies = [ "serde", ] @@ -1307,7 +1364,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", ] [[package]] @@ -1365,6 +1422,74 @@ dependencies = [ "serde", ] +[[package]] +name = "candle-core" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "855dfedff437d2681d68e1f34ae559d88b0dd84aa5a6b63f2c8e75ebdd875bbf" +dependencies = [ + "byteorder", + "candle-kernels", + "cudarc", + "gemm", + "half", + "memmap2", + "num-traits", + "num_cpus", + "rand 0.8.5", + "rand_distr", + "rayon", + "safetensors", + "thiserror 1.0.63", + "ug", + "ug-cuda", + "yoke", + "zip 1.1.4", +] + +[[package]] +name = "candle-kernels" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53343628fa470b7075c28c589b98735b4220b464e37ddbb8e117040e199f4787" +dependencies = [ + "bindgen_cuda", +] + +[[package]] +name = "candle-nn" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd3c6b2ee0dfd64af12ae5b07e4b7c517898981cdaeffcb10b71d7dd5c8f359" +dependencies = [ + "candle-core", + "half", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror 1.0.63", +] + +[[package]] +name = "candle-transformers" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4270cc692c4a3df2051c2e8c3c4da3a189746af7ca3a547b99ecd335582b92e1" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex", + "num-traits", + "rand 0.8.5", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + [[package]] name = "cargo-platform" version = "0.1.9" @@ -1670,7 +1795,7 @@ version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "memchr", ] @@ -1737,6 +1862,19 @@ dependencies = [ "url", ] +[[package]] +name = "console" +version = "0.15.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width 0.2.0", + "windows-sys 0.59.0", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2088,6 +2226,16 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "cudarc" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cd60a9a42ec83a2ed7effb0b1f073270264ea99da7acfc44f7e8d74dee0384" +dependencies = [ + "half", + "libloading 0.8.6", +] + [[package]] name = "darling" version = "0.13.4" @@ -2733,6 +2881,16 @@ version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + [[package]] name = "easyenv" version = "0.1.0" @@ -2790,7 +2948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40d9bd57d914cc66ce878f098f63ed7b5d5b64c30644a5adb950b008f874a6c6" dependencies = [ "base64 0.11.0", - "bytes 1.6.0", + "bytes 1.10.0", "dyn-clone", "lazy_static", "percent-encoding", @@ -2948,6 +3106,12 @@ version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ef6b89e5b37196644d8796de5268852ff179b44e96276cf4290264843743bb7" +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "encoding_rs" version = "0.8.34" @@ -2957,6 +3121,18 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "enums" version = "0.0.1" @@ -2999,9 +3175,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" dependencies = [ "anstream", "anstyle", @@ -3077,6 +3253,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -3096,20 +3281,30 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "exr" -version = "1.71.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "832a761f35ab3e6664babfbdc6cef35a4860e816ec3916dcfd0882954e98a8a8" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" dependencies = [ "bit_field", - "flume", "half", "lebe", - "miniz_oxide", + "miniz_oxide 0.8.4", "rayon-core", "smallvec 1.13.2", "zune-inflate", ] +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "1.9.0" @@ -3197,7 +3392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.7.2", ] [[package]] @@ -3542,6 +3737,124 @@ dependencies = [ "x11", ] +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3559,7 +3872,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" dependencies = [ - "unicode-width", + "unicode-width 0.1.11", ] [[package]] @@ -3796,7 +4109,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "fnv", "futures-core", "futures-sink", @@ -3816,7 +4129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ "atomic-waker", - "bytes 1.6.0", + "bytes 1.10.0", "fnv", "futures-core", "futures-sink", @@ -3830,11 +4143,16 @@ dependencies = [ [[package]] name = "half" -version = "2.2.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ + "bytemuck", + "cfg-if 1.0.0", "crunchy", + "num-traits", + "rand 0.8.5", + "rand_distr", ] [[package]] @@ -3904,6 +4222,23 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-hub" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b780635574b3d92f036890d8373433d6f9fc7abb320ee42a5c25897fc8ed732" +dependencies = [ + "dirs 5.0.1", + "indicatif", + "log", + "native-tls", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror 1.0.63", + "ureq", +] + [[package]] name = "hkdf" version = "0.12.3" @@ -4007,7 +4342,7 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "fnv", "itoa 1.0.11", ] @@ -4018,7 +4353,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "fnv", "itoa 1.0.11", ] @@ -4029,7 +4364,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "http 0.2.11", "pin-project-lite", ] @@ -4040,7 +4375,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "http 1.1.0", ] @@ -4050,7 +4385,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "futures-util", "http 1.1.0", "http-body 1.0.0", @@ -4122,7 +4457,7 @@ version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "futures-channel", "futures-core", "futures-util", @@ -4146,7 +4481,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "futures-channel", "futures-util", "h2 0.4.5", @@ -4196,7 +4531,7 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "hyper 0.14.28", "native-tls", "tokio", @@ -4209,7 +4544,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "http-body-util", "hyper 1.3.1", "hyper-util", @@ -4225,7 +4560,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "futures-channel", "futures-util", "http 1.1.0", @@ -4373,6 +4708,19 @@ dependencies = [ "serde", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width 0.2.0", + "web-time", +] + [[package]] name = "infer" version = "0.2.3" @@ -4847,7 +5195,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e9ec52138abedcc58dc17a7c6c0c00a2bdb4f3427c7f63fa97fd0d859155caf" dependencies = [ "gtk-sys", - "libloading", + "libloading 0.7.4", "once_cell", ] @@ -4877,6 +5225,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if 1.0.0", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.8" @@ -5039,6 +5397,22 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + [[package]] name = "magic-crypt" version = "3.1.13" @@ -5184,6 +5558,16 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", + "stable_deref_trait", +] + [[package]] name = "memoffset" version = "0.7.1" @@ -5312,6 +5696,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.6.23" @@ -5405,6 +5798,27 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "monostate" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "mp4" version = "0.14.0" @@ -5412,7 +5826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9ef834d5ed55e494a2ae350220314dc4aacd1c43a9498b00e320e0ea352a5c3" dependencies = [ "byteorder", - "bytes 1.6.0", + "bytes 1.10.0", "num-rational", "serde", "serde_json", @@ -5489,7 +5903,7 @@ dependencies = [ "log", "md-5 0.10.6", "memchr", - "miniz_oxide", + "miniz_oxide 0.7.2", "native-tls", "num-bigint", "num-integer", @@ -5691,9 +6105,9 @@ checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" [[package]] name = "num" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -5705,11 +6119,10 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ - "autocfg", "num-integer", "num-traits", ] @@ -5733,10 +6146,11 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ + "bytemuck", "num-traits", ] @@ -5759,19 +6173,18 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -5780,11 +6193,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -5841,6 +6253,12 @@ dependencies = [ "libc", ] +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "objc" version = "0.2.7" @@ -6083,6 +6501,28 @@ version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "opaque-debug" version = "0.3.0" @@ -6623,9 +7063,15 @@ dependencies = [ "crc32fast", "fdeflate", "flate2", - "miniz_oxide", + "miniz_oxide 0.7.2", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + [[package]] name = "powerfmt" version = "0.2.0" @@ -6789,7 +7235,7 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "prost-derive", ] @@ -6865,6 +7311,18 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + [[package]] name = "qoi" version = "0.4.1" @@ -6916,7 +7374,7 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "pin-project-lite", "quinn-proto", "quinn-udp", @@ -6934,7 +7392,7 @@ version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "rand 0.8.5", "ring", "rustc-hash", @@ -7091,6 +7549,16 @@ dependencies = [ "getrandom 0.2.14", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + [[package]] name = "rand_hc" version = "0.2.0" @@ -7159,6 +7627,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "raw-window-handle" version = "0.6.2" @@ -7175,6 +7652,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools 0.11.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.12.1" @@ -7194,6 +7682,12 @@ dependencies = [ "rand_core 0.3.1", ] +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redis" version = "0.13.0" @@ -7379,7 +7873,7 @@ checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "async-compression", "base64 0.21.7", - "bytes 1.6.0", + "bytes 1.10.0", "encoding_rs", "futures-core", "futures-util", @@ -7420,7 +7914,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", - "bytes 1.6.0", + "bytes 1.10.0", "encoding_rs", "futures-channel", "futures-core", @@ -7515,7 +8009,7 @@ checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" dependencies = [ "bitvec", "bytecheck", - "bytes 1.6.0", + "bytes 1.10.0", "hashbrown 0.12.3", "ptr_meta", "rend", @@ -7624,7 +8118,7 @@ dependencies = [ "aws-creds", "aws-region", "base64 0.22.1", - "bytes 1.6.0", + "bytes 1.10.0", "cfg-if 1.0.0", "futures 0.3.29", "hex", @@ -7656,7 +8150,7 @@ checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" dependencies = [ "arrayvec", "borsh", - "bytes 1.6.0", + "bytes 1.10.0", "num-traits", "rand 0.8.5", "rkyv", @@ -7724,6 +8218,7 @@ version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", @@ -7790,7 +8285,7 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65bc94b580d0f5a6b7a2d604e597513d3c673154b52ddeccd1d5c32360d945ee" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "rxml_validation", ] @@ -7809,6 +8304,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "safetensors" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -8013,6 +8518,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + [[package]] name = "serde" version = "1.0.210" @@ -8206,6 +8717,19 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.2.6", + "itoa 1.0.11", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "serial_test" version = "3.0.0" @@ -8517,6 +9041,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlformat" version = "0.2.2" @@ -8551,7 +9087,7 @@ dependencies = [ "atoi", "bigdecimal", "byteorder", - "bytes 1.6.0", + "bytes 1.10.0", "chrono", "crc", "crossbeam-queue", @@ -8638,7 +9174,7 @@ dependencies = [ "bigdecimal", "bitflags 2.5.0", "byteorder", - "bytes 1.6.0", + "bytes 1.10.0", "chrono", "crc", "digest 0.10.7", @@ -9213,6 +9749,31 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.5.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.63", + "walkdir", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -9336,7 +9897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "58a998b6be84104ca05c7e9a21f2180ddec020c8b84ea59a8fc8530a2a19588d" dependencies = [ "anyhow", - "bytes 1.6.0", + "bytes 1.10.0", "dirs 6.0.0", "dunce", "embed_plist", @@ -9756,6 +10317,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d" +dependencies = [ + "aho-corasick", + "clap", + "derive_builder", + "esaxx-rs", + "getrandom 0.2.14", + "indicatif", + "itertools 0.12.1", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.8.5", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror 1.0.63", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokens" version = "0.0.1" @@ -9781,7 +10375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", - "bytes 1.6.0", + "bytes 1.10.0", "libc", "mio 1.0.2", "parking_lot 0.12.2", @@ -9944,7 +10538,7 @@ version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "futures-core", "futures-sink", "pin-project-lite", @@ -10042,7 +10636,7 @@ dependencies = [ "async-trait", "axum", "base64 0.21.7", - "bytes 1.6.0", + "bytes 1.10.0", "h2 0.3.26", "http 0.2.11", "http-body 0.4.6", @@ -10250,6 +10844,33 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "ug" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4eef2ebfc18c67a6dbcacd9d8a4d85e0568cc58c82515552382312c2730ea13" +dependencies = [ + "half", + "num", + "serde", + "serde_json", + "thiserror 1.0.63", +] + +[[package]] +name = "ug-cuda" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c4dcab280ad0ef3957e153a82dcad608c954d02cf253b695322f502d1f8902e" +dependencies = [ + "cudarc", + "half", + "serde", + "serde_json", + "thiserror 1.0.63", + "ug", +] + [[package]] name = "unic-char-property" version = "0.9.0" @@ -10321,6 +10942,15 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec 1.13.2", +] + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -10333,6 +10963,12 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -10348,12 +10984,37 @@ dependencies = [ "void", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "native-tls", + "once_cell", + "rustls 0.23.13", + "rustls-pki-types", + "serde", + "serde_json", + "url", + "webpki-roots 0.26.2", +] + [[package]] name = "url" version = "2.5.2" @@ -10775,11 +11436,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web_scrapers" version = "0.0.1" dependencies = [ - "bytes 1.6.0", + "bytes 1.10.0", "chrono", "enums", "errors", @@ -11474,6 +12145,30 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.33" @@ -11494,6 +12189,27 @@ dependencies = [ "syn 2.0.87", ] +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" diff --git a/app_config.yaml b/app_config.yaml new file mode 100644 index 0000000000..924b3516dc --- /dev/null +++ b/app_config.yaml @@ -0,0 +1,7 @@ +image_width: 512 +image_height: 512 +#scheduler_steps: 15 +# NB(bt): It looks like we can get down to 2 sampling steps, perhaps ? +scheduler_steps: 2 +scheduler_samples: 1 + diff --git a/crates/desktop/src-tauri/Cargo.toml b/crates/desktop/src-tauri/Cargo.toml index 4e34dd0ef8..06585d0e3e 100644 --- a/crates/desktop/src-tauri/Cargo.toml +++ b/crates/desktop/src-tauri/Cargo.toml @@ -18,11 +18,30 @@ crate-type = ["staticlib", "cdylib", "rlib"] tauri-build = { version = "2.0.4", features = [] } [dependencies] -serde_json = "1.0" -serde = { version = "1.0", features = ["derive"] } -log = "0.4" -tauri = { version = "2.2.4", features = [] } -tauri-plugin-log = "2.0.0-rc" -once_cell = "1.20.3" -image = "0.25.5" + +# Internal +filesys = { path = "../../lib/files/filesys" } +memory_caching = { path = "../../lib/caching/memory_caching" } + +# External +anyhow = "1.0" +bytes = "1.10" base64 = "0.22.1" +candle-core = { version = "0.8.2", features = ["cuda"] } +candle-nn = { version = "0.8.2", features = ["cuda"] } +candle-transformers = { version = "0.8.2", features = ["cuda"] } +hf-hub = "0.3.2" +image = "0.25.5" +serde_yaml = "0.9" +log = "0.4" +once_cell = "1.20.3" +rand = "0.8" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +tauri = { version = "2.2.4", features = [] } +tauri-plugin-log = "2" +tokenizers = "0.15.2" + +[features] +default = ["cuda"] +cuda = ["candle-core/cuda", "candle-nn/cuda", "candle-transformers/cuda"] diff --git a/crates/desktop/src-tauri/capabilities/default.json b/crates/desktop/src-tauri/capabilities/default.json index c135d7f15d..f3543164cb 100644 --- a/crates/desktop/src-tauri/capabilities/default.json +++ b/crates/desktop/src-tauri/capabilities/default.json @@ -6,6 +6,7 @@ "main" ], "permissions": [ - "core:default" + "core:default", + "log:default" ] -} +} \ No newline at end of file diff --git a/crates/desktop/src-tauri/src/image_endpoint.rs b/crates/desktop/src-tauri/src/endpoints/flip_image.rs similarity index 79% rename from crates/desktop/src-tauri/src/image_endpoint.rs rename to crates/desktop/src-tauri/src/endpoints/flip_image.rs index a40e1a590f..cc6b2a7fea 100644 --- a/crates/desktop/src-tauri/src/image_endpoint.rs +++ b/crates/desktop/src-tauri/src/endpoints/flip_image.rs @@ -1,11 +1,11 @@ -use std::io::Cursor; +use base64::prelude::BASE64_STANDARD; use base64::Engine; -use base64::prelude::{BASE64_STANDARD, BASE64_URL_SAFE}; use image::codecs::png::{CompressionType, FilterType, PngEncoder}; use image::ImageReader; +use std::io::Cursor; #[tauri::command] -pub fn infer_image(image: &str) -> Result { +pub fn flip_image(image: &str) -> Result { println!("infer_image called; processing image..."); let bytes = BASE64_STANDARD.decode(image) @@ -18,17 +18,19 @@ pub fn infer_image(image: &str) -> Result { .map_err(|err| format!("Image decode error: {}", err))?; let rotated_image = image.rotate180(); - + let mut buffer = Vec::new(); // TODO: Preallocate, better data structure let mut writer = Cursor::new(&mut buffer); + + let encoder = PngEncoder::new_with_quality + (&mut writer, CompressionType::Fast, FilterType::Adaptive); - let encoder = PngEncoder::new_with_quality(&mut writer, CompressionType::Fast, FilterType::Adaptive); rotated_image.write_with_encoder(encoder) .map_err(|err| format!("Image encoding error: {}", err))?; let encoded = BASE64_STANDARD.encode(&buffer); - + println!("Encoded: {:?}", encoded.split_at(10).0); - + Ok(encoded) } diff --git a/crates/desktop/src-tauri/src/endpoints/image_endpoint.rs b/crates/desktop/src-tauri/src/endpoints/image_endpoint.rs new file mode 100644 index 0000000000..50a5bdff0b --- /dev/null +++ b/crates/desktop/src-tauri/src/endpoints/image_endpoint.rs @@ -0,0 +1,92 @@ +use crate::endpoints::sd::{run, Args}; +use crate::ml::model_cache::ModelCache; +use crate::ml::prompt_cache::PromptCache; +use crate::state::app_config::AppConfig; +use crate::state::yaml_config::YamlConfig; +use anyhow::Result; +use base64::{engine::general_purpose::STANDARD as BASE64_STANDARD, Engine}; +use bytes::{BufMut, BytesMut}; +use image::imageops::FilterType; +use image::{DynamicImage, EncodableLayout, ImageFormat, ImageReader}; +use log::{error, info}; +use std::io::Cursor; +use std::path::PathBuf; +use tauri::State; + +const PROMPT: &str = "A beautiful landscape with mountains and a lake"; + +#[tauri::command] +pub fn infer_image( + image: &str, + model_config: State, + model_cache: State, + prompt_cache: State, +) -> Result { + + let bytes = BASE64_STANDARD.decode(image) + .map_err(|err| format!("Base64 decode error: {}", err))?; + + let prompt_file = PathBuf::from("prompt.txt").canonicalize() + .unwrap_or_else(|_| PathBuf::from("prompt.txt")); + + let prompt = std::fs::read_to_string(&prompt_file) + .map_err(|err| format!("Failed to read prompt file: {}", err)) + .unwrap_or_else(|_| { + error!("Failed to read prompt file: {:?}", prompt_file); + PROMPT.to_string() + }) + .trim() + .to_string(); + + let image = ImageReader::new(Cursor::new(bytes)) + .with_guessed_format() + .map_err(|err| format!("Image format error: {}", err))? + .decode() + .map_err(|err| format!("Image decode error: {}", err))?; + + // TODO(bt,2025-02-17): Running out of vram with full image buffer size + let image = image.resize(512, 512, FilterType::CatmullRom); + + let result = do_infer_image(&prompt, image, &model_config, &model_cache, prompt_cache); + + if let Err(err) = result.as_deref() { + error!("There was an error: {:?}", err); + } + + result +} + +fn do_infer_image( + prompt: &str, + image: DynamicImage, + config: &AppConfig, + model_cache: &ModelCache, + prompt_cache: State, +) -> Result { + println!("infer_image called; generating image with SDXL Turbo..."); + + let args = Args { + image: &image, + prompt: prompt.to_string(), + uncond_prompt: "".to_string(), + //guidance_scale: Some(0.0), + guidance_scale: None, + model_cache, + configs: config, + prompt_cache: &prompt_cache, + }; + + match run(args) { + Ok(image) => { + let mut bytes = Vec::with_capacity(1024*1024); + + image.write_to(&mut Cursor::new(&mut bytes), ImageFormat::Png) + .map_err(|err| format!("failure to encode image: {:?}", err))?; + + let bytes = BASE64_STANDARD.encode(bytes); + + Ok(bytes) + }, + Err(e) => Err(format!("Failed to generate image: {}", e)), + } +} diff --git a/crates/desktop/src-tauri/src/endpoints/mod.rs b/crates/desktop/src-tauri/src/endpoints/mod.rs new file mode 100644 index 0000000000..3727a5824f --- /dev/null +++ b/crates/desktop/src-tauri/src/endpoints/mod.rs @@ -0,0 +1,4 @@ +pub mod flip_image; +pub mod image_endpoint; +pub mod sd; +pub mod test_counter; diff --git a/crates/desktop/src-tauri/src/endpoints/sd.rs b/crates/desktop/src-tauri/src/endpoints/sd.rs new file mode 100644 index 0000000000..c7d5c1f7dd --- /dev/null +++ b/crates/desktop/src-tauri/src/endpoints/sd.rs @@ -0,0 +1,172 @@ +use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}; + +use crate::ml::image::dynamic_image_to_tensor::dynamic_image_to_tensor; +use crate::ml::image::tensor_to_image_buffer::{tensor_to_image_buffer, RgbImage}; +use crate::ml::infer_clip_text_embeddings::infer_clip_text_embeddings; +use crate::ml::model_cache::ModelCache; +use crate::ml::prompt_cache::PromptCache; +use crate::ml::stable_diffusion::get_vae_scale::get_vae_scale; +use crate::state::app_config::AppConfig; +use anyhow::{Error as E, Result}; +use candle_core::{DType, IndexOp, D}; +use candle_transformers::models::stable_diffusion::vae::DiagonalGaussianDistribution; +use image::DynamicImage; +use log::info; +use rand::Rng; + +// TODO: Clean up + +// Note for Kasisnu: I'm going to start using lifetimes as long as that doesn't slow your velocity +// Basically the args o this telescopic args struct are guaranteed to live as long as the struct +// itself with the 'a lifetime. +pub struct Args<'a> { + pub image: &'a DynamicImage, + pub prompt: String, + pub uncond_prompt: String, + pub guidance_scale: Option, + pub configs: &'a AppConfig, + pub model_cache: &'a ModelCache, + pub prompt_cache: &'a PromptCache, +} + +pub fn run(args: Args<'_>) -> Result { + println!("Starting image generation with the following configuration:"); + println!(" Model: {:?}", args.configs.sd_version); + println!(" Prompt: {}", args.prompt); + println!(" Steps: {}", args.configs.scheduler_steps); + println!(" Device: {:?}", args.configs.device); + + + println!("Model dimensions: {}x{}", args.configs.sd_config.width, args.configs.sd_config.height); + + println!("Building scheduler..."); + let mut scheduler = args.configs.sd_config.build_scheduler( + args.configs.scheduler_steps)?; + + let seed = args.configs.seed.unwrap_or_else(|| rand::thread_rng().gen()); + + println!("Using seed: {}", seed); + args.configs.device.set_seed(seed)?; + + info!("Checking if prompt is cached"); + let maybe_cached = args.prompt_cache.get_copy(&args.prompt)?; + + let text_embeddings = if let Some(tensor) = maybe_cached { + info!("Prompt is cached!"); + tensor + } else { + info!("Prompt is NOT cached! Calculating embedding..."); + let tensor = infer_clip_text_embeddings( + &args.prompt, + &args.uncond_prompt, + None, // tokenizer + None, // clip_weights + None, // clip2_weights + args.configs.sd_version, + &args.configs.sd_config, + false, // use_f16 + &args.configs.device, + args.configs.dtype, + false, // use_guide_scale + )?; + args.prompt_cache.store_copy(&args.prompt, &tensor)?; + tensor + }; + + println!("Text embeddings shape: {:?}", text_embeddings.shape()); + + println!("Loading input image into tensor..."); + + let input_image = dynamic_image_to_tensor( + args.image, + &args.configs.device, + args.configs.dtype)?; + + println!("Reference image shape: {:?}", input_image.shape()); + + let vae_scale = get_vae_scale(args.configs.sd_version); + + let init_latent_dist : DiagonalGaussianDistribution = args.model_cache.vae_encode(&input_image)?; + + // TODO(bt,2025-02-18): This takes a little bit to generate. + println!("Generating latents from input image..."); + let latents = (init_latent_dist.sample()? * vae_scale)?; + + //.to_device(&args.configs.device)?; + + println!("Initial latents shape: {:?}", latents.shape()); + + println!("Calculating start step for diffusion process..."); + let img2img_strength = 0.75f64; + + let t_start = { + let start = args.configs.scheduler_steps - (args.configs.scheduler_steps as f64 * img2img_strength) as usize; + + println!("Starting from step {} of {} (strength: {})", start, args.configs.scheduler_steps, img2img_strength); + start + }; + + let latents = if t_start < scheduler.timesteps().len() { + println!("Adding noise to latents..."); + let noise = latents.randn_like(0f64, 1f64)?; + scheduler.add_noise(&latents, noise, scheduler.timesteps()[t_start])? + } else { + println!("Using latents directly (no noise addition needed)"); + latents + }; + + println!("Latents initialized successfully"); + + let timesteps: Vec<_> = scheduler.timesteps().iter().copied().collect(); + + // Initial noise shape: [1, 4, 64, 64] + println!("Initial noise shape: {:?}", latents.shape()); + + println!("Starting diffusion process..."); + + let mut latents = latents; + + for (timestep_index, ×tep) in timesteps.iter().enumerate() { + if timestep_index < t_start { + continue; + } + + println!("Processing step {}/{} @ timestamp = {}", timestep_index + 1, timesteps.len(), timestep); + + // TODO(bt,2025-08-18): Do we have to clone the tensor? + let latent_model_input = scheduler.scale_model_input(latents.clone(), timestep)?; + + let noise_pred = match args.model_cache.unet_inference(&latent_model_input, timestep as f64, &text_embeddings) { + Ok(pred) => { + pred + }, + Err(e) => { + println!("UNet inference failed with error: {}", e); + return Err(anyhow::anyhow!("UNet inference failed: {}", e)); + }, + }; + + latents = scheduler.step(&noise_pred, timestep, &latents)?; + } + + println!("Diffusion process completed, decoding image..."); + let image = args.model_cache.vae_decode(&(latents / vae_scale)?)?; + + println!("VAE decode completed"); + + println!("Scaling image back"); + let image = ((image / 2.)? + 0.5)?; + + // TODO(bt,2025-08-18): This normalization is slow. + println!("Normalized image values"); + let image = (image.clamp(0f32, 1.)? * 255.)?; + + println!("Convert to int8"); + let image = image.to_dtype(DType::U8)?; + + println!("Converted to 8-bit format"); + + let image = tensor_to_image_buffer(&image.i(0)?)?; + + Ok(image) +} diff --git a/crates/desktop/src-tauri/src/endpoints/test_counter.rs b/crates/desktop/src-tauri/src/endpoints/test_counter.rs new file mode 100644 index 0000000000..f377b8c90e --- /dev/null +++ b/crates/desktop/src-tauri/src/endpoints/test_counter.rs @@ -0,0 +1,24 @@ +use std::sync::{Arc, RwLock}; +use once_cell::sync::Lazy; + +static COUNTER : Lazy>> = Lazy::new(|| Arc::new(RwLock::new(0))); + +#[tauri::command] +pub fn test_counter() -> u64 { + let value : u64; + { + match COUNTER.write() { + Ok(mut counter) => { + *counter += 1; + value = *counter; + }, + Err(_e) => { + value = 0; + } + } + } + + println!("I was invoked from JavaScript! {:?}", value); + + value +} diff --git a/crates/desktop/src-tauri/src/lib.rs b/crates/desktop/src-tauri/src/lib.rs index eb94f6a947..625e8f1c96 100644 --- a/crates/desktop/src-tauri/src/lib.rs +++ b/crates/desktop/src-tauri/src/lib.rs @@ -1,55 +1,60 @@ -mod image_endpoint; +pub mod endpoints; +pub mod ml; +pub mod state; -use std::sync::{Arc, RwLock}; -use once_cell::sync::Lazy; -use image_endpoint::infer_image; +use crate::ml::model_cache::ModelCache; +use crate::ml::prompt_cache::PromptCache; +use crate::state::app_config::AppConfig; +use endpoints::image_endpoint::infer_image; +use endpoints::test_counter::test_counter; +use tauri_plugin_log::Target; +use tauri_plugin_log::TargetKind; #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { + // NB: Tauri wants to install the logger itself, so we can't rely on the logger crate + // until the tauri runtime begins. + println!("Loading model config..."); + + let config = AppConfig::init() + .expect("config should load"); + + let prompt_cache = PromptCache::with_capacity(8) + .expect("prompt cache should load"); + + println!("Creating model cache..."); + + let model_cache = ModelCache::new( + config.device.clone(), + config.dtype, + config.sd_version.clone(), + config.sd_config.clone() + ).expect("Model cache should create"); + + println!("Initializing backend runtime..."); + tauri::Builder::default() + .plugin(tauri_plugin_log::Builder::new() + .level(log::LevelFilter::Info) + .targets(vec![Target::new(TargetKind::Stdout)]) + .build()) .setup(|app| { - if cfg!(debug_assertions) { - app.handle().plugin( - tauri_plugin_log::Builder::default() - .level(log::LevelFilter::Info) - .build(), - )?; - } + //if cfg!(debug_assertions) { + // app.handle().plugin( + // tauri_plugin_log::Builder::default() + // .level(log::LevelFilter::Info) + // .build(), + // )?; + //} Ok(()) }) + .manage(config) + .manage(prompt_cache) + .manage(model_cache) .invoke_handler(tauri::generate_handler![ - my_custom_command, - test_round_trip, + test_counter, infer_image, ]) .run(tauri::generate_context!()) .expect("error while running tauri application"); } - -#[tauri::command] -fn my_custom_command() { - println!("I was invoked from JavaScript!"); -} - -static COUNTER : Lazy>> = Lazy::new(|| Arc::new(RwLock::new(0))); - -#[tauri::command] -fn test_round_trip() -> u64 { - let value : u64; - { - match COUNTER.write() { - Ok(mut counter) => { - *counter += 1; - value = *counter; - }, - Err(_e) => { - value = 0; - } - } - } - - println!("I was invoked from JavaScript! {:?}", value); - - value -} - diff --git a/crates/desktop/src-tauri/src/ml/create_inpainting_tensors.rs b/crates/desktop/src-tauri/src/ml/create_inpainting_tensors.rs new file mode 100644 index 0000000000..90e61d3a1b --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/create_inpainting_tensors.rs @@ -0,0 +1,80 @@ +use std::path::PathBuf; +use crate::ml::model_file::StableDiffusionVersion; +use crate::ml::models::lazy_load_vae_model::LazyLoadVaeModel; +use candle_core::{DType, Device, Tensor}; +use crate::ml::model_cache::ModelCache; + +/// Generates the mask latents, scaled mask and mask_4 for inpainting. +/// Returns a tuple of None if inpainting is not being used. +#[allow(clippy::too_many_arguments)] +pub fn create_inpainting_tensors( + sd_version: StableDiffusionVersion, + mask_path: Option, + dtype: DType, + device: &Device, + use_guide_scale: bool, + model_cache: &ModelCache, + image: Option, + vae_scale: f64, +) -> anyhow::Result<(Option, Option, Option)> { + match sd_version { + StableDiffusionVersion::XlInpaint + | StableDiffusionVersion::V2Inpaint + | StableDiffusionVersion::V1_5Inpaint => { + let inpaint_mask = mask_path.ok_or_else(|| { + anyhow::anyhow!("An inpainting model was requested but mask-path is not provided.") + })?; + // Get the mask image with shape [1, 1, 128, 128] + let mask = mask_preprocess(inpaint_mask)? + .to_device(device)? + .to_dtype(dtype)?; + // Generate the masked image from the image and the mask with shape [1, 3, 1024, 1024] + let xmask = mask.le(0.5)?.repeat(&[1, 3, 1, 1])?.to_dtype(dtype)?; + let image = &image + .ok_or_else(|| anyhow::anyhow!( + "An inpainting model was requested but img2img which is used as the input image is not provided." + ))?; + let masked_img = (image * xmask)?; + // Scale down the mask + let shape = masked_img.shape(); + let (w, h) = (shape.dims()[3] / 8, shape.dims()[2] / 8); + let mask = mask.interpolate2d(w, h)?; + // shape: [1, 4, 128, 128] + let mask_latents = model_cache.vae_encode(&masked_img)?; + let mask_latents = (mask_latents.sample()? * vae_scale)?.to_device(device)?; + + let mask_4 = mask.as_ref().repeat(&[1, 4, 1, 1])?; + let (mask_latents, mask) = if use_guide_scale { + ( + Tensor::cat(&[&mask_latents, &mask_latents], 0)?, + Tensor::cat(&[&mask, &mask], 0)?, + ) + } else { + (mask_latents, mask) + }; + Ok((Some(mask_latents), Some(mask), Some(mask_4))) + } + _ => Ok((None, None, None)), + } +} + +fn mask_preprocess>(path: T) -> anyhow::Result { + let img = image::open(path)?.to_luma8(); + let (new_width, new_height) = { + let (width, height) = img.dimensions(); + (width - width % 32, height - height % 32) + }; + let img = image::imageops::resize( + &img, + new_width, + new_height, + image::imageops::FilterType::CatmullRom, + ) + .into_raw(); + let mask = Tensor::from_vec(img, (new_height as usize, new_width as usize), &Device::Cpu)? + .unsqueeze(0)? + .to_dtype(DType::F32)? + //.div(255.0)? // TODO(bt): Need to figure out what changed in candle to no longer permit division by f64 + .unsqueeze(0)?; + Ok(mask) +} diff --git a/crates/desktop/src-tauri/src/ml/image/dynamic_image_to_tensor.rs b/crates/desktop/src-tauri/src/ml/image/dynamic_image_to_tensor.rs new file mode 100644 index 0000000000..47d83deda9 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/image/dynamic_image_to_tensor.rs @@ -0,0 +1,17 @@ +use anyhow::Result; +use candle_core::{DType, Device, Tensor}; +use image::DynamicImage; + +pub fn dynamic_image_to_tensor(image: &DynamicImage, device: &Device, datatype: DType) -> Result { + let (height, width) = (image.height() as usize, image.width() as usize); + let height = height - height % 32; + let width = width - width % 32; + let image = image.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom); + let image = image.to_rgb8(); + let image = image.into_raw(); + Ok(Tensor::from_vec(image, (height, width, 3), device)? + .permute((2, 0, 1))? + .to_dtype(datatype)? + .affine(2. / 255., -1.)? + .unsqueeze(0)?) +} diff --git a/crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor.rs b/crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor.rs new file mode 100644 index 0000000000..0d89b8bb7e --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor.rs @@ -0,0 +1,28 @@ +use candle_core::{DType, Device, Tensor}; + +pub fn load_image_file_to_tensor>(path: T, device: &Device) -> anyhow::Result { + let img = image::ImageReader::open(path)?.decode()?; + let (height, width) = (img.height() as usize, img.width() as usize); + println!("INPUT IMAGE WIDTH AND HEIGHT: {}x{}", width, height); + + // TODO: Downscale from 1024 -- YES! THIS HACK DOES THE TRICK OF GETTING THE TENSOR SHAPE TO [1,4,64,64] !! + let height = height / 2; + let width = width / 2; + + let height = height - height % 32; + let width = width - width % 32; + let img = img.resize_to_fill( + width as u32, + height as u32, + image::imageops::FilterType::CatmullRom, + ); + println!("RESIZED IMAGE WIDTH AND HEIGHT: {}x{}", width, height); + let img = img.to_rgb8(); + let img = img.into_raw(); + let img = Tensor::from_vec(img, (height, width, 3), device)? + .permute((2, 0, 1))? + .to_dtype(DType::F32)? + .affine(2. / 255., -1.)? + .unsqueeze(0)?; + Ok(img) +} diff --git a/crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor_2.rs b/crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor_2.rs new file mode 100644 index 0000000000..6eb5ff4aea --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/image/load_image_file_to_tensor_2.rs @@ -0,0 +1,18 @@ +use anyhow::Result; +use candle_core::{DType, Device, Tensor}; + +pub fn load_image_file_to_tensor_2>(path: T) -> Result { + let img = image::ImageReader::open(path)?.decode()?; + let (height, width) = (img.height() as usize, img.width() as usize); + let height = height - height % 32; + let width = width - width % 32; + let img = img.resize_to_fill(width as u32, height as u32, image::imageops::FilterType::CatmullRom); + let img = img.to_rgb8(); + let img = img.into_raw(); + let img = Tensor::from_vec(img, (height, width, 3), &Device::Cpu)? + .permute((2, 0, 1))? + .to_dtype(DType::F32)? + .affine(2. / 255., -1.)? + .unsqueeze(0)?; + Ok(img) +} diff --git a/crates/desktop/src-tauri/src/ml/image/mod.rs b/crates/desktop/src-tauri/src/ml/image/mod.rs new file mode 100644 index 0000000000..b6059fd6e3 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/image/mod.rs @@ -0,0 +1,5 @@ +pub mod dynamic_image_to_tensor; +pub mod load_image_file_to_tensor; +pub mod load_image_file_to_tensor_2; +pub mod save_image_from_tensor; +pub mod tensor_to_image_buffer; \ No newline at end of file diff --git a/crates/desktop/src-tauri/src/ml/image/save_image_from_tensor.rs b/crates/desktop/src-tauri/src/ml/image/save_image_from_tensor.rs new file mode 100644 index 0000000000..145ac69e50 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/image/save_image_from_tensor.rs @@ -0,0 +1,28 @@ +use anyhow::{Error as E, Result}; +use candle_core::Tensor; +use image::{ImageBuffer, Rgb}; + +pub fn save_image_from_tensor>(img: &Tensor, p: P) -> Result<()> { + println!("Saving image to {:?}", p.as_ref()); + let p = p.as_ref(); + let (channel, height, width) = img.dims3()?; + println!("Image dimensions: {}x{} with {} channels", width, height, channel); + + if channel != 3 { + anyhow::bail!("save_image expects an input of shape (3, height, width)") + } + let img = img.permute((1, 2, 0))?.flatten_all()?; + let pixels = img.to_vec1::()?; + println!("Converting tensor to image buffer..."); + + let image: ImageBuffer, Vec> = + match ImageBuffer::from_raw(width as u32, height as u32, pixels) { + Some(image) => image, + None => anyhow::bail!("error saving image {p:?}"), + }; + println!("Successfully created image buffer"); + + image.save(p).map_err(E::from)?; + println!("Successfully saved image to disk"); + Ok(()) +} diff --git a/crates/desktop/src-tauri/src/ml/image/tensor_to_image_buffer.rs b/crates/desktop/src-tauri/src/ml/image/tensor_to_image_buffer.rs new file mode 100644 index 0000000000..362f954b24 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/image/tensor_to_image_buffer.rs @@ -0,0 +1,18 @@ +use candle_core::Tensor; +use image::{ImageBuffer, Rgb}; + +pub type RgbImage = ImageBuffer, Vec>; + +pub fn tensor_to_image_buffer(img: &Tensor) -> anyhow::Result { + let (channel, height, width) = img.dims3()?; + + if channel != 3 { + anyhow::bail!("save_image expects an input of shape (3, height, width)") + } + + let img = img.permute((1, 2, 0))?.flatten_all()?; + let pixels = img.to_vec1::()?; + + ImageBuffer::from_raw(width as u32, height as u32, pixels) + .ok_or_else(|| anyhow::anyhow!("error loading image from raw tensor")) +} diff --git a/crates/desktop/src-tauri/src/ml/infer_clip_text_embeddings.rs b/crates/desktop/src-tauri/src/ml/infer_clip_text_embeddings.rs new file mode 100644 index 0000000000..eed7961c00 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/infer_clip_text_embeddings.rs @@ -0,0 +1,153 @@ +#[cfg(feature = "accelerate")] +extern crate accelerate_src; +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +use candle_transformers::models::stable_diffusion; + +use crate::ml::model_file::{ModelFile, StableDiffusionVersion}; +use anyhow::Error as E; +use candle_core::{DType, Device, Module, Tensor, D}; +use log::info; +use tokenizers::Tokenizer; + +#[allow(clippy::too_many_arguments)] +pub fn infer_clip_text_embeddings( + prompt: &str, + uncond_prompt: &str, + tokenizer: Option, + clip_weights: Option, + clip2_weights: Option, + sd_version: StableDiffusionVersion, + sd_config: &stable_diffusion::StableDiffusionConfig, + use_f16: bool, + device: &Device, + dtype: DType, + use_guide_scale: bool, +) -> anyhow::Result { + info!("Preparing text embeddings..."); + + let which = match sd_version { + StableDiffusionVersion::Xl + | StableDiffusionVersion::XlInpaint + | StableDiffusionVersion::Turbo => vec![true, false], + _ => vec![true], + }; + + let text_embeddings = which + .iter() + .map(|first| { + do_infer_clip_text_embeddings( + &prompt, + &uncond_prompt, + None, // tokenizer + None, // clip_weights + None, // clip2_weights + sd_version, + &sd_config, + false, // use_f16 + &device, + dtype, + false, // use_guide_scale + *first, + ) + }) + .collect::>>()?; + + let text_embeddings = Tensor::cat(&text_embeddings, D::Minus1)?; + + Ok(text_embeddings) +} + +#[allow(clippy::too_many_arguments)] +fn do_infer_clip_text_embeddings( + prompt: &str, + uncond_prompt: &str, + tokenizer: Option, + clip_weights: Option, + clip2_weights: Option, + sd_version: StableDiffusionVersion, + sd_config: &stable_diffusion::StableDiffusionConfig, + use_f16: bool, + device: &Device, + dtype: DType, + use_guide_scale: bool, + first: bool, + ) -> anyhow::Result { + + let tokenizer_file = if first { + ModelFile::Tokenizer + } else { + ModelFile::Tokenizer2 + }; + let tokenizer = tokenizer_file.get(tokenizer, sd_version, use_f16)?; + let tokenizer = Tokenizer::from_file(tokenizer).map_err(E::msg)?; + let pad_id = match &sd_config.clip.pad_with { + Some(padding) => *tokenizer.get_vocab(true).get(padding.as_str()).unwrap(), + None => *tokenizer.get_vocab(true).get("<|endoftext|>").unwrap(), + }; + + println!("Running with prompt \"{prompt}\"."); + let mut tokens = tokenizer + .encode(prompt, true) + .map_err(E::msg)? + .get_ids() + .to_vec(); + if tokens.len() > sd_config.clip.max_position_embeddings { + anyhow::bail!( + "the prompt is too long, {} > max-tokens ({})", + tokens.len(), + sd_config.clip.max_position_embeddings + ) + } + while tokens.len() < sd_config.clip.max_position_embeddings { + tokens.push(pad_id) + } + let tokens = Tensor::new(tokens.as_slice(), device)?.unsqueeze(0)?; + + println!("Building the Clip transformer."); + let clip_weights_file = if first { + ModelFile::Clip + } else { + ModelFile::Clip2 + }; + let clip_weights = if first { + clip_weights_file.get(clip_weights, sd_version, use_f16)? + } else { + clip_weights_file.get(clip2_weights, sd_version, use_f16)? + }; + let clip_config = if first { + &sd_config.clip + } else { + sd_config.clip2.as_ref().unwrap() + }; + let text_model = stable_diffusion::build_clip_transformer(clip_config, clip_weights, device, DType::F32)?; + + let text_embeddings = text_model.forward(&tokens)?; + + let text_embeddings = if use_guide_scale { + let mut uncond_tokens = tokenizer + .encode(uncond_prompt, true) + .map_err(E::msg)? + .get_ids() + .to_vec(); + if uncond_tokens.len() > sd_config.clip.max_position_embeddings { + anyhow::bail!( + "the negative prompt is too long, {} > max-tokens ({})", + uncond_tokens.len(), + sd_config.clip.max_position_embeddings + ) + } + while uncond_tokens.len() < sd_config.clip.max_position_embeddings { + uncond_tokens.push(pad_id) + } + + let uncond_tokens = Tensor::new(uncond_tokens.as_slice(), device)?.unsqueeze(0)?; + let uncond_embeddings = text_model.forward(&uncond_tokens)?; + + Tensor::cat(&[uncond_embeddings, text_embeddings], 0)?.to_dtype(dtype)? + } else { + text_embeddings.to_dtype(dtype)? + }; + Ok(text_embeddings) +} diff --git a/crates/desktop/src-tauri/src/ml/mod.rs b/crates/desktop/src-tauri/src/ml/mod.rs new file mode 100644 index 0000000000..3f95f0499e --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/mod.rs @@ -0,0 +1,8 @@ +pub mod create_inpainting_tensors; +pub mod image; +pub mod infer_clip_text_embeddings; +pub mod model_cache; +pub mod model_file; +pub mod models; +pub mod prompt_cache; +pub mod stable_diffusion; \ No newline at end of file diff --git a/crates/desktop/src-tauri/src/ml/model_cache.rs b/crates/desktop/src-tauri/src/ml/model_cache.rs new file mode 100644 index 0000000000..1cbbc76ba9 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/model_cache.rs @@ -0,0 +1,118 @@ +use crate::ml::models::unet_model::UNetModel; +use crate::ml::model_file::StableDiffusionVersion; +use anyhow::anyhow; +use candle_core::{DType, Device, Tensor}; +use candle_transformers::models::stable_diffusion::StableDiffusionConfig; +use hf_hub::api::sync::Api; +use std::sync::{Arc, RwLock}; +use candle_transformers::models::stable_diffusion::vae::DiagonalGaussianDistribution; +use crate::ml::models::lazy_load_vae_model::LazyLoadVaeModel; +// TODO: This data structure is gross. Generalize the locking and loading semantics for reuse. + +// Simple registry for now. We can build complex machinery that aids in +// VRAM utilization, disk space saving, intelligent scheduling, etc. in the future. +pub struct ModelCache { + // TODO(bt,2025-02-14): Support for more than one device in the future. + device: Device, + dtype: DType, + hf_api: Api, + sd_config: StableDiffusionConfig, + sd_version: StableDiffusionVersion, + + unet: Arc>>, + lazy_vae: LazyLoadVaeModel, +} + +impl ModelCache { + pub fn new( + device: Device, + dtype: DType, + sd_version: StableDiffusionVersion, + sd_config: StableDiffusionConfig, + ) -> anyhow::Result { + let api = Api::new()?; + + Ok(Self { + device: device.clone(), + dtype, + hf_api: api.clone(), + unet: Arc::new(RwLock::new(None)), + lazy_vae: LazyLoadVaeModel::new( + &sd_config, + &sd_version, + &api, + &device, + dtype, + )?, + sd_config, + sd_version, + }) + } + + pub fn vae_encode(&self, xs: &Tensor) -> anyhow::Result { + self.lazy_vae.encode(xs) + } + + pub fn vae_decode(&self, xs: &Tensor) -> anyhow::Result { + self.lazy_vae.decode(xs) + } + + pub fn unet_inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result { + println!("unet_inference"); + match self.try_unet_inference(latent_model_input, timestep, text_embeddings) { + Err(err) => Err(err), + Ok(Some(result)) => Ok(result), + Ok(None) => { + // Model wasn't previously loaded + self.load_unet()?; + self.try_unet_inference(latent_model_input, timestep, text_embeddings)? + .ok_or(anyhow!("model was not previously loaded")) + } + } + } + + fn load_unet(&self) -> anyhow::Result<()> { + println!("load_unet"); + match self.unet.write() { + Err(err) => return Err(anyhow!("lock error: {:?}", err)), + Ok(mut maybe_model) => { + match &*maybe_model { + Some(_model) => {} // Fall through + None => { + let repo = self.sd_version.repo(); + + println!("Downloading UNET model files from: {} ...", repo); + + let unet_file = self.hf_api.model(repo.to_string()) + .get("unet/diffusion_pytorch_model.safetensors") + .map_err(|err| anyhow!("error fetching model: {:?}", err))?; + + println!("Building UNET model... (3)"); + let unet = UNetModel::new(&self.sd_config, unet_file, &self.device, self.dtype) + .map_err(|err| anyhow!("error initializing unet model: {:?}", err))?; + + println!("Built UNET..."); + *maybe_model = Some(unet); + } + } + } + } + Ok(()) + } + + pub fn try_unet_inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result> { + println!("try_unet_inference"); + match self.unet.read() { + Ok(mut model) => { + match &*model { + None => Ok(None), // Model is not yet loaded + Some(model) => { + Ok(Some(model.inference(latent_model_input, timestep, text_embeddings)?)) + } + } + } + Err(err) => Err(anyhow!("lock error: {:?}", err)), + } + } + +} diff --git a/crates/desktop/src-tauri/src/ml/model_file.rs b/crates/desktop/src-tauri/src/ml/model_file.rs new file mode 100644 index 0000000000..bdfe9b8cc0 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/model_file.rs @@ -0,0 +1,83 @@ +#[cfg(feature = "accelerate")] +extern crate accelerate_src; +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +use candle_transformers::models::stable_diffusion; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModelFile { + Tokenizer, + Tokenizer2, + Clip, + Clip2, + Unet, + Vae, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StableDiffusionVersion { + V1_5, + V1_5Inpaint, + V2_1, + V2Inpaint, + Xl, + XlInpaint, + Turbo, +} + +impl StableDiffusionVersion { + pub fn repo(&self) -> &'static str { + match self { + Self::XlInpaint => "diffusers/stable-diffusion-xl-1.0-inpainting-0.1", + Self::Xl => "stabilityai/stable-diffusion-xl-base-1.0", + Self::V2Inpaint => "stabilityai/stable-diffusion-2-inpainting", + Self::V2_1 => "stabilityai/stable-diffusion-2-1", + Self::V1_5 => "runwayml/stable-diffusion-v1-5", + Self::V1_5Inpaint => "stable-diffusion-v1-5/stable-diffusion-inpainting", + Self::Turbo => "stabilityai/sdxl-turbo", + } + } +} + +impl ModelFile { + pub fn get( + &self, + filename: Option, + version: StableDiffusionVersion, + use_f16: bool, + ) -> anyhow::Result { + use hf_hub::api::sync::Api; + match filename { + Some(filename) => Ok(std::path::PathBuf::from(filename)), + None => { + let (repo, path) = match self { + Self::Tokenizer => { + let tokenizer_repo = match version { + StableDiffusionVersion::V1_5 + | StableDiffusionVersion::V2_1 + | StableDiffusionVersion::V1_5Inpaint + | StableDiffusionVersion::V2Inpaint => "openai/clip-vit-base-patch32", + StableDiffusionVersion::Xl + | StableDiffusionVersion::XlInpaint + | StableDiffusionVersion::Turbo => { + "openai/clip-vit-large-patch14" + } + }; + (tokenizer_repo, "tokenizer.json") + } + Self::Tokenizer2 => { + ("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", "tokenizer.json") + } + Self::Clip => (version.repo(), "text_encoder/model.safetensors"), + Self::Clip2 => (version.repo(), "text_encoder_2/model.safetensors"), + Self::Unet => (version.repo(), "unet/diffusion_pytorch_model.safetensors"), + Self::Vae => (version.repo(), "vae/diffusion_pytorch_model.safetensors"), + }; + let filename = Api::new()?.model(repo.to_string()).get(path)?; + Ok(filename) + } + } + } +} + diff --git a/crates/desktop/src-tauri/src/ml/models/lazy_load_vae_model.rs b/crates/desktop/src-tauri/src/ml/models/lazy_load_vae_model.rs new file mode 100644 index 0000000000..dfe5e7f3f1 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/models/lazy_load_vae_model.rs @@ -0,0 +1,130 @@ +use crate::ml::model_file::StableDiffusionVersion; +use anyhow::anyhow; +use candle_core::{DType, Device, Tensor}; +use candle_transformers::models::stable_diffusion::vae::{AutoEncoderKL, DiagonalGaussianDistribution}; +use candle_transformers::models::stable_diffusion::StableDiffusionConfig; +use hf_hub::api::sync::Api; +use std::sync::{Arc, RwLock}; + +pub struct LazyLoadVaeModel { + lazy_model: Arc>>, + model_configs: VaeConfigs, +} + +// TODO: Maybe we pass these via a central config read at startup, passed as an Arc +pub struct VaeConfigs { + sd_config: StableDiffusionConfig, + sd_version: StableDiffusionVersion, + device: Device, + dtype: DType, + hf_api: Api, +} + +impl LazyLoadVaeModel { + pub fn new( + sd_config: &StableDiffusionConfig, + sd_version: &StableDiffusionVersion, + hf_api: &Api, + device: &Device, + dtype: DType, + ) -> anyhow::Result { + println!("building vae model... (1)"); + + Ok(Self { + lazy_model: Arc::new(RwLock::new(None)), + model_configs: VaeConfigs { + sd_config: sd_config.clone(), + sd_version: sd_version.clone(), + device: device.clone(), + hf_api: hf_api.clone(), + dtype + } + }) + } + + pub fn encode(&self, xs: &Tensor) -> anyhow::Result { + println!("vae encode..."); + match self.try_encode(xs) { + Err(err) => Err(err), + Ok(Some(result)) => Ok(result), + Ok(None) => { + // Model wasn't previously loaded + self.load_model()?; + self.try_encode(xs)? + .ok_or(anyhow!("model was not previously loaded")) + } + } + } + + pub fn decode(&self, xs: &Tensor) -> anyhow::Result { + println!("vae decode..."); + match self.try_decode(xs) { + Err(err) => Err(err), + Ok(Some(result)) => Ok(result), + Ok(None) => { + // Model wasn't previously loaded + self.load_model()?; + self.try_decode(xs)? + .ok_or(anyhow!("model was not previously loaded")) + } + } + } + + fn try_encode(&self, xs: &Tensor) -> anyhow::Result> { + match self.lazy_model.read() { + Err(err) => Err(anyhow!("lock error: {:?}", err)), + Ok(model) => { + match &*model { + None => Ok(None), // Model is not yet loaded + Some(model) => { + Ok(Some(model.encode(xs)?)) + } + } + } + } + } + + fn try_decode(&self, xs: &Tensor) -> anyhow::Result> { + match self.lazy_model.read() { + Err(err) => Err(anyhow!("lock error: {:?}", err)), + Ok(model) => { + match &*model { + None => Ok(None), // Model is not yet loaded + Some(model) => { + Ok(Some(model.decode(xs)?)) + } + } + } + } + } + + fn load_model(&self) -> anyhow::Result<()> { + println!("load vae"); + match self.lazy_model.write() { + Err(err) => return Err(anyhow!("lock error: {:?}", err)), + Ok(mut maybe_model) => { + match &*maybe_model { + Some(_model) => {} // Already loaded; fall through + None => { + let repo = self.model_configs.sd_version.repo(); + + println!("Building VAE model from : {:?} ... (3)", repo); + + let vae_file = self.model_configs.hf_api.model(repo.to_string()) + .get("vae/diffusion_pytorch_model.safetensors")?; + + println!("Building VAE model from file {:?}...", &vae_file); + let vae = self.model_configs + .sd_config + .build_vae(vae_file, &self.model_configs.device, self.model_configs.dtype)?; + + println!("Built VAE..."); + *maybe_model = Some(vae); + } + } + } + } + Ok(()) + } + +} diff --git a/crates/desktop/src-tauri/src/ml/models/mod.rs b/crates/desktop/src-tauri/src/ml/models/mod.rs new file mode 100644 index 0000000000..be97348798 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/models/mod.rs @@ -0,0 +1,2 @@ +pub mod unet_model; +pub mod lazy_load_vae_model; \ No newline at end of file diff --git a/crates/desktop/src-tauri/src/ml/models/unet_model.rs b/crates/desktop/src-tauri/src/ml/models/unet_model.rs new file mode 100644 index 0000000000..2483525c96 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/models/unet_model.rs @@ -0,0 +1,35 @@ +use candle_core::{DType, Device, Tensor}; +use candle_transformers::models::stable_diffusion::unet_2d::UNet2DConditionModel; +use candle_transformers::models::stable_diffusion::StableDiffusionConfig; +use std::path::Path; + +pub struct UNetModel { + model: UNet2DConditionModel, +} + +impl UNetModel { + pub fn new>( + sd_config: &StableDiffusionConfig, + unet_weights_file: P, + device: &Device, + dtype: DType, + ) -> anyhow::Result { + println!("building unet model... (1)"); + + let model = sd_config.build_unet( + unet_weights_file, + &device, + 4, + false, + dtype + )?; + + Ok(Self { + model, + }) + } + + pub fn inference(&self, latent_model_input: &Tensor, timestep: f64, text_embeddings: &Tensor) -> anyhow::Result { + Ok(self.model.forward(&latent_model_input, timestep, &text_embeddings)?) + } +} diff --git a/crates/desktop/src-tauri/src/ml/prompt_cache.rs b/crates/desktop/src-tauri/src/ml/prompt_cache.rs new file mode 100644 index 0000000000..6816d8b798 --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/prompt_cache.rs @@ -0,0 +1,22 @@ +use candle_core::Tensor; +use memory_caching::arc_sieve::ArcSieve; + +pub struct PromptCache { + cache: ArcSieve, +} + +impl PromptCache { + pub fn with_capacity(capacity: usize) -> anyhow::Result { + Ok(Self { + cache: ArcSieve::with_capacity(capacity)?, + }) + } + + pub fn get_copy(&self, key: &String) -> anyhow::Result> { + self.cache.get_copy(key) + } + + pub fn store_copy(&self, key: &String, value: &Tensor) -> anyhow::Result<()> { + self.cache.store_copy(key, value) + } +} diff --git a/crates/desktop/src-tauri/src/ml/stable_diffusion/get_vae_scale.rs b/crates/desktop/src-tauri/src/ml/stable_diffusion/get_vae_scale.rs new file mode 100644 index 0000000000..23b8c19f9e --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/stable_diffusion/get_vae_scale.rs @@ -0,0 +1,13 @@ +use crate::ml::model_file::StableDiffusionVersion; + +pub fn get_vae_scale(sd_version: StableDiffusionVersion) -> f64 { + match sd_version { + StableDiffusionVersion::V1_5 + | StableDiffusionVersion::V1_5Inpaint + | StableDiffusionVersion::V2_1 + | StableDiffusionVersion::V2Inpaint + | StableDiffusionVersion::XlInpaint + | StableDiffusionVersion::Xl => 0.18215, + StableDiffusionVersion::Turbo => 0.13025, + } +} diff --git a/crates/desktop/src-tauri/src/ml/stable_diffusion/mod.rs b/crates/desktop/src-tauri/src/ml/stable_diffusion/mod.rs new file mode 100644 index 0000000000..8e9407a18e --- /dev/null +++ b/crates/desktop/src-tauri/src/ml/stable_diffusion/mod.rs @@ -0,0 +1 @@ +pub mod get_vae_scale; \ No newline at end of file diff --git a/crates/desktop/src-tauri/src/state/app_config.rs b/crates/desktop/src-tauri/src/state/app_config.rs new file mode 100644 index 0000000000..3c5f5e0a7c --- /dev/null +++ b/crates/desktop/src-tauri/src/state/app_config.rs @@ -0,0 +1,93 @@ +use crate::ml::model_file::StableDiffusionVersion; +use crate::state::yaml_config::YamlConfig; +use candle_core::{DType, Device}; +use candle_transformers::models::stable_diffusion::StableDiffusionConfig; +use filesys::recursively_find_file_by_name::recursively_find_file_by_name; +use hf_hub::api::sync::Api; +use log::info; + +const CONFIG_FILENAME : &str = "app_config.yaml"; + +const DEFAULT_SD_IMAGE_WIDTH: usize = 512; +const DEFAULT_SD_IMAGE_HEIGHT: usize = 512; + +const DEFAULT_SD_NUMERIC_DATATYPE: DType = DType::F32; +const SD_VERSION: StableDiffusionVersion = StableDiffusionVersion::Turbo; + +/// A central place to configure app-wide details. +pub struct AppConfig { + /// We only support one device type currently, and we can't multiplex + /// over several devices yet. + pub device: Device, + + /// The core numeric type to use for models. + pub dtype: DType, + + pub sd_version: StableDiffusionVersion, + pub sd_config: StableDiffusionConfig, + + pub image_height: usize, + pub image_width: usize, + pub scheduler_steps: usize, + pub scheduler_samples: usize, + + pub seed: Option, + + /// Probably shouldn't be here. + pub hf_api: Api, +} + +impl AppConfig { + + pub fn init() -> anyhow::Result { + println!("Configuration filename: {}", CONFIG_FILENAME); + let yaml_configs = load_yaml_configs(); + + println!("Configs: {:?}", yaml_configs); + + let device = Device::new_cuda(0) + .unwrap_or_else(|e| { + info!("CUDA not available ({}), falling back to CPU", e); + Device::Cpu + }); + + let sd_config = match SD_VERSION { + StableDiffusionVersion::Turbo => { + StableDiffusionConfig::sdxl_turbo(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH)) + } + _ => { + StableDiffusionConfig::v2_1(None, Some(DEFAULT_SD_IMAGE_HEIGHT), Some(DEFAULT_SD_IMAGE_WIDTH)) + } + }; + + let hf_api = Api::new()?; + + Ok(Self { + device, + dtype: DEFAULT_SD_NUMERIC_DATATYPE, + sd_version: SD_VERSION, + sd_config, + image_height: yaml_configs.image_height.unwrap_or(DEFAULT_SD_IMAGE_HEIGHT), + image_width: yaml_configs.image_width.unwrap_or(DEFAULT_SD_IMAGE_WIDTH), + scheduler_steps: yaml_configs.scheduler_steps.unwrap_or(1), + scheduler_samples: yaml_configs.scheduler_samples.unwrap_or(15), + seed: yaml_configs.seed, + hf_api, + }) + } +} + +fn load_yaml_configs() -> YamlConfig { + let maybe_path = recursively_find_file_by_name(CONFIG_FILENAME, "../../../", 5) + .ok() + .flatten(); + if let Some(path) = maybe_path { + let path = path.canonicalize().unwrap_or(path); + println!("Attempting to load configs from: {:?}", &path); + if let Ok(config) = YamlConfig::read_from_file(path) { + return config; + } + } + println!("Loading default configs."); + YamlConfig::default() +} diff --git a/crates/desktop/src-tauri/src/state/mod.rs b/crates/desktop/src-tauri/src/state/mod.rs new file mode 100644 index 0000000000..196a0ce480 --- /dev/null +++ b/crates/desktop/src-tauri/src/state/mod.rs @@ -0,0 +1,2 @@ +pub mod yaml_config; +pub mod app_config; \ No newline at end of file diff --git a/crates/desktop/src-tauri/src/state/yaml_config.rs b/crates/desktop/src-tauri/src/state/yaml_config.rs new file mode 100644 index 0000000000..cd0626338c --- /dev/null +++ b/crates/desktop/src-tauri/src/state/yaml_config.rs @@ -0,0 +1,24 @@ +use std::fs::File; +use std::io::BufReader; +use std::path::Path; + +/// Configuration read or derived from a YAML config file at startup +/// This is not meant to contain any runtime config values. +/// This may not be exhaustive. +#[derive(serde::Deserialize, serde::Serialize, Clone, Debug, Default)] +pub struct YamlConfig { + pub image_height: Option, + pub image_width: Option, + pub scheduler_steps: Option, + pub scheduler_samples: Option, + pub seed: Option, +} + +impl YamlConfig { + pub fn read_from_file>(path: P) -> anyhow::Result { + let file = File::open(path)?; + let reader = BufReader::new(file); + let config = serde_yaml::from_reader(reader)?; + Ok(config) + } +} diff --git a/crates/lib/files/filesys/src/lib.rs b/crates/lib/files/filesys/src/lib.rs index f2781858d1..604dff5349 100644 --- a/crates/lib/files/filesys/src/lib.rs +++ b/crates/lib/files/filesys/src/lib.rs @@ -31,4 +31,5 @@ pub mod file_size; pub mod filename_concat; pub mod is_filesystem_full_error; pub mod path_to_string; +pub mod recursively_find_file_by_name; pub mod rename_across_devices; diff --git a/crates/lib/files/filesys/src/recursively_find_file_by_name.rs b/crates/lib/files/filesys/src/recursively_find_file_by_name.rs new file mode 100644 index 0000000000..a454238611 --- /dev/null +++ b/crates/lib/files/filesys/src/recursively_find_file_by_name.rs @@ -0,0 +1,56 @@ +use std::path::{Path, PathBuf}; +use walkdir::WalkDir; +use errors::AnyhowResult; + +pub fn recursively_find_file_by_name, Q: AsRef>( + search_filename: P, + search_root: Q, + max_depth: usize, +) -> AnyhowResult> { + + let walker = WalkDir::new(search_root) + .max_depth(max_depth); + + for entry in walker { + let entry = entry?; + if entry.path().is_dir() { + let check_path = entry.path().join(search_filename.as_ref()); + if check_path.exists() && check_path.is_file() { + return Ok(Some(check_path)); + } + } + } + + Ok(None) +} + +#[cfg(test)] +mod tests { + use testing::test_file_path::test_file_path; + use crate::recursively_find_file_by_name::recursively_find_file_by_name; + + #[test] + fn test_path_exists() { + let test_data_path = test_file_path("test_data/").expect("should work"); + let result = recursively_find_file_by_name( + "super_mario_rpg_beware_the_forests_mushrooms.mp3", + test_data_path, + 5 + ); + let result = result.expect("should be ok"); + let result = result.expect("should exist"); + assert!(result.ends_with("test_data/audio/mp3/super_mario_rpg_beware_the_forests_mushrooms.mp3")); + } + + #[test] + fn test_path_does_not_exist() { + let test_data_path = test_file_path("test_data/").expect("should work"); + let result = recursively_find_file_by_name( + "fake_name.foo", + test_data_path, + 5 + ); + let result = result.expect("should be ok"); + assert!(result.is_none()); + } +} \ No newline at end of file diff --git a/script/build_production_realtime_desktop.sh b/script/build_production_realtime_desktop.sh new file mode 100755 index 0000000000..076029513a --- /dev/null +++ b/script/build_production_realtime_desktop.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# Tauri doesn't let you configure the frontend project directory statically, though they do provide an +# environment variable to pass it to the CLI. Without doing this, the tauri cli randomly walks the +# filesystem and finds the wrong frontend code. + +# TODO(bt,2025-02-13): This is not the correct way to get the root dir +root_dir=$(pwd) +export TAURI_FRONTEND_PATH="${root_dir}/frontend/realtime" + +cargo tauri build + diff --git a/script/launch_realtime_desktop.sh b/script/launch_realtime_desktop.sh index 55f0faec6a..15ba9f4ed2 100755 --- a/script/launch_realtime_desktop.sh +++ b/script/launch_realtime_desktop.sh @@ -8,4 +8,4 @@ root_dir=$(pwd) export TAURI_FRONTEND_PATH="${root_dir}/frontend/realtime" -cargo tauri dev +cargo tauri dev --no-watch