diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 84748bc785..a9bebf1881 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -3,6 +3,7 @@ arpabet backfill + canonicalized cashapp cheermote cidrs @@ -22,6 +23,8 @@ hparams huggingface lipsync + mochi + nestedness pathbuf replayable retryably diff --git a/Cargo.lock b/Cargo.lock index 9e048f0c1f..3a68aa5e70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -965,6 +965,12 @@ dependencies = [ "url_config", ] +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -1268,6 +1274,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "combine" version = "3.8.1" @@ -1562,6 +1574,12 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-bigint" version = "0.3.2" @@ -2111,6 +2129,22 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "exr" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1e481eb11a482815d3e9d618db8c42a93207134662873809335a92327440c18" +dependencies = [ + "bit_field", + "flume", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec 1.10.0", + "zune-inflate", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -2120,6 +2154,15 @@ dependencies = [ "instant", ] +[[package]] +name = "fdeflate" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10" +dependencies = [ + "simd-adler32", +] + [[package]] name = "filesys" version = "0.0.1" @@ -2154,6 +2197,7 @@ checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" dependencies = [ "futures-core", "futures-sink", + "nanorand", "pin-project 1.0.7", "spin 0.9.4", ] @@ -2400,8 +2444,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if 1.0.0", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", +] + +[[package]] +name = "gif" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045" +dependencies = [ + "color_quant", + "weezl", ] [[package]] @@ -2454,6 +2510,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +dependencies = [ + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2788,6 +2853,35 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-rational 0.4.1", + "num-traits", + "png", + "qoi", + "tiff", +] + +[[package]] +name = "images" +version = "0.0.1" +dependencies = [ + "errors", + "image", + "log", + "testing", +] + [[package]] name = "indexmap" version = "1.9.2" @@ -2982,6 +3076,15 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.61" @@ -3037,6 +3140,12 @@ dependencies = [ "spin 0.5.2", ] +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "lexical-sort" version = "0.3.1" @@ -3338,6 +3447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", + "simd-adler32", ] [[package]] @@ -3457,6 +3567,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.8", +] + [[package]] name = "native-tls" version = "0.2.10" @@ -3548,7 +3667,7 @@ dependencies = [ "num-complex", "num-integer", "num-iter", - "num-rational", + "num-rational 0.1.42", "num-traits", ] @@ -3635,6 +3754,17 @@ dependencies = [ "rustc-serialize", ] +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.15" @@ -4049,6 +4179,19 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +[[package]] +name = "png" +version = "0.17.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -4118,6 +4261,15 @@ dependencies = [ "unicase", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -4273,6 +4425,26 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils 0.8.16", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -4973,6 +5145,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" version = "0.3.10" @@ -5749,6 +5927,13 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" +[[package]] +name = "testing" +version = "0.0.1" +dependencies = [ + "errors", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -5793,6 +5978,17 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tiff" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d172b0f4d3fba17ba89811858b9d3d97f928aece846475bbda076ca46736211" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "tiger" version = "0.1.0" @@ -6979,6 +7175,12 @@ dependencies = [ "uuid 0.8.2", ] +[[package]] +name = "weezl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" + [[package]] name = "widestring" version = "0.4.3" @@ -7289,3 +7491,12 @@ dependencies = [ "libc", "pkg-config", ] + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml index 5bb79076f2..69d64b6bc4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,6 @@ members = [ # Shared libraries "crates/lib/actix_helpers", "crates/lib/api_clients/cloud_storage", - #"crates/lib/api_clients/fakeyou_client", "crates/lib/api_clients/google_drive_common", "crates/lib/api_clients/twitch_common", "crates/lib/bootstrap", @@ -34,6 +33,7 @@ members = [ "crates/lib/files/archives", "crates/lib/files/filesys", "crates/lib/files/hashing", + "crates/lib/files/images", "crates/lib/files/media", "crates/lib/files/mimetypes", "crates/lib/idempotency", @@ -41,13 +41,15 @@ members = [ "crates/lib/mysql_queries", "crates/lib/redis_common", "crates/lib/server_environment", - #"crates/lib/sqlite_queries", "crates/lib/sqlx_mysql_helpers", "crates/lib/subprocess_common", + "crates/lib/testing", "crates/lib/tts_common", "crates/lib/url_config", "crates/lib/user_input_common", "crates/lib/web_scrapers", + #"crates/lib/api_clients/fakeyou_client", + #"crates/lib/sqlite_queries", # Programs: Desktop apps #"crates/programs/aichatbot-sidecar", diff --git a/crates/lib/files/images/Cargo.toml b/crates/lib/files/images/Cargo.toml new file mode 100644 index 0000000000..8fc138ee3d --- /dev/null +++ b/crates/lib/files/images/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "images" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +# NB: Public "images" crate doesn't exist; "image" crate is the heavily used public library, +# so perhaps this naming scheme causes confusion (?) +name = "images" +path = "src/lib.rs" + +[dependencies] + +# Internal +errors = { path = "../../errors" } + +# External +image = "0.24.1" +log = "0.4.14" + +[dev-dependencies] +testing = { path = "../../testing" } diff --git a/crates/lib/files/images/src/lib.rs b/crates/lib/files/images/src/lib.rs new file mode 100644 index 0000000000..dbb0728069 --- /dev/null +++ b/crates/lib/files/images/src/lib.rs @@ -0,0 +1,22 @@ +//! images +//! +//! Even though the "image" library is the most widely used crate, if we declare a dependency here +//! we can update every downstream caller in unison. This will also let us collect common algorithms +//! and functions. +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod resize_preserving_aspect; diff --git a/crates/lib/files/images/src/resize_preserving_aspect.rs b/crates/lib/files/images/src/resize_preserving_aspect.rs new file mode 100644 index 0000000000..b0487eed46 --- /dev/null +++ b/crates/lib/files/images/src/resize_preserving_aspect.rs @@ -0,0 +1,81 @@ +use image::DynamicImage; +use image::imageops::FilterType; + +pub fn resize_preserving_aspect(source_image: &DynamicImage, new_width: u32, new_height: u32) -> DynamicImage { + source_image.resize(new_width, new_height, FilterType::Lanczos3) +} + +#[cfg(test)] +mod tests { + use image::DynamicImage; + + use errors::AnyhowResult; + use testing::test_file_path::test_file_path; + + use crate::resize_preserving_aspect::resize_preserving_aspect; + + fn open_test_image(filename: &str) -> AnyhowResult { + let path = test_file_path(filename)?; + Ok(image::open(path)?) + } + + fn open_tall_image() -> AnyhowResult { + let image = open_test_image("test_data/image/mochi.jpg").unwrap(); + assert!(image.width() < image.height()); + assert_eq!(image.width(), 753); + assert_eq!(image.height(), 1000); + Ok(image) + } + + fn open_wide_image() -> AnyhowResult { + let image = open_test_image("test_data/image/juno.jpg").unwrap(); + assert!(image.width() > image.height()); + assert_eq!(image.width(), 1000); + assert_eq!(image.height(), 750); + Ok(image) + } + + #[test] + fn downsize_wide_image_preserving_aspect() { + let image = open_wide_image().unwrap(); + assert!(image.width() > image.height()); + + let resized = resize_preserving_aspect(&image, 500, 500); + assert!(resized.width() > resized.height()); + assert_eq!(resized.width(), 500); + assert_eq!(resized.height(), 375); + } + + #[test] + fn downsize_tall_image_preserving_aspect() { + let image = open_tall_image().unwrap(); + assert!(image.width() < image.height()); + + let resized = resize_preserving_aspect(&image, 500, 500); + assert!(resized.width() < resized.height()); + assert_eq!(resized.width(), 377); + assert_eq!(resized.height(), 500); + } + + #[test] + fn enlarge_wide_image_preserving_aspect() { + let image = open_wide_image().unwrap(); + assert!(image.width() > image.height()); + + let resized = resize_preserving_aspect(&image, 1200, 1200); + assert!(resized.width() > resized.height()); + assert_eq!(resized.width(), 1200); + assert_eq!(resized.height(), 900); + } + + #[test] + fn enlarge_tall_image_preserving_aspect() { + let image = open_tall_image().unwrap(); + assert!(image.width() < image.height()); + + let resized = resize_preserving_aspect(&image, 1200, 1200); + assert!(resized.width() < resized.height()); + assert_eq!(resized.width(), 904); + assert_eq!(resized.height(), 1200); + } +} diff --git a/crates/lib/testing/Cargo.toml b/crates/lib/testing/Cargo.toml new file mode 100644 index 0000000000..24a2283604 --- /dev/null +++ b/crates/lib/testing/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "testing" +edition = "2018" +version = "0.0.1" +authors = [ + "Brandon Thomas ", + "Brandon Thomas ", +] +publish = false + +[lib] +# NB: Public "testing" crate is used by the "swc" project as an internal project. +# It isn't a crate meant to be consumed by others, so we'll use it internally. +name = "testing" +path = "src/lib.rs" + +[dependencies] + +# Internal +errors = { path = "../errors" } + +# External +# None yet + +[dev-dependencies] +# None yet diff --git a/crates/lib/testing/src/lib.rs b/crates/lib/testing/src/lib.rs new file mode 100644 index 0000000000..3726de55a9 --- /dev/null +++ b/crates/lib/testing/src/lib.rs @@ -0,0 +1,26 @@ +//! testing +//! +//! This crate collects some of our common code around test setup and execution. +//! In the future we may have several such crates that deal with specific functions (http serving, +//! mocks, database, etc.). This crate is meant for general testing concerns. It will also import +//! and re-export the most common testing libraries we use. +//! +//! (The public "testing" crate is used for another project's internal tooling and isn't meant for +//! public consumption.) +//! + +// Never allow these +#![forbid(private_in_public)] +#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things + +// Okay to toggle +#![forbid(unreachable_patterns)] +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +// Always allow +#![allow(dead_code)] +#![allow(non_snake_case)] + +pub mod test_file_path; \ No newline at end of file diff --git a/crates/lib/testing/src/test_file_path.rs b/crates/lib/testing/src/test_file_path.rs new file mode 100644 index 0000000000..76c4c4ce97 --- /dev/null +++ b/crates/lib/testing/src/test_file_path.rs @@ -0,0 +1,38 @@ +use std::path::{Path, PathBuf}; +use errors::{anyhow, AnyhowResult}; + +/// Refer to test files using the path from repo root and return a canonical path. +/// +/// NB: The path must exist or the canonicalization will return an error! +/// +/// (If each library had to deal with this problem separately, they would have to concern +/// themselves with the number of directories between the Cargo.toml project declaration +/// and the test directory, ie. this differs based on code nestedness.) +pub fn test_file_path>(path_from_repo_root: P) -> AnyhowResult { + // https://doc.rust-lang.org/cargo/reference/environment-variables.html + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let file = path_from_repo_root + .as_ref() + .to_str() + .ok_or(anyhow!("path cannot convert to string"))?; + path.push(format!("../../../{}", file)); + Ok(path.canonicalize()?) +} + +#[cfg(test)] +mod tests { + use super::test_file_path; + + #[test] + fn assert_canonicalized() { + // NB: We can't assert the exact paths since each machine (dev, CI) differs, + // but we can make assertions + let str_path = "test_data/audio/flac/zelda_ocarina_small_item.flac"; + let path = test_file_path(str_path).unwrap(); + + assert!(path.is_absolute()); + assert!(path.exists()); + assert!(path.ends_with(str_path)); + assert_eq!(path.canonicalize().unwrap(), path); + } +} diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs index 22a49e091a..fb0e9a0cc4 100644 --- a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/mod.rs @@ -2,5 +2,6 @@ pub mod download_audio_file; pub mod download_image_file; pub mod model_downloaders; pub mod process_job; +pub mod resize_image; pub mod sad_talker_inference_command; pub mod validate_job; \ No newline at end of file diff --git a/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/resize_image.rs b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/resize_image.rs new file mode 100644 index 0000000000..9e59dce92c --- /dev/null +++ b/crates/service/job/inference_job/src/job/job_types/lipsync/sad_talker/resize_image.rs @@ -0,0 +1,8 @@ +use std::path::Path; +use errors::AnyhowResult; + +pub fn resize_image, Q: AsRef>(input_file: P, output_file: Q) -> AnyhowResult<()> { + + + +} diff --git a/test_data/image/juno.jpg b/test_data/image/juno.jpg new file mode 100644 index 0000000000..b8f148b49a Binary files /dev/null and b/test_data/image/juno.jpg differ diff --git a/test_data/image/mochi.jpg b/test_data/image/mochi.jpg new file mode 100644 index 0000000000..243ca3cd8e Binary files /dev/null and b/test_data/image/mochi.jpg differ