Merge pull request #61 from storytold/testing-and-images-libs

Libraries for testing and image processing
This commit is contained in:
Brandon Thomas authored and GitHub committed 2023-10-02 16:21:28 -04:00
commit dbae9803e2
13 files changed
+448 -3

No files matched your search

+3
View File
@@ -3,6 +3,7 @@
<words>
<w>arpabet</w>
<w>backfill</w>
<w>canonicalized</w>
<w>cashapp</w>
<w>cheermote</w>
<w>cidrs</w>
@@ -22,6 +23,8 @@
<w>hparams</w>
<w>huggingface</w>
<w>lipsync</w>
<w>mochi</w>
<w>nestedness</w>
<w>pathbuf</w>
<w>replayable</w>
<w>retryably</w>
Generated
+212 -1
View File
@@ -965,6 +965,12 @@ dependencies = [
"url_config",
]
[[package]]
name = "bit_field"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61"
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -1268,6 +1274,12 @@ dependencies = [
"bitflags",
]
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "combine"
version = "3.8.1"
@@ -1562,6 +1574,12 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crunchy"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
name = "crypto-bigint"
version = "0.3.2"
@@ -2111,6 +2129,22 @@ version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]]
name = "exr"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1e481eb11a482815d3e9d618db8c42a93207134662873809335a92327440c18"
dependencies = [
"bit_field",
"flume",
"half",
"lebe",
"miniz_oxide",
"rayon-core",
"smallvec 1.10.0",
"zune-inflate",
]
[[package]]
name = "fastrand"
version = "1.7.0"
@@ -2120,6 +2154,15 @@ dependencies = [
"instant",
]
[[package]]
name = "fdeflate"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d329bdeac514ee06249dabc27877490f17f5d371ec693360768b838e19f3ae10"
dependencies = [
"simd-adler32",
]
[[package]]
name = "filesys"
version = "0.0.1"
@@ -2154,6 +2197,7 @@ checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"pin-project 1.0.7",
"spin 0.9.4",
]
@@ -2400,8 +2444,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
]
[[package]]
name = "gif"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80792593675e051cf94a4b111980da2ba60d4a83e43e0048c5693baab3977045"
dependencies = [
"color_quant",
"weezl",
]
[[package]]
@@ -2454,6 +2510,15 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0"
dependencies = [
"crunchy",
]
[[package]]
name = "hashbrown"
version = "0.12.3"
@@ -2788,6 +2853,35 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "image"
version = "0.24.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f3dfdbdd72063086ff443e297b61695500514b1e41095b6fb9a5ab48a70a711"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"exr",
"gif",
"jpeg-decoder",
"num-rational 0.4.1",
"num-traits",
"png",
"qoi",
"tiff",
]
[[package]]
name = "images"
version = "0.0.1"
dependencies = [
"errors",
"image",
"log",
"testing",
]
[[package]]
name = "indexmap"
version = "1.9.2"
@@ -2982,6 +3076,15 @@ dependencies = [
"libc",
]
[[package]]
name = "jpeg-decoder"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e"
dependencies = [
"rayon",
]
[[package]]
name = "js-sys"
version = "0.3.61"
@@ -3037,6 +3140,12 @@ dependencies = [
"spin 0.5.2",
]
[[package]]
name = "lebe"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]]
name = "lexical-sort"
version = "0.3.1"
@@ -3338,6 +3447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7"
dependencies = [
"adler",
"simd-adler32",
]
[[package]]
@@ -3457,6 +3567,15 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "nanorand"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
dependencies = [
"getrandom 0.2.8",
]
[[package]]
name = "native-tls"
version = "0.2.10"
@@ -3548,7 +3667,7 @@ dependencies = [
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-rational 0.1.42",
"num-traits",
]
@@ -3635,6 +3754,17 @@ dependencies = [
"rustc-serialize",
]
[[package]]
name = "num-rational"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.15"
@@ -4049,6 +4179,19 @@ version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]]
name = "png"
version = "0.17.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd75bf2d8dd3702b9707cdbc56a5b9ef42cec752eb8b3bafc01234558442aa64"
dependencies = [
"bitflags",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
]
[[package]]
name = "ppv-lite86"
version = "0.2.10"
@@ -4118,6 +4261,15 @@ dependencies = [
"unicase",
]
[[package]]
name = "qoi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001"
dependencies = [
"bytemuck",
]
[[package]]
name = "quick-error"
version = "1.2.3"
@@ -4273,6 +4425,26 @@ dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rayon"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
dependencies = [
"crossbeam-deque",
"crossbeam-utils 0.8.16",
]
[[package]]
name = "rdrand"
version = "0.4.0"
@@ -4973,6 +5145,12 @@ dependencies = [
"libc",
]
[[package]]
name = "simd-adler32"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]]
name = "siphasher"
version = "0.3.10"
@@ -5749,6 +5927,13 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
[[package]]
name = "testing"
version = "0.0.1"
dependencies = [
"errors",
]
[[package]]
name = "textwrap"
version = "0.11.0"
@@ -5793,6 +5978,17 @@ dependencies = [
"num_cpus",
]
[[package]]
name = "tiff"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d172b0f4d3fba17ba89811858b9d3d97f928aece846475bbda076ca46736211"
dependencies = [
"flate2",
"jpeg-decoder",
"weezl",
]
[[package]]
name = "tiger"
version = "0.1.0"
@@ -6979,6 +7175,12 @@ dependencies = [
"uuid 0.8.2",
]
[[package]]
name = "weezl"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb"
[[package]]
name = "widestring"
version = "0.4.3"
@@ -7289,3 +7491,12 @@ dependencies = [
"libc",
"pkg-config",
]
[[package]]
name = "zune-inflate"
version = "0.2.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02"
dependencies = [
"simd-adler32",
]
+4 -2
View File
@@ -16,7 +16,6 @@ members = [
# Shared libraries
"crates/lib/actix_helpers",
"crates/lib/api_clients/cloud_storage",
#"crates/lib/api_clients/fakeyou_client",
"crates/lib/api_clients/google_drive_common",
"crates/lib/api_clients/twitch_common",
"crates/lib/bootstrap",
@@ -34,6 +33,7 @@ members = [
"crates/lib/files/archives",
"crates/lib/files/filesys",
"crates/lib/files/hashing",
"crates/lib/files/images",
"crates/lib/files/media",
"crates/lib/files/mimetypes",
"crates/lib/idempotency",
@@ -41,13 +41,15 @@ members = [
"crates/lib/mysql_queries",
"crates/lib/redis_common",
"crates/lib/server_environment",
#"crates/lib/sqlite_queries",
"crates/lib/sqlx_mysql_helpers",
"crates/lib/subprocess_common",
"crates/lib/testing",
"crates/lib/tts_common",
"crates/lib/url_config",
"crates/lib/user_input_common",
"crates/lib/web_scrapers",
#"crates/lib/api_clients/fakeyou_client",
#"crates/lib/sqlite_queries",
# Programs: Desktop apps
#"crates/programs/aichatbot-sidecar",
+27
View File
@@ -0,0 +1,27 @@
[package]
name = "images"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
# NB: Public "images" crate doesn't exist; "image" crate is the heavily used public library,
# so perhaps this naming scheme causes confusion (?)
name = "images"
path = "src/lib.rs"
[dependencies]
# Internal
errors = { path = "../../errors" }
# External
image = "0.24.1"
log = "0.4.14"
[dev-dependencies]
testing = { path = "../../testing" }
+22
View File
@@ -0,0 +1,22 @@
//! images
//!
//! Even though the "image" library is the most widely used crate, if we declare a dependency here
//! we can update every downstream caller in unison. This will also let us collect common algorithms
//! and functions.
//!
// Never allow these
#![forbid(private_in_public)]
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
pub mod resize_preserving_aspect;
@@ -0,0 +1,81 @@
use image::DynamicImage;
use image::imageops::FilterType;
pub fn resize_preserving_aspect(source_image: &DynamicImage, new_width: u32, new_height: u32) -> DynamicImage {
source_image.resize(new_width, new_height, FilterType::Lanczos3)
}
#[cfg(test)]
mod tests {
use image::DynamicImage;
use errors::AnyhowResult;
use testing::test_file_path::test_file_path;
use crate::resize_preserving_aspect::resize_preserving_aspect;
fn open_test_image(filename: &str) -> AnyhowResult<DynamicImage> {
let path = test_file_path(filename)?;
Ok(image::open(path)?)
}
fn open_tall_image() -> AnyhowResult<DynamicImage> {
let image = open_test_image("test_data/image/mochi.jpg").unwrap();
assert!(image.width() < image.height());
assert_eq!(image.width(), 753);
assert_eq!(image.height(), 1000);
Ok(image)
}
fn open_wide_image() -> AnyhowResult<DynamicImage> {
let image = open_test_image("test_data/image/juno.jpg").unwrap();
assert!(image.width() > image.height());
assert_eq!(image.width(), 1000);
assert_eq!(image.height(), 750);
Ok(image)
}
#[test]
fn downsize_wide_image_preserving_aspect() {
let image = open_wide_image().unwrap();
assert!(image.width() > image.height());
let resized = resize_preserving_aspect(&image, 500, 500);
assert!(resized.width() > resized.height());
assert_eq!(resized.width(), 500);
assert_eq!(resized.height(), 375);
}
#[test]
fn downsize_tall_image_preserving_aspect() {
let image = open_tall_image().unwrap();
assert!(image.width() < image.height());
let resized = resize_preserving_aspect(&image, 500, 500);
assert!(resized.width() < resized.height());
assert_eq!(resized.width(), 377);
assert_eq!(resized.height(), 500);
}
#[test]
fn enlarge_wide_image_preserving_aspect() {
let image = open_wide_image().unwrap();
assert!(image.width() > image.height());
let resized = resize_preserving_aspect(&image, 1200, 1200);
assert!(resized.width() > resized.height());
assert_eq!(resized.width(), 1200);
assert_eq!(resized.height(), 900);
}
#[test]
fn enlarge_tall_image_preserving_aspect() {
let image = open_tall_image().unwrap();
assert!(image.width() < image.height());
let resized = resize_preserving_aspect(&image, 1200, 1200);
assert!(resized.width() < resized.height());
assert_eq!(resized.width(), 904);
assert_eq!(resized.height(), 1200);
}
}
+26
View File
@@ -0,0 +1,26 @@
[package]
name = "testing"
edition = "2018"
version = "0.0.1"
authors = [
"Brandon Thomas <bt@brand.io>",
"Brandon Thomas <echelon@gmail.com>",
]
publish = false
[lib]
# NB: Public "testing" crate is used by the "swc" project as an internal project.
# It isn't a crate meant to be consumed by others, so we'll use it internally.
name = "testing"
path = "src/lib.rs"
[dependencies]
# Internal
errors = { path = "../errors" }
# External
# None yet
[dev-dependencies]
# None yet
+26
View File
@@ -0,0 +1,26 @@
//! testing
//!
//! This crate collects some of our common code around test setup and execution.
//! In the future we may have several such crates that deal with specific functions (http serving,
//! mocks, database, etc.). This crate is meant for general testing concerns. It will also import
//! and re-export the most common testing libraries we use.
//!
//! (The public "testing" crate is used for another project's internal tooling and isn't meant for
//! public consumption.)
//!
// Never allow these
#![forbid(private_in_public)]
#![forbid(unused_must_use)] // NB: It's unsafe to not close/check some things
// Okay to toggle
#![forbid(unreachable_patterns)]
#![forbid(unused_imports)]
#![forbid(unused_mut)]
#![forbid(unused_variables)]
// Always allow
#![allow(dead_code)]
#![allow(non_snake_case)]
pub mod test_file_path;
+38
View File
@@ -0,0 +1,38 @@
use std::path::{Path, PathBuf};
use errors::{anyhow, AnyhowResult};
/// Refer to test files using the path from repo root and return a canonical path.
///
/// NB: The path must exist or the canonicalization will return an error!
///
/// (If each library had to deal with this problem separately, they would have to concern
/// themselves with the number of directories between the Cargo.toml project declaration
/// and the test directory, ie. this differs based on code nestedness.)
pub fn test_file_path<P: AsRef<Path>>(path_from_repo_root: P) -> AnyhowResult<PathBuf> {
// https://doc.rust-lang.org/cargo/reference/environment-variables.html
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
let file = path_from_repo_root
.as_ref()
.to_str()
.ok_or(anyhow!("path cannot convert to string"))?;
path.push(format!("../../../{}", file));
Ok(path.canonicalize()?)
}
#[cfg(test)]
mod tests {
use super::test_file_path;
#[test]
fn assert_canonicalized() {
// NB: We can't assert the exact paths since each machine (dev, CI) differs,
// but we can make assertions
let str_path = "test_data/audio/flac/zelda_ocarina_small_item.flac";
let path = test_file_path(str_path).unwrap();
assert!(path.is_absolute());
assert!(path.exists());
assert!(path.ends_with(str_path));
assert_eq!(path.canonicalize().unwrap(), path);
}
}
@@ -2,5 +2,6 @@ pub mod download_audio_file;
pub mod download_image_file;
pub mod model_downloaders;
pub mod process_job;
pub mod resize_image;
pub mod sad_talker_inference_command;
pub mod validate_job;
@@ -0,0 +1,8 @@
use std::path::Path;
use errors::AnyhowResult;
pub fn resize_image<P: AsRef<Path>, Q: AsRef<Path>>(input_file: P, output_file: Q) -> AnyhowResult<()> {
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 164 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 250 KiB