daniel543x/craft-libs
1
0 Fork 0
mirror of https://github.com/storytold/craft-libs.git synced 2026-11-13 22:12:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #259 from aiongg/i18n-batch

Browse Source 
i18n batch: isolation, region, sizes, folding and sorting, ICU4X numbers, number parsing, craft-fonts-build
This commit is contained in:
aiongg authored and GitHub committed 2026-10-23 22:42:02 -04:00
2 parents 2c6b3b5392 ec65b95a65
commit 53837472d6
45 files changed
+5098 -91
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/i18n.yml
+2 -2
View File
@@ -25,5 +25,5 @@ jobs:
with:
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-i18n --all-targets --locked -- -D warnings
- run: cargo test -p craft-i18n --locked -- --nocapture
- run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings
- run: cargo test -p craft-i18n --features collate --locked -- --nocapture
ATTRIBUTION.md
+1
View File
@@ -9,6 +9,7 @@ author, source and license. Library code and original assets are under the repos
| `crates/cms/profiles/photocraft-coated-cmyk.icc` | PhotoCraft contributors; generated by `craft_cms::synth::coated_cmyk` | [PhotoCraft `crates/cms`](https://github.com/storytold/photocraft/tree/b3222ee8e7e6bcb4799c37f904c595f20f9baf5e/crates/cms) | CC0-1.0 |
| `crates/cms/profiles/craft-coated-fogra39.icc` | craft-libs contributors; generated by `crates/cms/tests/fogra39.rs` from the FOGRA39 data below | this repository | CC0-1.0 |
| `crates/cms/tests/data/FOGRA39.txt` (unmodified) | Fogra | [Fogra characterisation data](https://fogra.org/en/downloads/work-tools/characterisation-data), `FOGRA39_to_FOGRA60_v2.zip` | Fogra terms: free use and redistribution, unmodified, with Fogra identified as the source; "FOGRA39" identifies the reference data only and implies no certification, approval or endorsement by Fogra (full text in [NOTICE](NOTICE)) |
| `crates/i18n/data/collation.postcard` | Unicode, Inc. (ICU 78.1 collation data, CLDR 48.2.1); generated by `crates/i18n/data/generate-collation.sh` | [ICU](https://github.com/unicode-org/icu/releases/tag/release-78.1rc) | Unicode-3.0 |
Original assets are covered by the repository licence.
Cargo.lock
Generated
+264 -35
View File
@@ -624,6 +624,15 @@ dependencies = [
"x509-cert",
]
[[package]]
name = "cobs"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "codespan-reporting"
version = "0.13.1"
@@ -786,15 +795,43 @@ dependencies = [
name = "craft-docpath"
version = "0.1.0"
[[package]]
name = "craft-fonts-build"
version = "0.1.0"
dependencies = [
"craft-fonts-build-consumer",
"skera",
"skrifa 0.48.0",
]
[[package]]
name = "craft-fonts-build-consumer"
version = "0.1.0"
dependencies = [
"craft-fonts-build",
]
[[package]]
name = "craft-i18n"
version = "0.1.0"
dependencies = [
"intl_pluralrules",
"fixed_decimal",
"icu_collator",
"icu_decimal",
"icu_locale_core",
"icu_locale_fallback",
"icu_normalizer",
"icu_plurals",
"icu_properties",
"icu_provider",
"icu_provider_adapters",
"icu_provider_blob",
"log",
"objc2-foundation 0.3.2",
"sys-locale",
"unic-langid",
"unicode-bidi",
"windows-registry",
"writeable",
]
[[package]]
@@ -1252,6 +1289,18 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "embedded-io"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"
[[package]]
name = "embedded-io"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d"
[[package]]
name = "encoding_rs"
version = "0.8.42"
@@ -1324,7 +1373,7 @@ dependencies = [
"ecolor",
"emath",
"epaint_default_fonts",
"font-types 0.12.6",
"font-types 0.12.7",
"harfrust",
"log",
"nohash-hasher",
@@ -1438,6 +1487,17 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
[[package]]
name = "fixed_decimal"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79c3c892f121fff406e5dd6b28c1b30096b95111c30701a899d4f2b18da6d1bd"
dependencies = [
"displaydoc",
"smallvec",
"writeable",
]
[[package]]
name = "flagset"
version = "0.4.7"
@@ -1509,9 +1569,9 @@ dependencies = [
[[package]]
name = "font-types"
version = "0.12.6"
version = "0.12.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c92d9081c7ab3bd83ef1eecdfad090f3d966e3ce74106779664114cfe1a6274"
checksum = "835ae9d781464a79026cde387f9cbc650c9a4909d7cf7e658b127e3c35d2f35d"
dependencies = [
"bytemuck",
]
@@ -2064,6 +2124,24 @@ dependencies = [
"cc",
]
[[package]]
name = "icu_collator"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08984ed58ac439ebf3e13d2cf26b0c46a60afcd21721c1d14087c0b240344dda"
dependencies = [
"icu_collections",
"icu_locale_core",
"icu_normalizer",
"icu_properties",
"icu_provider",
"serde",
"smallvec",
"utf16_iter",
"utf8_iter",
"zerovec",
]
[[package]]
name = "icu_collections"
version = "2.3.0"
@@ -2072,12 +2150,30 @@ checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
dependencies = [
"displaydoc",
"potential_utf",
"serde",
"utf8_iter",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
name = "icu_decimal"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9eb8f655bba2d0c0459e43a5b0e6cd3cd388109898fb3d85d048165e8b0abd08"
dependencies = [
"displaydoc",
"fixed_decimal",
"icu_locale_core",
"icu_pattern",
"icu_plurals",
"icu_provider",
"serde",
"writeable",
"zerovec",
]
[[package]]
name = "icu_locale_core"
version = "2.3.0"
@@ -2086,11 +2182,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
dependencies = [
"displaydoc",
"litemap",
"serde",
"tinystr",
"writeable",
"zerovec",
]
[[package]]
name = "icu_locale_fallback"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9"
dependencies = [
"icu_locale_core",
"icu_locale_fallback_data",
"icu_provider",
"potential_utf",
"tinystr",
"zerovec",
]
[[package]]
name = "icu_locale_fallback_data"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8"
[[package]]
name = "icu_normalizer"
version = "2.3.0"
@@ -2101,7 +2218,11 @@ dependencies = [
"icu_normalizer_data",
"icu_properties",
"icu_provider",
"serde",
"smallvec",
"utf16_iter",
"utf8_iter",
"write16",
"zerovec",
]
@@ -2111,6 +2232,32 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
[[package]]
name = "icu_pattern"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b3db611446bd92c246b03e4aa0bafa71ddb194af431ddbb42b9d70e4bfce1a0"
dependencies = [
"displaydoc",
"either",
"serde",
"writeable",
"zerovec",
]
[[package]]
name = "icu_plurals"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e475e6766ef87b1d3c1f97be6363995b0bfaeddbc789671615df598a97ff0593"
dependencies = [
"displaydoc",
"fixed_decimal",
"icu_provider",
"serde",
"zerovec",
]
[[package]]
name = "icu_properties"
version = "2.3.0"
@@ -2122,6 +2269,7 @@ dependencies = [
"icu_locale_core",
"icu_properties_data",
"icu_provider",
"serde",
"zerotrie",
"zerovec",
]
@@ -2140,6 +2288,9 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
dependencies = [
"displaydoc",
"icu_locale_core",
"postcard",
"serde",
"stable_deref_trait",
"writeable",
"yoke",
"zerofrom",
@@ -2147,6 +2298,30 @@ dependencies = [
"zerovec",
]
[[package]]
name = "icu_provider_adapters"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df887bc4f55dbc773c7c45a0c7dfa75be886f7b7812830a71ab7271cc1801820"
dependencies = [
"icu_locale_fallback",
"icu_provider",
]
[[package]]
name = "icu_provider_blob"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5916b44fac319d3f58e4d8d805a627df19e27937b582cd99fcde3bb6b7ce1b0e"
dependencies = [
"icu_provider",
"postcard",
"serde",
"writeable",
"zerotrie",
"zerovec",
]
[[package]]
name = "idna"
version = "1.1.0"
@@ -2214,15 +2389,6 @@ dependencies = [
"hashbrown 0.17.1",
]
[[package]]
name = "intl_pluralrules"
version = "7.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972"
dependencies = [
"unic-langid",
]
[[package]]
name = "ipnet"
version = "2.12.2"
@@ -2467,6 +2633,9 @@ name = "litemap"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
dependencies = [
"serde_core",
]
[[package]]
name = "litrs"
@@ -3334,12 +3503,26 @@ dependencies = [
"portable-atomic",
]
[[package]]
name = "postcard"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24"
dependencies = [
"cobs",
"embedded-io 0.4.0",
"embedded-io 0.6.1",
"serde",
]
[[package]]
name = "potential_utf"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
dependencies = [
"serde_core",
"writeable",
"zerovec",
]
@@ -3618,7 +3801,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "046a7d674daf459825b32f5062056d6882db0d2f5a479fbd76ccfc870ac18709"
dependencies = [
"bytemuck",
"font-types 0.12.6",
"font-types 0.12.7",
"once_cell",
]
@@ -3629,10 +3812,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5a0cf4bb6cbce4c29756270d189a5e400ac05e91192161d83a0bcb6e68492e8"
dependencies = [
"bytemuck",
"font-types 0.12.6",
"font-types 0.12.7",
"once_cell",
]
[[package]]
name = "read-fonts"
version = "0.45.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4edc8db0d972de88e53a085b1f01fb3fbdae7bb0cb7a0ce2b2f8285a412fe3ef"
dependencies = [
"bytemuck",
"foldhash",
"font-types 0.12.7",
"once_cell",
"yoke",
]
[[package]]
name = "redox_syscall"
version = "0.4.1"
@@ -4333,6 +4529,18 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256"
[[package]]
name = "skera"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "531045338e36ad05b9652912a47ba38a935d0eaa0293c3e33d8112cfa414c14b"
dependencies = [
"hashbrown 0.17.1",
"skrifa 0.48.0",
"thiserror 2.0.21",
"write-fonts 0.54.1",
]
[[package]]
name = "skrifa"
version = "0.42.1"
@@ -4363,6 +4571,16 @@ dependencies = [
"read-fonts 0.44.0",
]
[[package]]
name = "skrifa"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5213b4825bac3587046330052d61bfac7bd3bd0d203c758d7c2ef726fcea034"
dependencies = [
"bytemuck",
"read-fonts 0.45.1",
]
[[package]]
name = "slab"
version = "0.4.12"
@@ -4522,7 +4740,7 @@ dependencies = [
"kurbo 0.13.1",
"rustc-hash 2.1.3",
"skrifa 0.42.1",
"write-fonts",
"write-fonts 0.48.1",
]
[[package]]
@@ -5002,24 +5220,6 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
[[package]]
name = "unic-langid"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05"
dependencies = [
"unic-langid-impl",
]
[[package]]
name = "unic-langid-impl"
version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658"
dependencies = [
"tinystr",
]
[[package]]
name = "unicode-bidi"
version = "0.3.18"
@@ -5129,6 +5329,12 @@ dependencies = [
"xmlwriter",
]
[[package]]
name = "utf16_iter"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
[[package]]
name = "utf8_iter"
version = "1.0.4"
@@ -6079,11 +6285,31 @@ dependencies = [
"read-fonts 0.39.2",
]
[[package]]
name = "write-fonts"
version = "0.54.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55716cfa7e8bcdb90289e07dfac478ccb2e3ec0f9a604fe682c7a03a7984d705"
dependencies = [
"font-types 0.12.7",
"log",
"read-fonts 0.45.1",
]
[[package]]
name = "write16"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
[[package]]
name = "writeable"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
dependencies = [
"either",
]
[[package]]
name = "x11rb"
@@ -6347,8 +6573,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
dependencies = [
"displaydoc",
"litemap",
"serde_core",
"yoke",
"zerofrom",
"zerovec",
]
[[package]]
Cargo.toml
+1
View File
@@ -24,6 +24,7 @@ thiserror = "2"
craft-atomic = { path = "crates/atomic" }
craft-cms = { path = "crates/cms" }
craft-docpath = { path = "crates/docpath" }
craft-fonts-build = { path = "crates/fonts-build" }
craft-i18n = { path = "crates/i18n" }
craft-media = { path = "crates/media" }
craft-print = { path = "crates/print" }
README.md
+3
View File
@@ -37,6 +37,9 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
- [`craft-docpath`](crates/docpath/src/lib.rs): decides whether a path or `file:` URL stored in a
document may be opened. Paths stored in documents are followed only when they are local or on the
document's own network share; other network paths, device paths and non-file URLs are refused.
- [`craft-fonts-build`](crates/fonts-build/README.md): embeds the optional craft-fonts faces
from an app's `build.rs` (`CRAFT_FONTS_DIR`, `CRAFT_FONTS_REQUIRED`, script filters) as a table
of `EmbeddedFace`s. No dependencies; the app picks the faces.
- [`craft-i18n`](crates/i18n/README.md): translations over the embedded craft-i18n catalogs:
lookup with fallback (`pt-br` → `pt` → English), plurals, language names, the system locale and
a pseudo locale. The catalogs stay a fully optional build input; apps keep `tl!` and their
crates/fonts-build/Cargo.toml
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "craft-fonts-build"
description = "Embeds the optional craft-fonts faces from an app's build.rs, plus the EmbeddedFace type the generated table uses."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[features]
# Cut the `role ui` faces to the catalogs' characters (`Embed::subset_to`). Enable it only in an
# app's `[build-dependencies]`, so the subsetter stays out of the app itself.
subset = ["dep:skera", "dep:skrifa"]
[dependencies]
skera = { version = "0.8", optional = true }
skrifa = { version = "0.48", default-features = false, features = ["std"], optional = true }
[dev-dependencies]
craft-fonts-build-consumer = { path = "tests/consumer" }
[lints]
workspace = true
crates/fonts-build/README.md
+168
View File
@@ -0,0 +1,168 @@
# craft-fonts-build
Embeds the optional [craft-fonts](https://github.com/storytold/craft-fonts) faces from an app's
`build.rs`, and defines `EmbeddedFace`, the plain data type of the generated table. No
dependencies without the `subset` feature, and no egui, so crates below the UI layer (text engines,
exporters) can use it.
Apps never list craft-fonts in `Cargo.toml`: Cargo would fetch the font repository for every
build. The build reads a local checkout named by `CRAFT_FONTS_DIR` instead.
| Environment | Result |
|---|---|
| `CRAFT_FONTS_DIR` unset or empty | Empty table. |
| `CRAFT_FONTS_DIR` = a craft-fonts checkout | Every selected face of `fonts/manifest.txt`, embedded with `include_bytes!`. |
| `CRAFT_FONTS_DIR` set, but no manifest or a missing font file | `cargo::warning`, empty table. |
| `CRAFT_FONTS_REQUIRED=1` (exactly `1`; other values are ignored) | Error instead when the directory is unset, unreadable, has no selected face, or has no face for a script passed to `scripts`. |
Give `CRAFT_FONTS_DIR` as an absolute path: Cargo runs build scripts in the package's directory.
## Using it
`Cargo.toml` (same `rev` as the app's other craft-libs crates). The `subset` feature goes on the
build-dependency only: it brings in the subsetter ([skera](https://crates.io/crates/skera) and the
fontations crates), which the app itself never needs. Cargo's resolver 2 and later (edition 2021
and later) keep build-dependency features apart from the app's, so the dependency stays without
it.
```toml
[dependencies]
craft-fonts-build = { git = "https://github.com/storytold/craft-libs", rev = "…" }
[build-dependencies]
craft-fonts-build = { git = "https://github.com/storytold/craft-libs", rev = "…", features = ["subset"] }
```
`build.rs` (FilmCraft's `crates/text/build.rs`, in full):
```rust
use std::path::PathBuf;
fn main() {
println!("cargo::rerun-if-changed=build.rs");
let wasm = craft_fonts_build::target_is_wasm();
let embedded = craft_fonts_build::Embed::new()
// CJK faces only as the interface faces, cut to the characters FilmCraft's catalogs use.
.subset_to(catalogs("filmcraft"))
// Web builds have a size budget: embed only the Japanese interface face there.
.filter(move |face| !wasm || face.has_script("Jpan"))
.write("craft_fonts.rs");
if let Err(e) = embedded {
println!("cargo::error={e}");
}
}
/// Every locale's shared and app catalog in the craft-i18n checkout `CRAFT_I18N_DIR` names, the
/// files the app's translations embed; none when it is unset.
fn catalogs(app: &str) -> Vec<PathBuf> {
println!("cargo::rerun-if-env-changed=CRAFT_I18N_DIR");
let Some(dir) = std::env::var_os("CRAFT_I18N_DIR").filter(|dir| !dir.is_empty()) else { return Vec::new() };
let locales = PathBuf::from(dir).join("locales");
println!("cargo::rerun-if-changed={}", locales.display());
let mut paths: Vec<PathBuf> = std::fs::read_dir(&locales)
.into_iter()
.flatten()
.flatten()
.flat_map(|locale| ["common.tsv".to_owned(), format!("{app}.tsv")].map(|file| locale.path().join(file)))
.filter(|path| path.is_file())
.collect();
paths.sort();
paths
}
```
The crate:
```rust
use craft_fonts_build::EmbeddedFace;
/// craft-fonts faces; empty unless built with `CRAFT_FONTS_DIR`.
pub static CRAFT_FONTS: &[EmbeddedFace] = include!(concat!(env!("OUT_DIR"), "/craft_fonts.rs"));
```
`Embed::new()` selects every face except the sources of the interface faces (`role` `ui`, see
below), and `Embed::documents(&["Noto Sans", …])` only the families it names (see
[Document fonts](#document-fonts)). The builder narrows the set:
- `scripts(&["Jpan", "Arab", "Hant"])` keeps faces with any of these script tags. A tag matches
itself and its region forms (`Hant` matches `Hant-HK`). With `CRAFT_FONTS_REQUIRED=1`, each tag
must match at least one face.
- `filter(|entry| …)` keeps faces by any manifest field (family, style, licence, index).
- `for_ui()` keeps CJK faces only as the interface faces (see below).
- `subset_to(catalogs)` (feature `subset`) cuts the interface faces to the catalogs' characters;
it implies `for_ui()`.
- `check_text(paths)` warns about characters in these files that no embedded face covers.
- `embedded_elsewhere(&["Noto Sans Math", …])` names families another table of the app embeds,
so `check_text` and the catalog check count their characters as covered.
- `dir(path)` and `required(bool)` override the two environment variables.
`write` returns the embedded `ManifestEntry`s, for build-time checks such as "the interface face
is present". The generated file is a slice expression of `EmbeddedFace { family, style, scripts,
index, bytes }` literals; any struct with those fields and that name in scope works too.
`EmbeddedFace` itself has `name()`, `has_script(tag)`, `is_cjk()` and `weight()`.
## Document fonts
An app that draws document text (titles, layouts, vector text) embeds the families its documents
use, and only those, in a table of its own:
```rust
let embedded = craft_fonts_build::Embed::documents(&["Noto Sans", "Shippori Mincho"])
// The interface table holds the CJK cuts and the symbol faces.
.embedded_elsewhere(&["Noto Sans Math", "Noto Sans Symbols 2"])
.write("document_fonts.rs");
```
Each named family brings all its styles, as whole files: a CJK document face takes 5–17 MB, so a
release embeds exactly the faces the app's documents need. A family named here is embedded even
when it is also an interface face's source (`BIZ UDPGothic` brings its Regular too). A name the
manifest lacks is an `Error::UnknownFamilies` listing the manifest's families: a build error with
`CRAFT_FONTS_REQUIRED=1`, and a `cargo::warning` with an empty table otherwise, as for any
checkout that can't be used. `scripts` and `filter` narrow the families further, for example to
their Regular faces.
## Interface faces
The full CJK faces take 5–17 MB each. For the interface, craft-fonts marks one Regular TrueType
face per CJK script with the role `ui`: `Noto Sans SC UI` (`Hans`), `Noto Sans TC UI` (`Hant`),
`Noto Sans HK UI` (`Hant-HK`), `Noto Sans KR UI` (`Kore`) and BIZ UDPGothic Regular (`Jpan`). The
app's build cuts them to the characters its translation catalogs use; for FilmCraft's catalogs
the five cuts take about 3 MB together.
- `for_ui()` drops every face with a CJK script tag (`Hans`, `Hant`, `Hant-HK`, `Jpan`, `Kore`,
`Hani`) that is not an interface face, and keeps the rest. Document text embeds the full faces
it names with `Embed::documents` instead. Without `for_ui()` or `subset_to`, the interface
faces' sources are left out unless `Embed::documents` names their family.
- `subset_to(catalogs)` cuts each interface face to the characters the catalogs use from U+2E80 up
(CJK and later blocks, without emoji, variation selectors and tags), plus CJK punctuation
(U+3000–303F), kana (U+3040–30FF, U+31F0–31FF) and full-width forms (U+FF00–FFEF), and keeps
every layout feature, so vertical forms (`vert`) still work. Every face gets the characters of
every catalog, so a Japanese label in a Chinese catalog still draws. The cuts are written to
`OUT_DIR/craft-fonts-ui/` and embedded from there; the same inputs give the same bytes, for
reproducible releases. Cutting all five faces takes well under a second.
- Without `subset_to`, or with an empty catalog list, `for_ui()` leaves the interface faces out
with a `cargo::warning`: a full CJK face is never embedded for the interface by accident. A
checkout whose manifest has no interface face is a `cargo::warning` too. With `CRAFT_FONTS_REQUIRED=1` that
makes `scripts(&["Hant"])` an error; with catalogs, an interface face satisfies it.
- An interface face with CFF outlines is an error (`Error::NotTrueType`), even when not required:
the subsetter cuts TrueType (`glyf`) faces correctly, but not CFF ones.
The catalogs come from craft-i18n. An app's build reads them from the checkout `CRAFT_I18N_DIR`
names (craft-i18n's `docs/integration.md`); the fonts build script reads the same variable, as in
the example above. Catalogs kept in the app's own repository are passed by path, relative to the
package directory.
A character the faces lack still shows: craft-ui's system font provider draws it with an installed
font. The build says so: for each catalog, and each file passed to `check_text`, it prints one
`cargo::warning` with the count and the first few characters from U+2E80 up that no embedded face
covers. Since the cuts hold every catalog character their sources have, these are the characters
the sources lack. Faces the app embeds in another table count as covering their characters when
named with `embedded_elsewhere`; they are read whole from the checkout, so a face cut elsewhere is
exact only for the catalogs it was cut to. It also reruns the build script when a file changes.
## The manifest
One face per line, eight or nine fields separated by `" | "`; craft-fonts' `docs/integration.md`
defines them. Lines starting with `#` are comments. A file field ending in `#<n>` names face `n` of
a font collection (`fonts/noto-sans-cjk/NotoSansCJK.ttc#2`); without it the index is 0. The ninth
field, `role`, is `ui` on interface faces and absent elsewhere. Fields after the ninth are ignored.
crates/fonts-build/src/cmap.rs
+118
View File
@@ -0,0 +1,118 @@
//! Just enough of an OpenType reader to ask which characters a face maps: the `cmap` table's
//! Unicode subtables in formats 4 and 12, in a single font or one face of a collection.
/// The Unicode character maps of one face.
pub(crate) struct Cmap<'a> {
subtables: Vec<&'a [u8]>,
}
impl<'a> Cmap<'a> {
/// The cmap of face `index` of `font` (a `.ttf`/`.otf`, or a `.ttc` collection); `None` when the
/// file isn't a font or has no Unicode subtable in format 4 or 12.
pub(crate) fn read(font: &'a [u8], index: u32) -> Option<Self> {
let face = if font.get(0..4)? == b"ttcf" {
if index >= u32_at(font, 8)? {
return None;
}
usize::try_from(u32_at(font, 12 + 4 * usize::try_from(index).ok()?)?).ok()?
} else {
0
};
let tables = usize::from(u16_at(font, face + 4)?);
let record = (0..tables).map(|i| face + 12 + 16 * i).find(|&r| font.get(r..r + 4) == Some(b"cmap"))?;
let cmap = slice(font, u32_at(font, record + 8)?, u32_at(font, record + 12)?)?;
let mut subtables = Vec::new();
for i in 0..usize::from(u16_at(cmap, 2)?) {
let (platform, encoding) = (u16_at(cmap, 4 + 8 * i)?, u16_at(cmap, 6 + 8 * i)?);
// Unicode platform, or Windows Unicode BMP (1) and full repertoire (10).
if platform != 0 && !(platform == 3 && matches!(encoding, 1 | 10)) {
continue;
}
let start = usize::try_from(u32_at(cmap, 8 + 8 * i)?).ok()?;
let rest = cmap.get(start..)?;
// Bound each subtable by its own length so that reads can't stray into the next one.
// Some large CJK fonts declare a format 4 length past the end of the table, so that
// one is clamped; a format 12 length that doesn't fit is skipped.
let len = match u16_at(rest, 0) {
Some(4) => u16_at(rest, 2).map(|len| usize::from(len).min(rest.len())),
Some(12) => u32_at(rest, 4).and_then(|len| usize::try_from(len).ok()),
_ => None,
};
if let Some(subtable) = len.and_then(|len| rest.get(..len)) {
subtables.push(subtable);
}
}
(!subtables.is_empty()).then_some(Self { subtables })
}
/// Whether any Unicode subtable maps `c` to a glyph other than `.notdef`.
pub(crate) fn covers(&self, c: char) -> bool {
self.subtables.iter().any(|t| glyph(t, u32::from(c)).is_some_and(|g| g != 0))
}
}
fn glyph(subtable: &[u8], c: u32) -> Option<u32> {
match u16_at(subtable, 0)? {
4 => format4(subtable, u16::try_from(c).ok()?),
12 => format12(subtable, c),
_ => None,
}
}
fn format4(t: &[u8], c: u16) -> Option<u32> {
let seg_x2 = usize::from(u16_at(t, 6)?);
let (ends, starts, deltas, range_offsets) = (14, 16 + seg_x2, 16 + 2 * seg_x2, 16 + 3 * seg_x2);
// End codes are sorted: the first segment ending at or after `c` is the only candidate.
let segments = seg_x2 / 2;
let i = partition_point(segments, |i| u16_at(t, ends + 2 * i).is_some_and(|end| end < c));
if i == segments {
return None;
}
let start = u16_at(t, starts + 2 * i)?;
if c < start {
return None;
}
let delta = u16_at(t, deltas + 2 * i)?;
let range_offset_at = range_offsets + 2 * i;
let range_offset = usize::from(u16_at(t, range_offset_at)?);
if range_offset == 0 {
return Some(u32::from(c.wrapping_add(delta)));
}
let g = u16_at(t, range_offset_at + range_offset + 2 * usize::from(c - start))?;
Some(if g == 0 { 0 } else { u32::from(g.wrapping_add(delta)) })
}
fn format12(t: &[u8], c: u32) -> Option<u32> {
let groups = usize::try_from(u32_at(t, 12)?).ok()?;
// Groups are sorted by start code and don't overlap.
let i = partition_point(groups, |i| u32_at(t, 16 + 12 * i + 4).is_some_and(|end| end < c));
if i == groups {
return None;
}
let g = 16 + 12 * i;
let (start, end, glyph) = (u32_at(t, g)?, u32_at(t, g + 4)?, u32_at(t, g + 8)?);
(start..=end).contains(&c).then(|| glyph.wrapping_add(c - start))
}
/// The first index in `0..len` for which `before` is false, given that it is true for a prefix.
fn partition_point(len: usize, before: impl Fn(usize) -> bool) -> usize {
let (mut lo, mut hi) = (0, len);
while lo < hi {
let mid = lo + (hi - lo) / 2;
if before(mid) { lo = mid + 1 } else { hi = mid }
}
lo
}
fn slice(data: &[u8], offset: u32, len: u32) -> Option<&[u8]> {
let offset = usize::try_from(offset).ok()?;
data.get(offset..offset.checked_add(usize::try_from(len).ok()?)?)
}
fn u16_at(data: &[u8], at: usize) -> Option<u16> {
Some(u16::from_be_bytes(data.get(at..at.checked_add(2)?)?.try_into().ok()?))
}
fn u32_at(data: &[u8], at: usize) -> Option<u32> {
Some(u32::from_be_bytes(data.get(at..at.checked_add(4)?)?.try_into().ok()?))
}
crates/fonts-build/src/embed.rs
+516
View File
@@ -0,0 +1,516 @@
//! The build-script side: read `CRAFT_FONTS_DIR`, select faces, write the generated table.
#[cfg(feature = "subset")]
use std::collections::BTreeSet;
use std::collections::HashSet;
use std::ffi::OsString;
use std::fmt::{self, Write as _};
use std::io;
use std::path::{Path, PathBuf};
use crate::cmap::Cmap;
use crate::manifest::{ManifestEntry, ParseError, parse_manifest, script_matches};
const DIR_VAR: &str = "CRAFT_FONTS_DIR";
const REQUIRED_VAR: &str = "CRAFT_FONTS_REQUIRED";
const EMPTY_TABLE: &str = "&[]\n";
/// Whether the build script is building for wasm32 (web builds usually embed fewer faces).
pub fn target_is_wasm() -> bool {
std::env::var("CARGO_CFG_TARGET_ARCH").is_ok_and(|arch| arch == "wasm32")
}
type Filter = Box<dyn Fn(&ManifestEntry) -> bool>;
/// Which faces to embed, and from where. [`Embed::write`] does the work.
#[derive(Default)]
pub struct Embed {
scripts: Option<Vec<String>>,
filter: Option<Filter>,
dir: Option<PathBuf>,
required: Option<bool>,
ui: bool,
catalogs: Option<Vec<PathBuf>>,
texts: Vec<PathBuf>,
families: Option<Vec<String>>,
elsewhere: Vec<String>,
}
/// A selected face and the file the table embeds it from.
struct Face {
entry: ManifestEntry,
path: PathBuf,
index: u32,
/// An interface face to cut to the catalogs' characters before it is embedded.
#[cfg_attr(not(feature = "subset"), expect(dead_code))]
cut: bool,
}
impl Embed {
/// Every face except the interface faces' sources (`role` `ui`), from `CRAFT_FONTS_DIR`,
/// required when `CRAFT_FONTS_REQUIRED=1`. Narrow it with [`Embed::scripts`] and
/// [`Embed::filter`]; a table of document fonts starts from [`Embed::documents`] instead.
pub fn new() -> Self {
Self::default()
}
/// Every face of these families, whole, and nothing else: an app's document fonts. A family
/// is named as the manifest spells it (`"Noto Sans"`, `"Shippori Mincho"`) and contributes all
/// its styles, interface faces included (`"BIZ UDPGothic"` brings its Regular as well).
/// A name the manifest lacks is an [`Error::UnknownFamilies`], so the build fails with
/// `CRAFT_FONTS_REQUIRED=1` and warns and embeds nothing otherwise. Narrow it further with
/// [`Embed::scripts`] and [`Embed::filter`].
pub fn documents(families: &[&str]) -> Self {
Self { families: Some(families.iter().map(|f| (*f).to_owned()).collect()), ..Self::default() }
}
/// Embed only faces with at least one of these script tags (see
/// [`ManifestEntry::has_script`]). With `CRAFT_FONTS_REQUIRED=1`, each tag must match a face.
pub fn scripts(mut self, tags: &[&str]) -> Self {
self.scripts = Some(tags.iter().map(|t| (*t).to_owned()).collect());
self
}
/// Embed only faces for which `keep` returns true (applied after [`Embed::scripts`]).
pub fn filter(mut self, keep: impl Fn(&ManifestEntry) -> bool + 'static) -> Self {
self.filter = Some(Box::new(keep));
self
}
/// Embed faces for an app's interface: CJK faces only as the interface faces (`role` `ui` in
/// the manifest), cut to the characters of the catalogs given to `Embed::subset_to`. Other
/// CJK faces are dropped and the rest are kept. Without catalogs the interface faces are left
/// out too, with a `cargo::warning`, so a full CJK face is never embedded for the interface.
/// A manifest with no interface face is also a `cargo::warning` when that drops a CJK face.
/// Combines with [`Embed::scripts`] and [`Embed::filter`]; with `CRAFT_FONTS_REQUIRED=1`, an
/// interface face satisfies its script tags.
pub fn for_ui(mut self) -> Self {
self.ui = true;
self
}
/// Cut the interface faces to the characters these translation catalogs use, and embed the
/// cut faces, written to `OUT_DIR`. Implies [`Embed::for_ui`]. An empty list counts as no
/// catalogs: the interface faces are left out, with a `cargo::warning`.
///
/// The cut keeps the catalogs' characters from U+2E80 up (CJK and later blocks, without emoji,
/// variation selectors and tags), plus CJK punctuation (U+3000–303F), kana (U+3040–30FF,
/// U+31F0–31FF) and full-width forms (U+FF00–FFEF), with every layout feature. The same inputs
/// give the same bytes. A face with CFF outlines is an [`Error::NotTrueType`].
///
/// The catalogs are also checked as by [`Embed::check_text`]. Relative paths are under the
/// package directory; an unreadable catalog is a `cargo::warning`.
#[cfg(feature = "subset")]
pub fn subset_to<P: Into<PathBuf>>(mut self, catalogs: impl IntoIterator<Item = P>) -> Self {
self.ui = true;
self.catalogs.get_or_insert_default().extend(catalogs.into_iter().map(Into::into));
self
}
/// Check that the embedded faces cover these text files, such as translation catalogs: one
/// `cargo::warning` per file that has characters from U+2E80 up (CJK and later blocks, without
/// emoji and variation selectors) that no embedded face maps. Those characters fall back to
/// system fonts at run time. Relative paths are under the package directory. The check runs
/// only when a checkout was read.
pub fn check_text<P: Into<PathBuf>>(mut self, paths: impl IntoIterator<Item = P>) -> Self {
self.texts.extend(paths.into_iter().map(Into::into));
self
}
/// Families the app embeds in another table, such as its interface faces next to its document
/// fonts: [`Embed::check_text`] and the catalog check count their characters as covered. Each
/// is read from the checkout whole, so for a face cut elsewhere this is exact only for the
/// catalogs it was cut to. A name the manifest lacks is a `cargo::warning`.
pub fn embedded_elsewhere(mut self, families: &[&str]) -> Self {
self.elsewhere.extend(families.iter().map(|f| (*f).to_owned()));
self
}
/// Read this checkout instead of `CRAFT_FONTS_DIR`.
pub fn dir(mut self, dir: impl Into<PathBuf>) -> Self {
self.dir = Some(dir.into());
self
}
/// Override `CRAFT_FONTS_REQUIRED` (required only when it is exactly `1`).
pub fn required(mut self, required: bool) -> Self {
self.required = Some(required);
self
}
/// Write the face table to `out_file` and print the `cargo::` lines; returns the faces
/// embedded.
///
/// Without a checkout the table is empty. A checkout that can't be read is a
/// `cargo::warning` and an empty table, or an [`Error`] when required. The table is written
/// in every case, so on an error the build fails only on the message the caller prints.
pub fn write(&self, out_file: impl AsRef<Path>) -> Result<Vec<ManifestEntry>, Error> {
let env = Env { dir: std::env::var_os(DIR_VAR), required: std::env::var_os(REQUIRED_VAR), out_dir: std::env::var_os("OUT_DIR") };
let mut cargo = Vec::new();
let result = self.run(&env, out_file.as_ref(), &mut cargo);
for line in cargo {
println!("{line}");
}
result
}
pub(crate) fn run(&self, env: &Env, out_file: &Path, cargo: &mut Vec<String>) -> Result<Vec<ManifestEntry>, Error> {
cargo.push(format!("cargo::rerun-if-env-changed={DIR_VAR}"));
cargo.push(format!("cargo::rerun-if-env-changed={REQUIRED_VAR}"));
for text in self.checked_files() {
cargo.push(format!("cargo::rerun-if-changed={}", text.display()));
}
let out = match (out_file.is_absolute(), &env.out_dir) {
(true, _) => out_file.to_path_buf(),
(false, Some(out_dir)) => Path::new(out_dir).join(out_file),
(false, None) => return Err(Error::NoOutDir),
};
let required = self.required.unwrap_or_else(|| env.required.as_ref().is_some_and(|v| v == "1"));
let dir = self.dir.clone().or_else(|| env.dir.clone().filter(|d| !d.is_empty()).map(PathBuf::from));
let selected = match &dir {
None => Err(Error::NotSet),
Some(dir) => self.select(dir, cargo).and_then(|faces| if required { self.check_required(dir, faces) } else { Ok(faces) }),
};
let mut selected = selected;
if let Ok(faces) = &mut selected {
// A face that can't be cut is a broken manifest line: an error even when not required.
if let Err(e) = self.cut(faces, out.parent().unwrap_or(Path::new(""))) {
write(&out, EMPTY_TABLE)?;
return Err(e);
}
if let Some(dir) = &dir {
self.check_coverage(faces, dir, cargo);
}
}
let tabled = selected.and_then(|faces| Ok((table(&faces)?, faces.into_iter().map(|face| face.entry).collect())));
let (table, faces) = match tabled {
Ok(tabled) => tabled,
Err(e) if required => {
write(&out, EMPTY_TABLE)?;
return Err(e);
}
Err(Error::NotSet) => (EMPTY_TABLE.to_owned(), Vec::new()),
Err(e) => {
cargo.push(format!("cargo::warning=building without craft-fonts: {e}"));
(EMPTY_TABLE.to_owned(), Vec::new())
}
};
write(&out, &table)?;
Ok(faces)
}
/// The wanted manifest entries with their canonical file paths.
fn select(&self, dir: &Path, cargo: &mut Vec<String>) -> Result<Vec<Face>, Error> {
let manifest = dir.join("fonts/manifest.txt");
let text = std::fs::read_to_string(&manifest).map_err(|source| Error::Read { path: manifest.clone(), source })?;
// Cargo treats a missing watched file as changed, which would rerun the build every time.
cargo.push(format!("cargo::rerun-if-changed={}", manifest.display()));
let entries = parse_manifest(&text).map_err(|error| Error::Parse { manifest: manifest.clone(), error })?;
if let Some(families) = &self.families {
let unknown: Vec<String> = families.iter().filter(|f| !entries.iter().any(|e| &e.family == *f)).cloned().collect();
if !unknown.is_empty() {
let mut known: Vec<String> = Vec::new();
for entry in &entries {
if !known.contains(&entry.family) {
known.push(entry.family.clone());
}
}
return Err(Error::UnknownFamilies { manifest, families: unknown, known });
}
}
let has_ui = entries.iter().any(ManifestEntry::is_ui);
let no_catalogs = self.catalogs.as_ref().is_none_or(Vec::is_empty);
let mut faces = Vec::new();
let mut uncut = Vec::new();
let mut dropped_cjk = false;
for entry in entries {
// Interface faces are the sources of the cuts: embedded whole only when named.
let named = match &self.families {
Some(families) => families.contains(&entry.family),
None => self.ui || !entry.is_ui(),
};
if !named {
continue;
}
let wanted = self.scripts.as_ref().is_none_or(|tags| tags.iter().any(|t| entry.has_script(t)));
let ui_dropped = self.ui && entry.is_cjk() && !entry.is_ui();
if !wanted || ui_dropped || self.filter.as_ref().is_some_and(|keep| !keep(&entry)) {
dropped_cjk |= wanted && ui_dropped;
continue;
}
let cut = self.ui && entry.is_ui();
// An empty list would cut the faces to kana and punctuation, so it counts as none.
if cut && no_catalogs {
uncut.push(format!("{} {}", entry.family, entry.style));
continue;
}
let file = dir.join(&entry.file);
// `include_bytes!` resolves relative paths from the including source file.
let path = file.canonicalize().map_err(|source| Error::Read { path: file, source })?;
cargo.push(format!("cargo::rerun-if-changed={}", path.display()));
let index = entry.index;
faces.push(Face { entry, path, index, cut });
}
if !uncut.is_empty() {
let why = if self.catalogs.is_some() { "Embed::subset_to was given no translation catalogs" } else { "no translation catalogs" };
cargo.push(format!(
"cargo::warning=craft-fonts: not embedding the interface faces {}: {why}; pass the catalogs to \
Embed::subset_to (feature `subset`) to embed them cut to the catalogs' characters",
uncut.join(", ")
));
}
if dropped_cjk && !has_ui {
cargo.push(format!(
"cargo::warning=craft-fonts: not embedding any CJK face for the interface: {} has no interface face \
(role `ui`); use a craft-fonts checkout that has them",
manifest.display()
));
}
Ok(faces)
}
/// Replace each interface face with its cut, written to `out_dir`.
#[cfg(feature = "subset")]
fn cut(&self, faces: &mut [Face], out_dir: &Path) -> Result<(), Error> {
let Some(catalogs) = &self.catalogs else { return Ok(()) };
// `select` leaves the faces to cut out when there are no catalogs.
if !faces.iter().any(|face| face.cut) {
return Ok(());
}
let mut chars = BTreeSet::new();
// An unreadable catalog adds nothing here; `check_coverage` reports it.
for text in catalogs.iter().filter_map(|path| std::fs::read_to_string(path).ok()) {
chars.extend(text.chars().filter(|&c| needs_glyph(c)));
}
for (start, end) in UI_RANGES {
chars.extend(start..=end);
}
let dir = out_dir.join("craft-fonts-ui");
std::fs::create_dir_all(&dir).map_err(|source| Error::Write { path: dir.clone(), source })?;
for face in faces.iter_mut().filter(|face| face.cut) {
let source = std::fs::read(&face.path).map_err(|source| Error::Read { path: face.path.clone(), source })?;
let bytes = crate::subset::subset(&source, face.index, &chars).map_err(|failure| match failure {
crate::subset::Failure::NotTrueType => Error::NotTrueType { path: face.path.clone() },
crate::subset::Failure::Other(reason) => Error::Subset { path: face.path.clone(), reason },
})?;
let name: String =
format!("{}-{}", face.entry.family, face.entry.style).chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }).collect();
let out = dir.join(format!("{name}.ttf"));
// Rewriting identical bytes would make Cargo rebuild the crate that embeds them.
if std::fs::read(&out).ok().as_deref() != Some(&bytes[..]) {
std::fs::write(&out, &bytes).map_err(|source| Error::Write { path: out.clone(), source })?;
}
face.path = out;
face.index = 0;
}
Ok(())
}
/// Without the subsetter there are no catalogs, so [`Embed::select`] left out every face to cut.
#[cfg(not(feature = "subset"))]
fn cut(&self, _faces: &mut [Face], _out_dir: &Path) -> Result<(), Error> {
Ok(())
}
/// The catalogs, then the other [`Embed::check_text`] files.
fn checked_files(&self) -> Vec<&PathBuf> {
let mut files: Vec<&PathBuf> = self.catalogs.iter().flatten().collect();
for text in &self.texts {
if !files.contains(&text) {
files.push(text);
}
}
files
}
/// The manifest entries of the [`Embed::embedded_elsewhere`] families, with a
/// `cargo::warning` for a family the manifest lacks.
fn elsewhere(&self, dir: &Path, cargo: &mut Vec<String>) -> Vec<ManifestEntry> {
if self.elsewhere.is_empty() {
return Vec::new();
}
let manifest = dir.join("fonts/manifest.txt");
// `select` read and parsed this manifest already.
let entries = std::fs::read_to_string(&manifest).ok().and_then(|text| parse_manifest(&text).ok()).unwrap_or_default();
for family in self.elsewhere.iter().filter(|f| !entries.iter().any(|e| &e.family == *f)) {
cargo.push(format!("cargo::warning=craft-fonts: embedded_elsewhere names {family}, which {} lacks", manifest.display()));
}
entries.into_iter().filter(|e| self.elsewhere.contains(&e.family)).collect()
}
/// Warn about each catalog's and [`Embed::check_text`] file's characters that neither an
/// embedded face nor an [`Embed::embedded_elsewhere`] face covers.
fn check_coverage(&self, faces: &[Face], dir: &Path, cargo: &mut Vec<String>) {
let files = self.checked_files();
if files.is_empty() {
return;
}
let elsewhere = self.elsewhere(dir, cargo);
let fonts: Vec<(&ManifestEntry, u32, Option<Vec<u8>>)> = faces
.iter()
.map(|face| (&face.entry, face.index, std::fs::read(&face.path).ok()))
.chain(elsewhere.iter().map(|entry| (entry, entry.index, std::fs::read(dir.join(&entry.file)).ok())))
.collect();
let mut cmaps = Vec::new();
for (entry, index, bytes) in &fonts {
match bytes.as_deref().and_then(|bytes| Cmap::read(bytes, *index)) {
Some(cmap) => cmaps.push(cmap),
None => cargo
.push(format!("cargo::warning=craft-fonts: cannot read the character map of {} {} ({})", entry.family, entry.style, entry.file)),
}
}
for path in files {
let text = match std::fs::read_to_string(path) {
Ok(text) => text,
Err(e) => {
cargo.push(format!("cargo::warning=craft-fonts: cannot check {}: {e}", path.display()));
continue;
}
};
let mut seen = HashSet::new();
let missing: Vec<char> = text.chars().filter(|&c| needs_glyph(c) && seen.insert(c) && !cmaps.iter().any(|cmap| cmap.covers(c))).collect();
if missing.is_empty() {
continue;
}
const SHOWN: usize = 8;
let shown: String = missing.iter().take(SHOWN).collect();
let more = if missing.len() > SHOWN { " …" } else { "" };
cargo.push(format!(
"cargo::warning=craft-fonts: {} has {} character{} no embedded face covers ({shown}{more}); system fonts will draw them.",
path.display(),
missing.len(),
if missing.len() == 1 { "" } else { "s" },
));
}
}
fn check_required(&self, dir: &Path, faces: Vec<Face>) -> Result<Vec<Face>, Error> {
let manifest = dir.join("fonts/manifest.txt");
if faces.is_empty() {
return Err(Error::NoFaces { manifest });
}
let missing: Vec<String> = self
.scripts
.iter()
.flatten()
.filter(|tag| !faces.iter().any(|face| face.entry.scripts.iter().any(|s| script_matches(s, tag))))
.cloned()
.collect();
if missing.is_empty() { Ok(faces) } else { Err(Error::MissingScripts { manifest, scripts: missing }) }
}
}
/// Whether the interface faces need `c`: CJK and later blocks, except emoji and other pictographs
/// (the emoji faces draw them), variation selectors, tags and a byte order mark.
fn needs_glyph(c: char) -> bool {
c >= '\u{2E80}' && !matches!(c, '\u{FE00}'..='\u{FE0F}' | '\u{FEFF}' | '\u{1F000}'..='\u{1FFFF}' | '\u{E0000}'..)
}
/// Kept in every cut, whatever the catalogs use: CJK punctuation, kana and full-width forms.
#[cfg(feature = "subset")]
const UI_RANGES: [(char, char); 4] = [('\u{3000}', '\u{303F}'), ('\u{3040}', '\u{30FF}'), ('\u{31F0}', '\u{31FF}'), ('\u{FF00}', '\u{FFEF}')];
/// The environment a build script sees, separated out for tests.
pub(crate) struct Env {
pub dir: Option<OsString>,
pub required: Option<OsString>,
pub out_dir: Option<OsString>,
}
/// The generated source; `include_bytes!` takes a string, so a face under a path that isn't UTF-8
/// is an error rather than a lossy path the consumer's build can't find.
fn table(faces: &[Face]) -> Result<String, Error> {
let mut src = String::from("// Generated by craft-fonts-build. Do not edit.\n&[\n");
for Face { entry, path, index, .. } in faces {
let file = path.to_str().ok_or_else(|| Error::NonUtf8Path { path: path.clone() })?;
let scripts: Vec<String> = entry.scripts.iter().map(|s| format!("{s:?}")).collect();
let _ = writeln!(
src,
" EmbeddedFace {{ family: {:?}, style: {:?}, scripts: &[{}], index: {}, bytes: include_bytes!({:?}) }},",
entry.family,
entry.style,
scripts.join(", "),
index,
file,
);
}
src.push_str("]\n");
Ok(src)
}
fn write(path: &Path, contents: &str) -> Result<(), Error> {
std::fs::write(path, contents).map_err(|source| Error::Write { path: path.to_path_buf(), source })
}
/// Why the faces could not be embedded. Print it as `cargo::error={error}`.
#[derive(Debug)]
#[non_exhaustive]
pub enum Error {
/// `CRAFT_FONTS_REQUIRED=1` but `CRAFT_FONTS_DIR` is unset or empty.
NotSet,
/// The manifest or a font file it lists can't be read.
Read { path: PathBuf, source: io::Error },
/// The manifest has a malformed line.
Parse { manifest: PathBuf, error: ParseError },
/// Required, but no face in the manifest was selected.
NoFaces { manifest: PathBuf },
/// Required, but no face in the manifest has these requested script tags.
MissingScripts { manifest: PathBuf, scripts: Vec<String> },
/// The generated table can't be written.
Write { path: PathBuf, source: io::Error },
/// `out_file` is relative and `OUT_DIR` is unset (not running as a build script).
NoOutDir,
/// An interface face has CFF outlines, which can't be cut correctly; it must be TrueType.
NotTrueType { path: PathBuf },
/// An interface face can't be cut.
Subset { path: PathBuf, reason: String },
/// [`Embed::documents`] names families the manifest lacks; `known` lists the manifest's.
UnknownFamilies { manifest: PathBuf, families: Vec<String>, known: Vec<String> },
/// A face to embed has a path that isn't UTF-8, which `include_bytes!` can't name.
NonUtf8Path { path: PathBuf },
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NotSet => {
write!(f, "{REQUIRED_VAR}=1 but {DIR_VAR} is not: set {DIR_VAR} to the absolute path of a storytold/craft-fonts checkout")
}
Self::Read { path, source } => {
write!(f, "cannot read {} ({DIR_VAR} must be the absolute path of a storytold/craft-fonts checkout): {source}", path.display())
}
Self::Parse { manifest, error } => write!(f, "{}: {error}", manifest.display()),
Self::NoFaces { manifest } => write!(f, "{REQUIRED_VAR}=1 but no face in {} was selected", manifest.display()),
Self::MissingScripts { manifest, scripts } => {
write!(f, "{REQUIRED_VAR}=1 but {} has no face for {}", manifest.display(), scripts.join(", "))
}
Self::Write { path, source } => write!(f, "cannot write {}: {source}", path.display()),
Self::NoOutDir => write!(f, "OUT_DIR is not set: call craft-fonts-build from a build script, or pass an absolute out_file"),
Self::NotTrueType { path } => write!(
f,
"{} has CFF outlines, which craft-fonts-build can't subset correctly: an interface face (role ui) must be a TrueType (glyf) font",
path.display()
),
Self::Subset { path, reason } => write!(f, "cannot subset {}: {reason}", path.display()),
Self::UnknownFamilies { manifest, families, known } => write!(
f,
"{} has no family {}; Embed::documents names families as the manifest spells them: {}",
manifest.display(),
families.join(", "),
known.join(", ")
),
Self::NonUtf8Path { path } => write!(f, "cannot embed {}: include_bytes! needs a UTF-8 path", path.display()),
}
}
}
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Read { source, .. } | Self::Write { source, .. } => Some(source),
Self::Parse { error, .. } => Some(error),
_ => None,
}
}
}
crates/fonts-build/src/lib.rs
+125
View File
@@ -0,0 +1,125 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! Embed the optional [craft-fonts](https://github.com/storytold/craft-fonts) faces from an app's
//! `build.rs`.
//!
//! Apps never depend on craft-fonts in `Cargo.toml` (Cargo would fetch the font repository for
//! every build). The build reads a local checkout named by `CRAFT_FONTS_DIR` instead:
//!
//! - **Unset or empty:** the table is empty and the app uses its own and system fonts.
//! - **Set:** every selected face in `fonts/manifest.txt` is embedded with `include_bytes!`.
//! - **Set, but not readable as a checkout:** a `cargo::warning` and an empty table.
//! - **`CRAFT_FONTS_REQUIRED=1`** (release builds; any other value is ignored): a missing or
//! unreadable checkout, no selected face, or a requested script without a face is an [`Error`].
//!
//! Add this crate both as a build-dependency (for [`Embed`]) and as a dependency (for
//! [`EmbeddedFace`]). The build-dependency enables the `subset` feature for `Embed::subset_to`,
//! which cuts the CJK interface faces to the translation catalogs' characters; the dependency
//! leaves it off, so the subsetter is not part of the app. In `build.rs`'s `main`:
//!
//! ```ignore
//! println!("cargo::rerun-if-changed=build.rs");
//! let wasm = craft_fonts_build::target_is_wasm();
//! let catalogs: Vec<std::path::PathBuf> = /* the app's translation catalogs */;
//! let embedded = craft_fonts_build::Embed::new()
//! // CJK faces only as the interface faces, cut to the characters the catalogs use.
//! .subset_to(catalogs)
//! // Web builds have a size budget: embed only the Japanese interface face there.
//! .filter(move |face| !wasm || face.has_script("Jpan"))
//! .write("craft_fonts.rs");
//! if let Err(e) = embedded {
//! println!("cargo::error={e}");
//! }
//! ```
//!
//! and in the crate:
//!
//! ```ignore
//! use craft_fonts_build::EmbeddedFace;
//!
//! pub static CRAFT_FONTS: &[EmbeddedFace] = include!(concat!(env!("OUT_DIR"), "/craft_fonts.rs"));
//! ```
//!
//! The generated file is a slice expression of `EmbeddedFace { .. }` literals, so any struct with
//! the same fields and that name in scope works too.
//!
//! An app's document fonts go in a table of their own that names its families,
//! `Embed::documents(&["Noto Sans", "Shippori Mincho"])`: nothing is embedded for documents
//! unless the app names it.
mod cmap;
mod embed;
mod manifest;
#[cfg(feature = "subset")]
mod subset;
#[cfg(test)]
mod tests;
pub use embed::{Embed, Error, target_is_wasm};
pub use manifest::{CJK_SCRIPTS, ManifestEntry, ParseError, UI_ROLE, parse_manifest};
/// A craft-fonts face embedded by [`Embed`]: one line of `fonts/manifest.txt`.
#[derive(Clone, Copy)]
pub struct EmbeddedFace {
/// Family name, e.g. `"Noto Sans Arabic"`.
pub family: &'static str,
/// Style, e.g. `"Regular"` or `"Bold"`.
pub style: &'static str,
/// Script tags: ISO 15924 codes such as `Arab`, `Jpan`, `Hans`, `Hant`, optionally with a
/// region subtag (`Hant-HK` for Hong Kong forms, which have no ISO 15924 code).
pub scripts: &'static [&'static str],
/// Face index in a font collection (`.ttc`); `0` for a single font.
pub index: u32,
pub bytes: &'static [u8],
}
impl EmbeddedFace {
/// `"<family> <style>"`, e.g. `"Noto Sans Arabic Regular"`.
pub fn name(&self) -> String {
format!("{} {}", self.family, self.style)
}
/// Whether any script tag matches `tag`: `Hant` matches `Hant` and `Hant-HK`.
pub fn has_script(&self, tag: &str) -> bool {
self.scripts.iter().any(|s| manifest::script_matches(s, tag))
}
/// Whether the face has a CJK script tag ([`CJK_SCRIPTS`]).
pub fn is_cjk(&self) -> bool {
CJK_SCRIPTS.iter().any(|tag| self.has_script(tag))
}
/// CSS/OpenType weight implied by the style name (`Bold` → 700); 400 when it names none.
pub fn weight(&self) -> u16 {
let style = self.style.to_ascii_lowercase().replace([' ', '-'], "");
const WEIGHTS: [(&str, u16); 14] = [
("extralight", 200),
("ultralight", 200),
("semibold", 600),
("demibold", 600),
("extrabold", 800),
("ultrabold", 800),
("thin", 100),
("light", 300),
("medium", 500),
("bold", 700),
("black", 900),
("heavy", 900),
("regular", 400),
("book", 400),
];
WEIGHTS.iter().find(|(name, _)| style.contains(name)).map_or(400, |(_, weight)| *weight)
}
}
impl std::fmt::Debug for EmbeddedFace {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("EmbeddedFace")
.field("family", &self.family)
.field("style", &self.style)
.field("scripts", &self.scripts)
.field("index", &self.index)
.field("bytes", &format_args!("[{} bytes]", self.bytes.len()))
.finish()
}
}
crates/fonts-build/src/manifest.rs
+127
View File
@@ -0,0 +1,127 @@
//! craft-fonts' `fonts/manifest.txt`: one font face per line, eight or nine fields separated by
//! `" | "`:
//!
//! ```text
//! family | style | file | scripts | licence | licence file | sha256 | source [| role]
//! ```
//!
//! Blank lines and lines starting with `#` are skipped. `file` is relative to the checkout and may
//! end in `#<index>` to pick one face of a font collection (`.ttc`). The optional `role` is `ui` for
//! an interface face: a full Regular TrueType face that an app's build cuts to the characters of its
//! translation catalogs. Fields after the ninth are ignored, so a manifest that gains a field still
//! embeds.
use std::fmt;
/// One face listed in the manifest.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ManifestEntry {
/// Family name, as the font's `name` table spells it (e.g. `Noto Sans Arabic`).
pub family: String,
/// Style within the family (e.g. `Regular`, `Bold`).
pub style: String,
/// Font file, relative to the checkout, without the `#<index>` suffix.
pub file: String,
/// Face index in a font collection; `0` when the file field has no `#<index>`.
pub index: u32,
/// Script tags: ISO 15924 codes (`Jpan`, `Arab`), optionally with a region (`Hant-HK`).
pub scripts: Vec<String>,
/// SPDX licence identifier.
pub licence: String,
/// Licence text, relative to the checkout.
pub licence_file: String,
/// Hex SHA-256 of the font file.
pub sha256: String,
/// Where the file came from.
pub source: String,
/// The optional ninth field: `Some("ui")` for an interface face, `None` when the line has no
/// role.
pub role: Option<String>,
}
impl ManifestEntry {
/// Whether any of the face's script tags matches `tag`. `Hant` matches `Hant` and `Hant-HK`;
/// `Hant-HK` matches only `Hant-HK`.
pub fn has_script(&self, tag: &str) -> bool {
self.scripts.iter().any(|s| script_matches(s, tag))
}
/// Whether the face is an interface face (`role` is `ui`).
pub fn is_ui(&self) -> bool {
self.role.as_deref() == Some(UI_ROLE)
}
/// Whether the face has a CJK script tag ([`CJK_SCRIPTS`]).
pub fn is_cjk(&self) -> bool {
CJK_SCRIPTS.iter().any(|tag| self.has_script(tag))
}
}
/// The `role` of an interface face.
pub const UI_ROLE: &str = "ui";
/// Script tags of the CJK faces, which craft-fonts also ships as interface faces. `Hant` covers
/// `Hant-HK`.
pub const CJK_SCRIPTS: &[&str] = &["Hans", "Hant", "Jpan", "Kore", "Hani"];
/// `face_tag` is `wanted` or `wanted` plus a `-` subtag.
pub(crate) fn script_matches(face_tag: &str, wanted: &str) -> bool {
face_tag.strip_prefix(wanted).is_some_and(|rest| rest.is_empty() || rest.starts_with('-'))
}
/// A manifest line that could not be read.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ParseError {
/// 1-based line number.
pub line: usize,
pub reason: String,
}
impl fmt::Display for ParseError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "manifest line {}: {}", self.line, self.reason)
}
}
impl std::error::Error for ParseError {}
/// Parse a whole manifest.
pub fn parse_manifest(text: &str) -> Result<Vec<ManifestEntry>, ParseError> {
let mut entries = Vec::new();
for (i, raw) in text.lines().enumerate() {
let line = i + 1;
let trimmed = raw.trim();
if trimmed.is_empty() || trimmed.starts_with('#') {
continue;
}
let fields: Vec<&str> = trimmed.split(" | ").map(str::trim).collect();
let [family, style, file, scripts, licence, licence_file, sha256, source, rest @ ..] = fields.as_slice() else {
return Err(ParseError { line, reason: format!("expected 8 fields separated by \" | \", found {}", fields.len()) });
};
for (name, value) in [("family", family), ("style", style), ("file", file), ("scripts", scripts)] {
if value.is_empty() {
return Err(ParseError { line, reason: format!("the {name} field is empty") });
}
}
let (file, index) = match file.rsplit_once('#') {
Some((path, index)) => {
let index = index.parse().map_err(|_| ParseError { line, reason: format!("bad face index in {file:?}") })?;
(path, index)
}
None => (*file, 0),
};
entries.push(ManifestEntry {
family: (*family).to_owned(),
style: (*style).to_owned(),
file: file.to_owned(),
index,
scripts: scripts.split(',').map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned).collect(),
licence: (*licence).to_owned(),
licence_file: (*licence_file).to_owned(),
sha256: (*sha256).to_owned(),
source: (*source).to_owned(),
role: rest.first().filter(|role| !role.is_empty()).map(|role| (*role).to_owned()),
});
}
Ok(entries)
}
crates/fonts-build/src/subset.rs
+45
View File
@@ -0,0 +1,45 @@
//! Cutting an interface face down to the characters the translation catalogs use, with skera.
use std::collections::BTreeSet;
use skera::{Plan, SubsetFlags, subset_font};
use skrifa::raw::collections::IntSet;
use skrifa::raw::types::NameId;
use skrifa::{FontRef, GlyphId, Tag};
/// Why a face could not be cut.
#[derive(Debug)]
pub(crate) enum Failure {
/// The face has CFF outlines: skera's CFF output draws the wrong glyphs.
NotTrueType,
Other(String),
}
/// Face `index` of `font` with the glyphs of those `chars` it maps and the glyphs its layout tables
/// reach from them (vertical forms, ligatures). Every layout feature and script is kept; the name
/// table keeps IDs 0–6 in US English, as fonttools' subsetter does. The same input gives the same
/// bytes.
pub(crate) fn subset(font: &[u8], index: u32, chars: &BTreeSet<char>) -> Result<Vec<u8>, Failure> {
let font = FontRef::from_index(font, index).map_err(|e| Failure::Other(e.to_string()))?;
if font.table_data(Tag::new(b"glyf")).is_none() {
return Err(Failure::NotTrueType);
}
let mut unicodes = IntSet::<u32>::empty();
unicodes.extend(chars.iter().map(|&c| u32::from(c)));
let mut name_ids = IntSet::<NameId>::empty();
name_ids.insert_range(NameId::new(0)..=NameId::new(6));
let mut languages = IntSet::<u16>::empty();
languages.insert(0x0409);
let plan = Plan::new(
&IntSet::<GlyphId>::empty(),
&unicodes,
&font,
SubsetFlags::default(),
&IntSet::<Tag>::empty(),
&IntSet::<Tag>::all(),
&IntSet::<Tag>::all(),
&name_ids,
&languages,
);
subset_font(&font, &plan).map_err(|e| Failure::Other(e.to_string()))
}
crates/fonts-build/src/tests.rs
+562
View File
@@ -0,0 +1,562 @@
use std::ffi::OsString;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::cmap::Cmap;
use crate::embed::Env;
use crate::{Embed, EmbeddedFace, Error, parse_manifest};
fn fixture() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture/craft-fonts")
}
/// A fresh scratch directory under the system temp dir.
struct Scratch(PathBuf);
impl Scratch {
fn new(name: &str) -> Self {
let dir = std::env::temp_dir().join(format!("craft-fonts-build-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
Self(dir)
}
}
impl Drop for Scratch {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.0);
}
}
struct Run {
result: Result<Vec<String>, Error>,
cargo: Vec<String>,
table: String,
/// `OUT_DIR`, which holds the table and the cut faces.
#[cfg_attr(not(feature = "subset"), expect(dead_code))]
out: Scratch,
}
/// Run `embed` with `CRAFT_FONTS_DIR` and `CRAFT_FONTS_REQUIRED` as given; returns the embedded
/// face names, the cargo lines and the generated table.
fn run(embed: Embed, dir: Option<&Path>, required: Option<&str>) -> Run {
static RUNS: AtomicUsize = AtomicUsize::new(0);
let scratch = Scratch::new(&format!("run-{}", RUNS.fetch_add(1, Ordering::Relaxed)));
let env = Env { dir: dir.map(OsString::from), required: required.map(OsString::from), out_dir: Some(scratch.0.clone().into()) };
let mut cargo = Vec::new();
let result = embed.run(&env, Path::new("craft_fonts.rs"), &mut cargo);
let table = std::fs::read_to_string(scratch.0.join("craft_fonts.rs")).unwrap();
Run { result: result.map(|faces| faces.iter().map(|f| format!("{} {}", f.family, f.style)).collect()), cargo, table, out: scratch }
}
#[test]
fn parses_the_manifest_with_collection_indices() {
let text = std::fs::read_to_string(fixture().join("fonts/manifest.txt")).unwrap();
let entries = parse_manifest(&text).unwrap();
assert_eq!(entries.len(), 6);
assert_eq!((entries[0].family.as_str(), entries[0].style.as_str()), ("Sans JP", "Regular"));
assert_eq!(entries[0].scripts, ["Jpan", "Latn"]);
assert_eq!((entries[0].file.as_str(), entries[0].index), ("fonts/sans-jp/SansJP-Regular.ttf", 0));
assert_eq!((entries[4].file.as_str(), entries[4].index), ("fonts/sans-cjk/SansCJK.ttc", 3));
assert_eq!(entries[4].scripts, ["Hant-HK", "Latn"]);
assert_eq!(entries[4].licence, "OFL-1.1");
let short = "# comment\n\nA | Regular | a.ttf | Latn\n";
let err = parse_manifest(short).unwrap_err();
assert_eq!(err.line, 3);
assert!(err.to_string().contains("expected 8 fields"), "{err}");
let bad_index = "A | Regular | a.ttc#x | Latn | OFL-1.1 | OFL.txt | 00 | src\n";
assert!(parse_manifest(bad_index).unwrap_err().to_string().contains("face index"));
let ui = "A UI | Regular | a.ttf | Hant | OFL-1.1 | OFL.txt | 00 | src | ui | future field\nA | Regular | a.ttf | Hant | OFL-1.1 | OFL.txt | 00 | src |\n";
let entries = parse_manifest(ui).unwrap();
assert_eq!(entries[0].role.as_deref(), Some("ui"));
assert!(entries[0].is_ui() && entries[0].is_cjk());
assert_eq!(entries[1].role, None);
assert!(!entries[1].is_ui());
assert!(entries.iter().all(|e| e.role.is_none() || e.is_ui()));
}
#[test]
fn unset_dir_writes_an_empty_table() {
let r = run(Embed::new(), None, None);
assert_eq!(r.result.unwrap(), Vec::<String>::new());
assert_eq!(r.table.trim(), "&[]");
assert!(r.cargo.contains(&"cargo::rerun-if-env-changed=CRAFT_FONTS_DIR".to_owned()));
assert!(r.cargo.contains(&"cargo::rerun-if-env-changed=CRAFT_FONTS_REQUIRED".to_owned()));
assert!(!r.cargo.iter().any(|l| l.starts_with("cargo::warning")));
// Only `1` means required.
for value in ["0", "true", "yes", " 1"] {
assert!(run(Embed::new(), None, Some(value)).result.is_ok(), "{value:?}");
}
assert!(run(Embed::new(), Some(Path::new("")), Some("")).result.is_ok());
}
#[test]
fn required_fails_without_a_usable_checkout() {
for dir in [None, Some(Path::new(""))] {
let r = run(Embed::new(), dir, Some("1"));
assert!(matches!(r.result, Err(Error::NotSet)), "{dir:?}");
assert_eq!(r.table.trim(), "&[]");
}
let err = run(Embed::new(), None, Some("1")).result.unwrap_err().to_string();
assert!(err.contains("CRAFT_FONTS_DIR") && err.contains("CRAFT_FONTS_REQUIRED"), "{err}");
let missing = Path::new("/nonexistent/craft-fonts");
assert!(matches!(run(Embed::new(), Some(missing), Some("1")).result, Err(Error::Read { .. })));
let r = run(Embed::new(), Some(missing), None);
assert!(r.result.unwrap().is_empty());
// Watching a missing manifest would rerun the build script on every build.
assert!(!r.cargo.iter().any(|l| l.starts_with("cargo::rerun-if-changed")), "{:?}", r.cargo);
assert!(r.cargo.iter().any(|l| l.starts_with("cargo::warning=building without craft-fonts")), "{:?}", r.cargo);
let scratch = Scratch::new("missing-file");
std::fs::create_dir_all(scratch.0.join("fonts")).unwrap();
std::fs::write(scratch.0.join("fonts/manifest.txt"), "A | Regular | fonts/a.ttf | Latn | OFL-1.1 | OFL.txt | 00 | src\n").unwrap();
assert!(matches!(run(Embed::new(), Some(&scratch.0), Some("1")).result, Err(Error::Read { .. })));
std::fs::write(scratch.0.join("fonts/manifest.txt"), "# no faces\n").unwrap();
assert!(matches!(run(Embed::new(), Some(&scratch.0), Some("1")).result, Err(Error::NoFaces { .. })));
}
#[cfg(unix)]
#[test]
fn rejects_a_checkout_under_a_non_utf8_path() {
use std::os::unix::ffi::OsStrExt;
let scratch = Scratch::new("non-utf8");
let dir = scratch.0.join(std::ffi::OsStr::from_bytes(b"craft-fonts-\xff"));
std::fs::create_dir_all(dir.join("fonts")).unwrap();
std::fs::copy(fixture().join("fonts/sans-arabic/SansArabic.ttf"), dir.join("fonts/a.ttf")).unwrap();
std::fs::write(dir.join("fonts/manifest.txt"), "A | Regular | fonts/a.ttf | Arab | OFL-1.1 | OFL.txt | 00 | src\n").unwrap();
let r = run(Embed::new(), Some(&dir), Some("1"));
assert!(matches!(r.result, Err(Error::NonUtf8Path { .. })), "{:?}", r.result);
assert_eq!(r.table.trim(), "&[]");
}
#[test]
fn embeds_the_selected_scripts() {
let r = run(Embed::new(), Some(&fixture()), Some("1"));
assert_eq!(r.result.unwrap().len(), 6);
let r = run(Embed::new().scripts(&["Jpan", "Hant"]), Some(&fixture()), Some("1"));
assert_eq!(r.result.unwrap(), ["Sans JP Regular", "Sans JP Bold", "Sans CJK TC Regular", "Sans CJK HK Regular"]);
let manifest = fixture().join("fonts/manifest.txt");
assert!(r.cargo.contains(&format!("cargo::rerun-if-changed={}", manifest.display())));
let jp = fixture().join("fonts/sans-jp/SansJP-Regular.ttf").canonicalize().unwrap();
assert!(r.cargo.contains(&format!("cargo::rerun-if-changed={}", jp.display())));
assert!(r.table.contains(&format!("include_bytes!({:?})", jp.display().to_string())), "{}", r.table);
assert!(r.table.contains("family: \"Sans CJK HK\", style: \"Regular\", scripts: &[\"Hant-HK\", \"Latn\"], index: 3"), "{}", r.table);
let r = run(Embed::new().scripts(&["Hant-HK"]).filter(|f| f.index != 0), Some(&fixture()), None);
assert_eq!(r.result.unwrap(), ["Sans CJK HK Regular"]);
let r = run(Embed::new().scripts(&["Arab", "Kore", "Geor"]), Some(&fixture()), Some("1"));
let Err(Error::MissingScripts { scripts, .. }) = r.result else { panic!("{:?}", r.result) };
assert_eq!(scripts, ["Kore", "Geor"]);
// Not required: whatever matched is embedded.
let r = run(Embed::new().scripts(&["Arab", "Kore"]), Some(&fixture()), None);
assert_eq!(r.result.unwrap(), ["Sans Arabic Regular"]);
}
#[test]
fn embedded_face_helpers() {
let face = |style: &'static str| EmbeddedFace { family: "Sans", style, scripts: &["Hant-HK", "Latn"], index: 0, bytes: b"" };
assert_eq!(face("Regular").name(), "Sans Regular");
assert!(face("Regular").has_script("Hant") && face("Regular").has_script("Hant-HK") && !face("Regular").has_script("Han"));
let weights: Vec<u16> = ["Regular", "Bold", "SemiBold", "Extra Light", "Black Italic", "Italic"].map(|s| face(s).weight()).into();
assert_eq!(weights, [400, 700, 600, 200, 900, 400]);
assert!(face("Regular").is_cjk());
assert!(!EmbeddedFace { scripts: &["Arab"], ..face("Regular") }.is_cjk());
}
#[test]
fn documents_embeds_only_the_named_families() {
let r = run(Embed::documents(&["Sans JP", "Sans Thai"]), Some(&fixture()), Some("1"));
assert_eq!(r.result.unwrap(), ["Sans JP Regular", "Sans JP Bold", "Sans Thai Regular"]);
let r = run(Embed::documents(&["Sans JP", "Sans Thai"]).filter(|f| f.style == "Bold"), Some(&fixture()), Some("1"));
assert_eq!(r.result.unwrap(), ["Sans JP Bold"]);
let r = run(Embed::documents(&[]), Some(&fixture()), None);
assert_eq!(r.result.unwrap(), Vec::<String>::new());
let r = run(Embed::documents(&["Sans JP", "Sans Jp", "Serif"]), Some(&fixture()), Some("1"));
let Err(Error::UnknownFamilies { families, .. }) = &r.result else { panic!("{:?}", r.result) };
assert_eq!(families, &["Sans Jp", "Serif"]);
let message = r.result.unwrap_err().to_string();
assert!(
message.contains("has no family Sans Jp, Serif") && message.ends_with("Sans JP, Sans Arabic, Sans CJK TC, Sans CJK HK, Sans Thai"),
"{message}"
);
assert_eq!(r.table.trim(), "&[]");
// Not required: a warning and an empty table, as for any checkout that can't be used.
let r = run(Embed::documents(&["Serif"]), Some(&fixture()), None);
assert!(r.result.as_ref().unwrap().is_empty());
assert!(warnings(&r).iter().any(|w| w.contains("building without craft-fonts") && w.contains("no family Serif")), "{:?}", r.cargo);
}
/// A font with only a `cmap` table: a Windows format 4 subtable with `bmp` segments `(first,
/// last, glyphs)`, mapped to glyphs 1.. when `glyphs` is `None` and through the glyph array
/// otherwise, and a format 12 subtable with `groups` `(first, last)` when there are any.
fn tiny_font(bmp: &[(u16, u16, Option<&[u16]>)], groups: &[(u32, u32)]) -> Vec<u8> {
let be16 = |v: &mut Vec<u8>, x: u16| v.extend_from_slice(&x.to_be_bytes());
let be32 = |v: &mut Vec<u8>, x: u32| v.extend_from_slice(&x.to_be_bytes());
let mut segments = bmp.to_vec();
segments.push((0xFFFF, 0xFFFF, None));
let n = u16::try_from(segments.len()).unwrap();
let (mut ends, mut starts, mut deltas, mut offsets, mut glyphs) = (vec![], vec![], vec![], vec![], vec![]);
for (i, &(first, last, ids)) in segments.iter().enumerate() {
be16(&mut ends, last);
be16(&mut starts, first);
match ids {
None => {
// The last segment maps 0xFFFF to glyph 0, as fonts do.
be16(&mut deltas, 1u16.wrapping_sub(if first == 0xFFFF { 0 } else { first }));
be16(&mut offsets, 0);
}
Some(ids) => {
be16(&mut deltas, 0);
be16(&mut offsets, 2 * (n - u16::try_from(i).unwrap()) + u16::try_from(glyphs.len()).unwrap());
ids.iter().for_each(|&g| be16(&mut glyphs, g));
}
}
}
let mut format4 = vec![];
for x in [4, 0, 0, 2 * n, 0, 0, 0] {
be16(&mut format4, x);
}
format4.extend(ends.iter().chain(&[0, 0]).chain(&starts).chain(&deltas).chain(&offsets).chain(&glyphs));
let len = u16::try_from(format4.len()).unwrap();
format4[2..4].copy_from_slice(&len.to_be_bytes());
let mut format12 = vec![];
be16(&mut format12, 12);
be16(&mut format12, 0);
be32(&mut format12, 16 + 12 * u32::try_from(groups.len()).unwrap());
be32(&mut format12, 0);
be32(&mut format12, u32::try_from(groups.len()).unwrap());
for &(first, last) in groups {
[first, last, 1].into_iter().for_each(|x| be32(&mut format12, x));
}
let subtables: Vec<(u16, Vec<u8>)> = [(1, format4), (10, format12)].into_iter().filter(|(enc, _)| *enc == 1 || !groups.is_empty()).collect();
let mut cmap = vec![];
be16(&mut cmap, 0);
be16(&mut cmap, u16::try_from(subtables.len()).unwrap());
let mut at = 4 + 8 * u32::try_from(subtables.len()).unwrap();
for (encoding, table) in &subtables {
[3, *encoding].into_iter().for_each(|x| be16(&mut cmap, x));
be32(&mut cmap, at);
at += u32::try_from(table.len()).unwrap();
}
subtables.iter().for_each(|(_, table)| cmap.extend(table));
let mut font = vec![];
be32(&mut font, 0x0001_0000);
for x in [1, 0, 0, 0] {
be16(&mut font, x);
}
font.extend_from_slice(b"cmap");
for x in [0, 28, u32::try_from(cmap.len()).unwrap()] {
be32(&mut font, x);
}
font.extend(cmap);
font
}
/// A `.ttc` of fonts made by [`tiny_font`].
fn tiny_collection(fonts: &[Vec<u8>]) -> Vec<u8> {
let header = 12 + 4 * fonts.len();
let mut ttc = b"ttcf\0\x01\0\0".to_vec();
ttc.extend_from_slice(&u32::try_from(fonts.len()).unwrap().to_be_bytes());
let mut body = vec![];
for font in fonts {
let start = u32::try_from(header + body.len()).unwrap();
ttc.extend_from_slice(&start.to_be_bytes());
let mut font = font.clone();
// The table offsets of a collection's faces count from the start of the file.
let offset = u32::from_be_bytes(font[20..24].try_into().unwrap()) + start;
font[20..24].copy_from_slice(&offset.to_be_bytes());
body.extend(font);
}
ttc.extend(body);
ttc
}
#[test]
fn cmap_reads_formats_4_and_12_and_collections() {
let font = tiny_font(&[(0x41, 0x5A, None), (0x4E00, 0x4E02, Some(&[5, 0, 7]))], &[(0x2_0000, 0x2_0001)]);
let cmap = Cmap::read(&font, 0).unwrap();
let covered: Vec<char> = "AZa一丁丂七\u{FFFF}\u{20000}\u{20001}\u{20002}".chars().filter(|&c| cmap.covers(c)).collect();
assert_eq!(covered, ['A', 'Z', '一', '丂', '\u{20000}', '\u{20001}']);
let bmp_in_format12 = tiny_font(&[], &[(0x4E00, 0x4E0F)]);
assert!(Cmap::read(&bmp_in_format12, 0).unwrap().covers('丁'));
let ttc = tiny_collection(&[tiny_font(&[(0x41, 0x41, None)], &[]), tiny_font(&[(0x4E00, 0x4E00, None)], &[])]);
let second = Cmap::read(&ttc, 1).unwrap();
assert!(second.covers('一') && !second.covers('A'));
assert!(Cmap::read(&ttc, 0).unwrap().covers('A'));
assert!(Cmap::read(&ttc, 2).is_none());
assert!(Cmap::read(b"fixture: Sans JP Regular", 0).is_none());
assert!(Cmap::read(&font[..40], 0).is_none());
}
#[test]
fn cmap_reads_each_subtable_within_its_declared_length() {
// 'A' takes its glyph from an empty glyph array, so the read lands on the format 12 subtable
// that follows the format 4 one.
let font = tiny_font(&[(0x41, 0x41, Some(&[]))], &[(0x2_0000, 0x2_0000)]);
let cmap = Cmap::read(&font, 0).unwrap();
assert!(!cmap.covers('A') && cmap.covers('\u{20000}'));
// A format 4 length past the end of the table is clamped rather than rejected. The format 4
// subtable starts after the 28-byte font header, the 4-byte cmap header and two 8-byte encoding
// records.
let mut font = tiny_font(&[(0x41, 0x5A, None)], &[(0x2_0000, 0x2_0000)]);
let format4 = 28 + 4 + 2 * 8;
let format12 = format4 + usize::from(u16::from_be_bytes(font[format4 + 2..format4 + 4].try_into().unwrap()));
font[format4 + 2..format4 + 4].copy_from_slice(&u16::MAX.to_be_bytes());
assert!(Cmap::read(&font, 0).unwrap().covers('A'));
// A format 12 group past the declared length doesn't count.
font[format12 + 4..format12 + 8].copy_from_slice(&16u32.to_be_bytes());
let cmap = Cmap::read(&font, 0).unwrap();
assert!(cmap.covers('A') && !cmap.covers('\u{20000}'));
}
/// A craft-fonts checkout in `scratch` with these `(family, scripts, role, font)` faces, all
/// Regular.
fn checkout(scratch: &Scratch, faces: &[(&str, &str, &str, Vec<u8>)]) -> PathBuf {
let dir = scratch.0.join("craft-fonts");
std::fs::create_dir_all(dir.join("fonts")).unwrap();
let mut manifest = String::new();
for (i, (family, scripts, role, font)) in faces.iter().enumerate() {
std::fs::write(dir.join(format!("fonts/{i}.ttf")), font).unwrap();
manifest += &format!("{family} | Regular | fonts/{i}.ttf | {scripts} | OFL-1.1 | fonts/OFL.txt | 00 | src | {role}\n");
}
std::fs::write(dir.join("fonts/manifest.txt"), manifest).unwrap();
dir
}
fn warnings(r: &Run) -> Vec<String> {
r.cargo.iter().filter(|l| l.starts_with("cargo::warning")).cloned().collect()
}
#[test]
fn for_ui_embeds_cjk_faces_only_as_cut_interface_faces() {
let scratch = Scratch::new("for-ui");
let latin = || tiny_font(&[(0x20, 0x7E, None)], &[]);
let dir = checkout(
&scratch,
&[
("Noto Sans TC", "Hant,Latn", "", latin()),
("Noto Sans TC UI", "Hant,Latn", "ui", latin()),
("Noto Sans HK UI", "Hant-HK,Latn", "ui", latin()),
("Noto Sans Arabic", "Arab", "", latin()),
],
);
// Without catalogs to cut them to, the interface faces are left out too.
let r = run(Embed::new().for_ui(), Some(&dir), None);
assert_eq!(r.result.as_ref().unwrap(), &["Noto Sans Arabic Regular"]);
let w = warnings(&r);
assert_eq!(w.len(), 1, "{w:#?}");
assert!(w[0].contains("not embedding the interface faces Noto Sans TC UI Regular, Noto Sans HK UI Regular"), "{}", w[0]);
assert!(w[0].contains("Embed::subset_to"), "{}", w[0]);
// A manifest without interface faces loses every CJK face, which is worth a warning.
let r = run(Embed::new().for_ui(), Some(&fixture()), None);
assert!(r.cargo.contains(&format!("cargo::rerun-if-changed={}", fixture().join("fonts/manifest.txt").display())), "{:?}", r.cargo);
let w = warnings(&r);
assert_eq!(w.len(), 1, "{w:#?}");
assert!(w[0].contains("not embedding any CJK face for the interface") && w[0].contains("role `ui`"), "{}", w[0]);
let r = run(Embed::new().for_ui().scripts(&["Hant", "Arab"]), Some(&dir), Some("1"));
let Err(Error::MissingScripts { scripts, .. }) = r.result else { panic!("{:?}", r.result) };
assert_eq!(scripts, ["Hant"]);
assert_eq!(r.table.trim(), "&[]");
// Without `for_ui` the interface faces' sources are left out, unless a document table names them.
let r = run(Embed::new().scripts(&["Hant"]), Some(&dir), Some("1"));
assert_eq!(r.result.as_ref().unwrap(), &["Noto Sans TC Regular"]);
assert_eq!(warnings(&r), Vec::<String>::new());
let r = run(Embed::documents(&["Noto Sans TC UI", "Noto Sans TC"]), Some(&dir), Some("1"));
assert_eq!(r.result.as_ref().unwrap(), &["Noto Sans TC Regular", "Noto Sans TC UI Regular"]);
assert!(r.table.contains("fonts/1.ttf"), "{}", r.table);
}
#[test]
fn check_text_warns_about_characters_no_face_covers() {
let scratch = Scratch::new("check-text");
let dir = checkout(
&scratch,
&[
("Noto Sans TC", "Hant,Latn", "", tiny_font(&[(0x20, 0x7E, None), (0x4E00, 0x4E03, None)], &[])),
("Noto Sans Arabic", "Arab", "", tiny_font(&[(0x600, 0x6FF, None)], &[])),
("Noto Sans SC", "Hans,Latn", "", tiny_font(&[(0x20, 0x7E, None), (0x3000, 0x9FFF, None)], &[])),
],
);
let catalogs = scratch.0.join("i18n");
std::fs::create_dir_all(&catalogs).unwrap();
std::fs::write(catalogs.join("zh-TW.ftl"), "\u{FEFF}open = 一丁万。\nclose = 丁万\u{FE0F} \u{1F600}\n").unwrap();
std::fs::write(catalogs.join("ar.ftl"), "open = فتح\nok = 一\n").unwrap();
let long: String = (0x4E10..0x4E20).filter_map(char::from_u32).collect();
std::fs::write(catalogs.join("long.ftl"), format!("x = {long}\n")).unwrap();
let paths = ["zh-TW.ftl", "ar.ftl", "long.ftl", "missing.ftl"].map(|f| catalogs.join(f));
let r = run(Embed::new().scripts(&["Hant", "Arab"]).check_text(paths.clone()), Some(&dir), None);
assert_eq!(r.result.as_ref().unwrap().len(), 2);
for path in &paths {
assert!(r.cargo.contains(&format!("cargo::rerun-if-changed={}", path.display())), "{:?}", r.cargo);
}
let w = warnings(&r);
assert_eq!(w.len(), 3, "{w:#?}");
assert!(w[0].contains("zh-TW.ftl has 2 characters no embedded face covers (万。); system fonts"), "{}", w[0]);
assert!(w[1].contains("long.ftl has 16 characters") && w[1].contains("(丐丑丒专且丕世丗 …)"), "{}", w[1]);
assert!(w[2].contains("cannot check") && w[2].contains("missing.ftl"), "{}", w[2]);
// The full face covers everything; without a checkout there is nothing to check.
let r = run(Embed::new().check_text(paths[..3].to_vec()), Some(&dir), None);
assert_eq!(warnings(&r), Vec::<String>::new());
let r = run(Embed::new().check_text(paths.clone()), None, None);
assert_eq!(warnings(&r), Vec::<String>::new());
assert!(r.cargo.contains(&format!("cargo::rerun-if-changed={}", paths[0].display())));
// Faces another table embeds count as covering their characters.
let r = run(Embed::new().scripts(&["Arab"]).embedded_elsewhere(&["Noto Sans SC"]).check_text(paths[..3].to_vec()), Some(&dir), None);
assert_eq!(warnings(&r), Vec::<String>::new());
let r = run(
Embed::new().scripts(&["Arab"]).embedded_elsewhere(&["Noto Sans TC", "Noto Sans Math"]).check_text(paths[..2].to_vec()),
Some(&dir),
None,
);
let w = warnings(&r);
assert_eq!(w.len(), 2, "{w:#?}");
assert!(w[0].contains("embedded_elsewhere names Noto Sans Math, which") && w[0].ends_with("manifest.txt lacks"), "{}", w[0]);
assert!(w[1].contains("zh-TW.ftl has 2 characters no embedded face covers (万。)"), "{}", w[1]);
// The fixture's "fonts" are text, so they cover nothing.
let r = run(Embed::new().scripts(&["Arab"]).check_text([&paths[1]]), Some(&fixture()), None);
let w = warnings(&r);
assert!(w[0].contains("cannot read the character map of Sans Arabic Regular"), "{w:#?}");
assert!(w[1].contains("ar.ftl has 1 character no embedded face covers (一)"), "{w:#?}");
}
#[cfg(feature = "subset")]
mod subset {
use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use super::{Run, Scratch, checkout, run, tiny_font, warnings};
use crate::cmap::Cmap;
use crate::{Embed, Error};
/// `fixture/subset/make-fonts.py` writes these.
fn tiny(file: &str) -> Vec<u8> {
std::fs::read(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixture/subset").join(file)).unwrap()
}
/// The BMP characters `font` maps.
fn mapped(font: &[u8]) -> Vec<u32> {
let cmap = Cmap::read(font, 0).unwrap();
('\0'..='\u{FFFF}').filter(|&c| cmap.covers(c)).map(u32::from).collect()
}
fn table<'a>(font: &'a [u8], tag: &[u8; 4]) -> Option<&'a [u8]> {
let u16_at = |at: usize| usize::from(u16::from_be_bytes([font[at], font[at + 1]]));
let u32_at = |at: usize| usize::try_from(u32::from_be_bytes(font[at..at + 4].try_into().unwrap())).unwrap();
let record = (0..u16_at(4)).map(|i| 12 + 16 * i).find(|&r| &font[r..r + 4] == tag)?;
Some(&font[u32_at(record + 8)..][..u32_at(record + 12)])
}
/// The feature tags of a GSUB or GPOS table.
fn features(table: &[u8]) -> BTreeSet<String> {
let list = usize::from(u16::from_be_bytes([table[6], table[7]]));
let count = usize::from(u16::from_be_bytes([table[list], table[list + 1]]));
(0..count).map(|i| String::from_utf8_lossy(&table[list + 2 + 6 * i..][..4]).into_owned()).collect()
}
fn glyph_count(font: &[u8]) -> u16 {
let maxp = table(font, b"maxp").unwrap();
u16::from_be_bytes([maxp[4], maxp[5]])
}
fn cut_file(r: &Run, family: &str) -> PathBuf {
r.out.0.join("craft-fonts-ui").join(format!("{}-Regular.ttf", family.replace(' ', "-")))
}
#[test]
fn subset_to_cuts_interface_faces_to_the_catalogs() {
let scratch = Scratch::new("subset");
let latin = || tiny_font(&[(0x20, 0x7E, None)], &[]);
let dir = checkout(
&scratch,
&[
("Noto Sans TC", "Hant,Latn", "", latin()),
("Noto Sans TC UI", "Hant,Latn", "ui", tiny("Tiny-glyf.ttf")),
("Noto Sans HK UI", "Hant-HK,Latn", "ui", tiny("Tiny-glyf.ttf")),
("Noto Sans Arabic", "Arab", "", latin()),
],
);
let tw = scratch.0.join("zh-TW.tsv");
std::fs::write(&tw, "Open\t一万\n").unwrap();
let ko = scratch.0.join("ko.tsv");
std::fs::write(&ko, "Open\t丁\u{FE0F} \u{1F600}\n").unwrap();
let r = run(Embed::new().subset_to([&tw]).scripts(&["Hant"]), Some(&dir), Some("1"));
assert_eq!(r.result.as_ref().unwrap(), &["Noto Sans TC UI Regular", "Noto Sans HK UI Regular"]);
let tc = cut_file(&r, "Noto Sans TC UI");
assert!(r.table.contains(&format!("index: 0, bytes: include_bytes!({:?})", tc.display().to_string())), "{}", r.table);
assert!(r.cargo.contains(&format!("cargo::rerun-if-changed={}", tw.display())), "{:?}", r.cargo);
let cut = std::fs::read(&tc).unwrap();
// The catalog's 一, plus the punctuation, kana and full-width forms the source has; not
// its Latin, nor the 丁 and 七 no catalog uses.
assert_eq!(mapped(&cut), [0x3001, 0x300C, 0x3042, 0x4E00, 0xFF01]);
assert!(table(&cut, b"glyf").is_some());
assert_eq!(features(table(&cut, b"GSUB").unwrap()), BTreeSet::from(["vert".to_owned()]));
// .notdef, five characters and the vertical form of U+300C.
assert_eq!(glyph_count(&cut), 7);
// The source lacks 万, so system fonts draw it.
let w = warnings(&r);
assert_eq!(w.len(), 1, "{w:#?}");
assert!(w[0].contains("zh-TW.tsv has 1 character no embedded face covers (万)"), "{}", w[0]);
// The same bytes on every run. Another catalog adds its characters, but no emoji or
// variation selector, and a `check_text` file is only checked.
let again = run(Embed::new().for_ui().subset_to([&tw]), Some(&dir), None);
assert_eq!(std::fs::read(cut_file(&again, "Noto Sans TC UI")).unwrap(), cut);
let r = run(Embed::new().subset_to([&tw, &ko]).check_text([scratch.0.join("zh-TW.tsv")]), Some(&dir), None);
assert_eq!(mapped(&std::fs::read(cut_file(&r, "Noto Sans TC UI")).unwrap()), [0x3001, 0x300C, 0x3042, 0x4E00, 0x4E01, 0xFF01]);
assert_eq!(warnings(&r).len(), 1, "{:#?}", warnings(&r));
// No catalogs (an app whose CRAFT_I18N_DIR is unset) leaves the faces out, not cut to kana.
let r = run(Embed::new().subset_to(Vec::<PathBuf>::new()), Some(&dir), None);
assert_eq!(r.result.as_ref().unwrap(), &["Noto Sans Arabic Regular"]);
let w = warnings(&r);
assert_eq!(w.len(), 1, "{w:#?}");
assert!(w[0].contains("Noto Sans TC UI Regular") && w[0].contains("given no translation catalogs"), "{}", w[0]);
}
#[test]
fn subset_to_rejects_cff_interface_faces() {
let scratch = Scratch::new("subset-cff");
let dir = checkout(&scratch, &[("Noto Sans SC UI", "Hans", "ui", tiny("Tiny-cff.otf"))]);
let catalog = scratch.0.join("zh-CN.tsv");
std::fs::write(&catalog, "Open\t一\n").unwrap();
for required in [None, Some("1")] {
let r = run(Embed::new().subset_to([&catalog]), Some(&dir), required);
let Err(e @ Error::NotTrueType { .. }) = r.result else { panic!("{:?}", r.result) };
assert!(e.to_string().contains("0.ttf has CFF outlines"), "{e}");
assert_eq!(r.table.trim(), "&[]");
}
}
/// With `CRAFT_FONTS_DIR` set to a craft-fonts checkout, cut its BIZ UDPGothic Regular.
#[test]
fn cuts_biz_udpgothic() {
let Some(dir) = std::env::var_os("CRAFT_FONTS_DIR").filter(|dir| !dir.is_empty()) else { return };
let Ok(source) = std::fs::read(Path::new(&dir).join("fonts/biz-ud-pgothic/BIZUDPGothic-Regular.ttf")) else { return };
let chars: BTreeSet<char> = "ファイルを開く「新規」、保存。編集表示".chars().collect();
let cut = crate::subset::subset(&source, 0, &chars).unwrap();
let (source_cmap, cmap) = (Cmap::read(&source, 0).unwrap(), Cmap::read(&cut, 0).unwrap());
assert!(chars.iter().all(|&c| cmap.covers(c) == source_cmap.covers(c)));
assert!(!cmap.covers('A') && !cmap.covers('漢'));
assert!(features(table(&cut, b"GSUB").unwrap()).contains("vert"));
assert!(cut.len() < source.len() / 20, "{} of {} bytes", cut.len(), source.len());
assert_eq!(crate::subset::subset(&source, 0, &chars).unwrap(), cut);
}
}
crates/fonts-build/tests/consumer/Cargo.toml
+18
View File
@@ -0,0 +1,18 @@
[package]
name = "craft-fonts-build-consumer"
description = "Test crate: a build.rs that embeds the craft-fonts-build fixture the way an app does."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
publish = false
[dependencies]
craft-fonts-build = { path = "../.." }
[build-dependencies]
craft-fonts-build = { path = "../.." }
[lints]
workspace = true
crates/fonts-build/tests/consumer/build.rs
+16
View File
@@ -0,0 +1,16 @@
//! Embeds the fixture checkout in `../fixture/craft-fonts` the way an app's build.rs embeds
//! `CRAFT_FONTS_DIR`, so the generated table is compiled by a real `include!`.
use std::path::PathBuf;
fn main() {
let root = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap_or_default());
let embedded = craft_fonts_build::Embed::new()
.dir(root.join("../fixture/craft-fonts"))
.required(true)
.scripts(&["Jpan", "Hant"])
.filter(|face| face.style == "Regular")
.write("craft_fonts.rs");
if let Err(e) = embedded {
println!("cargo::error={e}");
}
}
crates/fonts-build/tests/consumer/src/lib.rs
+7
View File
@@ -0,0 +1,7 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! The fixture faces as an app's crate sees them; read by `craft-fonts-build`'s tests.
use craft_fonts_build::EmbeddedFace;
pub static CRAFT_FONTS: &[EmbeddedFace] = include!(concat!(env!("OUT_DIR"), "/craft_fonts.rs"));
crates/fonts-build/tests/fixture/craft-fonts/fonts/manifest.txt
+10
View File
@@ -0,0 +1,10 @@
# Test fixture in the craft-fonts manifest format. The "font files" are short text files:
# embedding copies bytes and never parses them.
# family | style | file | scripts | licence | licence file | sha256 | source
Sans JP | Regular | fonts/sans-jp/SansJP-Regular.ttf | Jpan,Latn | OFL-1.1 | fonts/sans-jp/OFL.txt | c9d08ef1fc69f15e87ce52bda8a9c24714e4125daf7e50a40edb1aa2be10dff1 | https://example.invalid/sans-jp
Sans JP | Bold | fonts/sans-jp/SansJP-Bold.ttf | Jpan,Latn | OFL-1.1 | fonts/sans-jp/OFL.txt | a135adabc1facd7a49fa4ef04771f74ffe69e67418e0e0f0b81a4cebba776e48 | https://example.invalid/sans-jp
Sans Arabic | Regular | fonts/sans-arabic/SansArabic.ttf | Arab | OFL-1.1 | fonts/sans-arabic/OFL.txt | 318d7469dd372ff3ad458d46ea714c0ea969696675794d6100396f8a11103514 | https://example.invalid/sans-arabic
Sans CJK TC | Regular | fonts/sans-cjk/SansCJK.ttc#2 | Hant,Latn | OFL-1.1 | fonts/sans-cjk/OFL.txt | 2fecde232d5dcda26a2299664dae1a2c50ad92b694575c5cfa70207010b339be | https://example.invalid/sans-cjk
Sans CJK HK | Regular | fonts/sans-cjk/SansCJK.ttc#3 | Hant-HK,Latn | OFL-1.1 | fonts/sans-cjk/OFL.txt | 2fecde232d5dcda26a2299664dae1a2c50ad92b694575c5cfa70207010b339be | https://example.invalid/sans-cjk
Sans Thai | Regular | fonts/sans-thai/SansThai.ttf | Thai | OFL-1.1 | fonts/sans-thai/OFL.txt | 5960276bcbad8e7949804d35651ebf42e17648f4d128c71ccdb76167a5337944 | https://example.invalid/sans-thai
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-arabic/OFL.txt
+1
View File
@@ -0,0 +1 @@
Fixture licence text.
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-arabic/SansArabic.ttf
+1
View File
@@ -0,0 +1 @@
fixture: Sans Arabic
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-cjk/OFL.txt
+1
View File
@@ -0,0 +1 @@
Fixture licence text.
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-cjk/SansCJK.ttc
+1
View File
@@ -0,0 +1 @@
fixture: Sans CJK collection
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-jp/OFL.txt
+1
View File
@@ -0,0 +1 @@
Fixture licence text.
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-jp/SansJP-Bold.ttf
+1
View File
@@ -0,0 +1 @@
fixture: Sans JP Bold
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-jp/SansJP-Regular.ttf
+1
View File
@@ -0,0 +1 @@
fixture: Sans JP Regular
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-thai/OFL.txt
+1
View File
@@ -0,0 +1 @@
Fixture licence text.
crates/fonts-build/tests/fixture/craft-fonts/fonts/sans-thai/SansThai.ttf
+1
View File
@@ -0,0 +1 @@
fixture: Sans Thai
crates/fonts-build/tests/fixture/subset/Tiny-cff.otf
BIN
View File
Binary file not shown.
crates/fonts-build/tests/fixture/subset/Tiny-glyf.ttf
BIN
View File
Binary file not shown.
crates/fonts-build/tests/fixture/subset/make-fonts.py
Executable
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env -S uv run --script
# /// script
# requires-python = ">=3.10"
# dependencies = ["fonttools==4.67.0"]
# ///
"""Write the subsetting fixtures next to this script: Tiny-glyf.ttf (TrueType outlines) and
Tiny-cff.otf (CFF outlines). Both map the same characters, have a `vert` feature that swaps
U+300C for a vertical form, and draw every glyph as a square.
"""
from pathlib import Path
from fontTools.feaLib.builder import addOpenTypeFeaturesFromString
from fontTools.fontBuilder import FontBuilder
from fontTools.pens.t2CharStringPen import T2CharStringPen
from fontTools.pens.ttGlyphPen import TTGlyphPen
HERE = Path(__file__).resolve().parent
CMAP = {
0x20: "space",
0x41: "A",
0x3001: "uni3001",
0x300C: "uni300C",
0x3042: "uni3042",
0x4E00: "uni4E00",
0x4E01: "uni4E01",
0x4E03: "uni4E03",
0xFF01: "uniFF01",
}
GLYPHS = [".notdef", *CMAP.values(), "uni300C.vert"]
FEATURES = "languagesystem DFLT dflt;\nfeature vert { sub uni300C by uni300C.vert; } vert;\n"
# 2026-01-01, so the files are the same on every run.
TIMESTAMP = 3_850_000_000
def square(pen, size):
pen.moveTo((100, 0))
pen.lineTo((100, size))
pen.lineTo((100 + size, size))
pen.lineTo((100 + size, 0))
pen.closePath()
def build(cff: bool) -> FontBuilder:
fb = FontBuilder(1000, isTTF=not cff)
fb.setupGlyphOrder(GLYPHS)
fb.setupCharacterMap(CMAP)
if cff:
charstrings = {}
for i, name in enumerate(GLYPHS):
pen = T2CharStringPen(1000, None)
square(pen, 100 + 10 * i)
charstrings[name] = pen.getCharString()
fb.setupCFF("Tiny-Regular", {"FullName": "Tiny Regular"}, charstrings, {})
else:
glyphs = {}
for i, name in enumerate(GLYPHS):
pen = TTGlyphPen(None)
square(pen, 100 + 10 * i)
glyphs[name] = pen.glyph()
fb.setupGlyf(glyphs)
fb.setupHorizontalMetrics({name: (1000, 100) for name in GLYPHS})
fb.setupHorizontalHeader(ascent=880, descent=-120)
fb.setupNameTable({"familyName": "Tiny", "styleName": "Regular"})
fb.setupOS2(sTypoAscender=880, sTypoDescender=-120, usWinAscent=880, usWinDescent=120)
fb.setupPost()
addOpenTypeFeaturesFromString(fb.font, FEATURES)
fb.font["head"].created = TIMESTAMP
fb.font["head"].modified = TIMESTAMP
fb.font.recalcTimestamp = False
return fb
def main() -> None:
build(cff=False).save(HERE / "Tiny-glyf.ttf")
build(cff=True).save(HERE / "Tiny-cff.otf")
if __name__ == "__main__":
main()
crates/fonts-build/tests/generated.rs
+16
View File
@@ -0,0 +1,16 @@
//! The table `craft-fonts-build-consumer`'s build.rs generated from the fixture compiles and holds
//! the selected faces' bytes.
use craft_fonts_build_consumer::CRAFT_FONTS;
#[test]
fn generated_table_embeds_the_selected_faces() {
let names: Vec<String> = CRAFT_FONTS.iter().map(|f| f.name()).collect();
assert_eq!(names, ["Sans JP Regular", "Sans CJK TC Regular", "Sans CJK HK Regular"]);
assert_eq!(CRAFT_FONTS[0].bytes, include_bytes!("fixture/craft-fonts/fonts/sans-jp/SansJP-Regular.ttf"));
assert_eq!(CRAFT_FONTS[0].scripts, ["Jpan", "Latn"]);
let indices: Vec<u32> = CRAFT_FONTS.iter().map(|f| f.index).collect();
assert_eq!(indices, [0, 2, 3]);
assert_eq!(CRAFT_FONTS[2].bytes, include_bytes!("fixture/craft-fonts/fonts/sans-cjk/SansCJK.ttc"));
assert!(CRAFT_FONTS[2].has_script("Hant"));
}
crates/i18n/Cargo.toml
+28 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "craft-i18n"
description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale."
description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, numbers for a region, language names, the system locale, search folding and locale sorting."
version.workspace = true
edition.workspace = true
license.workspace = true
@@ -8,14 +8,39 @@ rust-version.workspace = true
repository.workspace = true
[dependencies]
intl_pluralrules = "=7.0.2"
unic-langid = "0.9"
# Numbers and plural rules, on CLDR data (data/numbers.postcard) generated for one ICU4X minor
# version, hence `~2.3`.
fixed_decimal = { version = "0.7.2", default-features = false }
icu_decimal = { version = "~2.3", default-features = false, features = ["serde"] }
icu_locale_core = { version = "~2.3", default-features = false }
icu_locale_fallback = { version = "~2.3", default-features = false, features = ["compiled_data"] }
icu_plurals = { version = "~2.3", default-features = false, features = ["serde"] }
icu_provider = { version = "~2.3", default-features = false, features = ["sync"] }
icu_provider_adapters = { version = "~2.3", default-features = false }
icu_provider_blob = { version = "~2.3", default-features = false }
writeable = "0.6"
log = "0.4"
sys-locale = "0.3"
icu_normalizer = "~2.3"
icu_properties = "~2.3"
# Generated data fits one ICU4X minor version; data/generate-collation.sh names its datagen.
icu_collator = { version = "~2.3.1", default-features = false, features = ["serde"], optional = true }
unicode-bidi = "0.3"
[target.'cfg(target_arch = "wasm32")'.dependencies]
sys-locale = { version = "0.3", features = ["js"] }
# The regional format (Region::system); safe registry reads, MIT OR Apache-2.0.
[target.'cfg(windows)'.dependencies]
windows-registry = "0.6"
# The current locale (Region::system); already in the apps' Cargo.lock through eframe.
[target.'cfg(target_vendor = "apple")'.dependencies]
objc2-foundation = { version = "0.3.2", default-features = false, features = ["std", "NSDateFormatter", "NSFormatter", "NSLocale", "NSString"] }
[features]
# Locale sorting (`collate`), with ICU4X collation data for the suite's locales embedded.
collate = ["dep:icu_collator", "icu_provider/serde", "icu_provider_blob/alloc"]
[lints]
workspace = true
crates/i18n/README.md
+78 -6
View File
@@ -45,10 +45,21 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l
→ `pt`, `zh-hant-hk` → `zh-hant`); `sr-latn` never falls back to Cyrillic `sr`. A missing
translation never errors.
- **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an
`@id` command id, else the English label), `trn` (a count, `@plural` rows, `{n}` filled), `get`
(one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row
`@id` command id, else the English label), `trn` and `trn_decimal` (a count, `@plural` rows,
`{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row
or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}`
placeholders, filled in one pass.
- **Isolation:** when a message or a value inserted into it has a right-to-left letter (bidi
class R or AL), or a value has a bidi control (LRE, RLE, PDF, LRO, RLO, LRI, RLI, FSI, PDI,
LRM, RLM, ALM), `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn`,
`file_size` and `measure` wrap `{n}` in a left-to-right one (LRI…PDI), so a Latin file name in an Arabic sentence, a
Hebrew name in an English one, or an ASCII file name with a right-to-left override can't
reorder the words around it. Embeddings, overrides and isolates a value opens are closed inside
its isolate, and a stray PDF or PDI in it is dropped. Wrap numbers, paths and codes in `ltr`
before passing them; a value that already is one isolate keeps it. Other text comes out
without marks, as if `fmt` didn't isolate at all. egui draws the marks as nothing. For text
that leaves the interface (file names, logs, terminals, text parsed later), use `fmt_plain` or
`strip_isolates`.
- **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice,
of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case)
are skipped; craft-i18n's CI rejects them before they ship. `I18n::new` parses nothing: it
@@ -56,10 +67,13 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l
`log::warn!`) on the first lookup that reaches it, including as a fallback. `I18n::check_all()`
parses every locale and returns all problems (`<locale>/<common|app>: line N: …`), for a test.
- **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories
whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the
last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br`
`pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and
`plural_form_counts` holds the same counts.
whole numbers 0 to 1000 reach, from ICU4X's CLDR rules; `plural_form_counts` holds the counts
craft-i18n's checker expects. `trn_decimal` picks the form from the digits shown (ICU4X plural
operands), so English `1.0` is plural and French `1,5` singular. A count in a category integers
never reach takes the `few` form if there is one (Russian and Ukrainian decimals, `1,5 файла`),
else the last. Rules are per language, so `sr-latn` and `zh-hant` take `sr`'s and `zh`'s. CLDR's
`pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` `pt`'s. A language
CLDR has no plural rules for shows English forms.
- **Names:** a language's name comes from the app catalog's `@id language.name` row, else the
shared catalog's, else its code; English is "English".
- **Scripts and direction:** for any language tag, `scripts` lists the ISO 15924 scripts its
@@ -68,13 +82,71 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l
gives the regional forms of its Han characters for picking a CJK face: `zh-CN`/`zh-SG`/`zh-Hans` →
SC, `zh-TW`/`zh-US`/`zh-Hant` → TC, `zh-HK`/`zh-MO` → HK, `ja` → JP, `ko` → KR. `I18n::rtl` is
`direction` of the language's code; putting text into display order is the app's job.
- **Reading numbers:** `Format::parse(text)` reads what the format writes (its digits, decimal
and grouping separators, the system's included) and what people type or paste: any Unicode
decimal digits (ASCII, Arabic-Indic, Persian, Devanagari, Bengali, Thai, full-width, …) from one
set; `٫` as a decimal mark everywhere; grouping by the locale's other mark of `.` and `,`, spaces
(NBSP, narrow NBSP), `'`, `’` or `٬`, in threes or the Indian 2-2-3; a leading `-`, `−` or `+`;
bidi marks ignored. A lone `.` in a comma locale is decimal when it can't be grouping (`1.5`,
`0.250`, `1234.5`), while `1.234` stays 1234 in German; the same holds for `,` in a point
locale. Percent signs and units (`50 %`, `12 px`) are not read: strip them first. Other input
that doesn't fit is an error, never a guess. `parse_number(text, locale)` is the same for
`Format::from_tag(locale)`.
- **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix
`LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and
Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → `zh-hant`,
`pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never
crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`.
A region that implies a script counts as that script: `pa-PK` picks `pa-arab`, not Gurmukhi `pa`.
- **Sizes and units:** `file_size(lang, bytes, style)` writes `1.5 MB` or `512 bytes`; `FileSize`
picks steps of 1024 named KB (`BINARY`), 1024 named KiB (`IEC`) or 1000 named KB (`DECIMAL`),
the decimals per unit, no decimals from a given value up, the smallest and largest unit (below
the smallest, sizes round up), and `B` or words for bytes. `measure(lang, value, decimals,
"{n} px")` writes any other measure. Unit names are rows in context `unit`, keyed by the English
template (`{n} MB` → `{n} Mo`, `{n}%` → `{n} %`) and read with `get`, so a plain `B` row (bold)
is never used; bytes in words are the `@plural` row `{n} byte|{n} bytes`. Numbers are written
by `I18n::format`.
- **Numbers:** `Format` writes numbers for a `Region` on ICU4X's decimal formatter: `integer`
(`1.234.567`, Indian `12,34,567`), `number(value, decimals)` and `percent(ratio, decimals)`
(`50 %`, Turkish `%50`). It follows the region, not the interface language, with the locale's
digits (`١٬٢٣٤٫٥` in Egypt, `۱٬۲۳۴` in Iran) or, with `Digits::Latin`, `0`–`9`. With Latin
digits, a decimal or grouping separator the system sets (Windows' and macOS's own settings)
replaces the locale's. `I18n::format` is the format `trn`, `file_size` and `measure` use: the
system region's until `I18n::set_format` (the user's region setting or digit choice), and again
after `I18n::reset_format`. A count in `trn` is therefore grouped and in the region's digits
(`1.234 Dateien`, `١٬٢٣٤ ملفًا`); for names written into documents or files (`Copy 1234`), use
`trn_plain`, which writes `0`–`9` without grouping or isolate marks.
- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 10 KB) and
percent patterns (`data/locales.rs`) cover every CLDR locale; `data/generate.sh` rebuilds both
from CLDR. The ICU4X crates are pinned to one minor version (`~2.3`), the one the data is
generated for. The ICU4X code and data add about 90 KB to a stripped release build and 80 KB to
wasm.
- **Formatting region:** `Region` holds how numbers, times and measurements are written, apart
from the interface language: the number and time locales (BCP 47), metric or US measurement, A4
or Letter, and the decimal and grouping separators (or no grouping) and clock when the system
sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once:
Windows' regional format with the user's changes (`HKCU\Control Panel\International`), Apple's
current locale (`en_PL`, `en_GB@rg=uszzzz`), `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`,
`LC_MEASUREMENT` and `LC_PAPER` each on its own over `LANG` on Unix (`C` there is English on A4,
metric, with a `.` decimal and no grouping; `LANG=C` gives way to the preferred language), the
preferred language on the web and Android. `Region::from_setting` takes the app's saved choice
(`auto` or a tag like `pl-PL`). Paper and measurement follow CLDR's territory lists; tags may
carry `-u-ms-`, `-u-hc-`, `-u-rg-` and `-u-nu-` (Apple's `@numbers=`), and a chosen digit
system stays in the number locale.
Ported from WordCraft's `Region`.
- **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get`
included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through
translation. `looked_up()` lists the source texts looked up so far, for an audit against a
complete catalog.
- **Search** (`Folder`): `Folder::new(locale).fold(text)` gives a key for case- and
accent-insensitive matching (`keep_accents()` makes accents count) and a map back to the original
text, so `Folded::find` returns the byte range to highlight. Compatibility forms fold to their
plain letters (`fi`, full-width `Ａ`), `ß` to `ss`, `ς` to `σ`, digits of every script to ASCII
(`Layer ١٢` finds `layer 12`); ignoring accents, Persian and Arabic letter forms match (`ک` and
`ك`, `ی` and `ي`, `ة` and `ه`, `أ` and `ا`); invisible format characters (soft hyphens, joiners,
bidi isolates) are dropped. Turkish and Azerbaijani fold `I` to `ı` when accents count; ignoring
accents, `I`, `i`, `İ` and `ı` all match.
- **Sorting** (`Collator`, feature `collate`): ICU4X collation for the locales craft-i18n ships,
Persian and Hindi, with digits compared by value (`Layer 2` before `Layer 10`) and `-u-co-pinyin`
/ `-u-co-stroke` for Chinese. The data (754 KB) is regenerated by `data/generate-collation.sh`;
the feature adds about 1 MB to a release binary or wasm module (320 KB gzipped).
crates/i18n/data/collation.postcard
BIN
View File
Binary file not shown.
crates/i18n/data/generate-collation.sh
Executable
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
# Regenerates the collation data `collate` embeds, a postcard blob for the locales craft-i18n
# ships (its `locales/` directory) plus English, Persian and Hindi: each locale's default order,
# and Chinese pinyin and stroke orders.
# Needs icu4x-datagen at the version of the icu_collator dependency:
# cargo install icu4x-datagen --version 2.3.0 --locked
set -eu
cd "$(dirname "$0")"
icu4x-datagen --format blob --overwrite --out collation.postcard --deduplication maximal \
--cldr-tag 48.2.1 --icuexport-tag release-78.1rc \
--attribute-filter 'collator=/^(stroke|pinyin)?$/' \
--markers CollationRootV1 CollationTailoringV1 CollationDiacriticsV1 CollationJamoV1 \
CollationMetadataV1 CollationReorderingV1 CollationSpecialPrimariesV1 \
--locales en ar bg cs da de el es et fa fi fr hi hr hu id it ja ka kk ko lt nl pl pt-BR ru sl \
sr sr-Latn sv te tr uk vi zh-Hans zh-Hant
crates/i18n/data/generate.sh
Executable
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env bash
# Regenerates the number and plural data in this directory from Unicode CLDR, for every CLDR
# locale: 10 KB, of which CLDR's modern-coverage locales alone would be 7 KB. Lookups fall back
# at run time, so a locale whose data matches its parent's is left out.
#
# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X minor version in Cargo.toml)
# and python3. Downloads CLDR's JSON release once into $CACHE.
set -euo pipefail
CLDR=48.2.1
HERE=$(cd "$(dirname "$0")" && pwd)
CACHE=${CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/craft-i18n-data}
mkdir -p "$CACHE"
TMPDIR=$CACHE icu4x-datagen --format blob --overwrite \
--cldr-tag "$CLDR" \
--markers DecimalSymbolsV1 DecimalDigitsV1 PluralsCardinalV1 \
--locales full --deduplication maximal \
--out "$HERE/numbers.postcard"
ZIP=$CACHE/icu4x-source-cache/github.com/unicode-org/cldr-json/releases/download/$CLDR/cldr-$CLDR-json-full.zip
python3 -I "$HERE/locales.py" "$ZIP" "$CLDR" > "$HERE/locales.rs.tmp"
mv "$HERE/locales.rs.tmp" "$HERE/locales.rs"
crates/i18n/data/locales.py
+70
View File
@@ -0,0 +1,70 @@
"""Writes locales.rs from CLDR's JSON release: the languages with plural rules and every locale's
percent affixes.
Usage: locales.py <cldr-json-full.zip> <cldr version>
"""
import json
import re
import sys
import zipfile
def rust(s):
return '"' + "".join(c if " " <= c <= "~" and c not in '"\\' else f"\\u{{{ord(c):x}}}" for c in s) + '"'
def affixes(numbers, numsys):
pattern = numbers[f"percentFormats-numberSystem-{numsys}"]["standard"].split(";")[0].replace("'", "")
sign = numbers[f"symbols-numberSystem-{numsys}"]["percentSign"]
number = re.search(r"[#0,.]+", pattern)
return pattern[: number.start()].replace("%", sign), pattern[number.end() :].replace("%", sign)
def main():
path, version = sys.argv[1], sys.argv[2]
found = {}
with zipfile.ZipFile(path) as z:
plurals = json.loads(z.read("cldr-core/supplemental/plurals.json"))["supplemental"]["plurals-type-cardinal"]
languages = sorted(l for l in plurals if l != "root" and "-" not in l)
for name in z.namelist():
m = re.fullmatch(r"cldr-numbers-full/main/([^/]+)/numbers\.json", name)
if not m or m.group(1) == "root":
continue
locale = m.group(1)
numbers = json.loads(z.read(name))["main"][locale]["numbers"]
found[(locale, True)] = affixes(numbers, "latn")
default = numbers["defaultNumberingSystem"]
if default != "latn":
found[(locale, False)] = affixes(numbers, default)
# Kept only where the lookup by dropping trailing subtags would find something else.
kept = {}
for (locale, latin), value in sorted(found.items(), key=lambda e: (e[0][0].count("-"), e[0])):
parts = locale.split("-")
inherited = ("", "%")
for n in range(len(parts) - 1, 0, -1):
if ("-".join(parts[:n]), latin) in kept:
inherited = kept[("-".join(parts[:n]), latin)]
break
if value != inherited:
kept[(locale, latin)] = value
print(f"// @generated by generate.sh from Unicode CLDR {version} (Unicode License).")
print()
print("/// The languages CLDR has plural rules for; the baked data has them all.")
print("pub(crate) const PLURAL_LANGUAGES: &[&str] = &[")
for i in range(0, len(languages), 16):
print(" " + ", ".join(rust(l) for l in languages[i : i + 16]) + ",")
print("];")
print()
print("/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin")
print("/// digits or the locale's own, text before the number, text after it. A locale missing here")
print("/// takes its parent's by dropping trailing subtags, then `{n}%`.")
print("pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[")
for (locale, latin), (before, after) in sorted(kept.items()):
print(f" ({rust(locale)}, {str(latin).lower()}, {rust(before)}, {rust(after)}),")
print("];")
main()
crates/i18n/data/locales.rs
+144
View File
@@ -0,0 +1,144 @@
// @generated by generate.sh from Unicode CLDR 48.2.1 (Unicode License).
/// The languages CLDR has plural rules for; the baked data has them all.
pub(crate) const PLURAL_LANGUAGES: &[&str] = &[
"af", "ak", "am", "an", "ar", "ars", "as", "asa", "ast", "az", "bal", "be", "bem", "bez", "bg", "bho",
"blo", "bm", "bn", "bo", "br", "brx", "bs", "ca", "ce", "ceb", "cgg", "chr", "ckb", "cs", "csw", "cv",
"cy", "da", "de", "doi", "dsb", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff",
"fi", "fil", "fo", "fr", "fur", "fy", "ga", "gd", "gl", "gsw", "gu", "guw", "gv", "ha", "haw", "he",
"hi", "hnj", "hr", "hsb", "hu", "hy", "ia", "id", "ie", "ig", "ii", "io", "is", "it", "iu", "ja",
"jbo", "jgo", "jmc", "jv", "jw", "ka", "kab", "kaj", "kcg", "kde", "kea", "kk", "kkj", "kl", "km", "kn",
"ko", "kok", "ks", "ksb", "ksh", "ku", "kw", "ky", "lag", "lb", "lg", "lij", "lkt", "lld", "ln", "lo",
"lt", "lv", "mas", "mg", "mgo", "mk", "ml", "mn", "mo", "mr", "ms", "mt", "my", "nah", "naq", "nb",
"nd", "ne", "nl", "nn", "nnh", "no", "nqo", "nr", "nso", "ny", "nyn", "om", "or", "os", "osa", "pa",
"pap", "pcm", "pl", "prg", "ps", "pt", "rm", "ro", "rof", "ru", "rwk", "sah", "saq", "sat", "sc", "scn",
"sd", "sdh", "se", "seh", "ses", "sg", "sgs", "sh", "shi", "si", "sk", "sl", "sma", "smi", "smj", "smn",
"sms", "sn", "so", "sq", "sr", "ss", "ssy", "st", "su", "sv", "sw", "syr", "ta", "te", "teo", "th",
"ti", "tig", "tk", "tl", "tn", "to", "tpi", "tr", "ts", "tzm", "ug", "uk", "und", "ur", "uz", "ve",
"vec", "vi", "vo", "vun", "wa", "wae", "wo", "xh", "xog", "yi", "yo", "yue", "zh", "zu",
];
/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin
/// digits or the locale's own, text before the number, text after it. A locale missing here
/// takes its parent's by dropping trailing subtags, then `{n}%`.
pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[
("ar", true, "", "\u{200e}%\u{200e}"),
("ar-BH", false, "", "\u{66a}\u{61c}"),
("ar-DJ", false, "", "\u{66a}\u{61c}"),
("ar-EG", false, "", "\u{66a}\u{61c}"),
("ar-ER", false, "", "\u{66a}\u{61c}"),
("ar-IL", false, "", "\u{66a}\u{61c}"),
("ar-IQ", false, "", "\u{66a}\u{61c}"),
("ar-JO", false, "", "\u{66a}\u{61c}"),
("ar-KM", false, "", "\u{66a}\u{61c}"),
("ar-KW", false, "", "\u{66a}\u{61c}"),
("ar-LB", false, "", "\u{66a}\u{61c}"),
("ar-MR", false, "", "\u{66a}\u{61c}"),
("ar-OM", false, "", "\u{66a}\u{61c}"),
("ar-PS", false, "", "\u{66a}\u{61c}"),
("ar-QA", false, "", "\u{66a}\u{61c}"),
("ar-SA", false, "", "\u{66a}\u{61c}"),
("ar-SA", true, "", "\u{66a}"),
("ar-SD", false, "", "\u{66a}\u{61c}"),
("ar-SO", false, "", "\u{66a}\u{61c}"),
("ar-SO", true, "", "\u{66a}"),
("ar-SS", false, "", "\u{66a}\u{61c}"),
("ar-SY", false, "", "\u{66a}\u{61c}"),
("ar-TD", false, "", "\u{66a}\u{61c}"),
("ar-YE", false, "", "\u{66a}\u{61c}"),
("az-Arab", false, "", "\u{66a}"),
("bas", true, "", "\u{a0}%"),
("be", true, "", "\u{a0}%"),
("bgn", false, "", "\u{66a}"),
("bgn", true, "", "\u{66a}"),
("blo", true, "%\u{a0}", ""),
("br", true, "", "\u{a0}%"),
("bua", true, "", "\u{a0}%"),
("ca", true, "", "\u{a0}%"),
("ce", true, "", "\u{a0}%"),
("ckb", false, "", "\u{a0}\u{66a}"),
("cs", true, "", "\u{a0}%"),
("cv", true, "", "\u{a0}%"),
("da", true, "", "\u{a0}%"),
("de", true, "", "\u{a0}%"),
("de-CH", true, "", "%"),
("de-LI", true, "", "%"),
("dsb", true, "", "\u{a0}%"),
("dua", true, "", "\u{a0}%"),
("dz", false, "", "\u{a0}%"),
("dz", true, "", "\u{a0}%"),
("en-AT", true, "", "\u{a0}%"),
("en-DE", true, "", "\u{a0}%"),
("en-DK", true, "", "\u{a0}%"),
("en-FI", true, "", "\u{a0}%"),
("en-NO", true, "", "\u{a0}%"),
("en-SE", true, "", "\u{a0}%"),
("es", true, "", "\u{a0}%"),
("es-419", true, "", "%"),
("es-AR", true, "", "%"),
("es-BO", true, "", "%"),
("es-BR", true, "", "%"),
("es-BZ", true, "", "%"),
("es-CL", true, "", "%"),
("es-CO", true, "", "%"),
("es-CR", true, "", "%"),
("es-CU", true, "", "%"),
("es-DO", true, "", "%"),
("es-EC", true, "", "%"),
("es-GT", true, "", "%"),
("es-HN", true, "", "%"),
("es-MX", true, "", "%"),
("es-NI", true, "", "%"),
("es-PA", true, "", "%"),
("es-PE", true, "", "%"),
("es-PR", true, "", "%"),
("es-PY", true, "", "%"),
("es-SV", true, "", "%"),
("es-US", true, "", "%"),
("es-UY", true, "", "%"),
("es-VE", true, "", "%"),
("eu", true, "%\u{a0}", ""),
("fa", false, "", "\u{66a}"),
("fi", true, "", "\u{a0}%"),
("fo", true, "", "\u{a0}%"),
("fr", true, "", "\u{a0}%"),
("fr-CH", true, "", "%"),
("gl", true, "", "\u{a0}%"),
("gsw", true, "", "\u{a0}%"),
("hr", true, "", "\u{a0}%"),
("hsb", true, "", "\u{a0}%"),
("ht", true, "", "\u{a0}%"),
("kl", true, "", "\u{a0}%"),
("ks", false, "", "\u{66a}"),
("ksh", true, "", "\u{a0}%"),
("ku", true, "%", ""),
("ku-Arab", true, "", "%"),
("lb", true, "", "\u{a0}%"),
("lrc", false, "", "\u{66a}"),
("lt", true, "", "\u{a0}%"),
("mk", true, "", "\u{a0}%"),
("mzn", false, "", "\u{66a}"),
("nb", true, "", "\u{a0}%"),
("nds", true, "", "\u{a0}%"),
("nn", true, "", "\u{a0}%"),
("no", true, "", "\u{a0}%"),
("oc", true, "", "\u{202f}%"),
("pa-Arab", false, "", "\u{66a}"),
("ps", false, "", "\u{66a}"),
("qu", true, "", "\u{a0}%"),
("rn", true, "", "\u{a0}%"),
("ro", true, "", "\u{a0}%"),
("ru", true, "", "\u{a0}%"),
("sd", false, "", "\u{66a}\u{61c}"),
("sdh", false, "", "\u{66a}\u{61c}"),
("se", true, "", "\u{a0}%"),
("sk", true, "", "\u{a0}%"),
("sl", true, "", "\u{a0}%"),
("smn", true, "", "\u{a0}%"),
("sv", true, "", "\u{a0}%"),
("tk", true, "", "\u{a0}%"),
("tr", true, "%", ""),
("tt", true, "", "\u{a0}%"),
("uz-Arab", false, "", "\u{66a}"),
("zgh", true, "", "\u{a0}%"),
];
crates/i18n/data/numbers.postcard
BIN
View File
Binary file not shown.
crates/i18n/src/collate.rs
+234
View File
@@ -0,0 +1,234 @@
//! Locale sorting for lists people see: fonts, layers, presets, files.
//!
//! ICU4X collation with data embedded for the locales craft-i18n ships (regenerated by
//! `data/generate-collation.sh`). A locale without data of its own sorts in the Unicode root
//! order, which suits most languages that have no tailoring.
use std::cmp::Ordering;
use icu_collator::CollatorPreferences;
use icu_collator::options::CollatorOptions;
use icu_collator::preferences::CollationNumericOrdering;
use icu_collator::provider::{
CollationDiacriticsV1, CollationJamoV1, CollationMetadataV1, CollationReorderingV1, CollationRootV1, CollationSpecialPrimariesV1,
CollationTailoringV1,
};
use icu_locale_core::Locale;
use icu_locale_fallback::LocaleFallbacker;
use icu_normalizer::provider::{NormalizerNfdDataV1, NormalizerNfdTablesV1};
use icu_provider::buf::AsDeserializingBufferProvider;
use icu_provider::prelude::*;
use icu_provider_adapters::fallback::LocaleFallbackProvider;
use icu_provider_blob::BlobDataProvider;
/// Compares text in one locale's order, with runs of digits by their value: `Layer 2` before
/// `Layer 10`.
#[derive(Debug)]
pub struct Collator(Option<icu_collator::Collator>);
impl Collator {
/// Sorting for `locale`, a language code such as `sv`, `pt-br` or `zh-hant` (any case, `-`
/// or `_`, glibc forms such as `sv_SE.UTF-8` and `sr_RS@latin`). A BCP 47 collation type
/// picks another order where there is one: `zh-hant-u-co-pinyin`, `zh-hans-u-co-stroke`.
pub fn new(locale: &str) -> Collator {
let mut prefs: CollatorPreferences = Locale::try_from_str(&bcp47(locale)).unwrap_or(Locale::UNKNOWN).into();
prefs.numeric_ordering = Some(CollationNumericOrdering::True);
let options = CollatorOptions::default();
let collator = Data::new().ok().and_then(|data| {
icu_collator::Collator::try_new_unstable(&data, prefs, options)
.or_else(|_| icu_collator::Collator::try_new_unstable(&data, CollatorPreferences::default(), options))
.ok()
});
Collator(collator)
}
/// `a` against `b`. Text that differs only in case or accents still has an order, and text
/// the collation counts as equal (`Layer 2` and `Layer 02`) is in code point order, so a sort
/// doesn't depend on the input order. Without collation data (never with the embedded data)
/// this is code point order.
pub fn compare(&self, a: &str, b: &str) -> Ordering {
match &self.0 {
Some(c) => c.as_borrowed().compare(a, b).then_with(|| a.cmp(b)),
None => a.cmp(b),
}
}
/// Sort `items` by the text `key` gives each one.
pub fn sort_by_key<T>(&self, items: &mut [T], key: impl Fn(&T) -> &str) {
items.sort_by(|a, b| self.compare(key(a), key(b)));
}
}
/// `locale` as a BCP 47 tag: `_` → `-`, a glibc codeset dropped and its script modifier made a
/// subtag (`sr_RS.UTF-8@latin` → `sr-latn-RS`).
fn bcp47(locale: &str) -> String {
let locale = locale.trim();
let mut parts: Vec<&str> = locale.split(['.', '@']).next().unwrap_or_default().split(['-', '_']).collect();
let script = match locale.split_once('@').map(|(_, m)| m.to_ascii_lowercase()).as_deref() {
Some("latin") => Some("latn"),
Some("cyrillic") => Some("cyrl"),
_ => None,
};
let has_script = parts.get(1).is_some_and(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic()));
if let Some(script) = script
&& !has_script
{
parts.insert(1.min(parts.len()), script);
}
parts.join("-")
}
/// The embedded collation data, with locale fallback, and the decomposition tables the collator
/// also needs taken from `icu_normalizer`'s compiled data rather than embedded a second time.
struct Data(LocaleFallbackProvider<BlobDataProvider>);
impl Data {
fn new() -> Result<Data, DataError> {
let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/collation.postcard"))?;
Ok(Data(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned())))
}
}
macro_rules! from_blob {
($($marker:ty),*) => {$(
impl DataProvider<$marker> for Data {
fn load(&self, req: DataRequest) -> Result<DataResponse<$marker>, DataError> {
self.0.as_deserializing().load(req)
}
}
)*};
}
from_blob!(
CollationRootV1,
CollationTailoringV1,
CollationDiacriticsV1,
CollationJamoV1,
CollationMetadataV1,
CollationReorderingV1,
CollationSpecialPrimariesV1
);
impl DataProvider<NormalizerNfdDataV1> for Data {
fn load(&self, req: DataRequest) -> Result<DataResponse<NormalizerNfdDataV1>, DataError> {
icu_normalizer::provider::Baked.load(req)
}
}
impl DataProvider<NormalizerNfdTablesV1> for Data {
fn load(&self, req: DataRequest) -> Result<DataResponse<NormalizerNfdTablesV1>, DataError> {
icu_normalizer::provider::Baked.load(req)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn sorted<'a>(locale: &str, items: &[&'a str]) -> Vec<&'a str> {
let mut v = items.to_vec();
Collator::new(locale).sort_by_key(&mut v, |s| s);
v
}
/// Each tailored locale puts a pair in another order than the root collation does, which a
/// locale whose data is missing (it falls back to root) wouldn't.
#[test]
fn every_shipped_locale_has_its_data() {
let root = Collator::new("und");
for (locale, first, second) in [
("ar", "ب", "a"),
("bg", "б", "a"),
("cs", "hrad", "chata"),
("da", "zebra", "år"),
("el", "β", "a"),
("es", "nube", "ñu"),
("et", "wc", "õun"),
("fa", "ب", "a"),
("fi", "zeta", "äiti"),
("hi", "क", "a"),
("hr", "cvijet", "čaj"),
("hu", "cukor", "csak"),
("ja", "亜", "一"),
("ka", "ა", "a"),
("kk", "б", "a"),
("ko", "가", "a"),
("lt", "yra", "jau"),
("pl", "az", "ąb"),
("ru", "б", "a"),
("sl", "cvet", "čaj"),
("sr", "б", "a"),
("sr-latn", "cvijet", "čaj"),
("sv", "zebra", "år"),
("te", "క", "a"),
("tr", "cz", "ça"),
("uk", "б", "a"),
("vi", "az", "ăn"),
("zh-hans", "阿", "中"),
("zh-hant", "人", "丈"),
] {
assert_eq!(Collator::new(locale).compare(first, second), Ordering::Less, "{locale}: {first} < {second}");
assert_eq!(root.compare(first, second), Ordering::Greater, "root: {second} < {first}");
}
// CLDR doesn't tailor these: they sort in the root order.
for locale in ["en", "de", "fr", "id", "it", "nl", "pt-br", "not a locale"] {
let collator = Collator::new(locale);
assert!(collator.0.is_some(), "{locale}");
assert_eq!(collator.compare("čaj", "cvijet"), Ordering::Less, "{locale}");
}
}
#[test]
fn digits_sort_by_value() {
let layers = ["Layer 10", "Layer 2", "layer 1", "Layer 1", "Layer 2b", "Layer 02"];
assert_eq!(sorted("en", &layers), ["layer 1", "Layer 1", "Layer 02", "Layer 2", "Layer 2b", "Layer 10"]);
let reversed: Vec<&str> = layers.iter().rev().copied().collect();
assert_eq!(sorted("en", &reversed), sorted("en", &layers));
assert_eq!(sorted("de", &["Ebene ٣", "Ebene 10", "Ebene ２"]), ["Ebene ２", "Ebene ٣", "Ebene 10"]);
}
#[test]
fn locales_sort_their_own_letters() {
let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
assert_eq!(sorted("sv", &words), ["apa", "oxe", "zebra", "år", "äpple", "öl"]);
assert_eq!(sorted("de", &words), ["apa", "äpple", "år", "öl", "oxe", "zebra"]);
assert_eq!(sorted("de", &["Muller", "Müller", "Mueller", "Mahler"]), ["Mahler", "Mueller", "Muller", "Müller"]);
assert_eq!(
sorted("tr", &["şeker", "sabun", "ılık", "iğne", "çay", "cam", "uzun", "üzüm"]),
["cam", "çay", "ılık", "iğne", "sabun", "şeker", "uzun", "üzüm"]
);
assert_eq!(sorted("en", &["ılık", "iğne", "cam", "çay"]), ["cam", "çay", "iğne", "ılık"]);
assert_eq!(sorted("es", &["ñu", "nube", "oso"]), ["nube", "ñu", "oso"]);
// Japanese: Latin, then kana in gojūon order (hiragana and katakana together), then kanji.
assert_eq!(sorted("ja", &["漢字", "カメラ", "あめ", "いぬ", "Zoom"]), ["Zoom", "あめ", "いぬ", "カメラ", "漢字"]);
}
#[test]
fn glibc_locale_names() {
let words = ["öl", "zebra", "år", "äpple", "apa", "oxe"];
assert_eq!(sorted("sv_SE.UTF-8", &words), sorted("sv", &words));
assert_ne!(sorted("sv_SE.UTF-8", &words), sorted("en", &words));
let cities = ["天津", "北京", "上海"];
assert_eq!(sorted("zh_TW.UTF-8", &cities), sorted("zh-hant", &cities));
assert_eq!(sorted("zh_CN.UTF-8", &cities), sorted("zh-hans", &cities));
// Serbian Cyrillic sorts Cyrillic first; Latin Serbian sorts Latin first, `lj` after `lu`.
let mixed = ["ljubav", "lula", "абв"];
assert_eq!(sorted("sr_RS.UTF-8", &mixed), ["абв", "ljubav", "lula"]);
assert_eq!(sorted("sr_RS.UTF-8@latin", &mixed), ["lula", "ljubav", "абв"]);
assert_eq!(bcp47("sr_RS.UTF-8@latin"), "sr-latn-RS");
assert_eq!(bcp47("sr_Latn_RS@latin"), "sr-Latn-RS");
assert_eq!(bcp47("de_DE@euro"), "de-DE");
}
#[test]
fn chinese_sorts_by_pinyin_or_stroke() {
// 北 běi (5 strokes), 上 shàng (3), 天 tiān (4).
let cities = ["天津", "北京", "上海"];
let pinyin = ["北京", "上海", "天津"];
let stroke = ["上海", "天津", "北京"];
assert_eq!(sorted("zh-hans", &cities), pinyin);
assert_eq!(sorted("zh-hant", &cities), stroke);
assert_eq!(sorted("zh-hans-u-co-stroke", &cities), stroke);
assert_eq!(sorted("zh-hant-u-co-pinyin", &cities), pinyin);
}
}
crates/i18n/src/fold.rs
+424
View File
@@ -0,0 +1,424 @@
//! Case- and accent-insensitive matching for search fields, with a map back to the original text
//! for highlighting.
//!
//! [`Folder::fold`] turns text into a search key: compatibility forms decomposed (`fi` → `fi`,
//! full-width `Ａ` → `a`, `㎏` → `kg`), lower case, `ß` → `ss`, final `ς` → `σ`, and with accents
//! ignored (the default) combining accents, Hebrew points and Arabic vowel marks dropped and the
//! few letters with built-in strokes mapped to their base (`ø`, `ł`, `đ`, `ı`, `æ` → `ae`), and
//! Persian letter forms mapped to the Arabic ones (`ک` → `ك`, `ی` → `ي`, `ة` → `ه`). Decimal
//! digits of every script fold to ASCII (`١٢` → `12`). Invisible format characters (soft
//! hyphens, zero-width joiners, bidi isolates) are dropped too.
//!
//! Dotted and dotless i follow the locale when accents count. In Turkish and Azerbaijani, `I`
//! folds to `ı` and `İ` to `i`; elsewhere both fold to `i` and `ı` stays a letter of its own.
//! Ignoring accents, all four fold to `i`, so `istanbul` finds `İSTANBUL` and `irmak` finds
//! `IRMAK` and `ırmak`. Plain `str::to_lowercase` follows no locale and turns `İ` into `i` plus a
//! combining dot, which nothing typed would match.
//!
//! The Turkish i handling and sigma folding come from PdfCraft's `fold_case` (snipeTR,
//! nevicia2010-hub).
use std::ops::Range;
use icu_normalizer::DecomposingNormalizerBorrowed;
use icu_properties::props::{GraphemeExtend, HangulSyllableType};
use icu_properties::{CodePointMapData, CodePointSetData};
/// Folds text for one locale. Cheap to build and copy.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Folder {
turkic: bool,
accents: bool,
}
impl Folder {
/// Case- and accent-insensitive folding for `locale`, a language code such as `tr`, `pt-br`
/// or `zh-hant` (any case, `-` or `_`).
pub fn new(locale: &str) -> Folder {
let language = locale.split(['-', '_', '.', '@']).next().unwrap_or_default();
Folder { turkic: language.eq_ignore_ascii_case("tr") || language.eq_ignore_ascii_case("az"), accents: false }
}
/// Accents count: `é` no longer matches `e`, and a match can't end on a letter whose accent it
/// leaves out.
pub fn keep_accents(self) -> Folder {
Folder { accents: true, ..self }
}
/// `s` folded, with the map back to `s`.
pub fn fold(&self, s: &str) -> Folded {
let nfkd = DecomposingNormalizerBorrowed::new_nfkd();
let extend = CodePointSetData::new::<GraphemeExtend>();
let hangul = CodePointMapData::<HangulSyllableType>::new();
let mut text = String::with_capacity(s.len());
let mut starts = Vec::new();
// A cluster is a character and the combining marks after it, folded and mapped as one;
// conjoining jamo that spell one syllable are one cluster too.
let mut cluster_start = 0;
let mut chars = s.char_indices();
let mut prev = chars.next().map(|(_, c)| hangul.get(c));
for (i, c) in chars {
let kind = hangul.get(c);
if !extend.contains(c) && !prev.is_some_and(|p| continues_syllable(p, kind)) {
starts.push((text.len(), cluster_start));
self.fold_cluster(&nfkd.normalize(&s[cluster_start..i]), &mut text);
cluster_start = i;
}
prev = Some(kind);
}
if !s.is_empty() {
starts.push((text.len(), cluster_start));
self.fold_cluster(&nfkd.normalize(&s[cluster_start..]), &mut text);
}
starts.push((text.len(), s.len()));
Folded { text, starts }
}
/// `s` folded, without the map: a key to compare or store.
pub fn key(&self, s: &str) -> String {
self.fold(s).text
}
/// Whether `haystack` contains `needle`, both folded.
pub fn contains(&self, haystack: &str, needle: &str) -> bool {
self.fold(haystack).find(&self.fold(needle)).is_some()
}
/// One cluster, already in NFKD.
fn fold_cluster(&self, cluster: &str, out: &mut String) {
let mut chars = cluster.chars().peekable();
while let Some(c) = chars.next() {
match c {
// `İ` decomposes to `I` and a combining dot above: the dot of a lower-case `i`.
'I' if chars.peek() == Some(&'\u{307}') => {
chars.next();
out.push('i');
}
'I' if self.turkic && self.accents => out.push('ı'),
// Persian heh with hamza above (ۀ) decomposes to ae (ە) and hamza.
'\u{6D5}' if !self.accents && chars.peek() == Some(&'\u{654}') => {
chars.next();
out.push('ه');
}
'ß' | 'ẞ' => out.push_str("ss"),
c if let Some(d) = ascii_digit(c) => out.push(d),
'Σ' | 'σ' | 'ς' => out.push('σ'),
c if ignorable(c) || (!self.accents && accent(c)) => {}
c => {
for l in c.to_lowercase() {
match base_letter(l) {
Some(base) if !self.accents => out.push_str(base),
_ => out.push(l),
}
}
}
}
}
}
}
/// The base of a lower-case letter whose diacritic is built in rather than combining, and the
/// Arabic form of a letter that Persian and Urdu write differently: keheh (ک) is kaf (ك), Farsi
/// yeh (ی) and alef maksura (ى) are yeh (ي), teh marbuta (ة) is heh (ه), alef wasla (ٱ) is alef.
fn base_letter(c: char) -> Option<&'static str> {
Some(match c {
'ک' => "ك",
'ی' | 'ى' => "ي",
'ة' => "ه",
'ٱ' => "ا",
'ı' => "i",
'ø' => "o",
'ł' => "l",
'đ' => "d",
'ħ' => "h",
'ŧ' => "t",
'æ' => "ae",
'œ' => "oe",
_ => return None,
})
}
/// Invisible characters that only steer layout: soft hyphen, Arabic letter mark, the kashida
/// that stretches Arabic words, Mongolian variation selectors, zero-width space and joiners, direction marks, embeddings and isolates,
/// word joiner, variation selectors, byte order mark and tags.
fn ignorable(c: char) -> bool {
matches!(c,
'\u{AD}' | '\u{61C}' | '\u{640}' | '\u{180B}'..='\u{180F}' | '\u{200B}'..='\u{200F}' | '\u{202A}'..='\u{202E}'
| '\u{2060}'..='\u{206F}' | '\u{FE00}'..='\u{FE0F}' | '\u{FEFF}' | '\u{E0000}'..='\u{E0FFF}')
}
/// Marks that searching without accents ignores: Latin, Greek and Cyrillic combining diacritics,
/// Hebrew points and cantillation, Arabic vowel marks.
/// Kana voicing marks, Indic and Thai vowel signs are letters' parts, not accents, and stay.
fn accent(c: char) -> bool {
matches!(c,
'\u{300}'..='\u{36F}' | '\u{1AB0}'..='\u{1AFF}' | '\u{1DC0}'..='\u{1DFF}' | '\u{20D0}'..='\u{20FF}'
| '\u{FE20}'..='\u{FE2F}'
| '\u{591}'..='\u{5BD}' | '\u{5BF}' | '\u{5C1}' | '\u{5C2}' | '\u{5C4}' | '\u{5C5}' | '\u{5C7}'
| '\u{610}'..='\u{61A}' | '\u{64B}'..='\u{65F}' | '\u{670}' | '\u{6D6}'..='\u{6DC}'
| '\u{6DF}'..='\u{6E4}' | '\u{6E7}' | '\u{6E8}' | '\u{6EA}'..='\u{6ED}')
}
/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode
/// 16.0.
const ZEROS: [u32; 76] = [
0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0,
0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0,
0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950,
0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6,
0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0,
];
/// A decimal digit of any script as its ASCII digit: `١٢` and `१२` search as `12`.
fn ascii_digit(c: char) -> Option<char> {
let c = u32::from(c);
let zero = match ZEROS.binary_search(&c) {
Ok(i) => ZEROS.get(i),
Err(i) => ZEROS.get(i.checked_sub(1)?),
}?;
let value = c - zero;
if value < 10 { char::from_digit(value, 10) } else { None }
}
/// Whether a conjoining jamo of kind `next` continues the syllable a character of kind `prev`
/// ends: a vowel after a leading consonant, a final consonant after a vowel or an open syllable
/// (`가` and `ᆨ` spell `각`).
fn continues_syllable(prev: HangulSyllableType, next: HangulSyllableType) -> bool {
matches!(
(prev, next),
(HangulSyllableType::LeadingJamo, HangulSyllableType::VowelJamo)
| (HangulSyllableType::VowelJamo | HangulSyllableType::LVSyllable, HangulSyllableType::TrailingJamo)
)
}
/// Folded text and where each part came from.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Folded {
text: String,
/// One (folded offset, source offset) per source cluster, in order, then both lengths. A
/// cluster folds to the text up to the next entry's folded offset, possibly none.
starts: Vec<(usize, usize)>,
}
impl Folded {
/// The folded text.
pub fn as_str(&self) -> &str {
&self.text
}
/// The source byte range that folded bytes `range` came from, widened to whole source
/// characters with their accents: `ss` in the folding of `Straße` maps back to `ß`, `e` in
/// that of `é` to the whole `é`. An empty range maps to an empty one.
pub fn source_range(&self, range: Range<usize>) -> Range<usize> {
// The last cluster starting at or before `at` is the one holding it: clusters that fold to
// nothing share their offset with the next one.
let cluster = |at: usize| self.starts.partition_point(|&(f, _)| f <= at).saturating_sub(1);
if range.is_empty() {
let s = self.starts[cluster(range.start)].1;
return s..s;
}
let first = cluster(range.start);
let last = cluster(range.end - 1);
self.starts[first].1..self.starts[(last + 1).min(self.starts.len() - 1)].1
}
/// The first match of `needle` (folded the same way), as a byte range of the source text.
/// An empty needle matches at the start.
pub fn find(&self, needle: &Folded) -> Option<Range<usize>> {
self.matches(needle).next()
}
/// Every non-overlapping match of `needle` (folded the same way), as byte ranges of the
/// source text. A match may start or end inside an expansion (`ss` of `ß`, `fi` of `fi`) but
/// not leave out a mark that follows a letter, so `re` doesn't match `ré` when accents count.
pub fn matches<'a>(&'a self, needle: &'a Folded) -> impl Iterator<Item = Range<usize>> + 'a {
let extend = CodePointSetData::new::<GraphemeExtend>();
let hangul = CodePointMapData::<HangulSyllableType>::new();
// The vowel and final consonant of a decomposed Hangul syllable continue it, like a mark,
// but only inside one source cluster: standalone jamo (`ㅋㅠ`, `ㅠㅠ`) are letters.
let starts_mark = move |at: usize| {
let Some(c) = self.text[at..].chars().next() else { return false };
if extend.contains(c) {
return true;
}
let inside = self.starts.binary_search_by_key(&at, |&(f, _)| f).is_err();
inside && self.text[..at].chars().next_back().is_some_and(|p| continues_syllable(hangul.get(p), hangul.get(c)))
};
let n = needle.text.as_str();
let mut from = 0;
let mut done = false;
std::iter::from_fn(move || {
if done {
return None;
}
if n.is_empty() {
done = true;
return Some(0..0);
}
while let Some(i) = self.text.get(from..).and_then(|t| t.find(n)) {
let start = from + i;
let end = start + n.len();
if !starts_mark(start) && !starts_mark(end) {
from = end;
return Some(self.source_range(start..end));
}
from = start + self.text[start..].chars().next().map_or(1, char::len_utf8);
}
done = true;
None
})
}
}
#[cfg(test)]
mod tests {
use super::*;
fn key(locale: &str, s: &str) -> String {
Folder::new(locale).key(s)
}
fn key_accents(locale: &str, s: &str) -> String {
Folder::new(locale).keep_accents().key(s)
}
#[test]
fn folding_table() {
let nfd = |s: &str| DecomposingNormalizerBorrowed::new_nfd().normalize(s).into_owned();
// (locale, input, ignoring accents, keeping accents)
let table = [
("en", "Hello WORLD", "hello world".to_owned(), "hello world".to_owned()),
("en", "İSTANBUL", "istanbul".into(), "istanbul".into()),
("tr", "İSTANBUL", "istanbul".into(), "istanbul".into()),
("en", "IRMAK ırmak", "irmak irmak".into(), "irmak ırmak".into()),
("tr", "IRMAK ırmak", "irmak irmak".into(), "ırmak ırmak".into()),
("az", "İIıi", "iiii".into(), "iııi".into()),
("tr-TR", "Şişli", "sisli".into(), nfd("şişli")),
("de", "Straße STRAẞE", "strasse strasse".into(), "strasse strasse".into()),
("el", "ΟΔΥΣΣΕΥΣ οδυσσευς", "οδυσσευσ οδυσσευσ".into(), "οδυσσευσ οδυσσευσ".into()),
("el", "Ἀθῆναι", "αθηναι".into(), nfd("ἀθῆναι")),
("fr", "Œuvre ÉTÉ", "oeuvre ete".into(), nfd("œuvre été")),
("da", "Ærø", "aero".into(), nfd("ærø")),
("pl", "Łódź", "lodz".into(), nfd("łódź")),
("vi", "Đà Nẵng", "da nang".into(), nfd("đà nẵng")),
("en", "final ff", "final ff".into(), "final ff".into()),
("ja", "ＡＢＣ１２３　ｶﾞ", nfd("abc123 ガ"), nfd("abc123 ガ")),
("ar", "مُحَمَّد أحمـــد", nfd("محمد احمد"), nfd("مُحَمَّد أحمد")),
("he", "שָׁלוֹם", "שלום".into(), nfd("שָׁלוֹם")),
("en", "soft\u{AD}ware \u{2068}name\u{2069}", "software name".into(), "software name".into()),
("ru", "Ёлка", "елка".into(), nfd("ёлка")),
("ar", "Layer ١٢ ۳۴ १२", "layer 12 34 12".into(), "layer 12 34 12".into()),
("fa", "کتاب یک", "كتاب يك".into(), "کتاب یک".into()),
("fa", "خانۀ", "خانه".into(), nfd("خانۀ")),
("ar", "مدرسة على إسلام آمن", "مدرسه علي اسلام امن".into(), nfd("مدرسة على إسلام آمن")),
];
for (locale, input, ignoring, keeping) in table {
assert_eq!(key(locale, input), ignoring, "{locale} {input}");
assert_eq!(key_accents(locale, input), keeping, "{locale} {input}, accents kept");
}
}
#[test]
fn turkish_i_depends_on_locale_when_accents_count() {
let tr = Folder::new("tr").keep_accents();
let en = Folder::new("en").keep_accents();
assert!(tr.contains("IRMAK", "ırmak") && !tr.contains("IRMAK", "irmak"));
assert!(en.contains("IRMAK", "irmak") && !en.contains("IRMAK", "ırmak"));
assert!(tr.contains("İSTANBUL", "istanbul") && en.contains("İSTANBUL", "istanbul"));
for locale in ["tr", "en"] {
let f = Folder::new(locale);
assert!(f.contains("IRMAK", "ırmak") && f.contains("ırmak", "IRMAK") && f.contains("İzmir", "izmir"));
}
}
#[test]
fn matches_map_back_to_the_source() {
let f = Folder::new("en");
fn find<'a>(f: Folder, haystack: &'a str, needle: &str) -> Option<&'a str> {
f.fold(haystack).find(&f.fold(needle)).map(|r| &haystack[r])
}
let find = |haystack, needle| find(f, haystack, needle);
assert_eq!(find("Große Straße", "strasse"), Some("Straße"));
assert_eq!(find("Große Straße", "SS"), Some("ß"));
assert_eq!(find("Große Straße", "sse"), Some("ße"));
assert_eq!(find("Café crème", "creme"), Some("crème"));
assert_eq!(find("Cafe\u{301} noir", "cafe"), Some("Cafe\u{301}"));
assert_eq!(find("Ｆｉｌｅ ２", "file 2"), Some("Ｆｉｌｅ ２"));
assert_eq!(find("the final cut", "fin"), Some("fin"));
assert_eq!(find("Layer \u{2068}İSTANBUL\u{2069}", "istanbul"), Some("İSTANBUL"));
assert_eq!(find("abc", "x"), None);
assert_eq!(find("abc", ""), Some(""));
let all: Vec<_> = f.fold("Ab ab AB").matches(&f.fold("ab")).collect();
assert_eq!(all, [0..2, 3..5, 6..8]);
}
#[test]
fn persian_and_arabic_letters_match() {
let f = Folder::new("fa");
// Persian keheh and Farsi yeh against Arabic kaf and yeh, and hamza forms of alef.
assert!(f.contains("كتاب عربي", "کتاب عربی"));
assert!(f.contains("کتاب عربی", "كتاب عربي"));
assert!(f.contains("أحمد إبراهيم", "احمد ابراهیم"));
assert!(f.contains("مدرسة", "مدرسه"));
assert_eq!(f.fold("Layer ١٢").find(&f.fold("layer 12")), Some(0.."Layer ١٢".len()));
assert!(Folder::new("en").keep_accents().contains("Layer ١٢", "layer 12"));
let digits = ZEROS.iter().map(|&zero| char::from_u32(zero + 7).unwrap());
assert!(digits.clone().all(|d| d.is_numeric() && ascii_digit(d) == Some('7')));
assert_eq!(ascii_digit('\u{1D800}'), None);
assert_eq!(ascii_digit('/'), None);
}
#[test]
fn accents_count_only_whole_letters() {
let f = Folder::new("fr").keep_accents();
assert!(!f.contains("ré", "re"));
assert!(f.contains("ré", "RÉ") && f.contains("re\u{301}", "ré"));
assert!(f.contains("ré", "é"));
// Kana voicing marks stay even with accents ignored: か doesn't find が.
assert!(!Folder::new("ja").contains("が", "か"));
// Nor does a Hangul syllable without its final consonant: 하 doesn't find 한.
let ko = Folder::new("ko");
assert!(!ko.contains("한국", "하") && !ko.keep_accents().contains("한국", "하"));
assert!(ko.contains("한국", "한") && ko.contains("하늘", "하"));
// Conjoining jamo spell the same syllable; compatibility jamo stand alone.
assert!(!ko.contains("\u{1112}\u{1161}\u{11AB}", "하") && ko.contains("\u{1112}\u{1161}\u{11AB}", "한"));
assert!(!ko.contains("하\u{11AB}", "하"));
for (haystack, needle) in [("ㅠㅠ", "ㅠ"), ("ㅠㅠ", "ㅠㅠ"), ("ㅋㅠ", "ㅋ"), ("ㅋㅠ", "ㅠ"), ("ㅋㅋㅋ", "ㅋㅋ"), ("ㄳ", "ㄳ"), ("값ㄳ", "ㄳ")]
{
assert!(ko.contains(haystack, needle), "{haystack} {needle}");
}
assert_eq!(ko.fold("ㅋㅠㅠ").find(&ko.fold("ㅠ")).map(|r| r.start), Some("ㅋ".len()));
}
#[test]
fn source_ranges_round_trip() {
let f = Folder::new("de");
for s in ["Straße", "ffi Ａ", "e\u{301}\u{AD}x", "", "İı", "\u{301}a"] {
let folded = f.fold(s);
let text = folded.as_str();
// A leading mark folds to nothing and has no part in any match.
let whole = if s.starts_with('\u{301}') { 2..s.len() } else { 0..s.len() };
assert_eq!(folded.source_range(0..text.len()), whole, "{s:?}");
// Every folded character maps to a non-empty source range that folds back to text
// containing it.
for (i, c) in text.char_indices() {
let r = folded.source_range(i..i + c.len_utf8());
assert!(!r.is_empty() && f.key(&s[r.clone()]).contains(c), "{s:?} {i} {r:?}");
}
}
}
#[test]
fn folding_is_idempotent() {
// A folded query folds to itself, so a stored key can be searched with a fresh fold.
for locale in ["en", "tr"] {
for f in [Folder::new(locale), Folder::new(locale).keep_accents()] {
for c in ('\0'..='\u{FFFF}').chain('\u{1D400}'..='\u{1D7FF}') {
let once = f.key(c.encode_utf8(&mut [0; 4]));
assert_eq!(f.key(&once), once, "{locale} {:?} U+{:04X}", f, c as u32);
}
}
}
}
}
crates/i18n/src/format.rs
+392
View File
@@ -0,0 +1,392 @@
//! Numbers, percentages and counts written for a formatting [`Region`], on ICU4X's decimal
//! formatter with CLDR data for every CLDR locale (`data/generate.sh`).
use std::fmt;
use fixed_decimal::{Decimal, Sign};
use icu_decimal::{DecimalFormatter, parts};
use icu_locale_core::Locale;
use icu_locale_core::extensions::unicode::{key, value};
use writeable::{Part, PartsWrite, Writeable};
use crate::Region;
use crate::data::{self, PERCENT};
use crate::number::Marks;
/// Which digits numbers are written in.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]
pub enum Digits {
/// The locale's own: Arabic-Indic in Egypt (`١٢٣`), Persian in Iran (`۱۲۳`), else `0`–`9`.
#[default]
Locale,
/// `0`–`9` everywhere, for users who read Arabic or Persian with Latin digits.
Latin,
}
/// How numbers are written in a formatting region: digits, decimal and grouping separators,
/// grouping sizes (`1,234,567`, Indian `12,34,567`) and the percent sign. Build it once per region
/// and keep it; [`crate::I18n`] holds one for counts, sizes and measures.
#[derive(Debug)]
pub struct Format {
/// `None` only if the data doesn't load: numbers are then written as Rust writes them.
decimal: Option<DecimalFormatter>,
/// The system's separators where they differ from the locale's; an empty `group` is none.
group: Option<String>,
point: Option<String>,
percent: (&'static str, &'static str),
/// The marks [`Format::parse`] reads.
pub(crate) marks: Marks,
}
impl Format {
/// The format for `region`'s number locale. With Latin digits, a decimal or grouping separator
/// the region carries (Windows' and macOS's own settings) replaces the locale's.
pub fn new(region: &Region, digits: Digits) -> Format {
let mut locale = Locale::try_from_str(&region.numbers).unwrap_or(Locale::UNKNOWN);
if digits == Digits::Latin {
locale.extensions.unicode.keywords.set(key!("nu"), value!("latn"));
}
let decimal = data::provider().and_then(|p| DecimalFormatter::try_new_with_buffer_provider(p, (&locale).into(), Default::default()).ok());
let mut format = Format { decimal, group: None, point: None, percent: ("", "%"), marks: Marks::default() };
// The locale's own separators and digits, read from a number that shows them all.
let probe = format.parts(&Decimal::try_from_str("-1234567.5").unwrap_or_default());
let latin = probe.integer.starts_with(|c: char| c.is_ascii_digit());
if latin {
if let Some(point) = region.decimal.map(String::from).filter(|p| *p != probe.point) {
format.point = Some(point);
}
if let Some(g) = region.grouping.map(|g| g.map(String::from).unwrap_or_default()).filter(|g| *g != probe.group) {
format.group = Some(g);
}
// A grouping mark equal to the decimal mark (`1,234,5`) wouldn't read back. With a
// decimal comma and the locale's grouping, `1,234.5` becomes `1.234,5`; a grouping
// comma the system sets as well (Windows always sets one) leaves no grouping.
let point = format.point.as_deref().unwrap_or(&probe.point);
if format.group.as_deref().unwrap_or(&probe.group) == point {
let swapped = region.grouping.is_none() && probe.point != point;
format.group = Some(if swapped { probe.point.clone() } else { String::new() });
}
}
format.percent = percent_affixes(&region.numbers, latin);
let point = format.point.clone().unwrap_or(probe.point);
let group = format.group.clone().unwrap_or(probe.group);
// Latin digits typed into an Arabic-Indic format take the locale's Latin marks.
let comma = match (latin, digits) {
(true, _) => point == "," || group == ".",
(false, Digits::Locale) => !Format::new(region, Digits::Latin).marks.point,
(false, Digits::Latin) => false,
};
format.marks = Marks::new(&point, &group, !comma);
format
}
/// The format for a locale tag (`de-CH`, `ar-EG`), with the locale's own digits.
pub fn from_tag(tag: &str) -> Format {
Format::new(&Region::from_tag(tag), Digits::Locale)
}
/// A whole number: `1,234,567`, `1.234.567`, `١٬٢٣٤٬٥٦٧`.
pub fn integer(&self, n: impl Into<i128>) -> String {
self.write(&Decimal::from(n.into()))
}
/// `value` rounded to `decimals` digits after the decimal separator; infinities and NaN are
/// `∞`, `-∞` and `NaN`.
pub fn number(&self, value: f64, decimals: usize) -> String {
match to_decimal(value, decimals) {
Some(d) => self.write(&d),
None => non_finite(value).to_owned(),
}
}
/// [`Self::number`] without grouping separators, for values such as pixel sizes and years:
/// `1920`, `١٩٢٠`, `1234,5`.
pub fn number_ungrouped(&self, value: f64, decimals: usize) -> String {
match to_decimal(value, decimals) {
Some(d) => {
let p = self.parts_with(&d, Some(""));
p.sign + &p.text
}
None => non_finite(value).to_owned(),
}
}
/// `ratio` as a percentage with `decimals` digits after the decimal separator: `0.5` is
/// `50%`, `50 %` (French, German), `%50` (Turkish), `٥٠٪؜` (Egyptian Arabic).
pub fn percent(&self, ratio: f64, decimals: usize) -> String {
let Some(d) = to_decimal(ratio * 100.0, decimals) else {
return format!("{}{}{}", self.percent.0, non_finite(ratio), self.percent.1);
};
let p = self.parts(&d);
format!("{}{}{}{}", p.sign, self.percent.0, p.text, self.percent.1)
}
/// `d` as text, the sign in place.
pub(crate) fn write(&self, d: &Decimal) -> String {
let p = self.parts(d);
p.sign + &p.text
}
fn parts(&self, d: &Decimal) -> Written {
self.parts_with(d, self.group.as_deref())
}
/// `d` taken apart, with `group` in place of the locale's grouping separator if set (`""`
/// drops it).
fn parts_with(&self, d: &Decimal, group: Option<&str>) -> Written {
let mut sink = Sink { group, point: self.point.as_deref(), into: Target::Text, w: Written::default() };
let written = match &self.decimal {
Some(f) => f.format(d).write_to_parts(&mut sink),
None => fmt::Write::write_str(&mut sink, &d.to_string()),
};
if written.is_err() {
return Written { text: d.to_string(), ..Written::default() };
}
sink.w
}
}
/// A formatted number taken apart: its sign, the rest, and the separators and integer digits as
/// the locale wrote them.
#[derive(Default)]
struct Written {
sign: String,
text: String,
integer: String,
group: String,
point: String,
}
#[derive(Clone, Copy, PartialEq)]
enum Target {
Text,
Sign,
/// A separator the region replaces; the locale's goes to `group` or `point` only.
Replaced,
}
/// Writes the formatter's parts into [`Written`], swapping in the region's separators.
struct Sink<'a> {
group: Option<&'a str>,
point: Option<&'a str>,
into: Target,
w: Written,
}
impl fmt::Write for Sink<'_> {
fn write_str(&mut self, s: &str) -> fmt::Result {
match self.into {
Target::Text => self.w.text.push_str(s),
Target::Sign => self.w.sign.push_str(s),
Target::Replaced => {}
}
Ok(())
}
}
impl PartsWrite for Sink<'_> {
type SubPartsWrite = Self;
fn with_part(&mut self, part: Part, mut f: impl FnMut(&mut Self) -> fmt::Result) -> fmt::Result {
let outer = self.into;
let start = self.w.text.len();
let replacement = match part {
p if p == parts::GROUP => self.group,
p if p == parts::DECIMAL => self.point,
_ => None,
};
if let Some(r) = replacement {
self.w.text.push_str(r);
self.into = Target::Replaced;
} else if part == parts::MINUS_SIGN || part == parts::PLUS_SIGN {
self.into = Target::Sign;
}
let result = f(self);
self.into = outer;
if replacement.is_none() {
let written = self.w.text.get(start..).unwrap_or_default();
let slot = match part {
p if p == parts::GROUP => &mut self.w.group,
p if p == parts::DECIMAL => &mut self.w.point,
p if p == parts::INTEGER => &mut self.w.integer,
_ => return result,
};
if slot.is_empty() {
*slot = written.to_owned();
}
}
result
}
}
/// `value` rounded to `decimals` digits after the point; `None` for infinities and NaN. A value
/// that rounds to zero has no sign.
pub(crate) fn to_decimal(value: f64, decimals: usize) -> Option<Decimal> {
if !value.is_finite() {
return None;
}
let mut d = Decimal::try_from_str(&format!("{value:.decimals$}")).ok()?;
if d.absolute.is_zero() {
d.sign = Sign::None;
}
Some(d)
}
/// CLDR's root symbols for a value that isn't finite.
fn non_finite(value: f64) -> &'static str {
match value {
f64::INFINITY => "∞",
f64::NEG_INFINITY => "-∞",
_ => "NaN",
}
}
/// The percent affixes for a number locale (`ar-EG`), found by dropping trailing subtags.
fn percent_affixes(tag: &str, latin: bool) -> (&'static str, &'static str) {
let mut tag = tag;
loop {
if let Some((_, _, before, after)) = PERCENT.iter().find(|(l, lat, _, _)| *lat == latin && l.eq_ignore_ascii_case(tag)) {
return (before, after);
}
match tag.rsplit_once('-') {
Some((parent, _)) => tag = parent,
None => return ("", "%"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn latin(tag: &str) -> Format {
Format::new(&Region::from_tag(tag), Digits::Latin)
}
#[test]
fn locale_digits_and_separators() {
for (tag, integer, number) in [
("en", "1,234,567", "1,234.5"),
("ja", "1,234,567", "1,234.5"),
("de", "1.234.567", "1.234,5"),
("fr", "1\u{202f}234\u{202f}567", "1\u{202f}234,5"),
("hi", "12,34,567", "1,234.5"),
("ar-EG", "١٬٢٣٤٬٥٦٧", "١٬٢٣٤٫٥"),
("fa", "۱٬۲۳۴٬۵۶۷", "۱٬۲۳۴٫۵"),
] {
let f = Format::from_tag(tag);
assert_eq!((f.integer(1_234_567), f.number(1234.5, 1)), (integer.into(), number.into()), "{tag}");
}
// CLDR writes Arabic without a region in Latin digits.
assert_eq!(Format::from_tag("ar").number(1234.5, 1), "1,234.5");
}
#[test]
fn ungrouped_numbers() {
for (tag, want) in [("en", "1920"), ("de", "-1234567,5"), ("hi", "1234567"), ("ar-EG", "١٩٢٠")] {
let value = if tag == "de" {
-1_234_567.5
} else if tag == "hi" {
1_234_567.0
} else {
1920.0
};
let decimals = usize::from(tag == "de");
assert_eq!(Format::from_tag(tag).number_ungrouped(value, decimals), want, "{tag}");
}
let mut region = Region::from_tag("en-US");
region.decimal = Some(',');
assert_eq!(Format::new(&region, Digits::Locale).number_ungrouped(1234.5, 1), "1234,5", "the system's decimal mark stays");
assert_eq!(Format::from_tag("en").number_ungrouped(f64::INFINITY, 0), "∞");
}
#[test]
fn forced_latin_digits() {
assert_eq!(latin("ar-EG").number(1234.5, 1), "1,234.5");
assert_eq!(latin("fa").integer(1_234_567), "1,234,567");
assert_eq!(latin("de").number(1234.5, 1), "1.234,5", "a Latin locale is unchanged");
}
#[test]
fn cldr_separators_for_every_region() {
for (tag, want) in [
("en-PL", "1.234.567,5"),
("en-DE", "1.234.567,5"),
("ar-MA", "1.234.567,5"),
("de-CH", "1'234'567.5"),
("nb-NO", "1\u{a0}234\u{a0}567,5"),
("sk-SK", "1\u{a0}234\u{a0}567,5"),
("bn-BD", "১২,৩৪,৫৬৭.৫"),
("he-IL", "1,234,567.5"),
("km", "1,234,567.5"),
("lo", "1.234.567,5"),
] {
assert_eq!(Format::from_tag(tag).number(1_234_567.5, 1), want, "{tag}");
}
}
#[test]
fn system_separators() {
// A decimal comma the system sets moves the grouping out of its way.
let mut region = Region::from_tag("en-US");
region.decimal = Some(',');
assert_eq!(Format::new(&region, Digits::Locale).number(1234.5, 1), "1.234,5");
let mut region = Region::from_tag("de-DE");
region.grouping = Some(Some('\u{a0}'));
assert_eq!(Format::new(&region, Digits::Locale).number(1234.5, 1), "1\u{a0}234,5");
region.decimal = Some('.');
assert_eq!(Format::new(&region, Digits::Locale).number(1234.5, 1), "1\u{a0}234.5");
region.grouping = Some(None);
assert_eq!(Format::new(&region, Digits::Locale).integer(1_234_567), "1234567", "no grouping");
// A system's separators don't apply to Arabic-Indic digits.
let mut region = Region::from_tag("ar-EG");
region.grouping = Some(Some(','));
assert_eq!(Format::new(&region, Digits::Locale).number(1234.5, 1), "١٬٢٣٤٫٥");
}
/// Windows always sets `sThousand`, so a decimal comma the user picks arrives beside a grouping
/// comma: the numbers drop grouping rather than read `1,234,5`.
#[test]
fn a_system_grouping_mark_equal_to_the_decimal_mark_is_dropped() {
let reg = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string());
let region = crate::region::from_windows(reg(&[("LocaleName", "en-US"), ("sDecimal", ","), ("sThousand", ",")]), || None);
let format = Format::new(&region, Digits::Locale);
assert_eq!((format.number(1234.5, 1), format.integer(1_234_567)), ("1234,5".to_owned(), "1234567".to_owned()));
let region = crate::region::from_windows(reg(&[("LocaleName", "de-DE"), ("sDecimal", ","), ("sThousand", ",")]), || None);
assert_eq!(Format::new(&region, Digits::Locale).number(1234.5, 1), "1234,5");
let region = crate::region::from_windows(reg(&[("LocaleName", "en-US"), ("sDecimal", "."), ("sThousand", ".")]), || None);
assert_eq!(Format::new(&region, Digits::Locale).number(1234.5, 1), "1234.5");
}
#[test]
fn a_digit_system_the_region_names_is_used() {
assert_eq!(Format::new(&Region::from_tag("ar-EG-u-nu-latn"), Digits::Locale).number(1234.5, 1), "1,234.5");
assert_eq!(Format::new(&Region::from_tag("ar_EG@numbers=latn"), Digits::Locale).integer(5), "5");
assert_eq!(Format::new(&Region::from_tag("en-US-u-nu-arab"), Digits::Locale).integer(5), "٥");
}
#[test]
fn percentages() {
for (tag, want) in
[("en", "50%"), ("de", "50\u{a0}%"), ("fr", "12,5\u{a0}%"), ("tr", "%50"), ("ar-EG", "٥٠٪\u{61c}"), ("fa", "۵۰٪"), ("hi", "50%")]
{
let decimals = if tag == "fr" { 1 } else { 0 };
let ratio = if tag == "fr" { 0.125 } else { 0.5 };
assert_eq!(Format::from_tag(tag).percent(ratio, decimals), want, "{tag}");
}
assert_eq!(Format::from_tag("tr").percent(-0.2, 0), "-%20", "the sign leads the pattern");
assert_eq!(latin("fa").percent(0.5, 0), "50%");
}
#[test]
fn signs_and_odd_values() {
let en = Format::from_tag("en");
assert_eq!(en.number(-1234.5, 0), "-1,234");
assert_eq!(en.number(-0.04, 1), "0.0", "no sign on zero");
assert_eq!(en.number(f64::INFINITY, 1), "∞");
assert_eq!(en.number(f64::NEG_INFINITY, 1), "-∞");
assert_eq!(en.number(f64::NAN, 1), "NaN");
assert_eq!(Format::from_tag("de").percent(f64::NAN, 0), "NaN\u{a0}%");
assert_eq!(en.integer(-5_i64), "-5");
}
}
crates/i18n/src/lib.rs
+425 -45
View File
@@ -3,18 +3,60 @@
//! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time
//! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a
//! [`Lang`]. A missing translation is the English itself; nothing here fails.
//!
//! Counts ([`I18n::trn`]), file sizes and measures write their numbers as the formatting
//! [`Region`] does ([`I18n::format`]): grouped (`1.234`, `12,34,567`) and in the region's digits
//! (`١٬٢٣٤`). Text that goes into documents or file names should use [`I18n::trn_plain`], which
//! writes `0`–`9` without grouping.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod catalog;
#[cfg(feature = "collate")]
mod collate;
mod fold;
mod format;
mod number;
mod region;
mod scripts;
mod size;
/// The CLDR number and plural data for every CLDR locale, from `data/generate.sh`.
mod data {
use std::sync::OnceLock;
use icu_locale_fallback::LocaleFallbacker;
use icu_provider_adapters::fallback::LocaleFallbackProvider;
use icu_provider_blob::BlobDataProvider;
include!("../data/locales.rs");
/// The number and plural data, read once; `None` if the blob doesn't load.
pub(crate) fn provider() -> Option<&'static LocaleFallbackProvider<BlobDataProvider>> {
static PROVIDER: OnceLock<Option<LocaleFallbackProvider<BlobDataProvider>>> = OnceLock::new();
PROVIDER
.get_or_init(|| {
let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/numbers.postcard")).ok()?;
Some(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned()))
})
.as_ref()
}
}
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::{Mutex, OnceLock, PoisonError};
use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock};
use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules};
use unic_langid::LanguageIdentifier;
use icu_locale_core::Locale as IcuLocale;
use icu_plurals::{PluralCategory, PluralOperands, PluralRules};
use unicode_bidi::{BidiClass, bidi_class};
#[cfg(feature = "collate")]
pub use collate::Collator;
pub use fold::{Folded, Folder};
pub use format::{Digits, Format};
pub use number::{NumberError, parse_number};
pub use region::{HourCycle, Measurement, Paper, Region};
pub use scripts::{Direction, Han, Script, base_direction, direction, han, scripts};
pub use size::{FileSize, SizeUnit};
/// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog).
/// A missing catalog is `""`.
@@ -104,6 +146,8 @@ pub struct I18n {
pseudo: Mutex<BTreeMap<String, &'static str>>,
/// [`I18n::system_lang`], read once.
system: OnceLock<Lang>,
/// [`I18n::format`]; the system region's until [`I18n::set_format`].
format: RwLock<Option<Arc<Format>>>,
}
impl I18n {
@@ -147,7 +191,7 @@ impl I18n {
l.chain = chain;
}
}
(I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems)
(I18n { locales, pseudo: Mutex::default(), system: OnceLock::new(), format: RwLock::default() }, problems)
}
/// Parse every locale and list every problem, `<locale>/<common|app>: line N: …`, for a test or
@@ -209,9 +253,36 @@ impl I18n {
self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label)
}
/// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms;
/// they are also the key of the `@plural` row, and the fallback.
/// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers: grouped and
/// in the region's digits, as a left-to-right isolate ([`ltr`]) when the wording has a
/// right-to-left letter. `one` and `other` are the English forms; they are also the key of the
/// `@plural` row, and the fallback.
pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String {
let format = self.format();
self.plural(lang, Some(PluralOperands::from(n)), n == 1, &format.integer(n), one, other)
}
/// [`I18n::trn`] with `{n}` in `0`–`9` without grouping (`Copy 1234`) and no isolate marks
/// ([`strip_isolates`]), for text that goes into documents, file names or anything read back
/// later.
pub fn trn_plain(&self, lang: Lang, n: u64, one: &str, other: &str) -> String {
strip_isolates(&self.plural(lang, Some(PluralOperands::from(n)), n == 1, &n.to_string(), one, other))
}
/// [`I18n::trn`] for a count with `decimals` digits after the decimal separator (`1.5 files`).
/// The plural form follows the digits shown, so `1.0` takes English `other`; infinities and NaN
/// (`∞`, `NaN`) take `other` too.
pub fn trn_decimal(&self, lang: Lang, n: f64, decimals: usize, one: &str, other: &str) -> String {
let format = self.format();
let Some(d) = format::to_decimal(n, decimals) else {
return self.plural(lang, None, false, &format.number(n, decimals), one, other);
};
let english_one = d.to_string() == "1";
self.plural(lang, Some(PluralOperands::from(&d)), english_one, &format.write(&d), one, other)
}
/// `operands` `None` is the `other` category.
fn plural(&self, lang: Lang, operands: Option<PluralOperands>, english_one: bool, n: &str, one: &str, other: &str) -> String {
let key = format!("{one}|{other}");
let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice());
let text = chain
@@ -220,19 +291,45 @@ impl I18n {
let locale = self.locales.get(i)?;
let forms = locale.parsed().rows.get("@plural")?.get(&key)?;
let (rules, categories) = locale.plural.as_ref()?;
let category = rules.select(n).ok()?;
// A category whole numbers up to 1000 never reach (French `many`) takes the last form.
let index = categories.iter().position(|c| *c == category).unwrap_or(categories.len().saturating_sub(1));
let category = operands.map_or(PluralCategory::Other, |o| rules.category_for(o));
// A category whole numbers up to 1000 never reach has no form of its own. Russian
// and Ukrainian decimals (`other`) take the `few` form (`1,5 файла`); any other
// (French `many` for a million) takes the last.
let index = categories
.iter()
.position(|c| *c == category)
.or_else(|| categories.iter().position(|c| *c == PluralCategory::Few))
.unwrap_or(categories.len().saturating_sub(1));
forms.split('|').nth(index)
})
.unwrap_or(if n == 1 { one } else { other });
let text = fmt(text, &[("n", &n.to_string())]);
.unwrap_or(if english_one { one } else { other });
let text = fmt(text, &[("n", &ltr(n))]);
if lang == Lang::PSEUDO {
return marked(&text);
}
text
}
/// How counts, file sizes and measures write numbers: [`Region::system`]'s format with the
/// locale's digits until [`I18n::set_format`], and again after [`I18n::reset_format`].
pub fn format(&self) -> Arc<Format> {
if let Some(f) = self.format.read().unwrap_or_else(PoisonError::into_inner).as_ref() {
return Arc::clone(f);
}
let mut slot = self.format.write().unwrap_or_else(PoisonError::into_inner);
Arc::clone(slot.get_or_insert_with(|| Arc::new(Format::new(&Region::system(), Digits::Locale))))
}
/// Write numbers as `format` does from now on: the user's region setting, or Latin digits.
pub fn set_format(&self, format: Format) {
*self.format.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(format));
}
/// Write numbers as the system region does again, undoing [`I18n::set_format`].
pub fn reset_format(&self) {
*self.format.write().unwrap_or_else(PoisonError::into_inner) = None;
}
/// The languages to offer: English first, then the offered locales by code. Never the
/// pseudo locale.
pub fn languages(&self) -> Vec<Lang> {
@@ -311,29 +408,168 @@ impl I18n {
}
}
/// Fill `{name}` placeholders from `args` in one pass. Unknown placeholders stay as written, and
/// inserted values are never read as templates, so a file name containing `{n}` stays intact.
/// Fill `{name}` placeholders from `args` in one pass. When the template or an inserted value
/// has a right-to-left letter (Hebrew, Arabic and the like), or a value has a bidi control
/// (an embedding, override, isolate or mark), each value goes in as a first-strong isolate
/// (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name in an
/// Arabic message stays where the translator put it, and an override in a file name ends with
/// the name. Embeddings, overrides and isolates a value opens are closed inside its isolate. A
/// value that already is one isolate keeps it; pass numbers, paths and codes through [`ltr`].
/// Other text gets no marks, and a value's own isolate is taken off. Unknown placeholders stay as written,
/// and inserted values are never read as templates, so a file name containing `{n}` stays intact.
pub fn fmt(template: &str, args: &[(&str, &str)]) -> String {
let mut out = String::with_capacity(template.len());
fill(template, args, true)
}
/// [`fmt`] without isolates, for text that goes where the marks would get in the way: file
/// names, logs, terminals, text compared or parsed later. Every isolate mark is taken off, in
/// the template and in the values.
pub fn fmt_plain(template: &str, args: &[(&str, &str)]) -> String {
strip_isolates(&fill(template, args, false))
}
const LRI: char = '\u{2066}';
const RLI: char = '\u{2067}';
const FSI: char = '\u{2068}';
const PDI: char = '\u{2069}';
const LRE: char = '\u{202A}';
const RLE: char = '\u{202B}';
const PDF: char = '\u{202C}';
const LRO: char = '\u{202D}';
const RLO: char = '\u{202E}';
const LRM: char = '\u{200E}';
const RLM: char = '\u{200F}';
const ALM: char = '\u{061C}';
/// `value` as a first-strong isolate (FSI…PDI): it takes the direction of its first strong
/// letter and can't reorder the text around it. What [`fmt`] does to each value in text with a
/// right-to-left letter.
pub fn isolate(value: &str) -> String {
wrap(FSI, value)
}
/// `value` as a left-to-right isolate (LRI…PDI), for numbers, paths, file sizes, timecodes and
/// codes, which read left to right in every language. In an RTL first-strong isolate, the
/// segments of `/home/مستخدم/file.txt` would run right to left. [`fmt`] takes the isolate off
/// again in text with no right-to-left letter.
pub fn ltr(value: &str) -> String {
wrap(LRI, value)
}
/// `text` without isolate marks (LRI, RLI, FSI, PDI): a [`fmt`] or [`I18n::trn`] result made fit
/// for a file name, a log or a comparison.
pub fn strip_isolates(text: &str) -> String {
text.chars().filter(|c| !matches!(*c, LRI | RLI | FSI | PDI)).collect()
}
/// `value` between `initiator` and PDI, with every embedding, override and isolate it opens
/// closed inside: a stray PDF or PDI is dropped, and open ones are closed in order. Under the
/// bidi algorithm the outer PDI alone would close them, but a renderer that handles only
/// embeddings would let them run on into the rest of the text.
fn wrap(initiator: char, value: &str) -> String {
let mut out = String::with_capacity(value.len() + 6);
out.push(initiator);
// One entry per open initiator: true for an isolate, false for an embedding or override.
let mut open: Vec<bool> = Vec::new();
for c in value.chars() {
match c {
LRI | RLI | FSI => open.push(true),
LRE | RLE | LRO | RLO => open.push(false),
PDF if open.last() == Some(&false) => {
open.pop();
}
PDF => continue,
PDI if open.contains(&true) => {
while open.pop() == Some(false) {
out.push(PDF);
}
}
PDI => continue,
_ => {}
}
out.push(c);
}
while let Some(isolate) = open.pop() {
out.push(if isolate { PDI } else { PDF });
}
out.push(PDI);
out
}
/// Whether `value`, without its own isolate, has a bidi control: an embedding, override,
/// isolate or mark that could reorder the text around it unless it is contained.
fn has_bidi_control(value: &str) -> bool {
!value.is_ascii() && unwrap_isolate(value).chars().any(|c| matches!(c, LRE | RLE | PDF | LRO | RLO | LRI | RLI | FSI | PDI | LRM | RLM | ALM))
}
/// Whether `text` has a strong right-to-left character (bidi class R or AL): the text that needs
/// isolates.
fn has_rtl(text: &str) -> bool {
!text.is_ascii() && text.chars().any(|c| matches!(bidi_class(c), BidiClass::R | BidiClass::AL))
}
/// `value` without its isolate, if it is exactly one.
fn unwrap_isolate(value: &str) -> &str {
if is_one_isolate(value) {
// The initiator and PDI are three bytes each in UTF-8.
value.get(3..value.len() - 3).unwrap_or(value)
} else {
value
}
}
/// Whether `value` is exactly one isolate: an initiator, then the PDI that closes it last.
fn is_one_isolate(value: &str) -> bool {
let mut depth = 0_usize;
let mut chars = value.chars().peekable();
while let Some(c) = chars.next() {
match c {
LRI | RLI | FSI => depth += 1,
PDI if depth > 0 => {
depth -= 1;
if depth == 0 {
return chars.peek().is_none();
}
}
_ if depth == 0 => return false,
_ => {}
}
}
false
}
fn fill(template: &str, args: &[(&str, &str)], isolated: bool) -> String {
let mut pieces: Vec<(&str, bool)> = Vec::new();
let mut rest = template;
while let Some((before, after_open)) = rest.split_once('{') {
out.push_str(before);
pieces.push((before, false));
let Some((name, after_close)) = after_open.split_once('}') else {
out.push('{');
pieces.push(("{", false));
rest = after_open;
break;
};
match args.iter().find(|(k, _)| *k == name) {
Some((_, value)) => out.push_str(value),
None => {
out.push('{');
out.push_str(name);
out.push('}');
}
Some((_, value)) => pieces.push((value, true)),
None => pieces.extend([("{", false), (name, false), ("}", false)]),
}
rest = after_close;
}
out.push_str(rest);
pieces.push((rest, false));
let isolated = isolated && pieces.iter().any(|(text, value)| has_rtl(text) || (*value && has_bidi_control(text)));
let mut out = String::with_capacity(template.len());
for (text, value) in pieces {
match (value, isolated) {
(false, _) => out.push_str(text),
(true, false) => out.push_str(unwrap_isolate(text)),
(true, true) if text.is_empty() => {}
// Rebuilt rather than copied, so what it opens inside is closed too.
(true, true) if is_one_isolate(text) => {
let initiator = text.chars().next().unwrap_or(FSI);
out.push_str(&wrap(initiator, unwrap_isolate(text)));
}
(true, true) => out.push_str(&isolate(text)),
}
}
out
}
@@ -437,17 +673,22 @@ fn bare(code: &str) -> Locale {
}
/// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order
/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall
/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other
/// than `pt-br` takes `pt-PT`'s.
/// (zero, one, two, few, many, other): one `@plural` form each. The rules are looked up by
/// language alone: CLDR's parent of `sr-Latn` and `zh-Hant` is the root, whose only category is
/// `other`. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other than `pt-br` takes
/// `pt-PT`'s. `None` for a language CLDR has no plural rules for.
fn plural_rules(code: &str) -> Option<(PluralRules, Vec<PluralCategory>)> {
let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code };
let id: LanguageIdentifier = code.parse().ok()?;
let language = LanguageIdentifier::from_parts(id.language, None, None, &[]);
let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?;
let id = IcuLocale::try_from_str(code).ok()?.id;
let language = id.language.as_str();
if !data::PLURAL_LANGUAGES.contains(&language) {
return None;
}
let brazil = id.region.is_some_and(|r| r.as_str() == "BR");
let locale = IcuLocale::try_from_str(if language == "pt" && !brazil { "pt-PT" } else { language }).ok()?;
let rules = PluralRules::try_new_cardinal_with_buffer_provider(data::provider()?, (&locale).into()).ok()?;
let mut categories = Vec::new();
for n in 0..=1000_u32 {
let c = rules.select(n).ok()?;
let c = rules.category_for(n);
if !categories.contains(&c) {
categories.push(c);
}
@@ -458,12 +699,12 @@ fn plural_rules(code: &str) -> Option<(PluralRules, Vec<PluralCategory>)> {
fn rank(c: &PluralCategory) -> u8 {
match c {
PluralCategory::ZERO => 0,
PluralCategory::ONE => 1,
PluralCategory::TWO => 2,
PluralCategory::FEW => 3,
PluralCategory::MANY => 4,
PluralCategory::OTHER => 5,
PluralCategory::Zero => 0,
PluralCategory::One => 1,
PluralCategory::Two => 2,
PluralCategory::Few => 3,
PluralCategory::Many => 4,
PluralCategory::Other => 5,
}
}
@@ -487,10 +728,12 @@ mod tests {
("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""),
];
/// `embedded`, offering every locale in it.
/// `embedded`, offering every locale in it, with `en-US` numbers rather than the machine's.
fn all(embedded: &'static Embedded) -> (I18n, Vec<String>) {
let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect();
I18n::new(embedded, &codes)
let (i, problems) = I18n::new(embedded, &codes);
i.set_format(Format::from_tag("en-US"));
(i, problems)
}
fn i18n() -> I18n {
@@ -561,6 +804,8 @@ mod tests {
("fr", "@plural\t{n} page|{n} pages\t{n} page|{n} pages\n", ""),
("ru", "@plural\t{n} file|{n} files\t{n} файл|{n} файла|{n} файлов\n", ""),
("ja", "@plural\t{n} file|{n} files\t{n} 個\n", ""),
("ar", "@plural\t{n} file|{n} files\t{n} ملفات|{n} ملف واحد|{n} ملفان|{n} ملفات|{n} ملفًا|{n} ملف\n", ""),
("fa", "@plural\t{n} page|{n} pages\t{n} صفحه|{n} صفحه\n", ""),
];
#[test]
@@ -569,6 +814,76 @@ mod tests {
assert_eq!(fmt("{z} {", &[]), "{z} {");
}
#[test]
fn left_to_right_text_gets_no_marks() {
assert_eq!(fmt("Saved {name} in {dir}.", &[("name", "Report 2.pdf"), ("dir", &ltr("/home/sara"))]), "Saved Report 2.pdf in /home/sara.");
assert_eq!(fmt("{a}", &[("a", "x"), ("b", "שלום")]), "x", "an unused value doesn't count");
assert_eq!(fmt_plain("{a}", &[("a", &isolate("x"))]), "x");
assert_eq!(fmt_plain("\u{2066}{a} {b}", &[("a", "x\u{2069}"), ("b", &isolate(&ltr("y")))]), "x y");
let i = all(P).0;
assert_eq!(i.trn(Lang::EN, 2, "{n} page", "{n} pages"), "2 pages");
}
#[test]
fn right_to_left_text_isolates_values() {
let saved = fmt("تم حفظ {name} في {dir}.", &[("name", "Report 2.pdf"), ("dir", &ltr("/home/سارة/docs"))]);
assert_eq!(saved, "تم حفظ \u{2068}Report 2.pdf\u{2069} في \u{2066}/home/سارة/docs\u{2069}.");
assert_eq!(strip_isolates(&saved), "تم حفظ Report 2.pdf في /home/سارة/docs.");
assert_eq!(fmt("Opened {name}.", &[("name", "שלום.txt")]), "Opened \u{2068}שלום.txt\u{2069}.");
assert_eq!(fmt("[{a}] {b}", &[("a", ""), ("b", "ש")]), "[] \u{2068}ש\u{2069}");
let i = all(P).0;
assert_eq!(i.trn(lang(&i, "fa"), 3, "{n} page", "{n} pages"), "\u{2066}3\u{2069} صفحه");
}
#[test]
fn a_value_cannot_escape_its_isolate() {
assert_eq!(isolate("a\u{2069}b"), "\u{2068}ab\u{2069}");
assert_eq!(isolate("a\u{2067}b"), "\u{2068}a\u{2067}b\u{2069}\u{2069}");
let one = ltr("12");
assert_eq!(fmt("ש {n}", &[("n", &one)]), format!("ש {one}"));
// Two isolates side by side are not one, so they get wrapped.
assert_eq!(fmt("ש{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "ש\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}");
}
/// `text` in display order, as one line in a left-to-right paragraph, without bidi controls.
fn visual(text: &str) -> String {
let info = unicode_bidi::BidiInfo::new(text, Some(unicode_bidi::Level::ltr()));
let para = &info.paragraphs[0];
let shown = info.reorder_line(para, para.range.clone());
shown
.chars()
.filter(|c| bidi_class(*c) != BidiClass::BN && !"\u{202A}\u{202B}\u{202C}\u{202D}\u{202E}\u{2066}\u{2067}\u{2068}\u{2069}".contains(*c))
.collect()
}
#[test]
fn a_value_with_bidi_controls_cannot_reorder_the_sentence() {
let file_names = [
"\u{202E}fdp.exe",
"\u{202B}abc def",
"\u{202D}abc def",
"\u{2067}abc def",
"\u{2068}abc\u{2069}\u{202E}def",
"\u{202E}abc\u{2069}",
"x\u{200F}",
"x\u{061C}",
];
for name in file_names {
let text = fmt("Open {name} -> {name} now?", &[("name", name)]);
let shown = visual(&text);
assert!(shown.starts_with("Open ") && shown.contains(" -> ") && shown.ends_with(" now?"), "{name:?}: {shown:?}");
let i = all(P).0;
let wording = i.trn(Lang::EN, 2, "{n} page in {name}, done", "{n} pages in {name}, done");
let shown = visual(&fmt(&wording, &[("name", name)]));
assert!(shown.starts_with("2 pages in ") && shown.ends_with(", done"), "{name:?}: {shown:?}");
}
// An override the value opens is closed inside its isolate, and a stray PDF dropped.
assert_eq!(fmt("Open {name}.", &[("name", "\u{202E}abc")]), "Open \u{2068}\u{202E}abc\u{202C}\u{2069}.");
assert_eq!(fmt("Open {name}.", &[("name", "a\u{202C}b")]), "Open \u{2068}ab\u{2069}.");
assert_eq!(fmt("Open {n}.", &[("n", &ltr("\u{202B}1"))]), "Open \u{2066}\u{202B}1\u{202C}\u{2069}.");
assert_eq!(isolate("\u{2067}a\u{202E}b\u{2069}c"), "\u{2068}\u{2067}a\u{202E}b\u{202C}\u{2069}c\u{2069}");
}
#[test]
fn one_other() {
let (i, problems) = all(P);
@@ -603,7 +918,56 @@ mod tests {
#[test]
fn beyond_sweep() {
let i = all(P).0;
assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages");
i.set_format(Format::from_tag("fr"));
assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1\u{202f}000\u{202f}000 pages");
}
#[test]
fn counts_follow_the_region() {
let i = all(P).0;
i.set_format(Format::from_tag("ar-EG"));
assert_eq!(i.trn(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "\u{2066}١٬٢٣٤\u{2069} ملفًا");
i.set_format(Format::new(&Region::from_tag("ar-EG"), Digits::Latin));
assert_eq!(i.trn(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "\u{2066}1,234\u{2069} ملفًا");
// An English interface with a German region.
i.set_format(Format::from_tag("de-DE"));
assert_eq!(i.trn(Lang::EN, 12_345, "{n} file", "{n} files"), "12.345 files");
}
#[test]
fn plain_counts_and_the_system_format() {
let i = all(P).0;
i.set_format(Format::from_tag("ar-EG"));
assert_eq!(i.trn_plain(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "1234 ملفًا");
i.set_format(Format::from_tag("de-DE"));
assert_eq!(i.trn_plain(Lang::EN, 12_345, "copy {n}", "copy {n}"), "copy 12345");
i.reset_format();
assert_eq!(
i.trn(Lang::EN, 5, "{n} file", "{n} files"),
fmt("{n} files", &[("n", &Format::new(&Region::system(), Digits::Locale).integer(5))])
);
}
#[test]
fn decimal_counts() {
let i = all(P).0;
i.set_format(Format::from_tag("en"));
assert_eq!(i.trn_decimal(Lang::EN, 1.5, 1, "{n} file", "{n} files"), "1.5 files");
assert_eq!(i.trn_decimal(Lang::EN, 1.0, 1, "{n} file", "{n} files"), "1.0 files", "visible decimals are plural");
assert_eq!(i.trn_decimal(Lang::EN, 1.0, 0, "{n} file", "{n} files"), "1 file");
// Russian decimals are `other`, which integers never reach: the `few` form.
i.set_format(Format::from_tag("ru"));
let ru = lang(&i, "ru");
assert_eq!(i.trn_decimal(ru, 1.5, 1, "{n} file", "{n} files"), "1,5 файла");
assert_eq!(i.trn_decimal(ru, 21.0, 0, "{n} file", "{n} files"), "21 файл");
// Arabic decimals are `other`.
i.set_format(Format::from_tag("ar-EG"));
let ar = lang(&i, "ar");
assert_eq!(i.trn_decimal(ar, 2.5, 1, "{n} file", "{n} files"), "\u{2066}٢٫٥\u{2069} ملف");
assert_eq!(i.trn_decimal(ar, 3.0, 0, "{n} file", "{n} files"), "\u{2066}٣\u{2069} ملفات");
// Not a number: the translated `other` form.
assert_eq!(i.trn_decimal(ru, f64::NAN, 1, "{n} file", "{n} files"), "NaN файла");
assert_eq!(i.trn_decimal(Lang::EN, f64::INFINITY, 1, "{n} file", "{n} files"), "∞ files");
}
static L: &Embedded =
@@ -770,15 +1134,31 @@ mod tests {
assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo");
}
/// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog
/// whose `@plural` rows list any other.
/// The counts craft-i18n's checker expects, which rejects a catalog whose `@plural` rows list
/// any other: every directory in craft-i18n's `locales/`, then languages it may add.
#[test]
fn plural_form_counts() {
for (locale, n) in
[("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)]
{
#[rustfmt::skip]
let catalogs = [
("ar", 6), ("bg", 2), ("cs", 3), ("da", 2), ("de", 2), ("el", 2), ("es", 2), ("et", 2),
("fi", 2), ("fr", 2), ("hr", 3), ("hu", 2), ("id", 1), ("it", 2), ("ja", 1), ("ka", 2),
("kk", 2), ("ko", 1), ("lt", 3), ("nl", 2), ("pl", 3), ("pt-br", 2), ("ru", 3), ("sl", 4),
("sr", 3), ("sr-latn", 3), ("sv", 2), ("te", 2), ("tr", 2), ("uk", 3), ("vi", 1),
("zh-hans", 1), ("zh-hant", 1),
];
let more = [("en", 2), ("pt", 2), ("he", 3), ("ro", 3), ("nb", 2), ("ca", 2), ("sk", 3), ("th", 1), ("ms", 1), ("fa", 2), ("hi", 2)];
for (locale, n) in catalogs.into_iter().chain(more) {
assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}");
}
assert!(plural_rules("tlh").is_none(), "no CLDR rules");
}
#[test]
fn serbian_latin_plurals() {
let i = all(&[("sr-latn", "@plural\t{n} file|{n} files\t{n} fajl|{n} fajla|{n} fajlova\n", "")]).0;
let sr = lang(&i, "sr-latn");
let files = |n| i.trn(sr, n, "{n} file", "{n} files");
assert_eq!((files(1), files(3), files(5)), ("1 fajl".into(), "3 fajla".into(), "5 fajlova".into()));
}
#[test]
crates/i18n/src/number.rs
+395
View File
@@ -0,0 +1,395 @@
//! Reading numbers people type or paste, in any digits and with their locale's separators.
use std::fmt;
use crate::Format;
/// Why [`Format::parse`] rejected a text.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum NumberError {
/// No digits: empty, or only spaces and a sign.
Empty,
/// A character that isn't a digit, sign, decimal mark or grouping mark.
Unexpected(char),
/// Digits from more than one set (`1٢`).
MixedDigits,
/// A decimal or grouping mark where the locale puts none (`1.234,5` in English, `1,2.5` in
/// German).
Separator,
/// Too large for an `f64`.
TooLarge,
}
impl fmt::Display for NumberError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
NumberError::Empty => write!(f, "no digits"),
NumberError::Unexpected(c) => write!(f, "unexpected character {c:?}"),
NumberError::MixedDigits => write!(f, "digits from more than one script"),
NumberError::Separator => write!(f, "misplaced or ambiguous decimal or grouping mark"),
NumberError::TooLarge => write!(f, "number too large"),
}
}
}
impl std::error::Error for NumberError {}
/// The marks a [`Format`] writes, as [`Format::parse`] reads them.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) struct Marks {
/// `.` is the decimal mark and `,` grouping; else the other way round. For a format in
/// Arabic-Indic digits, as the locale writes Latin ones.
pub(crate) point: bool,
/// The format's own decimal and grouping marks when they are neither `.` nor `,` (`٫`, `'`, a
/// mark the system sets).
decimal: Option<char>,
group: Option<char>,
}
impl Default for Marks {
fn default() -> Self {
Marks { point: true, decimal: None, group: None }
}
}
impl Marks {
pub(crate) fn new(decimal: &str, group: &str, point: bool) -> Marks {
let own = |s: &str| {
let mut chars = s.chars();
chars.next().filter(|c| !matches!(c, '.' | ',') && chars.next().is_none())
};
Marks { point, decimal: own(decimal), group: own(group) }
}
}
impl Format {
/// The number in `text`: what this format writes, and what people type or paste.
///
/// - Digits: any Unicode decimal digits (ASCII, Arabic-Indic, Persian, Devanagari, Bengali,
/// Thai, full-width, …), all from one set.
/// - Decimal mark: the format's, `٫` in every format, and the locale's `.` or `,` when Latin
/// digits are typed into an Arabic-Indic format. A lone `.` in a comma locale is the decimal
/// mark when it can't be grouping (`1.5`, `0.250`, `1234.5`, `.5`), but `1.234` and
/// `1.234.567` stay grouped; the same holds for `,` in a point locale.
/// - Grouping: the format's, the other of `.` and `,`, spaces (NBSP, narrow NBSP, thin), `'`,
/// `’` and `٬`, one kind per number, in groups of three or the Indian 2-2-3 (`12,34,567`).
/// Never in the fraction.
/// - Sign: a leading `-`, `−` (U+2212), full-width `－` or `+`.
/// - Bidi marks and isolates (LRM, RLM, ALM, LRI…PDI) around or inside are ignored, so a
/// number formatted for an RTL interface reads back.
///
/// Percent signs, units and the space before a unit (`50 %`, `12 px`) are not read; strip them
/// first. Anything else is an error rather than a guess.
pub fn parse(&self, text: &str) -> Result<f64, NumberError> {
let marks = self.marks;
let text: String = text.chars().filter(|c| !is_bidi_mark(*c)).collect();
let mut rest = text.trim();
let negative = match rest.chars().next() {
Some(c @ ('-' | '−' | '－' | '+' | '＋')) => {
rest = rest.get(c.len_utf8()..).unwrap_or("");
matches!(c, '-' | '−' | '－')
}
_ => false,
};
let mut tokens = Vec::with_capacity(rest.len());
let mut zero = None;
for c in rest.chars() {
let token = match c {
c if Some(c) == marks.decimal => Token::Decimal,
c if Some(c) == marks.group => Token::Group(c),
'.' | '．' => Token::Point,
',' | '，' => Token::Comma,
'٫' => Token::Decimal,
'٬' => Token::Group('٬'),
'\'' | '’' => Token::Group('\''),
' ' | '\u{a0}' | '\u{202f}' | '\u{2009}' => Token::Group(' '),
_ => {
let (z, d) = digit(c).ok_or(NumberError::Unexpected(c))?;
if *zero.get_or_insert(z) != z {
return Err(NumberError::MixedDigits);
}
Token::Digit(d)
}
};
tokens.push(token);
}
if zero.is_none() {
return Err(if tokens.is_empty() { NumberError::Empty } else { NumberError::Separator });
}
let point = if lone_decimal(&tokens, marks.point) { !marks.point } else { marks.point };
let digits = read(&tokens, point).ok_or(NumberError::Separator)?;
let value: f64 = digits.parse().map_err(|_| NumberError::Separator)?;
if !value.is_finite() {
return Err(NumberError::TooLarge);
}
Ok(if negative { -value } else { value })
}
}
/// The number in `text` as `locale` (a language tag, `de-CH`, `ar_EG.UTF-8`) writes it:
/// [`Format::parse`] for [`Format::from_tag`]. Build the [`Format`] once to read many numbers. A
/// locale CLDR doesn't know reads as CLDR's root does (`1,234.5`).
pub fn parse_number(text: &str, locale: &str) -> Result<f64, NumberError> {
Format::from_tag(locale).parse(text)
}
#[derive(Clone, Copy, PartialEq)]
enum Token {
Digit(u8),
Point,
Comma,
/// A decimal mark in every reading: `٫`, or the format's own.
Decimal,
/// A grouping mark in every reading, by kind (space, apostrophe, `٬`, the format's own).
Group(char),
}
/// Whether the locale's grouping mark of `.` and `,` (`.` when `point` is false) is the decimal
/// mark here because it can't be grouping: it is the only mark of the two, there is one of it, and
/// the digits before it are none, start with 0 or are more than three, or the digits after it
/// aren't three.
fn lone_decimal(tokens: &[Token], point: bool) -> bool {
let (group, decimal) = if point { (Token::Comma, Token::Point) } else { (Token::Point, Token::Comma) };
if tokens.contains(&decimal) || tokens.contains(&Token::Decimal) || tokens.iter().filter(|t| **t == group).count() != 1 {
return false;
}
let Some(at) = tokens.iter().position(|t| *t == group) else { return false };
let (int, frac) = (tokens.get(..at).unwrap_or_default(), tokens.get(at + 1..).unwrap_or_default());
int.is_empty() || int.first() == Some(&Token::Digit(0)) || int.len() > 3 || frac.len() != 3
}
/// The ASCII number `tokens` spell when the decimal mark is `.` (`point`) or `,`; `None` when the
/// marks don't fit that reading.
fn read(tokens: &[Token], point: bool) -> Option<String> {
let tokens: Vec<Token> = tokens
.iter()
.map(|t| match t {
Token::Point if point => Token::Decimal,
Token::Point => Token::Group('.'),
Token::Comma if point => Token::Group(','),
Token::Comma => Token::Decimal,
t => *t,
})
.collect();
let (int, frac) = match tokens.iter().position(|t| *t == Token::Decimal) {
Some(i) => (tokens.get(..i)?, tokens.get(i + 1..)?),
None => (&tokens[..], &[][..]),
};
let ascii =
|part: &[Token]| -> Option<String> { part.iter().map(|t| if let Token::Digit(d) = t { Some(char::from(b'0' + d)) } else { None }).collect() };
let frac = ascii(frac)?;
let mut group = None;
let mut groups = Vec::new();
for chunk in int.split(|t| matches!(t, Token::Group(_))) {
groups.push(ascii(chunk)?);
}
for t in int {
if let Token::Group(g) = t
&& *group.get_or_insert(*g) != *g
{
return None;
}
}
if let [first, middle @ .., last] = groups.as_slice()
&& group.is_some()
{
let width = middle.first().map_or(3, String::len);
let first_max = if width == 2 { 2 } else { 3 };
let fits = (1..=first_max).contains(&first.len())
&& !first.starts_with('0')
&& last.len() == 3
&& middle.iter().all(|g| g.len() == width)
&& (2..=3).contains(&width);
if !fits {
return None;
}
}
let int = groups.concat();
if int.is_empty() && frac.is_empty() {
return None;
}
Some(format!("{}.{}", if int.is_empty() { "0" } else { &int }, if frac.is_empty() { "0" } else { &frac }))
}
/// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode
/// 16.0.
const ZEROS: [u32; 76] = [
0x30, 0x660, 0x6F0, 0x7C0, 0x966, 0x9E6, 0xA66, 0xAE6, 0xB66, 0xBE6, 0xC66, 0xCE6, 0xD66, 0xDE6, 0xE50, 0xED0, 0xF20, 0x1040, 0x1090, 0x17E0,
0x1810, 0x1946, 0x19D0, 0x1A80, 0x1A90, 0x1B50, 0x1BB0, 0x1C40, 0x1C50, 0xA620, 0xA8D0, 0xA900, 0xA9D0, 0xA9F0, 0xAA50, 0xABF0, 0xFF10, 0x104A0,
0x10D30, 0x10D40, 0x11066, 0x110F0, 0x11136, 0x111D0, 0x112F0, 0x11450, 0x114D0, 0x11650, 0x116C0, 0x116D0, 0x116DA, 0x11730, 0x118E0, 0x11950,
0x11BF0, 0x11C50, 0x11D50, 0x11DA0, 0x11F50, 0x16130, 0x16A60, 0x16AC0, 0x16B50, 0x16D70, 0x1CCF0, 0x1D7CE, 0x1D7D8, 0x1D7E2, 0x1D7EC, 0x1D7F6,
0x1E140, 0x1E2F0, 0x1E4F0, 0x1E5F1, 0x1E950, 0x1FBF0,
];
/// `c`'s digit set (its zero) and value.
fn digit(c: char) -> Option<(u32, u8)> {
let c = u32::from(c);
let zero = match ZEROS.binary_search(&c) {
Ok(i) => ZEROS.get(i),
Err(i) => ZEROS.get(i.checked_sub(1)?),
}?;
let value = c - zero;
u8::try_from(value).ok().filter(|v| *v < 10).map(|v| (*zero, v))
}
/// Marks bidi formatting adds around numbers: LRM, RLM, ALM, embeddings, overrides and isolates.
fn is_bidi_mark(c: char) -> bool {
matches!(c, '\u{200e}' | '\u{200f}' | '\u{061c}' | '\u{202a}'..='\u{202e}' | '\u{2066}'..='\u{2069}')
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Digits, Region};
#[test]
fn every_digit_set() {
for zero in ZEROS {
let d = |n: u32| char::from_u32(zero + n).unwrap();
let all: String = (0..10).rev().map(d).collect();
assert_eq!(parse_number(&all, "en"), Ok(9_876_543_210.0), "{zero:X}");
let grouped = format!("-{}{}{},{}{}{}.{}", d(1), d(2), d(3), d(4), d(5), d(6), d(7));
assert_eq!(parse_number(&grouped, "en"), Ok(-123_456.7), "{zero:X}");
assert!((0..10).all(|n| d(n).is_numeric()), "{zero:X}");
}
assert_eq!(digit('\u{1D800}'), None);
assert_eq!(digit('/'), None);
}
#[test]
fn locale_forms() {
for (text, locale, want) in [
("1,234.5", "en", 1234.5),
("1.234,5", "de", 1234.5),
("1 234,5", "fr", 1234.5),
("1\u{202f}234,5", "fr", 1234.5),
("1\u{a0}234,5", "ru", 1234.5),
("1’234.5", "de-CH", 1234.5),
("1'234.5", "de_CH.UTF-8", 1234.5),
("1,234", "en", 1234.0),
("1,234", "de", 1.234),
("1.234", "pt-br", 1234.0),
("1,5", "es", 1.5),
("1.5", "es-419", 1.5),
("١٬٢٣٤٫٥", "ar", 1234.5),
("١٢٫٥", "ar", 12.5),
("12.5", "ar", 12.5),
("12,5", "ar-MA", 12.5),
("۱۲٫۵", "fa", 12.5),
("−۴۲", "fa", -42.0),
("１２３，４５６．７", "ja", 123_456.7),
("－５", "ja", -5.0),
("१२,३४,५६७.८९", "hi", 1_234_567.89),
("১২,৩৪,৫৬৭", "bn", 1_234_567.0),
("౧,౨౩,౪౫౬", "te", 123_456.0),
("๑,๒๓๔.๕", "th", 1234.5),
("1,234,567", "en", 1_234_567.0),
("−3,5", "sv", -3.5),
("+7", "en", 7.0),
(".5", "en", 0.5),
("5.", "en", 5.0),
(" 42 ", "en", 42.0),
("\u{2066}-1,5\u{2069}", "de", -1.5),
("\u{061c}-١٢", "ar", -12.0),
("1,234.5", "xx", 1234.5),
("12", "xx", 12.0),
("1,5", "en-PL", 1.5),
("1.234,5", "en-DE", 1234.5),
("1.234,5", "lo", 1234.5),
("1,234.5", "km", 1234.5),
] {
assert_eq!(parse_number(text, locale), Ok(want), "{text:?} in {locale}");
}
}
#[test]
fn errors_instead_of_guesses() {
for (text, locale, want) in [
("", "en", NumberError::Empty),
(" ", "en", NumberError::Empty),
("-", "en", NumberError::Empty),
(",", "en", NumberError::Separator),
("1.2.3", "en", NumberError::Separator),
("1,,234", "en", NumberError::Separator),
("1,234,", "en", NumberError::Separator),
("1.234,5", "en", NumberError::Separator),
("1 234'567", "en", NumberError::Separator),
("1.234 567", "de", NumberError::Separator),
("0.123,4", "de", NumberError::Separator),
("1.5,2", "en", NumberError::Separator),
("123,45,678", "hi", NumberError::Separator),
("12٫3٫4", "ar", NumberError::Separator),
("1٫5.2", "ar", NumberError::Separator),
("1٢", "ar", NumberError::MixedDigits),
("12abc", "en", NumberError::Unexpected('a')),
("1e5", "en", NumberError::Unexpected('e')),
("--5", "en", NumberError::Unexpected('-')),
("5-", "en", NumberError::Unexpected('-')),
("- 5", "en", NumberError::Separator),
] {
assert_eq!(parse_number(text, locale), Err(want), "{text:?} in {locale}");
}
assert_eq!(parse_number(&"9".repeat(400), "en"), Err(NumberError::TooLarge));
for unit in ["50 %", "50%", "12 px", "1,5 MB"] {
assert!(parse_number(unit, "de").is_err(), "{unit}: units are the caller's to strip");
}
}
/// A lone mark that can't be grouping is the decimal mark; one that can stays grouping.
#[test]
fn lone_marks() {
for (text, locale, want) in [
("1.5", "de", 1.5),
("0.250", "de", 0.25),
("1234.5", "de", 1234.5),
(".5", "de", 0.5),
("-12.34", "ru", -12.34),
("1.234", "de", 1234.0),
("1.234.567", "de", 1_234_567.0),
("1,5", "en", 1.5),
("0,5", "en", 0.5),
("12,34", "en", 12.34),
("1234,567", "en", 1234.567),
("1,234", "en", 1234.0),
("12,345", "en", 12345.0),
] {
assert_eq!(parse_number(text, locale), Ok(want), "{text:?} in {locale}");
}
}
/// What a format writes reads back, in its digits and with its separators, the system's
/// included.
#[test]
fn reads_what_format_writes() {
let mut formats: Vec<Format> = ["en", "de", "de-CH", "fr", "hi", "ar-EG", "fa", "bn-BD", "ar-MA", "en-PL", "nb", "pt-PT", "sv"]
.iter()
.flat_map(|t| [Format::from_tag(t), Format::new(&Region::from_tag(t), Digits::Latin)])
.collect();
for (tag, decimal, grouping) in [
("en-US", Some(','), None),
("en-US", Some('.'), Some(Some('\u{a0}'))),
("en-US", None, Some(None)),
("en-US", Some('·'), Some(Some('_'))),
("de-DE", Some('·'), None),
] {
let mut region = Region::from_tag(tag);
region.decimal = decimal;
region.grouping = grouping;
formats.push(Format::new(&region, Digits::Locale));
}
// Windows sets both marks: a decimal comma the user picks arrives beside a grouping comma.
let reg = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string());
let windows = crate::region::from_windows(reg(&[("LocaleName", "en-US"), ("sDecimal", ","), ("sThousand", ",")]), || None);
formats.push(Format::new(&windows, Digits::Locale));
formats.push(Format::new(&Region::from_tag("ar-EG-u-nu-latn"), Digits::Locale));
formats.push(Format::new(&Region::from_tag("C"), Digits::Locale));
for f in &formats {
for value in [0.0, 1.5, -1234.25, 1_234_567.89, 12.0, 1234.0] {
let text = f.number(value, 2);
assert_eq!(f.parse(&text), Ok(value), "{text:?} ({f:?})");
}
let text = f.integer(1_234_567);
assert_eq!(f.parse(&text), Ok(1_234_567.0), "{text:?}");
}
}
}
crates/i18n/src/region.rs
+493
View File
@@ -0,0 +1,493 @@
//! The formatting region: how numbers, times and measurements are written, apart from the
//! interface language. An English interface in Poland still writes `2,54 cm` on A4.
//!
//! The territory lists come from Unicode CLDR's supplemental `measurementData` (Unicode License):
//! `paperSize` US-Letter and `measurementSystem` US. Decimal and grouping separators come from
//! the number locale's CLDR data ([`crate::Format`]); the region carries its own only where the
//! system sets them.
use std::sync::OnceLock;
/// Territories whose default paper is US Letter (CLDR `paperSize type="US-Letter"`).
const LETTER: &[&str] = &["BZ", "CA", "CL", "CO", "CR", "GT", "MX", "NI", "PA", "PH", "PR", "SV", "US", "VE"];
/// Territories that measure in inches (CLDR `measurementSystem type="US"`).
const INCHES: &[&str] = &["LR", "MM", "US"];
/// How lengths, distances and weights are measured.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Measurement {
/// Centimetres, kilometres and kilograms (CLDR `metric` and `UK`).
Metric,
/// Inches, miles and pounds (CLDR `US`).
Us,
}
/// The paper a new document starts on.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum Paper {
A4,
Letter,
}
/// A clock the user chose, where the system says so apart from the locale.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub enum HourCycle {
/// 1 to 12 with AM and PM.
H12,
/// 0 to 23.
H23,
}
/// The conventions numbers, times and measurements follow. Read it with [`Region::system`] or,
/// for the user's saved choice, [`Region::from_setting`]; the fields are public so an app can
/// keep a separate choice of, say, measurement.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Region {
/// The locale numbers are written in, BCP 47 (`pl-PL`, `sr-Latn-RS`).
pub numbers: String,
/// The locale dates and times are written in, BCP 47.
pub time: String,
pub measurement: Measurement,
pub paper: Paper,
/// The decimal separator the system sets (Windows, macOS); `None` follows the locale.
pub decimal: Option<char>,
/// The grouping separator the system sets (Windows, macOS): `Some(None)` for none at all
/// (`1234567`); `None` follows the locale.
pub grouping: Option<Option<char>>,
/// The clock the system sets (Windows, macOS); `None` follows the locale.
pub hour_cycle: Option<HourCycle>,
}
impl Default for Region {
/// `en-US`: Letter, inches.
fn default() -> Self {
Region::from_parts(&Tag::default())
}
}
impl Region {
/// The region a BCP 47 or POSIX locale tag names (`pl-PL`, `de_AT.UTF-8@euro`, `zh-Hans-CN`,
/// `en-US-u-ms-metric`, Apple's `en_US@rg=plzzzz`). The Unicode extensions `ms` (measurement),
/// `hc` (clock), `rg` (region override) and `nu` (digits; Apple's `@numbers=`) are read, and
/// `nu` stays in the number and time locales (`ar-EG-u-nu-latn`). `C` and `POSIX` are the C
/// locale: English, A4, metric, a `.` decimal and no grouping. An empty or unreadable tag is
/// `en-US`; a bare `en` is the United States, any other bare language has no territory.
pub fn from_tag(tag: &str) -> Region {
match Tag::parse(tag) {
Some(tag) => Region::from_parts(&tag),
None if is_c_locale(tag) => {
Region { measurement: Measurement::Metric, paper: Paper::A4, decimal: Some('.'), grouping: Some(None), ..Region::default() }
}
None => Region::default(),
}
}
/// The region for an app's saved region setting: `""` or `auto` (any case) is
/// [`Region::system`], anything else a locale tag for [`Region::from_tag`].
pub fn from_setting(setting: &str) -> Region {
let setting = setting.trim();
if setting.is_empty() || setting.eq_ignore_ascii_case("auto") { Region::system() } else { Region::from_tag(setting) }
}
/// The system's region, read once, then cached:
///
/// - Windows: the regional format and its customised separators, measurement system, paper
/// and short time format (Settings › Time & language › Language & region).
/// - macOS and iOS: the current locale (its identifier carries the Region setting), its
/// separators and metric setting, and its hour format.
/// - Linux and the BSDs: `LC_ALL`, else each of `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and
/// `LC_PAPER` on its own, else `LANG`.
/// - The web and Android: the preferred language, the only locale they expose.
pub fn system() -> Region {
static SYSTEM: OnceLock<Region> = OnceLock::new();
SYSTEM.get_or_init(read_system).clone()
}
fn from_parts(tag: &Tag) -> Region {
let territory = tag.territory.as_str();
let measurement = tag.measurement.unwrap_or(if INCHES.contains(&territory) { Measurement::Us } else { Measurement::Metric });
Region {
numbers: tag.bcp47(),
time: tag.bcp47(),
measurement,
paper: if LETTER.contains(&territory) { Paper::Letter } else { Paper::A4 },
decimal: None,
grouping: None,
hour_cycle: tag.hour_cycle,
}
}
}
/// The parts of a locale tag the region uses.
#[derive(Clone, Debug, PartialEq, Eq)]
struct Tag {
/// Lower case (`pl`).
language: String,
/// Title case (`Latn`); empty when the tag has none.
script: String,
/// Upper case (`PL`, or a UN M.49 number like `419`); empty when the tag has none.
territory: String,
measurement: Option<Measurement>,
hour_cycle: Option<HourCycle>,
/// The `nu` keyword (`latn`, `arab`); empty when the tag has none.
digits: String,
}
impl Default for Tag {
fn default() -> Self {
Tag { language: "en".into(), script: String::new(), territory: "US".into(), measurement: None, hour_cycle: None, digits: String::new() }
}
}
impl Tag {
fn parse(tag: &str) -> Option<Tag> {
// Locale tags are short; anything long is junk and only its start is read.
let tag: String = tag.chars().take(64).collect();
let (tag, modifier) = tag.split_once('@').unwrap_or((&tag, ""));
let tag = tag.split('.').next().unwrap_or_default().trim();
if is_c_locale(tag) {
return None;
}
let mut parts = tag.split(['-', '_']);
let language = parts.next().unwrap_or_default();
if !(2..=3).contains(&language.len()) || !language.chars().all(|c| c.is_ascii_alphabetic()) {
return None;
}
let mut out = Tag {
language: language.to_ascii_lowercase(),
script: String::new(),
territory: String::new(),
measurement: None,
hour_cycle: None,
digits: String::new(),
};
let mut extension = None;
for p in parts.by_ref() {
// A singleton starts the extensions (`-u-…`, `-x-…`).
if p.len() == 1 {
extension = Some(p.to_ascii_lowercase());
break;
}
if out.territory.is_empty() && out.script.is_empty() && p.len() == 4 && p.chars().all(|c| c.is_ascii_alphabetic()) {
out.script = title_case(p);
} else if out.territory.is_empty() && is_territory(p) {
out.territory = p.to_ascii_uppercase();
}
}
let mut keywords: Vec<(String, String)> = Vec::new();
if extension.as_deref() == Some("u") {
let mut key = String::new();
for p in parts {
if p.len() == 1 {
break;
}
if p.len() == 2 {
key = p.to_ascii_lowercase();
} else if !key.is_empty() {
keywords.push((std::mem::take(&mut key), p.to_ascii_lowercase()));
}
}
}
// Apple's and ICU's `@rg=plzzzz;ms=metric` keywords, and glibc's `@latin` script modifier.
for m in modifier.split(';') {
match m.split_once('=') {
Some((k, v)) => keywords.push((k.trim().to_ascii_lowercase(), v.trim().to_ascii_lowercase())),
None if m.eq_ignore_ascii_case("latin") => out.script = "Latn".into(),
None if m.eq_ignore_ascii_case("cyrillic") => out.script = "Cyrl".into(),
None => {}
}
}
for (key, value) in keywords {
match (key.as_str(), value.as_str()) {
("ms", "metric" | "uksystem") => out.measurement = Some(Measurement::Metric),
("ms", "ussystem") => out.measurement = Some(Measurement::Us),
("hc", "h11" | "h12") => out.hour_cycle = Some(HourCycle::H12),
("hc", "h23" | "h24") => out.hour_cycle = Some(HourCycle::H23),
("nu" | "numbers", v) if (3..=8).contains(&v.len()) && v.chars().all(|c| c.is_ascii_alphanumeric()) => out.digits = v.into(),
("rg", v) if v.len() == 6 && v.ends_with("zzzz") && v.get(..2).is_some_and(is_territory) => {
out.territory = v.get(..2).unwrap_or_default().to_ascii_uppercase();
}
_ => {}
}
}
if out.territory.is_empty() && out.language == "en" {
out.territory = "US".into();
}
Some(out)
}
fn bcp47(&self) -> String {
let tag = [self.language.as_str(), &self.script, &self.territory].into_iter().filter(|p| !p.is_empty()).collect::<Vec<_>>().join("-");
if self.digits.is_empty() { tag } else { format!("{tag}-u-nu-{}", self.digits) }
}
}
fn is_territory(p: &str) -> bool {
(p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit()))
}
fn title_case(p: &str) -> String {
let lower = p.to_ascii_lowercase();
let mut chars = lower.chars();
chars.next().map(|c| c.to_ascii_uppercase().to_string() + chars.as_str()).unwrap_or_default()
}
/// Whether a locale tag names the C locale (`C`, `C.UTF-8`, `POSIX`).
fn is_c_locale(tag: &str) -> bool {
let tag = tag.split(['@', '.']).next().unwrap_or_default().trim();
tag.eq_ignore_ascii_case("c") || tag.eq_ignore_ascii_case("posix")
}
/// The region from the POSIX locale variables `var` reads. An empty variable is unset; `C` in
/// `LC_ALL` or an `LC_*` variable is the C locale's conventions. `LANG` set to `C` or unset gives
/// way to the platform's preferred locale.
#[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))]
fn from_posix(var: impl Fn(&str) -> Option<String>, platform: impl FnOnce() -> Option<String>) -> Region {
let var = |name: &str| var(name).filter(|v| !v.trim().is_empty());
if let Some(all) = var("LC_ALL") {
return Region::from_tag(&all);
}
let lang = var("LANG").filter(|v| !is_c_locale(v)).or_else(platform).unwrap_or_default();
let tag = |name: &str| var(name).unwrap_or_else(|| lang.clone());
let time = Region::from_tag(&tag("LC_TIME"));
Region {
time: time.time,
hour_cycle: time.hour_cycle,
measurement: Region::from_tag(&tag("LC_MEASUREMENT")).measurement,
paper: Region::from_tag(&tag("LC_PAPER")).paper,
..Region::from_tag(&tag("LC_NUMERIC"))
}
}
/// The region from the values under `HKEY_CURRENT_USER\Control Panel\International` that `value`
/// reads, where Windows keeps the regional format and the user's changes to it. `LocaleName` is
/// the format (`pl-PL`), else the platform's preferred locale.
#[cfg(any(test, windows))]
pub(crate) fn from_windows(value: impl Fn(&str) -> Option<String>, platform: impl FnOnce() -> Option<String>) -> Region {
let mut region = Region::from_tag(&value("LocaleName").or_else(platform).unwrap_or_default());
region.decimal = value("sDecimal").as_deref().and_then(single_char);
region.grouping = value("sThousand").as_deref().and_then(grouping);
match value("iMeasure").as_deref().map(str::trim) {
Some("0") => region.measurement = Measurement::Metric,
Some("1") => region.measurement = Measurement::Us,
_ => {}
}
// DMPAPER_LETTER and DMPAPER_A4; other sizes keep the territory's.
match value("iPaperSize").as_deref().map(str::trim) {
Some("1") => region.paper = Paper::Letter,
Some("9") => region.paper = Paper::A4,
_ => {}
}
region.hour_cycle = value("sShortTime").as_deref().and_then(hour_cycle_of).or(region.hour_cycle);
region
}
/// The region from Apple's current locale: its identifier (`en_PL`, `en_GB@rg=uszzzz`), its
/// metric setting and separators, and the hour format its `j` skeleton gives (`HH`, `h a`).
#[cfg(any(test, target_vendor = "apple"))]
fn from_apple(identifier: &str, metric: bool, decimal: &str, grouping: &str, hours: Option<&str>) -> Region {
let mut region = Region::from_tag(identifier);
region.measurement = if metric { Measurement::Metric } else { Measurement::Us };
region.decimal = single_char(decimal);
region.grouping = self::grouping(grouping);
region.hour_cycle = hours.and_then(hour_cycle_of).or(region.hour_cycle);
region
}
/// A grouping separator the system sets: none at all when empty (Windows' `sThousand` the user
/// cleared), else one character.
#[cfg(any(test, windows, target_vendor = "apple"))]
fn grouping(s: &str) -> Option<Option<char>> {
if s.is_empty() { Some(None) } else { single_char(s).map(Some) }
}
/// A separator that is one character; Windows allows up to three, which the region can't carry.
#[cfg(any(test, windows, target_vendor = "apple"))]
fn single_char(s: &str) -> Option<char> {
let mut chars = s.chars();
let c = chars.next()?;
chars.next().is_none().then_some(c)
}
/// The clock a time pattern (`HH:mm`, `h:mm tt`, `h 'Uhr'`) uses; quoted literals are skipped.
#[cfg(any(test, windows, target_vendor = "apple"))]
fn hour_cycle_of(pattern: &str) -> Option<HourCycle> {
let mut quoted = false;
for c in pattern.chars() {
match c {
'\'' => quoted = !quoted,
'H' | 'k' if !quoted => return Some(HourCycle::H23),
'h' | 'K' if !quoted => return Some(HourCycle::H12),
_ => {}
}
}
None
}
#[cfg(windows)]
fn read_system() -> Region {
let key = windows_registry::CURRENT_USER.open(r"Control Panel\International").ok();
from_windows(|name| key.as_ref()?.get_string(name).ok(), sys_locale::get_locale)
}
#[cfg(target_vendor = "apple")]
fn read_system() -> Region {
use objc2_foundation::{NSDateFormatter, NSLocale, NSString};
let locale = NSLocale::currentLocale();
let hours = NSDateFormatter::dateFormatFromTemplate_options_locale(&NSString::from_str("j"), 0, Some(&locale)).map(|f| f.to_string());
from_apple(
&locale.localeIdentifier().to_string(),
locale.usesMetricSystem(),
&locale.decimalSeparator().to_string(),
&locale.groupingSeparator().to_string(),
hours.as_deref(),
)
}
#[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))]
fn read_system() -> Region {
from_posix(|name| std::env::var(name).ok(), sys_locale::get_locale)
}
#[cfg(not(any(windows, unix)))]
fn read_system() -> Region {
Region::from_tag(&sys_locale::get_locale().unwrap_or_default())
}
#[cfg(target_os = "android")]
fn read_system() -> Region {
Region::from_tag(&sys_locale::get_locale().unwrap_or_default())
}
#[cfg(test)]
mod tests {
use super::*;
fn conventions(r: &Region) -> (Paper, Measurement) {
(r.paper, r.measurement)
}
#[test]
fn reads_locale_tags() {
assert_eq!(Region::from_tag("pl-PL").numbers, "pl-PL");
assert_eq!(Region::from_tag("de_AT.UTF-8@euro").numbers, "de-AT");
assert_eq!(Region::from_tag("zh-hans-cn").numbers, "zh-Hans-CN");
assert_eq!(Region::from_tag("sr_RS@latin").numbers, "sr-Latn-RS");
assert_eq!(Region::from_tag("es-419").numbers, "es-419");
assert_eq!(Region::from_tag("fr").numbers, "fr");
assert_eq!(Region::from_tag("en").numbers, "en-US", "bare English is the US");
for junk in ["", "1234", "x", "--", "ünïcode-ÄÄ", &"a".repeat(10_000)] {
assert_eq!(Region::from_tag(junk), Region::default(), "{junk:?}");
}
}
/// glibc's C locale: A4 and metric (`C-paper.c`, `C-measurement.c`), `.` and no grouping.
#[test]
fn the_c_locale_is_a4_metric_and_ungrouped() {
for c in ["C", "POSIX", "C.UTF-8", "c.utf8", " C "] {
let r = Region::from_tag(c);
assert_eq!(
(r.numbers.as_str(), r.paper, r.measurement, r.decimal, r.grouping),
("en-US", Paper::A4, Measurement::Metric, Some('.'), Some(None)),
"{c:?}"
);
}
}
#[test]
fn a_chosen_digit_system_stays_in_the_locale() {
assert_eq!(Region::from_tag("ar-EG-u-nu-latn").numbers, "ar-EG-u-nu-latn");
let r = Region::from_tag("ar_EG@numbers=latn");
assert_eq!((r.numbers.as_str(), r.time.as_str()), ("ar-EG-u-nu-latn", "ar-EG-u-nu-latn"));
assert_eq!(Region::from_tag("en-US-u-nu-arab-ms-metric").numbers, "en-US-u-nu-arab");
assert_eq!(Region::from_tag("ar-EG-u-nu-x").numbers, "ar-EG", "not a numbering system");
}
#[test]
fn unicode_extensions_and_apple_keywords() {
let r = Region::from_tag("en-US-u-ms-metric-hc-h23");
assert_eq!((r.numbers.as_str(), r.measurement, r.hour_cycle), ("en-US", Measurement::Metric, Some(HourCycle::H23)));
assert_eq!(Region::from_tag("en-GB-u-hc-h12").hour_cycle, Some(HourCycle::H12));
assert_eq!(Region::from_tag("de-DE-x-ms-ussystem").measurement, Measurement::Metric, "private use isn't read");
let r = Region::from_tag("en_GB@rg=uszzzz");
assert_eq!((r.numbers.as_str(), r.paper, r.measurement), ("en-US", Paper::Letter, Measurement::Us));
assert_eq!(Region::from_tag("en_US@rg=plzzzz;ms=metric").numbers, "en-PL");
}
/// Paper and measurement follow the territory; separators are left to the number locale.
#[test]
fn conventions_follow_the_territory() {
assert_eq!(conventions(&Region::from_tag("pl-PL")), (Paper::A4, Measurement::Metric));
assert_eq!(conventions(&Region::from_tag("en-US")), (Paper::Letter, Measurement::Us));
assert_eq!(conventions(&Region::from_tag("fr-CA")), (Paper::Letter, Measurement::Metric));
assert_eq!(conventions(&Region::from_tag("en-GB")), (Paper::A4, Measurement::Metric));
let r = Region::from_tag("de-CH");
assert_eq!((r.decimal, r.grouping), (None, None));
assert_eq!(Region::from_tag("es-MX").paper, Paper::Letter);
assert_eq!(Region::from_tag("ja-JP").paper, Paper::A4);
}
#[test]
fn a_saved_setting_overrides_the_system() {
assert_eq!(Region::from_setting(" pl-PL "), Region::from_tag("pl-PL"));
assert_eq!(Region::from_setting("AUTO"), Region::system());
assert_eq!(Region::from_setting(""), Region::system());
}
#[test]
fn posix_variables_set_each_convention() {
let env = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string());
let none = || None;
let r = from_posix(env(&[("LANG", "en_US.UTF-8"), ("LC_NUMERIC", "pl_PL.UTF-8"), ("LC_TIME", "en_GB.UTF-8"), ("LC_PAPER", "de_DE")]), none);
assert_eq!((r.numbers.as_str(), r.time.as_str(), r.decimal), ("pl-PL", "en-GB", None));
assert_eq!((r.paper, r.measurement), (Paper::A4, Measurement::Us), "paper from LC_PAPER, measurement from LANG");
assert_eq!(from_posix(env(&[("LANG", "it_IT.UTF-8"), ("LC_ALL", "en_US"), ("LC_PAPER", "de_DE")]), none).paper, Paper::Letter);
assert_eq!(from_posix(env(&[("LANG", "en_US"), ("LC_MEASUREMENT", "it_IT")]), none).measurement, Measurement::Metric);
assert_eq!(from_posix(env(&[("LANG", "C.UTF-8"), ("LC_ALL", "")]), || Some("it-IT".into())).numbers, "it-IT");
assert_eq!(from_posix(env(&[]), none), Region::default());
// `C` in an LC_* variable is the C locale's conventions, not an unset variable.
let r = from_posix(env(&[("LANG", "de_DE.UTF-8"), ("LC_NUMERIC", "C"), ("LC_TIME", "C.UTF-8")]), none);
assert_eq!((r.numbers.as_str(), r.time.as_str(), r.paper), ("en-US", "en-US", Paper::A4));
assert_eq!(from_posix(env(&[("LANG", "de_DE.UTF-8"), ("LC_ALL", "C")]), none), Region::from_tag("C"));
let r = from_posix(env(&[("LANG", "en_US.UTF-8"), ("LC_PAPER", "C"), ("LC_MEASUREMENT", "C.UTF-8")]), none);
assert_eq!((r.paper, r.measurement, r.decimal), (Paper::A4, Measurement::Metric, None));
assert_eq!(from_posix(env(&[("LANG", "C"), ("LC_NUMERIC", "")]), || Some("it-IT".into())).numbers, "it-IT");
}
#[test]
fn windows_regional_format_and_its_changes() {
let reg = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string());
let r = from_windows(
reg(&[("LocaleName", "pl-PL"), ("sDecimal", ","), ("sThousand", "\u{a0}"), ("iMeasure", "0"), ("sShortTime", "HH:mm")]),
|| None,
);
assert_eq!((r.numbers.as_str(), r.decimal, r.grouping, r.hour_cycle), ("pl-PL", Some(','), Some(Some('\u{a0}')), Some(HourCycle::H23)));
// An English format in the US with the user's metric and A4 choices and a decimal comma.
let r = from_windows(
reg(&[("LocaleName", "en-US"), ("sDecimal", ","), ("iMeasure", "0"), ("iPaperSize", "9"), ("sShortTime", "h:mm tt")]),
|| None,
);
assert_eq!(
(r.decimal, r.grouping, r.measurement, r.paper, r.hour_cycle),
(Some(','), None, Measurement::Metric, Paper::A4, Some(HourCycle::H12))
);
let r = from_windows(reg(&[("sDecimal", "dec"), ("sThousand", "")]), || Some("de-DE".into()));
assert_eq!((r.numbers.as_str(), r.decimal), ("de-DE", None), "a separator longer than one character keeps the locale's");
assert_eq!(r.grouping, Some(None), "an empty grouping separator is no grouping");
assert_eq!(from_windows(reg(&[("sThousand", "abc")]), || None).grouping, None);
}
#[test]
fn apple_locale_values() {
let r = from_apple("en_PL", true, ",", "\u{a0}", Some("HH"));
assert_eq!(
(r.numbers.as_str(), r.paper, r.decimal, r.grouping, r.hour_cycle),
("en-PL", Paper::A4, Some(','), Some(Some('\u{a0}')), Some(HourCycle::H23))
);
let r = from_apple("en_US", false, ".", ",", Some("h a"));
assert_eq!((r.measurement, r.hour_cycle), (Measurement::Us, Some(HourCycle::H12)));
assert_eq!(from_apple("de_DE", true, ",", ".", Some("H 'Uhr'")).hour_cycle, Some(HourCycle::H23));
assert_eq!(hour_cycle_of("'h' mm"), None, "quoted letters are text");
}
}
crates/i18n/src/size.rs
+266
View File
@@ -0,0 +1,266 @@
//! File sizes and other measures with a unit, in the interface language.
//!
//! Unit names are catalog rows in context `unit`, keyed by the English template with the
//! number as `{n}` (`{n} MB`, `{n} px`, `{n}%`), so a translation can change the symbol and its
//! spacing (`{n} Mo`, `{n} %`). They are looked up with [`I18n::get`], never the plain row: a
//! plain `B` row means bold. A byte count in words is the `@plural` row `{n} byte|{n} bytes`.
use crate::{I18n, Lang, fmt, ltr};
/// The context unit rows are in.
const CONTEXT: &str = "unit";
/// A file-size unit.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum SizeUnit {
Byte,
Kilo,
Mega,
Giga,
Tera,
}
impl SizeUnit {
const ALL: [SizeUnit; 5] = [SizeUnit::Byte, SizeUnit::Kilo, SizeUnit::Mega, SizeUnit::Giga, SizeUnit::Tera];
}
/// How [`I18n::file_size`] writes a byte count: the step between units, their names, and the
/// digits after the decimal point. Start from [`FileSize::BINARY`], [`FileSize::IEC`] or
/// [`FileSize::DECIMAL`] and adjust:
///
/// ```
/// use craft_i18n::{FileSize, SizeUnit};
/// // `12 MB`, `3.4 MB`, `640 KB`: decimal steps, whole kilobytes, no decimals from 10 MB.
/// let style = FileSize::DECIMAL.smallest(SizeUnit::Kilo).largest(SizeUnit::Mega).unit_decimals(SizeUnit::Kilo, 0).whole_from(10);
/// ```
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
pub struct FileSize {
step: u64,
iec: bool,
/// Per unit, in [`SizeUnit::ALL`] order; bytes are always whole.
decimals: [u8; 5],
whole_from: u32,
smallest: SizeUnit,
largest: SizeUnit,
byte_symbol: bool,
}
impl FileSize {
/// Steps of 1024 named KB, MB, GB and TB, as Windows Explorer shows sizes; one decimal, bytes
/// in words.
pub const BINARY: FileSize = FileSize {
step: 1024,
iec: false,
decimals: [0, 1, 1, 1, 1],
whole_from: 0,
smallest: SizeUnit::Byte,
largest: SizeUnit::Tera,
byte_symbol: false,
};
/// Steps of 1024 named KiB, MiB, GiB and TiB.
pub const IEC: FileSize = FileSize { iec: true, ..FileSize::BINARY };
/// Steps of 1000 named KB, MB, GB and TB, as macOS shows sizes and as download and disk sizes
/// are sold.
pub const DECIMAL: FileSize = FileSize { step: 1000, ..FileSize::BINARY };
/// `n` digits after the decimal point in every unit above bytes.
pub const fn decimals(mut self, n: u8) -> FileSize {
self.decimals = [0, n, n, n, n];
self
}
/// `n` digits after the decimal point in `unit` (`0` for whole kilobytes, `2` for gigabytes).
pub const fn unit_decimals(mut self, unit: SizeUnit, n: u8) -> FileSize {
if !matches!(unit, SizeUnit::Byte) {
self.decimals[unit as usize] = n;
}
self
}
/// No decimals from `n` of a unit up: `whole_from(10)` writes `9.5 MB` and `12 MB`. `0`, the
/// default, keeps them.
pub const fn whole_from(mut self, n: u32) -> FileSize {
self.whole_from = n;
self
}
/// The smallest unit shown. Sizes below it are written in it, rounded up, so a file that
/// isn't empty never reads `0 KB`.
pub const fn smallest(mut self, unit: SizeUnit) -> FileSize {
self.smallest = unit;
self
}
/// The largest unit shown; larger sizes are counted in it (`2048 GB`).
pub const fn largest(mut self, unit: SizeUnit) -> FileSize {
self.largest = unit;
self
}
/// Bytes as the symbol `B` (`512 B`) instead of words (`512 bytes`).
pub const fn byte_symbol(mut self) -> FileSize {
self.byte_symbol = true;
self
}
/// The unit `bytes` is written in, the number in it and its digits after the point.
fn parts(&self, bytes: u64) -> (SizeUnit, f64, usize) {
let (smallest, largest) = (self.smallest.min(self.largest), self.largest);
let mut unit = smallest;
for &u in SizeUnit::ALL.iter().filter(|u| (smallest..=largest).contains(u)) {
if bytes as f64 >= self.scale(u) {
unit = u;
}
}
loop {
let value = bytes as f64 / self.scale(unit);
let round = |decimals: u8| {
let factor = 10f64.powi(i32::from(decimals));
(if unit == smallest && bytes > 0 && value < 1.0 { (value * factor).ceil() } else { (value * factor).round() }) / factor
};
let mut decimals = if unit == SizeUnit::Byte { 0 } else { self.decimals.get(unit as usize).copied().unwrap_or(0) };
let mut rounded = round(decimals);
// After rounding, so 9.96 MB, which rounds to 10.0, is written 10 MB.
if self.whole_from > 0 && rounded >= f64::from(self.whole_from) {
decimals = 0;
rounded = round(0);
}
// 1023.96 KB rounds to 1024.0 KB: the next unit writes it as 1.0 MB.
if rounded >= self.step as f64
&& let Some(&next) = SizeUnit::ALL.get(unit as usize + 1)
&& next <= largest
{
unit = next;
continue;
}
return (unit, rounded, usize::from(decimals));
}
}
fn scale(&self, unit: SizeUnit) -> f64 {
(self.step as f64).powi(unit as i32)
}
/// The English template for `unit`, which is also its catalog key.
fn template(&self, unit: SizeUnit) -> &'static str {
match (unit, self.iec) {
(SizeUnit::Byte, _) => "{n} B",
(SizeUnit::Kilo, false) => "{n} KB",
(SizeUnit::Mega, false) => "{n} MB",
(SizeUnit::Giga, false) => "{n} GB",
(SizeUnit::Tera, false) => "{n} TB",
(SizeUnit::Kilo, true) => "{n} KiB",
(SizeUnit::Mega, true) => "{n} MiB",
(SizeUnit::Giga, true) => "{n} GiB",
(SizeUnit::Tera, true) => "{n} TiB",
}
}
}
impl I18n {
/// `bytes` as a file size in `lang`, written the way `style` says (`1.5 MB`, `512 bytes`), the
/// number as [`I18n::format`] writes it (`1,5 Mo`).
pub fn file_size(&self, lang: Lang, bytes: u64, style: FileSize) -> String {
let (unit, n, decimals) = style.parts(bytes);
if unit == SizeUnit::Byte && !style.byte_symbol {
return self.trn(lang, bytes, "{n} byte", "{n} bytes");
}
self.unit_text(lang, style.template(unit), &self.format().number(n, decimals))
}
/// `value` with `decimals` digits after the decimal separator in the unit template `unit`, an
/// English catalog key with the number as `{n}` (`{n} px`, `{n} dB`), the number as
/// [`I18n::format`] writes it. Percentages are [`crate::Format::percent`].
pub fn measure(&self, lang: Lang, value: f64, decimals: usize, unit: &str) -> String {
self.unit_text(lang, unit, &self.format().number(value, decimals))
}
fn unit_text(&self, lang: Lang, template: &str, n: &str) -> String {
fmt(self.get(lang, CONTEXT, template).unwrap_or(template), &[("n", &ltr(n))])
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Embedded, Format};
static E: &Embedded = &[(
"fr",
"unit\t{n} KB\t{n} Ko\nunit\t{n} MB\t{n} Mo\nunit\t{n} B\t{n} o\nunit\t{n}%\t{n} %\n\tB\tG\n@plural\t{n} byte|{n} bytes\t{n} octet|{n} octets\n",
"",
)];
fn en(bytes: u64, style: FileSize) -> String {
let (i18n, _) = I18n::new(&[], &[]);
i18n.set_format(Format::from_tag("en-US"));
i18n.file_size(Lang::EN, bytes, style)
}
#[test]
fn binary_decimal_and_iec_steps() {
assert_eq!(en(512, FileSize::BINARY), "512 bytes");
assert_eq!(en(1023, FileSize::BINARY), "1,023 bytes");
assert_eq!(en(1, FileSize::BINARY), "1 byte");
assert_eq!(en(1536, FileSize::BINARY), "1.5 KB");
assert_eq!(en(1536, FileSize::IEC), "1.5 KiB");
assert_eq!(en(1500, FileSize::DECIMAL), "1.5 KB");
assert_eq!(en(3 << 30, FileSize::BINARY), "3.0 GB");
assert_eq!(en(5 << 40, FileSize::IEC.decimals(0)), "5 TiB");
assert_eq!(en(512, FileSize::BINARY.byte_symbol()), "512 B");
}
#[test]
fn precision_and_unit_range() {
// Rounding up to the next unit's step moves to that unit.
assert_eq!(en(1_048_575, FileSize::BINARY), "1.0 MB");
assert_eq!(en(999_999, FileSize::DECIMAL.decimals(0)), "1 MB");
assert_eq!(en(1_610_612_736, FileSize::BINARY.unit_decimals(SizeUnit::Giga, 2)), "1.50 GB");
// Lightcraft's denoise sizes: whole kilobytes, at least 1, no decimals from 10 MB.
let style = FileSize::DECIMAL.smallest(SizeUnit::Kilo).largest(SizeUnit::Mega).unit_decimals(SizeUnit::Kilo, 0).whole_from(10);
assert_eq!(en(1, style), "1 KB");
assert_eq!(en(0, style), "0 KB");
assert_eq!(en(640_400, style), "640 KB");
assert_eq!(en(3_400_000, style), "3.4 MB");
assert_eq!(en(12_600_000, style), "13 MB");
assert_eq!(en(9_960_000, style), "10 MB", "whole once rounded to 10");
assert_eq!(en(9_940_000, style), "9.9 MB");
assert_eq!(en(5_000_000_000, style), "5,000 MB");
// Only sizes below the smallest unit round up.
assert_eq!(en(1025, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB");
assert_eq!(en(100, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB");
}
#[test]
fn numbers_are_left_to_right_isolates_in_rtl_units() {
let (i18n, problems) = I18n::new(&[("ar", "unit\t{n} KB\t{n} ك.ب\n", "")], &["ar"]);
assert!(problems.is_empty(), "{problems:?}");
i18n.set_format(Format::from_tag("en-US"));
let ar = i18n.from_code("ar").unwrap();
assert_eq!(i18n.file_size(ar, 1536, FileSize::BINARY), "\u{2066}1.5\u{2069} ك.ب");
assert_eq!(en(1536, FileSize::BINARY), "1.5 KB", "no marks in left-to-right text");
}
#[test]
fn unit_names_are_catalog_rows() {
let (i18n, problems) = I18n::new(E, &["fr"]);
assert!(problems.is_empty(), "{problems:?}");
let fr = i18n.from_code("fr").unwrap();
i18n.set_format(Format::from_tag("fr-FR"));
assert_eq!(i18n.file_size(fr, 1536, FileSize::BINARY), "1,5 Ko");
assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY), "512 octets");
assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY.byte_symbol()), "512 o", "not the plain row B (bold)");
assert_eq!(i18n.file_size(fr, 3 << 30, FileSize::BINARY), "3,0 GB", "a missing row is English");
assert_eq!(i18n.measure(fr, 50.0, 0, "{n}%"), "50 %");
assert_eq!(i18n.measure(Lang::EN, 12.34, 1, "{n} px"), "12,3 px", "numbers follow the region, not the language");
}
#[test]
fn locale_digits() {
let (i18n, _) = I18n::new(&[], &[]);
i18n.set_format(Format::from_tag("ar-EG"));
assert_eq!(i18n.file_size(Lang::EN, 1536, FileSize::BINARY), "١٫٥ KB");
i18n.set_format(Format::from_tag("hi"));
assert_eq!(i18n.file_size(Lang::EN, 1_234_567, FileSize::DECIMAL.largest(SizeUnit::Byte).byte_symbol()), "12,34,567 B");
}
}