diff --git a/Cargo.lock b/Cargo.lock index 4d9de428..7fc992df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -491,6 +491,15 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "cobs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "codespan-reporting" version = "0.13.1" @@ -656,9 +665,18 @@ version = "0.1.0" name = "craft-i18n" version = "0.1.0" dependencies = [ - "intl_pluralrules", + "fixed_decimal", + "icu_decimal", + "icu_locale_core", + "icu_locale_fallback", + "icu_plurals", + "icu_provider", + "icu_provider_adapters", + "icu_provider_blob", + "objc2-foundation 0.3.2", "sys-locale", - "unic-langid", + "windows-registry", + "writeable", ] [[package]] @@ -1230,6 +1248,17 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fixed_decimal" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79c3c892f121fff406e5dd6b28c1b30096b95111c30701a899d4f2b18da6d1bd" +dependencies = [ + "displaydoc", + "smallvec", + "writeable", +] + [[package]] name = "flagset" version = "0.4.7" @@ -1859,6 +1888,23 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_decimal" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb8f655bba2d0c0459e43a5b0e6cd3cd388109898fb3d85d048165e8b0abd08" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_locale_core", + "icu_pattern", + "icu_plurals", + "icu_provider", + "serde", + "writeable", + "zerovec", +] + [[package]] name = "icu_locale_core" version = "2.3.0" @@ -1867,11 +1913,32 @@ checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", + "serde", "tinystr", "writeable", "zerovec", ] +[[package]] +name = "icu_locale_fallback" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "251af8e57c9400e3eb58242fe5b8b1152b2a64fdf4cf632f923c38ccee6f2fa9" +dependencies = [ + "icu_locale_core", + "icu_locale_fallback_data", + "icu_provider", + "potential_utf", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locale_fallback_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "decf2a22ec8fa68f1a0c1129a3f8583f8f8bc24e8b9ccbe98ead99f62a4dc3a8" + [[package]] name = "icu_normalizer" version = "2.3.0" @@ -1892,6 +1959,32 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +[[package]] +name = "icu_pattern" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b3db611446bd92c246b03e4aa0bafa71ddb194af431ddbb42b9d70e4bfce1a0" +dependencies = [ + "displaydoc", + "either", + "serde", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_plurals" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e475e6766ef87b1d3c1f97be6363995b0bfaeddbc789671615df598a97ff0593" +dependencies = [ + "displaydoc", + "fixed_decimal", + "icu_provider", + "serde", + "zerovec", +] + [[package]] name = "icu_properties" version = "2.3.0" @@ -1921,6 +2014,9 @@ checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", + "postcard", + "serde", + "stable_deref_trait", "writeable", "yoke", "zerofrom", @@ -1928,6 +2024,30 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_provider_adapters" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df887bc4f55dbc773c7c45a0c7dfa75be886f7b7812830a71ab7271cc1801820" +dependencies = [ + "icu_locale_fallback", + "icu_provider", +] + +[[package]] +name = "icu_provider_blob" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916b44fac319d3f58e4d8d805a627df19e27937b582cd99fcde3bb6b7ce1b0e" +dependencies = [ + "icu_provider", + "postcard", + "serde", + "writeable", + "zerotrie", + "zerovec", +] + [[package]] name = "idna" version = "1.1.0" @@ -1995,15 +2115,6 @@ dependencies = [ "hashbrown 0.17.1", ] -[[package]] -name = "intl_pluralrules" -version = "7.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" -dependencies = [ - "unic-langid", -] - [[package]] name = "ipnet" version = "2.12.2" @@ -2248,6 +2359,9 @@ name = "litemap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" +dependencies = [ + "serde_core", +] [[package]] name = "litrs" @@ -3079,12 +3193,24 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "postcard" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6764c3b5dd454e283a30e6dfe78e9b31096d9e32036b5d1eaac7a6119ccb9a24" +dependencies = [ + "cobs", + "serde", +] + [[package]] name = "potential_utf" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ + "serde_core", + "writeable", "zerovec", ] @@ -4715,24 +4841,6 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" -[[package]] -name = "unic-langid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28ba52c9b05311f4f6e62d5d9d46f094bd6e84cb8df7b3ef952748d752a7d05" -dependencies = [ - "unic-langid-impl", -] - -[[package]] -name = "unic-langid-impl" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce1bf08044d4b7a94028c93786f8566047edc11110595914de93362559bc658" -dependencies = [ - "tinystr", -] - [[package]] name = "unicode-bidi" version = "0.3.18" @@ -5792,6 +5900,9 @@ name = "writeable" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +dependencies = [ + "either", +] [[package]] name = "x11rb" @@ -5985,8 +6096,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", + "litemap", + "serde_core", "yoke", "zerofrom", + "zerovec", ] [[package]] diff --git a/crates/i18n/Cargo.toml b/crates/i18n/Cargo.toml index 45e21775..88e52d02 100644 --- a/crates/i18n/Cargo.toml +++ b/crates/i18n/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "craft-i18n" -description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, language names and the system locale." +description = "Translations for the Craft apps: the embedded craft-i18n catalogs, lookup with fallback, plurals, numbers for a region, language names and the system locale." version.workspace = true edition.workspace = true license.workspace = true @@ -8,12 +8,29 @@ rust-version.workspace = true repository.workspace = true [dependencies] -intl_pluralrules = "=7.0.2" -unic-langid = "0.9" +# Numbers and plural rules, on CLDR data (data/numbers.postcard) generated for one ICU4X minor +# version, hence `~2.3`. +fixed_decimal = { version = "0.7.2", default-features = false } +icu_decimal = { version = "~2.3", default-features = false, features = ["serde"] } +icu_locale_core = { version = "~2.3", default-features = false } +icu_locale_fallback = { version = "~2.3", default-features = false, features = ["compiled_data"] } +icu_plurals = { version = "~2.3", default-features = false, features = ["serde"] } +icu_provider = { version = "~2.3", default-features = false, features = ["sync"] } +icu_provider_adapters = { version = "~2.3", default-features = false } +icu_provider_blob = { version = "~2.3", default-features = false } +writeable = "0.6" sys-locale = "0.3" [target.'cfg(target_arch = "wasm32")'.dependencies] sys-locale = { version = "0.3", features = ["js"] } +# The regional format (Region::system); safe registry reads, MIT OR Apache-2.0. +[target.'cfg(windows)'.dependencies] +windows-registry = "0.6" + +# The current locale (Region::system); already in the apps' Cargo.lock through eframe. +[target.'cfg(target_vendor = "apple")'.dependencies] +objc2-foundation = { version = "0.3.2", default-features = false, features = ["std", "NSDateFormatter", "NSFormatter", "NSLocale", "NSString"] } + [lints] workspace = true diff --git a/crates/i18n/README.md b/crates/i18n/README.md index 6bb44dc2..d2cc7fda 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -45,8 +45,8 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l → `pt`, `zh-hant-hk` → `zh-hant`); `sr-latn` never falls back to Cyrillic `sr`. A missing translation never errors. - **Lookups:** `t` (plain), `tc` (with a context such as `menu` or `@msg`, else plain), `tid` (an - `@id` command id, else the English label), `trn` (a count, `@plural` rows, `{n}` filled), `get` - (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row + `@id` command id, else the English label), `trn` and `trn_decimal` (a count, `@plural` rows, + `{n}` filled), `get` (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, @@ -54,10 +54,13 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before they ship. - **Plurals:** forms are listed in CLDR order (zero, one, two, few, many, other) for the categories - whole numbers 0 to 1000 reach, from `intl_pluralrules`; a count in any other category takes the - last form. CLDR's `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` - `pt`'s. `intl_pluralrules` is pinned to the version craft-i18n's checker uses, and - `plural_form_counts` holds the same counts. + whole numbers 0 to 1000 reach, from ICU4X's CLDR rules; `plural_form_counts` holds the counts + craft-i18n's checker expects. `trn_decimal` picks the form from the digits shown (ICU4X plural + operands), so English `1.0` is plural and French `1,5` singular. A count in a category integers + never reach takes the `few` form if there is one (Russian and Ukrainian decimals, `1,5 файла`), + else the last. Rules are per language, so `sr-latn` and `zh-hant` take `sr`'s and `zh`'s. CLDR's + `pt` is Brazilian, so `pt` uses `pt-PT`'s rules (0 is plural) and `pt-br` `pt`'s. A language + CLDR has no plural rules for shows English forms. - **Names:** a language's name comes from its shared catalog's `@id language.name` row, else its code; English is "English". - **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and @@ -74,6 +77,40 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l `zh-hant`, `pt-PT` → `pt-br` when that is the only Portuguese, `C`/`POSIX` → English. A script is never crossed: `sr_RS@latin` and `sr-Latn-RS` pick `sr-latn`, never `sr`, while `sr-Cyrl-RS` picks `sr`. +- **Sizes and units:** `file_size(lang, bytes, style)` writes `1.5 MB` or `512 bytes`; `FileSize` + picks steps of 1024 named KB (`BINARY`), 1024 named KiB (`IEC`) or 1000 named KB (`DECIMAL`), + the decimals per unit, no decimals from a given value up, the smallest and largest unit (below + the smallest, sizes round up), and `B` or words for bytes. `measure(lang, value, decimals, + "{n} px")` writes any other measure. Unit names are rows in context `unit`, keyed by the English + template (`{n} MB` → `{n} Mo`, `{n}%` → `{n} %`) and read with `get`, so a plain `B` row (bold) + is never used; bytes in words are the `@plural` row `{n} byte|{n} bytes`. Numbers are written + by `I18n::format`. +- **Numbers:** `Format` writes numbers for a `Region` on ICU4X's decimal formatter: `integer` + (`1.234.567`, Indian `12,34,567`), `number(value, decimals)` and `percent(ratio, decimals)` + (`50 %`, Turkish `%50`). It follows the region, not the interface language, with the locale's + digits (`١٬٢٣٤٫٥` in Egypt, `۱٬۲۳۴` in Iran) or, with `Digits::Latin`, `0`–`9`. With Latin + digits, a decimal or grouping separator the system sets (Windows' and macOS's own settings) + replaces the locale's. `I18n::format` is the format `trn`, `file_size` and `measure` use: the + system region's until `I18n::set_format` (the user's region setting or digit choice), and again + after `I18n::reset_format`. A count in `trn` is therefore grouped and in the region's digits + (`1.234 Dateien`, `١٬٢٣٤ ملفًا`); for names written into documents or files (`Copy 1234`), use + `trn_plain`, which writes `0`–`9` without grouping. +- **CLDR data:** decimal symbols, digits and plural rules (`data/numbers.postcard`, about 10 KB) and + percent patterns (`data/locales.rs`) cover every CLDR locale; `data/generate.sh` rebuilds both + from CLDR. The ICU4X crates are pinned to one minor version (`~2.3`), the one the data is + generated for. The ICU4X code and data add about 90 KB to a stripped release build and 80 KB to + wasm. +- **Formatting region:** `Region` holds how numbers, times and measurements are written, apart + from the interface language: the number and time locales (BCP 47), metric or US measurement, A4 + or Letter, and the decimal and grouping separators (or no grouping) and clock when the system + sets them; otherwise the number locale's CLDR data decides. `Region::system()` reads it once: + Windows' regional format with the user's changes (`HKCU\Control Panel\International`), Apple's + current locale (`en_PL`, `en_GB@rg=uszzzz`), `LC_ALL` or else `LC_NUMERIC`, `LC_TIME`, + `LC_MEASUREMENT` and `LC_PAPER` each on its own over `LANG` on Unix (`C` there is `en-US`; + `LANG=C` gives way to the preferred language), the preferred language on the web and Android. + `Region::from_setting` takes the app's saved choice (`auto` or a tag like `pl-PL`). Paper and + measurement follow CLDR's territory lists; tags may carry `-u-ms-`, `-u-hc-` and `-u-rg-`. + Ported from WordCraft's `Region`. - **Pseudo locale** (`pseudo`, debug builds only, never in `languages()`): every lookup, `get` included, comes back as `⟦…⟧`, line by line, so text drawn without the marks never went through translation. `looked_up()` lists the source texts looked up so far, for an audit against a diff --git a/crates/i18n/data/generate.sh b/crates/i18n/data/generate.sh new file mode 100755 index 00000000..91b6ab94 --- /dev/null +++ b/crates/i18n/data/generate.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Regenerates the number and plural data in this directory from Unicode CLDR, for every CLDR +# locale: 10 KB, of which CLDR's modern-coverage locales alone would be 7 KB. Lookups fall back +# at run time, so a locale whose data matches its parent's is left out. +# +# Needs `cargo install icu4x-datagen --version 2.3.0` (the ICU4X minor version in Cargo.toml) +# and python3. Downloads CLDR's JSON release once into $CACHE. +set -euo pipefail + +CLDR=48.2.1 +HERE=$(cd "$(dirname "$0")" && pwd) +CACHE=${CACHE:-${XDG_CACHE_HOME:-$HOME/.cache}/craft-i18n-data} +mkdir -p "$CACHE" + +TMPDIR=$CACHE icu4x-datagen --format blob --overwrite \ + --cldr-tag "$CLDR" \ + --markers DecimalSymbolsV1 DecimalDigitsV1 PluralsCardinalV1 \ + --locales full --deduplication maximal \ + --out "$HERE/numbers.postcard" + +ZIP=$CACHE/icu4x-source-cache/github.com/unicode-org/cldr-json/releases/download/$CLDR/cldr-$CLDR-json-full.zip +python3 -I "$HERE/locales.py" "$ZIP" "$CLDR" > "$HERE/locales.rs" diff --git a/crates/i18n/data/locales.py b/crates/i18n/data/locales.py new file mode 100644 index 00000000..337bcfb9 --- /dev/null +++ b/crates/i18n/data/locales.py @@ -0,0 +1,70 @@ +"""Writes locales.rs from CLDR's JSON release: the languages with plural rules and every locale's +percent affixes. + +Usage: locales.py +""" + +import json +import re +import sys +import zipfile + + +def rust(s): + return '"' + "".join(c if " " <= c <= "~" and c not in '"\\' else f"\\u{{{ord(c):x}}}" for c in s) + '"' + + +def affixes(numbers, numsys): + pattern = numbers[f"percentFormats-numberSystem-{numsys}"]["standard"].split(";")[0].replace("'", "") + sign = numbers[f"symbols-numberSystem-{numsys}"]["percentSign"] + number = re.search(r"[#0,.]+", pattern) + return pattern[: number.start()].replace("%", sign), pattern[number.end() :].replace("%", sign) + + +def main(): + path, version = sys.argv[1], sys.argv[2] + found = {} + with zipfile.ZipFile(path) as z: + plurals = json.loads(z.read("cldr-core/supplemental/plurals.json"))["supplemental"]["plurals-type-cardinal"] + languages = sorted(l for l in plurals if l != "root" and "-" not in l) + for name in z.namelist(): + m = re.fullmatch(r"cldr-numbers-full/main/([^/]+)/numbers\.json", name) + if not m or m.group(1) == "root": + continue + locale = m.group(1) + numbers = json.loads(z.read(name))["main"][locale]["numbers"] + found[(locale, True)] = affixes(numbers, "latn") + default = numbers["defaultNumberingSystem"] + if default != "latn": + found[(locale, False)] = affixes(numbers, default) + + # Kept only where the lookup by dropping trailing subtags would find something else. + kept = {} + for (locale, latin), value in sorted(found.items(), key=lambda e: (e[0][0].count("-"), e[0])): + parts = locale.split("-") + inherited = ("", "%") + for n in range(len(parts) - 1, 0, -1): + if ("-".join(parts[:n]), latin) in kept: + inherited = kept[("-".join(parts[:n]), latin)] + break + if value != inherited: + kept[(locale, latin)] = value + + print(f"// @generated by generate.sh from Unicode CLDR {version} (Unicode License).") + print() + print("/// The languages CLDR has plural rules for; the baked data has them all.") + print("pub(crate) const PLURAL_LANGUAGES: &[&str] = &[") + for i in range(0, len(languages), 16): + print(" " + ", ".join(rust(l) for l in languages[i : i + 16]) + ",") + print("];") + print() + print("/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin") + print("/// digits or the locale's own, text before the number, text after it. A locale missing here") + print("/// takes its parent's by dropping trailing subtags, then `{n}%`.") + print("pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[") + for (locale, latin), (before, after) in sorted(kept.items()): + print(f" ({rust(locale)}, {str(latin).lower()}, {rust(before)}, {rust(after)}),") + print("];") + + +main() diff --git a/crates/i18n/data/locales.rs b/crates/i18n/data/locales.rs new file mode 100644 index 00000000..205a9661 --- /dev/null +++ b/crates/i18n/data/locales.rs @@ -0,0 +1,144 @@ +// @generated by generate.sh from Unicode CLDR 48.2.1 (Unicode License). + +/// The languages CLDR has plural rules for; the baked data has them all. +pub(crate) const PLURAL_LANGUAGES: &[&str] = &[ + "af", "ak", "am", "an", "ar", "ars", "as", "asa", "ast", "az", "bal", "be", "bem", "bez", "bg", "bho", + "blo", "bm", "bn", "bo", "br", "brx", "bs", "ca", "ce", "ceb", "cgg", "chr", "ckb", "cs", "csw", "cv", + "cy", "da", "de", "doi", "dsb", "dv", "dz", "ee", "el", "en", "eo", "es", "et", "eu", "fa", "ff", + "fi", "fil", "fo", "fr", "fur", "fy", "ga", "gd", "gl", "gsw", "gu", "guw", "gv", "ha", "haw", "he", + "hi", "hnj", "hr", "hsb", "hu", "hy", "ia", "id", "ie", "ig", "ii", "io", "is", "it", "iu", "ja", + "jbo", "jgo", "jmc", "jv", "jw", "ka", "kab", "kaj", "kcg", "kde", "kea", "kk", "kkj", "kl", "km", "kn", + "ko", "kok", "ks", "ksb", "ksh", "ku", "kw", "ky", "lag", "lb", "lg", "lij", "lkt", "lld", "ln", "lo", + "lt", "lv", "mas", "mg", "mgo", "mk", "ml", "mn", "mo", "mr", "ms", "mt", "my", "nah", "naq", "nb", + "nd", "ne", "nl", "nn", "nnh", "no", "nqo", "nr", "nso", "ny", "nyn", "om", "or", "os", "osa", "pa", + "pap", "pcm", "pl", "prg", "ps", "pt", "rm", "ro", "rof", "ru", "rwk", "sah", "saq", "sat", "sc", "scn", + "sd", "sdh", "se", "seh", "ses", "sg", "sgs", "sh", "shi", "si", "sk", "sl", "sma", "smi", "smj", "smn", + "sms", "sn", "so", "sq", "sr", "ss", "ssy", "st", "su", "sv", "sw", "syr", "ta", "te", "teo", "th", + "ti", "tig", "tk", "tl", "tn", "to", "tpi", "tr", "ts", "tzm", "ug", "uk", "und", "ur", "uz", "ve", + "vec", "vi", "vo", "vun", "wa", "wae", "wo", "xh", "xog", "yi", "yo", "yue", "zh", "zu", +]; + +/// Percent signs around the number (CLDR's standard percent pattern and sign): locale, Latin +/// digits or the locale's own, text before the number, text after it. A locale missing here +/// takes its parent's by dropping trailing subtags, then `{n}%`. +pub(crate) const PERCENT: &[(&str, bool, &str, &str)] = &[ + ("ar", true, "", "\u{200e}%\u{200e}"), + ("ar-BH", false, "", "\u{66a}\u{61c}"), + ("ar-DJ", false, "", "\u{66a}\u{61c}"), + ("ar-EG", false, "", "\u{66a}\u{61c}"), + ("ar-ER", false, "", "\u{66a}\u{61c}"), + ("ar-IL", false, "", "\u{66a}\u{61c}"), + ("ar-IQ", false, "", "\u{66a}\u{61c}"), + ("ar-JO", false, "", "\u{66a}\u{61c}"), + ("ar-KM", false, "", "\u{66a}\u{61c}"), + ("ar-KW", false, "", "\u{66a}\u{61c}"), + ("ar-LB", false, "", "\u{66a}\u{61c}"), + ("ar-MR", false, "", "\u{66a}\u{61c}"), + ("ar-OM", false, "", "\u{66a}\u{61c}"), + ("ar-PS", false, "", "\u{66a}\u{61c}"), + ("ar-QA", false, "", "\u{66a}\u{61c}"), + ("ar-SA", false, "", "\u{66a}\u{61c}"), + ("ar-SA", true, "", "\u{66a}"), + ("ar-SD", false, "", "\u{66a}\u{61c}"), + ("ar-SO", false, "", "\u{66a}\u{61c}"), + ("ar-SO", true, "", "\u{66a}"), + ("ar-SS", false, "", "\u{66a}\u{61c}"), + ("ar-SY", false, "", "\u{66a}\u{61c}"), + ("ar-TD", false, "", "\u{66a}\u{61c}"), + ("ar-YE", false, "", "\u{66a}\u{61c}"), + ("az-Arab", false, "", "\u{66a}"), + ("bas", true, "", "\u{a0}%"), + ("be", true, "", "\u{a0}%"), + ("bgn", false, "", "\u{66a}"), + ("bgn", true, "", "\u{66a}"), + ("blo", true, "%\u{a0}", ""), + ("br", true, "", "\u{a0}%"), + ("bua", true, "", "\u{a0}%"), + ("ca", true, "", "\u{a0}%"), + ("ce", true, "", "\u{a0}%"), + ("ckb", false, "", "\u{a0}\u{66a}"), + ("cs", true, "", "\u{a0}%"), + ("cv", true, "", "\u{a0}%"), + ("da", true, "", "\u{a0}%"), + ("de", true, "", "\u{a0}%"), + ("de-CH", true, "", "%"), + ("de-LI", true, "", "%"), + ("dsb", true, "", "\u{a0}%"), + ("dua", true, "", "\u{a0}%"), + ("dz", false, "", "\u{a0}%"), + ("dz", true, "", "\u{a0}%"), + ("en-AT", true, "", "\u{a0}%"), + ("en-DE", true, "", "\u{a0}%"), + ("en-DK", true, "", "\u{a0}%"), + ("en-FI", true, "", "\u{a0}%"), + ("en-NO", true, "", "\u{a0}%"), + ("en-SE", true, "", "\u{a0}%"), + ("es", true, "", "\u{a0}%"), + ("es-419", true, "", "%"), + ("es-AR", true, "", "%"), + ("es-BO", true, "", "%"), + ("es-BR", true, "", "%"), + ("es-BZ", true, "", "%"), + ("es-CL", true, "", "%"), + ("es-CO", true, "", "%"), + ("es-CR", true, "", "%"), + ("es-CU", true, "", "%"), + ("es-DO", true, "", "%"), + ("es-EC", true, "", "%"), + ("es-GT", true, "", "%"), + ("es-HN", true, "", "%"), + ("es-MX", true, "", "%"), + ("es-NI", true, "", "%"), + ("es-PA", true, "", "%"), + ("es-PE", true, "", "%"), + ("es-PR", true, "", "%"), + ("es-PY", true, "", "%"), + ("es-SV", true, "", "%"), + ("es-US", true, "", "%"), + ("es-UY", true, "", "%"), + ("es-VE", true, "", "%"), + ("eu", true, "%\u{a0}", ""), + ("fa", false, "", "\u{66a}"), + ("fi", true, "", "\u{a0}%"), + ("fo", true, "", "\u{a0}%"), + ("fr", true, "", "\u{a0}%"), + ("fr-CH", true, "", "%"), + ("gl", true, "", "\u{a0}%"), + ("gsw", true, "", "\u{a0}%"), + ("hr", true, "", "\u{a0}%"), + ("hsb", true, "", "\u{a0}%"), + ("ht", true, "", "\u{a0}%"), + ("kl", true, "", "\u{a0}%"), + ("ks", false, "", "\u{66a}"), + ("ksh", true, "", "\u{a0}%"), + ("ku", true, "%", ""), + ("ku-Arab", true, "", "%"), + ("lb", true, "", "\u{a0}%"), + ("lrc", false, "", "\u{66a}"), + ("lt", true, "", "\u{a0}%"), + ("mk", true, "", "\u{a0}%"), + ("mzn", false, "", "\u{66a}"), + ("nb", true, "", "\u{a0}%"), + ("nds", true, "", "\u{a0}%"), + ("nn", true, "", "\u{a0}%"), + ("no", true, "", "\u{a0}%"), + ("oc", true, "", "\u{202f}%"), + ("pa-Arab", false, "", "\u{66a}"), + ("ps", false, "", "\u{66a}"), + ("qu", true, "", "\u{a0}%"), + ("rn", true, "", "\u{a0}%"), + ("ro", true, "", "\u{a0}%"), + ("ru", true, "", "\u{a0}%"), + ("sd", false, "", "\u{66a}\u{61c}"), + ("sdh", false, "", "\u{66a}\u{61c}"), + ("se", true, "", "\u{a0}%"), + ("sk", true, "", "\u{a0}%"), + ("sl", true, "", "\u{a0}%"), + ("smn", true, "", "\u{a0}%"), + ("sv", true, "", "\u{a0}%"), + ("tk", true, "", "\u{a0}%"), + ("tr", true, "%", ""), + ("tt", true, "", "\u{a0}%"), + ("uz-Arab", false, "", "\u{66a}"), + ("zgh", true, "", "\u{a0}%"), +]; diff --git a/crates/i18n/data/numbers.postcard b/crates/i18n/data/numbers.postcard new file mode 100644 index 00000000..5ee84e03 Binary files /dev/null and b/crates/i18n/data/numbers.postcard differ diff --git a/crates/i18n/src/format.rs b/crates/i18n/src/format.rs new file mode 100644 index 00000000..6def7d18 --- /dev/null +++ b/crates/i18n/src/format.rs @@ -0,0 +1,318 @@ +//! Numbers, percentages and counts written for a formatting [`Region`], on ICU4X's decimal +//! formatter with CLDR data for every CLDR locale (`data/generate.sh`). + +use std::fmt; + +use fixed_decimal::{Decimal, Sign}; +use icu_decimal::{DecimalFormatter, parts}; +use icu_locale_core::Locale; +use icu_locale_core::extensions::unicode::{key, value}; +use writeable::{Part, PartsWrite, Writeable}; + +use crate::Region; +use crate::data::{self, PERCENT}; + +/// Which digits numbers are written in. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Digits { + /// The locale's own: Arabic-Indic in Egypt (`١٢٣`), Persian in Iran (`۱۲۳`), else `0`–`9`. + #[default] + Locale, + /// `0`–`9` everywhere, for users who read Arabic or Persian with Latin digits. + Latin, +} + +/// How numbers are written in a formatting region: digits, decimal and grouping separators, +/// grouping sizes (`1,234,567`, Indian `12,34,567`) and the percent sign. Build it once per region +/// and keep it; [`crate::I18n`] holds one for counts, sizes and measures. +#[derive(Debug)] +pub struct Format { + /// `None` only if the data doesn't load: numbers are then written as Rust writes them. + decimal: Option, + /// The system's separators where they differ from the locale's; an empty `group` is none. + group: Option, + point: Option, + percent: (&'static str, &'static str), +} + +impl Format { + /// The format for `region`'s number locale. With Latin digits, a decimal or grouping separator + /// the region carries (Windows' and macOS's own settings) replaces the locale's. + pub fn new(region: &Region, digits: Digits) -> Format { + let mut locale = Locale::try_from_str(®ion.numbers).unwrap_or(Locale::UNKNOWN); + if digits == Digits::Latin { + locale.extensions.unicode.keywords.set(key!("nu"), value!("latn")); + } + let decimal = data::provider().and_then(|p| DecimalFormatter::try_new_with_buffer_provider(p, (&locale).into(), Default::default()).ok()); + let mut format = Format { decimal, group: None, point: None, percent: ("", "%") }; + // The locale's own separators and digits, read from a number that shows them all. + let probe = format.parts(&Decimal::try_from_str("-1234567.5").unwrap_or_default()); + let latin = probe.integer.starts_with(|c: char| c.is_ascii_digit()); + if latin { + if let Some(point) = region.decimal.map(String::from).filter(|p| *p != probe.point) { + // `1,234.5` with a decimal comma becomes `1.234,5`, never `1,234,5`. + if probe.group == point && region.grouping.is_none() { + format.group = Some(probe.point.clone()); + } + format.point = Some(point); + } + if let Some(g) = region.grouping.map(|g| g.map(String::from).unwrap_or_default()).filter(|g| *g != probe.group) { + format.group = Some(g); + } + } + format.percent = percent_affixes(®ion.numbers, latin); + format + } + + /// The format for a locale tag (`de-CH`, `ar-EG`), with the locale's own digits. + pub fn from_tag(tag: &str) -> Format { + Format::new(&Region::from_tag(tag), Digits::Locale) + } + + /// A whole number: `1,234,567`, `1.234.567`, `١٬٢٣٤٬٥٦٧`. + pub fn integer(&self, n: impl Into) -> String { + self.write(&Decimal::from(n.into())) + } + + /// `value` rounded to `decimals` digits after the decimal separator; infinities and NaN are + /// `∞`, `-∞` and `NaN`. + pub fn number(&self, value: f64, decimals: usize) -> String { + match to_decimal(value, decimals) { + Some(d) => self.write(&d), + None => non_finite(value).to_owned(), + } + } + + /// `ratio` as a percentage with `decimals` digits after the decimal separator: `0.5` is + /// `50%`, `50 %` (French, German), `%50` (Turkish), `٥٠٪؜` (Egyptian Arabic). + pub fn percent(&self, ratio: f64, decimals: usize) -> String { + let Some(d) = to_decimal(ratio * 100.0, decimals) else { + return format!("{}{}{}", self.percent.0, non_finite(ratio), self.percent.1); + }; + let p = self.parts(&d); + format!("{}{}{}{}", p.sign, self.percent.0, p.text, self.percent.1) + } + + /// `d` as text, the sign in place. + pub(crate) fn write(&self, d: &Decimal) -> String { + let p = self.parts(d); + p.sign + &p.text + } + + fn parts(&self, d: &Decimal) -> Written { + let mut sink = Sink { group: self.group.as_deref(), point: self.point.as_deref(), into: Target::Text, w: Written::default() }; + let written = match &self.decimal { + Some(f) => f.format(d).write_to_parts(&mut sink), + None => fmt::Write::write_str(&mut sink, &d.to_string()), + }; + if written.is_err() { + return Written { text: d.to_string(), ..Written::default() }; + } + sink.w + } +} + +/// A formatted number taken apart: its sign, the rest, and the separators and integer digits as +/// the locale wrote them. +#[derive(Default)] +struct Written { + sign: String, + text: String, + integer: String, + group: String, + point: String, +} + +#[derive(Clone, Copy, PartialEq)] +enum Target { + Text, + Sign, + /// A separator the region replaces; the locale's goes to `group` or `point` only. + Replaced, +} + +/// Writes the formatter's parts into [`Written`], swapping in the region's separators. +struct Sink<'a> { + group: Option<&'a str>, + point: Option<&'a str>, + into: Target, + w: Written, +} + +impl fmt::Write for Sink<'_> { + fn write_str(&mut self, s: &str) -> fmt::Result { + match self.into { + Target::Text => self.w.text.push_str(s), + Target::Sign => self.w.sign.push_str(s), + Target::Replaced => {} + } + Ok(()) + } +} + +impl PartsWrite for Sink<'_> { + type SubPartsWrite = Self; + + fn with_part(&mut self, part: Part, mut f: impl FnMut(&mut Self) -> fmt::Result) -> fmt::Result { + let outer = self.into; + let start = self.w.text.len(); + let replacement = match part { + p if p == parts::GROUP => self.group, + p if p == parts::DECIMAL => self.point, + _ => None, + }; + if let Some(r) = replacement { + self.w.text.push_str(r); + self.into = Target::Replaced; + } else if part == parts::MINUS_SIGN || part == parts::PLUS_SIGN { + self.into = Target::Sign; + } + let result = f(self); + self.into = outer; + if replacement.is_none() { + let written = self.w.text.get(start..).unwrap_or_default(); + let slot = match part { + p if p == parts::GROUP => &mut self.w.group, + p if p == parts::DECIMAL => &mut self.w.point, + p if p == parts::INTEGER => &mut self.w.integer, + _ => return result, + }; + if slot.is_empty() { + *slot = written.to_owned(); + } + } + result + } +} + +/// `value` rounded to `decimals` digits after the point; `None` for infinities and NaN. A value +/// that rounds to zero has no sign. +pub(crate) fn to_decimal(value: f64, decimals: usize) -> Option { + if !value.is_finite() { + return None; + } + let mut d = Decimal::try_from_str(&format!("{value:.decimals$}")).ok()?; + if d.absolute.is_zero() { + d.sign = Sign::None; + } + Some(d) +} + +/// CLDR's root symbols for a value that isn't finite. +fn non_finite(value: f64) -> &'static str { + match value { + f64::INFINITY => "∞", + f64::NEG_INFINITY => "-∞", + _ => "NaN", + } +} + +/// The percent affixes for a number locale (`ar-EG`), found by dropping trailing subtags. +fn percent_affixes(tag: &str, latin: bool) -> (&'static str, &'static str) { + let mut tag = tag; + loop { + if let Some((_, _, before, after)) = PERCENT.iter().find(|(l, lat, _, _)| *lat == latin && l.eq_ignore_ascii_case(tag)) { + return (before, after); + } + match tag.rsplit_once('-') { + Some((parent, _)) => tag = parent, + None => return ("", "%"), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn latin(tag: &str) -> Format { + Format::new(&Region::from_tag(tag), Digits::Latin) + } + + #[test] + fn locale_digits_and_separators() { + for (tag, integer, number) in [ + ("en", "1,234,567", "1,234.5"), + ("ja", "1,234,567", "1,234.5"), + ("de", "1.234.567", "1.234,5"), + ("fr", "1\u{202f}234\u{202f}567", "1\u{202f}234,5"), + ("hi", "12,34,567", "1,234.5"), + ("ar-EG", "١٬٢٣٤٬٥٦٧", "١٬٢٣٤٫٥"), + ("fa", "۱٬۲۳۴٬۵۶۷", "۱٬۲۳۴٫۵"), + ] { + let f = Format::from_tag(tag); + assert_eq!((f.integer(1_234_567), f.number(1234.5, 1)), (integer.into(), number.into()), "{tag}"); + } + // CLDR writes Arabic without a region in Latin digits. + assert_eq!(Format::from_tag("ar").number(1234.5, 1), "1,234.5"); + } + + #[test] + fn forced_latin_digits() { + assert_eq!(latin("ar-EG").number(1234.5, 1), "1,234.5"); + assert_eq!(latin("fa").integer(1_234_567), "1,234,567"); + assert_eq!(latin("de").number(1234.5, 1), "1.234,5", "a Latin locale is unchanged"); + } + + #[test] + fn cldr_separators_for_every_region() { + for (tag, want) in [ + ("en-PL", "1.234.567,5"), + ("en-DE", "1.234.567,5"), + ("ar-MA", "1.234.567,5"), + ("de-CH", "1'234'567.5"), + ("nb-NO", "1\u{a0}234\u{a0}567,5"), + ("sk-SK", "1\u{a0}234\u{a0}567,5"), + ("bn-BD", "১২,৩৪,৫৬৭.৫"), + ("he-IL", "1,234,567.5"), + ("km", "1,234,567.5"), + ("lo", "1.234.567,5"), + ] { + assert_eq!(Format::from_tag(tag).number(1_234_567.5, 1), want, "{tag}"); + } + } + + #[test] + fn system_separators() { + // A decimal comma the system sets moves the grouping out of its way. + let mut region = Region::from_tag("en-US"); + region.decimal = Some(','); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1.234,5"); + let mut region = Region::from_tag("de-DE"); + region.grouping = Some(Some('\u{a0}')); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234,5"); + region.decimal = Some('.'); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "1\u{a0}234.5"); + region.grouping = Some(None); + assert_eq!(Format::new(®ion, Digits::Locale).integer(1_234_567), "1234567", "no grouping"); + // A system's separators don't apply to Arabic-Indic digits. + let mut region = Region::from_tag("ar-EG"); + region.grouping = Some(Some(',')); + assert_eq!(Format::new(®ion, Digits::Locale).number(1234.5, 1), "١٬٢٣٤٫٥"); + } + + #[test] + fn percentages() { + for (tag, want) in + [("en", "50%"), ("de", "50\u{a0}%"), ("fr", "12,5\u{a0}%"), ("tr", "%50"), ("ar-EG", "٥٠٪\u{61c}"), ("fa", "۵۰٪"), ("hi", "50%")] + { + let decimals = if tag == "fr" { 1 } else { 0 }; + let ratio = if tag == "fr" { 0.125 } else { 0.5 }; + assert_eq!(Format::from_tag(tag).percent(ratio, decimals), want, "{tag}"); + } + assert_eq!(Format::from_tag("tr").percent(-0.2, 0), "-%20", "the sign leads the pattern"); + assert_eq!(latin("fa").percent(0.5, 0), "50%"); + } + + #[test] + fn signs_and_odd_values() { + let en = Format::from_tag("en"); + assert_eq!(en.number(-1234.5, 0), "-1,234"); + assert_eq!(en.number(-0.04, 1), "0.0", "no sign on zero"); + assert_eq!(en.number(f64::INFINITY, 1), "∞"); + assert_eq!(en.number(f64::NEG_INFINITY, 1), "-∞"); + assert_eq!(en.number(f64::NAN, 1), "NaN"); + assert_eq!(Format::from_tag("de").percent(f64::NAN, 0), "NaN\u{a0}%"); + assert_eq!(en.integer(-5_i64), "-5"); + } +} diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 4ceaf2c2..7618f32a 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -3,18 +3,51 @@ //! An app embeds the [craft-i18n](https://github.com/benandez/craft-i18n) catalogs at build time //! as an [`Embedded`] static, builds one [`I18n`] from it, and looks English source text up in a //! [`Lang`]. A missing translation is the English itself; nothing here fails. +//! +//! Counts ([`I18n::trn`]), file sizes and measures write their numbers as the formatting +//! [`Region`] does ([`I18n::format`]): grouped (`1.234`, `12,34,567`) and in the region's digits +//! (`١٬٢٣٤`). Text that goes into documents or file names should use [`I18n::trn_plain`], which +//! writes `0`–`9` without grouping. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod catalog; +mod format; mod number; +mod region; +mod size; + +/// The CLDR number and plural data for every CLDR locale, from `data/generate.sh`. +mod data { + use std::sync::OnceLock; + + use icu_locale_fallback::LocaleFallbacker; + use icu_provider_adapters::fallback::LocaleFallbackProvider; + use icu_provider_blob::BlobDataProvider; + + include!("../data/locales.rs"); + + /// The number and plural data, read once; `None` if the blob doesn't load. + pub(crate) fn provider() -> Option<&'static LocaleFallbackProvider> { + static PROVIDER: OnceLock>> = OnceLock::new(); + PROVIDER + .get_or_init(|| { + let blob = BlobDataProvider::try_new_from_static_blob(include_bytes!("../data/numbers.postcard")).ok()?; + Some(LocaleFallbackProvider::new(blob, LocaleFallbacker::new().static_to_owned())) + }) + .as_ref() + } +} use std::collections::{BTreeMap, BTreeSet, HashMap}; -use std::sync::{Mutex, OnceLock, PoisonError}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError, RwLock}; -use intl_pluralrules::{PluralCategory, PluralRuleType, PluralRules}; -use unic_langid::LanguageIdentifier; +use icu_locale_core::Locale as IcuLocale; +use icu_plurals::{PluralCategory, PluralOperands, PluralRules}; +pub use format::{Digits, Format}; pub use number::{NumberError, parse_number}; +pub use region::{HourCycle, Measurement, Paper, Region}; +pub use size::{FileSize, SizeUnit}; /// The catalogs an app embeds, one entry per locale: (locale, shared catalog, the app's catalog). /// A missing catalog is `""`. @@ -55,6 +88,8 @@ pub struct I18n { pseudo: Mutex>, /// [`I18n::system_lang`], read once. system: OnceLock, + /// [`I18n::format`]; the system region's until [`I18n::set_format`]. + format: RwLock>>, } impl I18n { @@ -105,7 +140,7 @@ impl I18n { l.chain = chain; } } - (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new() }, problems) + (I18n { locales, pseudo: Mutex::default(), system: OnceLock::new(), format: RwLock::default() }, problems) } /// The language with this code (any case): `en`, an offered locale, or in debug builds @@ -161,9 +196,34 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; - /// they are also the key of the `@plural` row, and the fallback. + /// A count's wording in `lang`, `{n}` filled in as [`I18n::format`] writes numbers: grouped and + /// in the region's digits. `one` and `other` are the English forms; they are also the key of + /// the `@plural` row, and the fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + let format = self.format(); + self.plural(lang, Some(PluralOperands::from(n)), n == 1, &format.integer(n), one, other) + } + + /// [`I18n::trn`] with `{n}` in `0`–`9` without grouping (`Copy 1234`), for text that goes into + /// documents, file names or anything read back later. + pub fn trn_plain(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { + self.plural(lang, Some(PluralOperands::from(n)), n == 1, &n.to_string(), one, other) + } + + /// [`I18n::trn`] for a count with `decimals` digits after the decimal separator (`1.5 files`). + /// The plural form follows the digits shown, so `1.0` takes English `other`; infinities and NaN + /// (`∞`, `NaN`) take `other` too. + pub fn trn_decimal(&self, lang: Lang, n: f64, decimals: usize, one: &str, other: &str) -> String { + let format = self.format(); + let Some(d) = format::to_decimal(n, decimals) else { + return self.plural(lang, None, false, &format.number(n, decimals), one, other); + }; + let english_one = d.to_string() == "1"; + self.plural(lang, Some(PluralOperands::from(&d)), english_one, &format.write(&d), one, other) + } + + /// `operands` `None` is the `other` category. + fn plural(&self, lang: Lang, operands: Option, english_one: bool, n: &str, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); let text = chain @@ -172,19 +232,45 @@ impl I18n { let locale = self.locales.get(i)?; let forms = locale.rows.get("@plural")?.get(&key)?; let (rules, categories) = locale.plural.as_ref()?; - let category = rules.select(n).ok()?; - // A category whole numbers up to 1000 never reach (French `many`) takes the last form. - let index = categories.iter().position(|c| *c == category).unwrap_or(categories.len().saturating_sub(1)); + let category = operands.map_or(PluralCategory::Other, |o| rules.category_for(o)); + // A category whole numbers up to 1000 never reach has no form of its own. Russian + // and Ukrainian decimals (`other`) take the `few` form (`1,5 файла`); any other + // (French `many` for a million) takes the last. + let index = categories + .iter() + .position(|c| *c == category) + .or_else(|| categories.iter().position(|c| *c == PluralCategory::Few)) + .unwrap_or(categories.len().saturating_sub(1)); forms.split('|').nth(index) }) - .unwrap_or(if n == 1 { one } else { other }); - let text = fmt(text, &[("n", &n.to_string())]); + .unwrap_or(if english_one { one } else { other }); + let text = fmt(text, &[("n", n)]); if lang == Lang::PSEUDO { return marked(&text); } text } + /// How counts, file sizes and measures write numbers: [`Region::system`]'s format with the + /// locale's digits until [`I18n::set_format`], and again after [`I18n::reset_format`]. + pub fn format(&self) -> Arc { + if let Some(f) = self.format.read().unwrap_or_else(PoisonError::into_inner).as_ref() { + return Arc::clone(f); + } + let mut slot = self.format.write().unwrap_or_else(PoisonError::into_inner); + Arc::clone(slot.get_or_insert_with(|| Arc::new(Format::new(&Region::system(), Digits::Locale)))) + } + + /// Write numbers as `format` does from now on: the user's region setting, or Latin digits. + pub fn set_format(&self, format: Format) { + *self.format.write().unwrap_or_else(PoisonError::into_inner) = Some(Arc::new(format)); + } + + /// Write numbers as the system region does again, undoing [`I18n::set_format`]. + pub fn reset_format(&self) { + *self.format.write().unwrap_or_else(PoisonError::into_inner) = None; + } + /// The languages to offer: English first, then the offered locales by code. Never the /// pseudo locale. pub fn languages(&self) -> Vec { @@ -367,17 +453,22 @@ fn bare(code: &str) -> Locale { } /// `code`'s CLDR cardinal rules and the categories whole numbers 0..=1000 reach, in CLDR order -/// (zero, one, two, few, many, other): one `@plural` form each. Regional and scripted codes fall -/// back to the language's rules. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other -/// than `pt-br` takes `pt-PT`'s. +/// (zero, one, two, few, many, other): one `@plural` form each. The rules are looked up by +/// language alone: CLDR's parent of `sr-Latn` and `zh-Hant` is the root, whose only category is +/// `other`. CLDR's `pt` is Brazilian (0 is singular), so Portuguese other than `pt-br` takes +/// `pt-PT`'s. `None` for a language CLDR has no plural rules for. fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { - let code = if code == "pt" || (code.starts_with("pt-") && code != "pt-br") { "pt-pt" } else { code }; - let id: LanguageIdentifier = code.parse().ok()?; - let language = LanguageIdentifier::from_parts(id.language, None, None, &[]); - let rules = PluralRules::create(id, PluralRuleType::CARDINAL).or_else(|_| PluralRules::create(language, PluralRuleType::CARDINAL)).ok()?; + let id = IcuLocale::try_from_str(code).ok()?.id; + let language = id.language.as_str(); + if !data::PLURAL_LANGUAGES.contains(&language) { + return None; + } + let brazil = id.region.is_some_and(|r| r.as_str() == "BR"); + let locale = IcuLocale::try_from_str(if language == "pt" && !brazil { "pt-PT" } else { language }).ok()?; + let rules = PluralRules::try_new_cardinal_with_buffer_provider(data::provider()?, (&locale).into()).ok()?; let mut categories = Vec::new(); for n in 0..=1000_u32 { - let c = rules.select(n).ok()?; + let c = rules.category_for(n); if !categories.contains(&c) { categories.push(c); } @@ -388,12 +479,12 @@ fn plural_rules(code: &str) -> Option<(PluralRules, Vec)> { fn rank(c: &PluralCategory) -> u8 { match c { - PluralCategory::ZERO => 0, - PluralCategory::ONE => 1, - PluralCategory::TWO => 2, - PluralCategory::FEW => 3, - PluralCategory::MANY => 4, - PluralCategory::OTHER => 5, + PluralCategory::Zero => 0, + PluralCategory::One => 1, + PluralCategory::Two => 2, + PluralCategory::Few => 3, + PluralCategory::Many => 4, + PluralCategory::Other => 5, } } @@ -408,10 +499,12 @@ mod tests { ("de", "\tBad\n@nope\tA\tB\n\tUndo\tRückgängig\n", ""), ]; - /// `embedded`, offering every locale in it. + /// `embedded`, offering every locale in it, with `en-US` numbers rather than the machine's. fn all(embedded: &'static Embedded) -> (I18n, Vec) { let codes: Vec<&str> = embedded.iter().map(|(c, _, _)| *c).collect(); - I18n::new(embedded, &codes) + let (i, problems) = I18n::new(embedded, &codes); + i.set_format(Format::from_tag("en-US")); + (i, problems) } fn i18n() -> I18n { @@ -484,6 +577,7 @@ mod tests { ("fr", "@plural\t{n} page|{n} pages\t{n} page|{n} pages\n", ""), ("ru", "@plural\t{n} file|{n} files\t{n} файл|{n} файла|{n} файлов\n", ""), ("ja", "@plural\t{n} file|{n} files\t{n} 個\n", ""), + ("ar", "@plural\t{n} file|{n} files\t{n} ملفات|{n} ملف واحد|{n} ملفان|{n} ملفات|{n} ملفًا|{n} ملف\n", ""), ]; #[test] @@ -526,7 +620,56 @@ mod tests { #[test] fn beyond_sweep() { let i = all(P).0; - assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); + i.set_format(Format::from_tag("fr")); + assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1\u{202f}000\u{202f}000 pages"); + } + + #[test] + fn counts_follow_the_region() { + let i = all(P).0; + i.set_format(Format::from_tag("ar-EG")); + assert_eq!(i.trn(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "١٬٢٣٤ ملفًا"); + i.set_format(Format::new(&Region::from_tag("ar-EG"), Digits::Latin)); + assert_eq!(i.trn(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "1,234 ملفًا"); + // An English interface with a German region. + i.set_format(Format::from_tag("de-DE")); + assert_eq!(i.trn(Lang::EN, 12_345, "{n} file", "{n} files"), "12.345 files"); + } + + #[test] + fn plain_counts_and_the_system_format() { + let i = all(P).0; + i.set_format(Format::from_tag("ar-EG")); + assert_eq!(i.trn_plain(lang(&i, "ar"), 1234, "{n} file", "{n} files"), "1234 ملفًا"); + i.set_format(Format::from_tag("de-DE")); + assert_eq!(i.trn_plain(Lang::EN, 12_345, "copy {n}", "copy {n}"), "copy 12345"); + i.reset_format(); + assert_eq!( + i.trn(Lang::EN, 5, "{n} file", "{n} files"), + fmt("{n} files", &[("n", &Format::new(&Region::system(), Digits::Locale).integer(5))]) + ); + } + + #[test] + fn decimal_counts() { + let i = all(P).0; + i.set_format(Format::from_tag("en")); + assert_eq!(i.trn_decimal(Lang::EN, 1.5, 1, "{n} file", "{n} files"), "1.5 files"); + assert_eq!(i.trn_decimal(Lang::EN, 1.0, 1, "{n} file", "{n} files"), "1.0 files", "visible decimals are plural"); + assert_eq!(i.trn_decimal(Lang::EN, 1.0, 0, "{n} file", "{n} files"), "1 file"); + // Russian decimals are `other`, which integers never reach: the `few` form. + i.set_format(Format::from_tag("ru")); + let ru = lang(&i, "ru"); + assert_eq!(i.trn_decimal(ru, 1.5, 1, "{n} file", "{n} files"), "1,5 файла"); + assert_eq!(i.trn_decimal(ru, 21.0, 0, "{n} file", "{n} files"), "21 файл"); + // Arabic decimals are `other`. + i.set_format(Format::from_tag("ar-EG")); + let ar = lang(&i, "ar"); + assert_eq!(i.trn_decimal(ar, 2.5, 1, "{n} file", "{n} files"), "٢٫٥ ملف"); + assert_eq!(i.trn_decimal(ar, 3.0, 0, "{n} file", "{n} files"), "٣ ملفات"); + // Not a number: the translated `other` form. + assert_eq!(i.trn_decimal(ru, f64::NAN, 1, "{n} file", "{n} files"), "NaN файла"); + assert_eq!(i.trn_decimal(Lang::EN, f64::INFINITY, 1, "{n} file", "{n} files"), "∞ files"); } static L: &Embedded = @@ -677,15 +820,31 @@ mod tests { assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); } - /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog - /// whose `@plural` rows list any other. + /// The counts craft-i18n's checker expects, which rejects a catalog whose `@plural` rows list + /// any other: every directory in craft-i18n's `locales/`, then languages it may add. #[test] fn plural_form_counts() { - for (locale, n) in - [("en", 2), ("fr", 2), ("ja", 1), ("vi", 1), ("ru", 3), ("pl", 3), ("cs", 3), ("sl", 4), ("ar", 6), ("pt", 2), ("pt-br", 2), ("tr", 2)] - { + #[rustfmt::skip] + let catalogs = [ + ("ar", 6), ("bg", 2), ("cs", 3), ("da", 2), ("de", 2), ("el", 2), ("es", 2), ("et", 2), + ("fi", 2), ("fr", 2), ("hr", 3), ("hu", 2), ("id", 1), ("it", 2), ("ja", 1), ("ka", 2), + ("kk", 2), ("ko", 1), ("lt", 3), ("nl", 2), ("pl", 3), ("pt-br", 2), ("ru", 3), ("sl", 4), + ("sr", 3), ("sr-latn", 3), ("sv", 2), ("te", 2), ("tr", 2), ("uk", 3), ("vi", 1), + ("zh-hans", 1), ("zh-hant", 1), + ]; + let more = [("en", 2), ("pt", 2), ("he", 3), ("ro", 3), ("nb", 2), ("ca", 2), ("sk", 3), ("th", 1), ("ms", 1), ("fa", 2), ("hi", 2)]; + for (locale, n) in catalogs.into_iter().chain(more) { assert_eq!(plural_rules(locale).map(|(_, c)| c.len()), Some(n), "{locale}"); } + assert!(plural_rules("tlh").is_none(), "no CLDR rules"); + } + + #[test] + fn serbian_latin_plurals() { + let i = all(&[("sr-latn", "@plural\t{n} file|{n} files\t{n} fajl|{n} fajla|{n} fajlova\n", "")]).0; + let sr = lang(&i, "sr-latn"); + let files = |n| i.trn(sr, n, "{n} file", "{n} files"); + assert_eq!((files(1), files(3), files(5)), ("1 fajl".into(), "3 fajla".into(), "5 fajlova".into())); } #[test] diff --git a/crates/i18n/src/region.rs b/crates/i18n/src/region.rs new file mode 100644 index 00000000..bbebe4c9 --- /dev/null +++ b/crates/i18n/src/region.rs @@ -0,0 +1,453 @@ +//! The formatting region: how numbers, times and measurements are written, apart from the +//! interface language. An English interface in Poland still writes `2,54 cm` on A4. +//! +//! The territory lists come from Unicode CLDR's supplemental `measurementData` (Unicode License): +//! `paperSize` US-Letter and `measurementSystem` US. The separators numbers are written with +//! follow the number locale's CLDR data ([`crate::Format`]); the region carries them only where +//! the system sets its own. + +use std::sync::OnceLock; + +/// Territories whose default paper is US Letter (CLDR `paperSize type="US-Letter"`). +const LETTER: &[&str] = &["BZ", "CA", "CL", "CO", "CR", "GT", "MX", "NI", "PA", "PH", "PR", "SV", "US", "VE"]; + +/// Territories that measure in inches (CLDR `measurementSystem type="US"`). +const INCHES: &[&str] = &["LR", "MM", "US"]; + +/// How lengths, distances and weights are measured. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Measurement { + /// Centimetres, kilometres and kilograms (CLDR `metric` and `UK`). + Metric, + /// Inches, miles and pounds (CLDR `US`). + Us, +} + +/// The paper a new document starts on. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Paper { + A4, + Letter, +} + +/// A clock the user chose, where the system says so apart from the locale. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum HourCycle { + /// 1 to 12 with AM and PM. + H12, + /// 0 to 23. + H23, +} + +/// The conventions numbers, times and measurements follow. Read it with [`Region::system`] or, +/// for the user's saved choice, [`Region::from_setting`]; the fields are public so an app can +/// keep a separate choice of, say, measurement. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Region { + /// The locale numbers are written in, BCP 47 (`pl-PL`, `sr-Latn-RS`). + pub numbers: String, + /// The locale dates and times are written in, BCP 47. + pub time: String, + pub measurement: Measurement, + pub paper: Paper, + /// The decimal separator the system sets (Windows, macOS); `None` follows the locale. + pub decimal: Option, + /// The grouping separator the system sets (Windows, macOS): `Some(None)` for none at all + /// (`1234567`); `None` follows the locale. + pub grouping: Option>, + /// The clock the system sets (Windows, macOS); `None` follows the locale. + pub hour_cycle: Option, +} + +impl Default for Region { + /// `en-US`: Letter, inches. + fn default() -> Self { + Region::from_parts(&Tag::default()) + } +} + +impl Region { + /// The region a BCP 47 or POSIX locale tag names (`pl-PL`, `de_AT.UTF-8@euro`, `zh-Hans-CN`, + /// `en-US-u-ms-metric`, Apple's `en_US@rg=plzzzz`). The Unicode extensions `ms` (measurement), + /// `hc` (clock) and `rg` (region override) are read. An empty or unreadable tag, `C` and + /// `POSIX` are `en-US`; a bare `en` is the United States, any other bare language has no + /// territory. + pub fn from_tag(tag: &str) -> Region { + Region::from_parts(&Tag::parse(tag).unwrap_or_default()) + } + + /// The region for an app's saved region setting: `""` or `auto` (any case) is + /// [`Region::system`], anything else a locale tag for [`Region::from_tag`]. + pub fn from_setting(setting: &str) -> Region { + let setting = setting.trim(); + if setting.is_empty() || setting.eq_ignore_ascii_case("auto") { Region::system() } else { Region::from_tag(setting) } + } + + /// The system's region, read once, then cached: + /// + /// - Windows: the regional format and its customised separators, measurement system, paper + /// and short time format (Settings › Time & language › Language & region). + /// - macOS and iOS: the current locale (its identifier carries the Region setting), its + /// separators and metric setting, and its hour format. + /// - Linux and the BSDs: `LC_ALL`, else each of `LC_NUMERIC`, `LC_TIME`, `LC_MEASUREMENT` and + /// `LC_PAPER` on its own, else `LANG`. + /// - The web and Android: the preferred language, the only locale they expose. + pub fn system() -> Region { + static SYSTEM: OnceLock = OnceLock::new(); + SYSTEM.get_or_init(read_system).clone() + } + + fn from_parts(tag: &Tag) -> Region { + let territory = tag.territory.as_str(); + let measurement = tag.measurement.unwrap_or(if INCHES.contains(&territory) { Measurement::Us } else { Measurement::Metric }); + Region { + numbers: tag.bcp47(), + time: tag.bcp47(), + measurement, + paper: if LETTER.contains(&territory) { Paper::Letter } else { Paper::A4 }, + decimal: None, + grouping: None, + hour_cycle: tag.hour_cycle, + } + } +} + +/// The parts of a locale tag the region uses. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Tag { + /// Lower case (`pl`). + language: String, + /// Title case (`Latn`); empty when the tag has none. + script: String, + /// Upper case (`PL`, or a UN M.49 number like `419`); empty when the tag has none. + territory: String, + measurement: Option, + hour_cycle: Option, +} + +impl Default for Tag { + fn default() -> Self { + Tag { language: "en".into(), script: String::new(), territory: "US".into(), measurement: None, hour_cycle: None } + } +} + +impl Tag { + fn parse(tag: &str) -> Option { + // Locale tags are short; anything long is junk and only its start is read. + let tag: String = tag.chars().take(64).collect(); + let (tag, modifier) = tag.split_once('@').unwrap_or((&tag, "")); + let tag = tag.split('.').next().unwrap_or_default().trim(); + if tag.eq_ignore_ascii_case("c") || tag.eq_ignore_ascii_case("posix") { + return None; + } + let mut parts = tag.split(['-', '_']); + let language = parts.next().unwrap_or_default(); + if !(2..=3).contains(&language.len()) || !language.chars().all(|c| c.is_ascii_alphabetic()) { + return None; + } + let mut out = + Tag { language: language.to_ascii_lowercase(), script: String::new(), territory: String::new(), measurement: None, hour_cycle: None }; + let mut extension = None; + for p in parts.by_ref() { + // A singleton starts the extensions (`-u-…`, `-x-…`). + if p.len() == 1 { + extension = Some(p.to_ascii_lowercase()); + break; + } + if out.territory.is_empty() && out.script.is_empty() && p.len() == 4 && p.chars().all(|c| c.is_ascii_alphabetic()) { + out.script = title_case(p); + } else if out.territory.is_empty() && is_territory(p) { + out.territory = p.to_ascii_uppercase(); + } + } + let mut keywords: Vec<(String, String)> = Vec::new(); + if extension.as_deref() == Some("u") { + let mut key = String::new(); + for p in parts { + if p.len() == 1 { + break; + } + if p.len() == 2 { + key = p.to_ascii_lowercase(); + } else if !key.is_empty() { + keywords.push((std::mem::take(&mut key), p.to_ascii_lowercase())); + } + } + } + // Apple's and ICU's `@rg=plzzzz;ms=metric` keywords, and glibc's `@latin` script modifier. + for m in modifier.split(';') { + match m.split_once('=') { + Some((k, v)) => keywords.push((k.trim().to_ascii_lowercase(), v.trim().to_ascii_lowercase())), + None if m.eq_ignore_ascii_case("latin") => out.script = "Latn".into(), + None if m.eq_ignore_ascii_case("cyrillic") => out.script = "Cyrl".into(), + None => {} + } + } + for (key, value) in keywords { + match (key.as_str(), value.as_str()) { + ("ms", "metric" | "uksystem") => out.measurement = Some(Measurement::Metric), + ("ms", "ussystem") => out.measurement = Some(Measurement::Us), + ("hc", "h11" | "h12") => out.hour_cycle = Some(HourCycle::H12), + ("hc", "h23" | "h24") => out.hour_cycle = Some(HourCycle::H23), + ("rg", v) if v.len() == 6 && v.ends_with("zzzz") && v.get(..2).is_some_and(is_territory) => { + out.territory = v.get(..2).unwrap_or_default().to_ascii_uppercase(); + } + _ => {} + } + } + if out.territory.is_empty() && out.language == "en" { + out.territory = "US".into(); + } + Some(out) + } + + fn bcp47(&self) -> String { + [self.language.as_str(), &self.script, &self.territory].into_iter().filter(|p| !p.is_empty()).collect::>().join("-") + } +} + +fn is_territory(p: &str) -> bool { + (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit())) +} + +fn title_case(p: &str) -> String { + let lower = p.to_ascii_lowercase(); + let mut chars = lower.chars(); + chars.next().map(|c| c.to_ascii_uppercase().to_string() + chars.as_str()).unwrap_or_default() +} + +/// Whether a locale variable names the C locale (`C`, `C.UTF-8`, `POSIX`). +#[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))] +fn is_c_locale(v: &str) -> bool { + let v = v.trim(); + v.eq_ignore_ascii_case("c") || v.eq_ignore_ascii_case("posix") || v.starts_with("C.") +} + +/// The region from the POSIX locale variables `var` reads. An empty variable is unset; `C` in +/// `LC_ALL` or an `LC_*` variable is the C locale, written as `en-US`. `LANG` set to `C` or unset +/// gives way to the platform's preferred locale. +#[cfg(any(test, all(unix, not(any(target_vendor = "apple", target_os = "android")))))] +fn from_posix(var: impl Fn(&str) -> Option, platform: impl FnOnce() -> Option) -> Region { + let var = |name: &str| var(name).filter(|v| !v.trim().is_empty()); + if let Some(all) = var("LC_ALL") { + return Region::from_tag(&all); + } + let lang = var("LANG").filter(|v| !is_c_locale(v)).or_else(platform).unwrap_or_default(); + let tag = |name: &str| var(name).unwrap_or_else(|| lang.clone()); + let time = Region::from_tag(&tag("LC_TIME")); + Region { + time: time.time, + hour_cycle: time.hour_cycle, + measurement: Region::from_tag(&tag("LC_MEASUREMENT")).measurement, + paper: Region::from_tag(&tag("LC_PAPER")).paper, + ..Region::from_tag(&tag("LC_NUMERIC")) + } +} + +/// The region from the values under `HKEY_CURRENT_USER\Control Panel\International` that `value` +/// reads, where Windows keeps the regional format and the user's changes to it. `LocaleName` is +/// the format (`pl-PL`), else the platform's preferred locale. +#[cfg(any(test, windows))] +fn from_windows(value: impl Fn(&str) -> Option, platform: impl FnOnce() -> Option) -> Region { + let mut region = Region::from_tag(&value("LocaleName").or_else(platform).unwrap_or_default()); + region.decimal = value("sDecimal").as_deref().and_then(single_char); + region.grouping = value("sThousand").as_deref().and_then(grouping); + match value("iMeasure").as_deref().map(str::trim) { + Some("0") => region.measurement = Measurement::Metric, + Some("1") => region.measurement = Measurement::Us, + _ => {} + } + // DMPAPER_LETTER and DMPAPER_A4; other sizes keep the territory's. + match value("iPaperSize").as_deref().map(str::trim) { + Some("1") => region.paper = Paper::Letter, + Some("9") => region.paper = Paper::A4, + _ => {} + } + region.hour_cycle = value("sShortTime").as_deref().and_then(hour_cycle_of).or(region.hour_cycle); + region +} + +/// The region from Apple's current locale: its identifier (`en_PL`, `en_GB@rg=uszzzz`), its +/// metric setting and separators, and the hour format its `j` skeleton gives (`HH`, `h a`). +#[cfg(any(test, target_vendor = "apple"))] +fn from_apple(identifier: &str, metric: bool, decimal: &str, grouping: &str, hours: Option<&str>) -> Region { + let mut region = Region::from_tag(identifier); + region.measurement = if metric { Measurement::Metric } else { Measurement::Us }; + region.decimal = single_char(decimal); + region.grouping = self::grouping(grouping); + region.hour_cycle = hours.and_then(hour_cycle_of).or(region.hour_cycle); + region +} + +/// A grouping separator the system sets: none at all when empty (Windows' `sThousand` the user +/// cleared), else one character. +#[cfg(any(test, windows, target_vendor = "apple"))] +fn grouping(s: &str) -> Option> { + if s.is_empty() { Some(None) } else { single_char(s).map(Some) } +} + +/// A separator that is one character; Windows allows up to three, which the region can't carry. +#[cfg(any(test, windows, target_vendor = "apple"))] +fn single_char(s: &str) -> Option { + let mut chars = s.chars(); + let c = chars.next()?; + chars.next().is_none().then_some(c) +} + +/// The clock a time pattern (`HH:mm`, `h:mm tt`, `h 'Uhr'`) uses; quoted literals are skipped. +#[cfg(any(test, windows, target_vendor = "apple"))] +fn hour_cycle_of(pattern: &str) -> Option { + let mut quoted = false; + for c in pattern.chars() { + match c { + '\'' => quoted = !quoted, + 'H' | 'k' if !quoted => return Some(HourCycle::H23), + 'h' | 'K' if !quoted => return Some(HourCycle::H12), + _ => {} + } + } + None +} + +#[cfg(windows)] +fn read_system() -> Region { + let key = windows_registry::CURRENT_USER.open(r"Control Panel\International").ok(); + from_windows(|name| key.as_ref()?.get_string(name).ok(), sys_locale::get_locale) +} + +#[cfg(target_vendor = "apple")] +fn read_system() -> Region { + use objc2_foundation::{NSDateFormatter, NSLocale, NSString}; + let locale = NSLocale::currentLocale(); + let hours = NSDateFormatter::dateFormatFromTemplate_options_locale(&NSString::from_str("j"), 0, Some(&locale)).map(|f| f.to_string()); + from_apple( + &locale.localeIdentifier().to_string(), + locale.usesMetricSystem(), + &locale.decimalSeparator().to_string(), + &locale.groupingSeparator().to_string(), + hours.as_deref(), + ) +} + +#[cfg(all(unix, not(any(target_vendor = "apple", target_os = "android"))))] +fn read_system() -> Region { + from_posix(|name| std::env::var(name).ok(), sys_locale::get_locale) +} + +#[cfg(not(any(windows, unix)))] +fn read_system() -> Region { + Region::from_tag(&sys_locale::get_locale().unwrap_or_default()) +} + +#[cfg(target_os = "android")] +fn read_system() -> Region { + Region::from_tag(&sys_locale::get_locale().unwrap_or_default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn conventions(r: &Region) -> (Paper, Measurement) { + (r.paper, r.measurement) + } + + #[test] + fn reads_locale_tags() { + assert_eq!(Region::from_tag("pl-PL").numbers, "pl-PL"); + assert_eq!(Region::from_tag("de_AT.UTF-8@euro").numbers, "de-AT"); + assert_eq!(Region::from_tag("zh-hans-cn").numbers, "zh-Hans-CN"); + assert_eq!(Region::from_tag("sr_RS@latin").numbers, "sr-Latn-RS"); + assert_eq!(Region::from_tag("es-419").numbers, "es-419"); + assert_eq!(Region::from_tag("fr").numbers, "fr"); + assert_eq!(Region::from_tag("en").numbers, "en-US", "bare English is the US"); + for junk in ["", "C", "POSIX", "C.UTF-8", "1234", "x", "--", "ünïcode-ÄÄ", &"a".repeat(10_000)] { + assert_eq!(Region::from_tag(junk), Region::default(), "{junk:?}"); + } + } + + #[test] + fn unicode_extensions_and_apple_keywords() { + let r = Region::from_tag("en-US-u-ms-metric-hc-h23"); + assert_eq!((r.numbers.as_str(), r.measurement, r.hour_cycle), ("en-US", Measurement::Metric, Some(HourCycle::H23))); + assert_eq!(Region::from_tag("en-GB-u-hc-h12").hour_cycle, Some(HourCycle::H12)); + assert_eq!(Region::from_tag("de-DE-x-ms-ussystem").measurement, Measurement::Metric, "private use isn't read"); + let r = Region::from_tag("en_GB@rg=uszzzz"); + assert_eq!((r.numbers.as_str(), r.paper, r.measurement), ("en-US", Paper::Letter, Measurement::Us)); + assert_eq!(Region::from_tag("en_US@rg=plzzzz;ms=metric").numbers, "en-PL"); + } + + /// Paper and measurement follow the territory; separators are left to the number locale. + #[test] + fn conventions_follow_the_territory() { + assert_eq!(conventions(&Region::from_tag("pl-PL")), (Paper::A4, Measurement::Metric)); + assert_eq!(conventions(&Region::from_tag("en-US")), (Paper::Letter, Measurement::Us)); + assert_eq!(conventions(&Region::from_tag("fr-CA")), (Paper::Letter, Measurement::Metric)); + assert_eq!(conventions(&Region::from_tag("en-GB")), (Paper::A4, Measurement::Metric)); + let r = Region::from_tag("de-CH"); + assert_eq!((r.decimal, r.grouping), (None, None)); + assert_eq!(Region::from_tag("es-MX").paper, Paper::Letter); + assert_eq!(Region::from_tag("ja-JP").paper, Paper::A4); + } + + #[test] + fn a_saved_setting_overrides_the_system() { + assert_eq!(Region::from_setting(" pl-PL "), Region::from_tag("pl-PL")); + assert_eq!(Region::from_setting("AUTO"), Region::system()); + assert_eq!(Region::from_setting(""), Region::system()); + } + + #[test] + fn posix_variables_set_each_convention() { + let env = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string()); + let none = || None; + let r = from_posix(env(&[("LANG", "en_US.UTF-8"), ("LC_NUMERIC", "pl_PL.UTF-8"), ("LC_TIME", "en_GB.UTF-8"), ("LC_PAPER", "de_DE")]), none); + assert_eq!((r.numbers.as_str(), r.time.as_str(), r.decimal), ("pl-PL", "en-GB", None)); + assert_eq!((r.paper, r.measurement), (Paper::A4, Measurement::Us), "paper from LC_PAPER, measurement from LANG"); + assert_eq!(from_posix(env(&[("LANG", "it_IT.UTF-8"), ("LC_ALL", "en_US"), ("LC_PAPER", "de_DE")]), none).paper, Paper::Letter); + assert_eq!(from_posix(env(&[("LANG", "en_US"), ("LC_MEASUREMENT", "it_IT")]), none).measurement, Measurement::Metric); + assert_eq!(from_posix(env(&[("LANG", "C.UTF-8"), ("LC_ALL", "")]), || Some("it-IT".into())).numbers, "it-IT"); + assert_eq!(from_posix(env(&[]), none), Region::default()); + // `C` in an LC_* variable is the C locale's conventions, not an unset variable. + let r = from_posix(env(&[("LANG", "de_DE.UTF-8"), ("LC_NUMERIC", "C"), ("LC_TIME", "C.UTF-8")]), none); + assert_eq!((r.numbers.as_str(), r.time.as_str(), r.paper), ("en-US", "en-US", Paper::A4)); + assert_eq!(from_posix(env(&[("LANG", "de_DE.UTF-8"), ("LC_ALL", "C")]), none), Region::default()); + assert_eq!(from_posix(env(&[("LANG", "C"), ("LC_NUMERIC", "")]), || Some("it-IT".into())).numbers, "it-IT"); + } + + #[test] + fn windows_regional_format_and_its_changes() { + let reg = |pairs: &'static [(&'static str, &'static str)]| move |n: &str| pairs.iter().find(|(k, _)| *k == n).map(|(_, v)| v.to_string()); + let r = from_windows( + reg(&[("LocaleName", "pl-PL"), ("sDecimal", ","), ("sThousand", "\u{a0}"), ("iMeasure", "0"), ("sShortTime", "HH:mm")]), + || None, + ); + assert_eq!((r.numbers.as_str(), r.decimal, r.grouping, r.hour_cycle), ("pl-PL", Some(','), Some(Some('\u{a0}')), Some(HourCycle::H23))); + // An English format in the US with the user's metric and A4 choices and a decimal comma. + let r = from_windows( + reg(&[("LocaleName", "en-US"), ("sDecimal", ","), ("iMeasure", "0"), ("iPaperSize", "9"), ("sShortTime", "h:mm tt")]), + || None, + ); + assert_eq!( + (r.decimal, r.grouping, r.measurement, r.paper, r.hour_cycle), + (Some(','), None, Measurement::Metric, Paper::A4, Some(HourCycle::H12)) + ); + let r = from_windows(reg(&[("sDecimal", "dec"), ("sThousand", "")]), || Some("de-DE".into())); + assert_eq!((r.numbers.as_str(), r.decimal), ("de-DE", None), "a separator longer than one character keeps the locale's"); + assert_eq!(r.grouping, Some(None), "an empty grouping separator is no grouping"); + assert_eq!(from_windows(reg(&[("sThousand", "abc")]), || None).grouping, None); + } + + #[test] + fn apple_locale_values() { + let r = from_apple("en_PL", true, ",", "\u{a0}", Some("HH")); + assert_eq!( + (r.numbers.as_str(), r.paper, r.decimal, r.grouping, r.hour_cycle), + ("en-PL", Paper::A4, Some(','), Some(Some('\u{a0}')), Some(HourCycle::H23)) + ); + let r = from_apple("en_US", false, ".", ",", Some("h a")); + assert_eq!((r.measurement, r.hour_cycle), (Measurement::Us, Some(HourCycle::H12))); + assert_eq!(from_apple("de_DE", true, ",", ".", Some("H 'Uhr'")).hour_cycle, Some(HourCycle::H23)); + assert_eq!(hour_cycle_of("'h' mm"), None, "quoted letters are text"); + } +} diff --git a/crates/i18n/src/size.rs b/crates/i18n/src/size.rs new file mode 100644 index 00000000..5b5f0a0a --- /dev/null +++ b/crates/i18n/src/size.rs @@ -0,0 +1,256 @@ +//! File sizes and other measures with a unit, in the interface language. +//! +//! Unit names are catalog rows in context `unit`, keyed by the English template with the +//! number as `{n}` (`{n} MB`, `{n} px`, `{n}%`), so a translation can change the symbol and its +//! spacing (`{n} Mo`, `{n} %`). They are looked up with [`I18n::get`], never the plain row: a +//! plain `B` row means bold. A byte count in words is the `@plural` row `{n} byte|{n} bytes`. + +use crate::{I18n, Lang, fmt}; + +/// The context unit rows are in. +const CONTEXT: &str = "unit"; + +/// A file-size unit. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SizeUnit { + Byte, + Kilo, + Mega, + Giga, + Tera, +} + +impl SizeUnit { + const ALL: [SizeUnit; 5] = [SizeUnit::Byte, SizeUnit::Kilo, SizeUnit::Mega, SizeUnit::Giga, SizeUnit::Tera]; +} + +/// How [`I18n::file_size`] writes a byte count: the step between units, their names, and the +/// digits after the decimal point. Start from [`FileSize::BINARY`], [`FileSize::IEC`] or +/// [`FileSize::DECIMAL`] and adjust: +/// +/// ``` +/// use craft_i18n::{FileSize, SizeUnit}; +/// // `12 MB`, `3.4 MB`, `640 KB`: decimal steps, whole kilobytes, no decimals from 10 MB. +/// let style = FileSize::DECIMAL.smallest(SizeUnit::Kilo).largest(SizeUnit::Mega).unit_decimals(SizeUnit::Kilo, 0).whole_from(10); +/// ``` +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct FileSize { + step: u64, + iec: bool, + /// Per unit, in [`SizeUnit::ALL`] order; bytes are always whole. + decimals: [u8; 5], + whole_from: u32, + smallest: SizeUnit, + largest: SizeUnit, + byte_symbol: bool, +} + +impl FileSize { + /// Steps of 1024 named KB, MB, GB and TB, as Windows Explorer shows sizes; one decimal, bytes + /// in words. + pub const BINARY: FileSize = FileSize { + step: 1024, + iec: false, + decimals: [0, 1, 1, 1, 1], + whole_from: 0, + smallest: SizeUnit::Byte, + largest: SizeUnit::Tera, + byte_symbol: false, + }; + /// Steps of 1024 named KiB, MiB, GiB and TiB. + pub const IEC: FileSize = FileSize { iec: true, ..FileSize::BINARY }; + /// Steps of 1000 named KB, MB, GB and TB, as macOS shows sizes and as download and disk sizes + /// are sold. + pub const DECIMAL: FileSize = FileSize { step: 1000, ..FileSize::BINARY }; + + /// `n` digits after the decimal point in every unit above bytes. + pub const fn decimals(mut self, n: u8) -> FileSize { + self.decimals = [0, n, n, n, n]; + self + } + + /// `n` digits after the decimal point in `unit` (`0` for whole kilobytes, `2` for gigabytes). + pub const fn unit_decimals(mut self, unit: SizeUnit, n: u8) -> FileSize { + if !matches!(unit, SizeUnit::Byte) { + self.decimals[unit as usize] = n; + } + self + } + + /// No decimals from `n` of a unit up: `whole_from(10)` writes `9.5 MB` and `12 MB`. `0`, the + /// default, keeps them. + pub const fn whole_from(mut self, n: u32) -> FileSize { + self.whole_from = n; + self + } + + /// The smallest unit shown. Sizes below it are written in it, rounded up, so a file that + /// isn't empty never reads `0 KB`. + pub const fn smallest(mut self, unit: SizeUnit) -> FileSize { + self.smallest = unit; + self + } + + /// The largest unit shown; larger sizes are counted in it (`2048 GB`). + pub const fn largest(mut self, unit: SizeUnit) -> FileSize { + self.largest = unit; + self + } + + /// Bytes as the symbol `B` (`512 B`) instead of words (`512 bytes`). + pub const fn byte_symbol(mut self) -> FileSize { + self.byte_symbol = true; + self + } + + /// The unit `bytes` is written in, the number in it and its digits after the point. + fn parts(&self, bytes: u64) -> (SizeUnit, f64, usize) { + let (smallest, largest) = (self.smallest.min(self.largest), self.largest); + let mut unit = smallest; + for &u in SizeUnit::ALL.iter().filter(|u| (smallest..=largest).contains(u)) { + if bytes as f64 >= self.scale(u) { + unit = u; + } + } + loop { + let value = bytes as f64 / self.scale(unit); + let round = |decimals: u8| { + let factor = 10f64.powi(i32::from(decimals)); + (if unit == smallest && bytes > 0 && value < 1.0 { (value * factor).ceil() } else { (value * factor).round() }) / factor + }; + let mut decimals = if unit == SizeUnit::Byte { 0 } else { self.decimals.get(unit as usize).copied().unwrap_or(0) }; + let mut rounded = round(decimals); + // After rounding, so 9.96 MB, which rounds to 10.0, is written 10 MB. + if self.whole_from > 0 && rounded >= f64::from(self.whole_from) { + decimals = 0; + rounded = round(0); + } + // 1023.96 KB rounds to 1024.0 KB: the next unit writes it as 1.0 MB. + if rounded >= self.step as f64 + && let Some(&next) = SizeUnit::ALL.get(unit as usize + 1) + && next <= largest + { + unit = next; + continue; + } + return (unit, rounded, usize::from(decimals)); + } + } + + fn scale(&self, unit: SizeUnit) -> f64 { + (self.step as f64).powi(unit as i32) + } + + /// The English template for `unit`, which is also its catalog key. + fn template(&self, unit: SizeUnit) -> &'static str { + match (unit, self.iec) { + (SizeUnit::Byte, _) => "{n} B", + (SizeUnit::Kilo, false) => "{n} KB", + (SizeUnit::Mega, false) => "{n} MB", + (SizeUnit::Giga, false) => "{n} GB", + (SizeUnit::Tera, false) => "{n} TB", + (SizeUnit::Kilo, true) => "{n} KiB", + (SizeUnit::Mega, true) => "{n} MiB", + (SizeUnit::Giga, true) => "{n} GiB", + (SizeUnit::Tera, true) => "{n} TiB", + } + } +} + +impl I18n { + /// `bytes` as a file size in `lang`, written the way `style` says (`1.5 MB`, `512 bytes`), the + /// number as [`I18n::format`] writes it (`1,5 Mo`). + pub fn file_size(&self, lang: Lang, bytes: u64, style: FileSize) -> String { + let (unit, n, decimals) = style.parts(bytes); + if unit == SizeUnit::Byte && !style.byte_symbol { + return self.trn(lang, bytes, "{n} byte", "{n} bytes"); + } + self.unit_text(lang, style.template(unit), &self.format().number(n, decimals)) + } + + /// `value` with `decimals` digits after the decimal separator in the unit template `unit`, an + /// English catalog key with the number as `{n}` (`{n} px`, `{n} dB`), the number as + /// [`I18n::format`] writes it. Percentages are [`crate::Format::percent`]. + pub fn measure(&self, lang: Lang, value: f64, decimals: usize, unit: &str) -> String { + self.unit_text(lang, unit, &self.format().number(value, decimals)) + } + + fn unit_text(&self, lang: Lang, template: &str, n: &str) -> String { + fmt(self.get(lang, CONTEXT, template).unwrap_or(template), &[("n", n)]) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Embedded, Format}; + + static E: &Embedded = &[( + "fr", + "unit\t{n} KB\t{n} Ko\nunit\t{n} MB\t{n} Mo\nunit\t{n} B\t{n} o\nunit\t{n}%\t{n} %\n\tB\tG\n@plural\t{n} byte|{n} bytes\t{n} octet|{n} octets\n", + "", + )]; + + fn en(bytes: u64, style: FileSize) -> String { + let (i18n, _) = I18n::new(&[], &[]); + i18n.set_format(Format::from_tag("en-US")); + i18n.file_size(Lang::EN, bytes, style) + } + + #[test] + fn binary_decimal_and_iec_steps() { + assert_eq!(en(512, FileSize::BINARY), "512 bytes"); + assert_eq!(en(1023, FileSize::BINARY), "1,023 bytes"); + assert_eq!(en(1, FileSize::BINARY), "1 byte"); + assert_eq!(en(1536, FileSize::BINARY), "1.5 KB"); + assert_eq!(en(1536, FileSize::IEC), "1.5 KiB"); + assert_eq!(en(1500, FileSize::DECIMAL), "1.5 KB"); + assert_eq!(en(3 << 30, FileSize::BINARY), "3.0 GB"); + assert_eq!(en(5 << 40, FileSize::IEC.decimals(0)), "5 TiB"); + assert_eq!(en(512, FileSize::BINARY.byte_symbol()), "512 B"); + } + + #[test] + fn precision_and_unit_range() { + // Rounding up to the next unit's step moves to that unit. + assert_eq!(en(1_048_575, FileSize::BINARY), "1.0 MB"); + assert_eq!(en(999_999, FileSize::DECIMAL.decimals(0)), "1 MB"); + assert_eq!(en(1_610_612_736, FileSize::BINARY.unit_decimals(SizeUnit::Giga, 2)), "1.50 GB"); + // Lightcraft's denoise sizes: whole kilobytes, at least 1, no decimals from 10 MB. + let style = FileSize::DECIMAL.smallest(SizeUnit::Kilo).largest(SizeUnit::Mega).unit_decimals(SizeUnit::Kilo, 0).whole_from(10); + assert_eq!(en(1, style), "1 KB"); + assert_eq!(en(0, style), "0 KB"); + assert_eq!(en(640_400, style), "640 KB"); + assert_eq!(en(3_400_000, style), "3.4 MB"); + assert_eq!(en(12_600_000, style), "13 MB"); + assert_eq!(en(9_960_000, style), "10 MB", "whole once rounded to 10"); + assert_eq!(en(9_940_000, style), "9.9 MB"); + assert_eq!(en(5_000_000_000, style), "5,000 MB"); + // Only sizes below the smallest unit round up. + assert_eq!(en(1025, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); + assert_eq!(en(100, FileSize::BINARY.smallest(SizeUnit::Kilo).unit_decimals(SizeUnit::Kilo, 0)), "1 KB"); + } + + #[test] + fn unit_names_are_catalog_rows() { + let (i18n, problems) = I18n::new(E, &["fr"]); + assert!(problems.is_empty(), "{problems:?}"); + let fr = i18n.from_code("fr").unwrap(); + i18n.set_format(Format::from_tag("fr-FR")); + assert_eq!(i18n.file_size(fr, 1536, FileSize::BINARY), "1,5 Ko"); + assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY), "512 octets"); + assert_eq!(i18n.file_size(fr, 512, FileSize::BINARY.byte_symbol()), "512 o", "not the plain row B (bold)"); + assert_eq!(i18n.file_size(fr, 3 << 30, FileSize::BINARY), "3,0 GB", "a missing row is English"); + assert_eq!(i18n.measure(fr, 50.0, 0, "{n}%"), "50 %"); + assert_eq!(i18n.measure(Lang::EN, 12.34, 1, "{n} px"), "12,3 px", "numbers follow the region, not the language"); + } + + #[test] + fn locale_digits() { + let (i18n, _) = I18n::new(&[], &[]); + i18n.set_format(Format::from_tag("ar-EG")); + assert_eq!(i18n.file_size(Lang::EN, 1536, FileSize::BINARY), "١٫٥ KB"); + i18n.set_format(Format::from_tag("hi")); + assert_eq!(i18n.file_size(Lang::EN, 1_234_567, FileSize::DECIMAL.largest(SizeUnit::Byte).byte_symbol()), "12,34,567 B"); + } +}