From b010389df25cbb0b0425b48e4c19d554e6ca8d91 Mon Sep 17 00:00:00 2001 From: aiongg Date: Wed, 21 Oct 2026 05:54:08 +0000 Subject: [PATCH] i18n: Format::parse on the format's own marks; lone decimal marks Format::parse reads what a Format writes: its digits, its decimal and grouping separators (the system's included) and the locale's Latin marks in an Arabic-Indic format, plus every digit set. parse_number is Format::from_tag(locale).parse, so the decimal marks come from CLDR's data alone and the hand-made table is gone. A lone . in a comma locale is the decimal mark when it can't be grouping (1.5, 0.250, 1234.5, .5); 1.234 and 1.234.567 stay grouped in German. The same holds for , in a point locale. Percent signs and units are documented as the caller's to strip. --- crates/i18n/README.md | 14 +- crates/i18n/src/format.rs | 14 +- crates/i18n/src/number.rs | 271 ++++++++++++++++++++++++-------------- 3 files changed, 191 insertions(+), 108 deletions(-) diff --git a/crates/i18n/README.md b/crates/i18n/README.md index d2cc7fda..e486ae02 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -65,12 +65,16 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l code; English is "English". - **Right to left:** `rtl` is true for `ar`, `ckb`, `dv`, `fa`, `he`, `ps`, `sd`, `ug`, `ur` and `yi`; putting text into display order is the app's job. -- **Reading numbers:** `parse_number(text, locale)` reads what people type or paste: any Unicode +- **Reading numbers:** `Format::parse(text)` reads what the format writes (its digits, decimal + and grouping separators, the system's included) and what people type or paste: any Unicode decimal digits (ASCII, Arabic-Indic, Persian, Devanagari, Bengali, Thai, full-width, …) from one - set; the locale's decimal mark (`.` or `,`) and `٫` everywhere; grouping by the locale's other - mark, spaces (NBSP, narrow NBSP), `'`, `’` or `٬`, in threes or the Indian 2-2-3; a leading `-`, - `−` or `+`; bidi marks ignored. Input that doesn't fit is an error, never a guess: `1,234` is - 1234 in English, 1.234 in German and an error for a locale whose decimal mark isn't known. + set; `٫` as a decimal mark everywhere; grouping by the locale's other mark of `.` and `,`, spaces + (NBSP, narrow NBSP), `'`, `’` or `٬`, in threes or the Indian 2-2-3; a leading `-`, `−` or `+`; + bidi marks ignored. A lone `.` in a comma locale is decimal when it can't be grouping (`1.5`, + `0.250`, `1234.5`), while `1.234` stays 1234 in German; the same holds for `,` in a point + locale. Percent signs and units (`50 %`, `12 px`) are not read: strip them first. Other input + that doesn't fit is an error, never a guess. `parse_number(text, locale)` is the same for + `Format::from_tag(locale)`. - **System locale:** `system_lang` matches the system's list once and caches it: on desktop Unix `LANGUAGE`, `LC_ALL`, `LC_MESSAGES` and `LANG`, read raw; elsewhere `sys-locale`'s (macOS and Windows preferred languages, the browser's on the web). `fr_CA.UTF-8` → `fr`, `zh-TW` → diff --git a/crates/i18n/src/format.rs b/crates/i18n/src/format.rs index 6def7d18..bcb20d05 100644 --- a/crates/i18n/src/format.rs +++ b/crates/i18n/src/format.rs @@ -11,6 +11,7 @@ use writeable::{Part, PartsWrite, Writeable}; use crate::Region; use crate::data::{self, PERCENT}; +use crate::number::Marks; /// Which digits numbers are written in. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] @@ -33,6 +34,8 @@ pub struct Format { group: Option, point: Option, percent: (&'static str, &'static str), + /// The marks [`Format::parse`] reads. + pub(crate) marks: Marks, } impl Format { @@ -44,7 +47,7 @@ impl Format { locale.extensions.unicode.keywords.set(key!("nu"), value!("latn")); } let decimal = data::provider().and_then(|p| DecimalFormatter::try_new_with_buffer_provider(p, (&locale).into(), Default::default()).ok()); - let mut format = Format { decimal, group: None, point: None, percent: ("", "%") }; + let mut format = Format { decimal, group: None, point: None, percent: ("", "%"), marks: Marks::default() }; // The locale's own separators and digits, read from a number that shows them all. let probe = format.parts(&Decimal::try_from_str("-1234567.5").unwrap_or_default()); let latin = probe.integer.starts_with(|c: char| c.is_ascii_digit()); @@ -61,6 +64,15 @@ impl Format { } } format.percent = percent_affixes(®ion.numbers, latin); + let point = format.point.clone().unwrap_or(probe.point); + let group = format.group.clone().unwrap_or(probe.group); + // Latin digits typed into an Arabic-Indic format take the locale's Latin marks. + let comma = match (latin, digits) { + (true, _) => point == ",", + (false, Digits::Locale) => !Format::new(region, Digits::Latin).marks.point, + (false, Digits::Latin) => false, + }; + format.marks = Marks::new(&point, &group, !comma); format } diff --git a/crates/i18n/src/number.rs b/crates/i18n/src/number.rs index dfeb2e74..3de9c988 100644 --- a/crates/i18n/src/number.rs +++ b/crates/i18n/src/number.rs @@ -2,7 +2,9 @@ use std::fmt; -/// Why [`parse_number`] rejected a text. +use crate::Format; + +/// Why [`Format::parse`] rejected a text. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum NumberError { /// No digits: empty, or only spaces and a sign. @@ -11,8 +13,8 @@ pub enum NumberError { Unexpected(char), /// Digits from more than one set (`1٢`). MixedDigits, - /// A decimal or grouping mark where the locale puts none (`1,5` in English, `1.5` in German), - /// or one that could be read either way (`1,234` when the locale's decimal mark is unknown). + /// A decimal or grouping mark where the locale puts none (`1.234,5` in English, `1,2.5` in + /// German). Separator, /// Too large for an `f64`. TooLarge, @@ -32,68 +34,103 @@ impl fmt::Display for NumberError { impl std::error::Error for NumberError {} -/// The number in `text`, as someone using `locale` (a language tag, `de-CH`, `ar_EG.UTF-8`) writes -/// it. -/// -/// - Digits: any Unicode decimal digits (ASCII, Arabic-Indic, Persian, Devanagari, Bengali, Thai, -/// full-width, …), all from one set. -/// - Decimal mark: the locale's `.` or `,`; `٫` in every locale. A locale this crate doesn't know -/// takes either, when only one reading fits (`1,5`, `1.234,5`). -/// - Grouping: the locale's other mark of `.` and `,`, spaces (NBSP, narrow NBSP, thin), `'`, `’` -/// and `٬`, one kind per number, in groups of three or the Indian 2-2-3 (`12,34,567`). Never in -/// the fraction. -/// - Sign: a leading `-`, `−` (U+2212), full-width `－` or `+`. -/// - Bidi marks and isolates (LRM, RLM, ALM, LRI…PDI) around or inside are ignored, so a number -/// formatted for an RTL interface reads back. -/// -/// Anything else is an error rather than a guess: `1,234` is 1234 in English, 1.234 in German and -/// [`NumberError::Separator`] for an unknown locale. -pub fn parse_number(text: &str, locale: &str) -> Result { - let text: String = text.chars().filter(|c| !is_bidi_mark(*c)).collect(); - let mut rest = text.trim(); - let negative = match rest.chars().next() { - Some(c @ ('-' | '−' | '－' | '+' | '＋')) => { - rest = rest.get(c.len_utf8()..).unwrap_or(""); - matches!(c, '-' | '−' | '－') - } - _ => false, - }; - let mut tokens = Vec::with_capacity(rest.len()); - let mut zero = None; - for c in rest.chars() { - let token = match c { - '.' | '．' => Token::Point, - ',' | '，' => Token::Comma, - '٫' => Token::Decimal, - '٬' => Token::Group('٬'), - '\'' | '’' => Token::Group('\''), - ' ' | '\u{a0}' | '\u{202f}' | '\u{2009}' => Token::Group(' '), - _ => { - let (z, d) = digit(c).ok_or(NumberError::Unexpected(c))?; - if *zero.get_or_insert(z) != z { - return Err(NumberError::MixedDigits); - } - Token::Digit(d) - } +/// The marks a [`Format`] writes, as [`Format::parse`] reads them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Marks { + /// `.` is the decimal mark and `,` grouping; else the other way round. For a format in + /// Arabic-Indic digits, as the locale writes Latin ones. + pub(crate) point: bool, + /// The format's own decimal and grouping marks when they are neither `.` nor `,` (`٫`, `'`, a + /// mark the system sets). + decimal: Option, + group: Option, +} + +impl Default for Marks { + fn default() -> Self { + Marks { point: true, decimal: None, group: None } + } +} + +impl Marks { + pub(crate) fn new(decimal: &str, group: &str, point: bool) -> Marks { + let own = |s: &str| { + let mut chars = s.chars(); + chars.next().filter(|c| !matches!(c, '.' | ',') && chars.next().is_none()) }; - tokens.push(token); + Marks { point, decimal: own(decimal), group: own(group) } } - if zero.is_none() { - return Err(if tokens.is_empty() { NumberError::Empty } else { NumberError::Separator }); +} + +impl Format { + /// The number in `text`: what this format writes, and what people type or paste. + /// + /// - Digits: any Unicode decimal digits (ASCII, Arabic-Indic, Persian, Devanagari, Bengali, + /// Thai, full-width, …), all from one set. + /// - Decimal mark: the format's, `٫` in every format, and the locale's `.` or `,` when Latin + /// digits are typed into an Arabic-Indic format. A lone `.` in a comma locale is the decimal + /// mark when it can't be grouping (`1.5`, `0.250`, `1234.5`, `.5`), but `1.234` and + /// `1.234.567` stay grouped; the same holds for `,` in a point locale. + /// - Grouping: the format's, the other of `.` and `,`, spaces (NBSP, narrow NBSP, thin), `'`, + /// `’` and `٬`, one kind per number, in groups of three or the Indian 2-2-3 (`12,34,567`). + /// Never in the fraction. + /// - Sign: a leading `-`, `−` (U+2212), full-width `－` or `+`. + /// - Bidi marks and isolates (LRM, RLM, ALM, LRI…PDI) around or inside are ignored, so a + /// number formatted for an RTL interface reads back. + /// + /// Percent signs, units and the space before a unit (`50 %`, `12 px`) are not read; strip them + /// first. Anything else is an error rather than a guess. + pub fn parse(&self, text: &str) -> Result { + let marks = self.marks; + let text: String = text.chars().filter(|c| !is_bidi_mark(*c)).collect(); + let mut rest = text.trim(); + let negative = match rest.chars().next() { + Some(c @ ('-' | '−' | '－' | '+' | '＋')) => { + rest = rest.get(c.len_utf8()..).unwrap_or(""); + matches!(c, '-' | '−' | '－') + } + _ => false, + }; + let mut tokens = Vec::with_capacity(rest.len()); + let mut zero = None; + for c in rest.chars() { + let token = match c { + c if Some(c) == marks.decimal => Token::Decimal, + c if Some(c) == marks.group => Token::Group(c), + '.' | '．' => Token::Point, + ',' | '，' => Token::Comma, + '٫' => Token::Decimal, + '٬' => Token::Group('٬'), + '\'' | '’' => Token::Group('\''), + ' ' | '\u{a0}' | '\u{202f}' | '\u{2009}' => Token::Group(' '), + _ => { + let (z, d) = digit(c).ok_or(NumberError::Unexpected(c))?; + if *zero.get_or_insert(z) != z { + return Err(NumberError::MixedDigits); + } + Token::Digit(d) + } + }; + tokens.push(token); + } + if zero.is_none() { + return Err(if tokens.is_empty() { NumberError::Empty } else { NumberError::Separator }); + } + let point = if lone_decimal(&tokens, marks.point) { !marks.point } else { marks.point }; + let digits = read(&tokens, point).ok_or(NumberError::Separator)?; + let value: f64 = digits.parse().map_err(|_| NumberError::Separator)?; + if !value.is_finite() { + return Err(NumberError::TooLarge); + } + Ok(if negative { -value } else { value }) } - let digits = match decimal_mark(locale) { - Some(point) => read(&tokens, point), - None => match (read(&tokens, true), read(&tokens, false)) { - (Some(a), Some(b)) if a != b => None, - (a, b) => a.or(b), - }, - } - .ok_or(NumberError::Separator)?; - let value: f64 = digits.parse().map_err(|_| NumberError::Separator)?; - if !value.is_finite() { - return Err(NumberError::TooLarge); - } - Ok(if negative { -value } else { value }) +} + +/// The number in `text` as `locale` (a language tag, `de-CH`, `ar_EG.UTF-8`) writes it: +/// [`Format::parse`] for [`Format::from_tag`]. Build the [`Format`] once to read many numbers. A +/// locale CLDR doesn't know reads as CLDR's root does (`1,234.5`). +pub fn parse_number(text: &str, locale: &str) -> Result { + Format::from_tag(locale).parse(text) } #[derive(Clone, Copy, PartialEq)] @@ -101,12 +138,26 @@ enum Token { Digit(u8), Point, Comma, - /// `٫`, a decimal mark in every locale. + /// A decimal mark in every reading: `٫`, or the format's own. Decimal, - /// A grouping mark in every locale, by kind (space, apostrophe, `٬`). + /// A grouping mark in every reading, by kind (space, apostrophe, `٬`, the format's own). Group(char), } +/// Whether the locale's grouping mark of `.` and `,` (`.` when `point` is false) is the decimal +/// mark here because it can't be grouping: it is the only mark of the two, there is one of it, and +/// the digits before it are none, start with 0 or are more than three, or the digits after it +/// aren't three. +fn lone_decimal(tokens: &[Token], point: bool) -> bool { + let (group, decimal) = if point { (Token::Comma, Token::Point) } else { (Token::Point, Token::Comma) }; + if tokens.contains(&decimal) || tokens.contains(&Token::Decimal) || tokens.iter().filter(|t| **t == group).count() != 1 { + return false; + } + let Some(at) = tokens.iter().position(|t| *t == group) else { return false }; + let (int, frac) = (tokens.get(..at).unwrap_or_default(), tokens.get(at + 1..).unwrap_or_default()); + int.is_empty() || int.first() == Some(&Token::Digit(0)) || int.len() > 3 || frac.len() != 3 +} + /// The ASCII number `tokens` spell when the decimal mark is `.` (`point`) or `,`; `None` when the /// marks don't fit that reading. fn read(tokens: &[Token], point: bool) -> Option { @@ -160,36 +211,6 @@ fn read(tokens: &[Token], point: bool) -> Option { Some(format!("{}.{}", if int.is_empty() { "0" } else { &int }, if frac.is_empty() { "0" } else { &frac })) } -/// Whether `locale` writes decimals with `.` (`Some(true)`) or `,` (`Some(false)`); `None` for a -/// language not in the table (CLDR's decimal symbol for its Latin digits). -// ponytail: the languages a Craft app ships or could plausibly ship; add a row with the locale. -fn decimal_mark(locale: &str) -> Option { - let locale = locale.trim().to_ascii_lowercase(); - let base = locale.split(['.', '@']).next().unwrap_or("").replace('_', "-"); - let mut parts = base.split('-').filter(|p| !p.is_empty()); - let language = parts.next().unwrap_or(""); - let region = parts.find(|p| super::is_region(p)).unwrap_or(""); - Some(match (language, region) { - ("de" | "it", "ch" | "li") => true, - ("es", "419" | "mx" | "us" | "pr" | "do" | "gt" | "hn" | "ni" | "pa" | "pe" | "sv") => true, - ("en", "za" | "150" | "at" | "be" | "de" | "dk" | "fi" | "nl" | "se" | "si") => false, - ("ar", "dz" | "eh" | "ly" | "ma" | "tn") => false, - ( - "af" | "az" | "be" | "bg" | "bs" | "ca" | "cs" | "da" | "de" | "el" | "eo" | "es" | "et" | "eu" | "fi" | "fo" | "fr" | "fy" | "gl" | "hr" - | "hu" | "hy" | "id" | "is" | "it" | "ka" | "kk" | "km" | "ky" | "lb" | "lo" | "lt" | "lv" | "mk" | "nb" | "nl" | "nn" | "no" | "pl" - | "pt" | "ro" | "ru" | "sk" | "sl" | "sq" | "sr" | "sv" | "tg" | "tk" | "tr" | "tt" | "uk" | "uz" | "vi", - _, - ) => false, - ( - "am" | "ar" | "as" | "bn" | "bo" | "ckb" | "cy" | "dv" | "en" | "fa" | "fil" | "ga" | "gsw" | "gu" | "he" | "hi" | "ja" | "kn" | "ko" - | "ks" | "ml" | "mn" | "mr" | "ms" | "mt" | "my" | "ne" | "or" | "pa" | "ps" | "rm" | "sd" | "si" | "sw" | "ta" | "te" | "th" | "tl" - | "ug" | "ur" | "yue" | "zh" | "zu", - _, - ) => true, - _ => return None, - }) -} - /// The first code point of each run of ten Unicode decimal digits (general category Nd), Unicode /// 16.0. const ZEROS: [u32; 76] = [ @@ -219,6 +240,7 @@ fn is_bidi_mark(c: char) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::{Digits, Region}; #[test] fn every_digit_set() { @@ -269,11 +291,12 @@ mod tests { (" 42 ", "en", 42.0), ("\u{2066}-1,5\u{2069}", "de", -1.5), ("\u{061c}-١٢", "ar", -12.0), - ("1,5", "xx", 1.5), - ("1.234,5", "", 1234.5), ("1,234.5", "xx", 1234.5), - ("1.234.567", "xx", 1_234_567.0), ("12", "xx", 12.0), + ("1,5", "en-PL", 1.5), + ("1.234,5", "en-DE", 1234.5), + ("1.234,5", "lo", 1234.5), + ("1,234.5", "km", 1234.5), ] { assert_eq!(parse_number(text, locale), Ok(want), "{text:?} in {locale}"); } @@ -286,12 +309,7 @@ mod tests { (" ", "en", NumberError::Empty), ("-", "en", NumberError::Empty), (",", "en", NumberError::Separator), - ("1,234", "xx", NumberError::Separator), - ("1,5", "en", NumberError::Separator), - ("1.5", "de", NumberError::Separator), ("1.2.3", "en", NumberError::Separator), - ("12,34", "en", NumberError::Separator), - ("1234,567", "en", NumberError::Separator), ("1,,234", "en", NumberError::Separator), ("1,234,", "en", NumberError::Separator), ("1.234,5", "en", NumberError::Separator), @@ -312,5 +330,54 @@ mod tests { assert_eq!(parse_number(text, locale), Err(want), "{text:?} in {locale}"); } assert_eq!(parse_number(&"9".repeat(400), "en"), Err(NumberError::TooLarge)); + for unit in ["50 %", "50%", "12 px", "1,5 MB"] { + assert!(parse_number(unit, "de").is_err(), "{unit}: units are the caller's to strip"); + } + } + + /// A lone mark that can't be grouping is the decimal mark; one that can stays grouping. + #[test] + fn lone_marks() { + for (text, locale, want) in [ + ("1.5", "de", 1.5), + ("0.250", "de", 0.25), + ("1234.5", "de", 1234.5), + (".5", "de", 0.5), + ("-12.34", "ru", -12.34), + ("1.234", "de", 1234.0), + ("1.234.567", "de", 1_234_567.0), + ("1,5", "en", 1.5), + ("0,5", "en", 0.5), + ("12,34", "en", 12.34), + ("1234,567", "en", 1234.567), + ("1,234", "en", 1234.0), + ("12,345", "en", 12345.0), + ] { + assert_eq!(parse_number(text, locale), Ok(want), "{text:?} in {locale}"); + } + } + + /// What a format writes reads back, in its digits and with its separators, the system's + /// included. + #[test] + fn reads_what_format_writes() { + let mut formats: Vec = ["en", "de", "de-CH", "fr", "hi", "ar-EG", "fa", "bn-BD", "ar-MA", "en-PL", "nb", "pt-PT", "sv"] + .iter() + .flat_map(|t| [Format::from_tag(t), Format::new(&Region::from_tag(t), Digits::Latin)]) + .collect(); + for (decimal, grouping) in [(Some(','), None), (Some('.'), Some(Some('\u{a0}'))), (None, Some(None)), (Some('·'), Some(Some('_')))] { + let mut region = Region::from_tag("en-US"); + region.decimal = decimal; + region.grouping = grouping; + formats.push(Format::new(®ion, Digits::Locale)); + } + for f in &formats { + for value in [0.0, 1.5, -1234.25, 1_234_567.89, 12.0, 1234.0] { + let text = f.number(value, 2); + assert_eq!(f.parse(&text), Ok(value), "{text:?} ({f:?})"); + } + let text = f.integer(1_234_567); + assert_eq!(f.parse(&text), Ok(1_234_567.0), "{text:?}"); + } } }