diff --git a/crates/i18n/README.md b/crates/i18n/README.md index df337971..a34ce183 100644 --- a/crates/i18n/README.md +++ b/crates/i18n/README.md @@ -49,6 +49,13 @@ The app keeps its `tl!` macro, its language setting (`"auto"` → `I18N.system_l (one context along the locale chain, so `pt-br` can return a `pt` row, but never the plain row or English: a key cap's name under `key`, or "is there a translation?"), and `fmt` for `{name}` placeholders, filled in one pass. +- **Isolation:** `fmt` wraps each inserted value in a first-strong isolate (FSI…PDI), and `trn` + wraps `{n}` in a left-to-right one (LRI…PDI), so a Latin file name in an Arabic sentence, or a + Hebrew name in an English one, can't reorder the words around it. Wrap numbers, paths and codes + in `ltr` before passing them; a value that already is one isolate goes in as it is, and a stray + PDI in a value can't close its isolate early. egui draws the marks as nothing. For text that + leaves the interface (file names, logs, terminals, text parsed later), use `fmt_plain` or + `strip_isolates`. - **Bad rows** (malformed, an unknown `@` context, a wrong plural form count, a row defined twice, of which the first is kept) and bad locales (`en`, `pseudo`, a code embedded twice in any case) are skipped and returned by `I18n::new` for the app to log; craft-i18n's CI rejects them before diff --git a/crates/i18n/src/lib.rs b/crates/i18n/src/lib.rs index 7fa7a4ef..a394aef5 100644 --- a/crates/i18n/src/lib.rs +++ b/crates/i18n/src/lib.rs @@ -158,8 +158,9 @@ impl I18n { self.find(lang, &[("@id", id), ("", label)]).unwrap_or(label) } - /// A count's wording in `lang`, `{n}` filled in. `one` and `other` are the English forms; - /// they are also the key of the `@plural` row, and the fallback. + /// A count's wording in `lang`, `{n}` filled in as a left-to-right isolate ([`ltr`]). `one` + /// and `other` are the English forms; they are also the key of the `@plural` row, and the + /// fallback. pub fn trn(&self, lang: Lang, n: u64, one: &str, other: &str) -> String { let key = format!("{one}|{other}"); let chain = self.locale(lang).map_or(&[][..], |l| l.chain.as_slice()); @@ -175,7 +176,7 @@ impl I18n { forms.split('|').nth(index) }) .unwrap_or(if n == 1 { one } else { other }); - let text = fmt(text, &[("n", &n.to_string())]); + let text = fmt(text, &[("n", <r(&n.to_string()))]); if lang == Lang::PSEUDO { return marked(&text); } @@ -261,9 +262,88 @@ impl I18n { } } -/// Fill `{name}` placeholders from `args` in one pass. Unknown placeholders stay as written, and -/// inserted values are never read as templates, so a file name containing `{n}` stays intact. +/// Fill `{name}` placeholders from `args` in one pass, each value wrapped in a first-strong +/// isolate (FSI…PDI), so its direction can't reorder the sentence around it: a Latin file name +/// in an Arabic message stays where the translator put it. A value that already is one isolate +/// goes in as it is; pass numbers, paths and codes through [`ltr`]. Unknown placeholders stay as +/// written, and inserted values are never read as templates, so a file name containing `{n}` +/// stays intact. pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { + fill(template, args, true) +} + +/// [`fmt`] without isolates, for text that goes where the marks would get in the way: file +/// names, logs, terminals, text compared or parsed later. +pub fn fmt_plain(template: &str, args: &[(&str, &str)]) -> String { + fill(template, args, false) +} + +const LRI: char = '\u{2066}'; +const RLI: char = '\u{2067}'; +const FSI: char = '\u{2068}'; +const PDI: char = '\u{2069}'; + +/// `value` as a first-strong isolate (FSI…PDI): it takes the direction of its first strong +/// letter and can't reorder the text around it. What [`fmt`] does to each value. +pub fn isolate(value: &str) -> String { + wrap(FSI, value) +} + +/// `value` as a left-to-right isolate (LRI…PDI), for numbers, paths, file sizes, timecodes and +/// codes, which read left to right in every language. In an RTL first-strong isolate, the +/// segments of `/home/مستخدم/file.txt` would run right to left. +pub fn ltr(value: &str) -> String { + wrap(LRI, value) +} + +/// `text` without isolate marks (LRI, RLI, FSI, PDI): a [`fmt`] or [`I18n::trn`] result made fit +/// for a file name, a log or a comparison. +pub fn strip_isolates(text: &str) -> String { + text.chars().filter(|c| !matches!(*c, LRI | RLI | FSI | PDI)).collect() +} + +/// `value` between `initiator` and PDI. A stray PDI in the value would close the isolate early, so +/// it is dropped; isolates the value leaves open are closed. +fn wrap(initiator: char, value: &str) -> String { + let mut out = String::with_capacity(value.len() + 6); + out.push(initiator); + let mut open = 0_usize; + for c in value.chars() { + match c { + LRI | RLI | FSI => open += 1, + PDI if open == 0 => continue, + PDI => open -= 1, + _ => {} + } + out.push(c); + } + for _ in 0..=open { + out.push(PDI); + } + out +} + +/// Whether `value` is exactly one isolate: an initiator, then the PDI that closes it last. +fn is_one_isolate(value: &str) -> bool { + let mut depth = 0_usize; + let mut chars = value.chars().peekable(); + while let Some(c) = chars.next() { + match c { + LRI | RLI | FSI => depth += 1, + PDI if depth > 0 => { + depth -= 1; + if depth == 0 { + return chars.peek().is_none(); + } + } + _ if depth == 0 => return false, + _ => {} + } + } + false +} + +fn fill(template: &str, args: &[(&str, &str)], isolated: bool) -> String { let mut out = String::with_capacity(template.len()); let mut rest = template; while let Some((before, after_open)) = rest.split_once('{') { @@ -274,7 +354,8 @@ pub fn fmt(template: &str, args: &[(&str, &str)]) -> String { break; }; match args.iter().find(|(k, _)| *k == name) { - Some((_, value)) => out.push_str(value), + Some((_, value)) if !isolated || value.is_empty() || is_one_isolate(value) => out.push_str(value), + Some((_, value)) => out.push_str(&isolate(value)), None => { out.push('{'); out.push_str(name); @@ -485,17 +566,42 @@ mod tests { #[test] fn fmt_fills_once() { - assert_eq!(fmt("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); + assert_eq!(fmt_plain("{a} to {b}", &[("a", "x"), ("b", "{a}")]), "x to {a}"); assert_eq!(fmt("{z} {", &[]), "{z} {"); } + #[test] + fn inserted_values_are_isolated() { + let saved = fmt("تم حفظ {name} في {dir}.", &[("name", "Report 2.pdf"), ("dir", <r("/home/سارة/docs"))]); + assert_eq!(saved, "تم حفظ \u{2068}Report 2.pdf\u{2069} في \u{2066}/home/سارة/docs\u{2069}."); + assert_eq!(strip_isolates(&saved), "تم حفظ Report 2.pdf في /home/سارة/docs."); + assert_eq!(fmt_plain("{a}", &[("a", "x")]), "x"); + assert_eq!(fmt("[{a}]", &[("a", "")]), "[]"); + } + + #[test] + fn a_value_cannot_escape_its_isolate() { + assert_eq!(isolate("a\u{2069}b"), "\u{2068}ab\u{2069}"); + assert_eq!(isolate("a\u{2067}b"), "\u{2068}a\u{2067}b\u{2069}\u{2069}"); + let one = ltr("12"); + assert_eq!(fmt("{n}", &[("n", &one)]), one); + // Two isolates side by side are not one, so they get wrapped. + assert_eq!(fmt("{v}", &[("v", "\u{2066}a\u{2069}\u{2066}b\u{2069}")]), "\u{2068}\u{2066}a\u{2069}\u{2066}b\u{2069}\u{2069}"); + } + + #[test] + fn counts_are_left_to_right_isolates() { + let i = all(P).0; + assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} page", "{n} pages"), "\u{2066}2\u{2069} pages"); + } + #[test] fn one_other() { let (i, problems) = all(P); assert!(problems.is_empty(), "{problems:?}"); let fr = lang(&i, "fr"); - assert_eq!(i.trn(fr, 0, "{n} page", "{n} pages"), "0 page"); - assert_eq!(i.trn(fr, 2, "{n} page", "{n} pages"), "2 pages"); + assert_eq!(strip_isolates(&i.trn(fr, 0, "{n} page", "{n} pages")), "0 page"); + assert_eq!(strip_isolates(&i.trn(fr, 2, "{n} page", "{n} pages")), "2 pages"); } #[test] @@ -503,27 +609,27 @@ mod tests { let i = all(P).0; let ru = lang(&i, "ru"); for (n, want) in [(1, "1 файл"), (3, "3 файла"), (5, "5 файлов"), (11, "11 файлов"), (21, "21 файл")] { - assert_eq!(i.trn(ru, n, "{n} file", "{n} files"), want); + assert_eq!(strip_isolates(&i.trn(ru, n, "{n} file", "{n} files")), want); } } #[test] fn one_form_ja() { let i = all(P).0; - assert_eq!(i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files"), "7 個"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "ja"), 7, "{n} file", "{n} files")), "7 個"); } #[test] fn english_fallback() { let i = all(P).0; - assert_eq!(i.trn(Lang::EN, 1, "{n} file", "{n} files"), "1 file"); - assert_eq!(i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs"), "2 xs"); + assert_eq!(strip_isolates(&i.trn(Lang::EN, 1, "{n} file", "{n} files")), "1 file"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "fr"), 2, "{n} x", "{n} xs")), "2 xs"); } #[test] fn beyond_sweep() { let i = all(P).0; - assert_eq!(i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages"), "1000000 pages"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "fr"), 1_000_000, "{n} page", "{n} pages")), "1000000 pages"); } static L: &Embedded = @@ -590,7 +696,7 @@ mod tests { assert_eq!(i.t(Lang::PSEUDO, "A\nB"), "⟦A⟧\n⟦B⟧"); assert_eq!(i.tc(Lang::PSEUDO, "menu", "Open"), "⟦Open⟧"); assert_eq!(i.tid(Lang::PSEUDO, "file.save", "Save"), "⟦Save⟧"); - assert_eq!(i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs"), "⟦2 xs⟧"); + assert_eq!(strip_isolates(&i.trn(Lang::PSEUDO, 2, "{n} x", "{n} xs")), "⟦2 xs⟧"); assert!(!i.languages().contains(&Lang::PSEUDO)); assert_eq!(i.from_code("pseudo"), cfg!(debug_assertions).then_some(Lang::PSEUDO)); assert_eq!(i.get(Lang::PSEUDO, "key", "Esc"), Some("⟦Esc⟧")); @@ -669,9 +775,9 @@ mod tests { #[test] fn european_portuguese_zero_is_plural() { let i = all(PT).0; - assert_eq!(i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files"), "0 ficheiros"); - assert_eq!(i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files"), "1 ficheiro"); - assert_eq!(i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files"), "0 arquivo"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "pt"), 0, "{n} file", "{n} files")), "0 ficheiros"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "pt"), 1, "{n} file", "{n} files")), "1 ficheiro"); + assert_eq!(strip_isolates(&i.trn(lang(&i, "pt-br"), 0, "{n} file", "{n} files")), "0 arquivo"); } /// The same counts as craft-i18n's checker (`plural_form_counts`), which rejects a catalog