diff --git a/crates/proof/README.md b/crates/proof/README.md index 7ebfafe8..1230cd28 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -97,7 +97,8 @@ for `Other` (use `pack_locale()`). **Which pack serves a language:** its locale, then shorter prefixes (`fr-ca` → `fr`), then another region's pack of the same language (`nl` → `nl-be`), never another script's (`sr-latn` never reads -`sr`). The existing languages map their Hunspell names the same way (`pt_BR` → `pt-br`, `pt`, any +`sr`, which is Cyrillic). A tag naming its language's default script reads the script-less packs +(`sr-Cyrl-RS` → `sr`, `bs-Latn-BA` → `bs`), from a small default-script table. The existing languages map their Hunspell names the same way (`pt_BR` → `pt-br`, `pt`, any `pt-*`). Spelling dictionaries and hyphenation patterns for other languages come as **Language packs**: diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index c09d6f4d..4586a746 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -285,6 +285,31 @@ impl std::fmt::Display for PackLocale { } } +/// Is a subtag a script (four ASCII letters: `Latn`, `cyrl`)? +pub(crate) fn is_script(subtag: &str) -> bool { + subtag.len() == 4 && subtag.bytes().all(|b| b.is_ascii_alphabetic()) +} + +/// The script a language is written in when its tag names none, lowercase, for the languages +/// written in more than one (or whose packs may name it): Serbian is Cyrillic, Bosnian and +/// Croatian Latin… `None`: the language doesn't say, and a pack naming a script never serves it. +pub(crate) fn default_script(primary: &str) -> Option<&'static str> { + Some(match primary { + "sr" | "ru" | "uk" | "be" | "bg" | "mk" | "kk" | "ky" | "tg" | "mn" | "tt" | "ba" | "cv" | "os" | "ce" => "cyrl", + "bs" | "hr" | "sh" | "az" | "uz" | "tk" | "ms" | "ha" | "ro" | "sq" | "tr" | "ku" | "so" => "latn", + "zh" => "hans", + "pa" => "guru", + _ => return None, + }) +} + +/// The script of a tag (`sr-latn`, `sr_Latn`): the one it names, else its language's default. +pub(crate) fn effective_script(tag: &str) -> Option { + let mut parts = tag.split(['-', '_']); + let primary = parts.next().unwrap_or_default().to_ascii_lowercase(); + parts.find(|p| is_script(p)).map(str::to_ascii_lowercase).or_else(|| default_script(&primary).map(str::to_string)) +} + /// The region of a tag: the first two-letter or three-digit subtag after the language and /// script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored. fn region(tag: &str) -> Option<&str> { diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index bd9b5f1a..5f5bcebd 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1533,6 +1533,36 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// Review of #485: a tag naming its language's default script reads the script-less pack + /// (`sr-Cyrl-RS` reads `sr`, `bs-Latn-BA` reads `bs`); another script never does (`sr-Latn` + /// never reads `sr`, `sr` never reads `sr-latn`). + #[test] + fn packs_follow_the_script() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-script-packs-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let dic = |w: &str| format!("1\n{w}\n"); + let sr = write_pack(&root, "sr", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", dic("кућа").as_bytes())]); + let bs = write_pack(&root, "bs", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", dic("kuća").as_bytes())]); + packs::set_packs_dir(Some(root.clone())); + let file = |tag: &str| pack_dictionary_file(ProofLang::from_tag(Some(tag))); + assert_eq!(file("sr-Cyrl-RS"), Some(sr.join("spelling.dic"))); + assert_eq!(file("sr-RS"), Some(sr.join("spelling.dic"))); + assert_eq!(file("bs-Latn-BA"), Some(bs.join("spelling.dic"))); + assert_eq!(file("sr-Latn-RS"), None, "Latin Serbian never reads the Cyrillic pack"); + assert!(!is_correct_in("кућаа", ProofLang::from_tag(Some("sr-Cyrl")))); + let latin = write_pack(&root, "sr-latn", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", dic("kuća").as_bytes())]); + std::fs::remove_dir_all(&sr).unwrap(); + packs::reload(); + assert_eq!(file("sr-Latn-RS"), Some(latin.join("spelling.dic"))); + assert_eq!(file("sr"), None, "Serbian (Cyrillic) never reads the Latin pack"); + assert_eq!(file("sr-Cyrl-RS"), None); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + /// Pack languages for the cache tests: `qa`, `qb`… each knowing `word`. fn many_packs(root: &std::path::Path, n: usize) -> Vec { (0..n) diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 6c6e100e..6c116653 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -88,42 +88,61 @@ pub(crate) fn set_budget(bytes: u64) { } /// Hunspell file names for `locale`, best first: `fr-fr` → `fr_FR`, `fr-FR`, `fr`; -/// `sr-latn` → `sr_Latn`, `sr-Latn` (never plain `sr`, another script). +/// `sr-latn` → `sr_Latn`, `sr-Latn` (never plain `sr`, which is Cyrillic); `sr-cyrl-rs` → +/// `sr_Cyrl_RS`, `sr-Cyrl-RS`, `sr_Cyrl`, `sr-Cyrl`, `sr_RS`, `sr-RS`, `sr` (Cyrillic is Serbian's +/// default script, [`crate::lang::default_script`]); `sr` → `sr`, `sr_Cyrl`, `sr-Cyrl`. pub(crate) fn names(locale: PackLocale) -> Vec { let parts: Vec<&str> = locale.as_str().split('-').collect(); + let script = parts.iter().position(|p| crate::lang::is_script(p)); let cased: Vec = parts .iter() .enumerate() - .map(|(i, p)| match (i, p.len()) { - (0, _) => (*p).to_string(), - (_, 2) => p.to_ascii_uppercase(), - (_, 4) => { + .map(|(i, p)| { + if i == 0 { + (*p).to_string() + } else if Some(i) == script { let mut c = p.chars(); c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() + } else if p.len() == 2 { + p.to_ascii_uppercase() + } else { + (*p).to_string() } - _ => (*p).to_string(), }) .collect(); let mut out: Vec = Vec::new(); - let mut push = |n: String| { - if !out.contains(&n) { - out.push(n); + let mut push = |parts: &[String]| { + for sep in ["_", "-"] { + let n = parts.join(sep); + if !out.contains(&n) { + out.push(n); + } } }; - // Each shorter prefix too (`sr-latn-rs` → `sr-latn`), but not past a script subtag. - let script = parts.iter().position(|p| p.len() == 4).unwrap_or(usize::MAX); - for n in (1..=cased.len()).rev() { - if n <= script && script != usize::MAX { - break; - } - let prefix = cased.get(..n).unwrap_or_default(); - push(prefix.join("_")); - push(prefix.join("-")); + let primary = locale.primary(); + let default = crate::lang::default_script(primary); + // Each shorter prefix too (`sr-latn-rs` → `sr-latn`), never dropping the script subtag. + let keep = script.map_or(1, |s| s + 1); + for n in (keep..=cased.len()).rev() { + push(cased.get(..n).unwrap_or_default()); } - if script != usize::MAX { - let prefix = cased.get(..=script).unwrap_or_default(); - push(prefix.join("_")); - push(prefix.join("-")); + match script { + // The language's default script: its script-less names are in that script too. + Some(s) if parts.get(s).is_some_and(|p| Some(*p) == default) => { + let without: Vec = cased.iter().enumerate().filter(|(i, _)| *i != s).map(|(_, p)| p.clone()).collect(); + for n in (1..=without.len()).rev() { + push(without.get(..n).unwrap_or_default()); + } + } + Some(_) => {} + // No script named: the language's default one may be spelled out (`sr-Cyrl`). + None => { + if let Some(d) = default { + let mut c = d.chars(); + let d = c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default(); + push(&[primary.to_string(), d]); + } + } } out } @@ -259,7 +278,13 @@ mod tests { assert_eq!(names(l("fr-FR")), ["fr_FR", "fr-FR", "fr"]); assert_eq!(names(l("sr-Latn")), ["sr_Latn", "sr-Latn"]); assert_eq!(names(l("sr-Latn-RS")), ["sr_Latn_RS", "sr-Latn-RS", "sr_Latn", "sr-Latn"]); + assert_eq!(names(l("sr-Cyrl-RS")), ["sr_Cyrl_RS", "sr-Cyrl-RS", "sr_Cyrl", "sr-Cyrl", "sr_RS", "sr-RS", "sr"]); + assert_eq!(names(l("bs-Latn-BA")), ["bs_Latn_BA", "bs-Latn-BA", "bs_Latn", "bs-Latn", "bs_BA", "bs-BA", "bs"]); + assert_eq!(names(l("sr")), ["sr", "sr_Cyrl", "sr-Cyrl"]); + // A four-character subtag that isn't letters is no script. + assert_eq!(names(l("de-1996")), ["de_1996", "de-1996", "de"]); assert_eq!(hyphenation_locales(l("fr-CA")), ["fr-ca", "fr"]); assert_eq!(hyphenation_locales(l("sr-Latn")), ["sr-latn"]); + assert_eq!(hyphenation_locales(l("sr-Cyrl")), ["sr-cyrl", "sr"]); } } diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index a3ff36fa..1cc4ca56 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -362,16 +362,20 @@ fn locales_for(names: &[&str]) -> Vec { for n in names { push(n.to_ascii_lowercase().replace('_', "-")); } - // The primary language alone, unless a script is named (`sr_Latn` is never plain `sr`). - for n in names.iter().filter(|n| !has_script(n)) { - push(n.split(['_', '-']).next().unwrap_or_default().to_ascii_lowercase()); + // The primary language alone, when it is written in the same script (`sr_Latn` is never + // plain `sr`, which is Cyrillic; `sr_Cyrl` is). + for n in names { + let primary = n.split(['_', '-']).next().unwrap_or_default().to_ascii_lowercase(); + if same_script(n, &primary) { + push(primary); + } } out } -/// Does a tag or file name name a script (a four-letter subtag after the language: `sr-Latn`)? -fn has_script(tag: &str) -> bool { - tag.split(['_', '-']).skip(1).any(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic())) +/// Are two tags written in the same script ([`crate::lang::effective_script`])? +fn same_script(a: &str, b: &str) -> bool { + crate::lang::effective_script(a) == crate::lang::effective_script(b) } /// The installed packs to read for `locales`, in that order, then any other pack of their @@ -386,10 +390,10 @@ fn packs_for(locales: &[String], has: impl Fn(&Holds) -> bool) -> Vec { } } // Then another region's pack of the same language (`pt` → `pt-br`), never another script's. - let primaries: Vec<&str> = locales.iter().filter(|l| !has_script(l)).map(|l| l.split('-').next().unwrap_or_default()).collect(); for (p, _) in usable() { let primary = p.locale.split('-').next().unwrap_or_default(); - if primaries.contains(&primary) && !has_script(&p.locale) && !out.contains(&p.path) { + let fits = locales.iter().any(|l| l.split('-').next() == Some(primary) && same_script(l, &p.locale)); + if fits && !out.contains(&p.path) { out.push(p.path.clone()); } }