From 72e21ec7ca919008572c68aba9c6b3c11fb98620 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Tue, 3 Nov 2026 13:37:37 +0000 Subject: [PATCH 1/2] craft-proof: a variant after the region is looked up without the region first (ca-ES-valencia reads the ca-valencia pack) (review: FredApps) One region_index/without_index helper in lang.rs serves region(), pack_lang::names and packs::locales_for; tests pin the sr-RS-ijekavsk names and the length cut of sr-Cyrl-RS-ijekavsk. Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01FypgCYmv6UJmY2QcuhfXco --- crates/proof/src/lang.rs | 23 ++++++++++++++++++----- crates/proof/src/pack_lang.rs | 28 ++++++++++++++++++++++++++-- crates/proof/src/packs.rs | 12 +++++++++++- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 463406a9..e5fcf3bb 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -281,7 +281,8 @@ pub struct PackLocale { } impl PackLocale { - /// Longest locale kept; a longer tag is cut at a subtag boundary. + /// Longest locale kept; a longer tag is cut at the subtag that would pass it (whatever the + /// subtag is: `sr-Cyrl-RS-ijekavsk` loses its variant, `sr-RS-ijekavsk` keeps it). pub const MAX_LEN: usize = 15; /// The locale of a language tag (`nl-NL`, `sr_Latn`, `fr-FR.UTF-8`): lowercased, `_` as `-`, @@ -360,15 +361,27 @@ pub(crate) fn effective_script(tag: &str) -> Option { parts.find(|p| is_script(p)).map(str::to_ascii_lowercase).or_else(|| default_script(&primary).map(str::to_string)) } +/// The position of the region among a tag's subtags: the first two-letter or three-digit one +/// after the language (so a four-letter script is never one), before any singleton. +pub(crate) fn region_index>(parts: &[S]) -> Option { + parts.iter().enumerate().skip(1).take_while(|(_, p)| p.as_ref().len() != 1).find_map(|(i, p)| { + let p = p.as_ref(); + ((p.len() == 2 && p.bytes().all(|b| b.is_ascii_alphabetic())) || (p.len() == 3 && p.bytes().all(|b| b.is_ascii_digit()))).then_some(i) + }) +} + +/// `items` without the one at `index`. +pub(crate) fn without_index(items: &[T], index: usize) -> Vec { + items.iter().enumerate().filter(|(i, _)| *i != index).map(|(_, p)| p.clone()).collect() +} + /// The region of a tag: the first two-letter or three-digit subtag after the language and /// script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored. pub(crate) fn region(tag: &str) -> Option<&str> { let tag = tag.split(['.', '@']).next().unwrap_or(""); // Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions. - tag.split(['-', '_']) - .skip(1) - .take_while(|p| p.len() != 1) - .find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit()))) + let parts: Vec<&str> = tag.split(['-', '_']).collect(); + region_index(&parts).and_then(|i| parts.get(i).copied()) } /// Regions whose Spanish is Latin American ([`ProofLang::Es419`]): Latin America and the diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 7b4d1931..93ec1eb4 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -202,13 +202,23 @@ pub(crate) fn names(locale: PackLocale) -> Vec { let default = crate::lang::default_script(primary); // Each shorter prefix too (`sr-latn-rs` → `sr-latn`), never dropping the script subtag. let keep = script.map_or(1, |s| s + 1); + // The region subtag, when one follows the language (and script): a variant after it + // (`ca-ES-valencia`) is also looked up without it (`ca-valencia`), before the region alone, + // as packs and dictionaries name a variant by the language and the variant. + let region = crate::lang::region_index(&parts); for n in (keep..=cased.len()).rev() { - push(cased.get(..n).unwrap_or_default()); + let prefix = cased.get(..n).unwrap_or_default(); + push(prefix); + if let Some(r) = region + && r + 1 < n + { + push(&crate::lang::without_index(prefix, r)); + } } match script { // The language's default script: its script-less names are in that script too. Some(s) if parts.get(s).is_some_and(|p| Some(*p) == default) => { - let without: Vec = cased.iter().enumerate().filter(|(i, _)| *i != s).map(|(_, p)| p.clone()).collect(); + let without = crate::lang::without_index(&cased, s); for n in (1..=without.len()).rev() { push(without.get(..n).unwrap_or_default()); } @@ -420,6 +430,20 @@ mod tests { assert_eq!(names(l("sr")), ["sr", "sr_Cyrl", "sr-Cyrl"]); // A four-character subtag that isn't letters is no script. assert_eq!(names(l("de-1996")), ["de_1996", "de-1996", "de"]); + // A variant after the region is looked up without the region before the region alone + // (`ca-ES-valencia` is the `ca-valencia` pack's), and the region alone still is `ca-ES`. + assert_eq!(names(l("ca-ES-valencia")), ["ca_ES_valencia", "ca-ES-valencia", "ca_valencia", "ca-valencia", "ca_ES", "ca-ES", "ca"]); + assert_eq!(names(l("ca-ES")), ["ca_ES", "ca-ES", "ca"]); + // Any variant within the length limit counts, known or not: `sr-RS-ijekavsk` (14 bytes) + // keeps it, so its names are the full one, without the region, the region alone, the bare one, then the default script's. + assert_eq!(l("sr-RS-ijekavsk").as_str(), "sr-rs-ijekavsk"); + assert_eq!( + names(l("sr-RS-ijekavsk")), + ["sr_RS_ijekavsk", "sr-RS-ijekavsk", "sr_ijekavsk", "sr-ijekavsk", "sr_RS", "sr-RS", "sr", "sr_Cyrl", "sr-Cyrl"] + ); + // With a script the same variant passes the length limit (15 bytes), and is cut by it. + assert_eq!(l("sr-Cyrl-RS-ijekavsk").as_str(), "sr-cyrl-rs"); + assert_eq!(hyphenation_locales(l("ca-ES-valencia")), ["ca-es-valencia", "ca-valencia", "ca-es", "ca"]); assert_eq!(hyphenation_locales(l("fr-CA")), ["fr-ca", "fr"]); assert_eq!(hyphenation_locales(l("sr-Latn")), ["sr-latn"]); assert_eq!(hyphenation_locales(l("sr-Cyrl")), ["sr-cyrl", "sr"]); diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 2e558393..eebd406c 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -423,7 +423,15 @@ fn locales_for(names: &[&str]) -> Vec { } }; for n in names { - push(n.to_ascii_lowercase().replace('_', "-")); + let l = n.to_ascii_lowercase().replace('_', "-"); + // A variant after the region (`ca-es-valencia`) is the language-and-variant pack's + // (`ca-valencia`) before the region's or the language's alone. + let parts: Vec<&str> = l.split('-').collect(); + let without = crate::lang::region_index(&parts).filter(|r| r + 1 < parts.len()).map(|r| crate::lang::without_index(&parts, r).join("-")); + push(l); + if let Some(w) = without { + push(w); + } } // The primary language alone, when it is written in the same script (`sr_Latn` is never // plain `sr`, which is Cyrillic; `sr_Cyrl` is). @@ -683,6 +691,8 @@ mod tests { assert_eq!(region_locales(&["sr_Latn", "sr-Latn"]), ["sr-latn"]); assert!(region_locales(&[]).is_empty()); assert_eq!(locales_for(&["es_419", "es"]), ["es-419", "es"]); + assert_eq!(locales_for(&["ca_ES_valencia", "ca_ES", "ca"]), ["ca-es-valencia", "ca-valencia", "ca-es", "ca"]); + assert_eq!(locales_for(&["ca_ES", "ca"]), ["ca-es", "ca"]); assert_eq!(locales_for(&["../x", "a b"]), Vec::::new()); } From 7a334e5306d64b65f70a00f44129b42d485f81ba Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Tue, 3 Nov 2026 21:46:07 +0000 Subject: [PATCH 2/2] =?UTF-8?q?proof:=20a=20variant=20after=20the=20defaul?= =?UTF-8?q?t=20script=20and=20a=20region=20is=20looked=20up=20without=20th?= =?UTF-8?q?e=20region=20too=20(sr-Cyrl-RS-1994=20=E2=86=92=20sr-1994)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCYmv6UJmY2QcuhfXco --- crates/proof/src/pack_lang.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 93ec1eb4..30206661 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -219,8 +219,15 @@ pub(crate) fn names(locale: PackLocale) -> Vec { // The language's default script: its script-less names are in that script too. Some(s) if parts.get(s).is_some_and(|p| Some(*p) == default) => { let without = crate::lang::without_index(&cased, s); + let region = crate::lang::region_index(&without); for n in (1..=without.len()).rev() { - push(without.get(..n).unwrap_or_default()); + let prefix = without.get(..n).unwrap_or_default(); + push(prefix); + if let Some(r) = region + && r + 1 < n + { + push(&crate::lang::without_index(prefix, r)); + } } } Some(_) => {} @@ -443,6 +450,11 @@ mod tests { ); // With a script the same variant passes the length limit (15 bytes), and is cut by it. assert_eq!(l("sr-Cyrl-RS-ijekavsk").as_str(), "sr-cyrl-rs"); + // A variant that fits with the default script is looked up without the region too, script-less as well. + let n = names(l("sr-Cyrl-RS-1994")); + assert!(n.contains(&"sr_Cyrl_1994".to_string()), "{n:?}"); + assert!(n.contains(&"sr_1994".to_string()), "{n:?}"); + assert!(n.iter().position(|x| x == "sr_1994") < n.iter().position(|x| x == "sr_RS"), "{n:?}"); assert_eq!(hyphenation_locales(l("ca-ES-valencia")), ["ca-es-valencia", "ca-valencia", "ca-es", "ca"]); assert_eq!(hyphenation_locales(l("fr-CA")), ["fr-ca", "fr"]); assert_eq!(hyphenation_locales(l("sr-Latn")), ["sr-latn"]);