Merge pull request #590 from benandez/fix/variant-pack-locale

proof: a variant after the region is looked up without the region first (ca-ES-valencia → ca-valencia)

crates/proof/src/lang.rs

@@ -281,7 +281,8 @@ pub struct PackLocale {
 }
 
 impl PackLocale {
-    /// Longest locale kept; a longer tag is cut at a subtag boundary.
+    /// Longest locale kept; a longer tag is cut at the subtag that would pass it (whatever the
+    /// subtag is: `sr-Cyrl-RS-ijekavsk` loses its variant, `sr-RS-ijekavsk` keeps it).
     pub const MAX_LEN: usize = 15;
 
     /// The locale of a language tag (`nl-NL`, `sr_Latn`, `fr-FR.UTF-8`): lowercased, `_` as `-`,
@@ -360,15 +361,27 @@ pub(crate) fn effective_script(tag: &str) -> Option<String> {
     parts.find(|p| is_script(p)).map(str::to_ascii_lowercase).or_else(|| default_script(&primary).map(str::to_string))
 }
 
+/// The position of the region among a tag's subtags: the first two-letter or three-digit one
+/// after the language (so a four-letter script is never one), before any singleton.
+pub(crate) fn region_index<S: AsRef<str>>(parts: &[S]) -> Option<usize> {
+    parts.iter().enumerate().skip(1).take_while(|(_, p)| p.as_ref().len() != 1).find_map(|(i, p)| {
+        let p = p.as_ref();
+        ((p.len() == 2 && p.bytes().all(|b| b.is_ascii_alphabetic())) || (p.len() == 3 && p.bytes().all(|b| b.is_ascii_digit()))).then_some(i)
+    })
+}
+
+/// `items` without the one at `index`.
+pub(crate) fn without_index<T: Clone>(items: &[T], index: usize) -> Vec<T> {
+    items.iter().enumerate().filter(|(i, _)| *i != index).map(|(_, p)| p.clone()).collect()
+}
+
 /// The region of a tag: the first two-letter or three-digit subtag after the language and
 /// script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored.
 pub(crate) fn region(tag: &str) -> Option<&str> {
     let tag = tag.split(['.', '@']).next().unwrap_or("");
     // Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions.
-    tag.split(['-', '_'])
-        .skip(1)
-        .take_while(|p| p.len() != 1)
-        .find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit())))
+    let parts: Vec<&str> = tag.split(['-', '_']).collect();
+    region_index(&parts).and_then(|i| parts.get(i).copied())
 }
 
 /// Regions whose Spanish is Latin American ([`ProofLang::Es419`]): Latin America and the

crates/proof/src/pack_lang.rs

@@ -202,15 +202,32 @@ pub(crate) fn names(locale: PackLocale) -> Vec<String> {
     let default = crate::lang::default_script(primary);
     // Each shorter prefix too (`sr-latn-rs` → `sr-latn`), never dropping the script subtag.
     let keep = script.map_or(1, |s| s + 1);
+    // The region subtag, when one follows the language (and script): a variant after it
+    // (`ca-ES-valencia`) is also looked up without it (`ca-valencia`), before the region alone,
+    // as packs and dictionaries name a variant by the language and the variant.
+    let region = crate::lang::region_index(&parts);
     for n in (keep..=cased.len()).rev() {
-        push(cased.get(..n).unwrap_or_default());
+        let prefix = cased.get(..n).unwrap_or_default();
+        push(prefix);
+        if let Some(r) = region
+            && r + 1 < n
+        {
+            push(&crate::lang::without_index(prefix, r));
+        }
     }
     match script {
         // The language's default script: its script-less names are in that script too.
         Some(s) if parts.get(s).is_some_and(|p| Some(*p) == default) => {
-            let without: Vec<String> = cased.iter().enumerate().filter(|(i, _)| *i != s).map(|(_, p)| p.clone()).collect();
+            let without = crate::lang::without_index(&cased, s);
+            let region = crate::lang::region_index(&without);
             for n in (1..=without.len()).rev() {
-                push(without.get(..n).unwrap_or_default());
+                let prefix = without.get(..n).unwrap_or_default();
+                push(prefix);
+                if let Some(r) = region
+                    && r + 1 < n
+                {
+                    push(&crate::lang::without_index(prefix, r));
+                }
             }
         }
         Some(_) => {}
@@ -420,6 +437,25 @@ mod tests {
         assert_eq!(names(l("sr")), ["sr", "sr_Cyrl", "sr-Cyrl"]);
         // A four-character subtag that isn't letters is no script.
         assert_eq!(names(l("de-1996")), ["de_1996", "de-1996", "de"]);
+        // A variant after the region is looked up without the region before the region alone
+        // (`ca-ES-valencia` is the `ca-valencia` pack's), and the region alone still is `ca-ES`.
+        assert_eq!(names(l("ca-ES-valencia")), ["ca_ES_valencia", "ca-ES-valencia", "ca_valencia", "ca-valencia", "ca_ES", "ca-ES", "ca"]);
+        assert_eq!(names(l("ca-ES")), ["ca_ES", "ca-ES", "ca"]);
+        // Any variant within the length limit counts, known or not: `sr-RS-ijekavsk` (14 bytes)
+        // keeps it, so its names are the full one, without the region, the region alone, the bare one, then the default script's.
+        assert_eq!(l("sr-RS-ijekavsk").as_str(), "sr-rs-ijekavsk");
+        assert_eq!(
+            names(l("sr-RS-ijekavsk")),
+            ["sr_RS_ijekavsk", "sr-RS-ijekavsk", "sr_ijekavsk", "sr-ijekavsk", "sr_RS", "sr-RS", "sr", "sr_Cyrl", "sr-Cyrl"]
+        );
+        // With a script the same variant passes the length limit (15 bytes), and is cut by it.
+        assert_eq!(l("sr-Cyrl-RS-ijekavsk").as_str(), "sr-cyrl-rs");
+        // A variant that fits with the default script is looked up without the region too, script-less as well.
+        let n = names(l("sr-Cyrl-RS-1994"));
+        assert!(n.contains(&"sr_Cyrl_1994".to_string()), "{n:?}");
+        assert!(n.contains(&"sr_1994".to_string()), "{n:?}");
+        assert!(n.iter().position(|x| x == "sr_1994") < n.iter().position(|x| x == "sr_RS"), "{n:?}");
+        assert_eq!(hyphenation_locales(l("ca-ES-valencia")), ["ca-es-valencia", "ca-valencia", "ca-es", "ca"]);
         assert_eq!(hyphenation_locales(l("fr-CA")), ["fr-ca", "fr"]);
         assert_eq!(hyphenation_locales(l("sr-Latn")), ["sr-latn"]);
         assert_eq!(hyphenation_locales(l("sr-Cyrl")), ["sr-cyrl", "sr"]);

crates/proof/src/packs.rs

@@ -423,7 +423,15 @@ fn locales_for(names: &[&str]) -> Vec<String> {
         }
     };
     for n in names {
-        push(n.to_ascii_lowercase().replace('_', "-"));
+        let l = n.to_ascii_lowercase().replace('_', "-");
+        // A variant after the region (`ca-es-valencia`) is the language-and-variant pack's
+        // (`ca-valencia`) before the region's or the language's alone.
+        let parts: Vec<&str> = l.split('-').collect();
+        let without = crate::lang::region_index(&parts).filter(|r| r + 1 < parts.len()).map(|r| crate::lang::without_index(&parts, r).join("-"));
+        push(l);
+        if let Some(w) = without {
+            push(w);
+        }
     }
     // The primary language alone, when it is written in the same script (`sr_Latn` is never
     // plain `sr`, which is Cyrillic; `sr_Cyrl` is).
@@ -683,6 +691,8 @@ mod tests {
         assert_eq!(region_locales(&["sr_Latn", "sr-Latn"]), ["sr-latn"]);
         assert!(region_locales(&[]).is_empty());
         assert_eq!(locales_for(&["es_419", "es"]), ["es-419", "es"]);
+        assert_eq!(locales_for(&["ca_ES_valencia", "ca_ES", "ca"]), ["ca-es-valencia", "ca-valencia", "ca-es", "ca"]);
+        assert_eq!(locales_for(&["ca_ES", "ca"]), ["ca-es", "ca"]);
         assert_eq!(locales_for(&["../x", "a b"]), Vec::<String>::new());
     }