diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index ade16edd..3d7cef64 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -279,104 +279,13 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let input = word.to_lowercase(); let chars: Vec = input.chars().collect(); let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; - // 1. Letter case. - if d.contains(&input) && word != input { - col.keep(input.clone(), COST_CASE); - } else if d.contains(&capitalize(&input)) && word != capitalize(&input) { - col.keep(capitalize(&input), COST_CASE); - } - // 2. Accents and other MAP slips. + add_case(&mut col, word, &input); let map = groups(d, t); - let alts: Vec> = chars.iter().map(|&c| alternatives(c, &map)).collect(); - let mut variants = Vec::new(); - map_variants(&chars, &alts, t, 0, 0, &mut Vec::with_capacity(n), &mut variants); - variants.sort_by_key(|v| v.1); - for (v, cost) in &variants { - col.offer(v, *cost); - } - // 3. Common confusions. - let rep = d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); - for (from, to) in rep { - if from.is_empty() { - continue; - } - for (i, _) in input.match_indices(from) { - if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { - col.offer(&format!("{a}{to}{b}"), t.rep_cost); - } - } - } - // 4. A missing space; both halves must be dictionary words. - for (i, _) in input.char_indices().skip(1) { - let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; - if let Some(cost) = split_cost(&t.split, a, b) { - col.offer(&format!("{a} {b}"), cost); - } - } - // 5. Keyboard neighbours from the dictionary's KEY rows. - for row in d.key().split('|').take(32) { - let keys: Vec = row.chars().take(128).collect(); - for pair in keys.windows(2) { - let [a, b] = pair else { continue }; - for (i, &c) in chars.iter().enumerate() { - let replacement = if c == *a { - Some(*b) - } else if c == *b { - Some(*a) - } else { - None - }; - if let Some(replacement) = replacement { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = replacement; - } - col.offer(&v.iter().collect::(), COST_KEY); - } - } - } - } - // One typo, with the TRY letters (the language's when the dictionary has none), bounded and - // each once, in their order: a TRY line lists the commonest letters first, so equal-cost - // candidates come out in that order. - let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; - let mut letters: Vec = Vec::new(); - for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { - if !letters.contains(&c) { - letters.push(c); - } - } - let s = |v: &[char]| v.iter().collect::(); - for i in 0..n.saturating_sub(1) { - if chars.get(i) != chars.get(i + 1) { - let mut v = chars.clone(); - v.swap(i, i + 1); - col.offer(&s(&v), COST_SWAP); - } - } - for i in 0..n { - let mut v = chars.clone(); - v.remove(i); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - for i in 0..=n { - for &c in &letters { - let mut v = chars.clone(); - v.insert(i, c); - col.offer(&s(&v), COST_DROP_OR_ADD); - } - } - for i in 0..n { - for &c in &letters { - if chars.get(i) != Some(&c) { - let mut v = chars.clone(); - if let Some(x) = v.get_mut(i) { - *x = c; - } - col.offer(&s(&v), (t.mistype_cost)(&chars, i, c)); - } - } - } + add_map_slips(&mut col, &chars, &map, t); + add_confusions(&mut col, &input, t); + add_splits(&mut col, &input, &t.split); + add_key_neighbours(&mut col, &chars); + add_typos(&mut col, &chars, t); // 6. Further off: scan the stems that start like the word. if col.count_within(COST_MISTYPE) < max { scan(&mut col, &chars, &map); @@ -403,6 +312,123 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V out } +/// Round 1: letter case. +fn add_case(col: &mut Collector, word: &str, input: &str) { + if col.d.contains(input) && word != input { + col.keep(input.to_string(), COST_CASE); + } else if col.d.contains(&capitalize(input)) && word != capitalize(input) { + col.keep(capitalize(input), COST_CASE); + } +} + +/// Round 2: accents and other MAP slips. +fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec], t: &Tuning) { + let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); + let mut variants = Vec::new(); + map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants); + variants.sort_by_key(|v| v.1); + for (v, cost) in &variants { + col.offer(v, *cost); + } +} + +/// Round 3: common confusions. +fn add_confusions(col: &mut Collector, input: &str, t: &Tuning) { + let rep = col.d.rep().iter().map(|(from, to)| (from.as_str(), to.as_str())).chain(t.rep.iter().copied()); + for (from, to) in rep { + if from.is_empty() { + continue; + } + for (i, _) in input.match_indices(from) { + if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) { + col.offer(&format!("{a}{to}{b}"), t.rep_cost); + } + } + } +} + +/// Round 4: a missing space; both halves must be dictionary words. +fn add_splits(col: &mut Collector, input: &str, split: &Split) { + for (i, _) in input.char_indices().skip(1) { + let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue }; + if let Some(cost) = split_cost(split, a, b) { + col.offer(&format!("{a} {b}"), cost); + } + } +} + +/// Round 5: keyboard neighbours from the dictionary's KEY rows. +fn add_key_neighbours(col: &mut Collector, chars: &[char]) { + for row in col.d.key().split('|').take(32) { + let keys: Vec = row.chars().take(128).collect(); + for pair in keys.windows(2) { + let [a, b] = pair else { continue }; + for (i, &c) in chars.iter().enumerate() { + let replacement = if c == *a { + Some(*b) + } else if c == *b { + Some(*a) + } else { + None + }; + if let Some(replacement) = replacement { + let mut v = chars.to_vec(); + if let Some(x) = v.get_mut(i) { + *x = replacement; + } + col.offer(&v.iter().collect::(), COST_KEY); + } + } + } + } +} + +/// Round 5, second half: one typo, with the TRY letters (the language's when the dictionary has +/// none), bounded and each once, in their order: a TRY line lists the commonest letters first, so +/// equal-cost candidates come out in that order. +fn add_typos(col: &mut Collector, chars: &[char], t: &Tuning) { + let d = col.d; + let n = chars.len(); + let try_chars = if d.try_chars().is_empty() { t.alphabet } else { d.try_chars() }; + let mut letters: Vec = Vec::new(); + for c in try_chars.chars().filter(|c| c.is_lowercase()).take(128) { + if !letters.contains(&c) { + letters.push(c); + } + } + let s = |v: &[char]| v.iter().collect::(); + for i in 0..n.saturating_sub(1) { + if chars.get(i) != chars.get(i + 1) { + let mut v = chars.to_vec(); + v.swap(i, i + 1); + col.offer(&s(&v), COST_SWAP); + } + } + for i in 0..n { + let mut v = chars.to_vec(); + v.remove(i); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + for i in 0..=n { + for &c in &letters { + let mut v = chars.to_vec(); + v.insert(i, c); + col.offer(&s(&v), COST_DROP_OR_ADD); + } + } + for i in 0..n { + for &c in &letters { + if chars.get(i) != Some(&c) { + let mut v = chars.to_vec(); + if let Some(x) = v.get_mut(i) { + *x = c; + } + col.offer(&s(&v), (t.mistype_cost)(chars, i, c)); + } + } + } +} + /// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either /// case) by shared letter pairs, then keep the forms of the best within two typos. fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 2520a835..8f681408 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -276,84 +276,69 @@ mod tests { } } + fn assert_tags(tags: &[&str], expected: ProofLang) { + for tag in tags { + assert_eq!(ProofLang::from_tag(Some(tag)), expected, "{tag}"); + } + } + #[test] - fn tags_resolve() { - for t in ["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Pl, "{t}"); - } - for t in ["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Pt, "{t}"); - } - for t in ["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::It, "{t}"); - } - for t in ["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Cs, "{t}"); - } + fn tags_resolve_primary_languages() { + assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"], ProofLang::Pl); + assert_tags(&["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"], ProofLang::Pt); + assert_tags(&["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"], ProofLang::It); + assert_tags(&["cs", "cs-CZ", "CS-cz", "cs_CZ", " ces ", "CES", "cze", "CZE"], ProofLang::Cs); + assert_tags(&["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"], ProofLang::Id); assert_eq!(ProofLang::Cs.code(), "cs"); assert!(ProofLang::Cs.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Cs)); - for t in ["id", "id-ID", "ID-id", "id_ID", "id_ID.UTF-8", " id-ID ", "ind", "in", "in_ID"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Id, "{t}"); - } assert_eq!(ProofLang::Id.code(), "id"); assert!(ProofLang::Id.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Id)); - for t in ["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}"); - } - for t in [ - "en-GB", - "EN_gb", - "en_GB.UTF-8", - "en-AU", - "en_AU", - "en-NZ", - "en-IE", - "en-ZA", - "en-IN", - "en-Latn-GB", - "en-UK", - "en-150", - "eng-GB", - "en_GB@euro", - ] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EnGb, "{t}"); - } - for t in ["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EnCa, "{t}"); - } - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { - assert!(l.is_english() && l.is_proofed() && ProofLang::PROOFED.contains(&l), "{l:?}"); - assert_eq!(l.code(), "en"); + } + + #[test] + fn tags_resolve_english_regions() { + assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En); + assert_tags( + &[ + "en-GB", + "EN_gb", + "en_GB.UTF-8", + "en-AU", + "en_AU", + "en-NZ", + "en-IE", + "en-ZA", + "en-IN", + "en-Latn-GB", + "en-UK", + "en-150", + "eng-GB", + "en_GB@euro", + ], + ProofLang::EnGb, + ); + assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa); + for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}"); + assert_eq!(language.code(), "en"); } assert!(!ProofLang::Pl.is_english() && !ProofLang::Other.is_english()); // The region is the first two-letter or three-digit subtag: a script or variant isn't one. assert_eq!(ProofLang::from_tag(Some("en-Brai")), ProofLang::En); assert_eq!(ProofLang::from_tag(Some("en-GB-oxendict")), ProofLang::EnGb); - for t in ["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Bg, "{t}"); - } - for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}"); - } - for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}"); - } + } + + #[test] + fn tags_resolve_additional_locales_and_defaults() { + assert_tags(&["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"], ProofLang::Bg); + assert_tags(&["tr", "tr-TR", "tr-CY", "TR_tr", "tur"], ProofLang::Tr); + assert_tags(&["de", "de-DE", "de-CH", "DE_at", "deu", "ger"], ProofLang::De); + assert_tags(&["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"], ProofLang::Ru); assert!(!ProofLang::De.is_proofed()); - for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}"); - } - assert_eq!(ProofLang::from_tag(None), ProofLang::En); - assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En); - assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En); - assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed()); - // Bulgarian has hyphenation only: no spelling or grammar marks. assert!(!ProofLang::Bg.is_proofed()); assert_eq!(ProofLang::Bg.code(), "bg"); assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed()); assert!(ProofLang::Tr.is_proofed()); - for t in ["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"] { - assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Ru, "{t}"); - } assert!(ProofLang::Ru.is_proofed()); assert_eq!(ProofLang::Ru.code(), "ru"); assert!(ProofLang::PROOFED.contains(&ProofLang::Ru)); @@ -365,6 +350,15 @@ mod tests { assert_eq!(ProofLang::Hr.code(), "hr"); assert_eq!(ProofLang::Tr.code(), "tr"); assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed"); + assert_eq!(ProofLang::from_tag(None), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some("")), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some(" ")), ProofLang::En); + assert!(ProofLang::Pl.is_proofed() && !ProofLang::Other.is_proofed()); + } + + #[test] + fn tags_reject_unknown_or_partial_language_codes() { + assert_tags(&["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"], ProofLang::Other); } /// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and