diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs new file mode 100644 index 00000000..56b9f26f --- /dev/null +++ b/crates/proof/src/en_variants.rs @@ -0,0 +1,277 @@ +//! American and British spellings of the same English word (#1281, #1045): which forms each +//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, +//! and what it suggests instead (`colour` → `color` in American English). +//! +//! The pairs come from `assets/spelling/en-variants.txt`, our own list of base forms with a +//! policy per English. Inflections are generated here by rule for the regular patterns (-or/-our, +//! -er/-re, -ize/-ise, -ense/-ence…), so `colours`, `centred` and `organisation` follow their +//! base forms. A form is right when the shared English word list knows it or knows its other +//! spelling (`realisation` is right in British English because `realization` is a word), so the +//! rules never make a misspelling right. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use crate::ProofLang; + +/// Our list of American/British pairs: `POLICY AMERICAN BRITISH` per line, `#` comments. +const VARIANTS: &str = include_str!("../../../assets/spelling/en-variants.txt"); + +/// What one English does with a pair's two spellings. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Policy { + /// Only the American spelling is right. + American, + /// Only the British spelling is right. + British, + /// Both are right. + Both, +} + +impl Policy { + fn from_char(c: char) -> Option { + match c { + 'u' => Some(Policy::American), + 'b' => Some(Policy::British), + '*' => Some(Policy::Both), + _ => None, + } + } +} + +/// One line of the list: the policies for American, Canadian and British-spelling English, and +/// the two base forms. +struct Pair { + policy: [Policy; 3], + american: &'static str, + british: &'static str, +} + +/// The lines of [`VARIANTS`]; malformed lines are skipped (a test keeps the file clean). +fn pairs() -> &'static [Pair] { + static P: OnceLock> = OnceLock::new(); + P.get_or_init(|| VARIANTS.lines().filter_map(parse_line).collect()) +} + +fn parse_line(line: &'static str) -> Option { + let line = line.trim(); + if line.is_empty() || line.starts_with('#') { + return None; + } + let mut parts = line.split_whitespace(); + let (policy, american, british) = (parts.next()?, parts.next()?, parts.next()?); + if parts.next().is_some() || american == british { + return None; + } + let mut p = policy.chars().map(Policy::from_char); + let policy = [p.next()??, p.next()??, p.next()??]; + if p.next().is_some() { + return None; + } + let word = |w: &str| !w.is_empty() && w.chars().all(|c| c.is_ascii_lowercase()); + (word(american) && word(british)).then_some(Pair { policy, american, british }) +} + +/// The column of a pair's policy for `lang`; `None` for a language that isn't English. +fn column(lang: ProofLang) -> Option { + match lang { + ProofLang::En => Some(0), + ProofLang::EnCa => Some(1), + ProofLang::EnGb => Some(2), + _ => None, + } +} + +/// `word` with `suffix`: a final `e` drops before a vowel (`centre` + `ing` → `centring`), a +/// final consonant + `y` becomes `i` (`cosy` + `er` → `cosier`), `s` is the plural (`es` after a +/// sibilant, `ies` after a consonant + `y`). +fn add(word: &str, suffix: &str) -> String { + let b = word.as_bytes(); + let last = b.last().copied().unwrap_or(b'_'); + let before = b.len().checked_sub(2).and_then(|i| b.get(i)).copied().unwrap_or(b'_'); + let vowel = |c: u8| b"aeiou".contains(&c); + let head = word.get(..word.len().saturating_sub(1)).unwrap_or(word); + if suffix == "s" { + if last == b'y' && !vowel(before) { + return format!("{head}ies"); + } + if matches!(last, b's' | b'x' | b'z') || word.ends_with("ch") || word.ends_with("sh") { + return format!("{word}es"); + } + return format!("{word}s"); + } + let suffix_vowel = suffix.as_bytes().first().is_some_and(|&c| vowel(c)); + if last == b'e' && suffix_vowel { + return format!("{head}{suffix}"); + } + if last == b'y' && !vowel(before) && !suffix.starts_with('i') { + return format!("{head}i{suffix}"); + } + format!("{word}{suffix}") +} + +/// The suffixes both spellings of a pair take, by the pattern that tells them apart. +fn suffixes(american: &str, british: &str) -> &'static [&'static str] { + // Not -al, -ist or -ite: `humoral` and `humorist` are spelled so in British English too, and + // `behavioural` and `favourite` have lines of their own. + const OUR: &[&str] = &["s", "ed", "ing", "er", "ers", "ful", "fully", "less", "able", "ably"]; + const RE: &[&str] = &["s", "ed", "ing"]; + const IZE: &[&str] = &["s", "ed", "ing", "er", "ers", "ation", "ations", "ational", "able"]; + const E: &[&str] = &["s", "ed", "ing", "er", "ers"]; + const Y: &[&str] = &["s", "er", "est", "ly", "ness"]; + const PLURAL: &[&str] = &["s"]; + let stem = |w: &str, n: usize| w.get(..w.len().saturating_sub(n)).map(str::to_string); + if american.ends_with("or") && british.ends_with("our") && stem(american, 2) == stem(british, 3) { + return OUR; + } + if american.ends_with("er") && british.ends_with("re") && stem(american, 2) == stem(british, 2) { + return RE; + } + // One letter apart (`organize`/`organise`, `defense`/`defence`, `cozy`/`cosy`). + let differ = american.len() == british.len() && american.bytes().zip(british.bytes()).filter(|(a, b)| a != b).count() == 1; + if differ && american.ends_with('e') && british.ends_with('e') { + let ize = american.ends_with("ize") || american.ends_with("yze"); + return if ize { IZE } else { E }; + } + if differ && american.ends_with('y') && british.ends_with('y') { + return Y; + } + PLURAL +} + +/// One English's forms: those it flags (each with the form to suggest), and those it accepts +/// when the shared list knows them or their other spelling (each with that other spelling). +#[derive(Default)] +struct Forms { + foreign: HashMap, + accepted: HashMap, +} + +fn forms(lang: ProofLang) -> Option<&'static Forms> { + static F: [OnceLock; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()]; + let col = column(lang)?; + Some(F.get(col)?.get_or_init(|| build(col))) +} + +fn build(col: usize) -> Forms { + let mut f = Forms::default(); + for pair in pairs() { + let Some(&policy) = pair.policy.get(col) else { continue }; + let mut both = vec![(pair.american.to_string(), pair.british.to_string())]; + both.extend(suffixes(pair.american, pair.british).iter().map(|s| (add(pair.american, s), add(pair.british, s)))); + for (am, br) in both { + if am == br { + continue; + } + match policy { + Policy::American => { + f.foreign.insert(br.clone(), am.clone()); + f.accepted.insert(am, br); + } + Policy::British => { + f.foreign.insert(am.clone(), br.clone()); + f.accepted.insert(br, am); + } + Policy::Both => { + f.accepted.insert(am.clone(), br.clone()); + f.accepted.insert(br, am); + } + } + } + } + // A form one pair accepts is never flagged by another, and `analyses`, `paralyses`… are also + // the plurals of `analysis`, `paralysis`… everywhere. + f.foreign.retain(|w, _| !f.accepted.contains_key(w) && !w.ends_with("yses")); + f +} + +/// `lower` without a possessive `'s`. +fn without_possessive(lower: &str) -> &str { + lower.strip_suffix("'s").filter(|s| !s.is_empty()).unwrap_or(lower) +} + +/// The spelling `lang` wants instead of `lower` (a lower-case word, `'` apostrophes), when +/// `lower` is the other dialect's spelling of a word (`colour` in American English: `color`). +/// A possessive keeps its `'s` (`colour's` → `color's`). +pub fn preferred(lower: &str, lang: ProofLang) -> Option { + let f = forms(lang)?; + let base = without_possessive(lower); + let native = f.foreign.get(base)?; + Some(if base.len() < lower.len() { format!("{native}'s") } else { native.clone() }) +} + +/// Is `lower` right in `lang` because the shared word list knows its other spelling (`known` +/// says what the list knows)? `realisation` in British English, `organization` in all three. +pub fn known_variant(lower: &str, lang: ProofLang, known: impl Fn(&str) -> bool) -> bool { + let Some(f) = forms(lang) else { return false }; + let base = without_possessive(lower); + f.accepted.get(base).is_some_and(|other| known(other)) +} + +/// Is `lower` the other dialect's spelling in `lang` (flagged even though it is a word)? +pub fn is_foreign(lower: &str, lang: ProofLang) -> bool { + forms(lang).is_some_and(|f| f.foreign.contains_key(without_possessive(lower))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn every_line_parses() { + let lines = VARIANTS.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).count(); + assert_eq!(pairs().len(), lines, "a malformed line in en-variants.txt"); + assert!(pairs().len() > 250); + } + + #[test] + fn suffixes_follow_the_pattern() { + assert_eq!(add("centre", "ing"), "centring"); + assert_eq!(add("centre", "ed"), "centred"); + assert_eq!(add("center", "ed"), "centered"); + assert_eq!(add("organise", "ation"), "organisation"); + assert_eq!(add("cosy", "er"), "cosier"); + assert_eq!(add("cosy", "s"), "cosies"); + assert_eq!(add("armoury", "s"), "armouries"); + assert_eq!(add("analyse", "s"), "analyses"); + assert_eq!(add("colour", "ful"), "colourful"); + assert_eq!(add("", "s"), "s"); + assert_eq!(suffixes("color", "colour").len(), 10); + assert_eq!(suffixes("center", "centre"), ["s", "ed", "ing"]); + assert!(suffixes("organize", "organise").contains(&"ation")); + assert_eq!(suffixes("program", "programme"), ["s"], "no -ed: programmed is American too"); + } + + #[test] + fn no_form_is_both_flagged_and_accepted() { + for lang in [ProofLang::En, ProofLang::EnCa, ProofLang::EnGb] { + let Some(f) = forms(lang) else { panic!("{lang:?}") }; + for (w, native) in &f.foreign { + assert!(!f.accepted.contains_key(w), "{lang:?}: {w}"); + assert!(!f.foreign.contains_key(native), "{lang:?}: suggests {native}, itself flagged"); + } + } + // Words both dialects use alike are never flagged. + for w in [ + "programmed", + "tired", + "tires", + "curbs", + "humorous", + "honorary", + "humorist", + "humoral", + "meter", + "license", + "for", + "tour", + "analyses", + "paralyses", + ] { + for lang in [ProofLang::En, ProofLang::EnCa, ProofLang::EnGb] { + assert!(!is_foreign(w, lang), "{lang:?}: {w}"); + } + } + assert!(forms(ProofLang::Pl).is_none() && !is_foreign("color", ProofLang::Other)); + } +} diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 6129d03a..f94ec95f 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -5,11 +5,21 @@ /// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ProofLang { - /// English, any region (`en`, `en-US`, `en-GB`…): the public-domain Moby word list, the - /// English grammar rules and the English hyphenation dictionary and patterns. Also text with - /// no language at all (WordCraft's default editing language is US English, like Word's). + /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or + /// [`ProofLang::EnCa`]): the public-domain Moby word list, the English grammar rules and the + /// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`, + /// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also + /// text with no language at all (WordCraft's default editing language is US English, like + /// Word's). #[default] En, + /// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): + /// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are + /// flagged and `-ize` and `-ise` are both right, as in Oxford spelling. + EnGb, + /// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`, + /// `defence`) with American `-ize` (`organize`), `tire` and `program`. + EnCa, /// Polish (`pl`, `pl-PL`): the installed Hunspell dictionary (when there is one), the Polish grammar rules and the Polish /// hyphenation patterns. Pl, @@ -48,16 +58,28 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 9] = - [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id]; + pub const PROOFED: [ProofLang; 11] = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::EnCa, + ProofLang::Pl, + ProofLang::Pt, + ProofLang::It, + ProofLang::Hr, + ProofLang::Tr, + ProofLang::Ru, + ProofLang::Cs, + ProofLang::Id, + ]; - /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). Only the primary subtag - /// counts. No tag (or an empty one) is English, WordCraft's default editing language. + /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). The primary subtag + /// counts, and for English the region too ([`ProofLang::english`]). No tag (or an empty one) + /// is English, WordCraft's default editing language. pub fn from_tag(tag: Option<&str>) -> ProofLang { let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En }; let primary = tag.split(['-', '_']).next().unwrap_or(""); if primary.eq_ignore_ascii_case("en") || primary.eq_ignore_ascii_case("eng") { - ProofLang::En + ProofLang::english(tag) } else if primary.eq_ignore_ascii_case("pl") || primary.eq_ignore_ascii_case("pol") { ProofLang::Pl } else if primary.eq_ignore_ascii_case("bg") || primary.eq_ignore_ascii_case("bul") { @@ -81,10 +103,34 @@ impl ProofLang { } } + /// The English of an English tag, by its region (the first two-letter or three-digit subtag + /// after the language and script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored): Canada is + /// [`ProofLang::EnCa`], regions that spell the British way are [`ProofLang::EnGb`], and the + /// United States, any other region and no region are [`ProofLang::En`]. + fn english(tag: &str) -> ProofLang { + let tag = tag.split(['.', '@']).next().unwrap_or(""); + // Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions. + let region = tag + .split(['-', '_']) + .skip(1) + .take_while(|p| p.len() != 1) + .find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit()))); + match region { + Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, + Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, + _ => ProofLang::En, + } + } + + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? + pub fn is_english(self) -> bool { + matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) + } + /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`); empty for [`ProofLang::Other`]. pub fn code(self) -> &'static str { match self { - ProofLang::En => "en", + ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", ProofLang::Pl => "pl", ProofLang::Bg => "bg", ProofLang::Pt => "pt", @@ -104,6 +150,14 @@ impl ProofLang { } } +/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and +/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other +/// Commonwealth countries, the Crown dependencies and Europe (`150`). +const BRITISH_SPELLING: [&str; 26] = [ + "GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", + "PK", "150", +]; + /// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when /// `tag` isn't shaped like one: 1–35 ASCII letters, digits and `-`/`_` separators, starting with /// a 2–8 letter language (or `x`/`i` for private and grandfathered tags). `_` becomes `-`. @@ -183,9 +237,38 @@ mod tests { } assert_eq!(ProofLang::Id.code(), "id"); assert!(ProofLang::Id.is_proofed() && ProofLang::PROOFED.contains(&ProofLang::Id)); - for t in ["en", "en-US", "en-GB", "EN_au", "en-Latn-US"] { + for t in ["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}"); } + for t in [ + "en-GB", + "EN_gb", + "en_GB.UTF-8", + "en-AU", + "en_AU", + "en-NZ", + "en-IE", + "en-ZA", + "en-IN", + "en-Latn-GB", + "en-UK", + "en-150", + "eng-GB", + "en_GB@euro", + ] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EnGb, "{t}"); + } + for t in ["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EnCa, "{t}"); + } + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert!(l.is_english() && l.is_proofed() && ProofLang::PROOFED.contains(&l), "{l:?}"); + assert_eq!(l.code(), "en"); + } + assert!(!ProofLang::Pl.is_english() && !ProofLang::Other.is_english()); + // The region is the first two-letter or three-digit subtag: a script or variant isn't one. + assert_eq!(ProofLang::from_tag(Some("en-Brai")), ProofLang::En); + assert_eq!(ProofLang::from_tag(Some("en-GB-oxendict")), ProofLang::EnGb); for t in ["bg", "bg-BG", "BG_bg", "bg_BG.UTF-8", "bul"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Bg, "{t}"); } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 5afa42d7..9347fba7 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -46,6 +46,7 @@ pub mod affix; pub mod bg; pub mod cs; pub mod dict; +pub mod en_variants; pub mod hr; pub mod hunspell; pub mod hyphen; @@ -496,22 +497,76 @@ pub fn suggest(word: &str, max: usize) -> Vec { } } scored.sort_by(|a, b| (a.0, a.1, a.2.len().abs_diff(n)).cmp(&(b.0, b.1, b.2.len().abs_diff(n))).then(a.2.cmp(&b.2))); + scored.into_iter().take(max).map(|(_, _, w)| match_case(word, w)).collect() +} + +/// `w` (lower case) in the case of `word`: all capitals, an initial capital, or as it is. +fn match_case(word: &str, w: String) -> String { let cap = word.chars().next().is_some_and(char::is_uppercase); let upper = word.chars().all(|c| !c.is_lowercase()); - scored - .into_iter() - .take(max) - .map(|(_, _, w)| { - if upper && n > 1 { - w.to_uppercase() - } else if cap { - let mut c = w.chars(); - c.next().map(|f| f.to_uppercase().chain(c).collect()).unwrap_or(w) - } else { - w + if upper && word.chars().count() > 1 { + w.to_uppercase() + } else if cap { + let mut c = w.chars(); + c.next().map(|f| f.to_uppercase().chain(c).collect()).unwrap_or(w) + } else { + w + } +} + +/// `lower` in the spelling `lang` (an English) wants, when it or a part of a hyphenated compound +/// is the other dialect's spelling (`colour-blind` → `color-blind` in American English). +fn english_preferred(lower: &str, lang: ProofLang) -> Option { + let mut changed = false; + let parts: Vec = lower + .split('-') + .map(|p| match en_variants::preferred(p, lang) { + Some(n) => { + changed = true; + n } + None => p.to_string(), }) - .collect() + .collect(); + changed.then(|| parts.join("-")) +} + +/// Is `word` spelled right in the English `lang` ([`ProofLang::En`], [`ProofLang::EnGb`], +/// [`ProofLang::EnCa`])? The shared word list's words, less the other dialect's spellings +/// (`colour` in American English, `color` in British), plus this dialect's spellings of the +/// list's words ([`en_variants`]). The user dictionary overrules both. +fn is_correct_english(word: &str, lang: ProofLang, opts: &Options) -> bool { + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'').trim_matches('\''); + if accepted_by_options(w, opts) { + return true; + } + let lower = w.to_lowercase().replace('’', "'"); + if in_user_dictionary(&lower) { + return true; + } + let parts: Vec<&str> = lower.split('-').filter(|p| !p.is_empty()).collect(); + if parts.iter().any(|p| en_variants::is_foreign(p, lang)) { + return false; + } + is_correct_with(word, opts) || (!parts.is_empty() && parts.iter().all(|p| known_core(p) || en_variants::known_variant(p, lang, known_core))) +} + +/// Spelling suggestions in the English `lang`: this dialect's spelling first when `word` is the +/// other dialect's, then the shared suggestions in this dialect's spelling. +fn suggest_english(word: &str, max: usize, lang: ProofLang) -> Vec { + let lower = word.to_lowercase().replace('’', "'"); + let mut out: Vec = Vec::new(); + if let Some(p) = english_preferred(&lower, lang) { + out.push(match_case(word, p)); + } + for s in suggest(word, max.saturating_add(2)) { + let s = english_preferred(&s.to_lowercase(), lang).map_or(s, |p| match_case(word, p)); + if !out.contains(&s) { + out.push(s); + } + } + out.truncate(max); + out } /// Optimal string alignment distance, early-exit at 3. @@ -631,7 +686,7 @@ fn enclosing_token(text: &str, a: usize, b: usize) -> (usize, usize, bool) { fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Options, internet: bool) -> Option { let w = text.get(a..b)?; // A word with a Vietnamese-only letter (a name like Nguyễn in English text) isn't English. - if (opts.ignore_internet && internet) || (lang == ProofLang::En && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { + if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) { return None; } let message = match lang { @@ -656,7 +711,7 @@ pub fn is_correct_in(word: &str, lang: ProofLang) -> bool { /// [`is_correct_in`] with the File › Options › Proofing choices. pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { match lang { - ProofLang::En => is_correct_with(word, opts), + ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => is_correct_english(word, lang, opts), ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct(word), ProofLang::Hr => accepted_by_options(word, opts) || hr::is_correct(word), ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct(word), @@ -672,7 +727,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { /// Spelling suggestions for a word misspelled in `lang` (best first, at most `max`). pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { match lang { - ProofLang::En => suggest(word, max), + ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => suggest_english(word, max, lang), ProofLang::Pl => pl::suggest(word, max), ProofLang::Hr => hr::suggest(word, max), ProofLang::Pt => pt::suggest(word, max), @@ -697,6 +752,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), + ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), ProofLang::Pl => pl::grammar::check_with(text, capital_first), ProofLang::Hr => hr::grammar::check_with(text, capital_first), ProofLang::Pt => pt::grammar::check_with(text, capital_first), @@ -739,7 +795,7 @@ pub fn check_text_with(text: &str, spans: &[(Range, ProofLang)], opts: &O // A paragraph typed in Vietnamese under the default English language is Vietnamese, which // has no proofing tools: no marks (Word detects the language the same way). let default = if vi::looks_vietnamese(text) { - for (_, l) in merged.iter_mut().filter(|(_, l)| *l == ProofLang::En) { + for (_, l) in merged.iter_mut().filter(|(_, l)| l.is_english()) { *l = ProofLang::Other; } ProofLang::Other @@ -1100,7 +1156,197 @@ mod tests { ] { assert!(is_correct(w), "{w}"); } - assert!(check_spelling("The programme honoured her rigour and prioritised colourful, paediatric care.").is_empty()); + let text = "The programme honoured her rigour and prioritised colourful, paediatric care."; + for lang in [ProofLang::EnGb, ProofLang::EnCa] { + let flagged: Vec<&str> = check_spelling_in(text, lang).iter().map(|i| &text[i.start..i.end]).collect(); + let expect: &[&str] = if lang == ProofLang::EnCa { &["prioritised"] } else { &[] }; + assert_eq!(flagged, expect, "{lang:?}"); + } + // American English flags the British spellings (#1281). + let flagged: Vec<&str> = check_spelling(text).iter().map(|i| &text[i.start..i.end]).collect(); + assert_eq!(flagged, ["programme", "honoured", "rigour", "prioritised", "colourful", "paediatric"]); + } + + /// Each English accepts its own spellings and flags the other dialect's, suggesting its own + /// first, as Word does (#1281, #1045). + #[test] + fn english_dialects_flag_each_others_spellings() { + let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa); + // (word, right in en-US, right in en-CA, right in en-GB) + let cases = [ + ("color", true, false, false), + ("colour", false, true, true), + ("colours", false, true, true), + ("coloured", false, true, true), + ("Colourful", false, true, true), + ("colour's", false, true, true), + ("honorable", true, false, false), + ("neighbourhood", false, true, true), + ("center", true, false, false), + ("centre", false, true, true), + ("centred", false, true, true), + ("centring", false, true, true), + ("centering", true, false, false), + ("theater", true, false, false), + ("theatre", true, true, true), + ("organize", true, true, true), + ("organise", false, false, true), + ("organisation", false, false, true), + ("organisations", false, false, true), + ("realised", false, false, true), + ("recognising", false, false, true), + ("analyze", true, true, false), + ("analyse", false, true, true), + ("paralysed", false, true, true), + ("defense", true, false, false), + ("defence", false, true, true), + ("licence", false, true, true), + ("license", true, true, true), + ("practise", false, true, true), + ("traveled", true, false, false), + ("travelled", true, true, true), + ("catalog", true, false, false), + ("catalogue", true, true, true), + ("anaemia", false, true, true), + ("anemia", true, true, false), + ("tire", true, true, true), + ("tyre", false, false, true), + ("programme", false, true, true), + ("program", true, true, true), + ("meter", true, true, true), + ("metre", false, true, true), + ("plough", false, true, true), + ("colour-blind", false, true, true), + // Words spelled alike everywhere, and real misspellings, are unaffected. + ("humorous", true, true, true), + ("honorary", true, true, true), + ("tour", true, true, true), + ("four", true, true, true), + ("programmed", true, true, true), + ("colourr", false, false, false), + ("colorr", false, false, false), + ("organisationz", false, false, false), + ]; + for (w, in_us, in_ca, in_gb) in cases { + assert_eq!(is_correct_in(w, us), in_us, "en-US: {w}"); + assert_eq!(is_correct_in(w, ca), in_ca, "en-CA: {w}"); + assert_eq!(is_correct_in(w, gb), in_gb, "en-GB: {w}"); + assert_eq!(is_correct_for(w, Some("en-AU")), in_gb, "en-AU follows British spelling: {w}"); + } + // The dialect's own spelling comes first, in the word's case. + assert_eq!(suggest_in("colour", 5, us).first().map(String::as_str), Some("color")); + assert_eq!(suggest_in("Colours", 5, us).first().map(String::as_str), Some("Colors")); + assert_eq!(suggest_in("color", 5, gb).first().map(String::as_str), Some("colour")); + assert_eq!(suggest_in("organisation", 5, us).first().map(String::as_str), Some("organization")); + assert_eq!(suggest_in("organisation", 5, ca).first().map(String::as_str), Some("organization")); + assert_eq!(suggest_in("analyze", 5, gb).first().map(String::as_str), Some("analyse")); + assert_eq!(suggest_in("CENTER", 5, gb).first().map(String::as_str), Some("CENTRE")); + assert_eq!(suggest_in("colour's", 5, us).first().map(String::as_str), Some("color's")); + assert_eq!(suggest_in("colour-blind", 5, us).first().map(String::as_str), Some("color-blind")); + // Shared suggestions come in the dialect's spelling, never the other one's. + let gb_sugg = suggest_in("colouur", 8, gb); + assert!(gb_sugg.contains(&"colour".to_string()) && !gb_sugg.contains(&"color".to_string()), "{gb_sugg:?}"); + let us_sugg = suggest_in("colouur", 8, us); + assert!(us_sugg.contains(&"color".to_string()) && !us_sugg.contains(&"colour".to_string()), "{us_sugg:?}"); + // A word in the user dictionary is right in every English. + add_word("zzcolourzz"); + assert!(is_correct_in("zzcolourzz", us)); + // Issues carry the dialect, so suggestions on demand come from the same one. + let issues = check_text("The color is nice.", &[(0..18, gb)]); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!((issues[0].lang, &"The color is nice."[issues[0].start..issues[0].end]), (gb, "color")); + let grammar = check_text("the colour is nice.", &[(0..19, gb)]); + assert!(grammar.iter().any(|i| i.kind == IssueKind::Grammar && i.lang == gb), "{grammar:?}"); + } + + /// The words #40 (folded into #1281) reported as flagged in British English, with their + /// inflections: right in en-GB, and in en-US flagged or not as `en-variants.txt` decides. + #[test] + fn british_words_reported_in_issue_40_are_right_in_en_gb() { + let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa); + // (word, right in en-US, right in en-CA, right in en-GB) + let cases = [ + // Both spellings are right everywhere: `judgment` is also British legal usage, and + // American dictionaries list `judgement` as a variant. + ("judgement", true, true, true), + ("judgements", true, true, true), + ("judgment", true, true, true), + ("judgments", true, true, true), + ("defence", false, true, true), + ("defences", false, true, true), + ("defense", true, false, false), + // `license` is the British verb, so only American English flags `licence`. + ("licence", false, true, true), + ("licences", false, true, true), + ("license", true, true, true), + ("licensed", true, true, true), + ("licenses", true, true, true), + // `program` is British for computing. + ("programme", false, true, true), + ("programmes", false, true, true), + ("program", true, true, true), + ("programs", true, true, true), + // American English accepts both. + ("catalogue", true, true, true), + ("catalogues", true, true, true), + ("catalogued", true, true, true), + ("cataloguing", true, true, true), + ("catalog", true, false, false), + ("catalogs", true, false, false), + ]; + for (w, in_us, in_ca, in_gb) in cases { + assert_eq!(is_correct_in(w, us), in_us, "en-US: {w}"); + assert_eq!(is_correct_in(w, ca), in_ca, "en-CA: {w}"); + assert_eq!(is_correct_in(w, gb), in_gb, "en-GB: {w}"); + } + let text = "The judgement on the defence licence put the programme in the catalogue."; + assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); + assert_eq!(suggest_in("licences", 5, us).first().map(String::as_str), Some("licenses")); + assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs")); + } + + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, + /// with their inflections. + #[test] + fn more_british_spellings_are_right_in_en_gb() { + let gb = ProofLang::EnGb; + assert_eq!(ProofLang::from_tag(Some("en-AU")), gb); + for w in [ + "goitre", + "goitres", + "reconnoitre", + "sceptre", + "sceptres", + "enrol", + "enrols", + "enrolled", + "enrolment", + "fulfil", + "fulfils", + "fulfilled", + "fulfilment", + "instalment", + "instalments", + "skilful", + "skilfully", + "wilful", + "wilfully", + "encyclopaedia", + "encyclopaedias", + ] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + } + let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia."; + assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); + } + + #[test] + fn user_dictionary_accepts_the_other_dialects_spelling() { + add_word("savour"); + assert!(is_correct_in("savour", ProofLang::En), "a word the user added is never flagged"); + assert!(!is_correct_in("savor", ProofLang::EnGb)); + remove_word("savour"); + assert!(!is_correct_in("savour", ProofLang::En)); } #[test] @@ -1416,7 +1662,7 @@ mod tests { let _ = has_dictionary(Some(&s)); let _ = is_correct_for(&s, Some(&s)); let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); - for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] { let _ = check_spelling_in(&s, lang); let _ = check_grammar_in(&s, lang); let _ = is_correct_in(&s, lang); @@ -1460,12 +1706,26 @@ mod tests { } #[test] - fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..9), 0..6)) { - let langs = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other]; + fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..11), 0..6)) { + let langs = [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other]; let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % langs.len()])).collect(); for i in check_text(&s, &spans) { proptest::prop_assert!(i.start <= i.end && i.end <= s.len()); } } } + + proptest::proptest! { + // Suggestions scan the whole word list: fewer cases keep the test quick. + #![proptest_config(proptest::prelude::ProptestConfig::with_cases(24))] + #[test] + fn english_dialect_suggestions_never_panic(s in "[a-zA-Zéß'’ -]{0,16}|(colou?r|cent(er|re)|organi[sz]ation)['’s-]{0,4}") { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + let _ = suggest_in(&s, 4, lang); + let _ = is_correct_in(&s, lang); + let _ = en_variants::preferred(&s, lang); + let _ = en_variants::known_variant(&s, lang, |_| true); + } + } + } }