diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index a4f95d93..347793b6 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -25,11 +25,12 @@ jobs: with: persist-credentials: false # The Language data (craft-i18n `language-data/`), embedded by crates/proof/build.rs. - # Pinned to storytold/craft-i18n#109's merge commit (the SCOWL dictionaries, CP2.1). + # Pinned to storytold/craft-i18n#114's head (#109's SCOWL dictionaries + judgment/judgement, + # CP2.1); TODO(CP2.1): move to #114's merge commit once it merges. - uses: actions/checkout@v4 with: repository: storytold/craft-i18n - ref: efb1127ac65b238ccfe7df1893f33e91b924b96e + ref: c239f934ace90b7939afaceb23b472ceeafd6b8a path: craft-i18n persist-credentials: false - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable diff --git a/crates/proof/README.md b/crates/proof/README.md index 470b4c37..b1672faa 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | Language | Spelling | Grammar | Hyphenation | |---|---|---|---| -| English (en-US, en-GB and the British-spelling Englishes such as en-AU, en-CA) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | +| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | | German | — | — | TeX patterns | @@ -70,11 +70,16 @@ English spelling uses the first of these that is there; `english_source(lang)` s 3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` + `spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling - Englishes read en-GB. Suggestions come from the English's own dictionary; a word is right when - any of the three knows it (or `en-extra.txt` does), unless `en-variants.txt` says it is the other - dialect's spelling, and this dialect's spellings of their words are right too. SCOWL's size-60 - lists keep one spelling per dialect, and Word accepts more (`organize` and `organise` in - British English), so `en-variants.txt` stays. With SCOWL embedded the system locations aren't + Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own + dictionary knows it (or `en-extra.txt` does), never because another English's does + (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same + dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so + British English also takes the `-ize` spellings of its `-ise` words (`organize`, + `realization`), and `en-variants.txt` flags the other dialect's spellings and accepts this + dialect's other spellings of known words (`judgement` in en-US). English with no dialect + (`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only + between two lower-case words of three letters or more, after every one-word candidate, and no + name or abbreviation (`Col`) for a lower-case word. With SCOWL embedded the system locations aren't searched. 4. **System Hunspell:** the English's own pair in the system locations. Never another English's: an `en_US` dictionary would mark British spellings, so en-GB and en-CA without their own use diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 94175159..06433963 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -319,6 +319,10 @@ pub struct AffixDict { only_in_compound: Mask, /// `COMPOUNDMIN`, in characters. compound_min: usize, + /// The first characters of the stems a compound part can be (their flags are in a + /// `COMPOUNDRULE`), so a word starting otherwise is no compound: SCOWL's ordinals start with + /// a digit, and a misspelled word costs one lookup here instead of one per cut. + compound_firsts: Vec, } fn hash_str(s: &str) -> u64 { @@ -722,6 +726,7 @@ impl AffixDict { ..AffixDict::default() }; d.offs.push(0); + let compound_flags = d.compound_rules.iter().flatten().fold(Mask::EMPTY, |m, (bit, _)| m | *bit); let mut set_ids: HashMap = HashMap::new(); let mut i = 0; while i < stems.len() { @@ -739,6 +744,12 @@ impl AffixDict { if usize::from(id) == d.sets.len() { d.sets.push(mask); } + if mask.meets(compound_flags) + && let Some(c) = word.chars().next() + && !d.compound_firsts.contains(&c) + { + d.compound_firsts.push(c); + } d.words.push_str(word); d.offs.push(u32::try_from(d.words.len()).map_err(|_| "stems too large".to_string())?); d.flags.push(id); @@ -1002,26 +1013,35 @@ impl AffixDict { /// Is `w` a compound of two or more stems (no affixes) whose flags spell out one of the /// `COMPOUNDRULE` patterns, each part at least `COMPOUNDMIN` characters? `21st` is `2/m` + /// `1st/p` under `n*mp`. A dynamic programme over (byte offset, token, parts so far), so a - /// hostile pattern costs at most a few thousand lookups. + /// hostile pattern costs at most a few thousand lookups. A forbidden word is never rebuilt + /// from its parts. fn compound(&self, w: &str, deny: Mask) -> bool { - if self.compound_rules.is_empty() || w.len() > MAX_COMPOUND_BYTES { + if self.compound_rules.is_empty() + || w.len() > MAX_COMPOUND_BYTES + || !w.chars().next().is_some_and(|c| self.compound_firsts.contains(&c)) + || self.stem_mask(w).is_some_and(|m| m.meets(self.forbidden)) + { return false; } // Char boundaries of `w`, `0` and `w.len()` included. let cuts: Vec = w.char_indices().map(|(i, _)| i).chain(std::iter::once(w.len())).collect(); let n = cuts.len(); - // Flag mask of each part `w[cuts[a]..cuts[b]]` that is a stem, by (a, b). - let mut parts: Vec> = vec![None; n * n]; - for a in 0..n { - for b in a.saturating_add(self.compound_min)..n { - let (Some(&x), Some(&y)) = (cuts.get(a), cuts.get(b)) else { continue }; - let mask = w.get(x..y).and_then(|p| self.stem_mask(p)).filter(|m| !m.meets(self.forbidden | deny)); - if let Some(slot) = parts.get_mut(a * n + b) { - *slot = mask; - } + // Flag mask of each part `w[cuts[a]..cuts[b]]`, by (a, b), looked up when a rule first + // reaches cut `a` (`None`: not looked up yet; `Some(None)`: not a stem). Only the parts + // after a cut a rule reaches are looked up, so a word with no first part costs `n`. + let mut parts: Vec>> = vec![None; n * n]; + let min = self.compound_min; + let mut part = |a: usize, b: usize| -> Option { + if b < a.saturating_add(min) { + return None; } - } - self.compound_rules.iter().any(|rule| compound_matches(rule, &parts, n)) + let slot = parts.get_mut(a * n + b)?; + *slot.get_or_insert_with(|| { + let (Some(&x), Some(&y)) = (cuts.get(a), cuts.get(b)) else { return None }; + w.get(x..y).and_then(|p| self.stem_mask(p)).filter(|m| !m.meets(self.forbidden | deny)) + }) + }; + self.compound_rules.iter().any(|rule| compound_matches(rule, &mut part, n)) } /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must @@ -1259,11 +1279,13 @@ impl AffixDict { if !self.ignore.is_empty() { out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); } + if !self.only_in_compound.is_empty() + && let Some(c) = self.flag_char(self.only_in_compound) + { + out.push_str(&format!("ONLYINCOMPOUND {c}\n")); + } if !self.compound_text.is_empty() { out.push_str(&format!("COMPOUNDMIN {}\n", self.compound_min)); - if let Some(c) = self.flag_char(self.only_in_compound) { - out.push_str(&format!("ONLYINCOMPOUND {c}\n")); - } for p in &self.compound_text { out.push_str(&format!("COMPOUNDRULE {p}\n")); } @@ -1403,11 +1425,10 @@ impl AffixDict { } } -/// Does a compound rule's token list match a split of the word into two or more parts? `parts` -/// holds, for each pair of char cuts `(a, b)` at `a * n + b`, the flags of the stem -/// `w[a..b]` (`None`: not a stem). States are (cut, token, parts so far capped at 2), each -/// visited once. -fn compound_matches(rule: &[(Mask, Repeat)], parts: &[Option], n: usize) -> bool { +/// Does a compound rule's token list match a split of the word into two or more parts? `part(a, +/// b)` is the flags of the stem between char cuts `a` and `b` (`None`: not a stem). States are +/// (cut, token, parts so far capped at 2), each visited once. +fn compound_matches(rule: &[(Mask, Repeat)], part: &mut impl FnMut(usize, usize) -> Option, n: usize) -> bool { let t = rule.len(); let idx = |cut: usize, tok: usize, k: usize| (cut * (t + 1) + tok) * 3 + k; let mut seen = vec![false; n * (t + 1) * 3]; @@ -1428,7 +1449,7 @@ fn compound_matches(rule: &[(Mask, Repeat)], parts: &[Option], n: usize) - stack.push((cut, tok + 1, k)); } for next in cut + 1..n { - if parts.get(cut * n + next).copied().flatten().is_some_and(|m| m.meets(bit)) { + if part(cut, next).is_some_and(|m| m.meets(bit)) { let k = (k + 1).min(2); stack.push((next, if repeat == Repeat::Any { tok } else { tok + 1 }, k)); } @@ -1963,6 +1984,24 @@ pub(crate) mod tests { assert!(d.check("cdab")); } + /// craft-libs#483 review: a forbidden word isn't rebuilt from its parts. + #[test] + fn compound_never_rebuilds_a_forbidden_word() { + let d = AffixDict::from_hunspell("SET UTF-8\nFORBIDDENWORD X\nCOMPOUNDMIN 1\nCOMPOUNDRULE a*\n", "2\nx/a\nxx/X\n", &[]).unwrap(); + assert!(!d.check("xx") && d.check("xxx") && d.check("x")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(!back.check("xx") && back.check("xxx")); + } + + /// craft-libs#483 review: `ONLYINCOMPOUND` survives a round trip without compound rules too. + #[test] + fn onlyincompound_round_trips_without_compound_rules() { + let d = AffixDict::from_hunspell("SET UTF-8\nONLYINCOMPOUND c\n", "2\nfoo\nqux/c\n", &[]).unwrap(); + assert!(d.check("foo") && !d.check("qux")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("foo") && !back.check("qux"), "a part-only stem stays a part"); + } + #[test] fn hostile_compound_rules_are_refused_or_cheap() { for bad in ["COMPOUNDRULE *a", "COMPOUNDRULE (ab)", "COMPOUNDRULE a**", "COMPOUNDMIN x", "ONLYINCOMPOUND ab"] { diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 1456f81c..5029c310 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -43,6 +43,8 @@ const COST_SWAP: u32 = 14; const COST_DROP_OR_ADD: u32 = 15; pub(crate) const COST_MISTYPE: u32 = 16; const COST_SCAN: u32 = 20; +/// A missing space ranked after every one-word candidate (English: [`Split::Custom`]). +pub(crate) const COST_SPLIT_LAST: u32 = COST_SCAN + 4 * 3 + 1; /// What a language adds to its dictionary's tables. pub(crate) struct Tuning { @@ -62,6 +64,9 @@ pub(crate) struct Tuning { pub map_cost: fn(char, char) -> u32, /// The cost of typing letter `c` at `i` of the word (`chars`, lower case) instead. pub mistype_cost: fn(&[char], usize, char) -> u32, + /// Offer a word only the dictionary capitalizes (a name, an abbreviation: `Col`, `Wis`) only + /// when the word itself is capitalized, and never as half of a missing space (`Che Que`). + pub strict_case: bool, } /// Where a missing space may go. @@ -78,8 +83,17 @@ pub(crate) enum Split { impl Tuning { /// The dictionary's tables alone (Czech, Indonesian, Spanish). - pub(crate) const DEFAULT: Tuning = - Tuning { split: Split::Anywhere, alphabet: "", map: &[], rep: &[], rep_cost: COST_REP, max_map_changes: 4, map_cost, mistype_cost }; + pub(crate) const DEFAULT: Tuning = Tuning { + split: Split::Anywhere, + alphabet: "", + map: &[], + rep: &[], + rep_cost: COST_REP, + max_map_changes: 4, + map_cost, + mistype_cost, + strict_case: false, + }; } /// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the @@ -107,6 +121,10 @@ struct Collector<'a> { /// Suggestion → (cost, order found). found: HashMap, seq: usize, + /// May a word the dictionary only has capitalized be offered ([`Tuning::strict_case`])? + names: bool, + /// May half of a missing space be one ([`Tuning::strict_case`])? + split_names: bool, } impl Collector<'_> { @@ -125,9 +143,12 @@ impl Collector<'_> { } let mut words = Vec::new(); let mut cost = cost; + let names = if cand.contains(' ') { self.split_names } else { self.names }; for part in cand.split(' ') { if self.d.contains_suggestible(part) { words.push(part.to_string()); + } else if !names { + return; } else { let c = capitalize(part); if !self.d.contains_suggestible(&c) { @@ -278,7 +299,9 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V let case = casing(word); let input = word.to_lowercase(); let chars: Vec = input.chars().collect(); - let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + let names = !t.strict_case || case != Casing::Lower; + let split_names = !t.strict_case; + let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names }; add_case(&mut col, word, &input); let map = groups(d, t); add_map_slips(&mut col, &chars, &map, t); @@ -479,6 +502,9 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { if lf.len().abs_diff(chars.len()) > 2 { continue; } + if !col.names && form.chars().any(char::is_uppercase) { + continue; + } let dist = weighted_distance(&lf, chars, &related, limit); if dist <= limit && lf.as_slice() != chars { col.keep(form, COST_SCAN + dist * 3); @@ -504,7 +530,8 @@ mod tests { #[test] fn a_cheaper_later_offer_wins() { let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap(); - let mut col = Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0 }; + let mut col = + Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names: true, split_names: true }; col.offer("cafe", COST_DROP_DIACRITIC); col.offer("cafe", COST_REP); col.offer("cafe", COST_SCAN); diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index d614ae8b..17a9c0c0 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -5,8 +5,10 @@ //! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder). //! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2). //! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded -//! from craft-i18n at build time ([`crate::scowl`]); a word any of the three knows, or the -//! supplement ([`EXTRA`]), is right. When it is there, the system locations aren't searched. +//! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also +//! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is +//! right; never one only another English knows. When it is there, the system locations aren't +//! searched. //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the //! web). //! 5. **Built-in word list**: [`WORDS`], the whole public-domain Moby word list compiled into @@ -32,6 +34,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, OnceLock, PoisonError, RwLock}; use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; use crate::dict::Dictionary; use crate::{Options, ProofLang}; @@ -242,7 +245,7 @@ pub(crate) fn reload() { pub(crate) fn source(lang: ProofLang) -> EnglishSource { match speller(lang) { Speller::Hunspell(_, dic) => EnglishSource::Hunspell(dic), - Speller::Scowl(_) => EnglishSource::Embedded, + Speller::Scowl(..) => EnglishSource::Embedded, Speller::List => EnglishSource::CoreList, } } @@ -257,8 +260,9 @@ pub(crate) fn load_error(lang: ProofLang) -> Option { enum Speller { /// A dictionaries-folder or system Hunspell dictionary, and its `.dic` file. Hunspell(Arc, PathBuf), - /// The SCOWL dictionaries, `lang`'s first ([`crate::scowl::all`]). - Scowl(Vec<&'static AffixDict>), + /// `lang`'s own SCOWL dictionary (every English's, for English with no dialect: [`generic`]), + /// and whether British `-ize` spellings are right too ([`ise_spelling`]). + Scowl(Vec<&'static AffixDict>, bool), /// The built-in word list. List, } @@ -274,8 +278,22 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { if let Some((d, dic)) = found { return Speller::Hunspell(d, dic); } - let scowl = crate::scowl::all(lang); - if crate::scowl::dict(lang).is_some() && !scowl.is_empty() { Speller::Scowl(scowl) } else { Speller::List } + match crate::scowl::dict(lang) { + Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)), + None => Speller::List, + } +} + +/// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English +/// is right, as in the built-in list, which has both spellings. +fn generic(s: Speller) -> Speller { + match s { + Speller::Scowl(..) => { + let all: Vec<_> = [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa].into_iter().filter_map(crate::scowl::dict).collect(); + Speller::Scowl(all, true) + } + other => other, + } } impl Speller { @@ -283,7 +301,7 @@ impl Speller { fn knows(&self, w: &str) -> bool { match self { Speller::Hunspell(d, _) => affix_correct(&[d.as_ref()], w, false), - Speller::Scowl(ds) => affix_correct(ds, w, true), + Speller::Scowl(ds, ise) => affix_correct(ds, w, true) || (*ise && ise_spelling(w).is_some_and(|x| affix_correct(ds, &x, true))), Speller::List => list_correct(w), } } @@ -292,13 +310,14 @@ impl Speller { fn knows_lower(&self, p: &str) -> bool { match self { Speller::Hunspell(d, _) => d.check(p), - Speller::Scowl(ds) => ds.iter().any(|d| d.check(p)), + Speller::Scowl(ds, ise) => ds.iter().any(|d| d.check(p) || (*ise && ise_spelling(p).is_some_and(|x| d.check(&x)))), Speller::List => crate::known_in_word_list(p), } } } -/// Is `word` spelled right in `lang` (an English), against its dictionary ([`speller`]); with +/// Is `word` spelled right in `lang` (an English), against its own dictionary ([`speller`]); with +/// `policy` false, English with no dialect: any English's SCOWL dictionary ([`generic`]). With /// `policy`, also by the dialect policy of `en-variants.txt`: the other dialect's spellings are /// wrong and this dialect's spellings of known words right. The user dictionary overrules both. pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bool) -> bool { @@ -312,11 +331,36 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo if policy && parts.iter().any(|p| crate::en_variants::is_foreign(p, lang)) { return false; } - let s = speller(lang); + let s = if policy { speller(lang) } else { generic(speller(lang)) }; s.knows(&w) || (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x)))) } +/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/ +/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in +/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is +/// swapped (`citizen` → `citisen`, which no dictionary knows, so that does no harm). +pub(crate) fn ise_spelling(w: &str) -> Option { + let b = w.as_bytes(); + let mut out = b.to_vec(); + let mut changed = false; + for i in 1..b.len() { + let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c)); + if after + && matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y')) + && let Some(c) = out.get_mut(i) + { + match *c { + b'z' => *c = b's', + b'Z' => *c = b'S', + _ => continue, + } + changed = true; + } + } + if changed { String::from_utf8(out).ok() } else { None } +} + /// Against affix dictionaries (any of them): the word, or each part of a hyphenated one; with /// `supplement`, our supplement's words count too. fn affix_correct(ds: &[&AffixDict], w: &str, supplement: bool) -> bool { @@ -343,12 +387,22 @@ fn list_correct(w: &str) -> bool { /// Spelling suggestions in `lang` (an English), from its dictionary ([`speller`]). pub(crate) fn suggest(word: &str, max: usize, lang: ProofLang) -> Vec { match speller(lang) { - Speller::Hunspell(d, _) => crate::cs::suggest::suggest_in(&d, word, max), - Speller::Scowl(ds) => ds.first().map_or_else(Vec::new, |d| crate::cs::suggest::suggest_in(d, word, max)), + Speller::Hunspell(d, _) => crate::affix_suggest::suggest_in(&d, word, max, &TUNING), + Speller::Scowl(ds, _) => ds.first().map_or_else(Vec::new, |d| crate::affix_suggest::suggest_in(d, word, max, &TUNING)), Speller::List => list_suggest(word, max), } } +/// English suggestions from an affix dictionary: its own tables, a missing space only between two +/// words of three letters or more and after every one-word candidate (`publically` → `publicly` +/// before `public ally`), and no abbreviation or name offered for a lower-case word (`col`, `Wis`). +const TUNING: Tuning = Tuning { split: Split::Custom(split_cost), strict_case: true, ..Tuning::DEFAULT }; + +/// The cost of a missing space in English ([`TUNING`]). +fn split_cost(a: &str, b: &str) -> Option { + (a.chars().count() >= 3 && b.chars().count() >= 3).then_some(crate::affix_suggest::COST_SPLIT_LAST) +} + /// Letters an edit may add or put in a word's place. const ALPHABET: &str = "abcdefghijklmnopqrstuvwxyz'"; @@ -527,8 +581,12 @@ mod tests { assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}"); let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { - assert!(matches!(&fallback, Speller::Scowl(ds) if ds.len() == 3), "{lang:?}"); - assert!(fallback.knows("colour") && fallback.knows("color") && fallback.knows("21st") && !fallback.knows("becuase")); + assert!( + matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), + "{lang:?}" + ); + assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}"); + assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only"); } else { assert!(matches!(fallback, Speller::List), "{lang:?}"); } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index dac71a62..421c3db5 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -614,8 +614,9 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { || in_user_dictionary(&w.to_lowercase()) } -/// [`is_correct`] with the File › Options › Proofing choices: against American English's -/// dictionary ([`english_source`]), without the dialect policy. +/// [`is_correct`] with the File › Options › Proofing choices: English with no dialect, so a word +/// any English spells so is right (`color` and `colour`; American English's dictionary when it +/// isn't SCOWL: [`english_source`]), without the dialect policy. pub fn is_correct_with(word: &str, opts: &Options) -> bool { english::is_correct(word, ProofLang::En, opts, false) } @@ -1667,7 +1668,6 @@ mod tests { assert!(d.len() > 40_000, "{l:?}: {} stems", d.len()); assert_eq!(english_source(l), Some(EnglishSource::Embedded)); assert_eq!(dictionary_load_error(l), None); - assert_eq!(scowl::all(l).len(), 3); } assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB"); let notice = scowl::notice().unwrap(); @@ -1694,6 +1694,34 @@ mod tests { assert_eq!(suggest_in("color", 5, ProofLang::EnGb).first().map(String::as_str), Some("colour")); } + /// craft-libs#483 review: English suggestions offer a missing space only between two + /// lower-case words of three letters or more, after every one-word candidate, and no word the + /// dictionary only has capitalized (`Col`, `Wis`) for a lower-case word. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn english_suggestions_skip_junk_splits_and_names() { + for (w, l) in [ + ("colour", ProofLang::En), + ("theatr", ProofLang::En), + ("publically", ProofLang::En), + ("cheque", ProofLang::En), + ("colr", ProofLang::En), + ("wiss", ProofLang::En), + ("theatr", ProofLang::EnGb), + ] { + let got = suggest_in(w, 8, l); + assert!(!got.is_empty(), "{w}"); + assert!(got.iter().all(|s| !s.chars().any(char::is_uppercase)), "{w} in {l:?}: {got:?}"); + let first_split = got.iter().position(|s| s.contains(' ')).unwrap_or(got.len()); + assert!(got.iter().skip(first_split).all(|s| s.contains(' ')), "{w}: a split before a word: {got:?}"); + assert!(got.iter().filter(|s| s.contains(' ')).all(|s| s.split(' ').all(|p| p.chars().count() >= 3)), "{w}: {got:?}"); + } + assert_eq!(suggest_in("publically", 5, ProofLang::En).first().map(String::as_str), Some("publicly")); + assert_eq!(suggest_in("colour", 5, ProofLang::En).first().map(String::as_str), Some("color")); + assert_eq!(suggest_in("theatr", 5, ProofLang::EnGb).first().map(String::as_str), Some("theatre")); + assert!(suggest_in("Londn", 5, ProofLang::En).contains(&"London".to_string()), "a capitalized word may get a name"); + } + /// `en-variants.txt` stays with SCOWL: SCOWL's size-60 lists keep one spelling per dialect, /// and Word accepts more (both `-ize` and `-ise` in British English, `judgement` in American). #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] @@ -1707,6 +1735,52 @@ mod tests { assert!(is_correct_in("encyclopaedia", ProofLang::EnGb) && is_correct_in("encyclopedia", ProofLang::EnGb)); } + /// craft-libs#483 review: each English checks against its own SCOWL dictionary only. A stem + /// one English's dictionary has and another's lacks (`colourblind`, `greyscale` in en-GB; + /// `colorblind`, `esophagi` in en-US) doesn't cross, unless our dialect policy or the `-ize` + /// spellings of British English say so. Sampled from every pair of Englishes. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn scowl_dialects_dont_cross() { + let langs = [ + (ProofLang::En, crate::data::EN_US_SPELLING), + (ProofLang::EnGb, crate::data::EN_GB_SPELLING), + (ProofLang::EnCa, crate::data::EN_CA_SPELLING), + ]; + for (from, dic) in langs { + for (to, _) in langs { + if from == to { + continue; + } + let d = scowl::dict(to).unwrap(); + let widened = |w: &str| { + let x = english::ise_spelling(w); + (scowl::is_british(to) && x.as_deref().is_some_and(|x| d.check(x))) + || en_variants::known_variant(w, to, |x| d.check(x)) + || known_in_supplement(w) + }; + let only: Vec<&str> = dic + .lines() + .skip(1) + .map(|l| l.split('/').next().unwrap_or_default()) + .filter(|w| w.len() > 3 && w.chars().all(|c| c.is_ascii_lowercase()) && !d.check(w) && !widened(w)) + .collect(); + assert!(only.len() >= 40, "{from:?} → {to:?}: {}", only.len()); + let step = only.len() / 40; + let crossed: Vec<&str> = only.iter().step_by(step.max(1)).copied().filter(|w| is_correct_in(w, to)).collect(); + assert!(crossed.is_empty(), "{from:?} words right in {to:?}: {crossed:?}"); + } + } + for (w, l) in [("colourblind", ProofLang::En), ("greyscale", ProofLang::En), ("colorblind", ProofLang::EnGb), ("esophagi", ProofLang::EnGb)] { + assert!(!is_correct_in(w, l), "{w} in {l:?}"); + } + for (w, l) in + [("colorblind", ProofLang::En), ("colourblind", ProofLang::EnGb), ("organizational", ProofLang::EnGb), ("realized", ProofLang::EnGb)] + { + assert!(is_correct_in(w, l), "{w} in {l:?}"); + } + } + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] #[test] fn user_dictionary_accepts_the_other_dialects_spelling() { diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs index 5db78ec3..e050bb60 100644 --- a/crates/proof/src/no_data.rs +++ b/crates/proof/src/no_data.rs @@ -48,7 +48,7 @@ fn nothing_is_embedded() { assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty()); assert!(crate::scowl::notice().is_none()); for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { - assert!(crate::scowl::dict(l).is_none() && crate::scowl::all(l).is_empty() && crate::scowl::load_error(l).is_none()); + assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none()); } assert!(dict::Dictionary::en_us().is_empty()); for p in [patterns::Patterns::en_us(), patterns::Patterns::pl(), patterns::Patterns::pt(), patterns::Patterns::it(), patterns::Patterns::de()] { diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index c3b52a02..8d79c1d1 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -4,10 +4,11 @@ //! //! Each English has its own: American English reads en-US, the British-spelling Englishes //! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions -//! come from its own dictionary. A word is right when any of the three knows it, unless -//! `en-variants.txt` says it is the other dialect's spelling: SCOWL picks one spelling per -//! dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word accepts more. An -//! English dictionary in the dictionaries folder comes first; a SCOWL dictionary that isn't +//! come from its own dictionary, and a word is right when its own dictionary knows it: the other +//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL +//! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word +//! accepts more, so British English also takes the `-ize` spellings of its `-ise` words, and +//! `en-variants.txt` widens each dialect by its pairs (`crate::english`). An English dictionary in the dictionaries folder comes first; a SCOWL dictionary that isn't //! embedded, or fails to load, leaves that English to the next source (`crate::english`). //! //! SCOWL is under its own permissive licence and the affix files under Ispell's BSD licence; @@ -48,18 +49,9 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> { loaded.as_ref().ok() } -/// Every embedded SCOWL dictionary that loads, `lang`'s first: the shared English word list the -/// dialect policy (`en-variants.txt`) is applied to. -pub(crate) fn all(lang: ProofLang) -> Vec<&'static AffixDict> { - let mut out: Vec<&'static AffixDict> = dict(lang).into_iter().collect(); - for other in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { - if source(other).0 != source(lang).0 - && let Some(d) = dict(other) - { - out.push(d); - } - } - out +/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)? +pub(crate) fn is_british(lang: ProofLang) -> bool { + source(lang).0 == 1 } /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it.