Merge main into feat/doc-pcraft-pixels

This commit is contained in:
dotCooCoo committed 2026-11-02 18:30:48 -08:00
commit 3f6f29693c
33 files changed
+4148 -377

No files matched your search

+11 -9
View File
@@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = "<commit>
| Language | Spelling | Grammar | Hyphenation |
|---|---|---|---|
| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
| Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) |
| Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise |
| German | — | — | TeX patterns (embedded, or a Language pack) |
@@ -79,7 +79,7 @@ pub fn packs_dir() -> Option<PathBuf>; // +
pub fn installed() -> Vec<InstalledPack>; // InstalledPack { locale, name, licence, path }, by locale
pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/*
pub fn load_errors() -> Vec<(String, String)>; // + (locale or folder, reason): packs skipped, pattern files that failed
pub fn embedded_locales() -> &'static [&'static str]; // + what this build embeds ("en-us", "de", "pl", "pt", "it", "ru" by default; "en-us" with CRAFT_PROOF_EMBED=en)
pub fn embedded_locales() -> &'static [&'static str]; // + locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en)
// craft_proof (crate root)
pub enum ProofLang { …, Pack(PackLocale), Other } // + Pack: any language without tools of its own (nl, fr, sr-Latn…)
@@ -151,14 +151,16 @@ English spelling uses the first of these that is there; `english_source(lang)` s
or in WordCraft's own `dictionaries` folder. The user's choice comes first.
2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2).
3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from
`CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` +
`spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling
Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own
`CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`,
`spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU
(Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the
other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own
dictionary knows it (or `en-extra.txt` does), never because another English's does
(`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same
dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so
(`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian
English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only).
Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so
British English also takes the `-ize` spellings of its `-ise` words that American or Canadian
English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this
English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this
dialect's other spellings of known words (`judgement` in en-US). English with no dialect
(`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only
between two lower-case words of three letters or more, after every one-word candidate, and no
@@ -199,7 +201,7 @@ craft-i18n's copy.
The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's
notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window.
Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic`
Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic`
and the notice); the Moby list (432,905 bytes) stays for hyphenation.
## Check
+15 -4
View File
@@ -16,9 +16,8 @@
//! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded:
//! it is treated like a folder without the data.
//!
//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as
//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL
//! notice is embedded once (the four upstream READMEs differ only in their title and build line).
//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is
//! embedded once (the four upstream READMEs differ only in their title and build line).
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
@@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[
("EN_GB_SPELLING", "en-gb/spelling.dic", false),
("EN_CA_AFF", "en-ca/spelling.aff", false),
("EN_CA_SPELLING", "en-ca/spelling.dic", false),
("EN_AU_AFF", "en-au/spelling.aff", false),
("EN_AU_SPELLING", "en-au/spelling.dic", false),
("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false),
("EN_US_PAT", "en-us/hyphenation.pat", false),
("EXTRA_EN", "en-us/en-extra.txt", false),
@@ -152,8 +153,18 @@ fn main() {
files.sort();
for f in files {
println!("cargo::rerun-if-changed={}", f.display());
// `include_str!` fails the whole build on a file that isn't UTF-8 (a Latin-1
// licence text) and can't name a path that isn't either: leave such a file out.
let Some(path) = f.to_str() else {
println!("cargo::{level}=CRAFT_I18N_DIR: the path {} is not UTF-8; its licence text is not embedded", f.display());
continue;
};
if let Err(e) = std::fs::read_to_string(&f) {
println!("cargo::{level}=CRAFT_I18N_DIR: {path} can't be read as text ({e}); its licence text is not embedded");
continue;
}
let title = format!("{locale}: {}", f.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default());
let _ = write!(licences, "({title:?}, include_str!({:?})), ", f.display().to_string());
let _ = write!(licences, "({title:?}, include_str!({path:?})), ");
}
}
}
+182
View File
@@ -0,0 +1,182 @@
//! Compare the affix engine with another spell checker, black-box: load a Hunspell `.aff`/`.dic`
//! pair and answer for words, or make candidate compounds to ask both about.
//!
//! ```sh
//! # One word per line in, `word<TAB>1` (right) or `word<TAB>0` out, and the time taken on stderr.
//! cargo run --release -p craft-proof --example affix_check -- check de_DE.aff de_DE.dic < words.txt
//! # 2000 candidates (forms of random stems glued together, some with a letter changed), seed 7.
//! cargo run --release -p craft-proof --example affix_check -- gen de_DE.aff de_DE.dic 2000 7 > words.txt
//! # Single words instead: forms of random stems, and stems with another stem's ending.
//! cargo run --release -p craft-proof --example affix_check -- single de_DE.aff de_DE.dic 2000 7 > words.txt
//! # Suggestions: `word<TAB>microseconds<TAB>suggestion,suggestion…` per line.
//! cargo run --release -p craft-proof --example affix_check -- suggest de_DE.aff de_DE.dic < typos.txt
//! ```
//!
//! The answers are compared with `hunspell -d <path without extension> -l` on the same words
//! (the words it lists are the wrong ones). Nothing of the other checker is read but its answers.
use std::io::{BufRead, Write};
use craft_proof::affix::AffixDict;
fn load(aff: &str, dic: &str) -> Result<AffixDict, String> {
let aff = std::fs::read(aff).map_err(|e| format!("{aff}: {e}"))?;
let dic = std::fs::read(dic).map_err(|e| format!("{dic}: {e}"))?;
let (aff, dic) = craft_proof::hunspell::decode(&aff, &dic)?;
AffixDict::from_hunspell(&aff, &dic, &[])
}
/// xorshift64*: a reproducible stream of numbers, enough to pick stems.
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
self.0.wrapping_mul(0x2545_f491_4f6c_dd1d)
}
fn below(&mut self, n: usize) -> usize {
if n == 0 { 0 } else { (self.next() % n as u64) as usize }
}
}
/// `w` with its first letter in lower case (a compound's later parts).
fn decapitalize(w: &str) -> String {
let mut c = w.chars();
c.next().map(|f| f.to_lowercase().chain(c).collect()).unwrap_or_default()
}
/// Attempts per requested word before a generator gives up (a dictionary that yields none).
const MAX_TRIES_PER_WORD: usize = 1000;
fn generate(d: &AffixDict, n: usize, seed: u64) -> Vec<String> {
let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect();
let mut rng = Rng(seed.max(1));
let form = |rng: &mut Rng| -> String {
let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default();
let forms = d.forms(s);
forms.get(rng.below(forms.len())).cloned().unwrap_or_else(|| s.to_string())
};
let mut out = Vec::with_capacity(n);
// A dictionary without alphabetic stems yields no word: give up instead of looping forever.
let mut tries = n.saturating_mul(MAX_TRIES_PER_WORD);
while out.len() < n && !stems.is_empty() && tries > 0 {
tries -= 1;
let parts = 2 + rng.below(2);
let mut w = form(&mut rng);
for _ in 1..parts {
if rng.below(4) == 0 {
w.push('s');
}
w.push_str(&decapitalize(&form(&mut rng)));
}
if rng.below(3) == 0 {
// A letter dropped, doubled or changed: mostly wrong words.
let chars: Vec<char> = w.chars().collect();
let i = rng.below(chars.len());
let mut m: String = chars.iter().take(i).collect();
match rng.below(3) {
0 => {}
1 => m.extend(chars.get(i).into_iter().chain(chars.get(i))),
_ => m.push('e'),
}
m.extend(chars.iter().skip(i + 1));
w = m;
}
// Letters only: the other checker's command line splits words at anything else.
if w.chars().count() >= 4 && w.len() <= 100 && w.chars().all(char::is_alphabetic) {
out.push(w);
}
}
out
}
/// Single words (no compounds): forms of random stems, and stems with another stem's ending
/// (mostly wrong, some right: an ending the stem's own homonym allows or not).
fn single(d: &AffixDict, n: usize, seed: u64) -> Vec<String> {
let stems: Vec<&str> = d.stems().filter(|s| s.chars().all(char::is_alphabetic)).collect();
let mut rng = Rng(seed.max(1));
let mut endings: Vec<String> = Vec::new();
for _ in 0..200 {
let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default();
endings.extend(d.forms(s).iter().filter_map(|f| f.strip_prefix(s)).filter(|e| !e.is_empty()).map(str::to_string));
}
endings.sort();
endings.dedup();
let mut out = Vec::with_capacity(n);
let mut tries = n.saturating_mul(MAX_TRIES_PER_WORD);
while out.len() < n && !stems.is_empty() && tries > 0 {
tries -= 1;
let s = stems.get(rng.below(stems.len())).copied().unwrap_or_default();
let forms = d.forms(s);
if let Some(f) = forms.get(rng.below(forms.len())) {
out.push(f.clone());
}
if let Some(e) = endings.get(rng.below(endings.len())) {
out.push(format!("{s}{e}"));
}
out.retain(|w| w.chars().all(char::is_alphabetic));
}
// Letters only: the other checker's command line splits words at anything else.
out.retain(|w| w.chars().all(char::is_alphabetic));
out.truncate(n);
out
}
fn main() -> Result<(), String> {
let args: Vec<String> = std::env::args().skip(1).collect();
match args.as_slice() {
[cmd, aff, dic] if cmd == "check" => {
let d = load(aff, dic)?;
let words: Vec<String> = std::io::stdin().lock().lines().map_while(Result::ok).collect();
let t = std::time::Instant::now();
let mut slowest: Vec<(std::time::Duration, &str)> = Vec::new();
let answers: Vec<bool> = words
.iter()
.map(|w| {
let t = std::time::Instant::now();
let ok = d.check(w);
slowest.push((t.elapsed(), w));
ok
})
.collect();
let took = t.elapsed();
slowest.sort_by_key(|s| std::cmp::Reverse(s.0));
for (t, w) in slowest.iter().take(3) {
eprintln!("slow: {w} {t:?}");
}
let mut out = std::io::stdout().lock();
for (w, ok) in words.iter().zip(answers) {
writeln!(out, "{w}\t{}", u8::from(ok)).map_err(|e| e.to_string())?;
}
eprintln!("{} words in {took:?}", words.len());
Ok(())
}
[cmd, aff, dic] if cmd == "suggest" => {
let d = load(aff, dic)?;
let mut out = std::io::stdout().lock();
for w in std::io::stdin().lock().lines().map_while(Result::ok) {
let t = std::time::Instant::now();
let s = d.suggest(&w, 8);
writeln!(out, "{w}\t{}\t{}", t.elapsed().as_micros(), s.join(",")).map_err(|e| e.to_string())?;
}
Ok(())
}
[cmd, aff, dic, n, seed] if cmd == "gen" || cmd == "single" => {
let d = load(aff, dic)?;
let n: usize = n.parse().map_err(|_| "n is a number")?;
let seed: u64 = seed.parse().map_err(|_| "seed is a number")?;
let words = if cmd == "gen" { generate(&d, n, seed) } else { single(&d, n, seed) };
if words.len() < n {
return Err(format!("only {} of {n} words could be generated from this dictionary", words.len()));
}
for w in words {
println!("{w}");
}
Ok(())
}
_ => Err("usage: affix_check check|suggest <aff> <dic> < words | affix_check gen|single <aff> <dic> <n> <seed>".into()),
}
}
+1041 -164
View File
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+96 -8
View File
@@ -25,8 +25,12 @@ use crate::affix::{AffixDict, Casing, capitalize, casing};
/// Longest word suggestions are looked for, in letters.
pub(crate) const MAX_LETTERS: usize = 40;
const MAX_MAP_VARIANTS: usize = 4096;
/// Most candidates kept waiting for the compound check in one call (the rest are dropped).
const MAX_DEFERRED: usize = 1 << 14;
/// Stems the scan of round 6 expands.
const SCAN_STEMS: usize = 48;
/// Most affix rules tried in all to make those stems' forms (Mongolian has 12,000 rules).
const SCAN_FORMS_WORK: usize = 1 << 16;
const COST_CASE: u32 = 0;
pub(crate) const COST_ADD_DIACRITIC: u32 = 4;
@@ -125,6 +129,12 @@ struct Collector<'a> {
names: bool,
/// May half of a missing space be one ([`Tuning::strict_case`])?
split_names: bool,
/// Candidates no simple word, left for the compound check: (cost, order, candidate).
deferred: Vec<(u32, usize, String)>,
/// Steps left for the compound searches of this call (`MAX_SUGGEST_COMPOUND_WORK` at first).
compound_work: usize,
/// The pieces those searches analysed, shared between them.
pieces: crate::affix::Pieces,
}
impl Collector<'_> {
@@ -141,18 +151,68 @@ impl Collector<'_> {
self.tested.insert(cand.to_string(), cost);
}
}
// A candidate is checked as simple words here; one of a single word that isn't one waits
// for the compound check, which the cheapest candidates get first ([`Self::compounds`]).
if !self.accept(cand, cost, false) && self.d.flag_compounds() && !cand.contains(' ') && self.deferred.len() < MAX_DEFERRED {
self.seq += 1;
self.deferred.push((cost, self.seq, cand.to_string()));
}
}
/// Check the waiting candidates as compounds, cheapest (then earliest) first, in one budget
/// of steps per call: compound checks cost far more than lookups, and a one-letter slip of a
/// long compound is the likeliest intent.
///
/// A candidate changed where the word stops being parts (`ziekenhusopname`: `zieken` and
/// `opname` are, `hus` isn't) goes first.
fn compounds(&mut self) {
let mut waiting = std::mem::take(&mut self.deferred);
if waiting.is_empty() {
return;
}
let gap = self.d.compound_gap(&self.input, self.names, &mut self.compound_work, &mut self.pieces);
let input: Vec<char> = self.input.chars().collect();
let outside = |cand: &str| {
let Some((from, to)) = gap else { return false };
let at = cand.chars().zip(&input).position(|(a, b)| a != *b).unwrap_or(input.len().min(cand.chars().count()));
at + 1 < from || at > to + 1
};
let mut waiting: Vec<(bool, u32, usize, String)> = waiting.drain(..).map(|(cost, seq, cand)| (outside(&cand), cost, seq, cand)).collect();
waiting.sort_unstable();
for (_, cost, _, cand) in waiting {
if self.compound_work == 0 {
break;
}
// Offered again at a lower cost meanwhile: that offer is the one to check.
if self.tested.get(&cand).is_some_and(|&c| c < cost) || self.found.contains_key(&cand) {
continue;
}
self.accept(&cand, cost, true);
}
}
/// Keep `cand` if each of its words is in the dictionary (`compounds`: flag-based compounds
/// count); whether it was.
fn accept(&mut self, cand: &str, cost: u32, compounds: bool) -> bool {
let mut words = Vec::new();
let mut cost = cost;
let names = if cand.contains(' ') { self.split_names } else { self.names };
let d = self.d;
let mut work = self.compound_work;
let pieces = &mut self.pieces;
let mut is_word =
|w: &str| if compounds { d.flag_compound_spending_suggestible(w, &mut work, pieces) } else { d.contains_suggestible_with(w, false) };
for part in cand.split(' ') {
if self.d.contains_suggestible(part) {
if is_word(part) {
words.push(part.to_string());
} else if !names {
return;
self.compound_work = work;
return false;
} else {
let c = capitalize(part);
if !self.d.contains_suggestible(&c) {
return;
if !is_word(&c) {
self.compound_work = work;
return false;
}
words.push(c);
// A name ranks after a common word at the same distance (`moze` → `może`, then
@@ -160,7 +220,9 @@ impl Collector<'_> {
cost += 1;
}
}
self.compound_work = work;
self.keep(words.join(" "), cost);
true
}
/// Keep a suggestion known to be correct, unless the dictionary says never to offer it.
@@ -303,12 +365,25 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
return Vec::new();
}
// The simple-word checks of this call share one budget of affix rules.
let _steps = crate::affix::SuggestSteps::start();
let case = casing(word);
let input = word.to_lowercase();
let chars: Vec<char> = input.chars().collect();
let names = !t.strict_case || case != Casing::Lower;
let split_names = !t.strict_case;
let mut col = Collector { d, input: input.clone(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names, split_names };
let mut col = Collector {
d,
input: input.clone(),
tested: HashMap::new(),
found: HashMap::new(),
seq: 0,
names,
split_names,
deferred: Vec::new(),
compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK,
pieces: Default::default(),
};
add_case(&mut col, word, &input);
let map = groups(d, t);
add_map_slips(&mut col, &chars, &map, t);
@@ -316,9 +391,11 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V
add_splits(&mut col, &input, &t.split);
add_key_neighbours(&mut col, &chars);
add_typos(&mut col, &chars, t);
col.compounds();
// 6. Further off: scan the stems that start like the word.
if col.count_within(COST_MISTYPE) < max {
scan(&mut col, &chars, &map);
col.compounds();
}
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
// Among equal costs, the change nearest the word's start is the likelier intent
@@ -521,8 +598,9 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
let related = |a: char, b: char| fold(a, map) == fold(b, map);
let limit = 4;
let mut budget = SCAN_FORMS_WORK;
for (_, stem) in best {
for form in d.forms(stem) {
for form in d.forms_spending(stem, &mut budget) {
let lf: Vec<char> = form.to_lowercase().chars().collect();
if lf.len().abs_diff(chars.len()) > 2 {
continue;
@@ -640,8 +718,18 @@ mod tests {
#[test]
fn a_cheaper_later_offer_wins() {
let d = AffixDict::from_hunspell("", "1\ncafe\n", &[]).unwrap();
let mut col =
Collector { d: &d, input: "cafè".into(), tested: HashMap::new(), found: HashMap::new(), seq: 0, names: true, split_names: true };
let mut col = Collector {
d: &d,
input: "cafè".into(),
tested: HashMap::new(),
found: HashMap::new(),
seq: 0,
names: true,
split_names: true,
deferred: Vec::new(),
compound_work: crate::affix::MAX_SUGGEST_COMPOUND_WORK,
pieces: Default::default(),
};
col.offer("cafe", COST_DROP_DIACRITIC);
col.offer("cafe", COST_REP);
col.offer("cafe", COST_SCAN);
+2 -5
View File
@@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()]));
}
}
+10 -1
View File
@@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
// The whole word first, as Hunspell does: a hyphenated entry or user word.
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
@@ -142,6 +143,14 @@ mod tests {
}
}
/// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't.
#[test]
fn a_hyphenated_entry_is_looked_up_whole() {
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
assert!(is_correct_with(&d, "tic-tac"));
assert!(!is_correct_with(&d, "tic-tacc"));
}
#[test]
fn missing_dictionary_and_ignored_inputs_are_quiet() {
let empty = AffixDict::default();
+9
View File
@@ -75,4 +75,13 @@ mod tests {
assert!(!suggest("město", 4).contains(&"město".into()));
assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into()));
}
#[test]
fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() {
// wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed
// letters; the one-letter edits stay in range and dropping the dot still finds "ix".
let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap();
let suggestions = suggest_in(&d, "İx", 4);
assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}");
}
}
+3 -2
View File
@@ -1,5 +1,5 @@
//! American and British spellings of the same English word (#1281, #1045): which forms each
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags,
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags,
//! and what it suggests instead (`colour` → `color` in American English).
//!
//! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a
@@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option<usize> {
match lang {
ProofLang::En => Some(0),
ProofLang::EnCa => Some(1),
ProofLang::EnGb => Some(2),
// Australian English follows the British policy (`colour`, `centre`, `organise`).
ProofLang::EnGb | ProofLang::EnAu => Some(2),
_ => None,
}
}
+113 -22
View File
@@ -4,10 +4,11 @@
//! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options,
//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder).
//! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2).
//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded
//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded
//! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also
//! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is
//! right; never one only another English knows. When it is there, the system locations aren't
//! right; never one only another English knows, except that Australian English also takes
//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't
//! searched.
//! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the
//! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored
@@ -18,8 +19,9 @@
//! embedded from craft-i18n is the same list; it is used for hyphenation only.)
//!
//! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only
//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every
//! folder before the next name. Another English's dictionary is never used: it would mark this
//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then
//! `en_GB`, for Australian), each name in every folder before the next name. Another English's
//! dictionary is never used: it would mark this
//! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the
//! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE`
//! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read;
@@ -90,15 +92,47 @@ fn names(lang: ProofLang) -> &'static [&'static str] {
match lang {
ProofLang::EnGb => &["en_GB", "en-GB"],
ProofLang::EnCa => &["en_CA", "en-CA"],
// Australian English's own, else a British one (a system en_GB on a build without SCOWL):
// [`exact_candidates`] reads every en_AU source before any en_GB one.
ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"],
_ => &["en_US", "en-US"],
}
}
/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names,
/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's
/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before
/// any en_GB one.
fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> {
let mut out = find(own_names(lang));
if lang == ProofLang::EnAu {
for c in find(&names(lang)[own_names(lang).len()..]) {
if !out.iter().any(|(_, d)| *d == c.1) {
out.push(c);
}
}
}
out
}
/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair
/// only, not Australian English's British fallback (a user's `en_GB` is for British text).
fn own_names(lang: ProofLang) -> &'static [&'static str] {
match lang {
ProofLang::EnAu => &["en_AU", "en-AU"],
_ => names(lang),
}
}
/// How many Englishes have a slot of their own ([`slot`]).
const ENGLISHES: usize = 4;
/// The slot of each English in [`State::slots`].
fn slot(lang: ProofLang) -> usize {
match lang {
ProofLang::EnGb => 1,
ProofLang::EnCa => 2,
ProofLang::EnAu => 3,
_ => 0,
}
}
@@ -109,7 +143,7 @@ type Loaded = Option<(Arc<AffixDict>, PathBuf)>;
/// The English Hunspell dictionaries, under one lock.
struct State {
/// Per English (see [`slot`]): `None` until first use, then what was found.
slots: [Option<Loaded>; 3],
slots: [Option<Loaded>; ENGLISHES],
/// Every `.dic` file read, so Englishes that use the same file share one copy.
files: Vec<(PathBuf, Arc<AffixDict>)>,
/// Counts reloads, so a load that started before one is not kept.
@@ -118,7 +152,7 @@ struct State {
impl State {
const fn new() -> Self {
State { slots: [None, None, None], files: Vec::new(), reloads: 0 }
State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 }
}
}
@@ -181,7 +215,7 @@ fn already_read(state: &RwLock<State>, dic: &Path) -> Option<Arc<AffixDict>> {
/// Forget what `state` holds, so the next check looks again.
fn forget(state: &RwLock<State>) {
let mut s = state.write().unwrap_or_else(PoisonError::into_inner);
s.slots = [None, None, None];
s.slots = [None, None, None, None];
s.files.clear();
s.reloads = s.reloads.wrapping_add(1);
}
@@ -198,9 +232,9 @@ fn key(lang: ProofLang) -> &'static str {
/// would rather use), for [`note_error`].
fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) {
let candidates = if crate::scowl::dict(lang).is_some() {
crate::hunspell::user_candidates(names(lang))
crate::hunspell::user_candidates(own_names(lang))
} else {
crate::hunspell::candidates_exact(names(lang))
exact_candidates(lang, crate::hunspell::candidates_exact)
};
if candidates.is_empty() {
log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang));
@@ -240,7 +274,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) {
/// folder changed).
pub(crate) fn reload() {
forget(&HUNSPELL);
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] {
crate::hunspell::clear_load_error(key(lang));
}
}
@@ -283,10 +317,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller {
if let Some((d, dic)) = found {
return Speller::Hunspell(d, dic);
}
match crate::scowl::dict(lang) {
Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)),
None => Speller::List,
let Some(d) = crate::scowl::dict(lang) else { return Speller::List };
let mut ds = vec![d];
// SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB
// list), and Australian English takes both. Suggestions come from the first, en-AU.
if lang == ProofLang::EnAu
&& let Some(gb) = crate::scowl::dict(ProofLang::EnGb)
{
ds.push(gb);
}
Speller::Scowl(ds, crate::scowl::is_british(lang))
}
/// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English
@@ -348,11 +388,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo
|| (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x))))
}
/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/
/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in
/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is
/// swapped; the `-ize` form counts only when American or Canadian English spells the word so
/// ([`ize_elsewhere`]), or `wize` would pass as `wise`.
/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it
/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and
/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize`
/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or
/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so
/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says.
pub(crate) fn ise_spelling(w: &str) -> Option<String> {
let b = w.as_bytes();
let mut out = b.to_vec();
@@ -360,7 +401,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option<String> {
for i in 1..b.len() {
let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c));
if after
&& matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y'))
&& matches!(b.get(i - 1), Some(b'i' | b'I'))
&& let Some(c) = out.get_mut(i)
{
match *c {
@@ -512,6 +553,18 @@ fn list_suggest(word: &str, max: usize) -> Vec<String> {
mod tests {
use super::*;
/// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford
/// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American.
#[test]
fn ise_spelling_swaps_ize_only() {
assert_eq!(ise_spelling("organize").as_deref(), Some("organise"));
assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations"));
assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING"));
for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] {
assert_eq!(ise_spelling(w), None, "{w}");
}
}
#[test]
fn built_in_list_is_the_whole_moby_list() {
let d = words();
@@ -596,16 +649,16 @@ mod tests {
fn lookup_order() {
let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap());
let file = PathBuf::from("/folder/en_US.dic");
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}");
let fallback = pick(None, lang);
if crate::scowl::dict(lang).is_some() {
assert!(
matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())),
matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())),
"{lang:?}"
);
assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}");
assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only");
assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)");
} else {
assert!(matches!(fallback, Speller::List), "{lang:?}");
}
@@ -625,6 +678,44 @@ mod tests {
assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB");
assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]);
assert_eq!(found(names(ProofLang::En)), ["en_US"]);
// Australian English: its own, else the British one it read before it had its own.
assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]);
std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap();
std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap();
assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]);
// craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU.
assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]);
std::fs::remove_file(dir.join("en_AU.aff")).unwrap();
std::fs::remove_file(dir.join("en_AU.dic")).unwrap();
assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own");
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
assert_eq!(own_names(lang), names(lang));
}
let _ = std::fs::remove_dir_all(&dir);
}
/// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an
/// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU.
#[test]
fn australian_english_reads_every_en_au_before_any_en_gb() {
let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id()));
for sub in ["user", "system"] {
std::fs::create_dir_all(dir.join(sub)).unwrap();
}
let write = |sub: &str, name: &str| {
std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap();
};
write("user", "en_GB");
write("system", "en_AU");
let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]);
let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]);
let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec<String> {
v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect()
};
assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]);
assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]);
assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::<String>::new());
let _ = std::fs::remove_dir_all(&dir);
}
+12 -5
View File
@@ -75,11 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang));
}
}
@@ -194,6 +191,16 @@ mod tests {
.collect()
}
/// craft-libs#496 review: a run of spaces is one issue over the whole run.
#[test]
fn a_run_of_spaces_is_reported_whole() {
for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] {
let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect();
assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}");
}
assert!(messages("casa roja").is_empty());
}
#[test]
fn repeated_words_and_spaces_have_spanish_messages() {
let text = "Es es la casa , dijo.";
+2 -5
View File
@@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()]));
}
}
+10 -1
View File
@@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
return true;
};
if w.contains('-') {
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
// The whole word first, as Hunspell does: a hyphenated entry or user word.
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
}
known(d, w)
}
@@ -105,6 +106,14 @@ mod tests {
}
}
/// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't.
#[test]
fn a_hyphenated_entry_is_looked_up_whole() {
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
assert!(is_correct_with(&d, "tic-tac"));
assert!(!is_correct_with(&d, "tic-tacc"));
}
#[test]
fn missing_dictionary_is_silent() {
assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ"));
+38 -7
View File
@@ -196,19 +196,22 @@ pub(crate) fn ordered(
let first = names.first().map(|n| norm(n));
let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first);
let pack: &[&str] = &[crate::packs::SPELLING];
// (folder, names, a Language pack's: never read through a symbolic link, as the pack scan
// checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`).
let tiers = dirs
.iter()
.map(|d| (d, names))
.chain(region_packs.iter().map(|d| (d, pack)))
.chain(system.iter().map(|d| (d, &region[..])))
.chain(other_packs.iter().map(|d| (d, pack)))
.chain(system.iter().map(|d| (d, &rest[..])));
.map(|d| (d, names, false))
.chain(region_packs.iter().map(|d| (d, pack, true)))
.chain(system.iter().map(|d| (d, &region[..], false)))
.chain(other_packs.iter().map(|d| (d, pack, true)))
.chain(system.iter().map(|d| (d, &rest[..], false)));
let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() };
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
for (dir, names) in tiers {
for (dir, names, in_pack) in tiers {
for n in names {
let aff = dir.join(format!("{n}.aff"));
let dic = dir.join(format!("{n}.dic"));
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) {
out.push((aff, dic));
}
}
@@ -696,6 +699,34 @@ mod tests {
}
}
/// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link
/// (swapped in after the pack scan) is never a candidate; one in a system folder still is.
#[cfg(unix)]
#[test]
fn pack_files_are_never_read_through_a_link() {
let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere"));
for d in [&pack, &system, &elsewhere] {
std::fs::create_dir_all(d).unwrap();
}
std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap();
std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap();
std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap();
std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap();
let got: Vec<PathBuf> =
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never");
std::fs::remove_file(pack.join("spelling.dic")).unwrap();
std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap();
let got: Vec<PathBuf> =
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]);
let _ = std::fs::remove_dir_all(&root);
}
/// WordCraft's own dictionaries folder under the user's application data is searched on every
/// desktop system, after the setting and before the shared system locations (#1063).
#[test]
+36 -5
View File
@@ -217,18 +217,25 @@ fn composed_chars(core: &[char]) -> (Vec<char>, Vec<Option<usize>>) {
}
/// Does a pack's `pats` hyphenate `core` in `lang`? Russian, Bulgarian and Turkish: a word in
/// their script (the guard their rules use). Polish, Portuguese, Italian and German: every word,
/// as their embedded patterns do. Any other language: a word whose letters the patterns know.
/// their script (the guard their rules use) whose letters the patterns know, so a word with a
/// letter the pack never mentions (`ё`) goes on to the built-in rules. Polish, Portuguese, Italian
/// and German: every word, as their embedded patterns do. Any other language: a word whose letters
/// the patterns know.
fn pack_patterns_fit(lang: ProofLang, pats: &Patterns, core: &[char]) -> bool {
match lang {
ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)),
ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)),
ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)),
ProofLang::Ru => core.iter().all(|&c| crate::ru::is_cyrillic(c)) && pack_knows(pats, core),
ProofLang::Bg => core.iter().all(|&c| crate::bg::is_letter(c)) && pack_knows(pats, core),
ProofLang::Tr => core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) && pack_knows(pats, core),
ProofLang::Pl | ProofLang::Pt | ProofLang::It | ProofLang::De => true,
_ => pats.values(core).is_some(),
}
}
/// Do `pats` know `core`: an exception word, or a word whose letters the patterns mention?
fn pack_knows(pats: &Patterns, core: &[char]) -> bool {
pats.exception(&core.iter().collect::<String>()).is_some() || pats.values(core).is_some()
}
/// Polish: exception words, then the patterns, then the Polish syllable heuristic.
fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
core_breaks_pl_with(Patterns::pl(), core)
@@ -534,6 +541,30 @@ fn heuristic_points_pl(c: &[char]) -> Vec<usize> {
mod tests {
use super::*;
/// craft-libs#497 review: a Russian, Bulgarian or Turkish pack's patterns only take a word
/// whose letters they know; any other goes to the built-in rules, not the English heuristic.
#[test]
fn pack_patterns_for_ru_bg_tr_need_every_letter_of_the_word() {
let chars = |w: &str| w.chars().collect::<Vec<_>>();
let cases = [(ProofLang::Ru, "1на 1ка", "нака", "ёка"), (ProofLang::Bg, "1на 1ка", "нака", "жка"), (ProofLang::Tr, "a1b", "abab", "abğ")];
for (lang, pat, known, unknown) in cases {
let p = Patterns::parse(pat);
assert!(pack_patterns_fit(lang, &p, &chars(known)), "{lang:?} {known}");
assert!(!pack_patterns_fit(lang, &p, &chars(unknown)), "{lang:?} {unknown}");
}
}
/// An exception word of a pack keeps its breaks even with a letter the patterns never mention.
#[test]
fn pack_exception_words_need_no_pattern_letters() {
let chars = |w: &str| w.chars().collect::<Vec<_>>();
let p = Patterns::parse("1на 1ка ё-ка");
let w = chars("ёка");
assert!(pack_patterns_fit(ProofLang::Ru, &p, &w));
assert_eq!(core_breaks_patterns(&p, &w), (vec![1], Source::Dictionary));
assert!(!pack_patterns_fit(ProofLang::Ru, &p, &chars("ёна")));
}
/// CodeRabbit on craft-libs#484: a Turkish word with a suffix after an apostrophe is
/// hyphenated by the Turkish rules, part by part, never next to the apostrophe.
#[test]
+2 -5
View File
@@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()]));
}
}
+14 -1
View File
@@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|| w.split('-').all(|p| {
p.chars().count() <= 1
|| CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c))
|| (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6)
|| (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6))
|| known(d, p)
});
}
@@ -141,6 +141,19 @@ mod tests {
}
}
/// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no
/// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters
/// still is one (`KTP-nya`, `COVID-19` with numbers checked).
#[test]
fn caseless_parts_are_not_abbreviations() {
test_dict::install();
let d = dictionary();
let defaults = crate::Options::default();
assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults));
let numbers = crate::Options { ignore_numbers: false, ..defaults };
assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers));
}
#[test]
fn forbidden_words_are_rejected_in_every_case() {
test_dict::install();
+32 -8
View File
@@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool {
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
}
const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}'];
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
/// an abbreviation), then space, with nothing but quotes and brackets around?
fn ends_sentence(gap: &str, prev: &str) -> bool {
let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']);
let t = gap.trim_start_matches(CLOSING);
let Some(end) = t.chars().next() else { return false };
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
return false;
@@ -161,8 +163,16 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']);
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
let after_marks = t.trim_start_matches(['.', '!', '?']);
let rest = after_marks.trim_start_matches(CLOSING);
// (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.)
// `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or
// exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case.
if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) {
return false;
}
rest.starts_with(char::is_whitespace)
&& rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
}
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
@@ -203,11 +213,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()]));
}
}
@@ -403,6 +410,23 @@ mod tests {
}
}
/// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in
/// lower case.
#[test]
fn narration_after_a_quoted_question_is_not_a_new_sentence() {
for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] {
assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t));
}
assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence");
// After a period the dash opens the next line of speech: a sentence starts.
assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo."));
assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo."));
// An inline object (`\u{1}`) before a real closing quote is skipped.
assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui."));
// An inline object (`\u{1}`) is not a closing quote.
assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco."));
}
#[test]
fn po_takes_an_apostrophe() {
assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]);
+17 -1
View File
@@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
}
// The whole word first, as Hunspell does: a hyphenated entry or user word.
if w.contains('-') {
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p));
}
known_elided(d, w)
}
/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after
/// the apostrophe is what gets checked.
fn known_elided(d: &AffixDict, w: &str) -> bool {
if let Some((head, tail)) = w.split_once(['\'', '’'])
&& PROCLITICS.contains(&head.to_lowercase().as_str())
&& !tail.is_empty()
@@ -148,6 +154,16 @@ mod tests {
assert!(!is_correct_with(&d, "tic-tacc"));
}
/// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part.
#[test]
fn an_elision_in_a_hyphenated_part_is_read() {
let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap();
for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] {
assert!(is_correct_with(&d, w), "{w}");
}
assert!(!is_correct_with(&d, "l'exx-amico"));
}
#[test]
fn without_a_dictionary_nothing_is_flagged() {
let none = AffixDict::default();
+23 -30
View File
@@ -5,7 +5,7 @@
/// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub enum ProofLang {
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or
/// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the
/// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`,
/// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also
@@ -13,10 +13,14 @@ pub enum ProofLang {
/// Word's).
#[default]
En,
/// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
/// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
/// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are
/// flagged and `-ize` and `-ise` are both right, as in Oxford spelling.
EnGb,
/// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own
/// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and
/// spells some words the Australian way (`program`, `inquiry`).
EnAu,
/// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`,
/// `defence`) with American `-ize` (`organize`), `tire` and `program`.
EnCa,
@@ -86,9 +90,10 @@ pub enum ProofLang {
impl ProofLang {
/// Every language with a spelling dictionary, in display order.
pub const PROOFED: [ProofLang; 15] = [
pub const PROOFED: [ProofLang; 16] = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::EnAu,
ProofLang::EnCa,
ProofLang::Pl,
ProofLang::Pt,
@@ -155,12 +160,13 @@ impl ProofLang {
}
}
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions
/// that spell the British way are [`ProofLang::EnGb`], and the United States, any other
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia
/// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other
/// region and no region are [`ProofLang::En`].
fn english(tag: &str) -> ProofLang {
match region(tag) {
Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa,
Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu,
Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb,
_ => ProofLang::En,
}
@@ -189,9 +195,10 @@ impl ProofLang {
}
}
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])?
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`],
/// [`ProofLang::EnCa`])?
pub fn is_english(self) -> bool {
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa)
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa)
}
/// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`],
@@ -208,7 +215,7 @@ impl ProofLang {
pub fn code(self) -> &'static str {
match self {
ProofLang::Pack(_) => "",
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en",
ProofLang::Pl => "pl",
ProofLang::Bg => "bg",
ProofLang::Pt => "pt",
@@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> {
const LATIN_AMERICA: [&str; 22] =
["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"];
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and
/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own,
/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other
/// Commonwealth countries, the Crown dependencies and Europe (`150`).
const BRITISH_SPELLING: [&str; 26] = [
"GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH",
"PK", "150",
const BRITISH_SPELLING: [&str; 25] = [
"GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK",
"150",
];
/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when
@@ -469,26 +476,12 @@ mod tests {
fn tags_resolve_english_regions() {
assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En);
assert_tags(
&[
"en-GB",
"EN_gb",
"en_GB.UTF-8",
"en-AU",
"en_AU",
"en-NZ",
"en-IE",
"en-ZA",
"en-IN",
"en-Latn-GB",
"en-UK",
"en-150",
"eng-GB",
"en_GB@euro",
],
&["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"],
ProofLang::EnGb,
);
assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa);
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu);
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}");
assert_eq!(language.code(), "en");
}
+157 -13
View File
@@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String {
composed(&w.to_lowercase()).into_owned()
}
/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one
/// space). A run of three is one issue over three, not a pair and a leftover.
pub(crate) fn space_runs(text: &str) -> Vec<Range<usize>> {
let bytes = text.as_bytes();
let mut runs = Vec::new();
let mut i = 0;
while i < bytes.len() {
if bytes.get(i) != Some(&b' ') {
i += 1;
continue;
}
let start = i;
while bytes.get(i) == Some(&b' ') {
i += 1;
}
if i - start >= 2 {
runs.push(start..i);
}
}
runs
}
/// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language.
pub fn add_word(w: &str) {
if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) {
@@ -340,7 +362,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)>
/// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the
/// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]).
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) {
if lang.is_english() {
return english::load_error(lang);
}
if let Some(l) = lang.spelling_locale() {
@@ -669,7 +691,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool {
|| !w.chars().any(char::is_alphabetic)
|| (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit()))
// Only a word with a cased letter is in capitals: a Hebrew or Thai word has none.
|| (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()))
|| (opts.ignore_uppercase && spelling::in_capitals(w))
// The whole word: the languages split a hyphenated word into its parts, and an added
// `Unigranrio-RJ` is none of them (#2082).
|| in_user_dictionary(&w.to_lowercase())
@@ -953,7 +975,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec<Issue> {
fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
match lang {
ProofLang::En => grammar_en(text, capital_first),
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
// `pt-pt` shares Portuguese's rules, its issues in its own language.
_ => language(lang.built_in())
.map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()),
@@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec<Issue> {
lang: ProofLang::En,
});
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
for run in space_runs(text) {
v.push(Issue {
start: i,
end: i + 2,
start: run.start,
end: run.end,
kind: IssueKind::Grammar,
message: "Extra space".into(),
suggestions: vec![" ".into()],
@@ -1197,7 +1216,7 @@ mod tests {
}
// Every proofed language other than English has a Hunspell dictionary to report.
for l in ProofLang::PROOFED {
let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa);
let english = l.is_english();
assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}");
}
for l in [ProofLang::En, ProofLang::Other] {
@@ -1321,7 +1340,8 @@ mod tests {
}
// Text in a language without proofing is never checked with English.
assert!(check_spelling_in("hous teh zxqv", ProofLang::Other).is_empty());
assert!(check_text("hous teh", &[(0..8, ProofLang::De)]).is_empty());
// `Other` has no dictionary on any machine (German may: a system `de_DE` Hunspell).
assert!(check_text("hous teh", &[(0..8, ProofLang::Other)]).is_empty());
assert_eq!(english_source(ProofLang::Pl), None);
drop_english_files(&dir, english);
assert_ne!(english_source(ProofLang::En), Some(EnglishSource::Hunspell(dir.join("en_US.dic"))));
@@ -1709,6 +1729,31 @@ mod tests {
let _ = std::fs::remove_dir_all(&root);
}
/// Review follow-up (craft-libs#485): a word in another script than a pack language's
/// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a
/// misspelling in the dictionary's own script still is.
#[test]
fn pack_languages_leave_other_scripts_alone() {
crate::hold_dictionary_tests();
set_dictionaries_folder(Some(shared_test_folder()));
let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&root);
write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]);
write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]);
packs::set_packs_dir(Some(root.clone()));
let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR")));
assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl));
assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters");
assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl));
assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el));
let text = "Пушкин huiss";
let issues = check_text(text, &[(0..text.len(), nl)]);
assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::<Vec<_>>(), ["huiss"], "{issues:?}");
packs::set_packs_dir(None);
set_dictionaries_folder(Some(shared_test_folder()));
let _ = std::fs::remove_dir_all(&root);
}
/// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the
/// system's only dictionary folder.
fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option<std::path::PathBuf> {
@@ -2465,6 +2510,49 @@ mod tests {
assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs"));
}
/// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in
/// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn british_english_rejects_unlisted_yze_forms() {
crate::hold_dictionary_tests();
let gb = ProofLang::EnGb;
for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] {
assert!(!is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_in(w, ProofLang::En), "en-US: {w}");
}
for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] {
assert!(is_correct_in(w, gb), "en-GB: {w}");
}
}
/// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships
/// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells
/// them; British English doesn't take them. The British policy still applies: American
/// spellings are wrong, `-ise` and `-ize` both right.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn australian_english_reads_its_own_scowl() {
crate::hold_dictionary_tests();
let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb);
assert!(scowl::is_british(au));
assert_eq!(english_source(au), Some(EnglishSource::Embedded));
for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] {
assert!(is_correct_in(w, au), "en-AU: {w}");
assert!(!is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}");
}
for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] {
assert!(is_correct_in(w, au), "en-AU: {w}");
}
for w in ["color", "center", "analyze", "reanalyze", "defense"] {
assert!(!is_correct_in(w, au), "en-AU: {w}");
}
let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky.";
assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au));
assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb));
}
/// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae,
/// with their inflections.
#[test]
@@ -2472,7 +2560,7 @@ mod tests {
// English results depend on the dictionaries other tests install.
crate::hold_dictionary_tests();
let gb = ProofLang::EnGb;
assert_eq!(ProofLang::from_tag(Some("en-AU")), gb);
assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu);
for w in [
"goitre",
"goitres",
@@ -2497,11 +2585,66 @@ mod tests {
"encyclopaedias",
] {
assert!(is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}");
}
let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia.";
assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb));
}
/// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also
/// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only
/// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
#[test]
fn australian_english_also_takes_british_words() {
crate::hold_dictionary_tests();
let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb);
for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] {
assert!(is_correct_in(w, gb), "en-GB: {w}");
assert!(is_correct_in(w, au), "en-AU: {w}");
}
for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] {
assert!(!is_correct_in(w, gb), "en-GB: {w}");
assert!(!is_correct_in(w, au), "en-AU: {w}");
}
assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own");
assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au));
}
/// Review follow-up: a run of spaces is one issue over the whole run, in every language with a
/// double-space rule (a run of three used to leave two after the fix, four three).
#[test]
fn a_run_of_spaces_is_one_issue_in_every_language() {
assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]);
assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty());
let langs = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::Es,
ProofLang::It,
ProofLang::Pt,
ProofLang::Cs,
ProofLang::Hr,
ProofLang::Id,
ProofLang::Pl,
ProofLang::Ru,
ProofLang::Tr,
ProofLang::EnAu,
ProofLang::EnCa,
]
.into_iter()
.chain(PackLocale::new("pt-PT").map(ProofLang::Pack));
for lang in langs {
for n in 2..=5 {
let text = format!("Aaa{}bbb", " ".repeat(n));
let hits: Vec<_> =
check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect();
assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}");
assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}");
}
}
}
/// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English,
/// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood.
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
@@ -2513,7 +2656,8 @@ mod tests {
assert_eq!(english_source(l), Some(EnglishSource::Embedded));
assert_eq!(dictionary_load_error(l), None);
}
assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB");
assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own");
assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB");
let notice = scowl::notice().unwrap();
for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] {
assert!(notice.contains(credit), "{credit}");
+10 -4
View File
@@ -7,9 +7,10 @@
use crate::hyphen::{self, Limits, Source};
use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns};
const LANGS: [ProofLang; 18] = [
const LANGS: [ProofLang; 19] = [
ProofLang::En,
ProofLang::EnGb,
ProofLang::EnAu,
ProofLang::EnCa,
ProofLang::Pl,
ProofLang::Bg,
@@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [
#[test]
fn nothing_is_embedded() {
assert!(crate::data::EN_US_DIC.is_empty());
assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty());
assert!(
crate::data::EN_US_SPELLING.is_empty()
&& crate::data::EN_GB_SPELLING.is_empty()
&& crate::data::EN_CA_SPELLING.is_empty()
&& crate::data::EN_AU_SPELLING.is_empty()
);
assert!(crate::scowl::notice().is_none());
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none());
}
assert!(dict::Dictionary::en_us().is_empty());
@@ -63,7 +69,7 @@ fn nothing_is_embedded() {
#[test]
fn english_spelling_still_marks_misspellings() {
crate::hold_dictionary_tests();
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
let source = crate::english_source(l);
assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}");
// Any English dictionary marks junk and knows `the`; the word-by-word results below are
+154 -17
View File
@@ -6,6 +6,10 @@
//! files); the one used least recently goes first, but never one used during the current
//! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales
//! with no dictionary are remembered apart and take no room.
//!
//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic
//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another
//! script alone. A word in the dictionary's script is always checked, whatever its letters.
use std::cell::Cell;
use std::path::PathBuf;
@@ -20,7 +24,7 @@ use crate::lang::PackLocale;
pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20;
/// Estimated memory per byte of `.aff` and `.dic` file.
const BYTES_PER_FILE_BYTE: u64 = 6;
/// Most locales remembered as having no dictionary (the list is cleared when full).
/// Most locales remembered as having no dictionary (the oldest is forgotten first when full).
const MAX_MISSING: usize = 256;
/// The budget in force (tests lower it).
@@ -36,6 +40,8 @@ thread_local! {
struct Loaded {
locale: PackLocale,
dict: Arc<AffixDict>,
/// The scripts its words are written in ([`scripts`]), sorted.
scripts: Arc<[u32]>,
/// The `.dic` it came from.
file: Option<PathBuf>,
/// Estimated memory.
@@ -66,15 +72,88 @@ pub(crate) fn reload() {
/// over the budget.
pub(crate) fn in_one_check<R>(f: impl FnOnce() -> R) -> R {
let outer = CHECK_FROM.with(Cell::get);
if outer.is_none() {
let _end = outer.is_none().then(|| {
let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock;
CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1))));
}
let r = f();
if outer.is_none() {
EndCheck
});
f()
}
/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches
/// it), or every dictionary would stay pinned on this thread and the budget never hold again.
struct EndCheck;
impl Drop for EndCheck {
fn drop(&mut self) {
CHECK_FROM.with(|c| c.set(None));
}
r
}
/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is
/// forgotten (clearing them all would look every other one up again).
fn remember_missing(missing: &mut Vec<PackLocale>, locale: PackLocale) {
if missing.len() >= MAX_MISSING {
missing.remove(0);
}
missing.push(locale);
}
/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks
/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana,
/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named.
fn script_of(c: char) -> u32 {
const SCRIPTS: &[(u32, u32, u32)] = &[
(0x0041, 0x024F, 1),
(0x0250, 0x02AF, 1),
(0x1E00, 0x1EFF, 1),
(0x2C60, 0x2C7F, 1),
(0xA720, 0xA7FF, 1),
(0xAB30, 0xAB6F, 1),
(0xFF21, 0xFF5A, 1),
(0x0370, 0x03FF, 2),
(0x1F00, 0x1FFF, 2),
(0x0400, 0x052F, 3),
(0x1C80, 0x1C8F, 3),
(0x2DE0, 0x2DFF, 3),
(0xA640, 0xA69F, 3),
(0x0530, 0x058F, 4),
(0xFB13, 0xFB17, 4),
(0x0590, 0x05FF, 5),
(0xFB1D, 0xFB4F, 5),
(0x0600, 0x06FF, 6),
(0x0750, 0x077F, 6),
(0x08A0, 0x08FF, 6),
(0xFB50, 0xFDFF, 6),
(0xFE70, 0xFEFF, 6),
(0x10A0, 0x10FF, 7),
(0x1C90, 0x1CBF, 7),
(0x2D00, 0x2D2F, 7),
(0x1100, 0x11FF, 8),
(0x3130, 0x318F, 8),
(0xAC00, 0xD7AF, 8),
(0x3040, 0x30FF, 9),
(0x3400, 0x4DBF, 10),
(0x4E00, 0x9FFF, 10),
(0x1200, 0x139F, 11),
];
let u = u32::from(c);
SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s)
}
/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted.
fn scripts(d: &AffixDict) -> Arc<[u32]> {
let mut seen = std::collections::BTreeSet::new();
for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) {
seen.insert(script_of(c));
}
seen.into_iter().collect()
}
/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one
/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch).
fn other_script(word: &str, scripts: &[u32]) -> bool {
!scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok())
}
#[cfg(test)]
@@ -149,30 +228,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec<String> {
/// `locale`'s dictionary, loaded on first use; an empty one when there is none.
fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
let (d, file, _) = get_with_scripts(locale);
(d, file)
}
/// [`get`], with the dictionary's [`scripts`].
fn get_with_scripts(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>, Arc<[u32]>) {
let reloads = {
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
c.clock = c.clock.wrapping_add(1);
let now = c.clock;
if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) {
l.used = now;
return (l.dict.clone(), l.file.clone());
return (l.dict.clone(), l.file.clone(), l.scripts.clone());
}
if c.missing.contains(&locale) {
return (Arc::default(), None);
return (Arc::default(), None, Arc::from([]));
}
c.reloads
};
let (dict, file, bytes) = load(locale);
let scripts = scripts(&dict);
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) {
return (dict, file);
return (dict, file, scripts);
}
if dict.is_empty() {
if c.missing.len() >= MAX_MISSING {
c.missing.clear();
}
c.missing.push(locale);
return (dict, file);
remember_missing(&mut c.missing, locale);
return (dict, file, scripts);
}
LOADS.fetch_add(1, Ordering::Relaxed);
let budget = BUDGET.load(Ordering::Relaxed);
@@ -187,8 +270,8 @@ fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
total = total.saturating_sub(gone.bytes);
}
let used = c.clock;
c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used });
(dict, file)
c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used });
(dict, file, scripts)
}
/// Load `locale`'s dictionary, with its estimated memory.
@@ -229,11 +312,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option<String> {
/// Is `word` right in `locale`? Every word is when there is no dictionary.
pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool {
let (d, _) = get(locale);
let (d, _, scripts) = get_with_scripts(locale);
if d.is_empty() {
return true;
}
let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true };
if other_script(w, &scripts) {
return true;
}
let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase());
if w.contains('-') && !known(w) {
return w.split('-').all(|p| p.chars().count() <= 1 || known(p));
@@ -264,6 +350,57 @@ mod tests {
PackLocale::new(t).unwrap()
}
/// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its
/// oldest one, not all of them, so a document in more languages doesn't look every one up again.
#[test]
fn missing_locales_forget_the_oldest_first() {
let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char));
let mut missing = Vec::new();
for i in 0..MAX_MISSING + 10 {
remember_missing(&mut missing, tag(i));
assert!(missing.len() <= MAX_MISSING);
}
assert_eq!(missing.len(), MAX_MISSING);
assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first");
assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay");
}
/// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave
/// this thread's dictionaries pinned past the budget for good.
#[test]
fn a_panic_inside_a_check_ends_it() {
let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check")));
assert!(caught.is_err());
assert_eq!(CHECK_FROM.with(Cell::get), None);
in_one_check(|| {
assert!(CHECK_FROM.with(Cell::get).is_some());
in_one_check(|| ());
assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one");
});
assert_eq!(CHECK_FROM.with(Cell::get), None);
}
/// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is
/// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked,
/// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`).
#[test]
fn words_in_another_script_are_not_the_dictionarys() {
let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap();
let latin = scripts(&nl);
assert_eq!(latin.len(), 1);
for w in ["привет", "Привет", "αβγ", "שלום", "日本"] {
assert!(other_script(w, &latin), "{w}");
}
for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] {
assert!(!other_script(w, &latin), "{w}");
}
assert!(!other_script("привет", &[]), "no scripts: every word is checked");
let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap();
assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru)));
assert_ne!(script_of('ა'), script_of('a'));
assert_eq!(script_of('ă'), script_of('Ạ'));
}
#[test]
fn locales_and_file_names() {
assert_eq!(l("nl-NL").as_str(), "nl-nl");
+59 -9
View File
@@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling";
const HYPHENATION: &str = "hyphenation.pat";
/// Most languages whose pack patterns are remembered (a document may tag thousands).
const MAX_HYPHENATION_LOOKUPS: usize = 256;
/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]):
/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to
/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the
/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`).
const MAX_HYPHENATION_BYTES: u64 = 256 << 20;
/// Estimated memory per pattern (its packed key, its values and the map's own room).
const BYTES_PER_PATTERN: u64 = 64;
/// A pack's description.
const PACK_TOML: &str = "pack.toml";
/// A pack's licence texts.
@@ -80,15 +87,18 @@ struct State {
dir: Option<PathBuf>,
scan: Option<Arc<Scan>>,
/// Each language's pack patterns once looked up (`None`: no pack has them), at most
/// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full).
/// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full).
hyphenation: BTreeMap<ProofLang, Option<Arc<Patterns>>>,
/// The estimated memory of the patterns in [`State::hyphenation`].
hyphenation_bytes: u64,
/// (locale, reason) of pack hyphenation files that failed to load.
hyphenation_errors: Vec<(String, String)>,
/// Counts changes, so a scan or load that started before one is not kept.
changes: u64,
}
static STATE: RwLock<State> = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 });
static STATE: RwLock<State> =
RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 });
fn read() -> RwLockReadGuard<'static, State> {
STATE.read().unwrap_or_else(PoisonError::into_inner)
@@ -121,6 +131,7 @@ pub fn reload() {
fn forget(s: &mut State) {
s.scan = None;
s.hyphenation.clear();
s.hyphenation_bytes = 0;
s.hyphenation_errors.clear();
s.changes = s.changes.wrapping_add(1);
}
@@ -190,8 +201,9 @@ const BUILT_IN_MOBY_NOTICE: &str = "The English word list built into craft-proof
const EXTRA_EN_NOTICE: &str = "The English word supplement (en-extra.txt), common words the Moby list lacks, is written by \
WordCraft contributors and licensed MIT OR Apache-2.0.";
/// The locales whose Language data is embedded in this build: `en-us`, and by default `de`,
/// `pl`, `pt`, `it`, `ru` (until Language packs ship, CP4.4); a build with `CRAFT_PROOF_EMBED`
/// The locales whose upstream Language data is embedded in this build: `en-us`, and by default `de`,
/// `pl`, `pt`, `it` (until Language packs ship, CP4.4; our own `ru-extra.txt` is embedded but not
/// listed); a build with `CRAFT_PROOF_EMBED`
/// embeds English and the ones it names (`en`: English only).
pub fn embedded_locales() -> &'static [&'static str] {
crate::data::EMBEDDED_LOCALES
@@ -531,7 +543,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec<String> {
ProofLang::EsUy => &["es-uy", "es-419", "es"],
ProofLang::De => &["de", "de-1996", "de-de"],
ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l),
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[],
ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[],
};
names.iter().map(|n| (*n).to_string()).collect()
}
@@ -566,10 +578,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
let mut s = write();
if s.changes == changes && !s.hyphenation.contains_key(&lang) {
// ponytail: cleared when full, an LRU if many languages each need their patterns.
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS {
s.hyphenation.clear();
}
s.hyphenation.insert(lang, found.clone());
let bytes = found.as_deref().map_or(0, pattern_bytes);
remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES);
for e in errors {
if !s.hyphenation_errors.contains(&e) {
s.hyphenation_errors.push(e);
@@ -579,6 +589,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
found
}
/// Estimated memory of `p`.
fn pattern_bytes(p: &Patterns) -> u64 {
(p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN)
}
/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages
/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the
/// whole budget are still kept alone, or they would be read from disk for every word.
fn remember_patterns(s: &mut State, lang: ProofLang, found: Option<Arc<Patterns>>, bytes: u64, budget: u64) {
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget {
s.hyphenation.clear();
s.hyphenation_bytes = 0;
}
s.hyphenation.insert(lang, found);
s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes);
}
/// How many languages' pack patterns are remembered now.
#[cfg(test)]
pub(crate) fn hyphenation_lookups() -> usize {
@@ -600,6 +627,29 @@ fn load_patterns(path: &Path) -> Result<Patterns, String> {
mod tests {
use super::*;
/// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not
/// only a count, so 256 languages of large patterns can't each stay in memory.
#[test]
fn remembered_patterns_keep_to_a_memory_budget() {
let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 };
let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n"));
let bytes = pattern_bytes(&p);
assert_eq!(bytes, 9 * BYTES_PER_PATTERN);
let lang = |t: &str| ProofLang::from_tag(Some(t));
let budget = bytes * 2;
remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget);
remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget);
remember_patterns(&mut s, lang("sv"), None, 0, budget);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room");
remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten");
assert!(s.hyphenation.contains_key(&lang("da")));
remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone");
forget(&mut s);
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0));
}
/// Review of #485: a pack's name loses bidirectional and other format characters.
#[test]
fn pack_names_are_plain_text() {
+2 -5
View File
@@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()]));
}
}
+16 -6
View File
@@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
if end == '.' && abbreviation_or_number(prev) {
return false;
}
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
let after_marks = t.trim_start_matches(['.', '!', '?']);
let rest = after_marks.trim_start_matches(CLOSING);
// `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case.
if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) {
return false;
}
// A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes
// on in lower case.
rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c))
@@ -124,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()]));
}
}
@@ -225,6 +227,14 @@ mod tests {
// craft-libs#488 review: no space after the dash.
clean("—Vai? —disse ele.");
clean("— Vai? — disse ele.");
// craft-libs#496 review: a closing quote after the mark.
clean("\"Vai?\" disse ele.");
clean("«Vai!» gritou ela.");
assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence");
// An inline object (`\u{1}`) before a real closing quote is skipped.
clean("\"Vai?\u{1}\" disse ele.");
// An inline object (`\u{1}`) is not a closing quote.
assert!(!messages("Quem era?\u{1} era Marco.").is_empty());
clean("Ele disse: \"Volto amanhã\". E saiu.");
// A list item may start in lower case.
clean("maçãs,");
+2 -5
View File
@@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()]));
}
}
for (i, _) in text.match_indices(" ") {
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
continue;
}
v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()]));
for run in crate::space_runs(text) {
v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()]));
}
}
+5 -3
View File
@@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
return true;
}
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS {
return true;
}
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
@@ -321,7 +321,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
}
match lp.as_slice() {
// `кое-кто`, `кой-какой`.
[prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)),
[prefix, _] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)),
// `по-русски` (from `русский`), `по-новому`, `по-моему`.
["по", rest] => {
let Some(r) = parts.get(1) else { return false };
@@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
let last = parts.len() - 1;
parts.iter().enumerate().all(|(i, p)| {
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) {
return true;
}
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
@@ -485,5 +485,7 @@ mod tests {
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
// The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations.
assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits));
}
}
+17 -9
View File
@@ -2,10 +2,15 @@
//! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`,
//! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use.
//!
//! Each English has its own: American English reads en-US, the British-spelling Englishes
//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions
//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's
//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling
//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no
//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU
//! data, so it stays on en-GB. Its suggestions
//! come from its own dictionary, and a word is right when its own dictionary knows it: the other
//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL
//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except
//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words
//! (`enquiry`; its suggestions still come from en-AU). SCOWL
//! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word
//! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that
//! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs
@@ -19,16 +24,17 @@ use std::sync::OnceLock;
use crate::ProofLang;
use crate::affix::AffixDict;
use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE};
use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE};
/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there.
static DICTS: [OnceLock<Result<AffixDict, String>>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()];
/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there.
static DICTS: [OnceLock<Result<AffixDict, String>>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()];
/// The slot, `.aff` and `.dic` text of `lang`'s dictionary.
fn source(lang: ProofLang) -> (usize, &'static str, &'static str) {
match lang {
ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING),
ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING),
ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING),
_ => (0, EN_US_AFF, EN_US_SPELLING),
}
}
@@ -50,15 +56,17 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> {
loaded.as_ref().ok()
}
/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)?
/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)?
pub(crate) fn is_british(lang: ProofLang) -> bool {
source(lang).0 == 1
matches!(source(lang).0, 1 | 3)
}
/// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it.
pub(crate) fn load_error(lang: ProofLang) -> Option<String> {
let (i, ..) = source(lang);
DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}"))
let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}"));
// Australian English also reads the en-GB list (`crate::english`): its failure is reported too.
own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None })
}
/// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of
+28 -2
View File
@@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options)
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
return None;
}
// Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none.
if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
if opts.ignore_uppercase && in_capitals(w) && n <= 6 {
return None;
}
Some(w)
}
/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can
/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms.
pub(crate) fn in_capitals(w: &str) -> bool {
w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())
}
/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max`
/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A
/// part in a caseless script is no abbreviation.
pub(crate) fn capitals_part(part: &str, max: usize) -> bool {
part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic))
}
#[cfg(test)]
mod tests {
use super::*;
/// craft-libs#485's cased-letter rule, shared by every language: a word or part is in
/// capitals only with a cased letter; a part without letters counts as an abbreviation.
#[test]
fn capitals_need_a_cased_letter() {
for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] {
assert!(in_capitals(w) && capitals_part(w, 6), "{w}");
}
for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] {
assert!(!in_capitals(w), "{w}");
}
assert!(capitals_part("19", 6) && capitals_part("", 6));
assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6));
}
#[test]
fn words_needing_no_lookup() {
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
+23 -19
View File
@@ -128,8 +128,10 @@ const ABBREVIATIONS: &[&str] = &[
/// Conjunctions that must not be preceded by a comma.
const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"];
/// Conjunctions that must not be followed by a comma.
const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "gerek", "ister"];
/// Conjunctions that must not be followed by a comma. Not `ister` and `gerek`: they are
/// conjunctions only in `ister … ister` / `gerek … gerek`; otherwise `ister` is the verb
/// (`Kim ister, gelsin.`) and `gerek` a noun or adjective, and a comma after them is correct.
const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem"];
/// Check grammar and punctuation in a Turkish text block.
pub fn check(text: &str) -> Vec<Issue> {
@@ -209,23 +211,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
});
}
}
// Double space
if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') {
let mut end = idx + 1;
while end < text.len() && text.as_bytes().get(end) == Some(&b' ') {
end += 1;
}
v.push(Issue {
start: idx,
end,
kind: IssueKind::Grammar,
message: "Fazladan boşluk".into(),
suggestions: vec![" ".into()],
lang: ProofLang::Tr,
});
}
}
for run in crate::space_runs(text) {
v.push(Issue {
start: run.start,
end: run.end,
kind: IssueKind::Grammar,
message: "Fazladan boşluk".into(),
suggestions: vec![" ".into()],
lang: ProofLang::Tr,
});
}
// 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`)
for (i, &(w_start, w_end)) in ws.iter().enumerate() {
let word = match text.get(w_start..w_end) {
@@ -411,7 +407,15 @@ mod tests {
/// (craft-libs#488 review).
#[test]
fn a_comma_after_de_ne_or_ya_is_fine() {
for text in ["Ali de, Veli de geldi.", "Ne, sen mi geldin?", "Ya, bilmiyorum.", "Bu mesele de, haklı olan bizdik."] {
for text in [
"Ali de, Veli de geldi.",
"Ne, sen mi geldin?",
"Ya, bilmiyorum.",
"Bu mesele de, haklı olan bizdik.",
"Kim ister, gelsin.",
"Ali ister, Veli istemez.",
"Gerek, yok.",
] {
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
}
}
+15 -2
View File
@@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
}
}
// 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] {
// 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
for suffix in ["de", "da", "mi", "mı", "mu", "mü"] {
if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 {
let stem = &lower[..lower.len() - suffix.len()];
if d.is_empty() || d.contains(stem) {
@@ -333,6 +333,19 @@ mod tests {
assert!(sugs.contains(&"geldin mi".to_string()));
}
/// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te`
/// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written
/// attached, so `kitapte` is never offered as `kitap te`.
#[test]
fn the_detached_particle_is_never_te_or_ta() {
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap();
assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string()));
for w in ["kitapte", "kitapta"] {
let sugs = suggest_with(&d, w, 8);
assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}");
}
}
#[test]
fn overlong_words_get_no_suggestions_promptly() {
let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();