From e36fb5d06951c5813beb3668d2e6c31db38c0aa5 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 1 Nov 2026 23:57:16 +0000 Subject: [PATCH 01/23] =?UTF-8?q?FU.1:=20craft-proof=20=E2=80=94=20British?= =?UTF-8?q?=20English=20never=20swaps=20-yze=20for=20-yse=20(reanalyze,=20?= =?UTF-8?q?breathalyzer,=20paralyzingly=20stay=20American,=20as=20Hunspell?= =?UTF-8?q?=20en=5FGB=20says)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/english.rs | 25 +++++++++++++++++++------ crates/proof/src/lib.rs | 16 ++++++++++++++++ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index 0c2c2e13..e818465f 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -348,11 +348,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo || (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x)))) } -/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/ -/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in -/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is -/// swapped; the `-ize` form counts only when American or Canadian English spells the word so -/// ([`ize_elsewhere`]), or `wize` would pass as `wise`. +/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it +/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and +/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize` +/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or +/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so +/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says. pub(crate) fn ise_spelling(w: &str) -> Option { let b = w.as_bytes(); let mut out = b.to_vec(); @@ -360,7 +361,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option { for i in 1..b.len() { let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c)); if after - && matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y')) + && matches!(b.get(i - 1), Some(b'i' | b'I')) && let Some(c) = out.get_mut(i) { match *c { @@ -512,6 +513,18 @@ fn list_suggest(word: &str, max: usize) -> Vec { mod tests { use super::*; + /// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford + /// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American. + #[test] + fn ise_spelling_swaps_ize_only() { + assert_eq!(ise_spelling("organize").as_deref(), Some("organise")); + assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations")); + assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING")); + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] { + assert_eq!(ise_spelling(w), None, "{w}"); + } + } + #[test] fn built_in_list_is_the_whole_moby_list() { let d = words(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 5b58768c..9c19999d 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -2465,6 +2465,22 @@ mod tests { assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs")); } + /// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in + /// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn british_english_rejects_unlisted_yze_forms() { + crate::hold_dictionary_tests(); + let gb = ProofLang::EnGb; + for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::En), "en-US: {w}"); + } + for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + } + } + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, /// with their inflections. #[test] From 820695e95121f5b0959a0de6847b2f01812a1fb3 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:00:22 +0000 Subject: [PATCH 02/23] =?UTF-8?q?FU.2:=20craft-proof=20=E2=80=94=20Austral?= =?UTF-8?q?ian=20English=20reads=20SCOWL's=20own=20en-AU=20dictionary=20(P?= =?UTF-8?q?roofLang::EnAu),=20with=20en-GB's=20spelling=20policy;=20en-NZ?= =?UTF-8?q?=20stays=20on=20en-GB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SCOWL en-AU has 238 words en-GB lacks (billabong, brumby, larrikinism, Australian place names) and spells program, inquiry, livable, licorice, verandah the Australian way (no programme, enquiry, liveable, dialog or veranda, as Hunspell en_AU says); -ise/-ize policy and affix file are the same as en-GB's. craft-i18n already ships language-data/en-au. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/README.md | 11 +++---- crates/proof/build.rs | 7 +++-- crates/proof/src/en_variants.rs | 5 ++-- crates/proof/src/english.rs | 29 +++++++++++++----- crates/proof/src/lang.rs | 53 ++++++++++++++------------------- crates/proof/src/lib.rs | 39 ++++++++++++++++++++---- crates/proof/src/no_data.rs | 14 ++++++--- crates/proof/src/packs.rs | 2 +- crates/proof/src/scowl.rs | 18 ++++++----- 9 files changed, 113 insertions(+), 65 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 49a16473..4aeb677b 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | Language | Spelling | Grammar | Hyphenation | |---|---|---|---| -| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | +| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | | German | — | — | TeX patterns (embedded, or a Language pack) | @@ -151,14 +151,15 @@ English spelling uses the first of these that is there; `english_source(lang)` s or in WordCraft's own `dictionaries` folder. The user's choice comes first. 2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2). 3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from - `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` + - `spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling - Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own + `CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`, + `spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU + (Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the + other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own dictionary knows it (or `en-extra.txt` does), never because another English's does (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so British English also takes the `-ize` spellings of its `-ise` words that American or Canadian - English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this + English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this dialect's other spellings of known words (`judgement` in en-US). English with no dialect (`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only between two lower-case words of three letters or more, after every one-word candidate, and no diff --git a/crates/proof/build.rs b/crates/proof/build.rs index 8ff77fc4..82f5f87f 100644 --- a/crates/proof/build.rs +++ b/crates/proof/build.rs @@ -16,9 +16,8 @@ //! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded: //! it is treated like a folder without the data. //! -//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as -//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL -//! notice is embedded once (the four upstream READMEs differ only in their title and build line). +//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is +//! embedded once (the four upstream READMEs differ only in their title and build line). use std::fmt::Write as _; use std::path::{Path, PathBuf}; @@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[ ("EN_GB_SPELLING", "en-gb/spelling.dic", false), ("EN_CA_AFF", "en-ca/spelling.aff", false), ("EN_CA_SPELLING", "en-ca/spelling.dic", false), + ("EN_AU_AFF", "en-au/spelling.aff", false), + ("EN_AU_SPELLING", "en-au/spelling.dic", false), ("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false), ("EN_US_PAT", "en-us/hyphenation.pat", false), ("EXTRA_EN", "en-us/en-extra.txt", false), diff --git a/crates/proof/src/en_variants.rs b/crates/proof/src/en_variants.rs index 07c16bb1..c8a63fd8 100644 --- a/crates/proof/src/en_variants.rs +++ b/crates/proof/src/en_variants.rs @@ -1,5 +1,5 @@ //! American and British spellings of the same English word (#1281, #1045): which forms each -//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags, +//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags, //! and what it suggests instead (`colour` → `color` in American English). //! //! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a @@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option { match lang { ProofLang::En => Some(0), ProofLang::EnCa => Some(1), - ProofLang::EnGb => Some(2), + // Australian English follows the British policy (`colour`, `centre`, `organise`). + ProofLang::EnGb | ProofLang::EnAu => Some(2), _ => None, } } diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index e818465f..6617b1fb 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -4,7 +4,7 @@ //! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options, //! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder). //! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2). -//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded +//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded //! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also //! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is //! right; never one only another English knows. When it is there, the system locations aren't @@ -18,8 +18,9 @@ //! embedded from craft-i18n is the same list; it is used for hyphenation only.) //! //! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only -//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every -//! folder before the next name. Another English's dictionary is never used: it would mark this +//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then +//! `en_GB`, for Australian), each name in every folder before the next name. Another English's +//! dictionary is never used: it would mark this //! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the //! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE` //! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read; @@ -90,15 +91,22 @@ fn names(lang: ProofLang) -> &'static [&'static str] { match lang { ProofLang::EnGb => &["en_GB", "en-GB"], ProofLang::EnCa => &["en_CA", "en-CA"], + // Australian English's own, else a British one: the British-spelling dictionary it read + // before it had its own (a system en_GB on a build without SCOWL). + ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"], _ => &["en_US", "en-US"], } } +/// How many Englishes have a slot of their own ([`slot`]). +const ENGLISHES: usize = 4; + /// The slot of each English in [`State::slots`]. fn slot(lang: ProofLang) -> usize { match lang { ProofLang::EnGb => 1, ProofLang::EnCa => 2, + ProofLang::EnAu => 3, _ => 0, } } @@ -109,7 +117,7 @@ type Loaded = Option<(Arc, PathBuf)>; /// The English Hunspell dictionaries, under one lock. struct State { /// Per English (see [`slot`]): `None` until first use, then what was found. - slots: [Option; 3], + slots: [Option; ENGLISHES], /// Every `.dic` file read, so Englishes that use the same file share one copy. files: Vec<(PathBuf, Arc)>, /// Counts reloads, so a load that started before one is not kept. @@ -118,7 +126,7 @@ struct State { impl State { const fn new() -> Self { - State { slots: [None, None, None], files: Vec::new(), reloads: 0 } + State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 } } } @@ -181,7 +189,7 @@ fn already_read(state: &RwLock, dic: &Path) -> Option> { /// Forget what `state` holds, so the next check looks again. fn forget(state: &RwLock) { let mut s = state.write().unwrap_or_else(PoisonError::into_inner); - s.slots = [None, None, None]; + s.slots = [None, None, None, None]; s.files.clear(); s.reloads = s.reloads.wrapping_add(1); } @@ -240,7 +248,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) { /// folder changed). pub(crate) fn reload() { forget(&HUNSPELL); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] { crate::hunspell::clear_load_error(key(lang)); } } @@ -609,7 +617,7 @@ mod tests { fn lookup_order() { let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap()); let file = PathBuf::from("/folder/en_US.dic"); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}"); let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { @@ -638,6 +646,11 @@ mod tests { assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB"); assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]); assert_eq!(found(names(ProofLang::En)), ["en_US"]); + // Australian English: its own, else the British one it read before it had its own. + assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]); + std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap(); + assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]); let _ = std::fs::remove_dir_all(&dir); } diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 993e4030..463406a9 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -5,7 +5,7 @@ /// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)] pub enum ProofLang { - /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or + /// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or /// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the /// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`, /// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also @@ -13,10 +13,14 @@ pub enum ProofLang { /// Word's). #[default] En, - /// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): + /// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…): /// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are /// flagged and `-ize` and `-ise` are both right, as in Oxford spelling. EnGb, + /// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own + /// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and + /// spells some words the Australian way (`program`, `inquiry`). + EnAu, /// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`, /// `defence`) with American `-ize` (`organize`), `tire` and `program`. EnCa, @@ -86,9 +90,10 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 15] = [ + pub const PROOFED: [ProofLang; 16] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, @@ -155,12 +160,13 @@ impl ProofLang { } } - /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions - /// that spell the British way are [`ProofLang::EnGb`], and the United States, any other + /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia + /// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other /// region and no region are [`ProofLang::En`]. fn english(tag: &str) -> ProofLang { match region(tag) { Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, + Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu, Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, _ => ProofLang::En, } @@ -189,9 +195,10 @@ impl ProofLang { } } - /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`], + /// [`ProofLang::EnCa`])? pub fn is_english(self) -> bool { - matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) + matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa) } /// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], @@ -208,7 +215,7 @@ impl ProofLang { pub fn code(self) -> &'static str { match self { ProofLang::Pack(_) => "", - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en", ProofLang::Pl => "pl", ProofLang::Bg => "bg", ProofLang::Pt => "pt", @@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> { const LATIN_AMERICA: [&str; 22] = ["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"]; -/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and -/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other +/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own, +/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other /// Commonwealth countries, the Crown dependencies and Europe (`150`). -const BRITISH_SPELLING: [&str; 26] = [ - "GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", - "PK", "150", +const BRITISH_SPELLING: [&str; 25] = [ + "GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK", + "150", ]; /// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when @@ -469,26 +476,12 @@ mod tests { fn tags_resolve_english_regions() { assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En); assert_tags( - &[ - "en-GB", - "EN_gb", - "en_GB.UTF-8", - "en-AU", - "en_AU", - "en-NZ", - "en-IE", - "en-ZA", - "en-IN", - "en-Latn-GB", - "en-UK", - "en-150", - "eng-GB", - "en_GB@euro", - ], + &["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"], ProofLang::EnGb, ); assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa); - for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu); + for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}"); assert_eq!(language.code(), "en"); } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 9c19999d..d10708cf 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -340,7 +340,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> /// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the /// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]). pub fn dictionary_load_error(lang: ProofLang) -> Option { - if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) { + if lang.is_english() { return english::load_error(lang); } if let Some(l) = lang.spelling_locale() { @@ -953,7 +953,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), - ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), // `pt-pt` shares Portuguese's rules, its issues in its own language. _ => language(lang.built_in()) .map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()), @@ -1197,7 +1197,7 @@ mod tests { } // Every proofed language other than English has a Hunspell dictionary to report. for l in ProofLang::PROOFED { - let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa); + let english = l.is_english(); assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}"); } for l in [ProofLang::En, ProofLang::Other] { @@ -2481,6 +2481,33 @@ mod tests { } } + /// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships + /// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells + /// them; British English doesn't take them. The British policy still applies: American + /// spellings are wrong, `-ise` and `-ize` both right. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_reads_its_own_scowl() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + assert!(scowl::is_british(au)); + assert_eq!(english_source(au), Some(EnglishSource::Embedded)); + for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}"); + } + for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] { + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["color", "center", "analyze", "reanalyze", "defense"] { + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky."; + assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au)); + assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb)); + } + /// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae, /// with their inflections. #[test] @@ -2488,7 +2515,7 @@ mod tests { // English results depend on the dictionaries other tests install. crate::hold_dictionary_tests(); let gb = ProofLang::EnGb; - assert_eq!(ProofLang::from_tag(Some("en-AU")), gb); + assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu); for w in [ "goitre", "goitres", @@ -2513,6 +2540,7 @@ mod tests { "encyclopaedias", ] { assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}"); } let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia."; assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); @@ -2529,7 +2557,8 @@ mod tests { assert_eq!(english_source(l), Some(EnglishSource::Embedded)); assert_eq!(dictionary_load_error(l), None); } - assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB"); + assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own"); + assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB"); let notice = scowl::notice().unwrap(); for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] { assert!(notice.contains(credit), "{credit}"); diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs index 2f429985..cdd9025d 100644 --- a/crates/proof/src/no_data.rs +++ b/crates/proof/src/no_data.rs @@ -7,9 +7,10 @@ use crate::hyphen::{self, Limits, Source}; use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns}; -const LANGS: [ProofLang; 18] = [ +const LANGS: [ProofLang; 19] = [ ProofLang::En, ProofLang::EnGb, + ProofLang::EnAu, ProofLang::EnCa, ProofLang::Pl, ProofLang::Bg, @@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [ #[test] fn nothing_is_embedded() { assert!(crate::data::EN_US_DIC.is_empty()); - assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty()); + assert!( + crate::data::EN_US_SPELLING.is_empty() + && crate::data::EN_GB_SPELLING.is_empty() + && crate::data::EN_CA_SPELLING.is_empty() + && crate::data::EN_AU_SPELLING.is_empty() + ); assert!(crate::scowl::notice().is_none()); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none()); } assert!(dict::Dictionary::en_us().is_empty()); @@ -63,7 +69,7 @@ fn nothing_is_embedded() { #[test] fn english_spelling_still_marks_misspellings() { crate::hold_dictionary_tests(); - for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] { let source = crate::english_source(l); assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); // Any English dictionary marks junk and knows `the`; the word-by-word results below are diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 2fb291e9..7424cd11 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -531,7 +531,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec { ProofLang::EsUy => &["es-uy", "es-419", "es"], ProofLang::De => &["de", "de-1996", "de-de"], ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l), - ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[], + ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[], }; names.iter().map(|n| (*n).to_string()).collect() } diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index 9a71188f..e9f41b9e 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -2,8 +2,11 @@ //! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`, //! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use. //! -//! Each English has its own: American English reads en-US, the British-spelling Englishes -//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions +//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's +//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling +//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no +//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU +//! data, so it stays on en-GB. Its suggestions //! come from its own dictionary, and a word is right when its own dictionary knows it: the other //! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL //! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word @@ -19,16 +22,17 @@ use std::sync::OnceLock; use crate::ProofLang; use crate::affix::AffixDict; -use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; +use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE}; -/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there. -static DICTS: [OnceLock>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()]; +/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there. +static DICTS: [OnceLock>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()]; /// The slot, `.aff` and `.dic` text of `lang`'s dictionary. fn source(lang: ProofLang) -> (usize, &'static str, &'static str) { match lang { ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING), ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING), + ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING), _ => (0, EN_US_AFF, EN_US_SPELLING), } } @@ -50,9 +54,9 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> { loaded.as_ref().ok() } -/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)? +/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)? pub(crate) fn is_british(lang: ProofLang) -> bool { - source(lang).0 == 1 + matches!(source(lang).0, 1 | 3) } /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it. From 87defe169f90289ce074aaa2fabd94e301f9a0f0 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:04:07 +0000 Subject: [PATCH 03/23] =?UTF-8?q?FU.3:=20craft-proof=20pack=20languages=20?= =?UTF-8?q?=E2=80=94=20missing-dictionary=20list=20forgets=20its=20oldest,?= =?UTF-8?q?=20not=20all;=20a=20panic=20inside=20a=20check=20unpins=20its?= =?UTF-8?q?=20dictionaries;=20a=20word=20in=20another=20script=20than=20th?= =?UTF-8?q?e=20dictionary's=20isn't=20marked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from craft-libs#485: `CHECK_FROM` is reset by a drop guard; `remember_missing` drops the oldest of 256; a word none of whose letters the dictionary's words use (a Cyrillic word in a Dutch run) is left alone, as Word does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/lib.rs | 24 +++++++ crates/proof/src/pack_lang.rs | 130 +++++++++++++++++++++++++++++----- 2 files changed, 137 insertions(+), 17 deletions(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index d10708cf..4cfb9a5d 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1709,6 +1709,30 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// Review follow-up (craft-libs#485): a word in another script than a pack language's + /// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a + /// misspelling in the dictionary's own script still is. + #[test] + fn pack_languages_leave_other_scripts_alone() { + crate::hold_dictionary_tests(); + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]); + write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]); + packs::set_packs_dir(Some(root.clone())); + let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR"))); + assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl)); + assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl)); + assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el)); + let text = "Пушкин huiss"; + let issues = check_text(text, &[(0..text.len(), nl)]); + assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::>(), ["huiss"], "{issues:?}"); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + /// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the /// system's only dictionary folder. fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option { diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 6c116653..56e8d090 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -6,6 +6,9 @@ //! files); the one used least recently goes first, but never one used during the current //! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales //! with no dictionary are remembered apart and take no room. +//! +//! A word none of whose letters the dictionary's words use (a Cyrillic word in a Dutch run) isn't +//! this language's: it is never marked, as Word leaves a word in another script alone. use std::cell::Cell; use std::path::PathBuf; @@ -20,8 +23,10 @@ use crate::lang::PackLocale; pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20; /// Estimated memory per byte of `.aff` and `.dic` file. const BYTES_PER_FILE_BYTE: u64 = 6; -/// Most locales remembered as having no dictionary (the list is cleared when full). +/// Most locales remembered as having no dictionary (the oldest is forgotten first when full). const MAX_MISSING: usize = 256; +/// Most distinct letters kept in a dictionary's alphabet ([`alphabet`]). +const MAX_ALPHABET: usize = 4096; /// The budget in force (tests lower it). static BUDGET: AtomicU64 = AtomicU64::new(MAX_LOADED_BYTES); @@ -36,6 +41,8 @@ thread_local! { struct Loaded { locale: PackLocale, dict: Arc, + /// The lowercase letters its words use ([`alphabet`]), sorted. + alphabet: Arc<[char]>, /// The `.dic` it came from. file: Option, /// Estimated memory. @@ -66,15 +73,48 @@ pub(crate) fn reload() { /// over the budget. pub(crate) fn in_one_check(f: impl FnOnce() -> R) -> R { let outer = CHECK_FROM.with(Cell::get); - if outer.is_none() { + let _end = outer.is_none().then(|| { let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock; CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1)))); - } - let r = f(); - if outer.is_none() { + EndCheck + }); + f() +} + +/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches +/// it), or every dictionary would stay pinned on this thread and the budget never hold again. +struct EndCheck; + +impl Drop for EndCheck { + fn drop(&mut self) { CHECK_FROM.with(|c| c.set(None)); } - r +} + +/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is +/// forgotten (clearing them all would look every other one up again). +fn remember_missing(missing: &mut Vec, locale: PackLocale) { + if missing.len() >= MAX_MISSING { + missing.remove(0); + } + missing.push(locale); +} + +/// The lowercase letters the words of `d` use, sorted (at most [`MAX_ALPHABET`]). +fn alphabet(d: &AffixDict) -> Arc<[char]> { + let mut seen = std::collections::BTreeSet::new(); + for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase) { + if seen.len() >= MAX_ALPHABET { + break; + } + seen.insert(c); + } + seen.into_iter().collect() +} + +/// Is `word` in another script than `alphabet`'s words: none of its letters is in it? +fn other_script(word: &str, alphabet: &[char]) -> bool { + !alphabet.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase).any(|c| alphabet.binary_search(&c).is_ok()) } #[cfg(test)] @@ -149,30 +189,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec { /// `locale`'s dictionary, loaded on first use; an empty one when there is none. fn get(locale: PackLocale) -> (Arc, Option) { + let (d, file, _) = get_with_alphabet(locale); + (d, file) +} + +/// [`get`], with the dictionary's [`alphabet`]. +fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Arc<[char]>) { let reloads = { let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); c.clock = c.clock.wrapping_add(1); let now = c.clock; if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { l.used = now; - return (l.dict.clone(), l.file.clone()); + return (l.dict.clone(), l.file.clone(), l.alphabet.clone()); } if c.missing.contains(&locale) { - return (Arc::default(), None); + return (Arc::default(), None, Arc::from([])); } c.reloads }; let (dict, file, bytes) = load(locale); + let alphabet = alphabet(&dict); let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) { - return (dict, file); + return (dict, file, alphabet); } if dict.is_empty() { - if c.missing.len() >= MAX_MISSING { - c.missing.clear(); - } - c.missing.push(locale); - return (dict, file); + remember_missing(&mut c.missing, locale); + return (dict, file, alphabet); } LOADS.fetch_add(1, Ordering::Relaxed); let budget = BUDGET.load(Ordering::Relaxed); @@ -187,8 +231,8 @@ fn get(locale: PackLocale) -> (Arc, Option) { total = total.saturating_sub(gone.bytes); } let used = c.clock; - c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used }); - (dict, file) + c.loaded.push(Loaded { locale, dict: dict.clone(), alphabet: alphabet.clone(), file: file.clone(), bytes, used }); + (dict, file, alphabet) } /// Load `locale`'s dictionary, with its estimated memory. @@ -229,11 +273,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option { /// Is `word` right in `locale`? Every word is when there is no dictionary. pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { - let (d, _) = get(locale); + let (d, _, alphabet) = get_with_alphabet(locale); if d.is_empty() { return true; } let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true }; + if other_script(w, &alphabet) { + return true; + } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); if w.contains('-') && !known(w) { return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); @@ -264,6 +311,55 @@ mod tests { PackLocale::new(t).unwrap() } + /// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its + /// oldest one, not all of them, so a document in more languages doesn't look every one up again. + #[test] + fn missing_locales_forget_the_oldest_first() { + let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char)); + let mut missing = Vec::new(); + for i in 0..MAX_MISSING + 10 { + remember_missing(&mut missing, tag(i)); + assert!(missing.len() <= MAX_MISSING); + } + assert_eq!(missing.len(), MAX_MISSING); + assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first"); + assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay"); + } + + /// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave + /// this thread's dictionaries pinned past the budget for good. + #[test] + fn a_panic_inside_a_check_ends_it() { + let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check"))); + assert!(caught.is_err()); + assert_eq!(CHECK_FROM.with(Cell::get), None); + in_one_check(|| { + assert!(CHECK_FROM.with(Cell::get).is_some()); + in_one_check(|| ()); + assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one"); + }); + assert_eq!(CHECK_FROM.with(Cell::get), None); + } + + /// Review follow-up (craft-libs#485): a word none of whose letters the dictionary uses is in + /// another script (a Cyrillic word in a Dutch run); a word sharing any letter is checked. + #[test] + fn words_in_another_script_are_not_the_dictionarys() { + let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap(); + let a = alphabet(&nl); + assert_eq!( + &a[..], + ['a', 'h', 'i', 'k', 's', 't', 'u', 'é', 'n'].iter().copied().collect::>().into_iter().collect::>() + ); + for w in ["привет", "Привет", "αβγ", "שלום"] { + assert!(other_script(w, &a), "{w}"); + } + for w in ["huus", "kät", "huisп", "ÉÉN"] { + assert!(!other_script(w, &a), "{w}"); + } + assert!(!other_script("привет", &[]), "no alphabet: every word is checked"); + } + #[test] fn locales_and_file_names() { assert_eq!(l("nl-NL").as_str(), "nl-nl"); From 86345ebcf5cdff0c24e110e4592be9d82b8a95b9 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:05:20 +0000 Subject: [PATCH 04/23] =?UTF-8?q?FU.4:=20craft-proof=20packs=20=E2=80=94?= =?UTF-8?q?=20remembered=20hyphenation=20patterns=20keep=20to=20a=20256=20?= =?UTF-8?q?MB=20budget=20of=20their=20own;=20a=20pack's=20spelling=20files?= =?UTF-8?q?=20are=20never=20candidates=20through=20a=20symbolic=20link?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from craft-libs#485: up to 256 languages' pack patterns (64 MB files each) were kept by count only; `hunspell::ordered` checked pack files with `is_file`, which follows links, while the pack scan refuses them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/hunspell.rs | 45 +++++++++++++++++++++----- crates/proof/src/packs.rs | 61 ++++++++++++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 274af9eb..966ded72 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -196,19 +196,22 @@ pub(crate) fn ordered( let first = names.first().map(|n| norm(n)); let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first); let pack: &[&str] = &[crate::packs::SPELLING]; + // (folder, names, a Language pack's: never read through a symbolic link, as the pack scan + // checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`). let tiers = dirs .iter() - .map(|d| (d, names)) - .chain(region_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, ®ion[..]))) - .chain(other_packs.iter().map(|d| (d, pack))) - .chain(system.iter().map(|d| (d, &rest[..]))); + .map(|d| (d, names, false)) + .chain(region_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, ®ion[..], false))) + .chain(other_packs.iter().map(|d| (d, pack, true))) + .chain(system.iter().map(|d| (d, &rest[..], false))); + let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() }; let mut out: Vec<(PathBuf, PathBuf)> = Vec::new(); - for (dir, names) in tiers { + for (dir, names, in_pack) in tiers { for n in names { let aff = dir.join(format!("{n}.aff")); let dic = dir.join(format!("{n}.dic")); - if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) { + if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) { out.push((aff, dic)); } } @@ -696,6 +699,34 @@ mod tests { } } + /// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link + /// (swapped in after the pack scan) is never a candidate; one in a system folder still is. + #[cfg(unix)] + #[test] + fn pack_files_are_never_read_through_a_link() { + let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere")); + for d in [&pack, &system, &elsewhere] { + std::fs::create_dir_all(d).unwrap(); + } + std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap(); + std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap(); + std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap(); + std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never"); + std::fs::remove_file(pack.join("spelling.dic")).unwrap(); + std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap(); + let got: Vec = + ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect(); + assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]); + let _ = std::fs::remove_dir_all(&root); + } + /// WordCraft's own dictionaries folder under the user's application data is searched on every /// desktop system, after the setting and before the shared system locations (#1063). #[test] diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 7424cd11..0d6cfbc8 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling"; const HYPHENATION: &str = "hyphenation.pat"; /// Most languages whose pack patterns are remembered (a document may tag thousands). const MAX_HYPHENATION_LOOKUPS: usize = 256; +/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]): +/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to +/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the +/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`). +const MAX_HYPHENATION_BYTES: u64 = 256 << 20; +/// Estimated memory per pattern (its packed key, its values and the map's own room). +const BYTES_PER_PATTERN: u64 = 64; /// A pack's description. const PACK_TOML: &str = "pack.toml"; /// A pack's licence texts. @@ -80,15 +87,18 @@ struct State { dir: Option, scan: Option>, /// Each language's pack patterns once looked up (`None`: no pack has them), at most - /// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full). + /// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full). hyphenation: BTreeMap>>, + /// The estimated memory of the patterns in [`State::hyphenation`]. + hyphenation_bytes: u64, /// (locale, reason) of pack hyphenation files that failed to load. hyphenation_errors: Vec<(String, String)>, /// Counts changes, so a scan or load that started before one is not kept. changes: u64, } -static STATE: RwLock = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 }); +static STATE: RwLock = + RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }); fn read() -> RwLockReadGuard<'static, State> { STATE.read().unwrap_or_else(PoisonError::into_inner) @@ -121,6 +131,7 @@ pub fn reload() { fn forget(s: &mut State) { s.scan = None; s.hyphenation.clear(); + s.hyphenation_bytes = 0; s.hyphenation_errors.clear(); s.changes = s.changes.wrapping_add(1); } @@ -566,10 +577,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { let mut s = write(); if s.changes == changes && !s.hyphenation.contains_key(&lang) { // ponytail: cleared when full, an LRU if many languages each need their patterns. - if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS { - s.hyphenation.clear(); - } - s.hyphenation.insert(lang, found.clone()); + let bytes = found.as_deref().map_or(0, pattern_bytes); + remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES); for e in errors { if !s.hyphenation_errors.contains(&e) { s.hyphenation_errors.push(e); @@ -579,6 +588,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option> { found } +/// Estimated memory of `p`. +fn pattern_bytes(p: &Patterns) -> u64 { + (p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN) +} + +/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages +/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the +/// whole budget are still kept alone, or they would be read from disk for every word. +fn remember_patterns(s: &mut State, lang: ProofLang, found: Option>, bytes: u64, budget: u64) { + if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget { + s.hyphenation.clear(); + s.hyphenation_bytes = 0; + } + s.hyphenation.insert(lang, found); + s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes); +} + /// How many languages' pack patterns are remembered now. #[cfg(test)] pub(crate) fn hyphenation_lookups() -> usize { @@ -600,6 +626,29 @@ fn load_patterns(path: &Path) -> Result { mod tests { use super::*; + /// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not + /// only a count, so 256 languages of large patterns can't each stay in memory. + #[test] + fn remembered_patterns_keep_to_a_memory_budget() { + let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 }; + let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n")); + let bytes = pattern_bytes(&p); + assert_eq!(bytes, 9 * BYTES_PER_PATTERN); + let lang = |t: &str| ProofLang::from_tag(Some(t)); + let budget = bytes * 2; + remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget); + remember_patterns(&mut s, lang("sv"), None, 0, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room"); + remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten"); + assert!(s.hyphenation.contains_key(&lang("da"))); + remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone"); + forget(&mut s); + assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0)); + } + /// Review of #485: a pack's name loses bidirectional and other format characters. #[test] fn pack_names_are_plain_text() { From 1f345ea80974844ffaec1c9b5bfeb5099d5cb34c Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:06:36 +0000 Subject: [PATCH 05/23] =?UTF-8?q?FU.5:=20craft-proof=20id,=20ru=20?= =?UTF-8?q?=E2=80=94=20the=20cased-letter=20rule=20for=20words=20in=20capi?= =?UTF-8?q?tals=20(craft-libs#485):=20a=20caseless=20part=20is=20no=20abbr?= =?UTF-8?q?eviation;=20shared=20in=5Fcapitals=20/=20capitals=5Fpart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/id/mod.rs | 15 ++++++++++++++- crates/proof/src/lib.rs | 2 +- crates/proof/src/ru/mod.rs | 6 ++++-- crates/proof/src/spelling.rs | 30 ++++++++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 6 deletions(-) diff --git a/crates/proof/src/id/mod.rs b/crates/proof/src/id/mod.rs index c78e25d4..85dc825a 100644 --- a/crates/proof/src/id/mod.rs +++ b/crates/proof/src/id/mod.rs @@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { || w.split('-').all(|p| { p.chars().count() <= 1 || CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c)) - || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6) + || (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6)) || known(d, p) }); } @@ -141,6 +141,19 @@ mod tests { } } + /// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no + /// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters + /// still is one (`KTP-nya`, `COVID-19` with numbers checked). + #[test] + fn caseless_parts_are_not_abbreviations() { + test_dict::install(); + let d = dictionary(); + let defaults = crate::Options::default(); + assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults)); + let numbers = crate::Options { ignore_numbers: false, ..defaults }; + assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers)); + } + #[test] fn forbidden_words_are_rejected_in_every_case() { test_dict::install(); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 4cfb9a5d..561c3541 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -669,7 +669,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { || !w.chars().any(char::is_alphabetic) || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) // Only a word with a cased letter is in capitals: a Hebrew or Thai word has none. - || (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())) + || (opts.ignore_uppercase && spelling::in_capitals(w)) // The whole word: the languages split a hyphenated word into its parts, and an added // `Unigranrio-RJ` is none of them (#2082). || in_user_dictionary(&w.to_lowercase()) diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index b88978af..6e9ee34d 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) { return true; } - if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS { + if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS { return true; } // Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word. @@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { let last = parts.len() - 1; parts.iter().enumerate().all(|(i, p)| { // An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word. - if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) { + if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) { return true; } // A first part with a linking vowel: the stem (`северо` → `север`) or its adjective @@ -485,5 +485,7 @@ mod tests { assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too"); assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults"); + // The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations. + assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits)); } } diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs index c80f42fe..1b42f5ff 100644 --- a/crates/proof/src/spelling.rs +++ b/crates/proof/src/spelling.rs @@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options) if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { return None; } - // Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none. - if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + if opts.ignore_uppercase && in_capitals(w) && n <= 6 { return None; } Some(w) } +/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can +/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms. +pub(crate) fn in_capitals(w: &str) -> bool { + w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) +} + +/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max` +/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A +/// part in a caseless script is no abbreviation. +pub(crate) fn capitals_part(part: &str, max: usize) -> bool { + part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic)) +} + #[cfg(test)] mod tests { use super::*; + /// craft-libs#485's cased-letter rule, shared by every language: a word or part is in + /// capitals only with a cased letter; a part without letters counts as an abbreviation. + #[test] + fn capitals_need_a_cased_letter() { + for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] { + assert!(in_capitals(w) && capitals_part(w, 6), "{w}"); + } + for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] { + assert!(!in_capitals(w), "{w}"); + } + assert!(capitals_part("19", 6) && capitals_part("", 6)); + assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6)); + } + #[test] fn words_needing_no_lookup() { let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); From 8712aa105a88619458b2081eeb569f089af0cb66 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:09:16 +0000 Subject: [PATCH 06/23] =?UTF-8?q?FU.6:=20craft-proof=20affix=20=E2=80=94?= =?UTF-8?q?=20the=20stems'=20flags=20are=20budgeted=20while=20a=20dictiona?= =?UTF-8?q?ry=20is=20read=20(running=20counts=20of=20flag=20text=20and=20k?= =?UTF-8?q?eys),=20so=20a=20huge=20.dic=20fails=20early=20instead=20of=20a?= =?UTF-8?q?fter=20holding=20every=20entry's=20flags=20(#495=20P3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/affix.rs | 103 +++++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 17 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 63d6f289..c3ac63ab 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -74,6 +74,14 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; +/// Most flags all stems may carry together before their sets are merged and interned (the same +/// set on many stems counts each time; an `AF` alias counts once). Checked while reading, so a +/// huge `.dic` is refused before its flags fill memory, not after (#495: a 760 MB `.dic` once +/// held 3 GB before [`MAX_SET_DATA`] refused it). +const MAX_STEM_FLAGS: usize = 4 * MAX_SET_DATA; +/// Most bytes of flag text per flag of [`MAX_STEM_FLAGS`] the stem lines may hold in all while +/// they are read (`FLAG num`: `65535,` is six). +const FLAG_TEXT_PER_FLAG: usize = 6; /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; @@ -972,12 +980,19 @@ impl AffixDict { } fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result { + AffixDict::from_hunspell_capped(aff, dic, comments, english, MAX_STEM_FLAGS) + } + + /// [`AffixDict::from_hunspell_in`] with at most `max_flags` stem flags in all + /// ([`MAX_STEM_FLAGS`]; tests lower it). + fn from_hunspell_capped(aff: &str, dic: &str, comments: &[&str], english: bool, max_flags: usize) -> Result { // Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first // directive or turn the `.dic` count line into a stem. let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff); let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic); let rules = parse_aff(aff, english)?; let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(max_flags); for (n, line) in dic.lines().enumerate() { // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). @@ -1013,14 +1028,16 @@ impl AffixDict { } // With an `AF` table the field stays an alias number here: build() reads each alias // once, and every stem using it shares those flags. + flag_text.add(&flags)?; stems.push((word, flags)); } - AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY) + AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags) } /// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each /// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data). - fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize) -> Result { + /// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read. + fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec, entry_cap: usize, max_flags: usize) -> Result { let mode = rules.mode; // A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an // entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read, @@ -1049,20 +1066,24 @@ impl AffixDict { } Some((start, stem_keys.len(), word)) }; - let mut stems: Vec<(String, usize, usize, bool)> = stems - .into_iter() - .filter_map(|(w, f)| { - let range = match rules.alias_index(&f) { - Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), - None if rules.aliases_on => Some((0, 0, true)), - None => read(&f, &mut stem_keys), - }; - if range.is_none_or(|r| !r.2) { - skipped += 1; - } - range.map(|(a, b, word)| (w, a, b, word)) - }) - .collect(); + let mut read_stems: Vec<(String, usize, usize, bool)> = Vec::with_capacity(stems.len()); + for (w, f) in stems { + let range = match rules.alias_index(&f) { + Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)), + None if rules.aliases_on => Some((0, 0, true)), + None => read(&f, &mut stem_keys), + }; + if stem_keys.len() > max_flags { + return Err(format!("the stems carry more than {max_flags} flags in all")); + } + if range.is_none_or(|r| !r.2) { + skipped += 1; + } + if let Some((a, b, word)) = range { + read_stems.push((w, a, b, word)); + } + } + let mut stems = read_stems; if skipped > 0 { log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped"); } @@ -2059,6 +2080,7 @@ impl AffixDict { } let body = raw.get(words_at + 7..).unwrap_or_default(); let mut stems: Vec<(String, String)> = Vec::new(); + let mut flag_text = FlagText::new(MAX_STEM_FLAGS); let mut cur: Vec = Vec::new(); let mut i = 0; while let Some(&p) = body.get(i) { @@ -2081,9 +2103,32 @@ impl AffixDict { if word.is_empty() || word.len() > MAX_WORD_BYTES { return Err("affix dictionary: bad stem".into()); } + flag_text.add(flags).map_err(|e| format!("affix dictionary: {e}"))?; stems.push((word.to_string(), flags.to_string())); } - AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM) + AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM, MAX_STEM_FLAGS) + } +} + +/// A running count of the stems' flag text while a dictionary is read: past +/// [`FLAG_TEXT_PER_FLAG`] bytes for each of `max_flags` flags, reading stops with an error, before +/// the rest of a huge `.dic` is held ([`MAX_STEM_FLAGS`]). +struct FlagText { + bytes: usize, + cap: usize, +} + +impl FlagText { + fn new(max_flags: usize) -> Self { + FlagText { bytes: 0, cap: max_flags.saturating_mul(FLAG_TEXT_PER_FLAG) } + } + + fn add(&mut self, flags: &str) -> Result<(), String> { + self.bytes = self.bytes.saturating_add(flags.len()); + if self.bytes > self.cap { + return Err(format!("the stems' flags take more than {} bytes", self.cap)); + } + Ok(()) } } @@ -2133,6 +2178,30 @@ pub(crate) mod tests { AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap() } + /// #495 review: the stems' flags are counted as they are read, so a `.dic` whose flags pass + /// the budget is refused while reading (its flag text) or while its flags are read (the + /// keys), before they are all held; one within it loads. + #[test] + fn stem_flags_are_budgeted_while_reading() { + let aff = "SET UTF-8\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x .\n"; + let many: String = (0..100).map(|i| format!("w{i}/AB\n")).collect(); + let dic = format!("100\n{many}"); + // 200 flags: over a budget of 50 keys (whose flag text, 300 bytes, still fits). + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 50).unwrap_err(); + assert!(e.contains("more than 50 flags"), "{e}"); + let d = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 200).unwrap(); + assert!(d.check("w7s") && d.check("w99x")); + // Flag text past the budget (entries over the per-entry cap hold no keys, but their text + // still counts): refused while reading the `.dic`. + let long = "A".repeat(MAX_FLAGS_PER_ENTRY + 1); + let dic = format!("3\na/{long}\nb/{long}\nc/{long}\n"); + let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 2000).unwrap_err(); + assert!(e.contains("more than 12000 bytes"), "{e}"); + assert!(AffixDict::from_hunspell(aff, &dic, &[]).is_ok(), "within the real budget: skipped entries, no error"); + let mut text = FlagText::new(1); + assert!(text.add("ABCDEF").is_ok() && text.add("G").is_err()); + } + #[test] fn affix_headers_allow_comments_but_rule_conditions_stay_literal() { let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n"; From 65ce4f882a31c3b208287189c8cb16dfbbfbd640 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 00:24:58 +0000 Subject: [PATCH 07/23] =?UTF-8?q?FU.7:=20craft-proof=20pack=20languages=20?= =?UTF-8?q?=E2=80=94=20another=20script=20is=20told=20by=20the=20letters'?= =?UTF-8?q?=20Unicode=20scripts,=20not=20the=20dictionary's=20letter=20inv?= =?UTF-8?q?entory,=20so=20a=20misspelling=20in=20the=20dictionary's=20own?= =?UTF-8?q?=20script=20(`zzzz`=20in=20Dutch)=20is=20always=20checked?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by a local CodeRabbit review of #496 (major). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/src/lib.rs | 1 + crates/proof/src/pack_lang.rs | 119 +++++++++++++++++++++++----------- 2 files changed, 81 insertions(+), 39 deletions(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 561c3541..b4b25f5e 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -1723,6 +1723,7 @@ mod tests { packs::set_packs_dir(Some(root.clone())); let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR"))); assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl)); + assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters"); assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl)); assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el)); let text = "Пушкин huiss"; diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 56e8d090..7b4d1931 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -7,8 +7,9 @@ //! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales //! with no dictionary are remembered apart and take no room. //! -//! A word none of whose letters the dictionary's words use (a Cyrillic word in a Dutch run) isn't -//! this language's: it is never marked, as Word leaves a word in another script alone. +//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic +//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another +//! script alone. A word in the dictionary's script is always checked, whatever its letters. use std::cell::Cell; use std::path::PathBuf; @@ -25,8 +26,6 @@ pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20; const BYTES_PER_FILE_BYTE: u64 = 6; /// Most locales remembered as having no dictionary (the oldest is forgotten first when full). const MAX_MISSING: usize = 256; -/// Most distinct letters kept in a dictionary's alphabet ([`alphabet`]). -const MAX_ALPHABET: usize = 4096; /// The budget in force (tests lower it). static BUDGET: AtomicU64 = AtomicU64::new(MAX_LOADED_BYTES); @@ -41,8 +40,8 @@ thread_local! { struct Loaded { locale: PackLocale, dict: Arc, - /// The lowercase letters its words use ([`alphabet`]), sorted. - alphabet: Arc<[char]>, + /// The scripts its words are written in ([`scripts`]), sorted. + scripts: Arc<[u32]>, /// The `.dic` it came from. file: Option, /// Estimated memory. @@ -100,21 +99,61 @@ fn remember_missing(missing: &mut Vec, locale: PackLocale) { missing.push(locale); } -/// The lowercase letters the words of `d` use, sorted (at most [`MAX_ALPHABET`]). -fn alphabet(d: &AffixDict) -> Arc<[char]> { +/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks +/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana, +/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named. +fn script_of(c: char) -> u32 { + const SCRIPTS: &[(u32, u32, u32)] = &[ + (0x0041, 0x024F, 1), + (0x0250, 0x02AF, 1), + (0x1E00, 0x1EFF, 1), + (0x2C60, 0x2C7F, 1), + (0xA720, 0xA7FF, 1), + (0xAB30, 0xAB6F, 1), + (0xFF21, 0xFF5A, 1), + (0x0370, 0x03FF, 2), + (0x1F00, 0x1FFF, 2), + (0x0400, 0x052F, 3), + (0x1C80, 0x1C8F, 3), + (0x2DE0, 0x2DFF, 3), + (0xA640, 0xA69F, 3), + (0x0530, 0x058F, 4), + (0xFB13, 0xFB17, 4), + (0x0590, 0x05FF, 5), + (0xFB1D, 0xFB4F, 5), + (0x0600, 0x06FF, 6), + (0x0750, 0x077F, 6), + (0x08A0, 0x08FF, 6), + (0xFB50, 0xFDFF, 6), + (0xFE70, 0xFEFF, 6), + (0x10A0, 0x10FF, 7), + (0x1C90, 0x1CBF, 7), + (0x2D00, 0x2D2F, 7), + (0x1100, 0x11FF, 8), + (0x3130, 0x318F, 8), + (0xAC00, 0xD7AF, 8), + (0x3040, 0x30FF, 9), + (0x3400, 0x4DBF, 10), + (0x4E00, 0x9FFF, 10), + (0x1200, 0x139F, 11), + ]; + let u = u32::from(c); + SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s) +} + +/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted. +fn scripts(d: &AffixDict) -> Arc<[u32]> { let mut seen = std::collections::BTreeSet::new(); - for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase) { - if seen.len() >= MAX_ALPHABET { - break; - } - seen.insert(c); + for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) { + seen.insert(script_of(c)); } seen.into_iter().collect() } -/// Is `word` in another script than `alphabet`'s words: none of its letters is in it? -fn other_script(word: &str, alphabet: &[char]) -> bool { - !alphabet.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).flat_map(char::to_lowercase).any(|c| alphabet.binary_search(&c).is_ok()) +/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one +/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch). +fn other_script(word: &str, scripts: &[u32]) -> bool { + !scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok()) } #[cfg(test)] @@ -189,19 +228,19 @@ pub(crate) fn names(locale: PackLocale) -> Vec { /// `locale`'s dictionary, loaded on first use; an empty one when there is none. fn get(locale: PackLocale) -> (Arc, Option) { - let (d, file, _) = get_with_alphabet(locale); + let (d, file, _) = get_with_scripts(locale); (d, file) } -/// [`get`], with the dictionary's [`alphabet`]. -fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Arc<[char]>) { +/// [`get`], with the dictionary's [`scripts`]. +fn get_with_scripts(locale: PackLocale) -> (Arc, Option, Arc<[u32]>) { let reloads = { let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); c.clock = c.clock.wrapping_add(1); let now = c.clock; if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { l.used = now; - return (l.dict.clone(), l.file.clone(), l.alphabet.clone()); + return (l.dict.clone(), l.file.clone(), l.scripts.clone()); } if c.missing.contains(&locale) { return (Arc::default(), None, Arc::from([])); @@ -209,14 +248,14 @@ fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Ar c.reloads }; let (dict, file, bytes) = load(locale); - let alphabet = alphabet(&dict); + let scripts = scripts(&dict); let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) { - return (dict, file, alphabet); + return (dict, file, scripts); } if dict.is_empty() { remember_missing(&mut c.missing, locale); - return (dict, file, alphabet); + return (dict, file, scripts); } LOADS.fetch_add(1, Ordering::Relaxed); let budget = BUDGET.load(Ordering::Relaxed); @@ -231,8 +270,8 @@ fn get_with_alphabet(locale: PackLocale) -> (Arc, Option, Ar total = total.saturating_sub(gone.bytes); } let used = c.clock; - c.loaded.push(Loaded { locale, dict: dict.clone(), alphabet: alphabet.clone(), file: file.clone(), bytes, used }); - (dict, file, alphabet) + c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used }); + (dict, file, scripts) } /// Load `locale`'s dictionary, with its estimated memory. @@ -273,12 +312,12 @@ pub(crate) fn load_error(locale: PackLocale) -> Option { /// Is `word` right in `locale`? Every word is when there is no dictionary. pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { - let (d, _, alphabet) = get_with_alphabet(locale); + let (d, _, scripts) = get_with_scripts(locale); if d.is_empty() { return true; } let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true }; - if other_script(w, &alphabet) { + if other_script(w, &scripts) { return true; } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); @@ -341,23 +380,25 @@ mod tests { assert_eq!(CHECK_FROM.with(Cell::get), None); } - /// Review follow-up (craft-libs#485): a word none of whose letters the dictionary uses is in - /// another script (a Cyrillic word in a Dutch run); a word sharing any letter is checked. + /// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is + /// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked, + /// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`). #[test] fn words_in_another_script_are_not_the_dictionarys() { let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap(); - let a = alphabet(&nl); - assert_eq!( - &a[..], - ['a', 'h', 'i', 'k', 's', 't', 'u', 'é', 'n'].iter().copied().collect::>().into_iter().collect::>() - ); - for w in ["привет", "Привет", "αβγ", "שלום"] { - assert!(other_script(w, &a), "{w}"); + let latin = scripts(&nl); + assert_eq!(latin.len(), 1); + for w in ["привет", "Привет", "αβγ", "שלום", "日本"] { + assert!(other_script(w, &latin), "{w}"); } - for w in ["huus", "kät", "huisп", "ÉÉN"] { - assert!(!other_script(w, &a), "{w}"); + for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] { + assert!(!other_script(w, &latin), "{w}"); } - assert!(!other_script("привет", &[]), "no alphabet: every word is checked"); + assert!(!other_script("привет", &[]), "no scripts: every word is checked"); + let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap(); + assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru))); + assert_ne!(script_of('ა'), script_of('a')); + assert_eq!(script_of('ă'), script_of('Ạ')); } #[test] From 9ef9185131b03245e2f61d3b129e24211728ab57 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:43:25 +0000 Subject: [PATCH 08/23] =?UTF-8?q?FU.8:=20craft-proof=20cs=20=E2=80=94=20re?= =?UTF-8?q?gression=20test=20from=20wordcraft#4683:=20a=20letter=20that=20?= =?UTF-8?q?lowercases=20to=20two=20chars=20(=C4=B0)=20still=20gets=20its?= =?UTF-8?q?=20one-letter=20edits?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared suggestion code already edits across the whole lowercased word, so no code change is needed; the test keeps it that way. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/cs/suggest.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/crates/proof/src/cs/suggest.rs b/crates/proof/src/cs/suggest.rs index 05eba74a..1e31496e 100644 --- a/crates/proof/src/cs/suggest.rs +++ b/crates/proof/src/cs/suggest.rs @@ -75,4 +75,13 @@ mod tests { assert!(!suggest("město", 4).contains(&"město".into())); assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into())); } + + #[test] + fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() { + // wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed + // letters; the one-letter edits stay in range and dropping the dot still finds "ix". + let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap(); + let suggestions = suggest_in(&d, "İx", 4); + assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}"); + } } From 61030150dc71f0eacdef5708103d42d618cc71e7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 09/23] =?UTF-8?q?craft-proof=20cs,=20hr,=20it=20=E2=80=94?= =?UTF-8?q?=20a=20hyphenated=20word=20is=20looked=20up=20whole=20before=20?= =?UTF-8?q?its=20parts,=20and=20an=20elision=20(l'ex-amico)=20is=20read=20?= =?UTF-8?q?in=20each=20part=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/cs/mod.rs | 11 ++++++++++- crates/proof/src/hr/mod.rs | 11 ++++++++++- crates/proof/src/it/mod.rs | 18 +++++++++++++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 2be5e3f0..b1695140 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -142,6 +143,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_and_ignored_inputs_are_quiet() { let empty = AffixDict::default(); diff --git a/crates/proof/src/hr/mod.rs b/crates/proof/src/hr/mod.rs index 53edf067..e0bda809 100644 --- a/crates/proof/src/hr/mod.rs +++ b/crates/proof/src/hr/mod.rs @@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { return true; }; if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + // The whole word first, as Hunspell does: a hyphenated entry or user word. + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -105,6 +106,14 @@ mod tests { } } + /// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn missing_dictionary_is_silent() { assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ")); diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index a742eaa2..44550501 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { } // The whole word first, as Hunspell does: a hyphenated entry or user word. if w.contains('-') { - return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p)); } + known_elided(d, w) +} + +/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after +/// the apostrophe is what gets checked. +fn known_elided(d: &AffixDict, w: &str) -> bool { if let Some((head, tail)) = w.split_once(['\'', '’']) && PROCLITICS.contains(&head.to_lowercase().as_str()) && !tail.is_empty() @@ -148,6 +154,16 @@ mod tests { assert!(!is_correct_with(&d, "tic-tacc")); } + /// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part. + #[test] + fn an_elision_in_a_hyphenated_part_is_read() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap(); + for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] { + assert!(is_correct_with(&d, w), "{w}"); + } + assert!(!is_correct_with(&d, "l'exx-amico")); + } + #[test] fn without_a_dictionary_nothing_is_flagged() { let none = AffixDict::default(); From 15d3c0d370e470f2782a186ad17a9897608d71d5 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 10/23] =?UTF-8?q?craft-proof=20it,=20pt=20=E2=80=94=20narr?= =?UTF-8?q?ation=20after=20a=20quoted=20or=20dashed=20question=20or=20excl?= =?UTF-8?q?amation=20is=20not=20a=20new=20sentence=20(=C2=ABVai=3F=C2=BB?= =?UTF-8?q?=20disse=20lui)=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 22 ++++++++++++++++++++-- crates/proof/src/pt/grammar.rs | 11 ++++++++++- 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 57c62e28..e90e324b 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool { || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) } +const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}']; + /// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after /// an abbreviation), then space, with nothing but quotes and brackets around? fn ends_sentence(gap: &str, prev: &str) -> bool { - let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']); + let t = gap.trim_start_matches(CLOSING); let Some(end) = t.chars().next() else { return false }; if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { return false; @@ -161,7 +163,13 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']); + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or + // exclamation, or after a dialogue dash, goes on in lower case. + if rest.contains('—') || (end != '.' && after_marks.starts_with(CLOSING)) { + return false; + } rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) } @@ -403,6 +411,16 @@ mod tests { } } + /// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in + /// lower case. + #[test] + fn narration_after_a_quoted_question_is_not_a_new_sentence() { + for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] { + assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); + } + assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + } + #[test] fn po_takes_an_apostrophe() { assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]); diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 303929de..be8325bd 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end == '.' && abbreviation_or_number(prev) { return false; } - let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); + let after_marks = t.trim_start_matches(['.', '!', '?']); + let rest = after_marks.trim_start_matches(CLOSING); + // `"Vai?" disse ele.`: narration after a quoted question or exclamation goes on in lower case. + if end != '.' && after_marks.starts_with(CLOSING) { + return false; + } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes // on in lower case. rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c)) @@ -225,6 +230,10 @@ mod tests { // craft-libs#488 review: no space after the dash. clean("—Vai? —disse ele."); clean("— Vai? — disse ele."); + // craft-libs#496 review: a closing quote after the mark. + clean("\"Vai?\" disse ele."); + clean("«Vai!» gritou ela."); + assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); From bd3db196a56a545155c6c09ab535e5808e9b5ae2 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 11/23] =?UTF-8?q?craft-proof=20es=20=E2=80=94=20a=20run=20?= =?UTF-8?q?of=20spaces=20is=20one=20Espacio=20doble=20issue=20over=20the?= =?UTF-8?q?=20whole=20run=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/es/grammar.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 42eede50..687ecac3 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -75,11 +75,21 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + // Each run of two or more spaces once, over the whole run. + let bytes = text.as_bytes(); + let mut i = 0; + while i < bytes.len() { + if bytes.get(i) != Some(&b' ') { + i += 1; continue; } - v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang)); + let start = i; + while bytes.get(i) == Some(&b' ') { + i += 1; + } + if i - start >= 2 { + v.push(issue(start, i, "Espacio doble".into(), vec![" ".into()], lang)); + } } } @@ -194,6 +204,16 @@ mod tests { .collect() } + /// craft-libs#496 review: a run of spaces is one issue over the whole run. + #[test] + fn a_run_of_spaces_is_reported_whole() { + for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] { + let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect(); + assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}"); + } + assert!(messages("casa roja").is_empty()); + } + #[test] fn repeated_words_and_spaces_have_spanish_messages() { let text = "Es es la casa , dijo."; From d2563a77d416eef2bf4a6ee7e0c3cd2ddbced749 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 12/23] =?UTF-8?q?craft-proof=20tr=20=E2=80=94=20te=20and?= =?UTF-8?q?=20ta=20are=20the=20locative=20suffix,=20not=20a=20detached=20p?= =?UTF-8?q?article,=20so=20kitapte=20is=20not=20offered=20as=20kitap=20te?= =?UTF-8?q?=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/tr/suggest.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 0bb705b8..9dbff6f1 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { } } - // 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` - for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] { + // 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü` + for suffix in ["de", "da", "mi", "mı", "mu", "mü"] { if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 { let stem = &lower[..lower.len() - suffix.len()]; if d.is_empty() || d.contains(stem) { @@ -333,6 +333,19 @@ mod tests { assert!(sugs.contains(&"geldin mi".to_string())); } + /// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te` + /// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written + /// attached, so `kitapte` is never offered as `kitap te`. + #[test] + fn the_detached_particle_is_never_te_or_ta() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap(); + assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string())); + for w in ["kitapte", "kitapta"] { + let sugs = suggest_with(&d, w, 8); + assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}"); + } + } + #[test] fn overlong_words_get_no_suggestions_promptly() { let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap(); From 466919a8a5182e61dc3a7fd7420163d8bd8cf121 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 13/23] =?UTF-8?q?craft-proof=20english=20=E2=80=94=20with?= =?UTF-8?q?=20SCOWL=20embedded,=20a=20dictionaries-folder=20en=5FGB=20no?= =?UTF-8?q?=20longer=20outranks=20the=20embedded=20en-AU=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/english.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index 6617b1fb..cd214bac 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -98,6 +98,15 @@ fn names(lang: ProofLang) -> &'static [&'static str] { } } +/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair +/// only, not Australian English's British fallback (a user's `en_GB` is for British text). +fn own_names(lang: ProofLang) -> &'static [&'static str] { + match lang { + ProofLang::EnAu => &["en_AU", "en-AU"], + _ => names(lang), + } +} + /// How many Englishes have a slot of their own ([`slot`]). const ENGLISHES: usize = 4; @@ -206,7 +215,7 @@ fn key(lang: ProofLang) -> &'static str { /// would rather use), for [`note_error`]. fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { let candidates = if crate::scowl::dict(lang).is_some() { - crate::hunspell::user_candidates(names(lang)) + crate::hunspell::user_candidates(own_names(lang)) } else { crate::hunspell::candidates_exact(names(lang)) }; @@ -651,6 +660,14 @@ mod tests { std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap(); std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap(); assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]); + // craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU. + assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]); + std::fs::remove_file(dir.join("en_AU.aff")).unwrap(); + std::fs::remove_file(dir.join("en_AU.dic")).unwrap(); + assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own"); + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { + assert_eq!(own_names(lang), names(lang)); + } let _ = std::fs::remove_dir_all(&dir); } From 54ac5fbc4c94acb728f80f995f020173b915be2d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 21:56:44 +0000 Subject: [PATCH 14/23] =?UTF-8?q?craft-proof=20=E2=80=94=20README:=20embed?= =?UTF-8?q?ded=5Flocales()=20and=20the=20SCOWL=20byte=20count=20match=20bu?= =?UTF-8?q?ild.rs=20with=20en-AU;=20ru:=20drop=20an=20unused=20binding=20(?= =?UTF-8?q?#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/README.md | 4 ++-- crates/proof/src/ru/mod.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 4aeb677b..13ab9347 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -79,7 +79,7 @@ pub fn packs_dir() -> Option; // ＋ pub fn installed() -> Vec; // InstalledPack { locale, name, licence, path }, by locale pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/* pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed -pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us", "de", "pl", "pt", "it", "ru" by default; "en-us" with CRAFT_PROOF_EMBED=en) +pub fn embedded_locales() -> &'static [&'static str]; // ＋ locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en) // craft_proof (crate root) pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…) @@ -200,7 +200,7 @@ craft-i18n's copy. The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window. -Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic` +Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic` and the notice); the Moby list (432,905 bytes) stays for hyphenation. ## Check diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index 6e9ee34d..4a7cd3e0 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -321,7 +321,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool { } match lp.as_slice() { // `кое-кто`, `кой-какой`. - [prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), + [prefix, _] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)), // `по-русски` (from `русский`), `по-новому`, `по-моему`. ["по", rest] => { let Some(r) = parts.get(1) else { return false }; From 7b64b78cb016704a97480d07410064b1b40f5492 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:23:09 +0000 Subject: [PATCH 15/23] =?UTF-8?q?craft-proof=20english=20=E2=80=94=20Austr?= =?UTF-8?q?alian=20English=20also=20takes=20a=20word=20in=20SCOWL's=20en-G?= =?UTF-8?q?B=20list,=20and=20without=20SCOWL=20every=20en=5FAU=20source=20?= =?UTF-8?q?is=20read=20before=20any=20en=5FGB=20one=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/english.rs | 63 ++++++++++++++++++++++++++++++++----- crates/proof/src/lib.rs | 20 ++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index cd214bac..c27f01a9 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -91,13 +91,29 @@ fn names(lang: ProofLang) -> &'static [&'static str] { match lang { ProofLang::EnGb => &["en_GB", "en-GB"], ProofLang::EnCa => &["en_CA", "en-CA"], - // Australian English's own, else a British one: the British-spelling dictionary it read - // before it had its own (a system en_GB on a build without SCOWL). + // Australian English's own, else a British one (a system en_GB on a build without SCOWL): + // [`exact_candidates`] reads every en_AU source before any en_GB one. ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"], _ => &["en_US", "en-US"], } } +/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names, +/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's +/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before +/// any en_GB one. +fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> { + let mut out = find(own_names(lang)); + if lang == ProofLang::EnAu { + for c in find(&names(lang)[own_names(lang).len()..]) { + if !out.iter().any(|(_, d)| *d == c.1) { + out.push(c); + } + } + } + out +} + /// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair /// only, not Australian English's British fallback (a user's `en_GB` is for British text). fn own_names(lang: ProofLang) -> &'static [&'static str] { @@ -217,7 +233,7 @@ fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) { let candidates = if crate::scowl::dict(lang).is_some() { crate::hunspell::user_candidates(own_names(lang)) } else { - crate::hunspell::candidates_exact(names(lang)) + exact_candidates(lang, crate::hunspell::candidates_exact) }; if candidates.is_empty() { log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang)); @@ -300,10 +316,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { if let Some((d, dic)) = found { return Speller::Hunspell(d, dic); } - match crate::scowl::dict(lang) { - Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)), - None => Speller::List, + let Some(d) = crate::scowl::dict(lang) else { return Speller::List }; + let mut ds = vec![d]; + // SCOWL's en-AU list keeps one spelling per word (`enquiry`, not `inquiry`); the British list + // has the rest, which Australian English takes. Suggestions come from the first, en-AU. + if lang == ProofLang::EnAu + && let Some(gb) = crate::scowl::dict(ProofLang::EnGb) + { + ds.push(gb); } + Speller::Scowl(ds, crate::scowl::is_british(lang)) } /// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English @@ -631,11 +653,11 @@ mod tests { let fallback = pick(None, lang); if crate::scowl::dict(lang).is_some() { assert!( - matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), + matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())), "{lang:?}" ); assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}"); - assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only"); + assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)"); } else { assert!(matches!(fallback, Speller::List), "{lang:?}"); } @@ -671,6 +693,31 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an + /// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU. + #[test] + fn australian_english_reads_every_en_au_before_any_en_gb() { + let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id())); + for sub in ["user", "system"] { + std::fs::create_dir_all(dir.join(sub)).unwrap(); + } + let write = |sub: &str, name: &str| { + std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap(); + std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap(); + }; + write("user", "en_GB"); + write("system", "en_AU"); + let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]); + let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]); + let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec { + v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect() + }; + assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]); + assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::::new()); + let _ = std::fs::remove_dir_all(&dir); + } + /// craft-libs#482 review: a load that started before a reload (the dictionaries folder /// changed) serves its own check but isn't kept, so the next check reads the new folder; a /// load after it is kept, and shared with the Englishes that read the same file. diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index b4b25f5e..679a9a72 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -2571,6 +2571,26 @@ mod tests { assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb)); } + /// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also + /// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only + /// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list. + #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] + #[test] + fn australian_english_also_takes_british_words() { + crate::hold_dictionary_tests(); + let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb); + for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] { + assert!(is_correct_in(w, gb), "en-GB: {w}"); + assert!(is_correct_in(w, au), "en-AU: {w}"); + } + for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] { + assert!(!is_correct_in(w, gb), "en-GB: {w}"); + assert!(!is_correct_in(w, au), "en-AU: {w}"); + } + assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own"); + assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au)); + } + /// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English, /// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood. #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] From cb82e9142862e2921bbbc825bc4f95f9163747f6 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:23:15 +0000 Subject: [PATCH 16/23] =?UTF-8?q?craft-proof=20=E2=80=94=20a=20run=20of=20?= =?UTF-8?q?spaces=20is=20one=20double-space=20issue=20over=20the=20whole?= =?UTF-8?q?=20run=20in=20every=20language,=20through=20one=20shared=20spac?= =?UTF-8?q?e=5Fruns=20helper=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/cs/grammar.rs | 7 ++-- crates/proof/src/es/grammar.rs | 17 ++-------- crates/proof/src/hr/grammar.rs | 7 ++-- crates/proof/src/id/grammar.rs | 7 ++-- crates/proof/src/it/grammar.rs | 7 ++-- crates/proof/src/lib.rs | 60 ++++++++++++++++++++++++++++++---- crates/proof/src/pl/grammar.rs | 7 ++-- crates/proof/src/pt/grammar.rs | 7 ++-- crates/proof/src/ru/grammar.rs | 7 ++-- 9 files changed, 70 insertions(+), 56 deletions(-) diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs index 8a2590e6..e2d29954 100644 --- a/crates/proof/src/cs/grammar.rs +++ b/crates/proof/src/cs/grammar.rs @@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs index 687ecac3..00b20789 100644 --- a/crates/proof/src/es/grammar.rs +++ b/crates/proof/src/es/grammar.rs @@ -75,21 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); } } - // Each run of two or more spaces once, over the whole run. - let bytes = text.as_bytes(); - let mut i = 0; - while i < bytes.len() { - if bytes.get(i) != Some(&b' ') { - i += 1; - continue; - } - let start = i; - while bytes.get(i) == Some(&b' ') { - i += 1; - } - if i - start >= 2 { - v.push(issue(start, i, "Espacio doble".into(), vec![" ".into()], lang)); - } + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang)); } } diff --git a/crates/proof/src/hr/grammar.rs b/crates/proof/src/hr/grammar.rs index 2605ae88..f4997e9d 100644 --- a/crates/proof/src/hr/grammar.rs +++ b/crates/proof/src/hr/grammar.rs @@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/id/grammar.rs b/crates/proof/src/id/grammar.rs index 72ca10c9..6c13b7f1 100644 --- a/crates/proof/src/id/grammar.rs +++ b/crates/proof/src/id/grammar.rs @@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index e90e324b..cad690c4 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -211,11 +211,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 679a9a72..62df1b7e 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String { composed(&w.to_lowercase()).into_owned() } +/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one +/// space). A run of three is one issue over three, not a pair and a leftover. +pub(crate) fn space_runs(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut runs = Vec::new(); + let mut i = 0; + while i < bytes.len() { + if bytes.get(i) != Some(&b' ') { + i += 1; + continue; + } + let start = i; + while bytes.get(i) == Some(&b' ') { + i += 1; + } + if i - start >= 2 { + runs.push(start..i); + } + } + runs +} + /// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language. pub fn add_word(w: &str) { if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) { @@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec { lang: ProofLang::En, }); } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } + for run in space_runs(text) { v.push(Issue { - start: i, - end: i + 2, + start: run.start, + end: run.end, kind: IssueKind::Grammar, message: "Extra space".into(), suggestions: vec![" ".into()], @@ -2591,6 +2610,35 @@ mod tests { assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au)); } + /// Review follow-up: a run of spaces is one issue over the whole run, in every language with a + /// double-space rule (a run of three used to leave two after the fix, four three). + #[test] + fn a_run_of_spaces_is_one_issue_in_every_language() { + assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]); + assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty()); + let langs = [ + ProofLang::En, + ProofLang::EnGb, + ProofLang::Es, + ProofLang::It, + ProofLang::Pt, + ProofLang::Cs, + ProofLang::Hr, + ProofLang::Id, + ProofLang::Pl, + ProofLang::Ru, + ]; + for lang in langs { + for n in 2..=5 { + let text = format!("Aaa{}bbb", " ".repeat(n)); + let hits: Vec<_> = + check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect(); + assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}"); + assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}"); + } + } + } + /// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English, /// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood. #[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")] diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs index d514429e..2fd93f86 100644 --- a/crates/proof/src/pl/grammar.rs +++ b/crates/proof/src/pl/grammar.rs @@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index be8325bd..fc658e22 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -129,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()])); } } diff --git a/crates/proof/src/ru/grammar.rs b/crates/proof/src/ru/grammar.rs index ff8711fb..4d1ce3c2 100644 --- a/crates/proof/src/ru/grammar.rs +++ b/crates/proof/src/ru/grammar.rs @@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec) { v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()])); } } - for (i, _) in text.match_indices(" ") { - if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { - continue; - } - v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()])); + for run in crate::space_runs(text) { + v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()])); } } From a8b3bb74c8e51fdca1c4b156129c26a6e1b1385d Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:23:15 +0000 Subject: [PATCH 17/23] =?UTF-8?q?craft-proof=20it,=20pt=20=E2=80=94=20the?= =?UTF-8?q?=20object=20placeholder=20is=20not=20a=20closing=20quote,=20so?= =?UTF-8?q?=20Chi=20era=3F\u{1}=20era=20Marco.=20still=20gets=20its=20capi?= =?UTF-8?q?tal=20mark=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 5 ++++- crates/proof/src/pt/grammar.rs | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index cad690c4..620e284a 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -165,9 +165,10 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { } let after_marks = t.trim_start_matches(['.', '!', '?']); let rest = after_marks.trim_start_matches(CLOSING); + // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or // exclamation, or after a dialogue dash, goes on in lower case. - if rest.contains('—') || (end != '.' && after_marks.starts_with(CLOSING)) { + if rest.contains('—') || (end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { return false; } rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) @@ -416,6 +417,8 @@ mod tests { assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); } assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) is not a closing quote. + assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } #[test] diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index fc658e22..23891238 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -64,8 +64,8 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { } let after_marks = t.trim_start_matches(['.', '!', '?']); let rest = after_marks.trim_start_matches(CLOSING); - // `"Vai?" disse ele.`: narration after a quoted question or exclamation goes on in lower case. - if end != '.' && after_marks.starts_with(CLOSING) { + // `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case. + if end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c)) { return false; } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes @@ -231,6 +231,8 @@ mod tests { clean("\"Vai?\" disse ele."); clean("«Vai!» gritou ela."); assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) is not a closing quote. + assert!(!messages("Quem era?\u{1} era Marco.").is_empty()); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); From f3d411638d2b76cd9a7fd23fad0d9820329657de Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:47:08 +0000 Subject: [PATCH 18/23] =?UTF-8?q?craft-proof=20it=20=E2=80=94=20the=20dial?= =?UTF-8?q?ogue-dash=20exception=20applies=20after=20=3F=20and=20!=20only,?= =?UTF-8?q?=20so=20Fin=C3=AC.=20=E2=80=94domani=20partiamo.=20gets=20its?= =?UTF-8?q?=20capital=20mark=20again=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 620e284a..fd0799f7 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -167,8 +167,8 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { let rest = after_marks.trim_start_matches(CLOSING); // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or - // exclamation, or after a dialogue dash, goes on in lower case. - if rest.contains('—') || (end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { + // exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case. + if end != '.' && (rest.contains('—') || after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { return false; } rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) @@ -417,6 +417,8 @@ mod tests { assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t)); } assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); + // After a period the dash opens the next line of speech: a sentence starts. + assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); // An inline object (`\u{1}`) is not a closing quote. assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } From 4c8cb404ab16f275d93dff1baae3ef9c2491251b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:48:04 +0000 Subject: [PATCH 19/23] =?UTF-8?q?craft-proof=20it=20=E2=80=94=20a=20space?= =?UTF-8?q?=20after=20the=20dialogue=20dash=20no=20longer=20hides=20the=20?= =?UTF-8?q?capital=20mark=20(Fin=C3=AC.=20=E2=80=94=20domani=20partiamo.)?= =?UTF-8?q?=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index fd0799f7..15078e72 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -171,7 +171,8 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { if end != '.' && (rest.contains('—') || after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { return false; } - rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) + rest.starts_with(char::is_whitespace) + && rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—')) } fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { @@ -419,6 +420,7 @@ mod tests { assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence"); // After a period the dash opens the next line of speech: a sentence starts. assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); + assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo.")); // An inline object (`\u{1}`) is not a closing quote. assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } From 3e37fbcd7ef18a2bec10910f5318e4abf34a1242 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 20/23] =?UTF-8?q?craft-proof=20it,=20pt=20=E2=80=94=20an?= =?UTF-8?q?=20object=20placeholder=20between=20the=20mark=20and=20a=20real?= =?UTF-8?q?=20closing=20quote=20keeps=20the=20narration=20exemption=20(?= =?UTF-8?q?=C2=ABVai=3F\u{1}=C2=BB=20disse=20lui.)=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/it/grammar.rs | 4 +++- crates/proof/src/pt/grammar.rs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 15078e72..9019218c 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -168,7 +168,7 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { // (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.) // `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or // exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case. - if end != '.' && (rest.contains('—') || after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c))) { + if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) { return false; } rest.starts_with(char::is_whitespace) @@ -421,6 +421,8 @@ mod tests { // After a period the dash opens the next line of speech: a sentence starts. assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo.")); assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo.")); + // An inline object (`\u{1}`) before a real closing quote is skipped. + assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui.")); // An inline object (`\u{1}`) is not a closing quote. assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco.")); } diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 23891238..eb0f89a6 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -65,7 +65,7 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { let after_marks = t.trim_start_matches(['.', '!', '?']); let rest = after_marks.trim_start_matches(CLOSING); // `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case. - if end != '.' && after_marks.starts_with(|c| c != '\u{1}' && CLOSING.contains(&c)) { + if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) { return false; } // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes @@ -231,6 +231,8 @@ mod tests { clean("\"Vai?\" disse ele."); clean("«Vai!» gritou ela."); assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence"); + // An inline object (`\u{1}`) before a real closing quote is skipped. + clean("\"Vai?\u{1}\" disse ele."); // An inline object (`\u{1}`) is not a closing quote. assert!(!messages("Quem era?\u{1} era Marco.").is_empty()); clean("Ele disse: \"Volto amanhã\". E saiu."); From aa3730352f3e7a7216d86bef3613b652b843579a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 21/23] =?UTF-8?q?craft-proof=20=E2=80=94=20en-AU=20comment?= =?UTF-8?q?=20and=20docs:=20it=20also=20takes=20en-GB's=20words=20(enquiry?= =?UTF-8?q?=20is=20only=20in=20the=20en-GB=20list),=20suggestions=20from?= =?UTF-8?q?=20en-AU=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/README.md | 5 +++-- crates/proof/src/english.rs | 7 ++++--- crates/proof/src/scowl.rs | 4 +++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/crates/proof/README.md b/crates/proof/README.md index 13ab9347..97067014 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -156,8 +156,9 @@ English spelling uses the first of these that is there; `english_source(lang)` s (Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own dictionary knows it (or `en-extra.txt` does), never because another English's does - (`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same - dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so + (`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian + English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only). + Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so British English also takes the `-ize` spellings of its `-ise` words that American or Canadian English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this dialect's other spellings of known words (`judgement` in en-US). English with no dialect diff --git a/crates/proof/src/english.rs b/crates/proof/src/english.rs index c27f01a9..e821e801 100644 --- a/crates/proof/src/english.rs +++ b/crates/proof/src/english.rs @@ -7,7 +7,8 @@ //! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded //! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also //! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is -//! right; never one only another English knows. When it is there, the system locations aren't +//! right; never one only another English knows, except that Australian English also takes +//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't //! searched. //! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the //! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored @@ -318,8 +319,8 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller { } let Some(d) = crate::scowl::dict(lang) else { return Speller::List }; let mut ds = vec![d]; - // SCOWL's en-AU list keeps one spelling per word (`enquiry`, not `inquiry`); the British list - // has the rest, which Australian English takes. Suggestions come from the first, en-AU. + // SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB + // list), and Australian English takes both. Suggestions come from the first, en-AU. if lang == ProofLang::EnAu && let Some(gb) = crate::scowl::dict(ProofLang::EnGb) { diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index e9f41b9e..5e84ea48 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -8,7 +8,9 @@ //! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU //! data, so it stays on en-GB. Its suggestions //! come from its own dictionary, and a word is right when its own dictionary knows it: the other -//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL +//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except +//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words +//! (`enquiry`; its suggestions still come from en-AU). SCOWL //! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word //! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that //! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs From 3aca2642c5a3cb039a70a3819eb0e30fabf7ba6a Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 22/23] =?UTF-8?q?craft-proof=20tr=20=E2=80=94=20the=20doub?= =?UTF-8?q?le-space=20rule=20uses=20space=5Fruns;=20the=20all-languages=20?= =?UTF-8?q?test=20adds=20Tr,=20EnAu,=20EnCa=20and=20pt-PT=20(#496=20review?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/lib.rs | 7 ++++++- crates/proof/src/tr/grammar.rs | 26 ++++++++++---------------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 62df1b7e..2e804097 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -2627,7 +2627,12 @@ mod tests { ProofLang::Id, ProofLang::Pl, ProofLang::Ru, - ]; + ProofLang::Tr, + ProofLang::EnAu, + ProofLang::EnCa, + ] + .into_iter() + .chain(PackLocale::new("pt-PT").map(ProofLang::Pack)); for lang in langs { for n in 2..=5 { let text = format!("Aaa{}bbb", " ".repeat(n)); diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs index 81ba93c2..9daf5bfc 100644 --- a/crates/proof/src/tr/grammar.rs +++ b/crates/proof/src/tr/grammar.rs @@ -209,23 +209,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { }); } } - // Double space - if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') { - let mut end = idx + 1; - while end < text.len() && text.as_bytes().get(end) == Some(&b' ') { - end += 1; - } - v.push(Issue { - start: idx, - end, - kind: IssueKind::Grammar, - message: "Fazladan boşluk".into(), - suggestions: vec![" ".into()], - lang: ProofLang::Tr, - }); - } } - + for run in crate::space_runs(text) { + v.push(Issue { + start: run.start, + end: run.end, + kind: IssueKind::Grammar, + message: "Fazladan boşluk".into(), + suggestions: vec![" ".into()], + lang: ProofLang::Tr, + }); + } // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) for (i, &(w_start, w_end)) in ws.iter().enumerate() { let word = match text.get(w_start..w_end) { From 14cf444fcfcf34ad94e9a2134139b9233e4d5e85 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Mon, 2 Nov 2026 23:56:09 +0000 Subject: [PATCH 23/23] =?UTF-8?q?craft-proof=20scowl=20=E2=80=94=20load=5F?= =?UTF-8?q?error(en-AU)=20also=20reports=20a=20failed=20en-GB=20list,=20wh?= =?UTF-8?q?ich=20en-AU=20reads=20too=20(#496=20review)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/scowl.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/scowl.rs b/crates/proof/src/scowl.rs index 5e84ea48..cb101cd8 100644 --- a/crates/proof/src/scowl.rs +++ b/crates/proof/src/scowl.rs @@ -64,7 +64,9 @@ pub(crate) fn is_british(lang: ProofLang) -> bool { /// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it. pub(crate) fn load_error(lang: ProofLang) -> Option { let (i, ..) = source(lang); - DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}")) + let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}")); + // Australian English also reads the en-GB list (`crate::english`): its failure is reported too. + own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None }) } /// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of