From 3ad9ea765bf391d0f48b9e9e283e78fcf9567e37 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Sun, 1 Nov 2026 18:04:49 +0000 Subject: [PATCH] CP4.2: ProofLang::Pack(PackLocale) proofs any language from its installed pack or Hunspell dictionary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Languages without tools of their own (nl, fr, sr-Latn…) are spell-checked and given suggestions through the shared affix engine with default tuning, and hyphenated from their pack's patterns; no grammar. At most 8 such dictionaries stay loaded. Never another script's pack. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2 --- crates/proof/README.md | 46 ++++++-- crates/proof/src/lang.rs | 97 +++++++++++++++-- crates/proof/src/lib.rs | 115 +++++++++++++++++++- crates/proof/src/pack_lang.rs | 195 ++++++++++++++++++++++++++++++++++ crates/proof/src/packs.rs | 14 ++- 5 files changed, 446 insertions(+), 21 deletions(-) create mode 100644 crates/proof/src/pack_lang.rs diff --git a/crates/proof/README.md b/crates/proof/README.md index 64286ba8b..979fa005c 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -65,16 +65,49 @@ data folder runs neither set. ## Language packs +**API (fixed by `plan/pack-contract.md`, plus the additions marked ＋):** + +```rust +// craft_proof::packs +pub fn set_packs_dir(dir: Option); // /language-packs; bumps dictionary_generation() +pub fn reload(); // ＋ same folder, after a pack is installed or removed +pub fn packs_dir() -> Option; // ＋ +pub fn installed() -> Vec; // InstalledPack { locale, name, licence, path }, by locale +pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/* +pub fn load_errors() -> Vec<(String, String)>; // ＋ (locale or folder, reason): packs skipped, pattern files that failed +pub fn embedded_locales() -> &'static [&'static str]; // ＋ what this build embeds ("en-us"; more in a Language build) + +// craft_proof (crate root) +pub enum ProofLang { …, Pack(PackLocale), Other } // ＋ Pack: any language without tools of its own (nl, fr, sr-Latn…) +pub struct PackLocale; // ＋ Copy; PackLocale::new(tag) -> Option, .as_str() "nl-nl", .primary() "nl", Display +impl ProofLang { pub fn pack_locale(self) -> Option; } // ＋ +pub fn pack_dictionary_file(lang: ProofLang) -> Option; // ＋ the .dic a Pack language reads now; None = unchecked +``` + +`ProofLang::from_tag` now gives `Pack(locale)` for every valid language tag it has no tools for +(`nl-NL` → `Pack("nl-nl")`, `fr-FR`, `ja-JP`, `sr-Latn`…); `Other` is left for `x-none`, `und`, +`zxx`, `mul`, `mis`, `i-…` and malformed tags. A `Pack` language is spell-checked and given +suggestions from its Hunspell dictionary (dictionaries folder → installed pack → system Hunspell) +through the shared affix engine with default tuning, and hyphenated from its pack's +`hyphenation.pat`; it has no grammar rules and the generic English spelling message. Without a +dictionary or pack it behaves exactly as `Other` did (no marks, English hyphenation fallback). +`has_dictionary(tag)` / `ProofLang::is_proofed()` say whether a `Pack` language has a dictionary +now (read on first use, cached). At most 8 `Pack` dictionaries stay loaded (least recently used +goes first). Downstream `match`es on `ProofLang` need a `Pack(_)` arm; `code()` is `""` for it, as +for `Other` (use `pack_locale()`). + +**Which pack serves a language:** its locale, then shorter prefixes (`fr-ca` → `fr`), then another +region's pack of the same language (`nl` → `nl-be`), never another script's (`sr-latn` never reads +`sr`). The existing languages map their Hunspell names the same way (`pt_BR` → `pt-br`, `pt`, any +`pt-*`). + Spelling dictionaries and hyphenation patterns for other languages come as **Language packs**: the app's font store (`craft-font-store`, item kind `language-pack`) downloads, verifies and unpacks them; this crate reads them. ```rust -craft_proof::packs::set_packs_dir(Some(store_folder.join("language-packs"))); // at startup +craft_proof::packs::set_packs_dir(Some(store.language_packs_dir())); // at startup craft_proof::packs::reload(); // after a pack is installed or removed -let packs = craft_proof::packs::installed(); // Vec -let about = craft_proof::packs::licence_texts(); // Vec<(title, text)>: embedded notices, then each pack's -let problems = craft_proof::packs::load_errors(); // Vec<(locale, reason)>: packs skipped, files that failed ``` Both `set_packs_dir` and `reload` reload every dictionary without a restart and bump @@ -105,11 +138,6 @@ that leaves the pack (`..`, an absolute path, a drive, a backslash). Folders sta (the store's temporary folders) are ignored. A dictionary that fails to load is reported by `dictionary_load_error` like any other; nothing panics. -**Not yet:** packs only serve the languages this crate already proofs (the `ProofLang` slots). -A pack for a language without a `ProofLang` (Dutch, French…) is listed, its licence shown, but -its words aren't checked and `ProofLang::Other` keeps the English hyphenation fallback until that -language gets a slot. - ### English is always checked English spelling uses the first of these that is there; `english_source(lang)` says which. diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 3d15d2245..5e8c88bc7 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -70,7 +70,13 @@ pub enum ProofLang { /// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`], /// no spelling or grammar marks. De, - /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without + /// Any other language, by its tag (`nl`, `fr-FR`, `sr-Latn`): spelling and suggestions from + /// its Hunspell dictionary (the dictionaries folder, an installed Language pack, the system's) + /// through the shared affix engine with default tuning, and hyphenation from its Language + /// pack. No grammar rules. Without a dictionary or pack it behaves exactly as + /// [`ProofLang::Other`]: no marks, English hyphenation fallback. + Pack(PackLocale), + /// A language without proofing tools here (`x-none`, `zxx`, `und`, a malformed tag). Like Word without /// the language's proofing tools: no spelling or grammar marks, so its words are never /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before /// languages were told apart. @@ -129,7 +135,7 @@ impl ProofLang { } else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) { ProofLang::De } else { - ProofLang::Other + PackLocale::new(tag).map_or(ProofLang::Other, ProofLang::Pack) } } @@ -182,13 +188,86 @@ impl ProofLang { ProofLang::Id => "id", ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es", ProofLang::De => "de", - ProofLang::Other => "", + ProofLang::Pack(_) | ProofLang::Other => "", } } - /// Does WordCraft check spelling and grammar in this language? + /// The locale of a [`ProofLang::Pack`] language. + pub fn pack_locale(self) -> Option { + if let ProofLang::Pack(l) = self { Some(l) } else { None } + } + + /// Does WordCraft check spelling in this language? For a [`ProofLang::Pack`] language: is a + /// dictionary for it available now (read on first use, cached until the dictionaries folder or + /// the packs change)? pub fn is_proofed(self) -> bool { - Self::PROOFED.contains(&self) + match self { + ProofLang::Pack(l) => crate::pack_lang::has_dictionary(l), + _ => Self::PROOFED.contains(&self), + } + } +} + +/// The locale of a [`ProofLang::Pack`] language: a lowercase BCP 47 tag (`nl`, `fr-fr`, +/// `sr-latn`) of at most [`PackLocale::MAX_LEN`] bytes, stored inline so [`ProofLang`] stays `Copy`. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PackLocale { + len: u8, + bytes: [u8; PackLocale::MAX_LEN], +} + +impl PackLocale { + /// Longest locale kept; a longer tag is cut at a subtag boundary. + pub const MAX_LEN: usize = 15; + + /// The locale of a language tag (`nl-NL`, `sr_Latn`, `fr-FR.UTF-8`): lowercased, `_` as `-`, + /// without a POSIX `.…`/`@…` tail or anything from a singleton on (`-x-…`, `-u-…`). `None` for + /// a tag that isn't a language (`x-none`, `i-…`, `und`, `zxx`, `mul`, `mis`, malformed ones). + pub fn new(tag: &str) -> Option { + let tag = tag.trim().split(['.', '@']).next().unwrap_or(""); + let mut parts = tag.split(['-', '_']); + let primary = parts.next()?.to_ascii_lowercase(); + if !(2..=8).contains(&primary.len()) + || !primary.bytes().all(|b| b.is_ascii_lowercase()) + || ["und", "zxx", "mul", "mis"].contains(&primary.as_str()) + { + return None; + } + let mut out = primary; + for p in parts { + if p.len() <= 1 || p.len() > 8 || !p.bytes().all(|b| b.is_ascii_alphanumeric()) || out.len() + 1 + p.len() > Self::MAX_LEN { + break; + } + out.push('-'); + out.push_str(&p.to_ascii_lowercase()); + } + let mut bytes = [0u8; Self::MAX_LEN]; + for (b, c) in bytes.iter_mut().zip(out.bytes()) { + *b = c; + } + Some(PackLocale { len: u8::try_from(out.len()).ok()?, bytes }) + } + + /// The locale (`nl`, `fr-fr`). + pub fn as_str(&self) -> &str { + self.bytes.get(..usize::from(self.len)).and_then(|b| std::str::from_utf8(b).ok()).unwrap_or_default() + } + + /// The primary language subtag (`fr`). + pub fn primary(&self) -> &str { + self.as_str().split('-').next().unwrap_or_default() + } +} + +impl std::fmt::Debug for PackLocale { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "PackLocale({:?})", self.as_str()) + } +} + +impl std::fmt::Display for PackLocale { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) } } @@ -358,7 +437,13 @@ mod tests { #[test] fn tags_reject_unknown_or_partial_language_codes() { - assert_tags(&["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"], ProofLang::Other); + // A partial code is another language, never the one it starts like. + for tag in ["fr-FR", "dex", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax"] { + let lang = ProofLang::from_tag(Some(tag)); + assert_eq!(lang.pack_locale().map(|l| l.to_string()), Some(tag.to_ascii_lowercase()), "{tag}"); + assert_eq!(lang.code(), "", "{tag}"); + } + assert_tags(&["x-none", "e", "-", "und", "zxx", "i-klingon"], ProofLang::Other); } /// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 72a129a59..88024fb85 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -42,8 +42,11 @@ //! existing English fallback. //! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed //! spelling, MIT); no spelling or grammar marks yet. -//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without -//! the language's proofing tools. +//! - **Any other language** ([`ProofLang::Pack`], module [`packs`]): spelling and suggestions +//! from its Hunspell dictionary (an installed Language pack, the dictionaries folder or the +//! system's) with default tuning, hyphenation from its pack, no grammar rules; without a +//! dictionary no marks, like Word without the language's proofing tools. +//! - **No language** ([`ProofLang::Other`]: `x-none`, `zxx`, malformed tags): no marks. //! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese, //! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic. //! @@ -76,6 +79,7 @@ pub mod hyphen; pub mod id; pub mod it; pub mod lang; +mod pack_lang; pub mod packs; pub mod patterns; pub mod pl; @@ -110,7 +114,7 @@ impl Default for Options { } pub use english::EnglishSource; -pub use lang::ProofLang; +pub use lang::{PackLocale, ProofLang}; /// A problem found in text: byte range, kind, message and suggestions. #[derive(Clone, Debug, PartialEq)] @@ -314,9 +318,17 @@ pub(crate) fn reload_dictionaries() { l.slot.reload(); } english::reload(); + pack_lang::reload(); DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); } +/// The `.dic` file a [`ProofLang::Pack`] language's spelling is read from now (the dictionaries +/// folder, an installed Language pack or the system's); `None` when it has none (unchecked) or +/// `lang` isn't a pack language. Reads the dictionary once (it is cached afterwards). +pub fn pack_dictionary_file(lang: ProofLang) -> Option { + pack_lang::dictionary_file(lang.pack_locale()?) +} + /// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it /// now (`None`: not installed, so the language has no spelling marks). pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> { @@ -331,6 +343,9 @@ pub fn dictionary_load_error(lang: ProofLang) -> Option { if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) { return english::load_error(lang); } + if let ProofLang::Pack(l) = lang { + return pack_lang::load_error(l); + } hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason) } @@ -827,6 +842,9 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { if lang.is_english() { return is_correct_english(word, lang, opts); } + if let ProofLang::Pack(l) = lang { + return accepted_by_options(word, opts) || pack_lang::is_correct(word, l, opts); + } // A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks. language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts)) } @@ -838,6 +856,9 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { if lang.is_english() { return suggest_english(word, max, lang); } + if let ProofLang::Pack(l) = lang { + return pack_lang::suggest(word, max, l); + } language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang)) } @@ -1347,6 +1368,8 @@ mod tests { #[test] fn an_installed_pack_is_used_and_a_removed_one_is_not() { let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + // The test dictionaries on disk first, so a reload never leaves other tests without them. + set_dictionaries_folder(Some(shared_test_folder())); let root = std::env::temp_dir().join(format!("wc-proof-packs-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); let (folder, packs) = (root.join("dictionaries"), root.join("language-packs")); @@ -1414,12 +1437,98 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// CP4.2: a language with no tools of its own (Dutch) is spell-checked, given suggestions and + /// hyphenated from its installed Language pack; removed, it is unchecked again; English is + /// unaffected. No grammar rules for it. + #[test] + fn a_pack_proofs_a_language_without_its_own_tools() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + // The test dictionaries on disk first, so a reload never leaves other tests without them. + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-nl-pack-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let nl = ProofLang::from_tag(Some("nl-NL")); + assert_eq!(nl.pack_locale().map(|l| l.to_string()).as_deref(), Some("nl-nl")); + let text = "Het huis en de fietz."; + let english_before = check_text(text, &[]); + let unchecked = |lang: ProofLang| check_text(text, &[(0..text.len(), lang)]).iter().all(|i| i.kind == IssueKind::Grammar); + let before = (is_correct_in("fietz", nl), hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), pack_dictionary_file(nl)); + let pack = write_pack( + &root, + "nl", + &[ + ("spelling.aff", b"SET UTF-8\nTRY esianrtolducgmphbykfvwjzxq\nSFX E Y 1\nSFX E 0 en .\n"), + ("spelling.dic", b"5\nhet\nhuis/E\nen\nde\nfiets/E\n"), + ("hyphenation.pat", b"1na xx\n"), + ], + ); + packs::set_packs_dir(Some(root.clone())); + assert!(nl.is_proofed() && has_dictionary(Some("nl-BE")) && !has_dictionary(Some("fr-FR"))); + assert_eq!(pack_dictionary_file(nl), Some(pack.join("spelling.dic"))); + assert_eq!(pack_dictionary_file(ProofLang::from_tag(Some("nl"))), Some(pack.join("spelling.dic")), "a region falls back to the pack"); + for w in ["huis", "Huis", "huizen", "huisen", "fietsen", "HUIS"] { + assert_eq!(is_correct_in(w, nl), w != "huizen", "{w}"); + } + let issues = check_text(text, &[(0..text.len(), nl)]); + assert_eq!(issues.len(), 1, "{issues:?}"); + assert_eq!((issues[0].kind, &text[issues[0].start..issues[0].end], issues[0].lang), (IssueKind::Spelling, "fietz", nl)); + assert!(suggest_in("fietz", 5, nl).contains(&"fiets".to_string()), "{:?}", suggest_in("fietz", 5, nl)); + assert!(check_grammar_in("de de huis", nl).is_empty(), "no grammar rules"); + assert_eq!(hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), "xx-naxx"); + assert!(unchecked(ProofLang::from_tag(Some("fr-FR"))), "a language with no pack stays unchecked"); + assert_eq!(check_text(text, &[]), english_before, "English is unaffected"); + // A broken dictionary is reported, never a panic. + std::fs::write(pack.join("spelling.aff"), "SET SHIFT_JIS\n").unwrap(); + packs::reload(); + assert!(is_correct_in("fietz", nl) && !nl.is_proofed()); + assert!(dictionary_load_error(nl).unwrap_or_default().contains("unsupported encoding")); + // Removed: unchecked again. + std::fs::remove_dir_all(&pack).unwrap(); + packs::reload(); + assert_eq!( + (is_correct_in("fietz", nl), hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), pack_dictionary_file(nl)), + before + ); + assert!(unchecked(nl) || before.2.is_some()); + assert_eq!(check_text(text, &[]), english_before); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + + /// At most [`pack_lang::MAX_LOADED`] pack-language dictionaries stay loaded; others load again + /// when used. + #[test] + fn pack_language_dictionaries_are_capped() { + let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + // The test dictionaries on disk first, so a reload never leaves other tests without them. + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-many-packs-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let locales: Vec = (0..pack_lang::MAX_LOADED + 3).map(|i| format!("q{}", (b'a' + i as u8) as char)).collect(); + for l in &locales { + write_pack(&root, l, &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", format!("1\nword{l}\n").as_bytes())]); + } + packs::set_packs_dir(Some(root.clone())); + for _ in 0..2 { + for l in &locales { + let lang = ProofLang::from_tag(Some(l)); + assert!(is_correct_in(&format!("word{l}"), lang) && !is_correct_in("wordzz", lang), "{l}"); + } + } + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + /// CP4.2, hostile packs: a missing or broken `pack.toml`, a path outside the pack, a folder /// that isn't a language tag, a symbolic link, patterns that aren't UTF-8 or a dictionary that /// can't be read are reported, never a panic, and the pack (or that file) is skipped. #[test] fn corrupt_packs_are_reported_and_skipped() { let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + // The test dictionaries on disk first, so a reload never leaves other tests without them. + set_dictionaries_folder(Some(shared_test_folder())); let root = std::env::temp_dir().join(format!("wc-proof-bad-packs-{}", std::process::id())); let _ = std::fs::remove_dir_all(&root); // Good spelling (the test words, so Croatian tests running meanwhile are unaffected), bad diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs new file mode 100644 index 000000000..44dde95b6 --- /dev/null +++ b/crates/proof/src/pack_lang.rs @@ -0,0 +1,195 @@ +//! Proofing for any language by its locale ([`crate::ProofLang::Pack`]): its Hunspell dictionary (the +//! dictionaries folder, then an installed Language pack, then the system's), read by the shared +//! affix engine, with default suggestion tuning and no grammar rules. +//! +//! At most [`MAX_LOADED`] dictionaries stay loaded; the one used least recently goes first. + +use std::path::PathBuf; +use std::sync::{Arc, PoisonError, RwLock}; + +use crate::affix::AffixDict; +use crate::affix_suggest::Tuning; +use crate::lang::PackLocale; + +/// Most locale dictionaries kept loaded at once. +pub(crate) const MAX_LOADED: usize = 8; + +struct Loaded { + locale: PackLocale, + dict: Arc, + /// The `.dic` it came from (`None`: none found, the language is unchecked). + file: Option, + used: u64, +} + +struct Cache { + loaded: Vec, + clock: u64, + /// Counts reloads, so a load that started before one is not kept. + reloads: u64, +} + +static CACHE: RwLock = RwLock::new(Cache { loaded: Vec::new(), clock: 0, reloads: 0 }); + +/// Forget every loaded dictionary (the dictionaries folder or the packs changed). +pub(crate) fn reload() { + let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); + c.loaded.clear(); + c.reloads = c.reloads.wrapping_add(1); +} + +/// Hunspell file names for `locale`, best first: `fr-fr` → `fr_FR`, `fr-FR`, `fr`; +/// `sr-latn` → `sr_Latn`, `sr-Latn` (never plain `sr`, another script). +pub(crate) fn names(locale: PackLocale) -> Vec { + let parts: Vec<&str> = locale.as_str().split('-').collect(); + let cased: Vec = parts + .iter() + .enumerate() + .map(|(i, p)| match (i, p.len()) { + (0, _) => (*p).to_string(), + (_, 2) => p.to_ascii_uppercase(), + (_, 4) => { + let mut c = p.chars(); + c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default() + } + _ => (*p).to_string(), + }) + .collect(); + let mut out: Vec = Vec::new(); + let mut push = |n: String| { + if !out.contains(&n) { + out.push(n); + } + }; + // Each shorter prefix too (`sr-latn-rs` → `sr-latn`), but not past a script subtag. + let script = parts.iter().position(|p| p.len() == 4).unwrap_or(usize::MAX); + for n in (1..=cased.len()).rev() { + if n <= script && script != usize::MAX { + break; + } + let prefix = cased.get(..n).unwrap_or_default(); + push(prefix.join("_")); + push(prefix.join("-")); + } + if script != usize::MAX { + let prefix = cased.get(..=script).unwrap_or_default(); + push(prefix.join("_")); + push(prefix.join("-")); + } + out +} + +/// `locale`'s dictionary, loaded on first use; an empty one when there is none. +fn get(locale: PackLocale) -> (Arc, Option) { + let reloads = { + let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); + c.clock = c.clock.wrapping_add(1); + let now = c.clock; + if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) { + l.used = now; + return (l.dict.clone(), l.file.clone()); + } + c.reloads + }; + let (dict, file) = load(locale); + let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner); + if c.reloads == reloads && !c.loaded.iter().any(|l| l.locale == locale) { + if c.loaded.len() >= MAX_LOADED + && let Some(oldest) = c.loaded.iter().enumerate().min_by_key(|(_, l)| l.used).map(|(i, _)| i) + { + c.loaded.swap_remove(oldest); + } + let used = c.clock; + c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), used }); + } + (dict, file) +} + +fn load(locale: PackLocale) -> (Arc, Option) { + let names = names(locale); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + let key = names.first().map_or("", String::as_str); + let Some((aff, dic)) = crate::hunspell::find(&refs) else { + crate::hunspell::clear_load_error(key); + return (Arc::default(), None); + }; + match crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)) { + Ok(d) => (Arc::new(d), Some(dic)), + Err(e) => { + log::warn!("{locale} spelling: {e}"); + (Arc::default(), None) + } + } +} + +/// Is a dictionary for `locale` available (its spelling is checked)? +pub(crate) fn has_dictionary(locale: PackLocale) -> bool { + !get(locale).0.is_empty() +} + +/// The `.dic` file `locale`'s spelling is read from now. +pub(crate) fn dictionary_file(locale: PackLocale) -> Option { + let (d, file) = get(locale); + file.filter(|_| !d.is_empty()) +} + +/// Why `locale`'s dictionary failed to load, if it did. +pub(crate) fn load_error(locale: PackLocale) -> Option { + crate::hunspell::load_error(names(locale).first()?).map(|(_, e)| e) +} + +/// Is `word` right in `locale`? Every word is when there is no dictionary. +pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool { + let (d, _) = get(locale); + if d.is_empty() { + return true; + } + let Some(w) = crate::spelling::word_to_check(&d, word, opts) else { return true }; + let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); + if w.contains('-') && !known(w) { + return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); + } + known(w) +} + +/// Suggestions for `word` in `locale`, best first, at most `max`. +pub(crate) fn suggest(word: &str, max: usize, locale: PackLocale) -> Vec { + let (d, _) = get(locale); + if d.is_empty() { + return Vec::new(); + } + crate::affix_suggest::suggest_in(&d, word, max, &Tuning::DEFAULT) +} + +/// The pack locales whose hyphenation patterns serve a [`crate::ProofLang::Pack`] language: the locale, +/// then each shorter prefix not past a script subtag (`fr-ca` → `fr`). +pub(crate) fn hyphenation_locales(locale: PackLocale) -> Vec { + names(locale).into_iter().filter(|n| !n.contains('_')).map(|n| n.to_ascii_lowercase()).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn l(t: &str) -> PackLocale { + PackLocale::new(t).unwrap() + } + + #[test] + fn locales_and_file_names() { + assert_eq!(l("nl-NL").as_str(), "nl-nl"); + assert_eq!(l("fr_FR.UTF-8").as_str(), "fr-fr"); + assert_eq!(l("ca-ES-valencia").as_str(), "ca-es-valencia"); + assert_eq!(l("sr-Latn-RS-x-private").as_str(), "sr-latn-rs"); + assert_eq!(l("zh-Hant-TW-somethinglong").as_str(), "zh-hant-tw"); + for bad in ["x-none", "i-klingon", "und", "zxx", "mul", "e", "-", "", "1x", "näh"] { + assert_eq!(PackLocale::new(bad), None, "{bad}"); + } + assert_eq!(names(l("nl")), ["nl"]); + assert_eq!(names(l("fr-FR")), ["fr_FR", "fr-FR", "fr"]); + assert_eq!(names(l("sr-Latn")), ["sr_Latn", "sr-Latn"]); + assert_eq!(names(l("sr-Latn-RS")), ["sr_Latn_RS", "sr-Latn-RS", "sr_Latn", "sr-Latn"]); + assert_eq!(hyphenation_locales(l("fr-CA")), ["fr-ca", "fr"]); + assert_eq!(hyphenation_locales(l("sr-Latn")), ["sr-latn"]); + } +} diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index 2ab63a275..2e96bf66d 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -362,12 +362,18 @@ fn locales_for(names: &[&str]) -> Vec { for n in names { push(n.to_ascii_lowercase().replace('_', "-")); } - for n in names { + // The primary language alone, unless a script is named (`sr_Latn` is never plain `sr`). + for n in names.iter().filter(|n| !has_script(n)) { push(n.split(['_', '-']).next().unwrap_or_default().to_ascii_lowercase()); } out } +/// Does a tag or file name name a script (a four-letter subtag after the language: `sr-Latn`)? +fn has_script(tag: &str) -> bool { + tag.split(['_', '-']).skip(1).any(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic())) +} + /// The installed packs to read for `locales`, in that order, then any other pack of their /// primary languages (`pt` → `pt-br`, `pt-pt`), keeping those that `has` the data. fn packs_for(locales: &[String], has: impl Fn(&Holds) -> bool) -> Vec { @@ -379,10 +385,11 @@ fn packs_for(locales: &[String], has: impl Fn(&Holds) -> bool) -> Vec { out.push(p.path.clone()); } } - let primaries: Vec<&str> = locales.iter().map(|l| l.split('-').next().unwrap_or_default()).collect(); + // Then another region's pack of the same language (`pt` → `pt-br`), never another script's. + let primaries: Vec<&str> = locales.iter().filter(|l| !has_script(l)).map(|l| l.split('-').next().unwrap_or_default()).collect(); for (p, _) in usable() { let primary = p.locale.split('-').next().unwrap_or_default(); - if primaries.contains(&primary) && !out.contains(&p.path) { + if primaries.contains(&primary) && !has_script(&p.locale) && !out.contains(&p.path) { out.push(p.path.clone()); } } @@ -416,6 +423,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec { ProofLang::Es419 => &["es-419", "es"], ProofLang::EsUy => &["es-uy", "es-419", "es"], ProofLang::De => &["de", "de-1996", "de-de"], + ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l), ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[], }; names.iter().map(|n| (*n).to_string()).collect()