mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-12 21:42:13 +00:00
CP4.2: ProofLang::Pack(PackLocale) proofs any language from its installed pack or Hunspell dictionary
Languages without tools of their own (nl, fr, sr-Latn…) are spell-checked and given suggestions through the shared affix engine with default tuning, and hyphenated from their pack's patterns; no grammar. At most 8 such dictionaries stay loaded. Never another script's pack. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
1 parent
576cf914e3
commit
3ad9ea765b
5 files changed
+446
-21
No files matched your search
+37
-9
@@ -65,16 +65,49 @@ data folder runs neither set.
|
||||
|
||||
## Language packs
|
||||
|
||||
**API (fixed by `plan/pack-contract.md`, plus the additions marked +):**
|
||||
|
||||
```rust
|
||||
// craft_proof::packs
|
||||
pub fn set_packs_dir(dir: Option<PathBuf>); // <font-store folder>/language-packs; bumps dictionary_generation()
|
||||
pub fn reload(); // + same folder, after a pack is installed or removed
|
||||
pub fn packs_dir() -> Option<PathBuf>; // +
|
||||
pub fn installed() -> Vec<InstalledPack>; // InstalledPack { locale, name, licence, path }, by locale
|
||||
pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/*
|
||||
pub fn load_errors() -> Vec<(String, String)>; // + (locale or folder, reason): packs skipped, pattern files that failed
|
||||
pub fn embedded_locales() -> &'static [&'static str]; // + what this build embeds ("en-us"; more in a Language build)
|
||||
|
||||
// craft_proof (crate root)
|
||||
pub enum ProofLang { …, Pack(PackLocale), Other } // + Pack: any language without tools of its own (nl, fr, sr-Latn…)
|
||||
pub struct PackLocale; // + Copy; PackLocale::new(tag) -> Option<Self>, .as_str() "nl-nl", .primary() "nl", Display
|
||||
impl ProofLang { pub fn pack_locale(self) -> Option<PackLocale>; } // +
|
||||
pub fn pack_dictionary_file(lang: ProofLang) -> Option<PathBuf>; // + the .dic a Pack language reads now; None = unchecked
|
||||
```
|
||||
|
||||
`ProofLang::from_tag` now gives `Pack(locale)` for every valid language tag it has no tools for
|
||||
(`nl-NL` → `Pack("nl-nl")`, `fr-FR`, `ja-JP`, `sr-Latn`…); `Other` is left for `x-none`, `und`,
|
||||
`zxx`, `mul`, `mis`, `i-…` and malformed tags. A `Pack` language is spell-checked and given
|
||||
suggestions from its Hunspell dictionary (dictionaries folder → installed pack → system Hunspell)
|
||||
through the shared affix engine with default tuning, and hyphenated from its pack's
|
||||
`hyphenation.pat`; it has no grammar rules and the generic English spelling message. Without a
|
||||
dictionary or pack it behaves exactly as `Other` did (no marks, English hyphenation fallback).
|
||||
`has_dictionary(tag)` / `ProofLang::is_proofed()` say whether a `Pack` language has a dictionary
|
||||
now (read on first use, cached). At most 8 `Pack` dictionaries stay loaded (least recently used
|
||||
goes first). Downstream `match`es on `ProofLang` need a `Pack(_)` arm; `code()` is `""` for it, as
|
||||
for `Other` (use `pack_locale()`).
|
||||
|
||||
**Which pack serves a language:** its locale, then shorter prefixes (`fr-ca` → `fr`), then another
|
||||
region's pack of the same language (`nl` → `nl-be`), never another script's (`sr-latn` never reads
|
||||
`sr`). The existing languages map their Hunspell names the same way (`pt_BR` → `pt-br`, `pt`, any
|
||||
`pt-*`).
|
||||
|
||||
Spelling dictionaries and hyphenation patterns for other languages come as **Language packs**:
|
||||
the app's font store (`craft-font-store`, item kind `language-pack`) downloads, verifies and
|
||||
unpacks them; this crate reads them.
|
||||
|
||||
```rust
|
||||
craft_proof::packs::set_packs_dir(Some(store_folder.join("language-packs"))); // at startup
|
||||
craft_proof::packs::set_packs_dir(Some(store.language_packs_dir())); // at startup
|
||||
craft_proof::packs::reload(); // after a pack is installed or removed
|
||||
let packs = craft_proof::packs::installed(); // Vec<InstalledPack { locale, name, licence, path }>
|
||||
let about = craft_proof::packs::licence_texts(); // Vec<(title, text)>: embedded notices, then each pack's
|
||||
let problems = craft_proof::packs::load_errors(); // Vec<(locale, reason)>: packs skipped, files that failed
|
||||
```
|
||||
|
||||
Both `set_packs_dir` and `reload` reload every dictionary without a restart and bump
|
||||
@@ -105,11 +138,6 @@ that leaves the pack (`..`, an absolute path, a drive, a backslash). Folders sta
|
||||
(the store's temporary folders) are ignored. A dictionary that fails to load is reported by
|
||||
`dictionary_load_error` like any other; nothing panics.
|
||||
|
||||
**Not yet:** packs only serve the languages this crate already proofs (the `ProofLang` slots).
|
||||
A pack for a language without a `ProofLang` (Dutch, French…) is listed, its licence shown, but
|
||||
its words aren't checked and `ProofLang::Other` keeps the English hyphenation fallback until that
|
||||
language gets a slot.
|
||||
|
||||
### English is always checked
|
||||
|
||||
English spelling uses the first of these that is there; `english_source(lang)` says which.
|
||||
|
||||
@@ -70,7 +70,13 @@ pub enum ProofLang {
|
||||
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
|
||||
/// no spelling or grammar marks.
|
||||
De,
|
||||
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
|
||||
/// Any other language, by its tag (`nl`, `fr-FR`, `sr-Latn`): spelling and suggestions from
|
||||
/// its Hunspell dictionary (the dictionaries folder, an installed Language pack, the system's)
|
||||
/// through the shared affix engine with default tuning, and hyphenation from its Language
|
||||
/// pack. No grammar rules. Without a dictionary or pack it behaves exactly as
|
||||
/// [`ProofLang::Other`]: no marks, English hyphenation fallback.
|
||||
Pack(PackLocale),
|
||||
/// A language without proofing tools here (`x-none`, `zxx`, `und`, a malformed tag). Like Word without
|
||||
/// the language's proofing tools: no spelling or grammar marks, so its words are never
|
||||
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
|
||||
/// languages were told apart.
|
||||
@@ -129,7 +135,7 @@ impl ProofLang {
|
||||
} else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) {
|
||||
ProofLang::De
|
||||
} else {
|
||||
ProofLang::Other
|
||||
PackLocale::new(tag).map_or(ProofLang::Other, ProofLang::Pack)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,13 +188,86 @@ impl ProofLang {
|
||||
ProofLang::Id => "id",
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es",
|
||||
ProofLang::De => "de",
|
||||
ProofLang::Other => "",
|
||||
ProofLang::Pack(_) | ProofLang::Other => "",
|
||||
}
|
||||
}
|
||||
|
||||
/// Does WordCraft check spelling and grammar in this language?
|
||||
/// The locale of a [`ProofLang::Pack`] language.
|
||||
pub fn pack_locale(self) -> Option<PackLocale> {
|
||||
if let ProofLang::Pack(l) = self { Some(l) } else { None }
|
||||
}
|
||||
|
||||
/// Does WordCraft check spelling in this language? For a [`ProofLang::Pack`] language: is a
|
||||
/// dictionary for it available now (read on first use, cached until the dictionaries folder or
|
||||
/// the packs change)?
|
||||
pub fn is_proofed(self) -> bool {
|
||||
Self::PROOFED.contains(&self)
|
||||
match self {
|
||||
ProofLang::Pack(l) => crate::pack_lang::has_dictionary(l),
|
||||
_ => Self::PROOFED.contains(&self),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The locale of a [`ProofLang::Pack`] language: a lowercase BCP 47 tag (`nl`, `fr-fr`,
|
||||
/// `sr-latn`) of at most [`PackLocale::MAX_LEN`] bytes, stored inline so [`ProofLang`] stays `Copy`.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub struct PackLocale {
|
||||
len: u8,
|
||||
bytes: [u8; PackLocale::MAX_LEN],
|
||||
}
|
||||
|
||||
impl PackLocale {
|
||||
/// Longest locale kept; a longer tag is cut at a subtag boundary.
|
||||
pub const MAX_LEN: usize = 15;
|
||||
|
||||
/// The locale of a language tag (`nl-NL`, `sr_Latn`, `fr-FR.UTF-8`): lowercased, `_` as `-`,
|
||||
/// without a POSIX `.…`/`@…` tail or anything from a singleton on (`-x-…`, `-u-…`). `None` for
|
||||
/// a tag that isn't a language (`x-none`, `i-…`, `und`, `zxx`, `mul`, `mis`, malformed ones).
|
||||
pub fn new(tag: &str) -> Option<PackLocale> {
|
||||
let tag = tag.trim().split(['.', '@']).next().unwrap_or("");
|
||||
let mut parts = tag.split(['-', '_']);
|
||||
let primary = parts.next()?.to_ascii_lowercase();
|
||||
if !(2..=8).contains(&primary.len())
|
||||
|| !primary.bytes().all(|b| b.is_ascii_lowercase())
|
||||
|| ["und", "zxx", "mul", "mis"].contains(&primary.as_str())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let mut out = primary;
|
||||
for p in parts {
|
||||
if p.len() <= 1 || p.len() > 8 || !p.bytes().all(|b| b.is_ascii_alphanumeric()) || out.len() + 1 + p.len() > Self::MAX_LEN {
|
||||
break;
|
||||
}
|
||||
out.push('-');
|
||||
out.push_str(&p.to_ascii_lowercase());
|
||||
}
|
||||
let mut bytes = [0u8; Self::MAX_LEN];
|
||||
for (b, c) in bytes.iter_mut().zip(out.bytes()) {
|
||||
*b = c;
|
||||
}
|
||||
Some(PackLocale { len: u8::try_from(out.len()).ok()?, bytes })
|
||||
}
|
||||
|
||||
/// The locale (`nl`, `fr-fr`).
|
||||
pub fn as_str(&self) -> &str {
|
||||
self.bytes.get(..usize::from(self.len)).and_then(|b| std::str::from_utf8(b).ok()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// The primary language subtag (`fr`).
|
||||
pub fn primary(&self) -> &str {
|
||||
self.as_str().split('-').next().unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for PackLocale {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "PackLocale({:?})", self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for PackLocale {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -358,7 +437,13 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn tags_reject_unknown_or_partial_language_codes() {
|
||||
assert_tags(&["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"], ProofLang::Other);
|
||||
// A partial code is another language, never the one it starts like.
|
||||
for tag in ["fr-FR", "dex", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax"] {
|
||||
let lang = ProofLang::from_tag(Some(tag));
|
||||
assert_eq!(lang.pack_locale().map(|l| l.to_string()), Some(tag.to_ascii_lowercase()), "{tag}");
|
||||
assert_eq!(lang.code(), "", "{tag}");
|
||||
}
|
||||
assert_tags(&["x-none", "e", "-", "und", "zxx", "i-klingon"], ProofLang::Other);
|
||||
}
|
||||
|
||||
/// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and
|
||||
|
||||
+112
-3
@@ -42,8 +42,11 @@
|
||||
//! existing English fallback.
|
||||
//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed
|
||||
//! spelling, MIT); no spelling or grammar marks yet.
|
||||
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
|
||||
//! the language's proofing tools.
|
||||
//! - **Any other language** ([`ProofLang::Pack`], module [`packs`]): spelling and suggestions
|
||||
//! from its Hunspell dictionary (an installed Language pack, the dictionaries folder or the
|
||||
//! system's) with default tuning, hyphenation from its pack, no grammar rules; without a
|
||||
//! dictionary no marks, like Word without the language's proofing tools.
|
||||
//! - **No language** ([`ProofLang::Other`]: `x-none`, `zxx`, malformed tags): no marks.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese,
|
||||
//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic.
|
||||
//!
|
||||
@@ -76,6 +79,7 @@ pub mod hyphen;
|
||||
pub mod id;
|
||||
pub mod it;
|
||||
pub mod lang;
|
||||
mod pack_lang;
|
||||
pub mod packs;
|
||||
pub mod patterns;
|
||||
pub mod pl;
|
||||
@@ -110,7 +114,7 @@ impl Default for Options {
|
||||
}
|
||||
|
||||
pub use english::EnglishSource;
|
||||
pub use lang::ProofLang;
|
||||
pub use lang::{PackLocale, ProofLang};
|
||||
|
||||
/// A problem found in text: byte range, kind, message and suggestions.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
@@ -314,9 +318,17 @@ pub(crate) fn reload_dictionaries() {
|
||||
l.slot.reload();
|
||||
}
|
||||
english::reload();
|
||||
pack_lang::reload();
|
||||
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// The `.dic` file a [`ProofLang::Pack`] language's spelling is read from now (the dictionaries
|
||||
/// folder, an installed Language pack or the system's); `None` when it has none (unchecked) or
|
||||
/// `lang` isn't a pack language. Reads the dictionary once (it is cached afterwards).
|
||||
pub fn pack_dictionary_file(lang: ProofLang) -> Option<std::path::PathBuf> {
|
||||
pack_lang::dictionary_file(lang.pack_locale()?)
|
||||
}
|
||||
|
||||
/// Each language that reads an installed Hunspell dictionary, with the `.dic` file found for it
|
||||
/// now (`None`: not installed, so the language has no spelling marks).
|
||||
pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)> {
|
||||
@@ -331,6 +343,9 @@ pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
|
||||
if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) {
|
||||
return english::load_error(lang);
|
||||
}
|
||||
if let ProofLang::Pack(l) = lang {
|
||||
return pack_lang::load_error(l);
|
||||
}
|
||||
hunspell::load_error(language(lang)?.slot.name()).map(|(_, reason)| reason)
|
||||
}
|
||||
|
||||
@@ -827,6 +842,9 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
if lang.is_english() {
|
||||
return is_correct_english(word, lang, opts);
|
||||
}
|
||||
if let ProofLang::Pack(l) = lang {
|
||||
return accepted_by_options(word, opts) || pack_lang::is_correct(word, l, opts);
|
||||
}
|
||||
// A language WordCraft doesn't proof (Bulgarian, German, any other) has no spelling marks.
|
||||
language(lang).is_none_or(|l| accepted_by_options(word, opts) || (l.is_correct)(word, lang, opts))
|
||||
}
|
||||
@@ -838,6 +856,9 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
if lang.is_english() {
|
||||
return suggest_english(word, max, lang);
|
||||
}
|
||||
if let ProofLang::Pack(l) = lang {
|
||||
return pack_lang::suggest(word, max, l);
|
||||
}
|
||||
language(lang).map_or_else(Vec::new, |l| (l.suggest)(word, max, lang))
|
||||
}
|
||||
|
||||
@@ -1347,6 +1368,8 @@ mod tests {
|
||||
#[test]
|
||||
fn an_installed_pack_is_used_and_a_removed_one_is_not() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// The test dictionaries on disk first, so a reload never leaves other tests without them.
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-packs-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let (folder, packs) = (root.join("dictionaries"), root.join("language-packs"));
|
||||
@@ -1414,12 +1437,98 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// CP4.2: a language with no tools of its own (Dutch) is spell-checked, given suggestions and
|
||||
/// hyphenated from its installed Language pack; removed, it is unchecked again; English is
|
||||
/// unaffected. No grammar rules for it.
|
||||
#[test]
|
||||
fn a_pack_proofs_a_language_without_its_own_tools() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// The test dictionaries on disk first, so a reload never leaves other tests without them.
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-nl-pack-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let nl = ProofLang::from_tag(Some("nl-NL"));
|
||||
assert_eq!(nl.pack_locale().map(|l| l.to_string()).as_deref(), Some("nl-nl"));
|
||||
let text = "Het huis en de fietz.";
|
||||
let english_before = check_text(text, &[]);
|
||||
let unchecked = |lang: ProofLang| check_text(text, &[(0..text.len(), lang)]).iter().all(|i| i.kind == IssueKind::Grammar);
|
||||
let before = (is_correct_in("fietz", nl), hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), pack_dictionary_file(nl));
|
||||
let pack = write_pack(
|
||||
&root,
|
||||
"nl",
|
||||
&[
|
||||
("spelling.aff", b"SET UTF-8\nTRY esianrtolducgmphbykfvwjzxq\nSFX E Y 1\nSFX E 0 en .\n"),
|
||||
("spelling.dic", b"5\nhet\nhuis/E\nen\nde\nfiets/E\n"),
|
||||
("hyphenation.pat", b"1na xx\n"),
|
||||
],
|
||||
);
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
assert!(nl.is_proofed() && has_dictionary(Some("nl-BE")) && !has_dictionary(Some("fr-FR")));
|
||||
assert_eq!(pack_dictionary_file(nl), Some(pack.join("spelling.dic")));
|
||||
assert_eq!(pack_dictionary_file(ProofLang::from_tag(Some("nl"))), Some(pack.join("spelling.dic")), "a region falls back to the pack");
|
||||
for w in ["huis", "Huis", "huizen", "huisen", "fietsen", "HUIS"] {
|
||||
assert_eq!(is_correct_in(w, nl), w != "huizen", "{w}");
|
||||
}
|
||||
let issues = check_text(text, &[(0..text.len(), nl)]);
|
||||
assert_eq!(issues.len(), 1, "{issues:?}");
|
||||
assert_eq!((issues[0].kind, &text[issues[0].start..issues[0].end], issues[0].lang), (IssueKind::Spelling, "fietz", nl));
|
||||
assert!(suggest_in("fietz", 5, nl).contains(&"fiets".to_string()), "{:?}", suggest_in("fietz", 5, nl));
|
||||
assert!(check_grammar_in("de de huis", nl).is_empty(), "no grammar rules");
|
||||
assert_eq!(hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), "xx-naxx");
|
||||
assert!(unchecked(ProofLang::from_tag(Some("fr-FR"))), "a language with no pack stays unchecked");
|
||||
assert_eq!(check_text(text, &[]), english_before, "English is unaffected");
|
||||
// A broken dictionary is reported, never a panic.
|
||||
std::fs::write(pack.join("spelling.aff"), "SET SHIFT_JIS\n").unwrap();
|
||||
packs::reload();
|
||||
assert!(is_correct_in("fietz", nl) && !nl.is_proofed());
|
||||
assert!(dictionary_load_error(nl).unwrap_or_default().contains("unsupported encoding"));
|
||||
// Removed: unchecked again.
|
||||
std::fs::remove_dir_all(&pack).unwrap();
|
||||
packs::reload();
|
||||
assert_eq!(
|
||||
(is_correct_in("fietz", nl), hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), pack_dictionary_file(nl)),
|
||||
before
|
||||
);
|
||||
assert!(unchecked(nl) || before.2.is_some());
|
||||
assert_eq!(check_text(text, &[]), english_before);
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// At most [`pack_lang::MAX_LOADED`] pack-language dictionaries stay loaded; others load again
|
||||
/// when used.
|
||||
#[test]
|
||||
fn pack_language_dictionaries_are_capped() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// The test dictionaries on disk first, so a reload never leaves other tests without them.
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-many-packs-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let locales: Vec<String> = (0..pack_lang::MAX_LOADED + 3).map(|i| format!("q{}", (b'a' + i as u8) as char)).collect();
|
||||
for l in &locales {
|
||||
write_pack(&root, l, &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", format!("1\nword{l}\n").as_bytes())]);
|
||||
}
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
for _ in 0..2 {
|
||||
for l in &locales {
|
||||
let lang = ProofLang::from_tag(Some(l));
|
||||
assert!(is_correct_in(&format!("word{l}"), lang) && !is_correct_in("wordzz", lang), "{l}");
|
||||
}
|
||||
}
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// CP4.2, hostile packs: a missing or broken `pack.toml`, a path outside the pack, a folder
|
||||
/// that isn't a language tag, a symbolic link, patterns that aren't UTF-8 or a dictionary that
|
||||
/// can't be read are reported, never a panic, and the pack (or that file) is skipped.
|
||||
#[test]
|
||||
fn corrupt_packs_are_reported_and_skipped() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// The test dictionaries on disk first, so a reload never leaves other tests without them.
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-bad-packs-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
// Good spelling (the test words, so Croatian tests running meanwhile are unaffected), bad
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
//! Proofing for any language by its locale ([`crate::ProofLang::Pack`]): its Hunspell dictionary (the
|
||||
//! dictionaries folder, then an installed Language pack, then the system's), read by the shared
|
||||
//! affix engine, with default suggestion tuning and no grammar rules.
|
||||
//!
|
||||
//! At most [`MAX_LOADED`] dictionaries stay loaded; the one used least recently goes first.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, PoisonError, RwLock};
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
use crate::affix_suggest::Tuning;
|
||||
use crate::lang::PackLocale;
|
||||
|
||||
/// Most locale dictionaries kept loaded at once.
|
||||
pub(crate) const MAX_LOADED: usize = 8;
|
||||
|
||||
struct Loaded {
|
||||
locale: PackLocale,
|
||||
dict: Arc<AffixDict>,
|
||||
/// The `.dic` it came from (`None`: none found, the language is unchecked).
|
||||
file: Option<PathBuf>,
|
||||
used: u64,
|
||||
}
|
||||
|
||||
struct Cache {
|
||||
loaded: Vec<Loaded>,
|
||||
clock: u64,
|
||||
/// Counts reloads, so a load that started before one is not kept.
|
||||
reloads: u64,
|
||||
}
|
||||
|
||||
static CACHE: RwLock<Cache> = RwLock::new(Cache { loaded: Vec::new(), clock: 0, reloads: 0 });
|
||||
|
||||
/// Forget every loaded dictionary (the dictionaries folder or the packs changed).
|
||||
pub(crate) fn reload() {
|
||||
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
|
||||
c.loaded.clear();
|
||||
c.reloads = c.reloads.wrapping_add(1);
|
||||
}
|
||||
|
||||
/// Hunspell file names for `locale`, best first: `fr-fr` → `fr_FR`, `fr-FR`, `fr`;
|
||||
/// `sr-latn` → `sr_Latn`, `sr-Latn` (never plain `sr`, another script).
|
||||
pub(crate) fn names(locale: PackLocale) -> Vec<String> {
|
||||
let parts: Vec<&str> = locale.as_str().split('-').collect();
|
||||
let cased: Vec<String> = parts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(i, p)| match (i, p.len()) {
|
||||
(0, _) => (*p).to_string(),
|
||||
(_, 2) => p.to_ascii_uppercase(),
|
||||
(_, 4) => {
|
||||
let mut c = p.chars();
|
||||
c.next().map(|f| f.to_ascii_uppercase().to_string() + c.as_str()).unwrap_or_default()
|
||||
}
|
||||
_ => (*p).to_string(),
|
||||
})
|
||||
.collect();
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
let mut push = |n: String| {
|
||||
if !out.contains(&n) {
|
||||
out.push(n);
|
||||
}
|
||||
};
|
||||
// Each shorter prefix too (`sr-latn-rs` → `sr-latn`), but not past a script subtag.
|
||||
let script = parts.iter().position(|p| p.len() == 4).unwrap_or(usize::MAX);
|
||||
for n in (1..=cased.len()).rev() {
|
||||
if n <= script && script != usize::MAX {
|
||||
break;
|
||||
}
|
||||
let prefix = cased.get(..n).unwrap_or_default();
|
||||
push(prefix.join("_"));
|
||||
push(prefix.join("-"));
|
||||
}
|
||||
if script != usize::MAX {
|
||||
let prefix = cased.get(..=script).unwrap_or_default();
|
||||
push(prefix.join("_"));
|
||||
push(prefix.join("-"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `locale`'s dictionary, loaded on first use; an empty one when there is none.
|
||||
fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
|
||||
let reloads = {
|
||||
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
|
||||
c.clock = c.clock.wrapping_add(1);
|
||||
let now = c.clock;
|
||||
if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) {
|
||||
l.used = now;
|
||||
return (l.dict.clone(), l.file.clone());
|
||||
}
|
||||
c.reloads
|
||||
};
|
||||
let (dict, file) = load(locale);
|
||||
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
|
||||
if c.reloads == reloads && !c.loaded.iter().any(|l| l.locale == locale) {
|
||||
if c.loaded.len() >= MAX_LOADED
|
||||
&& let Some(oldest) = c.loaded.iter().enumerate().min_by_key(|(_, l)| l.used).map(|(i, _)| i)
|
||||
{
|
||||
c.loaded.swap_remove(oldest);
|
||||
}
|
||||
let used = c.clock;
|
||||
c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), used });
|
||||
}
|
||||
(dict, file)
|
||||
}
|
||||
|
||||
fn load(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
|
||||
let names = names(locale);
|
||||
let refs: Vec<&str> = names.iter().map(String::as_str).collect();
|
||||
let key = names.first().map_or("", String::as_str);
|
||||
let Some((aff, dic)) = crate::hunspell::find(&refs) else {
|
||||
crate::hunspell::clear_load_error(key);
|
||||
return (Arc::default(), None);
|
||||
};
|
||||
match crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)) {
|
||||
Ok(d) => (Arc::new(d), Some(dic)),
|
||||
Err(e) => {
|
||||
log::warn!("{locale} spelling: {e}");
|
||||
(Arc::default(), None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Is a dictionary for `locale` available (its spelling is checked)?
|
||||
pub(crate) fn has_dictionary(locale: PackLocale) -> bool {
|
||||
!get(locale).0.is_empty()
|
||||
}
|
||||
|
||||
/// The `.dic` file `locale`'s spelling is read from now.
|
||||
pub(crate) fn dictionary_file(locale: PackLocale) -> Option<PathBuf> {
|
||||
let (d, file) = get(locale);
|
||||
file.filter(|_| !d.is_empty())
|
||||
}
|
||||
|
||||
/// Why `locale`'s dictionary failed to load, if it did.
|
||||
pub(crate) fn load_error(locale: PackLocale) -> Option<String> {
|
||||
crate::hunspell::load_error(names(locale).first()?).map(|(_, e)| e)
|
||||
}
|
||||
|
||||
/// Is `word` right in `locale`? Every word is when there is no dictionary.
|
||||
pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool {
|
||||
let (d, _) = get(locale);
|
||||
if d.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let Some(w) = crate::spelling::word_to_check(&d, word, opts) else { return true };
|
||||
let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase());
|
||||
if w.contains('-') && !known(w) {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(p));
|
||||
}
|
||||
known(w)
|
||||
}
|
||||
|
||||
/// Suggestions for `word` in `locale`, best first, at most `max`.
|
||||
pub(crate) fn suggest(word: &str, max: usize, locale: PackLocale) -> Vec<String> {
|
||||
let (d, _) = get(locale);
|
||||
if d.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
crate::affix_suggest::suggest_in(&d, word, max, &Tuning::DEFAULT)
|
||||
}
|
||||
|
||||
/// The pack locales whose hyphenation patterns serve a [`crate::ProofLang::Pack`] language: the locale,
|
||||
/// then each shorter prefix not past a script subtag (`fr-ca` → `fr`).
|
||||
pub(crate) fn hyphenation_locales(locale: PackLocale) -> Vec<String> {
|
||||
names(locale).into_iter().filter(|n| !n.contains('_')).map(|n| n.to_ascii_lowercase()).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn l(t: &str) -> PackLocale {
|
||||
PackLocale::new(t).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn locales_and_file_names() {
|
||||
assert_eq!(l("nl-NL").as_str(), "nl-nl");
|
||||
assert_eq!(l("fr_FR.UTF-8").as_str(), "fr-fr");
|
||||
assert_eq!(l("ca-ES-valencia").as_str(), "ca-es-valencia");
|
||||
assert_eq!(l("sr-Latn-RS-x-private").as_str(), "sr-latn-rs");
|
||||
assert_eq!(l("zh-Hant-TW-somethinglong").as_str(), "zh-hant-tw");
|
||||
for bad in ["x-none", "i-klingon", "und", "zxx", "mul", "e", "-", "", "1x", "näh"] {
|
||||
assert_eq!(PackLocale::new(bad), None, "{bad}");
|
||||
}
|
||||
assert_eq!(names(l("nl")), ["nl"]);
|
||||
assert_eq!(names(l("fr-FR")), ["fr_FR", "fr-FR", "fr"]);
|
||||
assert_eq!(names(l("sr-Latn")), ["sr_Latn", "sr-Latn"]);
|
||||
assert_eq!(names(l("sr-Latn-RS")), ["sr_Latn_RS", "sr-Latn-RS", "sr_Latn", "sr-Latn"]);
|
||||
assert_eq!(hyphenation_locales(l("fr-CA")), ["fr-ca", "fr"]);
|
||||
assert_eq!(hyphenation_locales(l("sr-Latn")), ["sr-latn"]);
|
||||
}
|
||||
}
|
||||
@@ -362,12 +362,18 @@ fn locales_for(names: &[&str]) -> Vec<String> {
|
||||
for n in names {
|
||||
push(n.to_ascii_lowercase().replace('_', "-"));
|
||||
}
|
||||
for n in names {
|
||||
// The primary language alone, unless a script is named (`sr_Latn` is never plain `sr`).
|
||||
for n in names.iter().filter(|n| !has_script(n)) {
|
||||
push(n.split(['_', '-']).next().unwrap_or_default().to_ascii_lowercase());
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Does a tag or file name name a script (a four-letter subtag after the language: `sr-Latn`)?
|
||||
fn has_script(tag: &str) -> bool {
|
||||
tag.split(['_', '-']).skip(1).any(|p| p.len() == 4 && p.bytes().all(|b| b.is_ascii_alphabetic()))
|
||||
}
|
||||
|
||||
/// The installed packs to read for `locales`, in that order, then any other pack of their
|
||||
/// primary languages (`pt` → `pt-br`, `pt-pt`), keeping those that `has` the data.
|
||||
fn packs_for(locales: &[String], has: impl Fn(&Holds) -> bool) -> Vec<PathBuf> {
|
||||
@@ -379,10 +385,11 @@ fn packs_for(locales: &[String], has: impl Fn(&Holds) -> bool) -> Vec<PathBuf> {
|
||||
out.push(p.path.clone());
|
||||
}
|
||||
}
|
||||
let primaries: Vec<&str> = locales.iter().map(|l| l.split('-').next().unwrap_or_default()).collect();
|
||||
// Then another region's pack of the same language (`pt` → `pt-br`), never another script's.
|
||||
let primaries: Vec<&str> = locales.iter().filter(|l| !has_script(l)).map(|l| l.split('-').next().unwrap_or_default()).collect();
|
||||
for (p, _) in usable() {
|
||||
let primary = p.locale.split('-').next().unwrap_or_default();
|
||||
if primaries.contains(&primary) && !out.contains(&p.path) {
|
||||
if primaries.contains(&primary) && !has_script(&p.locale) && !out.contains(&p.path) {
|
||||
out.push(p.path.clone());
|
||||
}
|
||||
}
|
||||
@@ -416,6 +423,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec<String> {
|
||||
ProofLang::Es419 => &["es-419", "es"],
|
||||
ProofLang::EsUy => &["es-uy", "es-419", "es"],
|
||||
ProofLang::De => &["de", "de-1996", "de-de"],
|
||||
ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l),
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[],
|
||||
};
|
||||
names.iter().map(|n| (*n).to_string()).collect()
|
||||
|
||||
Reference in new issue
Block a user