mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
CP4.2: review — English tests hold the dictionaries lock; another region's pack never beats the system's exact region; bounded pack hyphenation lookups; NFC hyphenation; licences budgeted before reading; pack languages report like the others
P2: - Every test whose English results depend on installed dictionaries takes DICTIONARY_TESTS (the en-GB goitre CI failure: tiny en_US fixtures raced unlocked en-GB tests). - hunspell::candidates/find order: dictionaries folder, the first name's region pack (or the language alone), the system's exact-region dictionary, any other region's pack, the system's other names. pt_BR system beats a pt-pt pack; de_DE / de-ch, nl_NL / nl-be, es_ES / es-mx, en_US / en-gb likewise. P3: - Pack hyphenation lookups: a map, at most 256 languages (cleared when full); 3000 tags stay bounded and fast. - A decomposed word (a + U+0301) is hyphenated composed (NFC) and the breaks mapped back; combining marks stay in their word. - licence_texts checks each file's size against what is left before reading it; never reads past 16 MB. - German, Bulgarian and pack languages: spelling message in the language where we have it; missing_dictionary/unreadable_dictionary report them like the others. - build.rs: a missing file no longer says nothing is embedded; ru ignore message says CRAFT_I18N_DIR is enough. Built-in English list notice names the whole Moby list (en-words.dic). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FypgCX8LsYYE8fsj4byBm2
This commit is contained in:
1 parent
409c8facfd
commit
ee3be01b73
12 files changed
+389
-47
No files matched your search
@@ -102,7 +102,7 @@ fn main() {
|
||||
wanted_files += usize::from(wanted);
|
||||
let file = dir.as_ref().filter(|_| wanted).map(|d| d.join(rel)).filter(|f| f.is_file());
|
||||
if let (Some(d), None, true) = (&dir, &file, wanted) {
|
||||
println!("cargo::{level}=CRAFT_I18N_DIR: {} is missing; craft-proof embeds none", d.join(rel).display());
|
||||
println!("cargo::{level}=CRAFT_I18N_DIR: {} is missing; craft-proof embeds the rest without it", d.join(rel).display());
|
||||
}
|
||||
let (ty, empty, mac) = if bytes { ("&[u8]", "b\"\"", "include_bytes") } else { ("&str", "\"\"", "include_str") };
|
||||
// `language_data` only hands back a UTF-8 folder, so this is the path as it is.
|
||||
|
||||
@@ -163,6 +163,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn czech_dispatch_messages_options_and_mixed_spans() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
test_dict::install();
|
||||
assert!(crate::has_dictionary(Some("cs-CZ")));
|
||||
assert!(crate::is_correct_for("městě", Some("cze")));
|
||||
|
||||
@@ -245,6 +245,8 @@ mod tests {
|
||||
/// composed form (NFC) and keeps its own byte range, accents and all.
|
||||
#[test]
|
||||
fn decomposed_accents_are_checked_like_composed_ones() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
test_dict::install();
|
||||
for lang in VARIETIES {
|
||||
for text in ["esto también", "esto tambie\u{301}n", "una cancio\u{301}n y pingu\u{308}ino", "CANCIO\u{301}N", "nin\u{303}o"] {
|
||||
@@ -289,6 +291,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn spanish_dispatch_messages_options_and_mixed_spans() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
test_dict::install();
|
||||
for tag in ["es-ES", "es-MX", "es-419", "es-AR", "es-UY", "spa"] {
|
||||
assert!(crate::has_dictionary(Some(tag)), "{tag}");
|
||||
|
||||
@@ -150,27 +150,53 @@ fn xdg_data_home(xdg: Option<std::ffi::OsString>, home: Option<&Path>) -> Option
|
||||
xdg.map(PathBuf::from).filter(|p| p.is_absolute()).or_else(|| home.map(|h| h.join(".local/share")))
|
||||
}
|
||||
|
||||
/// The first `<name>.aff`/`<name>.dic` pair found for any of `names` (`pl_PL`, `pl`), searching
|
||||
/// [`search_dirs`] (including LibreOffice's extension folder for the first name).
|
||||
///
|
||||
/// Lookup order: the dictionaries folders, then an installed Language pack for one of `names`
|
||||
/// (its `spelling.aff`/`spelling.dic`, [`crate::packs`]), then the system's folders.
|
||||
/// The first `<name>.aff`/`<name>.dic` pair found for any of `names` (`pl_PL`, `pl`), in the
|
||||
/// order of [`candidates`].
|
||||
pub fn find(names: &[&str]) -> Option<(PathBuf, PathBuf)> {
|
||||
let lo = names.first().copied().map(lo_ext_dirs);
|
||||
let (dirs, system) = search_tiers(lo.as_deref());
|
||||
dirs.iter()
|
||||
.find_map(|dir| find_in(dir, names))
|
||||
.or_else(|| crate::packs::spelling_dirs(names).iter().find_map(|dir| find_in(dir, &[crate::packs::SPELLING])))
|
||||
.or_else(|| system.iter().find_map(|dir| find_in(dir, names)))
|
||||
candidates(names).into_iter().next()
|
||||
}
|
||||
|
||||
/// Every `<name>.aff`/`<name>.dic` pair for `names`, in the order [`find`] prefers them: the
|
||||
/// dictionaries folders, then installed Language packs for `names` (their `spelling.aff`/
|
||||
/// `spelling.dic`), then the system's folders; folder by folder, each folder's in the order of
|
||||
/// `names`. A caller that can't read one tries the next.
|
||||
/// Every `<name>.aff`/`<name>.dic` pair for `names`, best first. A caller that can't read one
|
||||
/// tries the next. The order:
|
||||
///
|
||||
/// 1. the dictionaries folders ([`set_dictionary_dir`] and the others of [`search_dirs`]), any
|
||||
/// of `names`;
|
||||
/// 2. an installed Language pack of the first name's region or of the language alone
|
||||
/// ([`crate::packs`], `spelling.aff`/`spelling.dic`);
|
||||
/// 3. the system's folders, the first name's region (`pt_BR`, `pt-BR`);
|
||||
/// 4. any other installed pack of the language (another region: `pt-pt`);
|
||||
/// 5. the system's folders, the other names (`pt`).
|
||||
pub fn candidates(names: &[&str]) -> Vec<(PathBuf, PathBuf)> {
|
||||
let lo = names.first().copied().map(lo_ext_dirs);
|
||||
candidates_in(&search_dirs(lo.as_deref()), names)
|
||||
let (dirs, system) = search_tiers(lo.as_deref());
|
||||
let (region_packs, other_packs) = crate::packs::spelling_dirs(names);
|
||||
ordered(names, &dirs, ®ion_packs, &system, &other_packs)
|
||||
}
|
||||
|
||||
/// [`candidates`] over the given folders.
|
||||
fn ordered(names: &[&str], dirs: &[PathBuf], region_packs: &[PathBuf], system: &[PathBuf], other_packs: &[PathBuf]) -> Vec<(PathBuf, PathBuf)> {
|
||||
let norm = |n: &str| n.to_ascii_lowercase().replace('_', "-");
|
||||
let first = names.first().map(|n| norm(n));
|
||||
let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first);
|
||||
let pack: &[&str] = &[crate::packs::SPELLING];
|
||||
let tiers = dirs
|
||||
.iter()
|
||||
.map(|d| (d, names))
|
||||
.chain(region_packs.iter().map(|d| (d, pack)))
|
||||
.chain(system.iter().map(|d| (d, ®ion[..])))
|
||||
.chain(other_packs.iter().map(|d| (d, pack)))
|
||||
.chain(system.iter().map(|d| (d, &rest[..])));
|
||||
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
|
||||
for (dir, names) in tiers {
|
||||
for n in names {
|
||||
let aff = dir.join(format!("{n}.aff"));
|
||||
let dic = dir.join(format!("{n}.dic"));
|
||||
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
|
||||
out.push((aff, dic));
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// [`candidates`] in the folders the user keeps dictionaries in for WordCraft ([`user_dirs`])
|
||||
@@ -569,6 +595,56 @@ fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Another region's Language pack never beats the system's dictionary of the right region
|
||||
/// (`pt-pt` pack vs system `pt_BR`; `de-ch` vs `de_DE`; `en-gb` vs `en_US`), but beats the
|
||||
/// system's other names; the right region's pack beats the system; the dictionaries folder
|
||||
/// beats everything.
|
||||
#[test]
|
||||
fn another_regions_pack_comes_after_the_systems_exact_region() {
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-order-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let pair = |dir: &Path, name: &str| {
|
||||
std::fs::create_dir_all(dir).unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.dic")), "1\nx\n").unwrap();
|
||||
dir.join(format!("{name}.dic"))
|
||||
};
|
||||
let (folder, system) = (root.join("folder"), root.join("system"));
|
||||
let en: &[&str] = &["en_US", "en-US", "en", "en_CA", "en_GB", "en-GB", "en_AU"];
|
||||
for (names, exact, other, cross) in [
|
||||
(&["pt_BR", "pt-BR", "pt"][..], "pt_BR", "pt", "pt-pt"),
|
||||
(&["de_DE", "de-DE", "de"][..], "de_DE", "de", "de-ch"),
|
||||
(&["nl_NL", "nl-NL", "nl"][..], "nl_NL", "nl", "nl-be"),
|
||||
(&["es_ES", "es-ES", "es_ANY", "es"][..], "es_ES", "es", "es-mx"),
|
||||
(en, "en_US", "en_GB", "en-gb"),
|
||||
] {
|
||||
let sys_exact = pair(&system, exact);
|
||||
let sys_other = pair(&system, other);
|
||||
let cross_pack = pair(&root.join("packs").join(cross), crate::packs::SPELLING);
|
||||
let dics = |v: Vec<(PathBuf, PathBuf)>| v.into_iter().map(|(_, d)| d).collect::<Vec<_>>();
|
||||
let got =
|
||||
dics(ordered(names, &[], &[], std::slice::from_ref(&system), std::slice::from_ref(&cross_pack.parent().unwrap().to_path_buf())));
|
||||
assert_eq!(got, [sys_exact.clone(), cross_pack.clone(), sys_other.clone()], "{names:?}");
|
||||
// The region's own pack first, then the system's exact region.
|
||||
let region_pack = pair(&root.join("packs").join(format!("{}-region", names[0])), crate::packs::SPELLING);
|
||||
let got = dics(ordered(names, &[], &[region_pack.parent().unwrap().to_path_buf()], std::slice::from_ref(&system), &[]));
|
||||
assert_eq!(got.first(), Some(®ion_pack), "{names:?}");
|
||||
assert_eq!(got.get(1), Some(&sys_exact), "{names:?}");
|
||||
// The dictionaries folder before all of them, whatever name.
|
||||
let mine = pair(&folder, other);
|
||||
let got = dics(ordered(
|
||||
names,
|
||||
std::slice::from_ref(&folder),
|
||||
&[region_pack.parent().unwrap().to_path_buf()],
|
||||
std::slice::from_ref(&system),
|
||||
&[],
|
||||
));
|
||||
assert_eq!(got.first(), Some(&mine), "{names:?}");
|
||||
assert_eq!(find_in(&folder, &[other]).map(|(_, d)| d), Some(mine));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
|
||||
/// WordCraft's own dictionaries folder under the user's application data is searched on every
|
||||
/// desktop system, after the setting and before the shared system locations (#1063).
|
||||
#[test]
|
||||
|
||||
@@ -119,7 +119,8 @@ fn is_apostrophe(c: char) -> bool {
|
||||
}
|
||||
|
||||
fn is_word_char(c: char) -> bool {
|
||||
c.is_alphabetic() || is_apostrophe(c)
|
||||
// A combining mark belongs to its letter (decomposed `a` + U+0301 is one letter, `á`).
|
||||
c.is_alphabetic() || is_apostrophe(c) || unicode_normalization::char::is_combining_mark(c)
|
||||
}
|
||||
|
||||
/// Unrestricted break points of one lowercase core (letters and `'` only, no leading/trailing `'`).
|
||||
@@ -184,6 +185,26 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
(heuristic_points(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// `core` composed (NFC), with the index in `core` where each composed character's base starts
|
||||
/// (`None` for a character that isn't the first of its base's group). Never longer than `core`.
|
||||
fn composed_chars(core: &[char]) -> (Vec<char>, Vec<Option<usize>>) {
|
||||
use unicode_normalization::UnicodeNormalization;
|
||||
use unicode_normalization::char::is_combining_mark;
|
||||
let mut nfc: Vec<char> = Vec::with_capacity(core.len());
|
||||
let mut starts: Vec<Option<usize>> = Vec::with_capacity(core.len());
|
||||
let mut i = 0;
|
||||
while i < core.len() {
|
||||
let end = core.get(i + 1..).and_then(|rest| rest.iter().position(|&c| !is_combining_mark(c))).map_or(core.len(), |n| i + 1 + n);
|
||||
let group: String = core.get(i..end).unwrap_or_default().iter().collect();
|
||||
for (k, c) in group.nfc().enumerate() {
|
||||
nfc.push(c);
|
||||
starts.push((k == 0).then_some(i));
|
||||
}
|
||||
i = end;
|
||||
}
|
||||
(nfc, starts)
|
||||
}
|
||||
|
||||
/// Does a pack's `pats` hyphenate `core` in `lang`? Russian, Bulgarian and Turkish: a word in
|
||||
/// their script (the guard their rules use). Polish, Portuguese, Italian and German: every word,
|
||||
/// as their embedded patterns do. Any other language: a word whose letters the patterns know.
|
||||
@@ -249,6 +270,16 @@ pub fn word_breaks(word: &str) -> (Vec<usize>, Source) {
|
||||
/// run is hyphenated separately). No break is ever placed next to an apostrophe or non-letter.
|
||||
pub fn word_breaks_in(word: &str, lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
let chars: Vec<char> = word.chars().collect();
|
||||
// Patterns are composed text (NFC), as spelling reads words: a decomposed word (`a` + U+0301)
|
||||
// is hyphenated composed, and the breaks mapped back to its own chars.
|
||||
if unicode_normalization::is_nfc_quick(word.chars()) != unicode_normalization::IsNormalized::Yes {
|
||||
let (nfc, starts) = composed_chars(&chars);
|
||||
if nfc.len() < chars.len() {
|
||||
let composed: String = nfc.iter().collect();
|
||||
let (pts, src) = word_breaks_in(&composed, lang);
|
||||
return (pts.into_iter().filter_map(|i| starts.get(i).copied().flatten()).collect(), src);
|
||||
}
|
||||
}
|
||||
let mut out = Vec::new();
|
||||
let mut src = Source::None;
|
||||
let mut i = 0;
|
||||
|
||||
@@ -170,6 +170,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn indonesian_dispatch_messages_options_and_mixed_spans() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
test_dict::install();
|
||||
assert!(crate::has_dictionary(Some("id-ID")));
|
||||
assert!(crate::is_correct_for("rumah", Some("ind")));
|
||||
|
||||
+167
-3
@@ -601,15 +601,44 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p
|
||||
|
||||
/// Did the Hunspell dictionary of `lang` load? `None` for languages that use none (English).
|
||||
fn dictionary_loaded(lang: ProofLang) -> Option<bool> {
|
||||
if let Some(l) = lang.spelling_locale() {
|
||||
return Some(pack_lang::has_dictionary(l));
|
||||
}
|
||||
language(lang).map(|l| l.slot.has_dictionary())
|
||||
}
|
||||
|
||||
/// The Hunspell name `lang` needs when `installed` is false (the first name its module looks for;
|
||||
/// a Spanish variety's is its own: `es_ES`, `es_MX`, `es_419`, `es_UY`).
|
||||
fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> {
|
||||
language(lang).filter(|_| !installed).map(|l| l.slot.name())
|
||||
if installed {
|
||||
return None;
|
||||
}
|
||||
if let Some(l) = lang.spelling_locale() {
|
||||
return pack_lang::names(l).first().and_then(|n| interned(n));
|
||||
}
|
||||
language(lang).map(|l| l.slot.name())
|
||||
}
|
||||
|
||||
/// `name` as a `&'static str` (the API hands out Hunspell names that way), for at most
|
||||
/// [`MAX_INTERNED`] names; `None` after that.
|
||||
fn interned(name: &str) -> Option<&'static str> {
|
||||
// ponytail: leaks each distinct pack dictionary name once, capped; a `Cow` API if more matter.
|
||||
static NAMES: std::sync::Mutex<Vec<&'static str>> = std::sync::Mutex::new(Vec::new());
|
||||
let mut names = NAMES.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if let Some(n) = names.iter().find(|n| **n == name) {
|
||||
return Some(n);
|
||||
}
|
||||
if names.len() >= MAX_INTERNED {
|
||||
return None;
|
||||
}
|
||||
let n: &'static str = Box::leak(name.to_string().into_boxed_str());
|
||||
names.push(n);
|
||||
Some(n)
|
||||
}
|
||||
|
||||
/// Most pack dictionary names [`interned`] keeps.
|
||||
const MAX_INTERNED: usize = 256;
|
||||
|
||||
/// Is `word` spelled correctly in the run's proofing language? A language without a dictionary
|
||||
/// can't be checked, so every word counts as correct (no red underlines for it).
|
||||
pub fn is_correct_for(word: &str, lang: Option<&str>) -> bool {
|
||||
@@ -806,6 +835,38 @@ pub(crate) fn is_address(token: &str) -> bool {
|
||||
token.contains("://") || token.contains('@') || token.starts_with("www.")
|
||||
}
|
||||
|
||||
/// The spelling mark's message in `lang`: a built-in language's own, a pack language's when we
|
||||
/// have it (German, Bulgarian and other common ones), else English.
|
||||
fn spelling_message(lang: ProofLang) -> &'static str {
|
||||
if let Some(l) = language(lang) {
|
||||
return l.message;
|
||||
}
|
||||
let pack = lang.pack_locale();
|
||||
let primary = match lang {
|
||||
ProofLang::De => "de",
|
||||
ProofLang::Bg => "bg",
|
||||
_ => pack.as_ref().map_or("", PackLocale::primary),
|
||||
};
|
||||
match primary {
|
||||
"de" => "Möglicher Rechtschreibfehler",
|
||||
"bg" => "Възможна правописна грешка",
|
||||
"nl" => "Mogelijke spelfout",
|
||||
"fr" => "Faute d’orthographe possible",
|
||||
"sv" => "Möjligt stavfel",
|
||||
"da" => "Mulig stavefejl",
|
||||
"nb" | "nn" | "no" => "Mulig stavefeil",
|
||||
"fi" => "Mahdollinen kirjoitusvirhe",
|
||||
"uk" => "Можлива орфографічна помилка",
|
||||
"el" => "Πιθανό ορθογραφικό λάθος",
|
||||
"ca" => "Possible error ortogràfic",
|
||||
"ro" => "Posibilă greșeală de ortografie",
|
||||
"hu" => "Lehetséges helyesírási hiba",
|
||||
"sk" => "Možná pravopisná chyba",
|
||||
"sl" => "Možna pravopisna napaka",
|
||||
_ => "Possible spelling mistake",
|
||||
}
|
||||
}
|
||||
|
||||
/// The spelling issue of the word at `a..b` of `text`, if it is misspelled in `lang`. Words in
|
||||
/// web and e-mail addresses are skipped (unless `opts` says otherwise); `internet` is the
|
||||
/// classification of the word's enclosing token, from [`enclosing_token`].
|
||||
@@ -815,7 +876,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
|
||||
if (opts.ignore_internet && internet) || (lang.is_english() && vi::is_vietnamese_word(w)) || is_correct_in_with(w, lang, opts) {
|
||||
return None;
|
||||
}
|
||||
let message = language(lang).map_or("Possible spelling mistake", |l| l.message);
|
||||
let message = spelling_message(lang);
|
||||
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
|
||||
}
|
||||
|
||||
@@ -1226,6 +1287,32 @@ mod tests {
|
||||
assert_ne!(english_source(ProofLang::En), Some(EnglishSource::Hunspell(dir.join("en_US.dic"))));
|
||||
}
|
||||
|
||||
/// A document tagging thousands of languages keeps the remembered pack hyphenation lookups
|
||||
/// bounded, and stays quick.
|
||||
#[test]
|
||||
fn many_pack_languages_keep_the_hyphenation_cache_bounded() {
|
||||
let _guard = DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-many-tags-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&root).unwrap();
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
let start = std::time::Instant::now();
|
||||
for i in 0..3000 {
|
||||
let lang = ProofLang::from_tag(Some(&format!(
|
||||
"q{}{}-{}",
|
||||
(b'a' + (i / 1000) as u8) as char,
|
||||
(b'a' + (i / 100 % 10) as u8) as char,
|
||||
100 + i % 100
|
||||
)));
|
||||
assert!(lang.pack_locale().is_some(), "{lang:?}");
|
||||
let _ = hyphen::hyphenate_word_in("hyphenation", &hyphen::Limits::default(), lang);
|
||||
assert!(packs::hyphenation_lookups() <= 256);
|
||||
}
|
||||
assert!(start.elapsed() < std::time::Duration::from_secs(5), "{:?}", start.elapsed());
|
||||
packs::set_packs_dir(None);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// CP4.2: with no embedded word list, English looks in the dictionaries folder, then an
|
||||
/// installed English Language pack, then the system's Hunspell, then the built-in list.
|
||||
#[test]
|
||||
@@ -1476,6 +1563,18 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// German, Bulgarian and pack languages mark spelling in their own language where we have
|
||||
/// the words for it, like the built-in ones; English otherwise.
|
||||
#[test]
|
||||
fn spelling_messages_are_in_the_language() {
|
||||
assert_eq!(spelling_message(ProofLang::De), "Möglicher Rechtschreibfehler");
|
||||
assert_eq!(spelling_message(ProofLang::Bg), "Възможна правописна грешка");
|
||||
assert_eq!(spelling_message(ProofLang::from_tag(Some("fr-CA"))), "Faute d’orthographe possible");
|
||||
assert_eq!(spelling_message(ProofLang::Pl), "Możliwy błąd pisowni");
|
||||
assert_eq!(spelling_message(ProofLang::from_tag(Some("xh"))), "Possible spelling mistake");
|
||||
assert_eq!(spelling_message(ProofLang::En), "Possible spelling mistake");
|
||||
}
|
||||
|
||||
/// CP4.2: a language with no tools of its own (Dutch) is spell-checked, given suggestions and
|
||||
/// hyphenated from its installed Language pack; removed, it is unchecked again; English is
|
||||
/// unaffected. No grammar rules for it.
|
||||
@@ -1511,6 +1610,10 @@ mod tests {
|
||||
let issues = check_text(text, &[(0..text.len(), nl)]);
|
||||
assert_eq!(issues.len(), 1, "{issues:?}");
|
||||
assert_eq!((issues[0].kind, &text[issues[0].start..issues[0].end], issues[0].lang), (IssueKind::Spelling, "fietz", nl));
|
||||
assert_eq!(issues[0].message, "Mogelijke spelfout", "the language's own message");
|
||||
assert_eq!((missing_dictionary(Some("nl-NL")), unreadable_dictionary(Some("nl-NL"))), (None, None));
|
||||
let missing_fr = missing_dictionary(Some("fr-FR"));
|
||||
assert!(missing_fr.is_none_or(|n| n == "fr_FR"), "{missing_fr:?}");
|
||||
assert!(suggest_in("fietz", 5, nl).contains(&"fiets".to_string()), "{:?}", suggest_in("fietz", 5, nl));
|
||||
assert!(check_grammar_in("de de huis", nl).is_empty(), "no grammar rules");
|
||||
assert_eq!(hyphen::hyphenate_word_in("xxnaxx", &hyphen::Limits::default(), nl), "xx-naxx");
|
||||
@@ -1521,6 +1624,10 @@ mod tests {
|
||||
packs::reload();
|
||||
assert!(is_correct_in("fietz", nl) && !nl.is_proofed());
|
||||
assert!(dictionary_load_error(nl).unwrap_or_default().contains("unsupported encoding"));
|
||||
let (name, file, reason) = unreadable_dictionary(Some("nl-NL")).unwrap();
|
||||
assert_eq!((name, file), ("nl_NL", pack.join("spelling.dic")));
|
||||
assert!(reason.contains("unsupported encoding"), "{reason}");
|
||||
assert_eq!(missing_dictionary(Some("nl-NL")), Some("nl_NL"), "as for the other languages");
|
||||
// Removed: unchecked again.
|
||||
std::fs::remove_dir_all(&pack).unwrap();
|
||||
packs::reload();
|
||||
@@ -1608,11 +1715,23 @@ mod tests {
|
||||
let (ru, el) = (ProofLang::Ru, ProofLang::from_tag(Some("el")));
|
||||
let before = [hyphen::hyphenate_word_in("hyphenation", &lim, ru), hyphen::hyphenate_word_in("hyphenation", &lim, el)];
|
||||
write_pack(&root, "ru", &[("hyphenation.pat", "1на 1ка\n".as_bytes())]);
|
||||
write_pack(&root, "el", &[("hyphenation.pat", "1να 1κα\n".as_bytes())]);
|
||||
write_pack(&root, "el", &[("hyphenation.pat", "1να 1κα 1κά\n".as_bytes())]);
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
assert_eq!([hyphen::hyphenate_word_in("hyphenation", &lim, ru), hyphen::hyphenate_word_in("hyphenation", &lim, el)], before);
|
||||
assert_eq!(hyphen::hyphenate_word_in("канака", &lim, ru), "ка-на-ка", "a Russian word uses the pack");
|
||||
assert_eq!(hyphen::hyphenate_word_in("κανακα", &lim, el), "κα-να-κα");
|
||||
// Decomposed text is composed for the patterns, as spelling does, and stays as it was
|
||||
// (patterns no fallback would give: `καν-ακά`).
|
||||
write_pack(&root, "grc", &[("hyphenation.pat", "ν1α ν1ά 2κ ά2\n".as_bytes())]);
|
||||
packs::reload();
|
||||
let grc = ProofLang::from_tag(Some("grc"));
|
||||
assert_eq!(hyphen::hyphenate_word_in("κανακά", &lim, grc), "καν-ακά");
|
||||
assert_eq!(hyphen::hyphenate_word_in("κανακα\u{301}", &lim, grc), "καν-ακα\u{301}");
|
||||
assert!(hyphen::hyphenate_word_in("κανάκανα", &lim, grc).starts_with("καν-ά"), "{}", hyphen::hyphenate_word_in("κανάκανα", &lim, grc));
|
||||
assert_eq!(
|
||||
hyphen::hyphenate_word_in("κανα\u{301}κανα", &lim, grc),
|
||||
hyphen::hyphenate_word_in("κανάκανα", &lim, grc).replace('ά', "α\u{301}")
|
||||
);
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
@@ -1631,8 +1750,11 @@ mod tests {
|
||||
std::fs::write(dir.join("LICENCES").join("BIG"), &big).unwrap();
|
||||
}
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
let before = packs::LICENCE_BYTES_READ.load(std::sync::atomic::Ordering::Relaxed);
|
||||
let from_packs: usize = packs::licence_texts().iter().filter(|(t, _)| t.contains(": BIG")).map(|(_, x)| x.len()).sum();
|
||||
assert!(from_packs > 0 && from_packs <= 16 << 20, "{from_packs}");
|
||||
let read = packs::LICENCE_BYTES_READ.load(std::sync::atomic::Ordering::Relaxed) - before;
|
||||
assert!(read <= 16 << 20, "{read} bytes read: a file over what is left is never read");
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
@@ -1803,6 +1925,8 @@ mod tests {
|
||||
/// with the rule; a plain paragraph's does, and later sentences in the item are still checked.
|
||||
#[test]
|
||||
fn list_item_first_word_needs_no_capital() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let grammar = |t: &str, lang: ProofLang, list: bool| -> Vec<String> {
|
||||
check_text_with(t, &[(0..t.len(), lang)], &Options::default(), true, list)
|
||||
.into_iter()
|
||||
@@ -1838,6 +1962,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn spelling() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for w in [
|
||||
"if",
|
||||
"we",
|
||||
@@ -1869,6 +1995,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn ically_adverbs_are_known() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for w in ["specifically", "scientifically", "automatically", "dramatically", "Specifically"] {
|
||||
assert!(is_correct(w), "{w}");
|
||||
}
|
||||
@@ -1882,6 +2010,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn non_ic_words_do_not_take_ally() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for (base, wrong) in [("desktop", "desktopally"), ("house", "houseally"), ("final", "finalally"), ("green", "greenally")] {
|
||||
assert!(is_correct(base), "{base}");
|
||||
assert!(!is_correct(wrong), "{wrong}");
|
||||
@@ -1892,6 +2022,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn common_modern_words() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for w in [
|
||||
"desktop",
|
||||
"Desktops",
|
||||
@@ -1929,6 +2061,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn british_spellings_are_known() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
for w in [
|
||||
"judgement",
|
||||
"defence",
|
||||
@@ -1976,6 +2110,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn english_dialects_flag_each_others_spellings() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa);
|
||||
// (word, right in en-US, right in en-CA, right in en-GB)
|
||||
let cases = [
|
||||
@@ -2069,6 +2205,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn british_words_reported_in_issue_40_are_right_in_en_gb() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let (us, gb, ca) = (ProofLang::En, ProofLang::EnGb, ProofLang::EnCa);
|
||||
// (word, right in en-US, right in en-CA, right in en-GB)
|
||||
let cases = [
|
||||
@@ -2115,6 +2253,8 @@ mod tests {
|
||||
/// with their inflections.
|
||||
#[test]
|
||||
fn more_british_spellings_are_right_in_en_gb() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let gb = ProofLang::EnGb;
|
||||
assert_eq!(ProofLang::from_tag(Some("en-AU")), gb);
|
||||
for w in [
|
||||
@@ -2292,6 +2432,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn user_dictionary_accepts_the_other_dialects_spelling() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
add_word("savour");
|
||||
assert!(is_correct_in("savour", ProofLang::En), "a word the user added is never flagged");
|
||||
assert!(!is_correct_in("savor", ProofLang::EnGb));
|
||||
@@ -2322,6 +2464,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn address_token_covers_every_word_split_out_of_it() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// #1335: one whitespace-delimited token, however punctuation splits it, has one
|
||||
// web/e-mail classification, shared by all its words.
|
||||
let t = "zxq@mail.example.com,blorp;qwirpy wugwug";
|
||||
@@ -2337,6 +2481,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn address_after_a_wide_space_is_an_internet_token() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// A web or e-mail address after a no-break space, a narrow no-break space, a figure space
|
||||
// or an ideographic space is skipped, as it is after a plain space.
|
||||
let t = "see\u{a0}https://xyzq.example/blorp and\u{3000}www.qwirpy.example or\u{a0}zxq@mail.example \
|
||||
@@ -2350,6 +2496,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn spelling_scales_over_one_long_token() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
// #1335: thousands of punctuation-separated words inside a single
|
||||
// whitespace-free token: one classification for the token, one issue per
|
||||
// misspelled word, through every public spelling entry point. Without the Language data
|
||||
@@ -2416,6 +2564,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn user_dictionary_works() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
assert!(!is_correct("zxqwordcrafty"));
|
||||
add_word("zxqwordcrafty");
|
||||
assert!(is_correct("zxqwordcrafty"));
|
||||
@@ -2449,6 +2599,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn hostile_dictionary_files_are_capped() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "hostile"));
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let path = dir.join("custom-dictionary.txt");
|
||||
@@ -2472,6 +2624,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn saving_merges_with_the_file() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let dir = std::env::temp_dir().join(format!("wc-dict-{}-{}", std::process::id(), "merge"));
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
let path = dir.join("custom-dictionary.txt");
|
||||
@@ -2490,6 +2644,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn ignored_words_are_not_dictionary_words() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
assert!(!is_correct("zxqignoredword"));
|
||||
ignore_word("ZxqIgnoredWord");
|
||||
assert!(is_correct("zxqignoredword"));
|
||||
@@ -2534,6 +2690,8 @@ mod tests {
|
||||
/// byte range of what was typed, and it is checked as its composed form in every language.
|
||||
#[test]
|
||||
fn decomposed_words_stay_whole() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let text = "tambie\u{301}n, cafe\u{301} \u{301}x";
|
||||
let ws: Vec<&str> = words(text).into_iter().map(|(a, b)| &text[a..b]).collect();
|
||||
assert_eq!(ws, ["tambie\u{301}n", "cafe\u{301}", "\u{301}x"]);
|
||||
@@ -2625,6 +2783,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn each_language_uses_its_own_tools() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
pl::test_dict::install();
|
||||
let pl = "Zażółć gęślą jaźń, chrząszcz brzmi w trzcinie.";
|
||||
assert!(check_spelling_in(pl, ProofLang::Pl).is_empty(), "{:?}", check_spelling_in(pl, ProofLang::Pl));
|
||||
@@ -2646,6 +2806,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn italian_uses_its_own_tools() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
it::test_dict::install();
|
||||
let it = "La città vecchia ha un bel caffè in piazza.";
|
||||
assert!(check_spelling_in(it, ProofLang::It).is_empty(), "{:?}", check_spelling_in(it, ProofLang::It));
|
||||
@@ -2662,6 +2824,8 @@ mod tests {
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn mixed_language_paragraphs_check_each_run_in_its_language() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
pl::test_dict::install();
|
||||
// A Polish paragraph quoting English: English spelling and rules inside the quote only.
|
||||
let t = "Napisał „the quick brown fox is is here” i poszedł spać, bo żeka wylała.";
|
||||
|
||||
@@ -118,6 +118,8 @@ fn hyphenation_is_off_where_it_needs_data() {
|
||||
/// Every public entry point, in every language, on every sample: no panic.
|
||||
#[test]
|
||||
fn every_public_function_runs_without_data() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let lim = Limits { min_word: 0, after_first: 0, before_last: 0, capitalized: true };
|
||||
let opts = Options { ignore_uppercase: false, ignore_numbers: false, ignore_internet: false };
|
||||
for text in SAMPLES {
|
||||
|
||||
+77
-24
@@ -21,6 +21,7 @@
|
||||
//! link); a pack whose `pack.toml` is missing, unreadable or names a path outside the pack is
|
||||
//! skipped and reported by [`load_errors`]. Nothing here panics on a hostile pack.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard};
|
||||
|
||||
@@ -31,6 +32,8 @@ use crate::patterns::Patterns;
|
||||
pub(crate) const SPELLING: &str = "spelling";
|
||||
/// A pack's hyphenation patterns.
|
||||
const HYPHENATION: &str = "hyphenation.pat";
|
||||
/// Most languages whose pack patterns are remembered (a document may tag thousands).
|
||||
const MAX_HYPHENATION_LOOKUPS: usize = 256;
|
||||
/// A pack's description.
|
||||
const PACK_TOML: &str = "pack.toml";
|
||||
/// A pack's licence texts.
|
||||
@@ -76,15 +79,16 @@ struct Holds {
|
||||
struct State {
|
||||
dir: Option<PathBuf>,
|
||||
scan: Option<Arc<Scan>>,
|
||||
/// Each language's pack patterns once looked up (`None`: no pack has them).
|
||||
hyphenation: Vec<(ProofLang, Option<Arc<Patterns>>)>,
|
||||
/// Each language's pack patterns once looked up (`None`: no pack has them), at most
|
||||
/// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full).
|
||||
hyphenation: BTreeMap<ProofLang, Option<Arc<Patterns>>>,
|
||||
/// (locale, reason) of pack hyphenation files that failed to load.
|
||||
hyphenation_errors: Vec<(String, String)>,
|
||||
/// Counts changes, so a scan or load that started before one is not kept.
|
||||
changes: u64,
|
||||
}
|
||||
|
||||
static STATE: RwLock<State> = RwLock::new(State { dir: None, scan: None, hyphenation: Vec::new(), hyphenation_errors: Vec::new(), changes: 0 });
|
||||
static STATE: RwLock<State> = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 });
|
||||
|
||||
fn read() -> RwLockReadGuard<'static, State> {
|
||||
STATE.read().unwrap_or_else(PoisonError::into_inner)
|
||||
@@ -148,22 +152,26 @@ pub fn licence_texts() -> Vec<(String, String)> {
|
||||
if !crate::data::EN_US_DIC.is_empty() {
|
||||
out.push(("English word list and hyphenation (Moby Hyphenator II)".into(), MOBY_NOTICE.into()));
|
||||
}
|
||||
out.push(("English core word list".into(), CORE_LIST_NOTICE.into()));
|
||||
out.push(("English word list built into craft-proof".into(), CORE_LIST_NOTICE.into()));
|
||||
out.extend(crate::data::EMBEDDED_LICENCES.iter().map(|(t, x)| ((*t).to_string(), (*x).to_string())));
|
||||
let mut total: u64 = 0;
|
||||
'packs: for (p, _) in &scan().packs {
|
||||
for (file, text) in licence_files(&p.path) {
|
||||
total = total.saturating_add(text.len() as u64);
|
||||
if total > MAX_LICENCE_TOTAL {
|
||||
log::warn!("Language pack licences: over {MAX_LICENCE_TOTAL} bytes in all; the rest aren't shown");
|
||||
break 'packs;
|
||||
}
|
||||
// Each file is checked against what is left before it is read, so no more is ever read.
|
||||
let mut left: u64 = MAX_LICENCE_TOTAL;
|
||||
for (p, _) in &scan().packs {
|
||||
for (file, text) in licence_files(&p.path, &mut left) {
|
||||
out.push((format!("{} ({}): {file}", p.name, p.locale), text));
|
||||
}
|
||||
if left == 0 {
|
||||
log::warn!("Language pack licences: {MAX_LICENCE_TOTAL} bytes in all; the rest aren't shown");
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Licence bytes read so far (tests check the cap holds while reading, not only after).
|
||||
#[cfg(test)]
|
||||
pub(crate) static LICENCE_BYTES_READ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
|
||||
|
||||
/// Most bytes of pack licence text [`licence_texts`] reads in all.
|
||||
const MAX_LICENCE_TOTAL: u64 = 16 << 20;
|
||||
|
||||
@@ -171,9 +179,9 @@ const MAX_LICENCE_TOTAL: u64 = 16 << 20;
|
||||
const MOBY_NOTICE: &str = "The Moby Hyphenator II word list by Grady Ward (Project Gutenberg eBook #3204) is in the public domain. \
|
||||
The English hyphenation patterns were trained from it for the Craft apps and are distributed with them under MIT OR Apache-2.0.";
|
||||
|
||||
/// The notice of the core word list built into the crate (`data/en-core.txt`).
|
||||
const CORE_LIST_NOTICE: &str = "The core English word list built into craft-proof is taken from the Moby Hyphenator II word list by Grady Ward \
|
||||
(public domain), ranked by public-domain Project Gutenberg books. It is in the public domain.";
|
||||
/// The notice of the word list built into the crate (`data/en-words.dic`).
|
||||
const CORE_LIST_NOTICE: &str = "The English word list built into craft-proof is the Moby Hyphenator II word list by Grady Ward \
|
||||
(Project Gutenberg eBook #3204), which is in the public domain.";
|
||||
|
||||
/// The locales whose Language data is embedded in this build (`en-us`…, and in a Language build
|
||||
/// the ones it names; `CRAFT_PROOF_EMBED`).
|
||||
@@ -354,7 +362,7 @@ fn read_capped(p: &Path, cap: u64) -> Result<Vec<u8>, String> {
|
||||
}
|
||||
|
||||
/// The licence texts in a pack's `LICENCES/` folder, by file name.
|
||||
fn licence_files(pack: &Path) -> Vec<(String, String)> {
|
||||
fn licence_files(pack: &Path, left: &mut u64) -> Vec<(String, String)> {
|
||||
let dir = pack.join(LICENCES);
|
||||
if !std::fs::symlink_metadata(&dir).is_ok_and(|m| m.is_dir()) {
|
||||
return Vec::new();
|
||||
@@ -370,7 +378,17 @@ fn licence_files(pack: &Path) -> Vec<(String, String)> {
|
||||
names
|
||||
.into_iter()
|
||||
.filter_map(|n| {
|
||||
let bytes = read_capped(&dir.join(&n), MAX_LICENCE).ok()?;
|
||||
let path = dir.join(&n);
|
||||
let size = std::fs::symlink_metadata(&path).ok().filter(std::fs::Metadata::is_file)?.len();
|
||||
if size > (*left).min(MAX_LICENCE) {
|
||||
// Over what is left: none of the rest is read either.
|
||||
*left = 0;
|
||||
return None;
|
||||
}
|
||||
let bytes = read_capped(&path, size.min(MAX_LICENCE)).ok()?;
|
||||
*left = left.saturating_sub(bytes.len() as u64);
|
||||
#[cfg(test)]
|
||||
LICENCE_BYTES_READ.fetch_add(bytes.len() as u64, std::sync::atomic::Ordering::Relaxed);
|
||||
Some((n, String::from_utf8_lossy(&bytes).into_owned()))
|
||||
})
|
||||
.collect()
|
||||
@@ -426,12 +444,33 @@ fn packs_for(locales: &[String], has: impl Fn(&Holds) -> bool) -> Vec<PathBuf> {
|
||||
}
|
||||
|
||||
/// The folders of the installed packs with a spelling dictionary for the Hunspell dictionary
|
||||
/// `names` (`pl_PL`, `pl`), best first ([`crate::hunspell::find`] reads them).
|
||||
pub(crate) fn spelling_dirs(names: &[&str]) -> Vec<PathBuf> {
|
||||
/// `names` (`pt_BR`, `pt-BR`, `pt`), best first ([`crate::hunspell::find`] reads them), in two
|
||||
/// tiers: the packs of the first name's own region (`pt-br`) or of the language alone (`pt`), then
|
||||
/// every other one (the other names' regions, then any other region of the language: `pt-pt`).
|
||||
/// The system's exact-region dictionaries are searched between the two, so another region's pack
|
||||
/// never beats them.
|
||||
pub(crate) fn spelling_dirs(names: &[&str]) -> (Vec<PathBuf>, Vec<PathBuf>) {
|
||||
if read().dir.is_none() {
|
||||
return Vec::new();
|
||||
return (Vec::new(), Vec::new());
|
||||
}
|
||||
packs_for(&locales_for(names), |h| h.spelling)
|
||||
let all = packs_for(&locales_for(names), |h| h.spelling);
|
||||
let region = region_locales(names);
|
||||
let scan = scan();
|
||||
let in_region = |path: &PathBuf| scan.packs.iter().any(|(p, _)| &p.path == path && region.contains(&p.locale));
|
||||
all.into_iter().partition(in_region)
|
||||
}
|
||||
|
||||
/// The pack locales of `names`' own region: the first name's (`pt_BR` → `pt-br`) and the language
|
||||
/// alone (`pt`) when it is written in the same script.
|
||||
pub(crate) fn region_locales(names: &[&str]) -> Vec<String> {
|
||||
let Some(first) = names.first() else { return Vec::new() };
|
||||
let mut out = vec![first.to_ascii_lowercase().replace('_', "-")];
|
||||
let primary = first.split(['_', '-']).next().unwrap_or_default().to_ascii_lowercase();
|
||||
if same_script(first, &primary) && !out.contains(&primary) {
|
||||
out.push(primary);
|
||||
}
|
||||
out.retain(|l| is_locale(l));
|
||||
out
|
||||
}
|
||||
|
||||
/// The pack locales whose patterns hyphenate `lang`, best first; none for English (embedded)
|
||||
@@ -465,7 +504,7 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
|
||||
let changes = {
|
||||
let s = read();
|
||||
s.dir.as_ref()?;
|
||||
if let Some((_, p)) = s.hyphenation.iter().find(|(l, _)| *l == lang) {
|
||||
if let Some(p) = s.hyphenation.get(&lang) {
|
||||
return p.clone();
|
||||
}
|
||||
s.changes
|
||||
@@ -486,8 +525,12 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
|
||||
}
|
||||
}
|
||||
let mut s = write();
|
||||
if s.changes == changes && !s.hyphenation.iter().any(|(l, _)| *l == lang) {
|
||||
s.hyphenation.push((lang, found.clone()));
|
||||
if s.changes == changes && !s.hyphenation.contains_key(&lang) {
|
||||
// ponytail: cleared when full, an LRU if many languages each need their patterns.
|
||||
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS {
|
||||
s.hyphenation.clear();
|
||||
}
|
||||
s.hyphenation.insert(lang, found.clone());
|
||||
for e in errors {
|
||||
if !s.hyphenation_errors.contains(&e) {
|
||||
s.hyphenation_errors.push(e);
|
||||
@@ -497,6 +540,12 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
|
||||
found
|
||||
}
|
||||
|
||||
/// How many languages' pack patterns are remembered now.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn hyphenation_lookups() -> usize {
|
||||
read().hyphenation.len()
|
||||
}
|
||||
|
||||
/// A pack's `hyphenation.pat` (UTF-8 TeX or plain Liang patterns, at most the dictionary cap).
|
||||
fn load_patterns(path: &Path) -> Result<Patterns, String> {
|
||||
let bytes = read_capped(path, crate::hunspell::MAX_FILE)?;
|
||||
@@ -534,6 +583,10 @@ mod tests {
|
||||
#[test]
|
||||
fn dictionary_names_map_to_pack_locales() {
|
||||
assert_eq!(locales_for(&["pt_BR", "pt-BR", "pt"]), ["pt-br", "pt"]);
|
||||
assert_eq!(region_locales(&["pt_BR", "pt-BR", "pt"]), ["pt-br", "pt"]);
|
||||
assert_eq!(region_locales(&["en_US", "en-US", "en", "en_GB"]), ["en-us", "en"]);
|
||||
assert_eq!(region_locales(&["sr_Latn", "sr-Latn"]), ["sr-latn"]);
|
||||
assert!(region_locales(&[]).is_empty());
|
||||
assert_eq!(locales_for(&["es_419", "es"]), ["es-419", "es"]);
|
||||
assert_eq!(locales_for(&["../x", "a b"]), Vec::<String>::new());
|
||||
}
|
||||
|
||||
@@ -156,7 +156,10 @@ fn forms(base: &str, class: &str) -> Vec<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[cfg_attr(not(craft_proof_embed = "ru"), ignore = "needs the ru Language data (CRAFT_I18N_DIR, CRAFT_PROOF_EMBED=ru)")]
|
||||
#[cfg_attr(
|
||||
not(craft_proof_embed = "ru"),
|
||||
ignore = "needs the ru Language data (CRAFT_I18N_DIR; ru-extra.txt is embedded in every build that has it)"
|
||||
)]
|
||||
#[test]
|
||||
fn supplement_words_decline() {
|
||||
for w in [
|
||||
|
||||
@@ -363,7 +363,10 @@ mod tests {
|
||||
assert!(suggest::suggest_in(&none, "превет", 5).is_empty());
|
||||
}
|
||||
|
||||
#[cfg_attr(not(craft_proof_embed = "ru"), ignore = "needs the ru Language data (CRAFT_I18N_DIR, CRAFT_PROOF_EMBED=ru)")]
|
||||
#[cfg_attr(
|
||||
not(craft_proof_embed = "ru"),
|
||||
ignore = "needs the ru Language data (CRAFT_I18N_DIR; ru-extra.txt is embedded in every build that has it)"
|
||||
)]
|
||||
#[test]
|
||||
fn russian_words_are_correct() {
|
||||
test_dict::install();
|
||||
|
||||
@@ -76,6 +76,8 @@ mod checker_tests {
|
||||
|
||||
#[test]
|
||||
fn vietnamese_paragraphs_under_english_get_no_marks() {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
let _guard = crate::DICTIONARY_TESTS.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let text = "Hôm nay tôi đi học, trời rất đẹp và mát mẻ.";
|
||||
let spans = [(0..text.len(), ProofLang::En)];
|
||||
assert!(check_text_with(text, &spans, &Options::default(), true, false).is_empty());
|
||||
|
||||
Reference in new issue
Block a user