mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
Merge pull request #496 from benandez/fu-proof-p3
craft-proof: review follow-ups (P3s from the proofing move)
This commit is contained in:
29 files changed
+858
-210
No files matched your search
+11
-9
@@ -13,7 +13,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = "<commit>
|
||||
|
||||
| Language | Spelling | Grammar | Hyphenation |
|
||||
|---|---|---|---|
|
||||
| English (en-US, en-CA, en-GB and the British-spelling Englishes such as en-AU, en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
|
||||
| English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic |
|
||||
| Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) |
|
||||
| Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise |
|
||||
| German | — | — | TeX patterns (embedded, or a Language pack) |
|
||||
@@ -79,7 +79,7 @@ pub fn packs_dir() -> Option<PathBuf>; // +
|
||||
pub fn installed() -> Vec<InstalledPack>; // InstalledPack { locale, name, licence, path }, by locale
|
||||
pub fn licence_texts() -> Vec<(String, String)>; // (title, text): embedded English notices, then each pack's LICENCES/*
|
||||
pub fn load_errors() -> Vec<(String, String)>; // + (locale or folder, reason): packs skipped, pattern files that failed
|
||||
pub fn embedded_locales() -> &'static [&'static str]; // + what this build embeds ("en-us", "de", "pl", "pt", "it", "ru" by default; "en-us" with CRAFT_PROOF_EMBED=en)
|
||||
pub fn embedded_locales() -> &'static [&'static str]; // + locales with embedded upstream data, not our supplements ("en-us", "en-gb", "en-ca", "en-au", "pl", "pt", "it", "de" by default; the four English ones with CRAFT_PROOF_EMBED=en)
|
||||
|
||||
// craft_proof (crate root)
|
||||
pub enum ProofLang { …, Pack(PackLocale), Other } // + Pack: any language without tools of its own (nl, fr, sr-Latn…)
|
||||
@@ -151,14 +151,16 @@ English spelling uses the first of these that is there; `english_source(lang)` s
|
||||
or in WordCraft's own `dictionaries` folder. The user's choice comes first.
|
||||
2. **Installed English pack:** reserved for Language packs (craft-libs#485, CP4.2).
|
||||
3. **Embedded SCOWL:** the English's own SCOWL `rel-2026.02.25` Hunspell dictionary from
|
||||
`CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`) or `en-ca`, `spelling.aff` +
|
||||
`spelling.dic`), read by the affix engine. en-AU, en-NZ, en-IE and the other British-spelling
|
||||
Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own
|
||||
`CRAFT_I18N_DIR` (`language-data/en-us`, `en-gb` (British `-ise`), `en-au` or `en-ca`,
|
||||
`spelling.aff` + `spelling.dic`), read by the affix engine. Australian English reads en-AU
|
||||
(Australian words such as `billabong`, with en-GB's spelling policy); en-NZ, en-IE and the
|
||||
other British-spelling Englishes read en-GB; Canadian English reads en-CA. A word is right when the English's own
|
||||
dictionary knows it (or `en-extra.txt` does), never because another English's does
|
||||
(`colourblind` is wrong in en-US, `colorblind` in en-GB); suggestions come from the same
|
||||
dictionary. SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so
|
||||
(`colourblind` is wrong in en-US, `colorblind` in en-GB); the one exception is Australian
|
||||
English, which also takes en-GB's words (`enquiry`: en-AU's list keeps `inquiry` only).
|
||||
Suggestions come from the English's own dictionary (en-AU's for Australian). SCOWL's size-60 lists keep one spelling per dialect, and Word accepts more, so
|
||||
British English also takes the `-ize` spellings of its `-ise` words that American or Canadian
|
||||
English spell so (`organize`, `realization`; never `wize` or `advize`), and `en-variants.txt` flags the other dialect's spellings and accepts this
|
||||
English spell so (`organize`, `realization`; never `wize` or `advize`, and never `-yze`: `reanalyze` stays American), and `en-variants.txt` flags the other dialect's spellings and accepts this
|
||||
dialect's other spellings of known words (`judgement` in en-US). English with no dialect
|
||||
(`is_correct`) takes a word any of the three knows. Suggestions offer a missing space only
|
||||
between two lower-case words of three letters or more, after every one-word candidate, and no
|
||||
@@ -199,7 +201,7 @@ craft-i18n's copy.
|
||||
The SCOWL notice (SCOWL's licence, the Ispell BSD licence of the affix files and Benjamin Titze's
|
||||
notice) is embedded with the dictionaries: `craft_proof::scowl::notice()`, for an About window.
|
||||
|
||||
Embedding SCOWL adds 1,542,396 bytes of Language data (the en-US, en-GB and en-CA `.aff`/`.dic`
|
||||
Embedding SCOWL adds 2,056,533 bytes of Language data (the en-US, en-GB, en-CA and en-AU `.aff`/`.dic`
|
||||
and the notice); the Moby list (432,905 bytes) stays for hyphenation.
|
||||
|
||||
## Check
|
||||
|
||||
@@ -16,9 +16,8 @@
|
||||
//! `include_str!` takes a string, so a `CRAFT_I18N_DIR` that isn't valid UTF-8 can't be embedded:
|
||||
//! it is treated like a folder without the data.
|
||||
//!
|
||||
//! English: SCOWL's en-US, en-GB (-ise) and en-CA dictionaries; en-AU is not embedded, as
|
||||
//! Australian English is checked as British-spelling English (`ProofLang::EnGb`). The SCOWL
|
||||
//! notice is embedded once (the four upstream READMEs differ only in their title and build line).
|
||||
//! English: SCOWL's en-US, en-GB (-ise), en-CA and en-AU dictionaries. The SCOWL notice is
|
||||
//! embedded once (the four upstream READMEs differ only in their title and build line).
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -34,6 +33,8 @@ const FILES: &[(&str, &str, bool)] = &[
|
||||
("EN_GB_SPELLING", "en-gb/spelling.dic", false),
|
||||
("EN_CA_AFF", "en-ca/spelling.aff", false),
|
||||
("EN_CA_SPELLING", "en-ca/spelling.dic", false),
|
||||
("EN_AU_AFF", "en-au/spelling.aff", false),
|
||||
("EN_AU_SPELLING", "en-au/spelling.dic", false),
|
||||
("SCOWL_NOTICE", "en-us/LICENCE-SCOWL.txt", false),
|
||||
("EN_US_PAT", "en-us/hyphenation.pat", false),
|
||||
("EXTRA_EN", "en-us/en-extra.txt", false),
|
||||
|
||||
+86
-17
@@ -74,6 +74,14 @@ const MAX_ALIAS_BYTES: usize = 4 << 20;
|
||||
/// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup
|
||||
/// tests each distinct continuation set it meets at most once, so this bounds its cost.
|
||||
const MAX_SET_DATA: usize = 1 << 23;
|
||||
/// Most flags all stems may carry together before their sets are merged and interned (the same
|
||||
/// set on many stems counts each time; an `AF` alias counts once). Checked while reading, so a
|
||||
/// huge `.dic` is refused before its flags fill memory, not after (#495: a 760 MB `.dic` once
|
||||
/// held 3 GB before [`MAX_SET_DATA`] refused it).
|
||||
const MAX_STEM_FLAGS: usize = 4 * MAX_SET_DATA;
|
||||
/// Most bytes of flag text per flag of [`MAX_STEM_FLAGS`] the stem lines may hold in all while
|
||||
/// they are read (`FLAG num`: `65535,` is six).
|
||||
const FLAG_TEXT_PER_FLAG: usize = 6;
|
||||
|
||||
/// Most distinct flags a dictionary may use: each gets a `u16` id.
|
||||
const MAX_DISTINCT_FLAGS: usize = 1 << 16;
|
||||
@@ -972,12 +980,19 @@ impl AffixDict {
|
||||
}
|
||||
|
||||
fn from_hunspell_in(aff: &str, dic: &str, comments: &[&str], english: bool) -> Result<AffixDict, String> {
|
||||
AffixDict::from_hunspell_capped(aff, dic, comments, english, MAX_STEM_FLAGS)
|
||||
}
|
||||
|
||||
/// [`AffixDict::from_hunspell_in`] with at most `max_flags` stem flags in all
|
||||
/// ([`MAX_STEM_FLAGS`]; tests lower it).
|
||||
fn from_hunspell_capped(aff: &str, dic: &str, comments: &[&str], english: bool, max_flags: usize) -> Result<AffixDict, String> {
|
||||
// Some dictionaries open with a UTF-8 BOM; it must not glue itself to the first
|
||||
// directive or turn the `.dic` count line into a stem.
|
||||
let aff = aff.strip_prefix('\u{feff}').unwrap_or(aff);
|
||||
let dic = dic.strip_prefix('\u{feff}').unwrap_or(dic);
|
||||
let rules = parse_aff(aff, english)?;
|
||||
let mut stems: Vec<(String, String)> = Vec::new();
|
||||
let mut flag_text = FlagText::new(max_flags);
|
||||
for (n, line) in dic.lines().enumerate() {
|
||||
// The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.
|
||||
// Leading blanks belong to it, as in Hunspell (` lab` is no `lab`).
|
||||
@@ -1013,14 +1028,16 @@ impl AffixDict {
|
||||
}
|
||||
// With an `AF` table the field stays an alias number here: build() reads each alias
|
||||
// once, and every stem using it shares those flags.
|
||||
flag_text.add(&flags)?;
|
||||
stems.push((word, flags));
|
||||
}
|
||||
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY)
|
||||
AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect(), MAX_FLAGS_PER_ENTRY, max_flags)
|
||||
}
|
||||
|
||||
/// Build from parsed rules and stems whose flag fields carry at most `entry_cap` flags each
|
||||
/// ([`MAX_FLAGS_PER_ENTRY`] for `.dic` entries, [`MAX_FLAGS_PER_STEM`] for compact data).
|
||||
fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec<String>, entry_cap: usize) -> Result<AffixDict, String> {
|
||||
/// The stems' flags may number `max_flags` in all ([`MAX_STEM_FLAGS`]), counted as they are read.
|
||||
fn build(rules: Rules, stems: Vec<(String, String)>, comments: Vec<String>, entry_cap: usize, max_flags: usize) -> Result<AffixDict, String> {
|
||||
let mode = rules.mode;
|
||||
// A `.dic` entry's malformed flag items read as Hunspell reads them ([`FlagMode`]); an
|
||||
// entry with more than MAX_FLAGS_PER_STEM flags, or compact data whose flags don't read,
|
||||
@@ -1049,20 +1066,24 @@ impl AffixDict {
|
||||
}
|
||||
Some((start, stem_keys.len(), word))
|
||||
};
|
||||
let mut stems: Vec<(String, usize, usize, bool)> = stems
|
||||
.into_iter()
|
||||
.filter_map(|(w, f)| {
|
||||
let range = match rules.alias_index(&f) {
|
||||
Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)),
|
||||
None if rules.aliases_on => Some((0, 0, true)),
|
||||
None => read(&f, &mut stem_keys),
|
||||
};
|
||||
if range.is_none_or(|r| !r.2) {
|
||||
skipped += 1;
|
||||
}
|
||||
range.map(|(a, b, word)| (w, a, b, word))
|
||||
})
|
||||
.collect();
|
||||
let mut read_stems: Vec<(String, usize, usize, bool)> = Vec::with_capacity(stems.len());
|
||||
for (w, f) in stems {
|
||||
let range = match rules.alias_index(&f) {
|
||||
Some(i) => *alias_ranges.entry(i).or_insert_with(|| read(rules.aliases.get(i).map_or("", String::as_str), &mut stem_keys)),
|
||||
None if rules.aliases_on => Some((0, 0, true)),
|
||||
None => read(&f, &mut stem_keys),
|
||||
};
|
||||
if stem_keys.len() > max_flags {
|
||||
return Err(format!("the stems carry more than {max_flags} flags in all"));
|
||||
}
|
||||
if range.is_none_or(|r| !r.2) {
|
||||
skipped += 1;
|
||||
}
|
||||
if let Some((a, b, word)) = range {
|
||||
read_stems.push((w, a, b, word));
|
||||
}
|
||||
}
|
||||
let mut stems = read_stems;
|
||||
if skipped > 0 {
|
||||
log::debug!("affix dictionary: {skipped} entries with unreadable or too many flags skipped");
|
||||
}
|
||||
@@ -2059,6 +2080,7 @@ impl AffixDict {
|
||||
}
|
||||
let body = raw.get(words_at + 7..).unwrap_or_default();
|
||||
let mut stems: Vec<(String, String)> = Vec::new();
|
||||
let mut flag_text = FlagText::new(MAX_STEM_FLAGS);
|
||||
let mut cur: Vec<u8> = Vec::new();
|
||||
let mut i = 0;
|
||||
while let Some(&p) = body.get(i) {
|
||||
@@ -2081,9 +2103,32 @@ impl AffixDict {
|
||||
if word.is_empty() || word.len() > MAX_WORD_BYTES {
|
||||
return Err("affix dictionary: bad stem".into());
|
||||
}
|
||||
flag_text.add(flags).map_err(|e| format!("affix dictionary: {e}"))?;
|
||||
stems.push((word.to_string(), flags.to_string()));
|
||||
}
|
||||
AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM)
|
||||
AffixDict::build(rules, stems, comments, MAX_FLAGS_PER_STEM, MAX_STEM_FLAGS)
|
||||
}
|
||||
}
|
||||
|
||||
/// A running count of the stems' flag text while a dictionary is read: past
|
||||
/// [`FLAG_TEXT_PER_FLAG`] bytes for each of `max_flags` flags, reading stops with an error, before
|
||||
/// the rest of a huge `.dic` is held ([`MAX_STEM_FLAGS`]).
|
||||
struct FlagText {
|
||||
bytes: usize,
|
||||
cap: usize,
|
||||
}
|
||||
|
||||
impl FlagText {
|
||||
fn new(max_flags: usize) -> Self {
|
||||
FlagText { bytes: 0, cap: max_flags.saturating_mul(FLAG_TEXT_PER_FLAG) }
|
||||
}
|
||||
|
||||
fn add(&mut self, flags: &str) -> Result<(), String> {
|
||||
self.bytes = self.bytes.saturating_add(flags.len());
|
||||
if self.bytes > self.cap {
|
||||
return Err(format!("the stems' flags take more than {} bytes", self.cap));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2133,6 +2178,30 @@ pub(crate) mod tests {
|
||||
AffixDict::from_hunspell(AFF, DIC, &["test data"]).unwrap()
|
||||
}
|
||||
|
||||
/// #495 review: the stems' flags are counted as they are read, so a `.dic` whose flags pass
|
||||
/// the budget is refused while reading (its flag text) or while its flags are read (the
|
||||
/// keys), before they are all held; one within it loads.
|
||||
#[test]
|
||||
fn stem_flags_are_budgeted_while_reading() {
|
||||
let aff = "SET UTF-8\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 x .\n";
|
||||
let many: String = (0..100).map(|i| format!("w{i}/AB\n")).collect();
|
||||
let dic = format!("100\n{many}");
|
||||
// 200 flags: over a budget of 50 keys (whose flag text, 300 bytes, still fits).
|
||||
let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 50).unwrap_err();
|
||||
assert!(e.contains("more than 50 flags"), "{e}");
|
||||
let d = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 200).unwrap();
|
||||
assert!(d.check("w7s") && d.check("w99x"));
|
||||
// Flag text past the budget (entries over the per-entry cap hold no keys, but their text
|
||||
// still counts): refused while reading the `.dic`.
|
||||
let long = "A".repeat(MAX_FLAGS_PER_ENTRY + 1);
|
||||
let dic = format!("3\na/{long}\nb/{long}\nc/{long}\n");
|
||||
let e = AffixDict::from_hunspell_capped(aff, &dic, &[], false, 2000).unwrap_err();
|
||||
assert!(e.contains("more than 12000 bytes"), "{e}");
|
||||
assert!(AffixDict::from_hunspell(aff, &dic, &[]).is_ok(), "within the real budget: skipped entries, no error");
|
||||
let mut text = FlagText::new(1);
|
||||
assert!(text.add("ABCDEF").is_ok() && text.add("G").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn affix_headers_allow_comments_but_rule_conditions_stay_literal() {
|
||||
let aff = "SET UTF-8\nPFX X Y 1 # prefix comment\nPFX X 0 ne . # rule comment\nSFX y Y 1 # suffix comment\nSFX y 0 y . # rule comment\n";
|
||||
|
||||
@@ -62,11 +62,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Mezera před interpunkcí".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Dvojitá mezera".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Dvojitá mezera".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -63,7 +63,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
return true;
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
// The whole word first, as Hunspell does: a hyphenated entry or user word.
|
||||
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
known(d, w)
|
||||
}
|
||||
@@ -142,6 +143,14 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't.
|
||||
#[test]
|
||||
fn a_hyphenated_entry_is_looked_up_whole() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
|
||||
assert!(is_correct_with(&d, "tic-tac"));
|
||||
assert!(!is_correct_with(&d, "tic-tacc"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_dictionary_and_ignored_inputs_are_quiet() {
|
||||
let empty = AffixDict::default();
|
||||
|
||||
@@ -75,4 +75,13 @@ mod tests {
|
||||
assert!(!suggest("město", 4).contains(&"město".into()));
|
||||
assert!(!suggest("zqzakázane", 8).contains(&"zqzakázané".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_letter_that_lowercases_to_two_chars_is_still_suggested_for() {
|
||||
// wordcraft#4683: "İx" lowercases to three chars ("i", U+0307, "x") but has two typed
|
||||
// letters; the one-letter edits stay in range and dropping the dot still finds "ix".
|
||||
let d = AffixDict::from_hunspell("", "1\nix\n", &[]).unwrap();
|
||||
let suggestions = suggest_in(&d, "İx", 4);
|
||||
assert!(suggestions.contains(&"Ix".into()), "{suggestions:?}");
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
//! American and British spellings of the same English word (#1281, #1045): which forms each
|
||||
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`]) accepts, which it flags,
|
||||
//! English ([`ProofLang::En`], [`ProofLang::EnCa`], [`ProofLang::EnGb`] and [`ProofLang::EnAu`]) accepts, which it flags,
|
||||
//! and what it suggests instead (`colour` → `color` in American English).
|
||||
//!
|
||||
//! The pairs come from craft-i18n's `language-data/en-us/en-variants.txt`, our own list of base forms with a
|
||||
@@ -77,7 +77,8 @@ fn column(lang: ProofLang) -> Option<usize> {
|
||||
match lang {
|
||||
ProofLang::En => Some(0),
|
||||
ProofLang::EnCa => Some(1),
|
||||
ProofLang::EnGb => Some(2),
|
||||
// Australian English follows the British policy (`colour`, `centre`, `organise`).
|
||||
ProofLang::EnGb | ProofLang::EnAu => Some(2),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
+113
-22
@@ -4,10 +4,11 @@
|
||||
//! user keeps for WordCraft ([`crate::hunspell::user_dirs`]: the folder set in the options,
|
||||
//! `WORDCRAFT_DICTIONARIES`, WordCraft's own `dictionaries` folder).
|
||||
//! 2. **Installed English pack**: reserved for Language packs (craft-libs#485, CP4.2).
|
||||
//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB or en-CA) embedded
|
||||
//! 3. **Embedded SCOWL**: the run's English's SCOWL dictionary (en-US, en-GB, en-AU or en-CA) embedded
|
||||
//! from craft-i18n at build time ([`crate::scowl`]); a word it knows (British English: also
|
||||
//! the `-ize` spelling of one it knows, [`ise_spelling`]), or the supplement ([`EXTRA`]), is
|
||||
//! right; never one only another English knows. When it is there, the system locations aren't
|
||||
//! right; never one only another English knows, except that Australian English also takes
|
||||
//! en-GB's words (its suggestions still come from en-AU). When it is there, the system locations aren't
|
||||
//! searched.
|
||||
//! 4. **System Hunspell**: a pair for the run's own English in the system locations (none on the
|
||||
//! web). `COMPOUNDRULE` (ordinals) and `OCONV` are read; flag-based compounding is ignored
|
||||
@@ -18,8 +19,9 @@
|
||||
//! embedded from craft-i18n is the same list; it is used for hyphenation only.)
|
||||
//!
|
||||
//! Hunspell dictionaries are found by [`crate::hunspell`] by their own English's names only
|
||||
//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English), each name in every
|
||||
//! folder before the next name. Another English's dictionary is never used: it would mark this
|
||||
//! (`en_US` for American, `en_GB` for British, `en_CA` for Canadian English; `en_AU`, then
|
||||
//! `en_GB`, for Australian), each name in every folder before the next name. Another English's
|
||||
//! dictionary is never used: it would mark this
|
||||
//! English's spellings (`colour` in an `en_US` one). A file that can't be read is skipped for the
|
||||
//! next name or folder, and one file is read once for every English that uses it. `COMPOUNDRULE`
|
||||
//! (ordinals) and `OCONV` (suggestions are written as the dictionary says: `don’t`) are read;
|
||||
@@ -90,15 +92,47 @@ fn names(lang: ProofLang) -> &'static [&'static str] {
|
||||
match lang {
|
||||
ProofLang::EnGb => &["en_GB", "en-GB"],
|
||||
ProofLang::EnCa => &["en_CA", "en-CA"],
|
||||
// Australian English's own, else a British one (a system en_GB on a build without SCOWL):
|
||||
// [`exact_candidates`] reads every en_AU source before any en_GB one.
|
||||
ProofLang::EnAu => &["en_AU", "en-AU", "en_GB", "en-GB"],
|
||||
_ => &["en_US", "en-US"],
|
||||
}
|
||||
}
|
||||
|
||||
/// The Hunspell pairs of `lang` without SCOWL, from `find` (a folder search for some names,
|
||||
/// [`crate::hunspell::candidates_exact`]): its own names first, and only then Australian English's
|
||||
/// British fallback, so every en_AU source (a dictionaries folder, a pack, the system) comes before
|
||||
/// any en_GB one.
|
||||
fn exact_candidates(lang: ProofLang, find: impl Fn(&[&str]) -> Vec<(PathBuf, PathBuf)>) -> Vec<(PathBuf, PathBuf)> {
|
||||
let mut out = find(own_names(lang));
|
||||
if lang == ProofLang::EnAu {
|
||||
for c in find(&names(lang)[own_names(lang).len()..]) {
|
||||
if !out.iter().any(|(_, d)| *d == c.1) {
|
||||
out.push(c);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The names [`names`] gives that outrank the dialect's own embedded SCOWL dictionary: its own pair
|
||||
/// only, not Australian English's British fallback (a user's `en_GB` is for British text).
|
||||
fn own_names(lang: ProofLang) -> &'static [&'static str] {
|
||||
match lang {
|
||||
ProofLang::EnAu => &["en_AU", "en-AU"],
|
||||
_ => names(lang),
|
||||
}
|
||||
}
|
||||
|
||||
/// How many Englishes have a slot of their own ([`slot`]).
|
||||
const ENGLISHES: usize = 4;
|
||||
|
||||
/// The slot of each English in [`State::slots`].
|
||||
fn slot(lang: ProofLang) -> usize {
|
||||
match lang {
|
||||
ProofLang::EnGb => 1,
|
||||
ProofLang::EnCa => 2,
|
||||
ProofLang::EnAu => 3,
|
||||
_ => 0,
|
||||
}
|
||||
}
|
||||
@@ -109,7 +143,7 @@ type Loaded = Option<(Arc<AffixDict>, PathBuf)>;
|
||||
/// The English Hunspell dictionaries, under one lock.
|
||||
struct State {
|
||||
/// Per English (see [`slot`]): `None` until first use, then what was found.
|
||||
slots: [Option<Loaded>; 3],
|
||||
slots: [Option<Loaded>; ENGLISHES],
|
||||
/// Every `.dic` file read, so Englishes that use the same file share one copy.
|
||||
files: Vec<(PathBuf, Arc<AffixDict>)>,
|
||||
/// Counts reloads, so a load that started before one is not kept.
|
||||
@@ -118,7 +152,7 @@ struct State {
|
||||
|
||||
impl State {
|
||||
const fn new() -> Self {
|
||||
State { slots: [None, None, None], files: Vec::new(), reloads: 0 }
|
||||
State { slots: [None, None, None, None], files: Vec::new(), reloads: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +215,7 @@ fn already_read(state: &RwLock<State>, dic: &Path) -> Option<Arc<AffixDict>> {
|
||||
/// Forget what `state` holds, so the next check looks again.
|
||||
fn forget(state: &RwLock<State>) {
|
||||
let mut s = state.write().unwrap_or_else(PoisonError::into_inner);
|
||||
s.slots = [None, None, None];
|
||||
s.slots = [None, None, None, None];
|
||||
s.files.clear();
|
||||
s.reloads = s.reloads.wrapping_add(1);
|
||||
}
|
||||
@@ -198,9 +232,9 @@ fn key(lang: ProofLang) -> &'static str {
|
||||
/// would rather use), for [`note_error`].
|
||||
fn load(lang: ProofLang) -> (Loaded, Option<(PathBuf, String)>) {
|
||||
let candidates = if crate::scowl::dict(lang).is_some() {
|
||||
crate::hunspell::user_candidates(names(lang))
|
||||
crate::hunspell::user_candidates(own_names(lang))
|
||||
} else {
|
||||
crate::hunspell::candidates_exact(names(lang))
|
||||
exact_candidates(lang, crate::hunspell::candidates_exact)
|
||||
};
|
||||
if candidates.is_empty() {
|
||||
log::info!("English spelling: no Hunspell {} dictionary to read first; using SCOWL or the built-in word list", key(lang));
|
||||
@@ -240,7 +274,7 @@ fn note_error(lang: ProofLang, error: Option<(PathBuf, String)>) {
|
||||
/// folder changed).
|
||||
pub(crate) fn reload() {
|
||||
forget(&HUNSPELL);
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::EnAu] {
|
||||
crate::hunspell::clear_load_error(key(lang));
|
||||
}
|
||||
}
|
||||
@@ -283,10 +317,16 @@ fn pick(found: Loaded, lang: ProofLang) -> Speller {
|
||||
if let Some((d, dic)) = found {
|
||||
return Speller::Hunspell(d, dic);
|
||||
}
|
||||
match crate::scowl::dict(lang) {
|
||||
Some(d) => Speller::Scowl(vec![d], crate::scowl::is_british(lang)),
|
||||
None => Speller::List,
|
||||
let Some(d) = crate::scowl::dict(lang) else { return Speller::List };
|
||||
let mut ds = vec![d];
|
||||
// SCOWL's en-AU list keeps one spelling per word (`inquiry`; `enquiry` is only in the en-GB
|
||||
// list), and Australian English takes both. Suggestions come from the first, en-AU.
|
||||
if lang == ProofLang::EnAu
|
||||
&& let Some(gb) = crate::scowl::dict(ProofLang::EnGb)
|
||||
{
|
||||
ds.push(gb);
|
||||
}
|
||||
Speller::Scowl(ds, crate::scowl::is_british(lang))
|
||||
}
|
||||
|
||||
/// [`speller`] for English with no dialect ([`crate::is_correct`]): a SCOWL word of any English
|
||||
@@ -348,11 +388,12 @@ pub(crate) fn is_correct(word: &str, lang: ProofLang, opts: &Options, policy: bo
|
||||
|| (policy && !parts.is_empty() && parts.iter().all(|p| s.knows_lower(p) || crate::en_variants::known_variant(p, lang, |x| s.knows_lower(x))))
|
||||
}
|
||||
|
||||
/// `w` with its `-ize`/`-yze` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`/
|
||||
/// `-yse`, when it has any: British English takes both (Word accepts `organize` and `organise` in
|
||||
/// en-GB), and SCOWL's en-GB dictionary has only `-ise`. A `z` between `i`/`y` and a vowel is
|
||||
/// swapped; the `-ize` form counts only when American or Canadian English spells the word so
|
||||
/// ([`ize_elsewhere`]), or `wize` would pass as `wise`.
|
||||
/// `w` with its `-ize` endings (and their forms: `-ization`, `-izing`…) spelled `-ise`, when it
|
||||
/// has any: British English takes both (Word accepts `organize` and `organise` in en-GB), and
|
||||
/// SCOWL's en-GB dictionary has only `-ise`. A `z` between `i` and a vowel is swapped; the `-ize`
|
||||
/// form counts only when American or Canadian English spells the word so ([`ize_elsewhere`]), or
|
||||
/// `wize` would pass as `wise`. Never `-yze`: Oxford spelling has `analyse`, `paralyse`, so
|
||||
/// `reanalyze` or `breathalyzer` is American in British English, as Hunspell's en_GB says.
|
||||
pub(crate) fn ise_spelling(w: &str) -> Option<String> {
|
||||
let b = w.as_bytes();
|
||||
let mut out = b.to_vec();
|
||||
@@ -360,7 +401,7 @@ pub(crate) fn ise_spelling(w: &str) -> Option<String> {
|
||||
for i in 1..b.len() {
|
||||
let after = b.get(i + 1).is_some_and(|c| b"aeiAEI".contains(c));
|
||||
if after
|
||||
&& matches!(b.get(i - 1), Some(b'i' | b'y' | b'I' | b'Y'))
|
||||
&& matches!(b.get(i - 1), Some(b'i' | b'I'))
|
||||
&& let Some(c) = out.get_mut(i)
|
||||
{
|
||||
match *c {
|
||||
@@ -512,6 +553,18 @@ fn list_suggest(word: &str, max: usize) -> Vec<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Review follow-up: only `-ize` is swapped for `-ise`, never `-yze` for `-yse` (Oxford
|
||||
/// spelling has `analyse`), so `reanalyze`, `breathalyzer`, `paralyzingly` stay American.
|
||||
#[test]
|
||||
fn ise_spelling_swaps_ize_only() {
|
||||
assert_eq!(ise_spelling("organize").as_deref(), Some("organise"));
|
||||
assert_eq!(ise_spelling("Realizations").as_deref(), Some("Realisations"));
|
||||
assert_eq!(ise_spelling("ORGANIZING").as_deref(), Some("ORGANISING"));
|
||||
for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "Analyzer", "ANALYZE"] {
|
||||
assert_eq!(ise_spelling(w), None, "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn built_in_list_is_the_whole_moby_list() {
|
||||
let d = words();
|
||||
@@ -596,16 +649,16 @@ mod tests {
|
||||
fn lookup_order() {
|
||||
let tiny = Arc::new(AffixDict::from_hunspell("", "1\nzzyzx\n", &[]).unwrap());
|
||||
let file = PathBuf::from("/folder/en_US.dic");
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
|
||||
assert!(matches!(pick(Some((tiny.clone(), file.clone())), lang), Speller::Hunspell(_, p) if p == file), "{lang:?}");
|
||||
let fallback = pick(None, lang);
|
||||
if crate::scowl::dict(lang).is_some() {
|
||||
assert!(
|
||||
matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == 1 && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())),
|
||||
matches!(&fallback, Speller::Scowl(ds, _) if ds.len() == if lang == ProofLang::EnAu { 2 } else { 1 } && std::ptr::eq(ds[0], crate::scowl::dict(lang).unwrap())),
|
||||
"{lang:?}"
|
||||
);
|
||||
assert!(fallback.knows("21st") && !fallback.knows("becuase"), "{lang:?}");
|
||||
assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary only");
|
||||
assert_eq!(fallback.knows("colour"), lang != ProofLang::En, "{lang:?}: its own dictionary (en-AU: and en-GB's)");
|
||||
} else {
|
||||
assert!(matches!(fallback, Speller::List), "{lang:?}");
|
||||
}
|
||||
@@ -625,6 +678,44 @@ mod tests {
|
||||
assert_eq!(found(names(ProofLang::EnGb)), ["en_GB"], "a user's en_US doesn't stand in for SCOWL's en-GB");
|
||||
assert_eq!(found(names(ProofLang::EnCa)), ["en_CA"]);
|
||||
assert_eq!(found(names(ProofLang::En)), ["en_US"]);
|
||||
// Australian English: its own, else the British one it read before it had its own.
|
||||
assert_eq!(found(names(ProofLang::EnAu)), ["en_GB"]);
|
||||
std::fs::write(dir.join("en_AU.aff"), "SET UTF-8\n").unwrap();
|
||||
std::fs::write(dir.join("en_AU.dic"), "1\nzzyzx\n").unwrap();
|
||||
assert_eq!(found(names(ProofLang::EnAu)), ["en_AU", "en_GB"]);
|
||||
// craft-libs#496 review: with SCOWL embedded, a user's en_GB must not outrank its en-AU.
|
||||
assert_eq!(found(own_names(ProofLang::EnAu)), ["en_AU"]);
|
||||
std::fs::remove_file(dir.join("en_AU.aff")).unwrap();
|
||||
std::fs::remove_file(dir.join("en_AU.dic")).unwrap();
|
||||
assert!(found(own_names(ProofLang::EnAu)).is_empty(), "an en_GB alone is not Australian English's own");
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
assert_eq!(own_names(lang), names(lang));
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Without SCOWL, every en_AU source comes before any en_GB one (a dictionaries folder's, an
|
||||
/// installed pack's, the system's): a user's en_GB doesn't outrank a system en_AU.
|
||||
#[test]
|
||||
fn australian_english_reads_every_en_au_before_any_en_gb() {
|
||||
let dir = std::env::temp_dir().join(format!("wc-proof-en-au-order-{}", std::process::id()));
|
||||
for sub in ["user", "system"] {
|
||||
std::fs::create_dir_all(dir.join(sub)).unwrap();
|
||||
}
|
||||
let write = |sub: &str, name: &str| {
|
||||
std::fs::write(dir.join(sub).join(format!("{name}.aff")), "SET UTF-8\n").unwrap();
|
||||
std::fs::write(dir.join(sub).join(format!("{name}.dic")), "1\nzzyzx\n").unwrap();
|
||||
};
|
||||
write("user", "en_GB");
|
||||
write("system", "en_AU");
|
||||
let (user, system) = (vec![dir.join("user")], vec![dir.join("system")]);
|
||||
let find = |n: &[&str]| crate::hunspell::ordered(n, &user, &[], &system, &[]);
|
||||
let stems = |v: Vec<(PathBuf, PathBuf)>| -> Vec<String> {
|
||||
v.iter().filter_map(|(_, d)| d.file_stem().map(|s| s.to_string_lossy().into_owned())).collect()
|
||||
};
|
||||
assert_eq!(stems(exact_candidates(ProofLang::EnAu, find)), ["en_AU", "en_GB"]);
|
||||
assert_eq!(stems(exact_candidates(ProofLang::EnGb, find)), ["en_GB"]);
|
||||
assert_eq!(stems(exact_candidates(ProofLang::En, find)), Vec::<String>::new());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
|
||||
@@ -75,11 +75,8 @@ fn spaces(text: &str, lang: ProofLang, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Espacio doble".into(), vec![" ".into()], lang));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,6 +191,16 @@ mod tests {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// craft-libs#496 review: a run of spaces is one issue over the whole run.
|
||||
#[test]
|
||||
fn a_run_of_spaces_is_reported_whole() {
|
||||
for (text, run) in [("casa roja", " "), ("casa roja", " "), ("casa roja", " "), ("casa roja", " ")] {
|
||||
let m: Vec<_> = messages(text).into_iter().filter(|(_, msg)| msg == "Espacio doble").collect();
|
||||
assert_eq!(m, [(run.to_string(), "Espacio doble".to_string())], "{text}");
|
||||
}
|
||||
assert!(messages("casa roja").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_words_and_spaces_have_spanish_messages() {
|
||||
let text = "Es es la casa , dijo.";
|
||||
|
||||
@@ -105,11 +105,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Razmak ispred interpunkcijskog znaka".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Dvostruki razmak".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Dvostruki razmak".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,7 +65,8 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
return true;
|
||||
};
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
// The whole word first, as Hunspell does: a hyphenated entry or user word.
|
||||
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
known(d, w)
|
||||
}
|
||||
@@ -105,6 +106,14 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// craft-libs#496 review: a hyphenated dictionary entry is a word even when its parts aren't.
|
||||
#[test]
|
||||
fn a_hyphenated_entry_is_looked_up_whole() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
|
||||
assert!(is_correct_with(&d, "tic-tac"));
|
||||
assert!(!is_correct_with(&d, "tic-tacc"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_dictionary_is_silent() {
|
||||
assert!(is_correct_with(&AffixDict::default(), "nepostojećariječ"));
|
||||
|
||||
@@ -196,19 +196,22 @@ pub(crate) fn ordered(
|
||||
let first = names.first().map(|n| norm(n));
|
||||
let (region, rest): (Vec<&str>, Vec<&str>) = names.iter().copied().partition(|n| Some(norm(n)) == first);
|
||||
let pack: &[&str] = &[crate::packs::SPELLING];
|
||||
// (folder, names, a Language pack's: never read through a symbolic link, as the pack scan
|
||||
// checks; a system folder's dictionaries often are links, `en_GB.dic` → `en_GB-ise.dic`).
|
||||
let tiers = dirs
|
||||
.iter()
|
||||
.map(|d| (d, names))
|
||||
.chain(region_packs.iter().map(|d| (d, pack)))
|
||||
.chain(system.iter().map(|d| (d, ®ion[..])))
|
||||
.chain(other_packs.iter().map(|d| (d, pack)))
|
||||
.chain(system.iter().map(|d| (d, &rest[..])));
|
||||
.map(|d| (d, names, false))
|
||||
.chain(region_packs.iter().map(|d| (d, pack, true)))
|
||||
.chain(system.iter().map(|d| (d, ®ion[..], false)))
|
||||
.chain(other_packs.iter().map(|d| (d, pack, true)))
|
||||
.chain(system.iter().map(|d| (d, &rest[..], false)));
|
||||
let file = |p: &Path, pack: bool| if pack { std::fs::symlink_metadata(p).is_ok_and(|m| m.is_file()) } else { p.is_file() };
|
||||
let mut out: Vec<(PathBuf, PathBuf)> = Vec::new();
|
||||
for (dir, names) in tiers {
|
||||
for (dir, names, in_pack) in tiers {
|
||||
for n in names {
|
||||
let aff = dir.join(format!("{n}.aff"));
|
||||
let dic = dir.join(format!("{n}.dic"));
|
||||
if aff.is_file() && dic.is_file() && !out.iter().any(|(_, d)| *d == dic) {
|
||||
if file(&aff, in_pack) && file(&dic, in_pack) && !out.iter().any(|(_, d)| *d == dic) {
|
||||
out.push((aff, dic));
|
||||
}
|
||||
}
|
||||
@@ -696,6 +699,34 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Review follow-up (craft-libs#485): a pack's `spelling.aff`/`.dic` that is a symbolic link
|
||||
/// (swapped in after the pack scan) is never a candidate; one in a system folder still is.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn pack_files_are_never_read_through_a_link() {
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-pack-links-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
let (pack, system, elsewhere) = (root.join("packs").join("nl"), root.join("system"), root.join("elsewhere"));
|
||||
for d in [&pack, &system, &elsewhere] {
|
||||
std::fs::create_dir_all(d).unwrap();
|
||||
}
|
||||
std::fs::write(elsewhere.join("x.aff"), "SET UTF-8\n").unwrap();
|
||||
std::fs::write(elsewhere.join("x.dic"), "1\nx\n").unwrap();
|
||||
std::fs::write(pack.join("spelling.aff"), "SET UTF-8\n").unwrap();
|
||||
std::os::unix::fs::symlink(elsewhere.join("x.dic"), pack.join("spelling.dic")).unwrap();
|
||||
std::os::unix::fs::symlink(elsewhere.join("x.aff"), system.join("nl.aff")).unwrap();
|
||||
std::os::unix::fs::symlink(elsewhere.join("x.dic"), system.join("nl.dic")).unwrap();
|
||||
let got: Vec<PathBuf> =
|
||||
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
|
||||
assert_eq!(got, [system.join("nl.dic")], "the system's link is followed, the pack's never");
|
||||
std::fs::remove_file(pack.join("spelling.dic")).unwrap();
|
||||
std::fs::write(pack.join("spelling.dic"), "1\nx\n").unwrap();
|
||||
let got: Vec<PathBuf> =
|
||||
ordered(&["nl"], &[], std::slice::from_ref(&pack), std::slice::from_ref(&system), &[]).into_iter().map(|(_, d)| d).collect();
|
||||
assert_eq!(got, [pack.join("spelling.dic"), system.join("nl.dic")]);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// WordCraft's own dictionaries folder under the user's application data is searched on every
|
||||
/// desktop system, after the setting and before the shared system locations (#1063).
|
||||
#[test]
|
||||
|
||||
@@ -56,11 +56,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Spasi sebelum tanda baca".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Spasi ganda".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Spasi ganda".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -76,7 +76,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
|| w.split('-').all(|p| {
|
||||
p.chars().count() <= 1
|
||||
|| CLITICS.iter().any(|c| p.eq_ignore_ascii_case(c))
|
||||
|| (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= 6)
|
||||
|| (opts.ignore_uppercase && crate::spelling::capitals_part(p, 6))
|
||||
|| known(d, p)
|
||||
});
|
||||
}
|
||||
@@ -141,6 +141,19 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// Review follow-up (craft-libs#485's cased-letter rule): a part in a caseless script is no
|
||||
/// abbreviation, so `buku-日本` is checked part by part; a part in capitals or without letters
|
||||
/// still is one (`KTP-nya`, `COVID-19` with numbers checked).
|
||||
#[test]
|
||||
fn caseless_parts_are_not_abbreviations() {
|
||||
test_dict::install();
|
||||
let d = dictionary();
|
||||
let defaults = crate::Options::default();
|
||||
assert!(!check(&d, "buku-日本", &defaults) && !check(&d, "buku-ไทย", &defaults));
|
||||
let numbers = crate::Options { ignore_numbers: false, ..defaults };
|
||||
assert!(check(&d, "KTP-nya", &defaults) && check(&d, "buku-19", &numbers));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forbidden_words_are_rejected_in_every_case() {
|
||||
test_dict::install();
|
||||
|
||||
@@ -150,10 +150,12 @@ fn abbreviation_or_number(prev: &str) -> bool {
|
||||
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
|
||||
}
|
||||
|
||||
const CLOSING: [char; 8] = ['"', '”', '’', '\'', ')', '»', ']', '\u{1}'];
|
||||
|
||||
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
|
||||
/// an abbreviation), then space, with nothing but quotes and brackets around?
|
||||
fn ends_sentence(gap: &str, prev: &str) -> bool {
|
||||
let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']);
|
||||
let t = gap.trim_start_matches(CLOSING);
|
||||
let Some(end) = t.chars().next() else { return false };
|
||||
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
|
||||
return false;
|
||||
@@ -161,8 +163,16 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
|
||||
if end == '.' && abbreviation_or_number(prev) {
|
||||
return false;
|
||||
}
|
||||
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']);
|
||||
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
|
||||
let after_marks = t.trim_start_matches(['.', '!', '?']);
|
||||
let rest = after_marks.trim_start_matches(CLOSING);
|
||||
// (Not the object placeholder `\u{1}`: `Chi era?\u{1} era Marco.` still starts a sentence.)
|
||||
// `«Vai?» disse lui.` and `—Vai? —disse lui.`: the narration after a quoted question or
|
||||
// exclamation (not after a period: `Finì. —domani` starts a sentence), or after a dialogue dash, goes on in lower case.
|
||||
if end != '.' && (rest.contains('—') || after_marks.trim_start_matches('\u{1}').starts_with(CLOSING)) {
|
||||
return false;
|
||||
}
|
||||
rest.starts_with(char::is_whitespace)
|
||||
&& rest.trim_start().chars().all(|c| c.is_whitespace() || matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
|
||||
}
|
||||
|
||||
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
|
||||
@@ -203,11 +213,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Spazio doppio".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,6 +410,23 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// craft-libs#496 review: narration after a quoted or dashed question or exclamation goes on in
|
||||
/// lower case.
|
||||
#[test]
|
||||
fn narration_after_a_quoted_question_is_not_a_new_sentence() {
|
||||
for t in ["«Vai?» disse lui.", "«Vieni!» gridò Anna.", "\"Vai?\" disse lui.", "—Vai? —disse lui.", "— Vai! — disse lui."] {
|
||||
assert!(fixes(t).is_empty(), "{t}: {:?}", fixes(t));
|
||||
}
|
||||
assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")), "a bare question still ends the sentence");
|
||||
// After a period the dash opens the next line of speech: a sentence starts.
|
||||
assert!(fixes("Finì. —domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. —domani partiamo."));
|
||||
assert!(fixes("Finì. — domani partiamo.").contains(&pair("domani", "Domani")), "{:?}", fixes("Finì. — domani partiamo."));
|
||||
// An inline object (`\u{1}`) before a real closing quote is skipped.
|
||||
assert!(fixes("«Vai?\u{1}» disse lui.").is_empty(), "{:?}", fixes("«Vai?\u{1}» disse lui."));
|
||||
// An inline object (`\u{1}`) is not a closing quote.
|
||||
assert!(fixes("Chi era?\u{1} era Marco.").contains(&pair("era", "Era")), "{:?}", fixes("Chi era?\u{1} era Marco."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn po_takes_an_apostrophe() {
|
||||
assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]);
|
||||
|
||||
@@ -97,8 +97,14 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
}
|
||||
// The whole word first, as Hunspell does: a hyphenated entry or user word.
|
||||
if w.contains('-') {
|
||||
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known_elided(d, p));
|
||||
}
|
||||
known_elided(d, w)
|
||||
}
|
||||
|
||||
/// [`known`] for a word that may start with an elided proclitic (`l'amico`, `l'ex`): the word after
|
||||
/// the apostrophe is what gets checked.
|
||||
fn known_elided(d: &AffixDict, w: &str) -> bool {
|
||||
if let Some((head, tail)) = w.split_once(['\'', '’'])
|
||||
&& PROCLITICS.contains(&head.to_lowercase().as_str())
|
||||
&& !tail.is_empty()
|
||||
@@ -148,6 +154,16 @@ mod tests {
|
||||
assert!(!is_correct_with(&d, "tic-tacc"));
|
||||
}
|
||||
|
||||
/// craft-libs#496 review: an elision inside a hyphenated word (`l'ex-amico`) is read per part.
|
||||
#[test]
|
||||
fn an_elision_in_a_hyphenated_part_is_read() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\n", "2\nex\namico\n", &[]).unwrap();
|
||||
for w in ["l'ex-amico", "dell’ex-amico", "ex-amico"] {
|
||||
assert!(is_correct_with(&d, w), "{w}");
|
||||
}
|
||||
assert!(!is_correct_with(&d, "l'exx-amico"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_dictionary_nothing_is_flagged() {
|
||||
let none = AffixDict::default();
|
||||
|
||||
+23
-30
@@ -5,7 +5,7 @@
|
||||
/// A language WordCraft can proof, or [`ProofLang::Other`] for one it can't.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub enum ProofLang {
|
||||
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`] or
|
||||
/// American English (`en`, `en-US`, and every region not listed for [`ProofLang::EnGb`], [`ProofLang::EnAu`] or
|
||||
/// [`ProofLang::EnCa`]): SCOWL's en-US dictionary ([`crate::scowl`]), the English grammar rules and the
|
||||
/// English hyphenation dictionary and patterns. British spellings (`colour`, `centre`,
|
||||
/// `organise`) are flagged, with the American form suggested ([`crate::en_variants`]). Also
|
||||
@@ -13,10 +13,14 @@ pub enum ProofLang {
|
||||
/// Word's).
|
||||
#[default]
|
||||
En,
|
||||
/// English with British spelling (`en-GB`, `en-AU`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
|
||||
/// English with British spelling (`en-GB`, `en-NZ`, `en-IE`, `en-ZA`, `en-IN`…):
|
||||
/// [`ProofLang::En`]'s tools, but American spellings (`color`, `center`, `analyze`) are
|
||||
/// flagged and `-ize` and `-ise` are both right, as in Oxford spelling.
|
||||
EnGb,
|
||||
/// Australian English (`en-AU`): [`ProofLang::EnGb`]'s spelling policy, against SCOWL's own
|
||||
/// en-AU dictionary, which knows Australian words (`billabong`, `brumby`, `larrikin`) and
|
||||
/// spells some words the Australian way (`program`, `inquiry`).
|
||||
EnAu,
|
||||
/// Canadian English (`en-CA`): British `-our`, `-re` and `-ence` (`colour`, `centre`,
|
||||
/// `defence`) with American `-ize` (`organize`), `tire` and `program`.
|
||||
EnCa,
|
||||
@@ -86,9 +90,10 @@ pub enum ProofLang {
|
||||
|
||||
impl ProofLang {
|
||||
/// Every language with a spelling dictionary, in display order.
|
||||
pub const PROOFED: [ProofLang; 15] = [
|
||||
pub const PROOFED: [ProofLang; 16] = [
|
||||
ProofLang::En,
|
||||
ProofLang::EnGb,
|
||||
ProofLang::EnAu,
|
||||
ProofLang::EnCa,
|
||||
ProofLang::Pl,
|
||||
ProofLang::Pt,
|
||||
@@ -155,12 +160,13 @@ impl ProofLang {
|
||||
}
|
||||
}
|
||||
|
||||
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions
|
||||
/// that spell the British way are [`ProofLang::EnGb`], and the United States, any other
|
||||
/// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], Australia
|
||||
/// [`ProofLang::EnAu`], other regions that spell the British way are [`ProofLang::EnGb`], and the United States, any other
|
||||
/// region and no region are [`ProofLang::En`].
|
||||
fn english(tag: &str) -> ProofLang {
|
||||
match region(tag) {
|
||||
Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa,
|
||||
Some(r) if r.eq_ignore_ascii_case("AU") => ProofLang::EnAu,
|
||||
Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb,
|
||||
_ => ProofLang::En,
|
||||
}
|
||||
@@ -189,9 +195,10 @@ impl ProofLang {
|
||||
}
|
||||
}
|
||||
|
||||
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])?
|
||||
/// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnAu`],
|
||||
/// [`ProofLang::EnCa`])?
|
||||
pub fn is_english(self) -> bool {
|
||||
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa)
|
||||
matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa)
|
||||
}
|
||||
|
||||
/// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`],
|
||||
@@ -208,7 +215,7 @@ impl ProofLang {
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
ProofLang::Pack(_) => "",
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => "en",
|
||||
ProofLang::Pl => "pl",
|
||||
ProofLang::Bg => "bg",
|
||||
ProofLang::Pt => "pt",
|
||||
@@ -371,12 +378,12 @@ pub(crate) fn region(tag: &str) -> Option<&str> {
|
||||
const LATIN_AMERICA: [&str; 22] =
|
||||
["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"];
|
||||
|
||||
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and
|
||||
/// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other
|
||||
/// Regions whose English uses British spelling ([`ProofLang::EnGb`]; Australia has its own,
|
||||
/// [`ProofLang::EnAu`]): the United Kingdom (and the non-standard `UK`), Ireland, New Zealand, South Africa, India and other
|
||||
/// Commonwealth countries, the Crown dependencies and Europe (`150`).
|
||||
const BRITISH_SPELLING: [&str; 26] = [
|
||||
"GB", "UK", "IE", "AU", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH",
|
||||
"PK", "150",
|
||||
const BRITISH_SPELLING: [&str; 25] = [
|
||||
"GB", "UK", "IE", "NZ", "ZA", "IN", "SG", "HK", "MY", "MT", "CY", "GI", "IM", "JE", "GG", "JM", "TT", "BS", "BB", "BZ", "KE", "NG", "GH", "PK",
|
||||
"150",
|
||||
];
|
||||
|
||||
/// A BCP 47 tag in canonical case (`pl-PL`, `en-GB`, `zh-Hant-TW`, `x-none`), or `None` when
|
||||
@@ -469,26 +476,12 @@ mod tests {
|
||||
fn tags_resolve_english_regions() {
|
||||
assert_tags(&["en", "en-US", "EN_us", "en-Latn-US", "en-PH", "en-001", "eng", "en_US.UTF-8", "en-x-gb"], ProofLang::En);
|
||||
assert_tags(
|
||||
&[
|
||||
"en-GB",
|
||||
"EN_gb",
|
||||
"en_GB.UTF-8",
|
||||
"en-AU",
|
||||
"en_AU",
|
||||
"en-NZ",
|
||||
"en-IE",
|
||||
"en-ZA",
|
||||
"en-IN",
|
||||
"en-Latn-GB",
|
||||
"en-UK",
|
||||
"en-150",
|
||||
"eng-GB",
|
||||
"en_GB@euro",
|
||||
],
|
||||
&["en-GB", "EN_gb", "en_GB.UTF-8", "en-NZ", "en-IE", "en-ZA", "en-IN", "en-Latn-GB", "en-UK", "en-150", "eng-GB", "en_GB@euro"],
|
||||
ProofLang::EnGb,
|
||||
);
|
||||
assert_tags(&["en-CA", "en_CA", "EN-ca", "en_CA.UTF-8", "en-Latn-CA"], ProofLang::EnCa);
|
||||
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
assert_tags(&["en-AU", "en_AU", "EN-au", "en_AU.UTF-8", "en-Latn-AU"], ProofLang::EnAu);
|
||||
for language in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
|
||||
assert!(language.is_english() && language.is_proofed() && ProofLang::PROOFED.contains(&language), "{language:?}");
|
||||
assert_eq!(language.code(), "en");
|
||||
}
|
||||
|
||||
+155
-12
@@ -152,6 +152,28 @@ fn dictionary_key(w: &str) -> String {
|
||||
composed(&w.to_lowercase()).into_owned()
|
||||
}
|
||||
|
||||
/// Each run of two or more spaces in `text`, once, over the whole run (a fix replaces it with one
|
||||
/// space). A run of three is one issue over three, not a pair and a leftover.
|
||||
pub(crate) fn space_runs(text: &str) -> Vec<Range<usize>> {
|
||||
let bytes = text.as_bytes();
|
||||
let mut runs = Vec::new();
|
||||
let mut i = 0;
|
||||
while i < bytes.len() {
|
||||
if bytes.get(i) != Some(&b' ') {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let start = i;
|
||||
while bytes.get(i) == Some(&b' ') {
|
||||
i += 1;
|
||||
}
|
||||
if i - start >= 2 {
|
||||
runs.push(start..i);
|
||||
}
|
||||
}
|
||||
runs
|
||||
}
|
||||
|
||||
/// Add a word to the user dictionary ("Add to Dictionary"). It applies to every language.
|
||||
pub fn add_word(w: &str) {
|
||||
if user_words().write().unwrap_or_else(std::sync::PoisonError::into_inner).insert(dictionary_key(w)) {
|
||||
@@ -340,7 +362,7 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)>
|
||||
/// checked yet reports `None` (#2830). For an English, its embedded SCOWL dictionary or the
|
||||
/// English Hunspell dictionary that failed; English then uses the next source ([`english_source`]).
|
||||
pub fn dictionary_load_error(lang: ProofLang) -> Option<String> {
|
||||
if matches!(lang, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) {
|
||||
if lang.is_english() {
|
||||
return english::load_error(lang);
|
||||
}
|
||||
if let Some(l) = lang.spelling_locale() {
|
||||
@@ -669,7 +691,7 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool {
|
||||
|| !w.chars().any(char::is_alphabetic)
|
||||
|| (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit()))
|
||||
// Only a word with a cased letter is in capitals: a Hebrew or Thai word has none.
|
||||
|| (opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()))
|
||||
|| (opts.ignore_uppercase && spelling::in_capitals(w))
|
||||
// The whole word: the languages split a hyphenated word into its parts, and an added
|
||||
// `Unigranrio-RJ` is none of them (#2082).
|
||||
|| in_user_dictionary(&w.to_lowercase())
|
||||
@@ -953,7 +975,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec<Issue> {
|
||||
fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
|
||||
match lang {
|
||||
ProofLang::En => grammar_en(text, capital_first),
|
||||
ProofLang::EnGb | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
|
||||
ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(),
|
||||
// `pt-pt` shares Portuguese's rules, its issues in its own language.
|
||||
_ => language(lang.built_in())
|
||||
.map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()),
|
||||
@@ -1098,13 +1120,10 @@ fn grammar_en(text: &str, capital_first: bool) -> Vec<Issue> {
|
||||
lang: ProofLang::En,
|
||||
});
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
for run in space_runs(text) {
|
||||
v.push(Issue {
|
||||
start: i,
|
||||
end: i + 2,
|
||||
start: run.start,
|
||||
end: run.end,
|
||||
kind: IssueKind::Grammar,
|
||||
message: "Extra space".into(),
|
||||
suggestions: vec![" ".into()],
|
||||
@@ -1197,7 +1216,7 @@ mod tests {
|
||||
}
|
||||
// Every proofed language other than English has a Hunspell dictionary to report.
|
||||
for l in ProofLang::PROOFED {
|
||||
let english = matches!(l, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa);
|
||||
let english = l.is_english();
|
||||
assert_eq!(dictionary_needed(l, false).is_some(), !english, "{l:?}");
|
||||
}
|
||||
for l in [ProofLang::En, ProofLang::Other] {
|
||||
@@ -1709,6 +1728,31 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// Review follow-up (craft-libs#485): a word in another script than a pack language's
|
||||
/// dictionary (a Russian name in a Dutch run, a Dutch word in a Greek one) isn't marked; a
|
||||
/// misspelling in the dictionary's own script still is.
|
||||
#[test]
|
||||
fn pack_languages_leave_other_scripts_alone() {
|
||||
crate::hold_dictionary_tests();
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-other-script-packs-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
write_pack(&root, "nl", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "2\nhuis\nkat\n".as_bytes())]);
|
||||
write_pack(&root, "el", &[("spelling.aff", b"SET UTF-8\n"), ("spelling.dic", "1\nσπίτι\n".as_bytes())]);
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
let (nl, el) = (ProofLang::from_tag(Some("nl-NL")), ProofLang::from_tag(Some("el-GR")));
|
||||
assert!(is_correct_in("huis", nl) && !is_correct_in("huiss", nl));
|
||||
assert!(!is_correct_in("zzzz", nl), "a misspelling in the dictionary's script, whatever its letters");
|
||||
assert!(is_correct_in("Пушкин", nl) && is_correct_in("σπίτι", nl));
|
||||
assert!(is_correct_in("σπίτι", el) && !is_correct_in("σπίτιι", el) && is_correct_in("huis", el));
|
||||
let text = "Пушкин huiss";
|
||||
let issues = check_text(text, &[(0..text.len(), nl)]);
|
||||
assert_eq!(issues.iter().map(|i| &text[i.start..i.end]).collect::<Vec<_>>(), ["huiss"], "{issues:?}");
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// The `.dic` file `lang`'s spelling is read from with the installed packs and `system` as the
|
||||
/// system's only dictionary folder.
|
||||
fn chosen_dictionary(lang: ProofLang, system: &std::path::Path) -> Option<std::path::PathBuf> {
|
||||
@@ -2465,6 +2509,49 @@ mod tests {
|
||||
assert_eq!(suggest_in("programmes", 5, us).first().map(String::as_str), Some("programs"));
|
||||
}
|
||||
|
||||
/// Review follow-up: British English takes `-ize` (Oxford spelling) but never `-yze`, also in
|
||||
/// forms en-variants.txt doesn't list. Hunspell 1.7.5 with SCOWL's en_GB rejects these too.
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn british_english_rejects_unlisted_yze_forms() {
|
||||
crate::hold_dictionary_tests();
|
||||
let gb = ProofLang::EnGb;
|
||||
for w in ["reanalyze", "reanalyzed", "breathalyze", "breathalyzer", "paralyzingly", "catalyze", "analyzer"] {
|
||||
assert!(!is_correct_in(w, gb), "en-GB: {w}");
|
||||
assert!(is_correct_in(w, ProofLang::En), "en-US: {w}");
|
||||
}
|
||||
for w in ["reanalyse", "breathalyser", "paralysingly", "catalyse", "organize", "realization"] {
|
||||
assert!(is_correct_in(w, gb), "en-GB: {w}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Review follow-up: Australian English reads SCOWL's own en-AU dictionary (craft-i18n ships
|
||||
/// it), not en-GB's: its Australian words are right and spelled as Hunspell's en_AU spells
|
||||
/// them; British English doesn't take them. The British policy still applies: American
|
||||
/// spellings are wrong, `-ise` and `-ize` both right.
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn australian_english_reads_its_own_scowl() {
|
||||
crate::hold_dictionary_tests();
|
||||
let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb);
|
||||
assert!(scowl::is_british(au));
|
||||
assert_eq!(english_source(au), Some(EnglishSource::Embedded));
|
||||
for w in ["billabong", "brumby", "larrikinism", "bunyip", "corroboree", "Bundaberg", "Rockhampton"] {
|
||||
assert!(is_correct_in(w, au), "en-AU: {w}");
|
||||
assert!(!is_correct_in(w, gb), "en-GB: {w}");
|
||||
assert!(is_correct_for(w, Some("en-AU")), "en-AU tag: {w}");
|
||||
}
|
||||
for w in ["colour", "centre", "organise", "organize", "analyse", "travelled", "defence"] {
|
||||
assert!(is_correct_in(w, au), "en-AU: {w}");
|
||||
}
|
||||
for w in ["color", "center", "analyze", "reanalyze", "defense"] {
|
||||
assert!(!is_correct_in(w, au), "en-AU: {w}");
|
||||
}
|
||||
let text = "The larrikin saw a brumby by the billabong and analysed the colour of the sky.";
|
||||
assert!(check_spelling_in(text, au).is_empty(), "{:?}", check_spelling_in(text, au));
|
||||
assert_eq!(check_spelling_in(text, gb).len(), 3, "larrikin, brumby, billabong: {:?}", check_spelling_in(text, gb));
|
||||
}
|
||||
|
||||
/// More British spellings that must stay right in en-GB and en-AU (#1281): -re, single l, ae,
|
||||
/// with their inflections.
|
||||
#[test]
|
||||
@@ -2472,7 +2559,7 @@ mod tests {
|
||||
// English results depend on the dictionaries other tests install.
|
||||
crate::hold_dictionary_tests();
|
||||
let gb = ProofLang::EnGb;
|
||||
assert_eq!(ProofLang::from_tag(Some("en-AU")), gb);
|
||||
assert_eq!(ProofLang::from_tag(Some("en-AU")), ProofLang::EnAu);
|
||||
for w in [
|
||||
"goitre",
|
||||
"goitres",
|
||||
@@ -2497,11 +2584,66 @@ mod tests {
|
||||
"encyclopaedias",
|
||||
] {
|
||||
assert!(is_correct_in(w, gb), "en-GB: {w}");
|
||||
assert!(is_correct_in(w, ProofLang::EnAu), "en-AU: {w}");
|
||||
}
|
||||
let text = "The wilful knight took the sceptre to fulfil his enrolment, paid the instalment and read the encyclopaedia.";
|
||||
assert!(check_spelling_in(text, gb).is_empty(), "{:?}", check_spelling_in(text, gb));
|
||||
}
|
||||
|
||||
/// Review follow-up: SCOWL's en-AU list keeps one spelling per word, so Australian English also
|
||||
/// takes a word the en-GB list has (`enquiry`, `guerrilla`, `liquorice`); American-only
|
||||
/// spellings, wrong in both, stay wrong. Suggestions still come from the en-AU list.
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
#[test]
|
||||
fn australian_english_also_takes_british_words() {
|
||||
crate::hold_dictionary_tests();
|
||||
let (au, gb) = (ProofLang::EnAu, ProofLang::EnGb);
|
||||
for w in ["enquiry", "enquire", "guerrilla", "guerrillas", "euthanise", "liveable", "liquorice", "veranda", "burka", "Anthea"] {
|
||||
assert!(is_correct_in(w, gb), "en-GB: {w}");
|
||||
assert!(is_correct_in(w, au), "en-AU: {w}");
|
||||
}
|
||||
for w in ["analyze", "catalyze", "color", "center", "defense", "zzyzxq"] {
|
||||
assert!(!is_correct_in(w, gb), "en-GB: {w}");
|
||||
assert!(!is_correct_in(w, au), "en-AU: {w}");
|
||||
}
|
||||
assert!(is_correct_in("billabong", au) && !is_correct_in("billabong", gb), "en-AU still has its own");
|
||||
assert!(suggest_in("billabon", 5, au).iter().any(|s| s == "billabong"), "{:?}", suggest_in("billabon", 5, au));
|
||||
}
|
||||
|
||||
/// Review follow-up: a run of spaces is one issue over the whole run, in every language with a
|
||||
/// double-space rule (a run of three used to leave two after the fix, four three).
|
||||
#[test]
|
||||
fn a_run_of_spaces_is_one_issue_in_every_language() {
|
||||
assert_eq!(space_runs("a b c d "), [3..5, 6..9, 10..14]);
|
||||
assert!(space_runs("a b c").is_empty() && space_runs("").is_empty() && space_runs(" ").is_empty());
|
||||
let langs = [
|
||||
ProofLang::En,
|
||||
ProofLang::EnGb,
|
||||
ProofLang::Es,
|
||||
ProofLang::It,
|
||||
ProofLang::Pt,
|
||||
ProofLang::Cs,
|
||||
ProofLang::Hr,
|
||||
ProofLang::Id,
|
||||
ProofLang::Pl,
|
||||
ProofLang::Ru,
|
||||
ProofLang::Tr,
|
||||
ProofLang::EnAu,
|
||||
ProofLang::EnCa,
|
||||
]
|
||||
.into_iter()
|
||||
.chain(PackLocale::new("pt-PT").map(ProofLang::Pack));
|
||||
for lang in langs {
|
||||
for n in 2..=5 {
|
||||
let text = format!("Aaa{}bbb", " ".repeat(n));
|
||||
let hits: Vec<_> =
|
||||
check_grammar_in(&text, lang).into_iter().filter(|i| i.start <= 3 && i.end > 3 && i.suggestions == [" "]).collect();
|
||||
assert_eq!(hits.len(), 1, "{lang:?} {n}: {hits:?}");
|
||||
assert_eq!((hits[0].start, hits[0].end), (3, 3 + n), "{lang:?} {n}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// English spelling reads SCOWL 2026 through the affix engine: one dictionary per English,
|
||||
/// its notice embedded, and its ordinals (`COMPOUNDRULE`) understood.
|
||||
#[cfg_attr(not(craft_proof_data), ignore = "needs the Language data (CRAFT_I18N_DIR)")]
|
||||
@@ -2513,7 +2655,8 @@ mod tests {
|
||||
assert_eq!(english_source(l), Some(EnglishSource::Embedded));
|
||||
assert_eq!(dictionary_load_error(l), None);
|
||||
}
|
||||
assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-AU"))).unwrap()), "en-AU reads en-GB");
|
||||
assert!(!std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::EnAu).unwrap()), "en-AU reads its own");
|
||||
assert!(std::ptr::eq(scowl::dict(ProofLang::EnGb).unwrap(), scowl::dict(ProofLang::from_tag(Some("en-NZ"))).unwrap()), "en-NZ reads en-GB");
|
||||
let notice = scowl::notice().unwrap();
|
||||
for credit in ["Kevin Atkinson", "Geoff Kuenning", "clause 4 removed", "Benjamin Titze"] {
|
||||
assert!(notice.contains(credit), "{credit}");
|
||||
|
||||
@@ -7,9 +7,10 @@
|
||||
use crate::hyphen::{self, Limits, Source};
|
||||
use crate::{EnglishSource, Options, ProofLang, dict, en_variants, patterns};
|
||||
|
||||
const LANGS: [ProofLang; 18] = [
|
||||
const LANGS: [ProofLang; 19] = [
|
||||
ProofLang::En,
|
||||
ProofLang::EnGb,
|
||||
ProofLang::EnAu,
|
||||
ProofLang::EnCa,
|
||||
ProofLang::Pl,
|
||||
ProofLang::Bg,
|
||||
@@ -45,9 +46,14 @@ const SAMPLES: [&str; 10] = [
|
||||
#[test]
|
||||
fn nothing_is_embedded() {
|
||||
assert!(crate::data::EN_US_DIC.is_empty());
|
||||
assert!(crate::data::EN_US_SPELLING.is_empty() && crate::data::EN_GB_SPELLING.is_empty() && crate::data::EN_CA_SPELLING.is_empty());
|
||||
assert!(
|
||||
crate::data::EN_US_SPELLING.is_empty()
|
||||
&& crate::data::EN_GB_SPELLING.is_empty()
|
||||
&& crate::data::EN_CA_SPELLING.is_empty()
|
||||
&& crate::data::EN_AU_SPELLING.is_empty()
|
||||
);
|
||||
assert!(crate::scowl::notice().is_none());
|
||||
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
|
||||
assert!(crate::scowl::dict(l).is_none() && crate::scowl::load_error(l).is_none());
|
||||
}
|
||||
assert!(dict::Dictionary::en_us().is_empty());
|
||||
@@ -63,7 +69,7 @@ fn nothing_is_embedded() {
|
||||
#[test]
|
||||
fn english_spelling_still_marks_misspellings() {
|
||||
crate::hold_dictionary_tests();
|
||||
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnAu, ProofLang::EnCa] {
|
||||
let source = crate::english_source(l);
|
||||
assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}");
|
||||
// Any English dictionary marks junk and knows `the`; the word-by-word results below are
|
||||
|
||||
+154
-17
@@ -6,6 +6,10 @@
|
||||
//! files); the one used least recently goes first, but never one used during the current
|
||||
//! [`in_one_check`] call, so a paragraph in many languages loads each dictionary once. Locales
|
||||
//! with no dictionary are remembered apart and take no room.
|
||||
//!
|
||||
//! A word none of whose letters is in a script the dictionary's words are written in (a Cyrillic
|
||||
//! word in a Dutch run) isn't this language's: it is never marked, as Word leaves a word in another
|
||||
//! script alone. A word in the dictionary's script is always checked, whatever its letters.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::path::PathBuf;
|
||||
@@ -20,7 +24,7 @@ use crate::lang::PackLocale;
|
||||
pub(crate) const MAX_LOADED_BYTES: u64 = 256 << 20;
|
||||
/// Estimated memory per byte of `.aff` and `.dic` file.
|
||||
const BYTES_PER_FILE_BYTE: u64 = 6;
|
||||
/// Most locales remembered as having no dictionary (the list is cleared when full).
|
||||
/// Most locales remembered as having no dictionary (the oldest is forgotten first when full).
|
||||
const MAX_MISSING: usize = 256;
|
||||
|
||||
/// The budget in force (tests lower it).
|
||||
@@ -36,6 +40,8 @@ thread_local! {
|
||||
struct Loaded {
|
||||
locale: PackLocale,
|
||||
dict: Arc<AffixDict>,
|
||||
/// The scripts its words are written in ([`scripts`]), sorted.
|
||||
scripts: Arc<[u32]>,
|
||||
/// The `.dic` it came from.
|
||||
file: Option<PathBuf>,
|
||||
/// Estimated memory.
|
||||
@@ -66,15 +72,88 @@ pub(crate) fn reload() {
|
||||
/// over the budget.
|
||||
pub(crate) fn in_one_check<R>(f: impl FnOnce() -> R) -> R {
|
||||
let outer = CHECK_FROM.with(Cell::get);
|
||||
if outer.is_none() {
|
||||
let _end = outer.is_none().then(|| {
|
||||
let now = CACHE.read().unwrap_or_else(PoisonError::into_inner).clock;
|
||||
CHECK_FROM.with(|c| c.set(Some(now.wrapping_add(1))));
|
||||
}
|
||||
let r = f();
|
||||
if outer.is_none() {
|
||||
EndCheck
|
||||
});
|
||||
f()
|
||||
}
|
||||
|
||||
/// Ends the outermost [`in_one_check`] when dropped: also when `f` panics (`Session::run` catches
|
||||
/// it), or every dictionary would stay pinned on this thread and the budget never hold again.
|
||||
struct EndCheck;
|
||||
|
||||
impl Drop for EndCheck {
|
||||
fn drop(&mut self) {
|
||||
CHECK_FROM.with(|c| c.set(None));
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
/// Remember that `locale` has no dictionary; when [`MAX_MISSING`] are remembered, the oldest is
|
||||
/// forgotten (clearing them all would look every other one up again).
|
||||
fn remember_missing(missing: &mut Vec<PackLocale>, locale: PackLocale) {
|
||||
if missing.len() >= MAX_MISSING {
|
||||
missing.remove(0);
|
||||
}
|
||||
missing.push(locale);
|
||||
}
|
||||
|
||||
/// The script of the letter `c`, as a number: the main alphabetic scripts by their Unicode blocks
|
||||
/// (Latin with its extensions, Greek, Cyrillic, Armenian, Hebrew, Arabic, Georgian, Hangul, kana,
|
||||
/// Han, Ethiopic); any other letter by its 128-character block. Only told apart, never named.
|
||||
fn script_of(c: char) -> u32 {
|
||||
const SCRIPTS: &[(u32, u32, u32)] = &[
|
||||
(0x0041, 0x024F, 1),
|
||||
(0x0250, 0x02AF, 1),
|
||||
(0x1E00, 0x1EFF, 1),
|
||||
(0x2C60, 0x2C7F, 1),
|
||||
(0xA720, 0xA7FF, 1),
|
||||
(0xAB30, 0xAB6F, 1),
|
||||
(0xFF21, 0xFF5A, 1),
|
||||
(0x0370, 0x03FF, 2),
|
||||
(0x1F00, 0x1FFF, 2),
|
||||
(0x0400, 0x052F, 3),
|
||||
(0x1C80, 0x1C8F, 3),
|
||||
(0x2DE0, 0x2DFF, 3),
|
||||
(0xA640, 0xA69F, 3),
|
||||
(0x0530, 0x058F, 4),
|
||||
(0xFB13, 0xFB17, 4),
|
||||
(0x0590, 0x05FF, 5),
|
||||
(0xFB1D, 0xFB4F, 5),
|
||||
(0x0600, 0x06FF, 6),
|
||||
(0x0750, 0x077F, 6),
|
||||
(0x08A0, 0x08FF, 6),
|
||||
(0xFB50, 0xFDFF, 6),
|
||||
(0xFE70, 0xFEFF, 6),
|
||||
(0x10A0, 0x10FF, 7),
|
||||
(0x1C90, 0x1CBF, 7),
|
||||
(0x2D00, 0x2D2F, 7),
|
||||
(0x1100, 0x11FF, 8),
|
||||
(0x3130, 0x318F, 8),
|
||||
(0xAC00, 0xD7AF, 8),
|
||||
(0x3040, 0x30FF, 9),
|
||||
(0x3400, 0x4DBF, 10),
|
||||
(0x4E00, 0x9FFF, 10),
|
||||
(0x1200, 0x139F, 11),
|
||||
];
|
||||
let u = u32::from(c);
|
||||
SCRIPTS.iter().find(|(lo, hi, _)| (*lo..=*hi).contains(&u)).map_or(0x100 + (u >> 7), |(_, _, s)| *s)
|
||||
}
|
||||
|
||||
/// The scripts the words of `d` are written in ([`script_of`] of their letters), sorted.
|
||||
fn scripts(d: &AffixDict) -> Arc<[u32]> {
|
||||
let mut seen = std::collections::BTreeSet::new();
|
||||
for c in d.stems().flat_map(str::chars).filter(|c| c.is_alphabetic()) {
|
||||
seen.insert(script_of(c));
|
||||
}
|
||||
seen.into_iter().collect()
|
||||
}
|
||||
|
||||
/// Is `word` in another script than the dictionary's (`scripts`): none of its letters is in one
|
||||
/// of them? A word in the dictionary's script is never, whatever its letters (`zzzz` in Dutch).
|
||||
fn other_script(word: &str, scripts: &[u32]) -> bool {
|
||||
!scripts.is_empty() && !word.chars().filter(|c| c.is_alphabetic()).any(|c| scripts.binary_search(&script_of(c)).is_ok())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -149,30 +228,34 @@ pub(crate) fn names(locale: PackLocale) -> Vec<String> {
|
||||
|
||||
/// `locale`'s dictionary, loaded on first use; an empty one when there is none.
|
||||
fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
|
||||
let (d, file, _) = get_with_scripts(locale);
|
||||
(d, file)
|
||||
}
|
||||
|
||||
/// [`get`], with the dictionary's [`scripts`].
|
||||
fn get_with_scripts(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>, Arc<[u32]>) {
|
||||
let reloads = {
|
||||
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
|
||||
c.clock = c.clock.wrapping_add(1);
|
||||
let now = c.clock;
|
||||
if let Some(l) = c.loaded.iter_mut().find(|l| l.locale == locale) {
|
||||
l.used = now;
|
||||
return (l.dict.clone(), l.file.clone());
|
||||
return (l.dict.clone(), l.file.clone(), l.scripts.clone());
|
||||
}
|
||||
if c.missing.contains(&locale) {
|
||||
return (Arc::default(), None);
|
||||
return (Arc::default(), None, Arc::from([]));
|
||||
}
|
||||
c.reloads
|
||||
};
|
||||
let (dict, file, bytes) = load(locale);
|
||||
let scripts = scripts(&dict);
|
||||
let mut c = CACHE.write().unwrap_or_else(PoisonError::into_inner);
|
||||
if c.reloads != reloads || c.loaded.iter().any(|l| l.locale == locale) || c.missing.contains(&locale) {
|
||||
return (dict, file);
|
||||
return (dict, file, scripts);
|
||||
}
|
||||
if dict.is_empty() {
|
||||
if c.missing.len() >= MAX_MISSING {
|
||||
c.missing.clear();
|
||||
}
|
||||
c.missing.push(locale);
|
||||
return (dict, file);
|
||||
remember_missing(&mut c.missing, locale);
|
||||
return (dict, file, scripts);
|
||||
}
|
||||
LOADS.fetch_add(1, Ordering::Relaxed);
|
||||
let budget = BUDGET.load(Ordering::Relaxed);
|
||||
@@ -187,8 +270,8 @@ fn get(locale: PackLocale) -> (Arc<AffixDict>, Option<PathBuf>) {
|
||||
total = total.saturating_sub(gone.bytes);
|
||||
}
|
||||
let used = c.clock;
|
||||
c.loaded.push(Loaded { locale, dict: dict.clone(), file: file.clone(), bytes, used });
|
||||
(dict, file)
|
||||
c.loaded.push(Loaded { locale, dict: dict.clone(), scripts: scripts.clone(), file: file.clone(), bytes, used });
|
||||
(dict, file, scripts)
|
||||
}
|
||||
|
||||
/// Load `locale`'s dictionary, with its estimated memory.
|
||||
@@ -229,11 +312,14 @@ pub(crate) fn load_error(locale: PackLocale) -> Option<String> {
|
||||
|
||||
/// Is `word` right in `locale`? Every word is when there is no dictionary.
|
||||
pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) -> bool {
|
||||
let (d, _) = get(locale);
|
||||
let (d, _, scripts) = get_with_scripts(locale);
|
||||
if d.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let Some(w) = crate::spelling::word_to_check_in(&d, word, opts) else { return true };
|
||||
if other_script(w, &scripts) {
|
||||
return true;
|
||||
}
|
||||
let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase());
|
||||
if w.contains('-') && !known(w) {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(p));
|
||||
@@ -264,6 +350,57 @@ mod tests {
|
||||
PackLocale::new(t).unwrap()
|
||||
}
|
||||
|
||||
/// Review follow-up (craft-libs#485): a full list of locales with no dictionary forgets its
|
||||
/// oldest one, not all of them, so a document in more languages doesn't look every one up again.
|
||||
#[test]
|
||||
fn missing_locales_forget_the_oldest_first() {
|
||||
let tag = |i: usize| l(&format!("q{}{}", (b'a' + (i / 26 % 26) as u8) as char, (b'a' + (i % 26) as u8) as char));
|
||||
let mut missing = Vec::new();
|
||||
for i in 0..MAX_MISSING + 10 {
|
||||
remember_missing(&mut missing, tag(i));
|
||||
assert!(missing.len() <= MAX_MISSING);
|
||||
}
|
||||
assert_eq!(missing.len(), MAX_MISSING);
|
||||
assert!(!missing.contains(&tag(0)) && !missing.contains(&tag(9)), "the oldest went first");
|
||||
assert!(missing.contains(&tag(10)) && missing.contains(&tag(MAX_MISSING + 9)), "the rest stay");
|
||||
}
|
||||
|
||||
/// Review follow-up (craft-libs#485): a panic inside a check (caught by the app) doesn't leave
|
||||
/// this thread's dictionaries pinned past the budget for good.
|
||||
#[test]
|
||||
fn a_panic_inside_a_check_ends_it() {
|
||||
let caught = std::panic::catch_unwind(|| in_one_check(|| panic!("a bug inside a check")));
|
||||
assert!(caught.is_err());
|
||||
assert_eq!(CHECK_FROM.with(Cell::get), None);
|
||||
in_one_check(|| {
|
||||
assert!(CHECK_FROM.with(Cell::get).is_some());
|
||||
in_one_check(|| ());
|
||||
assert!(CHECK_FROM.with(Cell::get).is_some(), "an inner check doesn't end the outer one");
|
||||
});
|
||||
assert_eq!(CHECK_FROM.with(Cell::get), None);
|
||||
}
|
||||
|
||||
/// Review follow-up (craft-libs#485): a word with no letter in the dictionary's scripts is
|
||||
/// in another script (a Cyrillic word in a Dutch run); a word in its script is always checked,
|
||||
/// even with letters no dictionary word uses (CodeRabbit on #496: `zzzz`).
|
||||
#[test]
|
||||
fn words_in_another_script_are_not_the_dictionarys() {
|
||||
let nl = AffixDict::from_hunspell("SET UTF-8\n", "3\nhuis\nkat\nÉén\n", &[]).unwrap();
|
||||
let latin = scripts(&nl);
|
||||
assert_eq!(latin.len(), 1);
|
||||
for w in ["привет", "Привет", "αβγ", "שלום", "日本"] {
|
||||
assert!(other_script(w, &latin), "{w}");
|
||||
}
|
||||
for w in ["huus", "kät", "huisп", "ÉÉN", "zzzz", "qxqw", "ŋʒ"] {
|
||||
assert!(!other_script(w, &latin), "{w}");
|
||||
}
|
||||
assert!(!other_script("привет", &[]), "no scripts: every word is checked");
|
||||
let ru = AffixDict::from_hunspell("SET UTF-8\n", "1\nдом\n", &[]).unwrap();
|
||||
assert!(!other_script("щщщ", &scripts(&ru)) && other_script("dom", &scripts(&ru)));
|
||||
assert_ne!(script_of('ა'), script_of('a'));
|
||||
assert_eq!(script_of('ă'), script_of('Ạ'));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn locales_and_file_names() {
|
||||
assert_eq!(l("nl-NL").as_str(), "nl-nl");
|
||||
|
||||
@@ -34,6 +34,13 @@ pub(crate) const SPELLING: &str = "spelling";
|
||||
const HYPHENATION: &str = "hyphenation.pat";
|
||||
/// Most languages whose pack patterns are remembered (a document may tag thousands).
|
||||
const MAX_HYPHENATION_LOOKUPS: usize = 256;
|
||||
/// The memory budget of the remembered pack patterns, in estimated bytes ([`pattern_bytes`]):
|
||||
/// a pack's patterns may be as large as a dictionary (64 MB of file), and up to
|
||||
/// [`MAX_HYPHENATION_LOOKUPS`] of them are kept, so they get a budget of their own, as the
|
||||
/// dictionaries do (`pack_lang::MAX_LOADED_BYTES`).
|
||||
const MAX_HYPHENATION_BYTES: u64 = 256 << 20;
|
||||
/// Estimated memory per pattern (its packed key, its values and the map's own room).
|
||||
const BYTES_PER_PATTERN: u64 = 64;
|
||||
/// A pack's description.
|
||||
const PACK_TOML: &str = "pack.toml";
|
||||
/// A pack's licence texts.
|
||||
@@ -80,15 +87,18 @@ struct State {
|
||||
dir: Option<PathBuf>,
|
||||
scan: Option<Arc<Scan>>,
|
||||
/// Each language's pack patterns once looked up (`None`: no pack has them), at most
|
||||
/// [`MAX_HYPHENATION_LOOKUPS`] languages (cleared when full).
|
||||
/// [`MAX_HYPHENATION_LOOKUPS`] languages and [`MAX_HYPHENATION_BYTES`] (cleared when full).
|
||||
hyphenation: BTreeMap<ProofLang, Option<Arc<Patterns>>>,
|
||||
/// The estimated memory of the patterns in [`State::hyphenation`].
|
||||
hyphenation_bytes: u64,
|
||||
/// (locale, reason) of pack hyphenation files that failed to load.
|
||||
hyphenation_errors: Vec<(String, String)>,
|
||||
/// Counts changes, so a scan or load that started before one is not kept.
|
||||
changes: u64,
|
||||
}
|
||||
|
||||
static STATE: RwLock<State> = RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_errors: Vec::new(), changes: 0 });
|
||||
static STATE: RwLock<State> =
|
||||
RwLock::new(State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 });
|
||||
|
||||
fn read() -> RwLockReadGuard<'static, State> {
|
||||
STATE.read().unwrap_or_else(PoisonError::into_inner)
|
||||
@@ -121,6 +131,7 @@ pub fn reload() {
|
||||
fn forget(s: &mut State) {
|
||||
s.scan = None;
|
||||
s.hyphenation.clear();
|
||||
s.hyphenation_bytes = 0;
|
||||
s.hyphenation_errors.clear();
|
||||
s.changes = s.changes.wrapping_add(1);
|
||||
}
|
||||
@@ -531,7 +542,7 @@ fn hyphenation_locales(lang: ProofLang) -> Vec<String> {
|
||||
ProofLang::EsUy => &["es-uy", "es-419", "es"],
|
||||
ProofLang::De => &["de", "de-1996", "de-de"],
|
||||
ProofLang::Pack(l) => return crate::pack_lang::hyphenation_locales(l),
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa | ProofLang::Other => &[],
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa | ProofLang::Other => &[],
|
||||
};
|
||||
names.iter().map(|n| (*n).to_string()).collect()
|
||||
}
|
||||
@@ -566,10 +577,8 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
|
||||
let mut s = write();
|
||||
if s.changes == changes && !s.hyphenation.contains_key(&lang) {
|
||||
// ponytail: cleared when full, an LRU if many languages each need their patterns.
|
||||
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS {
|
||||
s.hyphenation.clear();
|
||||
}
|
||||
s.hyphenation.insert(lang, found.clone());
|
||||
let bytes = found.as_deref().map_or(0, pattern_bytes);
|
||||
remember_patterns(&mut s, lang, found.clone(), bytes, MAX_HYPHENATION_BYTES);
|
||||
for e in errors {
|
||||
if !s.hyphenation_errors.contains(&e) {
|
||||
s.hyphenation_errors.push(e);
|
||||
@@ -579,6 +588,23 @@ pub(crate) fn hyphenation(lang: ProofLang) -> Option<Arc<Patterns>> {
|
||||
found
|
||||
}
|
||||
|
||||
/// Estimated memory of `p`.
|
||||
fn pattern_bytes(p: &Patterns) -> u64 {
|
||||
(p.len() as u64).saturating_add(p.exception_count() as u64).saturating_mul(BYTES_PER_PATTERN)
|
||||
}
|
||||
|
||||
/// Remember `lang`'s pack patterns (`bytes` of them): over [`MAX_HYPHENATION_LOOKUPS`] languages
|
||||
/// or `budget` bytes, the ones remembered so far are forgotten first. Patterns larger than the
|
||||
/// whole budget are still kept alone, or they would be read from disk for every word.
|
||||
fn remember_patterns(s: &mut State, lang: ProofLang, found: Option<Arc<Patterns>>, bytes: u64, budget: u64) {
|
||||
if s.hyphenation.len() >= MAX_HYPHENATION_LOOKUPS || s.hyphenation_bytes.saturating_add(bytes) > budget {
|
||||
s.hyphenation.clear();
|
||||
s.hyphenation_bytes = 0;
|
||||
}
|
||||
s.hyphenation.insert(lang, found);
|
||||
s.hyphenation_bytes = s.hyphenation_bytes.saturating_add(bytes);
|
||||
}
|
||||
|
||||
/// How many languages' pack patterns are remembered now.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn hyphenation_lookups() -> usize {
|
||||
@@ -600,6 +626,29 @@ fn load_patterns(path: &Path) -> Result<Patterns, String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Review follow-up (craft-libs#485): the remembered pack patterns have a memory budget, not
|
||||
/// only a count, so 256 languages of large patterns can't each stay in memory.
|
||||
#[test]
|
||||
fn remembered_patterns_keep_to_a_memory_budget() {
|
||||
let mut s = State { dir: None, scan: None, hyphenation: BTreeMap::new(), hyphenation_bytes: 0, hyphenation_errors: Vec::new(), changes: 0 };
|
||||
let p = Arc::new(Patterns::parse("hy3ph he2n hena4 hen5at 1na n2at 1tio 2io o2n"));
|
||||
let bytes = pattern_bytes(&p);
|
||||
assert_eq!(bytes, 9 * BYTES_PER_PATTERN);
|
||||
let lang = |t: &str| ProofLang::from_tag(Some(t));
|
||||
let budget = bytes * 2;
|
||||
remember_patterns(&mut s, lang("nl"), Some(p.clone()), bytes, budget);
|
||||
remember_patterns(&mut s, lang("fr"), Some(p.clone()), bytes, budget);
|
||||
remember_patterns(&mut s, lang("sv"), None, 0, budget);
|
||||
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (3, budget), "a language with no patterns takes no room");
|
||||
remember_patterns(&mut s, lang("da"), Some(p.clone()), bytes, budget);
|
||||
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, bytes), "over the budget: the others are forgotten");
|
||||
assert!(s.hyphenation.contains_key(&lang("da")));
|
||||
remember_patterns(&mut s, lang("fi"), Some(p.clone()), budget * 4, budget);
|
||||
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (1, budget * 4), "patterns over the whole budget are kept alone");
|
||||
forget(&mut s);
|
||||
assert_eq!((s.hyphenation.len(), s.hyphenation_bytes), (0, 0));
|
||||
}
|
||||
|
||||
/// Review of #485: a pack's name loses bidirectional and other format characters.
|
||||
#[test]
|
||||
fn pack_names_are_plain_text() {
|
||||
|
||||
@@ -304,11 +304,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Zbędna spacja przed znakiem interpunkcyjnym".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Podwójna spacja".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Podwójna spacja".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -62,7 +62,12 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
|
||||
if end == '.' && abbreviation_or_number(prev) {
|
||||
return false;
|
||||
}
|
||||
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
|
||||
let after_marks = t.trim_start_matches(['.', '!', '?']);
|
||||
let rest = after_marks.trim_start_matches(CLOSING);
|
||||
// `"Vai?" disse ele.` (not the object placeholder `\u{1}`): narration after a quoted question or exclamation goes on in lower case.
|
||||
if end != '.' && after_marks.trim_start_matches('\u{1}').starts_with(CLOSING) {
|
||||
return false;
|
||||
}
|
||||
// A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes
|
||||
// on in lower case.
|
||||
rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c))
|
||||
@@ -124,11 +129,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Espaço antes da pontuação".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Espaço duplo".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Espaço duplo".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -225,6 +227,14 @@ mod tests {
|
||||
// craft-libs#488 review: no space after the dash.
|
||||
clean("—Vai? —disse ele.");
|
||||
clean("— Vai? — disse ele.");
|
||||
// craft-libs#496 review: a closing quote after the mark.
|
||||
clean("\"Vai?\" disse ele.");
|
||||
clean("«Vai!» gritou ela.");
|
||||
assert!(!messages("Vai? \"sim\".").is_empty(), "a bare question still ends the sentence");
|
||||
// An inline object (`\u{1}`) before a real closing quote is skipped.
|
||||
clean("\"Vai?\u{1}\" disse ele.");
|
||||
// An inline object (`\u{1}`) is not a closing quote.
|
||||
assert!(!messages("Quem era?\u{1} era Marco.").is_empty());
|
||||
clean("Ele disse: \"Volto amanhã\". E saiu.");
|
||||
// A list item may start in lower case.
|
||||
clean("maçãs,");
|
||||
|
||||
@@ -503,11 +503,8 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Лишний пробел перед знаком препинания".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Двойной пробел".into(), vec![" ".into()]));
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(issue(run.start, run.end, "Двойной пробел".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,7 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(char::is_numeric)) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
if opts.ignore_uppercase && w.chars().all(|c| !c.is_lowercase()) && n <= ABBREVIATION_LETTERS {
|
||||
if opts.ignore_uppercase && crate::spelling::in_capitals(w) && n <= ABBREVIATION_LETTERS {
|
||||
return true;
|
||||
}
|
||||
// Camel case (`ВКонтакте`, `МирКрасок`): a brand or an identifier, not a dictionary word.
|
||||
@@ -321,7 +321,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
|
||||
}
|
||||
match lp.as_slice() {
|
||||
// `кое-кто`, `кой-какой`.
|
||||
[prefix, rest] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)),
|
||||
[prefix, _] if PRONOUN_PREFIXES.contains(prefix) => return parts.get(1).is_some_and(|r| known(d, r)),
|
||||
// `по-русски` (from `русский`), `по-новому`, `по-моему`.
|
||||
["по", rest] => {
|
||||
let Some(r) = parts.get(1) else { return false };
|
||||
@@ -333,7 +333,7 @@ fn hyphenated(d: &AffixDict, w: &str, opts: &crate::Options) -> bool {
|
||||
let last = parts.len() - 1;
|
||||
parts.iter().enumerate().all(|(i, p)| {
|
||||
// An abbreviation part (`ИИ-консультант`, `SMS-рассылка`) or a known word.
|
||||
if known(d, p) || (opts.ignore_uppercase && p.chars().all(|c| !c.is_lowercase()) && p.chars().count() <= ABBREVIATION_LETTERS) {
|
||||
if known(d, p) || (opts.ignore_uppercase && crate::spelling::capitals_part(p, ABBREVIATION_LETTERS)) {
|
||||
return true;
|
||||
}
|
||||
// A first part with a linking vowel: the stem (`северо` → `север`) or its adjective
|
||||
@@ -485,5 +485,7 @@ mod tests {
|
||||
assert!(!crate::is_correct_in_with("м²", lang, &check_digits), "a superscript is a number too");
|
||||
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
|
||||
assert!(is_correct("ПРЕВЕТ"), "direct checks keep the defaults");
|
||||
// The cased-letter rule (craft-libs#485): capitals and letterless parts are abbreviations.
|
||||
assert!(crate::is_correct_in_with("ИИ-2", lang, &check_digits) && crate::is_correct_in_with("ТУ-154М", lang, &check_digits));
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,15 @@
|
||||
//! dictionaries of craft-i18n's `language-data/en-us`, `en-gb` (British `-ise`) and `en-ca`,
|
||||
//! embedded unmodified at build time and read by the affix engine ([`crate::affix`]) on first use.
|
||||
//!
|
||||
//! Each English has its own: American English reads en-US, the British-spelling Englishes
|
||||
//! (en-GB, en-AU, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. Its suggestions
|
||||
//! Each English has its own: American English reads en-US, Australian English en-AU (SCOWL's
|
||||
//! Australian words: `billabong`, `brumby`; and `program`, `inquiry`), the other British-spelling
|
||||
//! Englishes (en-GB, en-NZ, en-IE…) read en-GB and Canadian English reads en-CA. SCOWL has no
|
||||
//! en-NZ: New Zealand English keeps `programme` and has few words of its own in SCOWL's en-AU
|
||||
//! data, so it stays on en-GB. Its suggestions
|
||||
//! come from its own dictionary, and a word is right when its own dictionary knows it: the other
|
||||
//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's). SCOWL
|
||||
//! dialects' dictionaries are never asked (en-US would take `colourblind` from en-GB's), except
|
||||
//! that en-AU's list keeps one spelling per word, so Australian English also takes en-GB's words
|
||||
//! (`enquiry`; its suggestions still come from en-AU). SCOWL
|
||||
//! picks one spelling per dialect (en-GB `-ise` has no `organize`, en-US no `judgement`), and Word
|
||||
//! accepts more, so British English also takes the `-ize` spellings of its `-ise` words that
|
||||
//! en-US or en-CA spells so, and `en-variants.txt` widens each dialect by its pairs
|
||||
@@ -19,16 +24,17 @@ use std::sync::OnceLock;
|
||||
|
||||
use crate::ProofLang;
|
||||
use crate::affix::AffixDict;
|
||||
use crate::data::{EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE};
|
||||
use crate::data::{EN_AU_AFF, EN_AU_SPELLING, EN_CA_AFF, EN_CA_SPELLING, EN_GB_AFF, EN_GB_SPELLING, EN_US_AFF, EN_US_SPELLING, SCOWL_NOTICE};
|
||||
|
||||
/// Per English (American, British, Canadian): the loaded dictionary, or why it isn't there.
|
||||
static DICTS: [OnceLock<Result<AffixDict, String>>; 3] = [OnceLock::new(), OnceLock::new(), OnceLock::new()];
|
||||
/// Per English (American, British, Canadian, Australian): the loaded dictionary, or why it isn't there.
|
||||
static DICTS: [OnceLock<Result<AffixDict, String>>; 4] = [OnceLock::new(), OnceLock::new(), OnceLock::new(), OnceLock::new()];
|
||||
|
||||
/// The slot, `.aff` and `.dic` text of `lang`'s dictionary.
|
||||
fn source(lang: ProofLang) -> (usize, &'static str, &'static str) {
|
||||
match lang {
|
||||
ProofLang::EnGb => (1, EN_GB_AFF, EN_GB_SPELLING),
|
||||
ProofLang::EnCa => (2, EN_CA_AFF, EN_CA_SPELLING),
|
||||
ProofLang::EnAu => (3, EN_AU_AFF, EN_AU_SPELLING),
|
||||
_ => (0, EN_US_AFF, EN_US_SPELLING),
|
||||
}
|
||||
}
|
||||
@@ -50,15 +56,17 @@ pub(crate) fn dict(lang: ProofLang) -> Option<&'static AffixDict> {
|
||||
loaded.as_ref().ok()
|
||||
}
|
||||
|
||||
/// Does `lang` read the British `-ise` dictionary (en-GB, en-AU, en-NZ…)?
|
||||
/// Does `lang` read a British `-ise` dictionary (en-GB, and en-NZ… through it; en-AU)?
|
||||
pub(crate) fn is_british(lang: ProofLang) -> bool {
|
||||
source(lang).0 == 1
|
||||
matches!(source(lang).0, 1 | 3)
|
||||
}
|
||||
|
||||
/// Why `lang`'s embedded SCOWL dictionary failed to load, once something tried to load it.
|
||||
pub(crate) fn load_error(lang: ProofLang) -> Option<String> {
|
||||
let (i, ..) = source(lang);
|
||||
DICTS.get(i)?.get()?.as_ref().err().map(|e| format!("embedded SCOWL dictionary: {e}"))
|
||||
let own = DICTS.get(i).and_then(|d| d.get()).and_then(|r| r.as_ref().err()).map(|e| format!("embedded SCOWL dictionary: {e}"));
|
||||
// Australian English also reads the en-GB list (`crate::english`): its failure is reported too.
|
||||
own.or_else(|| if lang == ProofLang::EnAu { load_error(ProofLang::EnGb).map(|e| format!("{e} (en-GB, also read for en-AU)")) } else { None })
|
||||
}
|
||||
|
||||
/// The SCOWL copyright and licence notices (the upstream README, with the Ispell BSD licence of
|
||||
|
||||
@@ -32,17 +32,43 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options)
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
return None;
|
||||
}
|
||||
// Only a word with a cased letter can be in capitals: Hebrew, Arabic, Thai… words have none.
|
||||
if opts.ignore_uppercase && w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
if opts.ignore_uppercase && in_capitals(w) && n <= 6 {
|
||||
return None;
|
||||
}
|
||||
Some(w)
|
||||
}
|
||||
|
||||
/// Is `w` in capitals: a cased letter and no lower-case one? Only a word with a cased letter can
|
||||
/// be: Hebrew, Arabic, Thai, Chinese… words have none, so they are never taken for acronyms.
|
||||
pub(crate) fn in_capitals(w: &str) -> bool {
|
||||
w.chars().any(char::is_uppercase) && w.chars().all(|c| !c.is_lowercase())
|
||||
}
|
||||
|
||||
/// Is `part` of a hyphenated word an abbreviation Ignore words in UPPERCASE skips: at most `max`
|
||||
/// characters, [`in_capitals`] or without letters (`SMS` in `SMS-ка`, `19` in `COVID-19`)? A
|
||||
/// part in a caseless script is no abbreviation.
|
||||
pub(crate) fn capitals_part(part: &str, max: usize) -> bool {
|
||||
part.chars().count() <= max && (in_capitals(part) || !part.chars().any(char::is_alphabetic))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// craft-libs#485's cased-letter rule, shared by every language: a word or part is in
|
||||
/// capitals only with a cased letter; a part without letters counts as an abbreviation.
|
||||
#[test]
|
||||
fn capitals_need_a_cased_letter() {
|
||||
for w in ["PKO", "SMS", "ÉTÉ", "ВКП", "B2B"] {
|
||||
assert!(in_capitals(w) && capitals_part(w, 6), "{w}");
|
||||
}
|
||||
for w in ["日本", "שלום", "ไทย", "Pko", "pko", "123", ""] {
|
||||
assert!(!in_capitals(w), "{w}");
|
||||
}
|
||||
assert!(capitals_part("19", 6) && capitals_part("", 6));
|
||||
assert!(!capitals_part("日本", 6) && !capitals_part("ไทย", 6) && !capitals_part("ABCDEFG", 6));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn words_needing_no_lookup() {
|
||||
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
|
||||
|
||||
@@ -209,23 +209,17 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
|
||||
});
|
||||
}
|
||||
}
|
||||
// Double space
|
||||
if c == ' ' && i + 1 < chars.len() && chars[i + 1].1 == ' ' && (i == 0 || chars[i - 1].1 != ' ') {
|
||||
let mut end = idx + 1;
|
||||
while end < text.len() && text.as_bytes().get(end) == Some(&b' ') {
|
||||
end += 1;
|
||||
}
|
||||
v.push(Issue {
|
||||
start: idx,
|
||||
end,
|
||||
kind: IssueKind::Grammar,
|
||||
message: "Fazladan boşluk".into(),
|
||||
suggestions: vec![" ".into()],
|
||||
lang: ProofLang::Tr,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for run in crate::space_runs(text) {
|
||||
v.push(Issue {
|
||||
start: run.start,
|
||||
end: run.end,
|
||||
kind: IssueKind::Grammar,
|
||||
message: "Fazladan boşluk".into(),
|
||||
suggestions: vec![" ".into()],
|
||||
lang: ProofLang::Tr,
|
||||
});
|
||||
}
|
||||
// 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`)
|
||||
for (i, &(w_start, w_end)) in ws.iter().enumerate() {
|
||||
let word = match text.get(w_start..w_end) {
|
||||
|
||||
@@ -97,8 +97,8 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Detached suffix check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
|
||||
for suffix in ["de", "da", "te", "ta", "mi", "mı", "mu", "mü"] {
|
||||
// 3. Detached particle check: `-de`, `-da`, `-mi`, `-mı`, `-mu`, `-mü`
|
||||
for suffix in ["de", "da", "mi", "mı", "mu", "mü"] {
|
||||
if lower.ends_with(suffix) && lower.len() > suffix.len() + 1 {
|
||||
let stem = &lower[..lower.len() - suffix.len()];
|
||||
if d.is_empty() || d.contains(stem) {
|
||||
@@ -333,6 +333,19 @@ mod tests {
|
||||
assert!(sugs.contains(&"geldin mi".to_string()));
|
||||
}
|
||||
|
||||
/// craft-libs#496 review: the particle is `de`/`da` whatever consonant comes before it. `te`
|
||||
/// and `ta` are only the locative suffix after a voiceless consonant (`kitapta`), written
|
||||
/// attached, so `kitapte` is never offered as `kitap te`.
|
||||
#[test]
|
||||
fn the_detached_particle_is_never_te_or_ta() {
|
||||
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nsonra\n").unwrap();
|
||||
assert!(suggest_with(&d, "kitapde", 8).contains(&"kitap de".to_string()));
|
||||
for w in ["kitapte", "kitapta"] {
|
||||
let sugs = suggest_with(&d, w, 8);
|
||||
assert!(!sugs.iter().any(|s| s == "kitap te" || s == "kitap ta"), "{w}: {sugs:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlong_words_get_no_suggestions_promptly() {
|
||||
let d = TrDict::from_hunspell("SET UTF-8\nSFX 1 Y 1\nSFX 1 0 lar .\n", "2\nkitap/1\nsonra\n").unwrap();
|
||||
|
||||
Reference in new issue
Block a user