mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
Italian proofing: it_IT and it_CH spelling, suggestions, grammar and hyphenation (storytold/wordcraft#951) (storytold/wordcraft#1090)
* Italian proofing: it-IT spelling, suggestions and grammar (storytold/wordcraft#951) * Italian proofing: Review Language menu and editing language * Italian proofing: interface test for editing language * Italian proofing: find the Swiss it_CH dictionary too * Italian proofing: TeX hyphenation patterns (MIT option) --------- Co-authored-by: Brandon Thomas <echelon@gmail.com>
This commit is contained in:
RegalDragoon200andBrandon Thomas authored and 
GitHub committed
1 parent
8caa51af04
commit
77e550ea0c
7 files changed
+1130
-4
No files matched your search
@@ -33,6 +33,9 @@ static EN_US_PAT: &str = include_str!("../../../assets/hyphenation/en-us.pat");
|
||||
static PL_PAT: &str = include_str!("../../../assets/hyphenation/pl.pat");
|
||||
/// hyph-utf8's `hyph-pt.tex`, unmodified (its BSD 3-clause header is in the file).
|
||||
static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat");
|
||||
/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose
|
||||
/// notice is in the file's header).
|
||||
static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat");
|
||||
|
||||
impl Dictionary {
|
||||
/// The bundled US English dictionary (decoded on first use).
|
||||
@@ -68,6 +71,12 @@ impl Patterns {
|
||||
static P: OnceLock<Patterns> = OnceLock::new();
|
||||
P.get_or_init(|| Patterns::parse(PT_PAT))
|
||||
}
|
||||
|
||||
/// The bundled Italian patterns (parsed on first use; the file has no exception words).
|
||||
pub fn it() -> &'static Patterns {
|
||||
static P: OnceLock<Patterns> = OnceLock::new();
|
||||
P.get_or_init(|| Patterns::parse(IT_PAT))
|
||||
}
|
||||
}
|
||||
|
||||
/// Hyphenation limits from the paragraph's Hyphenation settings.
|
||||
@@ -121,6 +130,9 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
if lang == ProofLang::Pt {
|
||||
return core_breaks_pt(core);
|
||||
}
|
||||
if lang == ProofLang::It {
|
||||
return core_breaks_it(core);
|
||||
}
|
||||
let dict = Dictionary::en_us();
|
||||
let s: String = core.iter().collect();
|
||||
let from_mask = |m: u64, len: usize| (1..len.min(64)).filter(|&i| m & (1 << i) != 0).collect::<Vec<_>>();
|
||||
@@ -175,6 +187,22 @@ fn core_breaks_pt(core: &[char]) -> (Vec<usize>, Source) {
|
||||
(heuristic_points(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words),
|
||||
/// then the generic heuristic.
|
||||
fn core_breaks_it(core: &[char]) -> (Vec<usize>, Source) {
|
||||
let pats = Patterns::it();
|
||||
let s: String = core.iter().collect();
|
||||
if let Some(p) = pats.exception(&s) {
|
||||
return (p.to_vec(), Source::Dictionary);
|
||||
}
|
||||
if !pats.is_empty()
|
||||
&& let Some(p) = pats.points(core)
|
||||
{
|
||||
return (p, Source::Patterns);
|
||||
}
|
||||
(heuristic_points(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// Unrestricted break points of an English `word` ([`word_breaks_in`] with [`ProofLang::En`]).
|
||||
pub fn word_breaks(word: &str) -> (Vec<usize>, Source) {
|
||||
word_breaks_in(word, ProofLang::En)
|
||||
@@ -706,6 +734,44 @@ mod tests {
|
||||
hyphenate_word_in(w, &Limits::default(), ProofLang::Pt)
|
||||
}
|
||||
|
||||
fn it(w: &str) -> String {
|
||||
hyphenate_word_in(w, &Limits::default(), ProofLang::It)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_patterns_load() {
|
||||
let p = Patterns::it();
|
||||
assert!(p.len() > 300, "{}", p.len());
|
||||
// The file has no exception words.
|
||||
assert_eq!(p.exception_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_words_hyphenate_by_italian_rules() {
|
||||
for (w, want) in [
|
||||
("famiglia", "fa-mi-glia"),
|
||||
("finestra", "fi-ne-stra"),
|
||||
("bicicletta", "bi-ci-clet-ta"),
|
||||
("passeggiata", "pas-seg-gia-ta"),
|
||||
("caffè", "caf-fè"),
|
||||
("tavolo", "ta-vo-lo"),
|
||||
("bambino", "bam-bi-no"),
|
||||
("scuola", "scuo-la"),
|
||||
("notte", "not-te"),
|
||||
("giorno", "gior-no"),
|
||||
("acqua", "ac-qua"),
|
||||
// Under five letters (`Limits::min_word`): no break.
|
||||
("casa", "casa"),
|
||||
("pane", "pane"),
|
||||
("libro", "li-bro"),
|
||||
("re", "re"),
|
||||
("più", "più"),
|
||||
] {
|
||||
assert_eq!(it(w), want, "{w}");
|
||||
}
|
||||
assert_eq!(word_breaks_in("famiglia", ProofLang::It).1, Source::Patterns);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn portuguese_patterns_load() {
|
||||
let p = Patterns::pt();
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
//! Italian grammar and punctuation checks: cheap, rule-based and tuned to stay quiet when
|
||||
//! unsure (a missed mistake costs less than a wrong mark on correct text). Messages are in
|
||||
//! Italian.
|
||||
//!
|
||||
//! - **Repeated word** (`la la`), except words Italian doubles on purpose (`piano piano`,
|
||||
//! `sì sì`, interjections).
|
||||
//! - **Space before punctuation** (`bene ,`) and **double spaces**; emoticons (` :)`) and
|
||||
//! ellipses are left alone.
|
||||
//! - **Capital letter at the start of a sentence**, knowing Italian abbreviations (`ecc.`,
|
||||
//! `dott.`, `prof.`, `sig.`), initials, ordinal numbers (`1. posto`) and ellipses.
|
||||
//! - **Doubled punctuation** (`,,`, `..`, `;;`).
|
||||
//!
|
||||
//! No comma rule and no English rule (such as `a`/`an`) applies to Italian text.
|
||||
|
||||
use crate::{Issue, IssueKind, ProofLang, words};
|
||||
|
||||
/// Words Italian repeats on purpose: emphatic doubling, interjections, onomatopoeia.
|
||||
const ALLOWED_REPEATS: &[&str] = &[
|
||||
"sì", "no", "piano", "bene", "male", "ecco", "oh", "ah", "eh", "mh", "bah", "ehi", "urrà", "evviva", "ciao", "addio", "adagio", "forte", "lento",
|
||||
"chiaro",
|
||||
];
|
||||
|
||||
/// Abbreviations written with a final period after which a sentence goes on in lower case
|
||||
/// (lower case, without the period).
|
||||
const ABBREVIATIONS: &[&str] = &[
|
||||
"ecc",
|
||||
"es",
|
||||
"dott",
|
||||
"prof",
|
||||
"profssa",
|
||||
"ing",
|
||||
"avv",
|
||||
"sig",
|
||||
"sigg",
|
||||
"sigra",
|
||||
"sigr",
|
||||
"geom",
|
||||
"rag",
|
||||
"dottssa",
|
||||
"avvoc",
|
||||
"pag",
|
||||
"pagg",
|
||||
"tel",
|
||||
"art",
|
||||
"artt",
|
||||
"fig",
|
||||
"figg",
|
||||
"vol",
|
||||
"voll",
|
||||
"ed",
|
||||
"sez",
|
||||
"cap",
|
||||
"capp",
|
||||
"par",
|
||||
"n",
|
||||
"num",
|
||||
"vs",
|
||||
"cfr",
|
||||
"cfrt",
|
||||
"op",
|
||||
"cit",
|
||||
"ibid",
|
||||
"sez",
|
||||
"all",
|
||||
"tav",
|
||||
"pag",
|
||||
"ditta",
|
||||
"soc",
|
||||
"spa",
|
||||
"srl",
|
||||
"on",
|
||||
"sen",
|
||||
"spett",
|
||||
"eg",
|
||||
"ca",
|
||||
"v",
|
||||
"vv",
|
||||
"sec",
|
||||
"min",
|
||||
"max",
|
||||
"lun",
|
||||
"mar",
|
||||
"mer",
|
||||
"gio",
|
||||
"ven",
|
||||
"sab",
|
||||
"dom",
|
||||
"gen",
|
||||
"feb",
|
||||
"mar",
|
||||
"apr",
|
||||
"mag",
|
||||
"giu",
|
||||
"lug",
|
||||
"ago",
|
||||
"set",
|
||||
"ott",
|
||||
"nov",
|
||||
"dic",
|
||||
"genn",
|
||||
"febb",
|
||||
"sett",
|
||||
"dottor",
|
||||
"professor",
|
||||
"cav",
|
||||
"commend",
|
||||
"mons",
|
||||
"fr",
|
||||
"suor",
|
||||
"s",
|
||||
"ss",
|
||||
"b",
|
||||
"beato",
|
||||
];
|
||||
|
||||
fn issue(start: usize, end: usize, message: String, suggestions: Vec<String>) -> Issue {
|
||||
Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::It }
|
||||
}
|
||||
|
||||
/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after
|
||||
/// which a lower-case word doesn't start a sentence)?
|
||||
fn abbreviation_or_number(prev: &str) -> bool {
|
||||
prev.chars().count() <= 1
|
||||
|| prev.chars().any(|c| c.is_ascii_digit())
|
||||
|| (prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')))
|
||||
|| ABBREVIATIONS.contains(&prev.to_lowercase().as_str())
|
||||
}
|
||||
|
||||
/// Does the gap between two words end a sentence: a `.`, `!` or `?` (not an ellipsis, not after
|
||||
/// an abbreviation), then space, with nothing but quotes and brackets around?
|
||||
fn ends_sentence(gap: &str, prev: &str) -> bool {
|
||||
let t = gap.trim_start_matches(['"', '”', '’', '\'', ')', '»', ']', '\u{1}']);
|
||||
let Some(end) = t.chars().next() else { return false };
|
||||
if !matches!(end, '.' | '!' | '?') || t.starts_with("..") {
|
||||
return false;
|
||||
}
|
||||
if end == '.' && abbreviation_or_number(prev) {
|
||||
return false;
|
||||
}
|
||||
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(['"', '”', '’', '\'', ')', '»', ']']);
|
||||
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| matches!(c, '"' | '“' | '«' | '(' | '[' | '\'' | '—'))
|
||||
}
|
||||
|
||||
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
|
||||
for pair in ws.windows(2) {
|
||||
let [(a0, b0), (a1, b1)] = pair else { continue };
|
||||
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
|
||||
if !gap.is_empty()
|
||||
&& gap.chars().all(char::is_whitespace)
|
||||
&& w0.chars().all(char::is_alphabetic)
|
||||
&& w0.to_lowercase() == w1.to_lowercase()
|
||||
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
|
||||
{
|
||||
v.push(issue(*a0, *b1, format!("Parola ripetuta: \"{w1}\""), vec![w0.to_string()]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
for (i, c) in text.char_indices() {
|
||||
if c != ' ' {
|
||||
continue;
|
||||
}
|
||||
let rest = text.get(i + 1..).unwrap_or("");
|
||||
let mut it = rest.chars();
|
||||
let Some(p) = it.next() else { continue };
|
||||
let after = it.next();
|
||||
let flag = match p {
|
||||
',' => true,
|
||||
// `;)` and `;-)` are emoticons.
|
||||
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
v.push(issue(i, i + 1 + p.len_utf8(), "Spazio prima della punteggiatura".into(), vec![p.to_string()]));
|
||||
}
|
||||
}
|
||||
for (i, _) in text.match_indices(" ") {
|
||||
if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) {
|
||||
continue;
|
||||
}
|
||||
v.push(issue(i, i + 2, "Spazio doppio".into(), vec![" ".into()]));
|
||||
}
|
||||
}
|
||||
|
||||
fn sentence_capitals(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
|
||||
let mut after_end = true;
|
||||
for (a, b) in ws {
|
||||
let Some(w) = text.get(*a..*b) else { continue };
|
||||
if after_end && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
|
||||
let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default();
|
||||
v.push(issue(*a, *b, "Maiuscola a inizio frase".into(), vec![fixed]));
|
||||
}
|
||||
after_end = ends_sentence(text.get(*b..).unwrap_or(""), w);
|
||||
}
|
||||
}
|
||||
|
||||
fn doubled_punctuation(text: &str, v: &mut Vec<Issue>) {
|
||||
for pat in [",,", "..", ";;", "::", "??", "!!"] {
|
||||
for (i, _) in text.match_indices(pat) {
|
||||
if pat == ".." && text.get(i + 2..).is_some_and(|r| r.starts_with('.')) {
|
||||
continue; // ellipsis
|
||||
}
|
||||
v.push(issue(i, i + pat.len(), "Punteggiatura doppia".into(), vec![pat[..1].into()]));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Grammar issues in a paragraph of Italian text, in order.
|
||||
pub fn check(text: &str) -> Vec<Issue> {
|
||||
let ws: Vec<(usize, usize)> = words(text);
|
||||
let mut v = Vec::new();
|
||||
repeated_words(text, &ws, &mut v);
|
||||
spaces(text, &mut v);
|
||||
sentence_capitals(text, &ws, &mut v);
|
||||
doubled_punctuation(text, &mut v);
|
||||
v.sort_by_key(|i| (i.start, i.end));
|
||||
v
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn italian_grammar_flags() {
|
||||
test_dict_install();
|
||||
let t = "questo questo è un test , fatto male.. Vedi dott. rossi";
|
||||
let v = check(t);
|
||||
let msgs: Vec<&str> = v.iter().map(|i| i.message.as_str()).collect();
|
||||
assert!(msgs.iter().any(|m| m.contains("ripetuta")), "{msgs:?}");
|
||||
assert!(msgs.iter().any(|m| m.contains("prima della punteggiatura")), "{msgs:?}");
|
||||
assert!(msgs.iter().any(|m| m.contains("doppio")), "{msgs:?}");
|
||||
assert!(msgs.iter().any(|m| m.contains("doppia")), "{msgs:?}");
|
||||
assert!(v.iter().all(|i| i.lang == ProofLang::It), "{v:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_grammar_stays_quiet() {
|
||||
test_dict_install();
|
||||
assert!(
|
||||
check("Vado a Roma. Il dott. Rossi è già lì... piano piano!").is_empty(),
|
||||
"{:?}",
|
||||
check("Vado a Roma. Il dott. Rossi è già lì... piano piano!")
|
||||
);
|
||||
assert!(check("Sì sì, bene bene :)").is_empty());
|
||||
}
|
||||
|
||||
fn test_dict_install() {
|
||||
crate::it::test_dict::install();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
//! Italian proofing.
|
||||
//!
|
||||
//! - **Spelling** against the Hunspell `it_IT` dictionary installed on the system (found by
|
||||
//! [`crate::hunspell`]: `it_IT.aff`/`it_IT.dic` from the OS, a package manager or
|
||||
//! LibreOffice; it isn't bundled — the upstream file is GPL-licensed). Without it Italian
|
||||
//! text gets no spelling marks. Elided forms check the word after the apostrophe
|
||||
//! (`l'amico`, `un'idea`, `c'è`); hyphenated compounds are checked part by part; the user
|
||||
//! dictionary applies as in English.
|
||||
//! - **Suggestions** tuned for Italian ([`suggest`]): missing accents first (`caffe` → `caffè`,
|
||||
//! `perche` → `perché`), then the dictionary's `REP` confusions, a missing space, one-letter
|
||||
//! typos, and a dictionary scan for anything further off.
|
||||
//! - **Grammar** ([`grammar`]): cheap, reliable rules with Italian messages.
|
||||
|
||||
pub mod grammar;
|
||||
mod suggest;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_dict;
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Italian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
|
||||
/// File names of an Italian Hunspell dictionary (Italy first, then Switzerland).
|
||||
const NAMES: &[&str] = &["it_IT", "it_CH", "it-IT", "it"];
|
||||
|
||||
/// The Italian dictionary: the installed Hunspell `it_IT` (read on first use), one given with
|
||||
/// [`install_dictionary`], or an empty one when none is available (Italian spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
let Some((aff, dic)) = crate::hunspell::find(NAMES) else {
|
||||
log::info!("Italian spelling: no Hunspell it_IT dictionary installed");
|
||||
return AffixDict::default();
|
||||
};
|
||||
crate::hunspell::load(&aff, &dic).unwrap_or_else(|e| {
|
||||
log::warn!("Italian spelling: {e}");
|
||||
AffixDict::default()
|
||||
})
|
||||
}
|
||||
|
||||
/// Use this Italian dictionary from now on (Hunspell `.aff` and `.dic` text, decoded to
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
}
|
||||
|
||||
/// Is an Italian dictionary available (spelling is checked)?
|
||||
pub fn has_dictionary() -> bool {
|
||||
!dictionary().is_empty()
|
||||
}
|
||||
|
||||
/// Is `c` a letter of the Latin script (the only one the Italian dictionary covers)?
|
||||
fn is_latin_letter(c: char) -> bool {
|
||||
c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c))
|
||||
}
|
||||
|
||||
/// Proclitics (and verb clippings) that elide before a vowel or `h`: the word after the
|
||||
/// apostrophe is what gets checked (`l'amico`, `un'idea`, `c'è`, `m'hai`).
|
||||
const PROCLITICS: &[&str] = &[
|
||||
"l", "un", "una", "uno", "all", "dall", "dell", "nell", "sull", "coll", "pell", "c", "m", "t", "s", "v", "d", "j", "quest", "quell", "bell",
|
||||
"sant", "grand", "poc",
|
||||
];
|
||||
|
||||
/// Is `word` spelled correctly in Italian? Numbers, single letters, short words in capitals
|
||||
/// (acronyms: `RAI`, `TG`), words with digits and words in other scripts are accepted, and so
|
||||
/// is every word when no Italian dictionary is installed.
|
||||
pub fn is_correct(word: &str) -> bool {
|
||||
is_correct_with(&dictionary(), word)
|
||||
}
|
||||
|
||||
/// [`is_correct`] against a given dictionary (an empty one accepts everything).
|
||||
pub fn is_correct_with(d: &AffixDict, word: &str) -> bool {
|
||||
if d.is_empty() {
|
||||
return true; // no Italian dictionary installed: no spelling marks
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
if w.chars().all(|c| !c.is_lowercase()) && n <= 6 {
|
||||
return true;
|
||||
}
|
||||
if !w.chars().any(is_latin_letter) {
|
||||
return true; // other scripts: no dictionary
|
||||
}
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
if let Some((head, tail)) = w.split_once(['\'', '’'])
|
||||
&& PROCLITICS.contains(&head.to_lowercase().as_str())
|
||||
&& !tail.is_empty()
|
||||
{
|
||||
return known(d, tail);
|
||||
}
|
||||
known(d, w)
|
||||
}
|
||||
|
||||
/// Is one word (no hyphens, no apostrophe) in the dictionary or the user dictionary?
|
||||
fn known(d: &AffixDict, w: &str) -> bool {
|
||||
d.check(w) || crate::in_user_dictionary(&w.to_lowercase())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_dictionary_loads() {
|
||||
test_dict::install();
|
||||
let d = dictionary();
|
||||
assert!(has_dictionary());
|
||||
assert!(d.len() > 50, "{}", d.len());
|
||||
assert!(d.rule_count() >= 4, "the test dictionary's affix rules");
|
||||
assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn swiss_dictionary_files_are_found() {
|
||||
// it_CH is searched too, so Swiss users with only it_CH installed get Italian spelling.
|
||||
let dir = std::env::temp_dir().join(format!("wc-it-ch-{}", std::process::id()));
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
std::fs::write(dir.join("it_CH.aff"), test_dict::AFF).expect("aff");
|
||||
std::fs::write(dir.join("it_CH.dic"), test_dict::DIC).expect("dic");
|
||||
let found = crate::hunspell::find_in(&dir, NAMES);
|
||||
let ok = found.as_ref().is_some_and(|(a, d)| a.ends_with("it_CH.aff") && d.ends_with("it_CH.dic"));
|
||||
assert!(ok, "{found:?}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_dictionary_nothing_is_flagged() {
|
||||
let none = AffixDict::default();
|
||||
for w in ["caffè", "perché", "xqzv", ""] {
|
||||
assert!(is_correct_with(&none, w), "{w}");
|
||||
}
|
||||
assert!(install_dictionary("FLAG long\n", "1\nkot\n").is_err(), "unsupported files are refused");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_words_are_correct() {
|
||||
test_dict::install();
|
||||
for w in [
|
||||
"casa", "case", "libro", "libri", "città", "caffè", "perché", "più", "può", "parlare", "parla", "parlano", "rifare", "bello", "belli",
|
||||
"nuovo", "bianco", "Roma", "Milano", "l'amico", "un'idea", "L'amico", "RAI", "2026", "e", "non",
|
||||
] {
|
||||
assert!(is_correct(w), "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn misspellings_are_flagged() {
|
||||
test_dict::install();
|
||||
for w in ["caffé", "perche", "lbiro", "parlre", "amicaa", "citta", "quelo"] {
|
||||
assert!(!is_correct(w), "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn user_dictionary_applies() {
|
||||
test_dict::install();
|
||||
assert!(!is_correct("zxqitalianaparola"));
|
||||
crate::add_word("Zxqitalianaparola");
|
||||
assert!(is_correct("zxqitalianaparola"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,435 @@
|
||||
//! Italian spelling suggestions, best first.
|
||||
//!
|
||||
//! Candidates come in rounds of rising cost, each checked against the dictionary:
|
||||
//! 1. letter case (`roma` → `Roma`);
|
||||
//! 2. missing or wrong accents, from the dictionary's `MAP` groups (`caffe` → `caffè`,
|
||||
//! `perche` → `perché`), up to four letters changed — Italian typed without accents is the
|
||||
//! most common slip, so adding one ranks first, while taking away an accent the writer typed
|
||||
//! ranks after the known confusions;
|
||||
//! 3. the dictionary's `REP` table of common confusions (`caffé` → `caffè`);
|
||||
//! 4. a missing space (`perme` → `per me`, `amilano` → `a milano`), cheaper when one half is a
|
||||
//! short function word;
|
||||
//! 5. one typo: a letter left out, added, swapped with the next or mistyped (`TRY` letters);
|
||||
//! 6. when that finds fewer than asked for: the stems that start like the word (accents
|
||||
//! folded) and share the most letter pairs with it, whose forms are ranked by an edit distance
|
||||
//! in which an accent slip costs half (two typos at most).
|
||||
//!
|
||||
//! The input is bounded (40 letters), and so is every round, so a call costs ten-odd thousand
|
||||
//! dictionary lookups at most.
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::affix::{AffixDict, Casing, capitalize, casing};
|
||||
|
||||
/// Short words often run into the next one (`per me`, `a Milano`, `di Roma`).
|
||||
const LEADING_WORDS: &[&str] = &[
|
||||
"a", "e", "è", "di", "da", "in", "con", "su", "per", "tra", "fra", "che", "chi", "cui", "non", "si", "ci", "vi", "ne", "più", "meno", "molto",
|
||||
"tanto", "come", "dove", "quando", "mai", "qui", "qua", "lì", "là", "ogni", "altro", "tutto", "questo", "quello", "bene", "male", "anche",
|
||||
"ancora", "sempre", "già", "poi", "ed", "od", "ad", "un", "uno", "una", "il", "lo", "la", "i", "gli", "le", "del", "al", "dal", "nel", "sul",
|
||||
"col", "pel", "allo", "dello", "nello", "sullo", "della", "alla", "dalla", "nella", "sulla", "degli", "agli", "dagli", "negli", "sugli", "delle",
|
||||
"alle", "dalle", "nelle", "sulle", "dei", "ai", "dai", "nei", "sui", "coi", "dei",
|
||||
];
|
||||
/// Short words often run into the one before (enclitic pronouns: `dimmelo` in writing without
|
||||
/// the doubling, or a dropped space: `perme` → `per me`).
|
||||
const TRAILING_WORDS: &[&str] = &[
|
||||
"mi", "ti", "si", "ci", "vi", "ne", "lo", "la", "li", "le", "gli", "me", "te", "lui", "lei", "noi", "voi", "loro", "che", "cosa", "come", "dove",
|
||||
"è",
|
||||
];
|
||||
const MAX_LETTERS: usize = 40;
|
||||
const MAX_MAP_CHANGES: usize = 4;
|
||||
const MAX_MAP_VARIANTS: usize = 4096;
|
||||
/// Stems the scan of round 6 expands.
|
||||
const SCAN_STEMS: usize = 48;
|
||||
|
||||
const COST_CASE: u32 = 0;
|
||||
const COST_ADD_DIACRITIC: u32 = 4;
|
||||
const COST_OTHER_MAP: u32 = 7;
|
||||
const COST_DROP_DIACRITIC: u32 = 11;
|
||||
const COST_REP: u32 = 10;
|
||||
const COST_SPLIT: u32 = 12;
|
||||
const COST_SPLIT_RARE: u32 = 18;
|
||||
const COST_SWAP: u32 = 14;
|
||||
const COST_DROP_OR_ADD: u32 = 15;
|
||||
const COST_MISTYPE: u32 = 16;
|
||||
const COST_SCAN: u32 = 20;
|
||||
|
||||
struct Collector<'a> {
|
||||
d: &'a AffixDict,
|
||||
/// The word, lower case.
|
||||
input: String,
|
||||
tested: HashSet<String>,
|
||||
/// Suggestion → (cost, order found).
|
||||
found: HashMap<String, (u32, usize)>,
|
||||
seq: usize,
|
||||
}
|
||||
|
||||
impl Collector<'_> {
|
||||
/// Check a lower-case candidate (a word, or two separated by a space) and keep it if every
|
||||
/// word is in the dictionary, as written or capitalized (names).
|
||||
fn offer(&mut self, cand: &str, cost: u32) {
|
||||
if cand.is_empty() || cand == self.input || !self.tested.insert(cand.to_string()) {
|
||||
return;
|
||||
}
|
||||
let mut words = Vec::new();
|
||||
let mut cost = cost;
|
||||
for part in cand.split(' ') {
|
||||
if self.d.contains(part) {
|
||||
words.push(part.to_string());
|
||||
} else {
|
||||
let c = capitalize(part);
|
||||
if !self.d.contains(&c) {
|
||||
return;
|
||||
}
|
||||
words.push(c);
|
||||
// A name ranks after a common word at the same distance.
|
||||
cost += 1;
|
||||
}
|
||||
}
|
||||
self.keep(words.join(" "), cost);
|
||||
}
|
||||
|
||||
/// Keep a suggestion known to be correct.
|
||||
fn keep(&mut self, text: String, cost: u32) {
|
||||
self.seq += 1;
|
||||
let seq = self.seq;
|
||||
let e = self.found.entry(text).or_insert((cost, seq));
|
||||
if cost < e.0 {
|
||||
*e = (cost, seq);
|
||||
}
|
||||
}
|
||||
|
||||
fn count_within(&self, cost: u32) -> usize {
|
||||
self.found.values().filter(|(c, _)| *c <= cost).count()
|
||||
}
|
||||
}
|
||||
|
||||
/// The MAP group of each letter (its other members are its likely confusions).
|
||||
fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
map.iter().filter(|g| g.contains(&c)).flat_map(|g| g.iter().copied()).filter(|&x| x != c).collect()
|
||||
}
|
||||
|
||||
/// The cost of changing letter `from` to its MAP relative `to`. Adding a missing accent is the
|
||||
/// commonest slip (Italian typed without accents: `citta`); an accent the writer did type
|
||||
/// is likelier right, so taking one away costs more than a known confusion from the REP table.
|
||||
fn map_cost(from: char, to: char) -> u32 {
|
||||
match (from.is_ascii(), to.is_ascii()) {
|
||||
(true, false) => COST_ADD_DIACRITIC,
|
||||
(false, true) => COST_DROP_DIACRITIC,
|
||||
_ => COST_OTHER_MAP,
|
||||
}
|
||||
}
|
||||
|
||||
/// Every way of changing up to [`MAX_MAP_CHANGES`] letters to a MAP relative, with its cost.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], i: usize, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
return;
|
||||
}
|
||||
let Some(&c) = chars.get(i) else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
return;
|
||||
};
|
||||
buf.push(c);
|
||||
map_variants(chars, alts, i + 1, changes, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < MAX_MAP_CHANGES {
|
||||
for &a in alts.get(i).map(Vec::as_slice).unwrap_or_default() {
|
||||
buf.push(a);
|
||||
map_variants(chars, alts, i + 1, changes + 1, cost + map_cost(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A letter's MAP representative (the first of its group), lower case: `à` → `a`.
|
||||
fn fold(c: char, map: &[Vec<char>]) -> char {
|
||||
let l = c.to_lowercase().next().unwrap_or(c);
|
||||
map.iter().find(|g| g.contains(&l)).and_then(|g| g.first().copied()).unwrap_or(l)
|
||||
}
|
||||
|
||||
/// Optimal string alignment distance where a MAP slip (`a`/`à`, `e`/`è`) costs 1 and any other
|
||||
/// edit 2; `limit + 1` once it is certainly above `limit`.
|
||||
fn weighted_distance(a: &[char], b: &[char], related: &dyn Fn(char, char) -> bool, limit: u32) -> u32 {
|
||||
let m = b.len();
|
||||
let over = limit + 1;
|
||||
let mut prev2 = vec![0u32; m + 1];
|
||||
let mut prev: Vec<u32> = (0..=m as u32).map(|j| j * 2).collect();
|
||||
let mut cur = vec![0u32; m + 1];
|
||||
for (i, &ai) in a.iter().enumerate() {
|
||||
if let Some(c) = cur.get_mut(0) {
|
||||
*c = (i as u32 + 1) * 2;
|
||||
}
|
||||
let mut row_min = u32::MAX;
|
||||
for (j, &bj) in b.iter().enumerate() {
|
||||
let sub = if ai == bj {
|
||||
0
|
||||
} else if related(ai, bj) {
|
||||
1
|
||||
} else {
|
||||
2
|
||||
};
|
||||
let at = |v: &[u32], k: usize| v.get(k).copied().unwrap_or(u32::MAX / 4);
|
||||
let mut v = (at(&prev, j + 1) + 2).min(at(&cur, j) + 2).min(at(&prev, j) + sub);
|
||||
if i > 0 && j > 0 && a.get(i - 1) == Some(&bj) && b.get(j - 1) == Some(&ai) {
|
||||
v = v.min(at(&prev2, j - 1) + 2);
|
||||
}
|
||||
if let Some(c) = cur.get_mut(j + 1) {
|
||||
*c = v;
|
||||
}
|
||||
row_min = row_min.min(v);
|
||||
}
|
||||
if m > 0 && row_min > limit {
|
||||
return over;
|
||||
}
|
||||
std::mem::swap(&mut prev2, &mut prev);
|
||||
std::mem::swap(&mut prev, &mut cur);
|
||||
}
|
||||
prev.get(m).copied().unwrap_or(over).min(over)
|
||||
}
|
||||
|
||||
/// Spelling suggestions for an Italian word, best first, at most `max`, in the word's letter case.
|
||||
pub fn suggest(word: &str, max: usize) -> Vec<String> {
|
||||
suggest_in(&super::dictionary(), word, max)
|
||||
}
|
||||
|
||||
/// [`suggest`] against a given dictionary.
|
||||
pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec<String> {
|
||||
let word = word.trim();
|
||||
let n = word.chars().count();
|
||||
if n == 0 || n > MAX_LETTERS || max == 0 || d.is_empty() || word.contains(char::is_whitespace) {
|
||||
return Vec::new();
|
||||
}
|
||||
let case = casing(word);
|
||||
let input = word.to_lowercase();
|
||||
let chars: Vec<char> = input.chars().collect();
|
||||
let mut col = Collector { d, input: input.clone(), tested: HashSet::new(), found: HashMap::new(), seq: 0 };
|
||||
// 1. Letter case.
|
||||
if d.contains(&input) && word != input {
|
||||
col.keep(input.clone(), COST_CASE);
|
||||
} else if d.contains(&capitalize(&input)) && word != capitalize(&input) {
|
||||
col.keep(capitalize(&input), COST_CASE);
|
||||
}
|
||||
// 2. Accents and other MAP slips.
|
||||
let map = d.map();
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(&chars, &alts, 0, 0, 0, &mut Vec::with_capacity(n), &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
}
|
||||
// 3. Common confusions.
|
||||
for (from, to) in d.rep() {
|
||||
if from.is_empty() {
|
||||
continue;
|
||||
}
|
||||
for (i, _) in input.match_indices(from.as_str()) {
|
||||
if let (Some(a), Some(b)) = (input.get(..i), input.get(i + from.len()..)) {
|
||||
col.offer(&format!("{a}{to}{b}"), COST_REP);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. A missing space: cheap after a short leading word or before a short trailing one,
|
||||
// dearer between two longer words, never leaving a stray short half.
|
||||
for (i, _) in input.char_indices().skip(1) {
|
||||
let (Some(a), Some(b)) = (input.get(..i), input.get(i..)) else { continue };
|
||||
let long = |p: &str| p.chars().count() >= 3;
|
||||
let cost = if (LEADING_WORDS.contains(&a) && long(b)) || (long(a) && TRAILING_WORDS.contains(&b)) {
|
||||
COST_SPLIT
|
||||
} else if long(a) && long(b) {
|
||||
COST_SPLIT_RARE
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
col.offer(&format!("{a} {b}"), cost);
|
||||
}
|
||||
// 5. One typo.
|
||||
let mut letters: Vec<char> = d.try_chars().chars().filter(|c| c.is_lowercase()).collect();
|
||||
if letters.is_empty() {
|
||||
letters = "aeiortnslcdupmghfbqvàèéìòóùz".chars().collect();
|
||||
}
|
||||
let s = |v: &[char]| v.iter().collect::<String>();
|
||||
for i in 0..n.saturating_sub(1) {
|
||||
if chars.get(i) != chars.get(i + 1) {
|
||||
let mut v = chars.clone();
|
||||
v.swap(i, i + 1);
|
||||
col.offer(&s(&v), COST_SWAP);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
let mut v = chars.clone();
|
||||
v.remove(i);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
for i in 0..=n {
|
||||
for &c in &letters {
|
||||
let mut v = chars.clone();
|
||||
v.insert(i, c);
|
||||
col.offer(&s(&v), COST_DROP_OR_ADD);
|
||||
}
|
||||
}
|
||||
for i in 0..n {
|
||||
for &c in &letters {
|
||||
if chars.get(i) != Some(&c) {
|
||||
let mut v = chars.clone();
|
||||
if let Some(x) = v.get_mut(i) {
|
||||
*x = c;
|
||||
}
|
||||
col.offer(&s(&v), COST_MISTYPE);
|
||||
}
|
||||
}
|
||||
}
|
||||
// 6. Further off: scan the stems that start like the word.
|
||||
if col.count_within(COST_MISTYPE) < max {
|
||||
scan(&mut col, &chars);
|
||||
}
|
||||
let mut found: Vec<(String, (u32, usize))> = col.found.into_iter().collect();
|
||||
// Among equal costs, the change nearest the word's start is the likelier intent.
|
||||
let diff_at = |text: &str| chars.iter().zip(text.chars()).position(|(a, b)| *a != b).unwrap_or(n);
|
||||
found.sort_by_key(|(text, (cost, seq))| (*cost, diff_at(text), *seq));
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for (text, _) in found {
|
||||
let t = match case {
|
||||
Casing::Initial => capitalize(&text),
|
||||
Casing::Upper if n > 1 => text.to_uppercase(),
|
||||
_ => text,
|
||||
};
|
||||
if t != word && !out.contains(&t) {
|
||||
out.push(t);
|
||||
}
|
||||
if out.len() >= max {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Round 6: rank the stems starting with the word's first letter (or a MAP relative of it, either
|
||||
/// case) by shared letter pairs, then keep the forms of the best within two typos.
|
||||
fn scan(col: &mut Collector, chars: &[char]) {
|
||||
let d = col.d;
|
||||
let map = d.map();
|
||||
let Some(&first) = chars.first() else { return };
|
||||
let folded: Vec<char> = chars.iter().map(|&c| fold(c, map)).collect();
|
||||
let pairs: HashSet<(char, char)> = std::iter::once(' ').chain(folded.iter().copied()).zip(folded.iter().copied()).collect();
|
||||
let mut firsts: Vec<char> = std::iter::once(first).chain(alternatives(first, map)).collect();
|
||||
let upper: Vec<char> = firsts.iter().flat_map(|c| c.to_uppercase()).collect();
|
||||
firsts.extend(upper);
|
||||
firsts.sort_unstable();
|
||||
firsts.dedup();
|
||||
let n = folded.len();
|
||||
let mut best: Vec<(u32, &str)> = Vec::new();
|
||||
for f in firsts {
|
||||
let prefix = f.to_string();
|
||||
for stem in d.stems_with_prefix(&prefix) {
|
||||
let sf: Vec<char> = stem.chars().map(|c| fold(c, map)).collect();
|
||||
let shared = std::iter::once(' ').chain(sf.iter().copied()).zip(sf.iter().copied()).filter(|p| pairs.contains(p)).count();
|
||||
if shared < 2 {
|
||||
continue;
|
||||
}
|
||||
// Shared pairs, less the stem's letters the word doesn't explain, less a length gap
|
||||
// beyond what an ending explains.
|
||||
let extra = sf.len().saturating_sub(shared);
|
||||
let gap = sf.len().abs_diff(n).saturating_sub(4);
|
||||
let score = (shared * 4).saturating_sub(extra * 3 + gap * 2) as u32;
|
||||
if score == 0 {
|
||||
continue;
|
||||
}
|
||||
if best.len() < SCAN_STEMS {
|
||||
best.push((score, stem));
|
||||
} else if let Some(min) = best.iter_mut().min_by_key(|b| b.0)
|
||||
&& score > min.0
|
||||
{
|
||||
*min = (score, stem);
|
||||
}
|
||||
}
|
||||
}
|
||||
best.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(b.1)));
|
||||
let related = |a: char, b: char| fold(a, map) == fold(b, map);
|
||||
let limit = 4;
|
||||
for (_, stem) in best {
|
||||
for form in d.forms(stem) {
|
||||
let lf: Vec<char> = form.to_lowercase().chars().collect();
|
||||
if lf.len().abs_diff(chars.len()) > 2 {
|
||||
continue;
|
||||
}
|
||||
let dist = weighted_distance(&lf, chars, &related, limit);
|
||||
if dist <= limit && lf.as_slice() != chars {
|
||||
col.keep(form, COST_SCAN + dist * 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn has(word: &str, want: &str) {
|
||||
let s = suggest(word, 8);
|
||||
assert!(s.iter().any(|x| x == want), "{word} → {s:?}, want {want}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_misspellings_get_the_right_suggestion() {
|
||||
crate::it::test_dict::install();
|
||||
has("caffe", "caffè");
|
||||
has("perche", "perché");
|
||||
has("cioe", "cioè");
|
||||
has("citta", "città");
|
||||
has("lbiro", "libro");
|
||||
has("parlre", "parlare");
|
||||
has("quelo", "quello");
|
||||
has("perme", "per me");
|
||||
has("amilano", "a Milano");
|
||||
has("roma", "Roma");
|
||||
has("Caffe", "Caffè");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_likeliest_fix_comes_first() {
|
||||
crate::it::test_dict::install();
|
||||
assert_eq!(suggest("caffe", 3).first().map(String::as_str), Some("caffè"));
|
||||
assert_eq!(suggest("perche", 3).first().map(String::as_str), Some("perché"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edge_cases_stay_empty_or_bounded() {
|
||||
crate::it::test_dict::install();
|
||||
assert!(suggest("", 5).is_empty());
|
||||
assert!(suggest("bene", 0).is_empty());
|
||||
assert!(suggest(&"a".repeat(41), 5).is_empty());
|
||||
assert!(suggest("due parole", 5).is_empty());
|
||||
assert!(suggest_in(&crate::affix::AffixDict::default(), "caffe", 5).is_empty());
|
||||
assert!(suggest("caffè", 5).len() <= 5);
|
||||
assert!(!suggest("caffè", 5).contains(&"caffè".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn distance_weighs_accents_half() {
|
||||
let map = vec![vec!['a', 'à'], vec!['e', 'é', 'è']];
|
||||
let related = |a: char, b: char| fold(a, &map) == fold(b, &map);
|
||||
let c = |s: &str| s.chars().collect::<Vec<_>>();
|
||||
assert_eq!(weighted_distance(&c("caffe"), &c("caffè"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("citta"), &c("città"), &related, 9), 1);
|
||||
assert_eq!(weighted_distance(&c("bene"), &c("bene"), &related, 9), 0);
|
||||
assert_eq!(weighted_distance(&c("bene"), &c("male"), &related, 2), 3);
|
||||
assert_eq!(weighted_distance(&c("bene"), &c("benissimo"), &related, 2), 3);
|
||||
assert_eq!(weighted_distance(&c(""), &c("ab"), &related, 9), 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suggestions_are_fast_enough() {
|
||||
crate::it::test_dict::install();
|
||||
// Warm the dictionary, then time a worst case (a long word nothing near): the scan round.
|
||||
let _ = suggest("bene", 1);
|
||||
let t = std::time::Instant::now();
|
||||
for w in ["xqzvweitalianaparola", "prsidenti", "parallellepipedo"] {
|
||||
let _ = suggest(w, 6);
|
||||
}
|
||||
let ms = t.elapsed().as_secs_f64() * 1000.0;
|
||||
eprintln!("3 suggestion calls: {ms:.0} ms");
|
||||
assert!(ms < 20_000.0, "{ms} ms");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
//! A tiny synthetic Italian Hunspell dictionary for tests (written for these tests; not taken
|
||||
//! from any real dictionary). Real dictionaries are installed on the system, never bundled.
|
||||
//!
|
||||
//! It exercises what the Italian dictionaries use: the default single-character `FLAG` type,
|
||||
//! suffixes (verb endings `-are`, noun plurals `-o` → `-i`), a prefix (`ri-`), the suggestion
|
||||
//! tables (`TRY`, `MAP` for accents, `REP` for common confusions) and elided forms with an
|
||||
//! apostrophe (`l'amico`).
|
||||
|
||||
pub const AFF: &str = "SET UTF-8
|
||||
TRY aeiortnslcdupmghfbqvàèéìòóùz
|
||||
MAP 5
|
||||
MAP aà
|
||||
MAP eéè
|
||||
MAP iì
|
||||
MAP oòó
|
||||
MAP uù
|
||||
REP 3
|
||||
REP caffé caffè
|
||||
REP perche perché
|
||||
REP cioe cioè
|
||||
SFX A Y 3
|
||||
SFX A 0 are .
|
||||
SFX A 0 a .
|
||||
SFX A 0 ano .
|
||||
SFX N Y 2
|
||||
SFX N o i o
|
||||
SFX N a e a
|
||||
PFX R Y 1
|
||||
PFX R 0 ri .
|
||||
";
|
||||
|
||||
/// Stems (the count line first, as in Hunspell files).
|
||||
pub const DIC: &str = "96
|
||||
città
|
||||
caffè
|
||||
perché
|
||||
più
|
||||
può
|
||||
è
|
||||
già
|
||||
così
|
||||
virtù
|
||||
tribù
|
||||
menù
|
||||
tè
|
||||
Mosca
|
||||
Roma
|
||||
Milano
|
||||
Torino
|
||||
casa/N
|
||||
libro/N
|
||||
cane
|
||||
gatto
|
||||
sole
|
||||
luna
|
||||
mare
|
||||
amico
|
||||
amica
|
||||
idea
|
||||
parl/A
|
||||
mangiare
|
||||
dormire
|
||||
fare/R
|
||||
scuola
|
||||
bambino/N
|
||||
albero
|
||||
fiore
|
||||
pane
|
||||
acqua
|
||||
terra
|
||||
cielo
|
||||
notte
|
||||
giorno
|
||||
vecchia
|
||||
vado
|
||||
con
|
||||
ha
|
||||
ho
|
||||
bel
|
||||
cioè
|
||||
la
|
||||
un
|
||||
in
|
||||
piazza
|
||||
lavoro
|
||||
tempo
|
||||
uomo
|
||||
donna
|
||||
bello/N
|
||||
buono
|
||||
grande
|
||||
piccolo
|
||||
nuovo/N
|
||||
vecchio
|
||||
bianco
|
||||
nero
|
||||
rosso
|
||||
verde
|
||||
giallo
|
||||
uno
|
||||
due
|
||||
tre
|
||||
tutto
|
||||
come
|
||||
quando
|
||||
dove
|
||||
senza
|
||||
sopra
|
||||
sotto
|
||||
tra
|
||||
molto
|
||||
sempre
|
||||
mai
|
||||
qui
|
||||
lì
|
||||
sì
|
||||
no
|
||||
non
|
||||
che
|
||||
chi
|
||||
cui
|
||||
mio
|
||||
tuo
|
||||
suo
|
||||
nostro
|
||||
a
|
||||
per
|
||||
me
|
||||
quello
|
||||
";
|
||||
|
||||
/// Make [`AFF`]/[`DIC`] the Italian dictionary (every test that needs one calls this; they all
|
||||
/// install the same one, so tests running in parallel agree).
|
||||
pub fn install() {
|
||||
if let Err(e) = super::install_dictionary(AFF, DIC) {
|
||||
panic!("test dictionary: {e}");
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,9 @@ pub enum ProofLang {
|
||||
/// friends, when there is one), the Portuguese grammar rules and the Portuguese hyphenation
|
||||
/// patterns.
|
||||
Pt,
|
||||
/// Italian (`it`, `it-IT`, `it-CH`): the installed Hunspell dictionary (`it_IT`, when there
|
||||
/// is one), the Italian grammar rules and the Italian hyphenation patterns.
|
||||
It,
|
||||
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
|
||||
/// the language's proofing tools: no spelling or grammar marks, so its words are never
|
||||
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
|
||||
@@ -30,7 +33,7 @@ pub enum ProofLang {
|
||||
|
||||
impl ProofLang {
|
||||
/// Every language with a spelling dictionary, in display order.
|
||||
pub const PROOFED: [ProofLang; 3] = [ProofLang::En, ProofLang::Pl, ProofLang::Pt];
|
||||
pub const PROOFED: [ProofLang; 4] = [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It];
|
||||
|
||||
/// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). Only the primary subtag
|
||||
/// counts. No tag (or an empty one) is English, WordCraft's default editing language.
|
||||
@@ -45,6 +48,8 @@ impl ProofLang {
|
||||
ProofLang::Bg
|
||||
} else if primary.eq_ignore_ascii_case("pt") || primary.eq_ignore_ascii_case("por") {
|
||||
ProofLang::Pt
|
||||
} else if primary.eq_ignore_ascii_case("it") || primary.eq_ignore_ascii_case("ita") {
|
||||
ProofLang::It
|
||||
} else {
|
||||
ProofLang::Other
|
||||
}
|
||||
@@ -57,6 +62,7 @@ impl ProofLang {
|
||||
ProofLang::Pl => "pl",
|
||||
ProofLang::Bg => "bg",
|
||||
ProofLang::Pt => "pt",
|
||||
ProofLang::It => "it",
|
||||
ProofLang::Other => "",
|
||||
}
|
||||
}
|
||||
@@ -133,6 +139,9 @@ mod tests {
|
||||
for t in ["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", " pt-PT ", "por"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Pt, "{t}");
|
||||
}
|
||||
for t in ["it", "it-IT", "IT_it", "it-CH", "it-CH.UTF-8", " it-IT ", "ita"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::It, "{t}");
|
||||
}
|
||||
for t in ["en", "en-US", "en-GB", "EN_au", "en-Latn-US"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::En, "{t}");
|
||||
}
|
||||
@@ -152,6 +161,8 @@ mod tests {
|
||||
assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed());
|
||||
assert_eq!(ProofLang::Pl.code(), "pl");
|
||||
assert_eq!(ProofLang::Pt.code(), "pt");
|
||||
assert!(ProofLang::It.is_proofed());
|
||||
assert_eq!(ProofLang::It.code(), "it");
|
||||
assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed");
|
||||
}
|
||||
}
|
||||
+34
-3
@@ -18,6 +18,11 @@
|
||||
//! dictionary installed on the system (VERO and friends; not bundled), suggestions tuned for
|
||||
//! Portuguese (missing accents, `seje`/`seja` confusions, missing spaces), conservative
|
||||
//! grammar rules with Portuguese messages, and `hyph-utf8`'s Portuguese patterns.
|
||||
//! - **Italian** ([`ProofLang::It`], module [`it`]): spelling against the Hunspell `it_IT`
|
||||
//! dictionary installed on the system (not bundled — the upstream file is GPL-licensed),
|
||||
//! suggestions tuned for Italian (missing accents, `REP` confusions, missing spaces),
|
||||
//! conservative grammar rules with Italian messages, and `hyph-utf8`'s Italian patterns
|
||||
//! (Beccari, MIT option).
|
||||
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
|
||||
//! the language's proofing tools.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese),
|
||||
@@ -31,6 +36,7 @@ pub mod bg;
|
||||
pub mod dict;
|
||||
pub mod hunspell;
|
||||
pub mod hyphen;
|
||||
pub mod it;
|
||||
pub mod lang;
|
||||
pub mod patterns;
|
||||
pub mod pl;
|
||||
@@ -499,6 +505,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option
|
||||
let message = match lang {
|
||||
ProofLang::Pl => "Możliwy błąd pisowni",
|
||||
ProofLang::Pt => "Possível erro de ortografia",
|
||||
ProofLang::It => "Possibile errore di ortografia",
|
||||
_ => "Possible spelling mistake",
|
||||
};
|
||||
Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang })
|
||||
@@ -515,6 +522,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
ProofLang::En => is_correct_with(word, opts),
|
||||
ProofLang::Pl => accepted_by_options(word, opts) || pl::is_correct(word),
|
||||
ProofLang::Pt => accepted_by_options(word, opts) || pt::is_correct(word),
|
||||
ProofLang::It => accepted_by_options(word, opts) || it::is_correct(word),
|
||||
ProofLang::Bg | ProofLang::Other => true,
|
||||
}
|
||||
}
|
||||
@@ -525,18 +533,20 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
ProofLang::En => suggest(word, max),
|
||||
ProofLang::Pl => pl::suggest(word, max),
|
||||
ProofLang::Pt => pt::suggest(word, max),
|
||||
ProofLang::It => it::suggest(word, max),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones
|
||||
/// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), or none for a language WordCraft
|
||||
/// doesn't proof.
|
||||
/// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), the Italian ones ([`it::grammar`]),
|
||||
/// or none for a language WordCraft doesn't proof.
|
||||
pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec<Issue> {
|
||||
match lang {
|
||||
ProofLang::En => check_grammar(text),
|
||||
ProofLang::Pl => pl::grammar::check(text),
|
||||
ProofLang::Pt => pt::grammar::check(text),
|
||||
ProofLang::It => it::grammar::check(text),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
@@ -989,6 +999,21 @@ mod tests {
|
||||
assert!(!g.iter().any(|i| i.message.contains("\"an\"")), "{g:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn italian_uses_its_own_tools() {
|
||||
it::test_dict::install();
|
||||
let it = "La città vecchia ha un bel caffè in piazza.";
|
||||
assert!(check_spelling_in(it, ProofLang::It).is_empty(), "{:?}", check_spelling_in(it, ProofLang::It));
|
||||
assert!(check_spelling_in(it, ProofLang::En).len() >= 3, "Italian words are English misspellings");
|
||||
let v = check_spelling_in("Vado a Roma con lbiro.", ProofLang::It);
|
||||
assert_eq!(flagged("Vado a Roma con lbiro.", &v), ["lbiro"]);
|
||||
assert_eq!((v[0].lang, v[0].message.as_str()), (ProofLang::It, "Possibile errore di ortografia"));
|
||||
assert!(suggest_in("caffe", 5, ProofLang::It).contains(&"caffè".to_string()));
|
||||
// Italian grammar speaks Italian and has no English a/an rule.
|
||||
let g = check_grammar_in("questo questo è un test", ProofLang::It);
|
||||
assert!(g.iter().any(|i| i.message.contains("ripetuta")), "{g:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_language_paragraphs_check_each_run_in_its_language() {
|
||||
pl::test_dict::install();
|
||||
@@ -1014,6 +1039,12 @@ mod tests {
|
||||
let v = check_text(t, &[(0..t.len(), ProofLang::Pt)]);
|
||||
assert_eq!(flagged(t, &v), ["coraçao", "frequencia"], "{v:?}");
|
||||
assert!(v.iter().all(|i| i.lang == ProofLang::Pt && i.message == "Possível erro de ortografia"), "{v:?}");
|
||||
// Italian checks in its span only, with the Italian message.
|
||||
it::test_dict::install();
|
||||
let t = "Vado a Roma con lbiro e caffe.";
|
||||
let v = check_text(t, &[(0..t.len(), ProofLang::It)]);
|
||||
assert_eq!(flagged(t, &v), ["lbiro", "caffe"], "{v:?}");
|
||||
assert!(v.iter().all(|i| i.lang == ProofLang::It && i.message == "Possibile errore di ortografia"), "{v:?}");
|
||||
}
|
||||
|
||||
proptest::proptest! {
|
||||
@@ -1025,7 +1056,7 @@ mod tests {
|
||||
let _ = has_dictionary(Some(&s));
|
||||
let _ = is_correct_for(&s, Some(&s));
|
||||
let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default());
|
||||
for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::Other] {
|
||||
for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Other] {
|
||||
let _ = check_spelling_in(&s, lang);
|
||||
let _ = check_grammar_in(&s, lang);
|
||||
let _ = is_correct_in(&s, lang);
|
||||
|
||||
Reference in new issue
Block a user