diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 1cbf3d1d..9bd6c1f8 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -11,7 +11,8 @@ //! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text //! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). //! -//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`), +//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`; +//! a variation selector after one, as in the Spanish dictionaries' `☎️`, belongs to it), //! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross //! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), //! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding, @@ -261,6 +262,18 @@ struct Rules { /// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. type RawRule = (char, bool, String, String, String, String); +/// Is `c` a Unicode variation selector (U+FE00–U+FE0F)? It only picks how the character before +/// it is drawn, so in a flag field it belongs to that flag: the RLA-ES Spanish dictionaries write +/// a flag as `☎️` (`☎` and U+FE0F), which Hunspell reads as `☎` (#3360). +fn is_variation_selector(c: char) -> bool { + ('\u{FE00}'..='\u{FE0F}').contains(&c) +} + +/// The flag characters of a flag field, without [variation selectors](is_variation_selector). +fn flag_chars(field: &str) -> impl Iterator + '_ { + field.chars().filter(|&c| !is_variation_selector(c)) +} + /// `0` means empty in Hunspell's strip and append fields. fn zero_empty(s: &str) -> String { if s == "0" { String::new() } else { s.to_string() } @@ -290,7 +303,7 @@ fn parse_aff(aff: &str) -> Result { } } directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars(); + let mut it = flag_chars(at(1)); match (it.next(), it.next()) { (Some(c), None) if !c.is_whitespace() => { if directive == "FORBIDDENWORD" { @@ -320,8 +333,8 @@ fn parse_aff(aff: &str) -> Result { } "PFX" | "SFX" => { let suffix = dir == "SFX"; - let mut flag_chars = at(1).chars(); - let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { + let mut flag_field = flag_chars(at(1)); + let (Some(flag), None) = (flag_field.next(), flag_field.next()) else { return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); }; // Only a header ignores trailing fields; a rule condition beginning with # is literal. @@ -341,7 +354,7 @@ fn parse_aff(aff: &str) -> Result { let cond = if f.len() >= 5 { at(4) } else { "." }; parse_cond(cond).map_err(|e| format!("line {}: {e}", n + 1))?; let c = cross.get(&(suffix, flag)).copied().unwrap_or(false); - let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), cont.to_string(), cond.to_string()); + let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), flag_chars(cont).collect(), cond.to_string()); if suffix { r.suffixes.push(rule); } else { @@ -377,7 +390,9 @@ impl AffixDict { let mut in_flags = false; while let Some(c) = chars.next() { if in_flags { - flags.push(c); + if !is_variation_selector(c) { + flags.push(c); + } } else if c == '\\' && chars.peek() == Some(&'/') { word.push('/'); chars.next(); @@ -1079,6 +1094,20 @@ pub(crate) mod tests { assert!(AffixDict::from_hunspell("FLAG UTF-8\n", "0\n", &[]).is_ok(), "FLAG alone is harmless"); } + /// #3360: the RLA-ES Spanish dictionaries write one flag as `☎️` (`☎` and the variation + /// selector U+FE0F) in the `.aff` and in the `.dic`; it is the flag `☎`, as Hunspell reads it, + /// not a two-character flag that refuses the whole dictionary. + #[test] + fn a_variation_selector_belongs_to_its_flag() { + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ✖\u{FE0F}\nSFX \u{260E}\u{FE0F} Y 1\nSFX \u{260E}\u{FE0F} 0 s/A\u{FE0F} .\nSFX A Y 1\nSFX A 0 x .\n"; + let d = AffixDict::from_hunspell(aff, "3\ncasa/\u{260E}\u{FE0F}\nmesa/\u{260E}\nzq/✖\u{FE0F}\n", &[]).unwrap(); + assert!(d.contains("casa") && d.contains("casas") && d.contains("casasx")); + assert!(d.contains("mesas"), "the bare flag is the same flag"); + assert!(!d.contains("zq"), "FORBIDDENWORD with a selector still forbids"); + assert_eq!(d.flag_chars.len(), 3, "{:?}", d.flag_chars); + assert!(AffixDict::from_hunspell("SFX ab Y 1\nSFX ab 0 s .\n", "0\n", &[]).is_err(), "two flag characters are still refused"); + } + #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin diff --git a/crates/proof/src/es/grammar.rs b/crates/proof/src/es/grammar.rs new file mode 100644 index 00000000..a22a6ff8 --- /dev/null +++ b/crates/proof/src/es/grammar.rs @@ -0,0 +1,188 @@ +//! Spanish grammar checks with Spanish messages: the generic repeated-word and spacing checks, +//! and the opening question and exclamation marks Spanish writes before a question or an +//! exclamation (`¿Vienes?`, `¡Qué bien!`): a closing `?` or `!` whose sentence has no opening mark +//! is flagged, with the opening mark offered at the sentence's start, and so is a space after an +//! opening mark (`¿ Vienes?`). Either opening mark may pair with either closing one (`¡Qué +//! dices?`), and `(?)` and `(!)`, which mark doubt or surprise, stand alone. No other +//! Spanish-specific grammar, comma or sentence-capital rules are inferred. + +use crate::{Issue, IssueKind, ProofLang, words}; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec, lang: ProofLang) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang } +} + +/// Grammar issues in a paragraph of Spanish text; `lang` is the variety (they explain the same in +/// every one). +pub fn check(text: &str, lang: ProofLang) -> Vec { + let mut issues = Vec::new(); + repeated_words(text, &words(text), lang, &mut issues); + spaces(text, lang, &mut issues); + opening_marks(text, lang, &mut issues); + issues.sort_by_key(|i| (i.start, i.end)); + issues +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], lang: ProofLang, v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() { + v.push(issue(*a0, *b1, format!("Palabra repetida: «{w1}»"), vec![w0.to_string()], lang)); + } + } +} + +fn spaces(text: &str, lang: ProofLang, v: &mut Vec) { + for (i, c) in text.char_indices() { + // A space after an opening mark: `¿ Vienes?`. + if matches!(c, '¿' | '¡') + && text.get(i + c.len_utf8()..).is_some_and(|r| r.starts_with(' ') && r.trim_start().chars().next().is_some_and(char::is_alphanumeric)) + { + v.push(issue(i, i + c.len_utf8() + 1, "Espacio después de un signo de apertura".into(), vec![c.to_string()], lang)); + continue; + } + if c != ' ' { + continue; + } + let rest = text.get(i + 1..).unwrap_or(""); + let mut it = rest.chars(); + let Some(p) = it.next() else { continue }; + let after = it.next(); + // A ratio/score keeps the spaces in digit-space-colon-space-digit. + let ratio = p == ':' + && text.get(..i).and_then(|before| before.chars().next_back()).is_some_and(|c| c.is_ascii_digit()) + && after == Some(' ') + && it.next().is_some_and(|c| c.is_ascii_digit()); + let flag = !ratio + && match p { + ',' => true, + // `;)` and `;-)` are emoticons. + ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), + // ` ...` is an ellipsis, ` .5` a number. + '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. + ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '»' | '!' | '?')), + _ => false, + }; + if flag { + v.push(issue(i, i + 1 + p.len_utf8(), "Espacio antes de un signo de puntuación".into(), vec![p.to_string()], lang)); + } + } + for (i, _) in text.match_indices(" ") { + if i > 0 && text.get(..i).is_some_and(|t| t.ends_with(' ')) { + continue; + } + v.push(issue(i, i + 2, "Espacio doble".into(), vec![" ".into()], lang)); + } +} + +/// Characters a question or exclamation can't reach back past: sentence ends, clause breaks, +/// line breaks, and the start of a quotation or parenthesis. +fn is_boundary(c: char) -> bool { + matches!(c, '.' | '?' | '!' | '…' | ':' | ';' | '\n' | '\r' | '\t' | '«' | '“' | '"' | '(' | '[' | '—') +} + +/// A closing `?` or `!` whose sentence has no opening `¿` or `¡`. +fn opening_marks(text: &str, lang: ProofLang, v: &mut Vec) { + for (i, c) in text.char_indices() { + let (opener, message) = match c { + '?' => ('¿', "Falta el signo de interrogación de apertura (¿)"), + '!' => ('¡', "Falta el signo de exclamación de apertura (¡)"), + _ => continue, + }; + let before = text.get(..i).unwrap_or(""); + let prev = before.chars().next_back(); + let next = text.get(i + 1..).and_then(|r| r.chars().next()); + // `?!` and `!!!` are handled at their first mark; `(?)` stands alone; `?q=` and `a!b` + // aren't sentence ends. + if matches!(prev, Some('?' | '!')) || (prev == Some('(') && next == Some(')')) || next.is_some_and(char::is_alphanumeric) { + continue; + } + let from = before.char_indices().rev().find(|&(_, b)| is_boundary(b) || matches!(b, '¿' | '¡')); + if from.is_some_and(|(_, b)| matches!(b, '¿' | '¡')) { + continue; + } + let start = from.map_or(0, |(j, b)| j + b.len_utf8()); + let Some(sentence) = text.get(start..i) else { continue }; + let lead = sentence.len() - sentence.trim_start().len(); + let start = start + lead; + let Some(sentence) = text.get(start..i + 1) else { continue }; + if !sentence.chars().any(char::is_alphabetic) { + continue; + } + v.push(issue(start, i + 1, message.into(), vec![format!("{opener}{sentence}")], lang)); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn messages(text: &str) -> Vec<(String, String)> { + check(text, ProofLang::Es).into_iter().map(|i| (text[i.start..i.end].to_string(), i.message)).collect() + } + + #[test] + fn repeated_words_and_spaces_have_spanish_messages() { + let text = "Es es la casa , dijo."; + let issues = check(text, ProofLang::EsMx); + assert_eq!(issues.len(), 3, "{issues:?}"); + assert_eq!(issues[0].message, "Palabra repetida: «es»"); + assert_eq!(issues[0].suggestions, ["Es"]); + assert!(issues.iter().any(|i| i.message == "Espacio antes de un signo de puntuación" && &text[i.start..i.end] == " ,")); + assert!(issues.iter().any(|i| i.message == "Espacio doble")); + assert!(issues.iter().all(|i| i.lang == ProofLang::EsMx && i.kind == IssueKind::Grammar)); + for text in ["La casa. Es grande", "Sr. García", "No, no lo sé", "Hola :) número .5 y puntos ...", "3 : 1", "teórico-práctico"] { + assert!(check(text, ProofLang::Es).is_empty(), "{text}: {:?}", check(text, ProofLang::Es)); + } + } + + #[test] + fn questions_and_exclamations_need_their_opening_mark() { + assert_eq!(messages("Vienes mañana?"), [("Vienes mañana?".into(), "Falta el signo de interrogación de apertura (¿)".into())]); + assert_eq!(check("Vienes mañana?", ProofLang::Es)[0].suggestions, ["¿Vienes mañana?"]); + assert_eq!(messages("Llegó. Qué bien!"), [("Qué bien!".into(), "Falta el signo de exclamación de apertura (¡)".into())]); + assert_eq!(messages("Le preguntó: vienes?")[0].0, "vienes?"); + assert_eq!(messages("Dijo «vienes?»")[0].0, "vienes?"); + assert_eq!(messages("Hola!!!").len(), 1); + assert_eq!(messages("Qué?!").len(), 1); + for text in [ + "¿Vienes mañana?", + "¡Qué bien!", + "Si vienes, ¿me llamas?", + "¡Qué dices?", + "¿Qué?! ¡No!", + "Nació en 1950 (?) en Montevideo.", + "Lo dijo (!) en público.", + "Ver https://example.com/?q=1 y a!b", + "¿Qué? ¿Cómo?", + "2 + 2 = ?", + "", + "?", + ] { + assert!(messages(text).iter().all(|(_, m)| !m.starts_with("Falta")), "{text}: {:?}", messages(text)); + } + } + + #[test] + fn a_space_after_an_opening_mark_is_flagged() { + let text = "¿ Vienes? ¡ Hola!"; + let issues = check(text, ProofLang::EsUy); + let spaced: Vec<&str> = + issues.iter().filter(|i| i.message == "Espacio después de un signo de apertura").map(|i| &text[i.start..i.end]).collect(); + assert_eq!(spaced, ["¿ ", "¡ "], "{issues:?}"); + assert!(issues.iter().all(|i| !i.message.starts_with("Falta")), "{issues:?}"); + assert!(check("¿ ", ProofLang::Es).is_empty()); + } + + proptest::proptest! { + #[test] + fn never_panics_and_ranges_are_valid(s in "\\PC{0,80}|[¿¡?!a-z .,(«»):]{0,40}") { + for issue in check(&s, ProofLang::Es) { + proptest::prop_assert!(issue.start <= issue.end && issue.end <= s.len()); + proptest::prop_assert!(s.is_char_boundary(issue.start) && s.is_char_boundary(issue.end)); + } + } + } +} diff --git a/crates/proof/src/es/mod.rs b/crates/proof/src/es/mod.rs new file mode 100644 index 00000000..2a0f98dc --- /dev/null +++ b/crates/proof/src/es/mod.rs @@ -0,0 +1,327 @@ +//! Spanish proofing (#3360, #3553) using the installed Hunspell dictionaries (never bundled; +//! LibreOffice ships them as its `dict-es` extension, Debian and Ubuntu as `hunspell-es`). Each +//! variety reads its own dictionary: Spain ([`ProofLang::Es`]) `es_ES`, Mexico +//! ([`ProofLang::EsMx`]) `es_MX`, Latin America ([`ProofLang::Es419`]) `es_419` and Uruguay +//! ([`ProofLang::EsUy`]) `es_UY`, each falling back to a nearby or generic one. Uruguayan Spanish +//! also accepts the `vos` forms of every verb in the dictionary ([`voseo`]: `tenés`, `vení`, +//! `decime`). Suggestions use the dictionary's own MAP/REP/TRY/KEY tables (the regional +//! dictionaries' REP tables carry their differences, such as `vosotros` forms in Mexico). +//! Grammar is limited to generic repeated-word and spacing checks and the Spanish opening +//! question and exclamation marks, with Spanish messages; hyphenation keeps the English fallback +//! (no Spanish patterns are bundled). + +pub mod grammar; +mod suggest; +#[cfg(test)] +pub(crate) mod test_dict; +pub mod voseo; + +use std::sync::{Arc, RwLock}; + +use crate::ProofLang; +use crate::affix::AffixDict; + +pub use suggest::suggest; + +/// The dictionary of each variety (Spain, Mexico, Latin America, Uruguay): `None` until first +/// use, then the installed one (or an empty one). +static DICTS: [RwLock>>; 4] = [const { RwLock::new(None) }; 4]; + +/// File names of a Spanish Hunspell dictionary for Spain, most specific first (`es_ANY` is the +/// generic one RLA-ES publishes). +const SPAIN: &[&str] = &["es_ES", "es-ES", "es_ANY", "es"]; +/// For Mexico: its own, then a Latin American or generic one. +const MEXICO: &[&str] = &["es_MX", "es-MX", "es_419", "es-419", "es_ANY", "es"]; +/// For Latin America: its own or a generic one, then the United States' and Mexico's, which +/// LibreOffice ships (it has no `es_419`) and which use Latin American spelling. +const LATIN_AMERICA: &[&str] = &["es_419", "es-419", "es_ANY", "es", "es_US", "es_MX"]; +/// For Uruguay: its own, then Argentina's (the same Rioplatense Spanish), then a Latin American +/// or generic one. +const URUGUAY: &[&str] = &["es_UY", "es-UY", "es_AR", "es-AR", "es_419", "es-419", "es_ANY", "es"]; + +/// The slot of `lang`'s dictionary in [`DICTS`] and its file names; Spain for any other language. +fn variety(lang: ProofLang) -> (usize, &'static [&'static str]) { + match lang { + ProofLang::EsMx => (1, MEXICO), + ProofLang::Es419 => (2, LATIN_AMERICA), + ProofLang::EsUy => (3, URUGUAY), + _ => (0, SPAIN), + } +} + +/// The variety's English name, for the log. +fn variety_name(lang: ProofLang) -> &'static str { + match lang { + ProofLang::EsMx => "Spanish (Mexico)", + ProofLang::Es419 => "Spanish (Latin America)", + ProofLang::EsUy => "Spanish (Uruguay)", + _ => "Spanish (Spain)", + } +} + +/// The Hunspell name each variety's dictionary is known by (`es_ES`, `es_MX`, `es_419`, +/// `es_UY`): the first of its file names, used for the status-bar hint and load errors. +pub fn dictionary_name(lang: ProofLang) -> &'static str { + variety(lang).1.first().copied().unwrap_or("es_ES") +} + +/// The Spanish dictionary of `lang`'s variety: the installed one (read on first use), one given +/// with [`install_dictionary`], or an empty one when none is available (spelling is off then; +/// see [`has_dictionary`]). +pub fn dictionary(lang: ProofLang) -> Arc { + let Some(slot) = DICTS.get(variety(lang).0) else { return Arc::new(AffixDict::default()) }; + if let Some(d) = slot.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + return d.clone(); + } + let loaded = Arc::new(load_installed(lang)); + let mut slot = slot.write().unwrap_or_else(|e| e.into_inner()); + // Another thread (or an install) may have filled it meanwhile: that one wins. + slot.get_or_insert(loaded).clone() +} + +fn load_installed(lang: ProofLang) -> AffixDict { + let key = dictionary_name(lang); + let Some((aff, dic)) = crate::hunspell::find(variety(lang).1) else { + crate::hunspell::clear_load_error(key); + log::info!("{} spelling: no Hunspell {key} dictionary installed", variety_name(lang)); + return AffixDict::default(); + }; + crate::hunspell::note_load(key, &dic, crate::hunspell::load(&aff, &dic)).unwrap_or_else(|e| { + log::warn!("{} spelling: {e}", variety_name(lang)); + AffixDict::default() + }) +} + +/// Use this dictionary for `lang`'s variety from now on (Hunspell `.aff` and `.dic` text, +/// decoded to UTF-8), for example one the user picked. On error the current dictionary stays. +pub fn install_dictionary(lang: ProofLang, aff: &str, dic: &str) -> Result<(), String> { + let d = AffixDict::from_hunspell(aff, dic, &[])?; + if let Some(slot) = DICTS.get(variety(lang).0) { + *slot.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + } + Ok(()) +} + +/// Forget every variety's loaded dictionary, so the next check looks for an installed one again +/// (after [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). +pub fn reload_dictionary() { + for slot in &DICTS { + *slot.write().unwrap_or_else(|e| e.into_inner()) = None; + } +} + +/// The `.dic` file of the Hunspell dictionary `lang`'s variety finds in the dictionaries folder +/// or the system locations now (File › Options › Proofing shows it); `None` when there is none. +/// A dictionary given with [`install_dictionary`] has no file and isn't reported. +pub fn installed_path(lang: ProofLang) -> Option { + crate::hunspell::find(variety(lang).1).map(|(_, dic)| dic) +} + +/// Is a dictionary available for `lang`'s variety (spelling is checked)? +pub fn has_dictionary(lang: ProofLang) -> bool { + !dictionary(lang).is_empty() +} + +/// Is `c` a letter of the Latin script (the only one the Spanish dictionaries cover)? +fn is_latin_letter(c: char) -> bool { + c.is_alphabetic() && (c < '\u{0250}' || ('\u{1E00}'..='\u{1EFF}').contains(&c)) +} + +/// Is `word` spelled correctly in `lang`'s Spanish? Numbers, single letters, short words in +/// capitals (acronyms: `ONU`, `CDMX`), words with digits and words in other scripts are accepted, +/// and so is every word when no dictionary is installed for the variety. Hyphenated compounds +/// are checked part by part (`teórico-práctico`, `franco-mexicano`). In Uruguayan Spanish the +/// `vos` forms of the dictionary's verbs are correct too ([`voseo`]). +pub fn is_correct(word: &str, lang: ProofLang) -> bool { + is_correct_with(&dictionary(lang), word, lang) +} + +/// [`is_correct`] against a given dictionary (an empty one accepts everything). +pub fn is_correct_with(d: &AffixDict, word: &str, lang: ProofLang) -> bool { + if d.is_empty() { + return true; // no Spanish dictionary installed: no spelling marks + } + let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + let w = w.trim_matches(['\'', '’']); + let n = w.chars().count(); + if n <= 1 || w.chars().any(|c| c.is_ascii_digit()) || !w.chars().any(char::is_alphabetic) { + return true; + } + if w.chars().all(|c| !c.is_lowercase()) && n <= 6 { + return true; + } + if !w.chars().any(is_latin_letter) { + return true; // other scripts: no dictionary + } + if w.contains('-') { + return known(d, w, lang) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p, lang)); + } + known(d, w, lang) +} + +/// Is one word in the dictionary or the user dictionary, or (in Uruguayan Spanish) the `vos` +/// form of one of the dictionary's verbs? +fn known(d: &AffixDict, w: &str, lang: ProofLang) -> bool { + d.check(w) || crate::in_user_dictionary(&w.to_lowercase()) || (lang == ProofLang::EsUy && voseo::is_vos_form(w, |inf| d.check(inf))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const VARIETIES: [ProofLang; 4] = [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy]; + + #[test] + fn each_variety_looks_for_its_own_dictionary_first() { + let dir = std::env::temp_dir().join(format!("wordcraft-es-aliases-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let (es, mx, la, uy) = (ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy); + // Alone in the folder, each file name is found by the varieties that list it. + for (name, found_by) in [ + ("es_ES", &[es][..]), + ("es-ES", &[es]), + ("es_MX", &[mx, la]), + ("es-MX", &[mx]), + ("es_419", &[mx, la, uy]), + ("es-419", &[mx, la, uy]), + ("es_US", &[la]), + ("es_UY", &[uy]), + ("es-UY", &[uy]), + ("es_AR", &[uy]), + ("es-AR", &[uy]), + ("es_ANY", &VARIETIES), + ("es", &VARIETIES), + ] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + for lang in VARIETIES { + let found = crate::hunspell::find_in(&dir, variety(lang).1); + assert_eq!(found.is_some(), found_by.contains(&lang), "{name}: {lang:?}"); + if let Some((aff, dic)) = found { + assert!(is_correct_with(&crate::hunspell::load(&aff, &dic).unwrap(), "canción", lang)); + } + } + std::fs::remove_file(dir.join(format!("{name}.aff"))).unwrap(); + std::fs::remove_file(dir.join(format!("{name}.dic"))).unwrap(); + } + // With them all, each variety reads its own. + for name in ["es_ES", "es_MX", "es_419", "es_UY", "es_AR", "es_ANY"] { + std::fs::write(dir.join(format!("{name}.aff")), test_dict::AFF).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), test_dict::DIC).unwrap(); + } + for (lang, name) in [(es, "es_ES"), (mx, "es_MX"), (la, "es_419"), (uy, "es_UY")] { + assert_eq!(crate::hunspell::find_in(&dir, variety(lang).1).map(|(_, dic)| dic), Some(dir.join(format!("{name}.dic")))); + assert_eq!(dictionary_name(lang), name); + } + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn spanish_words_affixes_case_and_hyphens() { + test_dict::install(); + let d = dictionary(ProofLang::Es); + assert!(has_dictionary(ProofLang::Es)); + assert!(d.rule_count() > 0); + for w in [ + "canción", + "canciones", + "prueba", + "pruebas", + "corrector", + "correctores", + "también", + "México", + "Canción", + "CANCIÓN", + "teórico-práctico", + "franco-mexicano", + "ONU", + "Año", + "niño", + "pingüino", + ] { + assert!(is_correct(w, ProofLang::Es), "{w}"); + } + for w in ["prueva", "corector", "tambien", "mexico", "cancion", "teórico-prático", "cAnción", "nino"] { + assert!(!is_correct(w, ProofLang::Es), "{w}"); + } + } + + #[test] + fn varieties_keep_their_own_dictionaries() { + test_dict::install(); + // The Americas' fixture lacks `vosotros` forms and has American words; Spain's is the other way round. + assert!(is_correct("habéis", ProofLang::Es) && !is_correct("habéis", ProofLang::EsMx)); + assert!(is_correct("platicar", ProofLang::EsMx) && !is_correct("platicar", ProofLang::Es)); + assert!(is_correct("platicar", ProofLang::Es419) && is_correct("computadora", ProofLang::Es419)); + assert!(!is_correct("habéis", ProofLang::Es419) && !is_correct("habéis", ProofLang::EsUy)); + assert!(is_correct("ordenador", ProofLang::Es) && !is_correct("ordenador", ProofLang::EsMx)); + } + + /// Uruguayan Spanish accepts the `vos` forms of the dictionary's verbs, which the other + /// varieties flag. + #[test] + fn uruguayan_spanish_accepts_vos_forms() { + test_dict::install(); + for w in ["tenés", "vení", "Vení", "decime", "decímelo", "andate", "mirá", "hablás", "querés", "venís", "fijate"] { + assert!(is_correct(w, ProofLang::EsUy), "{w}"); + assert!(!is_correct(w, ProofLang::Es419) && !is_correct(w, ProofLang::Es), "{w}"); + } + for w in ["tenéz", "venì", "decimelo", "decíme", "zqbailá"] { + assert!(!is_correct(w, ProofLang::EsUy), "{w}"); + } + } + + #[test] + fn forbidden_words_are_rejected_in_every_case() { + test_dict::install(); + for w in ["zqprohibida", "Zqprohibida", "ZQPROHIBIDA"] { + assert!(!is_correct(w, ProofLang::Es), "{w}"); + } + } + + #[test] + fn missing_dictionary_and_ignored_inputs_are_quiet() { + let empty = AffixDict::default(); + assert!(is_correct_with(&empty, "prueva", ProofLang::Es)); + assert!(install_dictionary(ProofLang::Es, "FLAG long\n", "1\ncasa\n").is_err()); + let d = AffixDict::from_hunspell(test_dict::AFF, test_dict::DIC, &[]).unwrap(); + for w in ["", "a", "2026", "EE", "1.º", "日本語", "-", "--"] { + assert!(is_correct_with(&d, w, ProofLang::EsUy), "{w}"); + } + } + + #[test] + fn user_dictionary_applies() { + test_dict::install(); + assert!(!is_correct("zqpalabrapropia", ProofLang::EsMx)); + crate::add_word("Zqpalabrapropia"); + assert!(is_correct("zqpalabrapropia", ProofLang::EsMx)); + } + + #[test] + fn spanish_dispatch_messages_options_and_mixed_spans() { + test_dict::install(); + for tag in ["es-ES", "es-MX", "es-419", "es-AR", "es-UY", "spa"] { + assert!(crate::has_dictionary(Some(tag)), "{tag}"); + } + assert!(crate::is_correct_for("canción", Some("es-MX"))); + assert!(crate::is_correct_for("tenés", Some("es-UY")) && !crate::is_correct_for("tenés", Some("es-MX"))); + // The sentence from #3553. + let text = "esto es una prueva del corector"; + for lang in VARIETIES { + let issues = crate::check_text(text, &[(0..text.len(), lang)]); + let flagged: Vec<&str> = issues.iter().map(|i| &text[i.start..i.end]).collect(); + assert_eq!(flagged, ["prueva", "corector"], "{lang:?}: {issues:?}"); + assert!(issues.iter().all(|i| i.message == "Posible error ortográfico" && i.lang == lang), "{issues:?}"); + assert!(crate::suggest_in("prueva", 8, lang).contains(&"prueba".into()), "{lang:?}"); + assert!(crate::suggest_in("corector", 8, lang).contains(&"corrector".into()), "{lang:?}"); + } + let text = "Canción this is is wrong"; + let issues = crate::check_text(text, &[(0..8, ProofLang::Es419), (9..text.len(), ProofLang::En)]); + assert!(issues.iter().any(|i| i.lang == ProofLang::En && i.message.starts_with("Repeated word"))); + assert!(issues.iter().all(|i| i.start >= 9)); + let opts = crate::Options { ignore_uppercase: false, ..Default::default() }; + assert!(!crate::is_correct_in_with("ZQPROHIBIDA", ProofLang::Es, &opts)); + } +} diff --git a/crates/proof/src/es/suggest.rs b/crates/proof/src/es/suggest.rs new file mode 100644 index 00000000..01c8e4a5 --- /dev/null +++ b/crates/proof/src/es/suggest.rs @@ -0,0 +1,99 @@ +//! Spanish suggestions. The ranking is the dictionary-driven one Czech uses +//! ([`crate::cs::suggest::suggest_in`]: case, MAP — the Spanish dictionaries map each vowel to +//! its accented forms and `n` to `ñ`, so `tambien` → `también` comes first —, REP, missing spaces, +//! KEY, TRY edits and a stem scan); every table comes from the variety's installed dictionary. +//! In Uruguayan Spanish a `vos` form typed without its accent is offered with it first +//! (`veni` → `vení`, `tenes` → `tenés`; [`super::voseo`]). + +use crate::ProofLang; +use crate::affix::{AffixDict, Casing, capitalize, casing}; + +/// Longest word suggestions are looked for, in letters (as in the shared ranking). +const MAX_LETTERS: usize = 40; + +/// Spelling suggestions for a word in `lang`'s Spanish, best first, at most `max`, in the word's +/// case. +pub fn suggest(word: &str, max: usize, lang: ProofLang) -> Vec { + suggest_in(&super::dictionary(lang), word, max, lang) +} + +/// [`suggest`] against a given dictionary. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize, lang: ProofLang) -> Vec { + let mut out = Vec::new(); + let word = word.trim(); + if lang == ProofLang::EsUy + && !d.is_empty() + && word.chars().count() <= MAX_LETTERS + && let Some(v) = super::voseo::accented(&word.to_lowercase(), |inf| d.check(inf)) + { + out.push(match casing(word) { + Casing::Upper => v.to_uppercase(), + Casing::Initial => capitalize(&v), + _ => v, + }); + } + for s in crate::cs::suggest::suggest_in(d, word, max) { + if !out.contains(&s) { + out.push(s); + } + } + out.truncate(max); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn spanish_suggestions_use_runtime_tables_and_case() { + crate::es::test_dict::install(); + for lang in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + for (word, want) in [ + ("tambien", "también"), + ("cancion", "canción"), + ("Cancion", "Canción"), + ("mexico", "México"), + ("nino", "niño"), + ("pinguino", "pingüino"), + ("prueva", "prueba"), + ("corector", "corrector"), + ("estoes", "esto es"), + ] { + assert!(suggest(word, 8, lang).contains(&want.into()), "{lang:?} {word}: {:?}", suggest(word, 8, lang)); + } + } + // The first suggestion for a missing accent is the accented word. + assert_eq!(suggest("tambien", 4, ProofLang::Es).first().map(String::as_str), Some("también")); + } + + #[test] + fn uruguayan_vos_forms_get_their_accent_back_first() { + crate::es::test_dict::install(); + for (word, want) in [("veni", "vení"), ("tenes", "tenés"), ("Veni", "Vení"), ("TENES", "TENÉS"), ("queres", "querés")] { + assert_eq!(suggest(word, 4, ProofLang::EsUy).first().map(String::as_str), Some(want), "{word}"); + assert!(!suggest(word, 8, ProofLang::Es419).contains(&want.into()), "{word}: no vos forms outside Uruguay"); + } + } + + #[test] + fn suggestions_stay_bounded_and_skip_forbidden_words() { + crate::es::test_dict::install(); + for lang in [ProofLang::Es, ProofLang::EsUy] { + for w in ["", "dos palabras", &"a".repeat(41)] { + assert!(suggest(w, 4, lang).is_empty(), "{w}"); + } + assert!(suggest("prueva", 0, lang).is_empty()); + assert!(suggest("veni", 0, lang).is_empty()); + assert!(suggest_in(&AffixDict::default(), "veni", 4, lang).is_empty()); + assert!(suggest("prueba", 4, lang).len() <= 4); + assert!(!suggest("prueba", 4, lang).contains(&"prueba".into())); + assert!(!suggest("zqprohibda", 8, lang).contains(&"zqprohibida".into())); + } + } + + #[test] + fn nosuggest_stems_and_forms_are_never_offered() { + crate::affix::tests::assert_nosuggest_is_honoured(|d, w, max| suggest_in(d, w, max, ProofLang::Es)); + } +} diff --git a/crates/proof/src/es/test_dict.rs b/crates/proof/src/es/test_dict.rs new file mode 100644 index 00000000..6b3e6b1f --- /dev/null +++ b/crates/proof/src/es/test_dict.rs @@ -0,0 +1,111 @@ +//! Hand-written synthetic Spanish test data. No words, rules or suggestion tables were copied +//! from a system dictionary. This is a small fixture, not a redistributable Spanish dictionary. +//! Spain's word list has `vosotros` forms and Spain's words; the Americas' list (Mexico, Latin +//! America and Uruguay) has theirs instead, so the tests can tell the varieties' dictionaries apart. + +use crate::ProofLang; + +pub const AFF: &str = "SET UTF-8 +FLAG UTF-8 +FORBIDDENWORD F +TRY aeosrnidlctumpbgvyqhfzjñáéíóúüxkw +KEY qwertyuiop|asdfghjklñ|zxcvbnm +MAP 6 +MAP aá +MAP eé +MAP ií +MAP oó +MAP uúü +MAP nñ +REP 2 +REP v b +REP b v +SFX S Y 2 # synthetic plurals +SFX S 0 s [aeiou] +SFX S ón ones ón +SFX E Y 1 +SFX E 0 es [^aeiou] +"; + +/// Spain. +pub const DIC: &str = "32 +hablar +tener +venir +decir +andar +mirar +querer +fijar +poner +canción/S +prueba/S +corrector/E +también +México +esto +es +una +del +de +la +el +y +niño/S +pingüino/S +año/S +teórico +práctico +mexicano +franco +habéis +ordenador/E +zqprohibida/F +"; + +/// Mexico, Latin America and Uruguay. +pub const DIC_AMERICAS: &str = "33 +hablar +tener +venir +decir +andar +mirar +querer +fijar +poner +canción/S +prueba/S +corrector/E +también +México +esto +es +una +del +de +la +el +y +niño/S +pingüino/S +año/S +teórico +práctico +mexicano +franco +platicar +computadora/S +han +zqprohibida/F +"; + +/// Install the fixture for every variety: [`DIC`] for Spain, [`DIC_AMERICAS`] for Mexico, Latin +/// America and Uruguay. +pub fn install() { + for (lang, dic) in [(ProofLang::Es, DIC), (ProofLang::EsMx, DIC_AMERICAS), (ProofLang::Es419, DIC_AMERICAS), (ProofLang::EsUy, DIC_AMERICAS)] { + if let Err(e) = super::install_dictionary(lang, AFF, dic) { + panic!("test dictionary: {e}"); + } + } +} diff --git a/crates/proof/src/es/voseo.rs b/crates/proof/src/es/voseo.rs new file mode 100644 index 00000000..d9753e12 --- /dev/null +++ b/crates/proof/src/es/voseo.rs @@ -0,0 +1,155 @@ +//! The `vos` forms of verbs in Uruguayan Spanish ([`crate::ProofLang::EsUy`]), which dictionaries +//! built around `tú` may lack (#3360). Rioplatense `voseo` builds them regularly from the +//! infinitive, stress on the ending, without the stem changes `tú` has: +//! +//! - the imperative drops the infinitive's `r` and accents the vowel: `hablar` → `hablá`, +//! `tener` → `tené`, `venir` → `vení`, `decir` → `decí` (`ir` uses `andá`, from `andar`); +//! - with one pronoun after it the accent goes (`decime`, `fijate`, `andate`, `poneme`), with two +//! it comes back (`decímelo`, `mandámela`); `-ír` verbs keep theirs (`sonreíte`); +//! - the present indicative ends in `-ás`, `-és`, `-ís`: `hablás`, `tenés`, `venís`. +//! +//! A form is accepted only when the dictionary has its verb's infinitive, so nothing but these +//! endings is hard-coded. Other `vos` forms (`sos`) are ordinary dictionary words. + +/// Pronouns that come first when two follow an imperative (`decímelo`, `dáselo`). +const FIRST_PRONOUNS: [&str; 4] = ["me", "te", "se", "nos"]; +/// Pronouns that come second (or alone). +const SECOND_PRONOUNS: [&str; 6] = ["lo", "la", "le", "los", "las", "les"]; + +/// Shortest infinitive considered (`dar` and `ir` have no `vos` imperative of their own here, and +/// short stems would let nouns such as `bar` pass for verbs). +const MIN_INFINITIVE: usize = 4; + +/// Is `word` (one word, any case) a `vos` form of a verb whose infinitive `is_word` accepts? +pub fn is_vos_form(word: &str, is_word: impl Fn(&str) -> bool) -> bool { + let w = word.to_lowercase(); + if w.chars().count() > crate::affix::MAX_WORD_BYTES { + return false; + } + infinitives(&w).iter().any(|inf| inf.chars().count() >= MIN_INFINITIVE && is_word(inf)) +} + +/// The infinitives `w` (lower case) could be a `vos` form of, by its ending. +fn infinitives(w: &str) -> Vec { + let mut out = Vec::new(); + // Present indicative: `hablás`, `tenés`, `venís`. + if let Some(stem) = w.strip_suffix('s') { + imperative(stem, &mut out); + } + // Imperative: `hablá`, `tené`, `vení`. + imperative(w, &mut out); + // With one pronoun the accent goes: `decime`, `fijate`; `-ír` verbs keep it: `sonreíte`. + for p in FIRST_PRONOUNS.iter().chain(SECOND_PRONOUNS.iter()) { + if let Some(base) = w.strip_suffix(p) + && base.ends_with(['a', 'e', 'i', 'í']) + { + out.push(format!("{base}r")); + } + } + // With two the accent comes back: `decímelo`, `mandámela`. + for second in SECOND_PRONOUNS { + let Some(rest) = w.strip_suffix(second) else { continue }; + for first in FIRST_PRONOUNS { + if let Some(base) = rest.strip_suffix(first) { + imperative(base, &mut out); + } + } + } + out +} + +/// The infinitives of an imperative ending in an accented vowel: `hablá` → `hablar`, and for `í` +/// both `vení` → `venir` and `sonreí` → `sonreír`. +fn imperative(form: &str, out: &mut Vec) { + let mut chars = form.chars(); + let Some(last) = chars.next_back() else { return }; + let stem = chars.as_str(); + match last { + 'á' => out.push(format!("{stem}ar")), + 'é' => out.push(format!("{stem}er")), + 'í' => { + out.push(format!("{stem}ir")); + out.push(format!("{stem}ír")); + } + _ => {} + } +} + +/// The `vos` form `word` (lower case, without its accent) was most likely meant as, when the +/// writer left the accent out: `veni` → `vení`, `tenes` → `tenés`. `None` when it isn't one. +pub fn accented(word: &str, is_word: impl Fn(&str) -> bool) -> Option { + let (body, s) = match word.strip_suffix('s') { + Some(b) => (b, "s"), + None => (word, ""), + }; + let mut chars = body.chars(); + let accent = match chars.next_back()? { + 'a' => 'á', + 'e' => 'é', + 'i' => 'í', + _ => return None, + }; + let form = format!("{}{accent}{s}", chars.as_str()); + is_vos_form(&form, is_word).then_some(form) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn verbs(w: &str) -> bool { + ["hablar", "tener", "venir", "decir", "andar", "mirar", "sonreír", "fijar", "mandar", "poner", "bar", "querer"].contains(&w) + } + + #[test] + fn imperatives_present_and_pronouns_come_from_the_infinitive() { + for w in [ + "hablá", + "tené", + "vení", + "decí", + "andá", + "mirá", + "poné", + "hablás", + "tenés", + "venís", + "querés", + "decime", + "fijate", + "andate", + "poneme", + "sonreíte", + "sonreí", + "decímelo", + "mandámela", + "decinos", + "Vení", + "TENÉS", + ] { + assert!(is_vos_form(w, verbs), "{w}"); + } + // Wrong accents, verbs the dictionary lacks, and short stems (`bame` isn't `bar`). + for w in ["habla", "decíme", "decínos", "decimelo", "tenéz", "cantá", "bá", "bame", "", "s", "á", "més", "lo", "melo"] { + assert!(!is_vos_form(w, verbs), "{w}"); + } + } + + #[test] + fn missing_accents_are_restored() { + assert_eq!(accented("veni", verbs).as_deref(), Some("vení")); + assert_eq!(accented("tenes", verbs).as_deref(), Some("tenés")); + assert_eq!(accented("habla", verbs).as_deref(), Some("hablá")); + for w in ["canta", "casa", "", "s", "pan", "tenés"] { + assert_eq!(accented(w, verbs), None, "{w}"); + } + } + + proptest::proptest! { + #[test] + fn never_panics(s in "\\PC{0,40}") { + let _ = is_vos_form(&s, |_| true); + let _ = accented(&s, |_| true); + } + } +} diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 9fdae033..a2adaba0 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -856,7 +856,7 @@ mod tests { #[test] fn languages_without_patterns_keep_the_english_ones() { for w in ["hyphenation", "typography", "internationalization"] { - for lang in [ProofLang::Other, ProofLang::Cs, ProofLang::Id] { + for lang in [ProofLang::Other, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { assert_eq!(hyphen_points_in(w, &Limits::default(), lang), hyphen_points(w, &Limits::default()), "{w}"); } } diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index f94ec95f..95587f69 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -49,6 +49,22 @@ pub enum ProofLang { /// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian /// messages. Hyphenation keeps the fallback English patterns. Id, + /// Spanish of Spain (`es`, `es-ES`, and every region not listed for [`ProofLang::EsMx`] or + /// [`ProofLang::Es419`]; also ISO 639-2 `spa`): the installed Hunspell `es_ES` dictionary + /// (or a generic Spanish one), dictionary-driven suggestions and generic checks with Spanish + /// messages ([`crate::es`]). Hyphenation keeps the fallback English patterns. + Es, + /// Mexican Spanish (`es-MX`): [`ProofLang::Es`]'s tools with the installed `es_MX` + /// dictionary, else a Latin American or generic Spanish one. + EsMx, + /// Latin American Spanish (`es-419`, and the Spanish of the Americas other than Mexico and + /// Uruguay: `es-AR`, `es-CO`, `es-US`…): [`ProofLang::Es`]'s tools with the installed `es_419` + /// dictionary, else a generic or another Latin American Spanish one. + Es419, + /// Uruguayan Spanish (`es-UY`): [`ProofLang::Es`]'s tools with the installed `es_UY` + /// dictionary (else the neighbouring Rioplatense `es_AR`, a Latin American or a generic one), + /// and the `vos` forms of verbs (`tenés`, `vení`, `decime`) accepted ([`crate::es::voseo`]). + EsUy, /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without /// the language's proofing tools: no spelling or grammar marks, so its words are never /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before @@ -58,7 +74,7 @@ pub enum ProofLang { impl ProofLang { /// Every language with a spelling dictionary, in display order. - pub const PROOFED: [ProofLang; 11] = [ + pub const PROOFED: [ProofLang; 15] = [ ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, @@ -70,11 +86,16 @@ impl ProofLang { ProofLang::Ru, ProofLang::Cs, ProofLang::Id, + ProofLang::Es, + ProofLang::EsMx, + ProofLang::Es419, + ProofLang::EsUy, ]; /// The proofing language of a BCP 47 tag (`pl-PL`, `en_GB`, `PL`…). The primary subtag - /// counts, and for English the region too ([`ProofLang::english`]). No tag (or an empty one) - /// is English, WordCraft's default editing language. + /// counts, and for English and Spanish the region too ([`ProofLang::english`], + /// [`ProofLang::spanish`]). No tag (or an empty one) is English, WordCraft's default editing + /// language. pub fn from_tag(tag: Option<&str>) -> ProofLang { let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En }; let primary = tag.split(['-', '_']).next().unwrap_or(""); @@ -98,36 +119,48 @@ impl ProofLang { ProofLang::Cs } else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") { ProofLang::Id + } else if primary.eq_ignore_ascii_case("es") || primary.eq_ignore_ascii_case("spa") { + ProofLang::spanish(tag) } else { ProofLang::Other } } - /// The English of an English tag, by its region (the first two-letter or three-digit subtag - /// after the language and script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored): Canada is - /// [`ProofLang::EnCa`], regions that spell the British way are [`ProofLang::EnGb`], and the - /// United States, any other region and no region are [`ProofLang::En`]. + /// The English of an English tag, by its [`region`]: Canada is [`ProofLang::EnCa`], regions + /// that spell the British way are [`ProofLang::EnGb`], and the United States, any other + /// region and no region are [`ProofLang::En`]. fn english(tag: &str) -> ProofLang { - let tag = tag.split(['.', '@']).next().unwrap_or(""); - // Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions. - let region = tag - .split(['-', '_']) - .skip(1) - .take_while(|p| p.len() != 1) - .find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit()))); - match region { + match region(tag) { Some(r) if r.eq_ignore_ascii_case("CA") => ProofLang::EnCa, Some(r) if BRITISH_SPELLING.iter().any(|b| b.eq_ignore_ascii_case(r)) => ProofLang::EnGb, _ => ProofLang::En, } } + /// The Spanish of a Spanish tag, by its [`region`]: Mexico is [`ProofLang::EsMx`], Uruguay + /// [`ProofLang::EsUy`], Latin America (`419`) and the other countries of the Americas are + /// [`ProofLang::Es419`], and Spain, any other region and no region are [`ProofLang::Es`]. + fn spanish(tag: &str) -> ProofLang { + match region(tag) { + Some(r) if r.eq_ignore_ascii_case("MX") => ProofLang::EsMx, + Some(r) if r.eq_ignore_ascii_case("UY") => ProofLang::EsUy, + Some(r) if LATIN_AMERICA.iter().any(|l| l.eq_ignore_ascii_case(r)) => ProofLang::Es419, + _ => ProofLang::Es, + } + } + /// Is this one of the Englishes ([`ProofLang::En`], [`ProofLang::EnGb`], [`ProofLang::EnCa`])? pub fn is_english(self) -> bool { matches!(self, ProofLang::En | ProofLang::EnGb | ProofLang::EnCa) } - /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`); empty for [`ProofLang::Other`]. + /// Is this one of the Spanishes ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], + /// [`ProofLang::EsUy`])? + pub fn is_spanish(self) -> bool { + matches!(self, ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy) + } + + /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `es`); empty for [`ProofLang::Other`]. pub fn code(self) -> &'static str { match self { ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en", @@ -140,6 +173,7 @@ impl ProofLang { ProofLang::Ru => "ru", ProofLang::Cs => "cs", ProofLang::Id => "id", + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es", ProofLang::Other => "", } } @@ -150,6 +184,24 @@ impl ProofLang { } } +/// The region of a tag: the first two-letter or three-digit subtag after the language and +/// script, before any singleton; a POSIX `.UTF-8` or `@…` tail is ignored. +fn region(tag: &str) -> Option<&str> { + let tag = tag.split(['.', '@']).next().unwrap_or(""); + // Subtags after a singleton (`x-…` private use, `u-…` extensions) aren't regions. + tag.split(['-', '_']) + .skip(1) + .take_while(|p| p.len() != 1) + .find(|p| (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit()))) +} + +/// Regions whose Spanish is Latin American ([`ProofLang::Es419`]): Latin America and the +/// Caribbean (`419`) and its parts (South America `005`, Central America `013`, the Caribbean +/// `029`), the Spanish-speaking countries of the Americas other than Mexico and Uruguay, and the +/// United States. +const LATIN_AMERICA: [&str; 22] = + ["419", "005", "013", "029", "AR", "BO", "CL", "CO", "CR", "CU", "DO", "EC", "GT", "HN", "NI", "PA", "PE", "PR", "PY", "SV", "US", "VE"]; + /// Regions whose English uses British spelling ([`ProofLang::EnGb`]): the United Kingdom (and /// the non-standard `UK`), Ireland, Australia, New Zealand, South Africa, India and other /// Commonwealth countries, the Crown dependencies and Europe (`150`). @@ -275,7 +327,7 @@ mod tests { for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}"); } - for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] { + for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}"); } assert_eq!(ProofLang::from_tag(None), ProofLang::En); @@ -302,4 +354,29 @@ mod tests { assert_eq!(ProofLang::Tr.code(), "tr"); assert!(ProofLang::PROOFED.contains(&ProofLang::Pt), "every proofed language is listed"); } + + /// #3360, #3553: Spanish is proofed, by region: Spain (and no region), Mexico, Uruguay, and + /// Latin America with the other countries of the Americas. + #[test] + fn spanish_tags_resolve_by_region() { + for t in ["es", "es-ES", "ES_es", "es_ES.UTF-8", " es-ES ", "spa", "es-Latn-ES", "es-GQ", "es-PH", "es-x-mx", "es_ES@euro"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Es, "{t}"); + } + for t in ["es-MX", "es_MX", "ES-mx", "es_MX.UTF-8", "spa-MX", "es-Latn-MX"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EsMx, "{t}"); + } + for t in ["es-419", "es_419", "es-AR", "es-CO", "es_CL.UTF-8", "es-US", "ES-ve", "es-PR", "es-Latn-419", "es-005", "spa-PE"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Es419, "{t}"); + } + for t in ["es-UY", "es_UY", "ES-uy", "es_UY.UTF-8", "spa-UY", "es-Latn-UY"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::EsUy, "{t}"); + } + for l in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + assert!(l.is_spanish() && l.is_proofed() && ProofLang::PROOFED.contains(&l), "{l:?}"); + assert!(!l.is_english()); + assert_eq!(l.code(), "es"); + } + assert!(!ProofLang::En.is_spanish() && !ProofLang::Pt.is_spanish() && !ProofLang::Other.is_spanish()); + assert_eq!(normalize_tag("ES_mx").as_deref(), Some("es-MX")); + } } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 9347fba7..e79b3e5d 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -34,6 +34,11 @@ //! - **Indonesian** ([`ProofLang::Id`], module [`id`]): the installed Hunspell `id_ID` //! dictionary (not bundled), dictionary-driven suggestions and generic checks with Indonesian //! messages. Hyphenation uses the existing English fallback. +//! - **Spanish** ([`ProofLang::Es`], [`ProofLang::EsMx`], [`ProofLang::Es419`], [`ProofLang::EsUy`], +//! module [`es`]): each variety's installed Hunspell dictionary (`es_ES`, `es_MX`, `es_419`, +//! `es_UY`, with fallbacks; not bundled), dictionary-driven suggestions, Uruguayan `vos` forms, +//! and generic checks plus the opening `¿`/`¡` with Spanish messages. Hyphenation uses the +//! existing English fallback. //! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without //! the language's proofing tools. //! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese), @@ -47,6 +52,7 @@ pub mod bg; pub mod cs; pub mod dict; pub mod en_variants; +pub mod es; pub mod hr; pub mod hunspell; pub mod hyphen; @@ -152,6 +158,7 @@ pub fn set_dictionaries_folder(dir: Option) { hr::reload_dictionary(); tr::reload_dictionary(); id::reload_dictionary(); + es::reload_dictionary(); DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); } @@ -165,6 +172,10 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> (ProofLang::Hr, hr::installed_path()), (ProofLang::Tr, tr::installed_path()), (ProofLang::Id, id::installed_path()), + (ProofLang::Es, es::installed_path(ProofLang::Es)), + (ProofLang::EsMx, es::installed_path(ProofLang::EsMx)), + (ProofLang::Es419, es::installed_path(ProofLang::Es419)), + (ProofLang::EsUy, es::installed_path(ProofLang::EsUy)), ] } @@ -179,6 +190,7 @@ pub fn dictionary_load_error(lang: ProofLang) -> Option { ProofLang::Hr => "hr_HR", ProofLang::Tr => "tr_TR", ProofLang::Id => "id_ID", + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::dictionary_name(lang), _ => return None, }; hunspell::load_error(key).map(|(_, reason)| reason) @@ -397,6 +409,7 @@ pub fn missing_dictionary(lang: Option<&str>) -> Option<&'static str> { match ProofLang::from_tag(lang) { ProofLang::Pl => dictionary_needed(ProofLang::Pl, pl::has_dictionary()), ProofLang::Pt => dictionary_needed(ProofLang::Pt, pt::has_dictionary()), + l if l.is_spanish() => dictionary_needed(l, es::has_dictionary(l)), _ => None, } } @@ -409,6 +422,7 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p let (proof_lang, loaded) = match ProofLang::from_tag(lang) { ProofLang::Pl => (ProofLang::Pl, pl::has_dictionary()), ProofLang::Pt => (ProofLang::Pt, pt::has_dictionary()), + l if l.is_spanish() => (l, es::has_dictionary(l)), _ => return None, }; let name = dictionary_needed(proof_lang, false)?; @@ -418,12 +432,14 @@ pub fn unreadable_dictionary(lang: Option<&str>) -> Option<(&'static str, std::p hunspell::load_error(name).map(|(file, reason)| (name, file, reason)) } -/// The Hunspell name `lang` needs when `installed` is false; only Polish and Portuguese use one. +/// The Hunspell name `lang` needs when `installed` is false; Polish, Portuguese and Spanish (per +/// variety: `es_ES`, `es_MX`, `es_419`, `es_UY`) name one. fn dictionary_needed(lang: ProofLang, installed: bool) -> Option<&'static str> { match lang { _ if installed => None, ProofLang::Pl => Some("pl_PL"), ProofLang::Pt => Some("pt_BR"), + l if l.is_spanish() => Some(es::dictionary_name(l)), _ => None, } } @@ -698,6 +714,7 @@ fn spelling_issue(text: &str, a: usize, b: usize, lang: ProofLang, opts: &Option ProofLang::Ru => "Возможна орфографическая ошибка", ProofLang::Cs => "Možná pravopisná chyba", ProofLang::Id => "Kemungkinan salah eja", + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "Posible error ortográfico", _ => "Possible spelling mistake", }; Some(Issue { start: a, end: b, kind: IssueKind::Spelling, message: message.into(), suggestions: Vec::new(), lang }) @@ -720,6 +737,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word), ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word), ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word), + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => accepted_by_options(word, opts) || es::is_correct(word, lang), ProofLang::Bg | ProofLang::Other => true, } } @@ -736,13 +754,15 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { ProofLang::Ru => ru::suggest(word, max), ProofLang::Cs => cs::suggest(word, max), ProofLang::Id => id::suggest(word, max), + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang), ProofLang::Bg | ProofLang::Other => Vec::new(), } } /// Grammar issues in a paragraph of text in `lang`: the English rules, the Polish ones /// ([`pl::grammar`]), the Portuguese ones ([`pt::grammar`]), the Italian ones ([`it::grammar`]), -/// the Turkish ones ([`tr::grammar`]), Czech and Indonesian generic checks ([`cs::grammar`], [`id::grammar`]), or none for a language WordCraft doesn't proof. +/// the Turkish ones ([`tr::grammar`]), Czech and Indonesian generic checks ([`cs::grammar`], [`id::grammar`]), the Spanish ones +/// ([`es::grammar`]), or none for a language WordCraft doesn't proof. pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { grammar_in(text, lang, true) } @@ -762,6 +782,8 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { // Czech has no sentence-capital rule, so a list item needs no exception. ProofLang::Cs => cs::grammar::check(text), ProofLang::Id => id::grammar::check(text), + // Spanish has no sentence-capital rule either. + ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang), ProofLang::Bg | ProofLang::Other => Vec::new(), } } @@ -951,13 +973,19 @@ mod tests { assert_eq!(dictionary_needed(ProofLang::Pt, false), Some("pt_BR")); assert_eq!(dictionary_needed(ProofLang::Pl, true), None); assert_eq!(dictionary_needed(ProofLang::Pt, true), None); + // #3360: each Spanish variety names its own. + for (l, name) in [(ProofLang::Es, "es_ES"), (ProofLang::EsMx, "es_MX"), (ProofLang::Es419, "es_419"), (ProofLang::EsUy, "es_UY")] { + assert_eq!(dictionary_needed(l, false), Some(name)); + assert_eq!(dictionary_needed(l, true), None); + } for l in [ProofLang::En, ProofLang::Bg, ProofLang::Other] { assert_eq!(dictionary_needed(l, false), None); } // Installed dictionaries and the built-in English list: no hint. pl::test_dict::install(); pt::test_dict::install(); - for tag in [Some("pl-PL"), Some("pt-BR"), Some("en-US"), Some("fr-FR"), None] { + es::test_dict::install(); + for tag in [Some("pl-PL"), Some("pt-BR"), Some("es-ES"), Some("es-MX"), Some("es-419"), Some("es-UY"), Some("en-US"), Some("fr-FR"), None] { assert_eq!(missing_dictionary(tag), None, "{tag:?}"); } } @@ -978,6 +1006,10 @@ mod tests { ("it_IT", it::test_dict::AFF, it::test_dict::DIC), ("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC), ("id_ID", id::test_dict::AFF, id::test_dict::DIC), + ("es_ES", es::test_dict::AFF, es::test_dict::DIC), + ("es_MX", es::test_dict::AFF, es::test_dict::DIC_AMERICAS), + ("es_419", es::test_dict::AFF, es::test_dict::DIC_AMERICAS), + ("es_UY", es::test_dict::AFF, es::test_dict::DIC_AMERICAS), ] { std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap(); @@ -985,10 +1017,19 @@ mod tests { let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); - // Turkish has no test dictionary to put in the folder; Indonesian is listed after it. + // Turkish has no test dictionary to put in the folder; Indonesian and the Spanishes are listed after it. let found = installed_dictionaries(); assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr)); - assert_eq!(found.last(), Some(&(ProofLang::Id, Some(dir.join("id_ID.dic"))))); + assert_eq!( + found[5..], + [ + (ProofLang::Id, Some(dir.join("id_ID.dic"))), + (ProofLang::Es, Some(dir.join("es_ES.dic"))), + (ProofLang::EsMx, Some(dir.join("es_MX.dic"))), + (ProofLang::Es419, Some(dir.join("es_419.dic"))), + (ProofLang::EsUy, Some(dir.join("es_UY.dic"))), + ] + ); assert_eq!( found[..4], [ @@ -999,6 +1040,7 @@ mod tests { ] ); assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); + assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy)); } /// #1741: a list item may be a fragment, so its first word needs no capital in any language @@ -1541,7 +1583,7 @@ mod tests { for l in [None, Some(""), Some(" "), Some("en"), Some("en-US"), Some("en-GB"), Some("EN-us"), Some("en_US")] { assert!(has_dictionary(l), "{l:?}"); } - for l in [Some("lt-LT"), Some("fr-FR"), Some("de"), Some("uk"), Some("es"), Some("-US"), Some("e")] { + for l in [Some("lt-LT"), Some("fr-FR"), Some("de"), Some("uk"), Some("-US"), Some("e")] { assert!(!has_dictionary(l), "{l:?}"); } // The gate follows the dictionaries, not English: every proofed language is proofed. @@ -1662,7 +1704,7 @@ mod tests { let _ = has_dictionary(Some(&s)); let _ = is_correct_for(&s, Some(&s)); let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); - for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] { + for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other] { let _ = check_spelling_in(&s, lang); let _ = check_grammar_in(&s, lang); let _ = is_correct_in(&s, lang); @@ -1706,8 +1748,17 @@ mod tests { } #[test] - fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..11), 0..6)) { - let langs = [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other]; + fn spanish_suggestions_never_panic(s in "[a-zA-ZáéíóúüñÁÉÍÓÚÜÑ¿¡'’ -]{0,14}") { + es::test_dict::install(); + for lang in [ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy] { + let _ = suggest_in(&s, 4, lang); + let _ = is_correct_in(&s, lang); + } + } + + #[test] + fn mixed_spans_never_panic(s in "\\PC{0,60}", cuts in proptest::collection::vec((0usize..80, 0usize..80, 0u8..15), 0..6)) { + let langs = [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other]; let spans: Vec<(Range, ProofLang)> = cuts.iter().map(|&(a, b, l)| (a.min(b)..a.max(b), langs[l as usize % langs.len()])).collect(); for i in check_text(&s, &spans) { proptest::prop_assert!(i.start <= i.end && i.end <= s.len());