diff --git a/crates/proof/README.md b/crates/proof/README.md index 970670143..b4b4b5ba8 100644 --- a/crates/proof/README.md +++ b/crates/proof/README.md @@ -16,6 +16,7 @@ craft-proof = { git = "https://github.com/storytold/craft-libs", rev = " | English (en-US, en-CA, en-AU, en-GB and the British-spelling Englishes such as en-NZ) | SCOWL 2026 Hunspell dictionary per English + `en-extra.txt`, dialect policy (`en-variants.txt`); a dictionaries-folder English first; without SCOWL the installed Hunspell English, else the built-in Moby list ([below](#english-is-always-checked)) | yes | Moby list, our patterns, heuristic | | Polish, Portuguese, Italian | the installed Hunspell dictionary | yes | TeX patterns (embedded, or a Language pack) | | Croatian, Turkish, Russian, Czech, Indonesian, Spanish (es, es-MX, es-419, es-UY) | the installed Hunspell dictionary (Russian also `ru-extra.txt`) | yes | Turkish and Russian rules; English fallback otherwise | +| French (fr-FR, fr-CA and other French pack locales) | the installed Language pack or Hunspell dictionary; dictionary-driven elisions and accent-oriented suggestions | conservative typography and basic checks, with Quebec punctuation spacing in fr-CA | the Language pack (generic fr fallback) | | German | — | — | TeX patterns (embedded, or a Language pack) | | Bulgarian | — | — | syllable rules | @@ -41,6 +42,12 @@ application data (`%LOCALAPPDATA%\WordCraft\dictionaries` and `%APPDATA%\WordCra `~/Library/Application Support/WordCraft/dictionaries`, `$XDG_DATA_HOME/wordcraft/dictionaries`), then the usual Hunspell and LibreOffice locations. None on the web. +French remains a `ProofLang::Pack` locale: use its full `tag()` for settings and issue identity. +The shared pack loader keeps regional dictionary overrides, fallback to `fr`, reloads and its +memory budget; no separate French dictionary slot or embedded French patterns are added. +Only French packs receive the French spelling tuning and typography; other pack languages +keep their existing behavior. French agreement and conjugation are outside these checks. + State is process-wide: `add_word`, `ignore_word`, `set_dictionaries_folder`, `load_user_dictionary`/`save_user_dictionary`. The crate reads no document model; callers pass plain text and language spans. @@ -221,6 +228,40 @@ CI: `.github/workflows/proof.yml`. storytold/craft-libs#30/#31 by brzuchal (Michał Marcin Brzuchalski) started the shared crate. +### French adaptation validation + +The French rules and original synthetic fixtures are adapted from +[totoz64’s WordCraft contribution #3185](https://github.com/storytold/wordcraft/pull/3185). +They use the shared affix reader and Language pack cache; WCAFF1 remains deliberately rejected. +The historical text fixtures retain their original expected data. Three observations for +Indonesian `bermainkanlah` are explicitly updated in the compatibility test because the +shared reader already supports that prefix plus two suffixes. This is a finite regression +check, not a universal compatibility claim. + +The ordinary tests cover French packs, regional overrides and reloads, options, user words, +NFC/NFD lookup with original ranges, suggestions and pack hyphenation. The opt-in real French +dictionary gate fails when its input is missing; it never silently skips its assertions: + +```sh +WORDCRAFT_DICTIONARIES=/absolute/isolated/french-dictionary cargo test -p craft-proof --test french_dictionary -- --ignored +``` + +For a reproducible real-dictionary run, use Dicollecte 7.7 from LibreOffice/dictionaries +revision `67032ee957631908098a72edf48dc81cac27e369`: +[fr.aff](https://raw.githubusercontent.com/LibreOffice/dictionaries/67032ee957631908098a72edf48dc81cac27e369/fr_FR/dictionaries/fr.aff) and +[fr.dic](https://raw.githubusercontent.com/LibreOffice/dictionaries/67032ee957631908098a72edf48dc81cac27e369/fr_FR/dictionaries/fr.dic). +Verify SHA-256 before invoking the gate (the test checks location and behavior, not hashes): + +| File | SHA-256 | +|---|---| +| fr.aff | `c176610cd5dc4846806a65ddd029f422d87978bf58f224aa44222662a16a2de5` | +| fr.dic | `b78a868e31dd6e373b6c3217969afb898a9acde828a5e7ef97308da42218c88c` | + +Use a dedicated directory containing only the intended `fr.aff`/`fr.dic` pair (or explicit +regional pairs for both regions). The gate requires more than 50,000 dictionary entries and +checks the actual source folder, spelling, accent/elision suggestions, regional dispatch, +combining-mark ranges and hyphenated compounds. No real dictionary is committed here. + ## Owners WordCraft maintainers: echelon, dylanmtaylor, aiongg. diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 0df68665b..32de07659 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -2212,6 +2212,37 @@ impl AffixDict { self.check_except(w, self.excluded(true)) } + /// Validate a collected suggestion in its displayed case, before OCONV. Only identity, + /// first-letter and uppercase variants are allowed (including uppercase that keeps sharp s). + /// An exact target excluded from suggestions cannot be rescued by a lowercase fallback. + /// This preserves the checker's homonym, compound and CHECKSHARPS rules without scanning + /// stems or changing which words the spelling checker accepts. + pub(crate) fn can_suggest_cased(&self, original: &str, changed: &str) -> bool { + if original.len() > MAX_INPUT_BYTES || changed.len() > MAX_INPUT_BYTES { + return false; + } + if changed != original + && changed != capitalize(original) + && changed != original.to_uppercase() + && changed != original.split('ß').map(str::to_uppercase).collect::>().join("ß") + { + return false; + } + let target = self.normalize(changed); + if self.is_forbidden(&target) { + return false; + } + // The exact simple-word path includes affixes; asking with no exclusions detects a + // target whose only derivations carry NOSUGGEST, SUBSTANDARD or FORBIDDENWORD. Spend + // the same shared suggestion budget as the normal checker, and abstain if it runs out. + match self.simple_word_budgeted(&target, Excluded::default(), true) { + None => return false, + Some(true) if !self.simple_word(&target, self.excluded(true)) => return false, + _ => {} + } + self.can_suggest(changed) + } + /// Whether `w` is a word only through a `KEEPCASE` stem, so it is correct in this letter case /// alone (`km` from `km/K`, which `Km` and `KM` are not). pub(crate) fn case_locked(&self, w: &str) -> bool { @@ -3298,6 +3329,41 @@ pub(crate) mod tests { assert!(d.check("ASSASSASSASSASS") && !d.check(&"SS".repeat(60))); } + #[test] + fn displayed_casing_rejects_exact_exclusions_before_case_fallback() { + for flag in ["N", "F", "S"] { + let aff = "SET UTF-8\nNOSUGGEST N\nFORBIDDENWORD F\nSUBSTANDARD S\nSFX A Y 1\nSFX A 0 s .\nSFX B Y 1\nSFX B 0 S .\n"; + let dic = format!("5\nnormal/A\nNORMAL/{flag}B\nmixCase\npour\nkm/K\n"); + let d = AffixDict::from_hunspell(&format!("{aff}KEEPCASE K\n"), &dic, &[]).unwrap(); + for (original, changed) in [("normal", "NORMAL"), ("normals", "NORMALS")] { + assert!(!d.can_suggest_cased(changed, changed), "{flag}: {changed}"); + assert!(!d.can_suggest_cased(original, changed), "{flag}: {original} → {changed}"); + } + assert!(d.can_suggest_cased("normal", "Normal")); + assert!(d.can_suggest_cased("mixCase", "MIXCASE")); + assert!(d.can_suggest_cased("km", "km") && !d.can_suggest_cased("km", "KM")); + assert!(!d.can_suggest_cased("normal", "pour"), "no arbitrary replacement"); + let back = AffixDict::from_bytes(&d.to_bytes().unwrap()).unwrap(); + assert!(!back.can_suggest_cased("normal", "NORMAL")); + } + } + + #[test] + fn displayed_casing_uses_normalization_and_native_homonym_policy() { + let aff = "SET UTF-8\nNOSUGGEST N\nSUBSTANDARD S\nIGNORE ()\nICONV 1\nICONV Q NORMAL\n"; + for flag in ["N", "S"] { + let d = AffixDict::from_hunspell(aff, &format!("3\nnormal\nNORMAL/{flag}\nNORMAL\n"), &[]).unwrap(); + // Native homonyms carry exclusions: a second entry does not erase NOSUGGEST. + assert!(!d.contains_suggestible("NORMAL")); + assert!(!d.can_suggest_cased("(normal)", "(NORMAL)")); + assert!(!d.can_suggest_cased("Q", "Q")); + } + let plain = AffixDict::from_hunspell(aff, "3\nnormal\nNORMAL\nNORMAL\n", &[]).unwrap(); + assert!(plain.can_suggest_cased("normal", "NORMAL")); + assert!(plain.can_suggest_cased("Q", "Q")); + assert!(!plain.can_suggest_cased(&"x".repeat(MAX_INPUT_BYTES + 1), "NORMAL")); + } + #[test] fn oconv_converts_suggestions_only() { // Answers as Hunspell 1.7 gives them on the same files: `kbt` → `k’t`. diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index 3b498542c..68336de16 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -426,6 +426,18 @@ pub(crate) fn suggest_in(d: &AffixDict, word: &str, max: usize, t: &Tuning) -> V _ => vec![text.clone()], }; for t in recased { + let eligible = |candidate: &str| { + let original: Vec<_> = text.split(' ').collect(); + let changed: Vec<_> = candidate.split(' ').collect(); + original.len() == changed.len() && original.iter().zip(changed).all(|(a, b)| d.can_suggest_cased(a, b)) + }; + let t = if eligible(&t) { + t + } else if eligible(&text) { + text.clone() + } else { + continue; + }; // Shown as the dictionary's OCONV table writes it (`'` → `’` in French). let t = d.output(&t).into_owned(); if t != word && !out.contains(&t) { @@ -763,6 +775,24 @@ mod tests { assert_eq!(suggest_in(&d, "Piesx", 4, &Tuning::DEFAULT).first().map(String::as_str), Some("Pies")); } + #[test] + fn recasing_keeps_valid_original_when_exact_target_is_excluded() { + for flag in ["N", "F", "S"] { + let aff = "SET UTF-8\nNOSUGGEST N\nFORBIDDENWORD F\nSUBSTANDARD S\nTRY normal\n"; + let d = AffixDict::from_hunspell(aff, &format!("2\nnormal\nNORMAL/{flag}\n"), &[]).unwrap(); + let got = suggest_in(&d, "NORMALL", 4, &Tuning::DEFAULT); + assert!(got.iter().any(|w| w == "normal"), "{flag}: {got:?}"); + assert!(!got.iter().any(|w| w == "NORMAL"), "{flag}: {got:?}"); + let split = AffixDict::from_hunspell(aff, &format!("3\nnormal\nNORMAL/{flag}\npour\n"), &[]).unwrap(); + let got = suggest_in(&split, "NORMALPOUR", 4, &Tuning::DEFAULT); + assert!(got.iter().any(|w| w == "normal pour"), "{flag}: {got:?}"); + assert!(!got.iter().any(|w| w == "NORMAL POUR"), "{flag}: {got:?}"); + } + let d = AffixDict::from_hunspell("NOSUGGEST N\n", "3\nnormal\nNORMAL/N\npour\n", &[]).unwrap(); + assert!(!d.can_suggest_cased("normal", "NORMAL")); + assert!(!d.can_suggest_cased("normal", "pour")); + } + /// A language's tables come after its dictionary's, a TRY alphabet stands in for a missing TRY /// line, and a split rule decides which missing spaces are offered. #[test] diff --git a/crates/proof/src/fr/grammar.rs b/crates/proof/src/fr/grammar.rs new file mode 100644 index 000000000..1d5449b1f --- /dev/null +++ b/crates/proof/src/fr/grammar.rs @@ -0,0 +1,665 @@ +//! French grammar and typography checks: cheap, rule-based and tuned to stay quiet when unsure +//! (a missed mistake costs less than a wrong mark on correct text). Messages are in French. +//! +//! - **Space before `;` `!` `?` `:`**: French typography puts a non-breaking space there — a +//! narrow one (U+202F) before `;` `!` `?`, a full one (U+00A0) before `:`. Only a missing +//! space is flagged (`Bonjour!`): an ordinary one stays quiet until typing turns it into a +//! non-breaking one (no French AutoFormat yet). Quebec French (`fr-CA`) has no space before +//! `;` `!` `?`, only before `:`. Addresses (`https://…`) and times (`10:30`) are left alone. +//! - **Space inside `« »`**: a missing one after `«` or before `»` (`«oui»`) → U+00A0. +//! - **Straight double quotes** around a phrase (`"bonjour"`) → `« bonjour »`; inch marks +//! (`27"`) and unbalanced quotes are left alone. +//! - **Repeated word** (`le le`, `que que`), except repetitions French makes on purpose +//! (`nous nous lavons`, `vous vous trompez`, `si si`, interjections) and a proper noun after +//! the same lower-case word (`de De…`). +//! - **Capital letter at the start of a sentence** after `.` `!` `?`, knowing French +//! abbreviations (`etc.`, `M.`, `p.`, `cf.`), initials, numbers, ellipses, addresses, +//! dialogue (`« Tu viens ? » demanda-t-elle`), inversions (`Quoi ? dit-il`) and exclamations +//! after an interjection (`Ah ! que c'est beau`). +//! - **Elision** (`de avoir` → `d’avoir`, `je ai` → `j’ai`, `le homme` → `l’homme`) for `le`, +//! `la`, `de`, `je`, `me`, `te`, `se`, `ne`, `que` before a vowel. Before an `h` only for words +//! known to have a mute `h`, so aspirated ones (`le hibou`, `la hache`, `le héros`) stay quiet; +//! never before a proper noun, `onze`, `oui`, `y…` words (`le yaourt`) or `la une`. +//! +//! No agreement or conjugation rule, and no English rule (such as `a`/`an`), applies to French +//! text. + +use crate::{Issue, IssueKind, ProofLang, words}; + +/// Non-breaking space, before `:` and inside `« »`. +const NBSP: char = '\u{A0}'; +/// Narrow non-breaking space, before `;` `!` `?`. +const NNBSP: char = '\u{202F}'; + +/// Words French repeats on purpose: pronominal verbs (`nous nous lavons`), emphasis, +/// interjections, onomatopoeia. +const ALLOWED_REPEATS: &[&str] = &[ + "nous", "vous", "si", "oui", "non", "très", "bon", "vite", "là", "ah", "oh", "eh", "ha", "hé", "ho", "hi", "bla", "chut", "toc", "pan", "bravo", + "merci", "allez", "encore", "beaucoup", "fort", "tchin", "bye", "couci", "tam", "miam", "glou", +]; + +/// Abbreviations written with a final period after which a sentence goes on in lower case +/// (lower case, without the period). +const ABBREVIATIONS: &[&str] = &[ + "etc", "cf", "ex", "env", "av", "apr", "mm", "mme", "mmes", "mlle", "mlles", "mgr", "dr", "pr", "me", "st", "ste", "sts", "pp", "vol", "chap", + "fig", "éd", "coll", "trad", "réf", "tél", "art", "al", "op", "cit", "ibid", "id", "resp", "min", "max", "sq", "sqq", "suiv", "hab", "janv", + "févr", "avr", "juil", "sept", "oct", "nov", "déc", "bd", "boul", "cie", "éts", "vve", "nb", "ps", "no", "nos", "vs", "qqch", "qqn", "ca", + "approx", "prof", "dir", "sté", "dépt", "adj", "adv", "fém", "masc", "sing", "plur", "à-d", "ms", "mss", "éds", "trim", "hôp", +]; + +/// Words after which `!` or `?` doesn't end the sentence (`Ah ! que c'est beau`, `Eh bien ? +/// et alors`). +const INTERJECTIONS: &[&str] = &[ + "ah", "oh", "eh", "hé", "hein", "hélas", "bah", "ô", "aïe", "ouf", "oups", "quoi", "comment", "ha", "ho", "bon", "tiens", "allons", "zut", + "chut", "bravo", "hourra", "dame", "bien", "là", "oui", "non", "pardon", "voyons", +]; + +/// Pronouns ending an inversion (`dit-il`, `demanda-t-elle`): after one, a `!` or `?` doesn't +/// end the sentence (`Quoi ? dit-il`). +const INVERTED: &[&str] = &["-il", "-elle", "-on", "-ils", "-elles", "-je", "-tu", "-nous", "-vous"]; + +/// Speech verbs of a dialogue tag: after one, a `!` or `?` doesn't end the sentence (`— Tu +/// viens ? demanda le voisin`). +const SPEECH_VERBS: &[&str] = &[ + "dit", + "demanda", + "répondit", + "cria", + "murmura", + "ajouta", + "reprit", + "fit", + "lança", + "souffla", + "répliqua", + "interrogea", + "hurla", + "poursuivit", + "continua", + "déclara", + "expliqua", + "protesta", + "soupira", + "chuchota", + "insista", + "s'écria", + "s’écria", + "s'exclama", + "s’exclama", + "objecta", + "conclut", + "dis", + "dit-on", +]; + +/// The words that elide before a vowel, and their elided form. +const ELIDING: &[(&str, &str)] = + &[("le", "l"), ("la", "l"), ("de", "d"), ("je", "j"), ("me", "m"), ("te", "t"), ("se", "s"), ("ne", "n"), ("que", "qu")]; + +/// Words starting with a vowel that don't take an elision (`le onze`, `le oui`, `de ouate`). +const NO_ELISION: &[&str] = &["onze", "onzième", "onzièmes", "oui", "ouistiti", "ouistitis", "ouate", "ouates", "uhlan", "uhlans"]; + +/// Words with a mute `h`, which take an elision (`l’homme`, `d’habitude`, `j’habite`). Only these +/// are flagged: an `h` word outside the list may be aspirated (`le hibou`, `la hache`, `le +/// héros`), and staying quiet is the safe side. +const MUTE_H: &[&str] = &[ + "homme", + "hommes", + "heure", + "heures", + "histoire", + "histoires", + "hiver", + "hivers", + "hôtel", + "hôtels", + "herbe", + "herbes", + "habitude", + "habitudes", + "habitant", + "habitants", + "habit", + "habits", + "hôpital", + "hôpitaux", + "humain", + "humains", + "humanité", + "honneur", + "honneurs", + "horizon", + "horizons", + "huile", + "huiles", + "hypothèse", + "hypothèses", + "harmonie", + "héritage", + "héroïne", + "héroïnes", + "hésitation", + "hommage", + "hommages", + "horaire", + "horaires", + "horreur", + "horreurs", + "humeur", + "humour", + "hygiène", + "hymne", + "habiter", + "hériter", + "hésiter", + "habiller", + "honorer", + "habite", + "habites", + "habitent", + "habitais", + "habitait", + "hésite", + "hésites", + "hésitent", + "hésitais", + "hésitait", + "habille", + "habillent", + "honore", + "hier", + "hôte", + "hôtes", + "hôtesse", + "humidité", + "haleine", + "hebdomadaire", + "hectare", + "hectares", + "hélicoptère", + "hémisphère", + "heureux", + "heureuse", + "hippopotame", + "hirondelle", + "honnête", + "honnêteté", + "huître", + "huîtres", + "humble", + "humilité", + "hydrogène", + "hypocrite", + "horloge", + "hôtellerie", + "hérédité", + "historien", + "historique", + "humanitaire", + "hormone", + "hormones", + "hospitalité", +]; + +fn issue(start: usize, end: usize, message: String, suggestions: Vec) -> Issue { + Issue { start, end, kind: IssueKind::Grammar, message, suggestions, lang: ProofLang::from_tag(Some("fr-FR")) } +} + +/// Is `c` a space that doesn't break (or a thin space some writers use instead)? +fn is_fixed_space(c: char) -> bool { + matches!(c, '\u{A0}' | '\u{202F}' | '\u{2007}' | '\u{2009}' | '\u{2060}') +} + +/// The byte range of the whitespace-delimited token around byte `i` of `text`. +fn token_bounds(text: &str, i: usize) -> (usize, usize) { + let start = text.get(..i).and_then(|t| t.char_indices().rev().find(|(_, c)| c.is_whitespace())).map_or(0, |(k, c)| k + c.len_utf8()); + let end = text.get(i..).and_then(|t| t.find(char::is_whitespace)).map_or(text.len(), |k| i + k); + (start, end) +} + +/// The whitespace-delimited token around byte `i` of `text`. +fn token_at(text: &str, i: usize) -> &str { + let (start, end) = token_bounds(text, i); + text.get(start..end).unwrap_or("") +} + +/// Is `token` a web or e-mail address, a path or an HTML entity, where punctuation isn't prose? +fn is_address(token: &str) -> bool { + token.contains("://") || token.contains('@') || token.starts_with("www.") || token.contains('&') || token.contains('\\') +} + +/// `«` + text + `»` in a message, with French spacing. +fn quoted(s: &str) -> String { + format!("«{NBSP}{s}{NBSP}»") +} + +/// `;` `!` `?` `:` without the non-breaking space French puts before them. +/// Quebec typography (`quebec`) puts no space before `;` `!` `?`, only before `:`. +fn high_punctuation(text: &str, quebec: bool, v: &mut Vec) { + let mut token: Option<(usize, usize, bool)> = None; + for (i, p) in text.char_indices() { + if !matches!(p, ';' | ':' | '!' | '?') || (quebec && p != ':') { + continue; + } + let before = text.get(..i).and_then(|t| t.chars().next_back()); + let after = text.get(i + p.len_utf8()..).and_then(|t| t.chars().next()); + // One look per token, so a long run of marks stays linear. + let address = match token { + Some((start, end, address)) if start <= i && i < end => address, + _ => { + let (start, end) = token_bounds(text, i); + let address = is_address(text.get(start..end).unwrap_or("")); + token = Some((start, end, address)); + address + } + }; + if address { + continue; + } + let (space, kind) = if p == ':' { (NBSP, "insécable") } else { (NNBSP, "fine insécable") }; + let message = format!("Espace {kind} attendue avant {}", quoted(&p.to_string())); + let fix = format!("{space}{p}"); + // Only a missing space is flagged: an ordinary one stays quiet until typing turns it into + // a non-breaking one by itself (French AutoFormat). + if before.is_some_and(char::is_alphabetic) { + // `10:30`, `a:b`, `C:` and `x;y` aren't prose: the mark must end the word. + let ends = match p { + ':' => after.is_none_or(char::is_whitespace), + _ => after.is_none_or(|a| a.is_whitespace() || matches!(a, '!' | '?' | '»' | '"' | '”' | ')')), + }; + if ends { + v.push(issue(i, i + p.len_utf8(), message, vec![fix])); + } + } + } +} + +/// `«` and `»` without a non-breaking space inside. +fn guillemets(text: &str, v: &mut Vec) { + for (i, c) in text.char_indices() { + let len = c.len_utf8(); + match c { + '«' => { + let mut next = text.get(i + len..).unwrap_or("").chars(); + let message = format!("Espace insécable attendue après {}", quoted("«")); + if next.next().is_some_and(char::is_alphanumeric) { + v.push(issue(i, i + len, message, vec![format!("«{NBSP}")])); + } + } + '»' => { + let prev = text.get(..i).unwrap_or("").chars().next_back(); + let message = format!("Espace insécable attendue avant {}", quoted("»")); + if prev.is_some_and(|n| n.is_alphanumeric() || matches!(n, '.' | '!' | '?' | '…')) { + v.push(issue(i, i + len, message, vec![format!("{NBSP}»")])); + } + } + _ => {} + } + } +} + +/// `"…"` around a phrase, where French uses `« … »`. +fn straight_quotes(text: &str, v: &mut Vec) { + let quotes: Vec = text.match_indices('"').map(|(i, _)| i).collect(); + // An odd count leaves no telling which quote opens and which closes. + if !quotes.len().is_multiple_of(2) { + return; + } + for pair in quotes.chunks(2) { + let [a, b] = pair else { continue }; + let (Some(inner), Some(before), Some(after)) = (text.get(a + 1..*b), text.get(..*a), text.get(b + 1..)) else { continue }; + let opens = before.chars().next_back().is_none_or(|c| c.is_whitespace() || matches!(c, '(' | '[' | '—' | '–')); + let closes = after.chars().next().is_none_or(|c| c.is_whitespace() || matches!(c, '.' | ',' | ';' | ':' | '!' | '?' | ')' | ']' | '…')); + let phrase = inner.trim(); + if !opens || !closes || !phrase.chars().any(char::is_alphabetic) || phrase.chars().count() > 120 || phrase.contains('\n') { + continue; + } + v.push(issue(*a, b + 1, format!("Guillemets français attendus{NBSP}: {}", quoted(phrase)), vec![quoted(phrase)])); + } +} + +fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + let lower = w0.to_lowercase(); + let starts_upper = |w: &str| w.chars().next().is_some_and(char::is_uppercase); + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && lower == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&lower.as_str()) + // `de De…`, `Bora Bora`: a capitalized second word is a name. + && !starts_upper(w1) + { + v.push(issue(*a0, *b1, format!("Mot répété{NBSP}: {}", quoted(w1)), vec![w0.to_string()])); + } + } +} + +/// Is `prev`, before a period, an abbreviation, an initial, a number or a Roman numeral (after +/// which a lower-case word doesn't start a sentence)? +fn abbreviation_or_number(prev: &str) -> bool { + prev.chars().count() <= 1 + || prev.chars().any(|c| c.is_ascii_digit()) + || prev.chars().all(|c| matches!(c, 'I' | 'V' | 'X' | 'L' | 'C' | 'D' | 'M')) + || ABBREVIATIONS.contains(&prev.to_lowercase().as_str()) +} + +/// Does `gap`, between the words `prev` and `next`, end a sentence: a `.`, `!` or `?` (French +/// puts a space before the last two), then a space, with only quotes and brackets around? Not +/// an ellipsis, not after an abbreviation; for `!` and `?`, not after an interjection, not before +/// a closing quote (dialogue goes on: `« Tu viens ? » demanda-t-elle`), not before an inversion +/// or a speech verb (`Quoi ? dit-il`, `— Viens ! cria-t-on`), and never in a `dialogue` line (one +/// opening with a dash). +fn ends_sentence(gap: &str, prev: &str, next: &str, dialogue: bool) -> bool { + let closing = ['"', '”', '’', '\'', ')', '»', ']']; + let t = gap.trim_start_matches(|c: char| c.is_whitespace() || is_fixed_space(c) || closing.contains(&c)); + let Some(end) = t.chars().next() else { return false }; + if !matches!(end, '.' | '!' | '?') || t.starts_with("..") { + return false; + } + if end == '.' && abbreviation_or_number(prev) { + return false; + } + let exclamation = end != '.'; + let lower_next = next.to_lowercase(); + if exclamation + && (dialogue + || INTERJECTIONS.contains(&prev.to_lowercase().as_str()) + || INVERTED.iter().any(|p| lower_next.ends_with(p)) + || SPEECH_VERBS.contains(&lower_next.as_str())) + { + return false; + } + let rest = t.trim_start_matches(['.', '!', '?']); + let after_marks = rest.trim_start_matches(is_fixed_space); + if after_marks.starts_with(closing) { + if exclamation { + return false; + } + return ends_sentence_after(after_marks.trim_start_matches(|c: char| closing.contains(&c) || is_fixed_space(c))); + } + ends_sentence_after(rest) +} + +/// Is `rest` (after the final mark) a space, then nothing but opening quotes and brackets? +fn ends_sentence_after(rest: &str) -> bool { + rest.starts_with(char::is_whitespace) + && rest.trim_start().chars().all(|c| c.is_whitespace() || is_fixed_space(c) || matches!(c, '"' | '“' | '«' | '(' | '[' | '—' | '–')) +} + +fn sentence_capitals(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + let dialogue = text.trim_start().starts_with(['—', '–']); + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(prev), Some(gap), Some(w)) = (text.get(*a0..*b0), text.get(*b0..*a1), text.get(*a1..*b1)) else { continue }; + let word_like = w.chars().all(|c| c.is_alphabetic() || matches!(c, '\'' | '’' | '-')); + if word_like && w.chars().next().is_some_and(char::is_lowercase) && ends_sentence(gap, prev, w, dialogue) && !is_address(token_at(text, *a1)) + { + let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default(); + v.push(issue(*a1, *b1, "Majuscule attendue en début de phrase".into(), vec![fixed])); + } + } +} + +/// Does `w` (lower case) start with a sound that makes `le`, `de`, `que`… elide? +fn takes_elision(w: &str) -> bool { + let Some(first) = w.chars().next() else { return false }; + if NO_ELISION.contains(&w) { + return false; + } + match first { + 'a' | 'à' | 'â' | 'ä' | 'e' | 'é' | 'è' | 'ê' | 'ë' | 'i' | 'î' | 'ï' | 'o' | 'ô' | 'ö' | 'u' | 'ù' | 'û' | 'ü' | 'œ' | 'æ' => { + true + } + 'h' => MUTE_H.contains(&w), + // Only `y` itself (`j’y vais`): `le yaourt`, `de yoga`. + 'y' => w == "y", + _ => false, + } +} + +fn elisions(text: &str, ws: &[(usize, usize)], v: &mut Vec) { + for pair in ws.windows(2) { + let [(a0, b0), (a1, b1)] = pair else { continue }; + let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; + if gap.is_empty() || !gap.chars().all(|c| c == ' ') { + continue; + } + let lower0 = w0.to_lowercase(); + let Some((_, elided)) = ELIDING.iter().find(|(full, _)| *full == lower0) else { continue }; + // `le`/`Le` only, not `LE`; never before a proper noun (`de Orléans`) or a capital. + let capital = w0.chars().next().is_some_and(char::is_uppercase); + if w0.chars().skip(1).any(char::is_uppercase) || !w1.chars().next().is_some_and(char::is_lowercase) { + continue; + } + if !w1.chars().all(|c| c.is_alphabetic() || c == '-') { + continue; + } + let lower1 = w1.to_lowercase(); + // `le un`, `la une` (the number, the front page); a single letter only where it is a word + // the elision is certain for (`n’a`, `qu’a`, `j’y`). + if matches!(lower0.as_str(), "le" | "la") && matches!(lower1.as_str(), "un" | "une") { + continue; + } + let certain = (lower1 == "y" && matches!(lower0.as_str(), "je" | "me" | "te" | "se" | "ne")) + || (lower1 == "a" && matches!(lower0.as_str(), "ne" | "que")); + if lower1.chars().count() == 1 && !certain { + continue; + } + if !takes_elision(&lower1) { + continue; + } + let head = if capital { crate::affix::capitalize(elided) } else { (*elided).to_string() }; + let fixed = format!("{head}’{w1}"); + v.push(issue(*a0, *b1, format!("Élision{NBSP}: {}", quoted(&fixed)), vec![fixed])); + } +} + +/// Grammar issues in a paragraph of French text (France's typography), in order. +pub fn check(text: &str) -> Vec { + check_in(text, ProofLang::from_tag(Some("fr-FR"))) +} + +/// Grammar issues in a French Language pack locale, with Quebec typography in `fr-CA`. +/// Other locales have no French issues; the original locale is retained on every issue. +pub fn check_in(text: &str, lang: ProofLang) -> Vec { + let Some(locale) = lang.pack_locale().filter(|l| l.primary() == "fr") else { return Vec::new() }; + let ws: Vec<(usize, usize)> = words(text); + let mut v = Vec::new(); + high_punctuation(text, crate::lang::region(locale.as_str()).is_some_and(|r| r.eq_ignore_ascii_case("CA")), &mut v); + guillemets(text, &mut v); + straight_quotes(text, &mut v); + repeated_words(text, &ws, &mut v); + sentence_capitals(text, &ws, &mut v); + elisions(text, &ws, &mut v); + for i in &mut v { + i.lang = lang; + } + v.sort_by_key(|i| (i.start, i.end)); + v +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The issues of `text` as (flagged text, first suggestion). + fn found(text: &str) -> Vec<(String, String)> { + check(text) + .iter() + .map(|i| (text.get(i.start..i.end).unwrap_or("?").to_string(), i.suggestions.first().cloned().unwrap_or_default())) + .collect() + } + + fn quiet(text: &str) { + let v = check(text); + assert!(v.is_empty(), "{text:?} → {v:?}"); + } + + fn one(text: &str, flagged: &str, fix: &str) { + assert_eq!(found(text), [(flagged.to_string(), fix.to_string())], "{text:?}"); + } + + #[test] + fn missing_space_before_high_punctuation() { + one("Bonjour!", "!", "\u{202F}!"); + one("Tu viens?", "?", "\u{202F}?"); + one("Il pleut; il vente.", ";", "\u{202F};"); + one("Note: rien.", ":", "\u{A0}:"); + // `?!`: only the first mark needs the space. + one("Quoi?!", "?", "\u{202F}?"); + assert!(check("Bonjour!").iter().all(|i| i.lang == ProofLang::from_tag(Some("fr-FR")) && i.message.contains("fine insécable"))); + assert!(check("Note: rien.").iter().all(|i| i.message.contains("insécable") && !i.message.contains("fine"))); + } + + #[test] + fn an_ordinary_space_stays_quiet_until_french_autoformat() { + // Typing can't produce a non-breaking space yet, so an ordinary one is accepted. + quiet("Vraiment ? Bonjour ! Il pleut ; note : rien."); + quiet("Il dit « oui » hier."); + } + + #[test] + fn quebec_has_no_space_before_semicolon_exclamation_and_question() { + let fr_ca = |t: &str| check_in(t, ProofLang::from_tag(Some("fr-CA"))); + assert!(fr_ca("Bonjour! Tu viens? Il pleut; il vente.").is_empty(), "{:?}", fr_ca("Bonjour! Tu viens?")); + // The colon and the guillemets still want their space in Quebec. + let v = fr_ca("Note: «oui»."); + assert_eq!(v.len(), 3, "{v:?}"); + assert!(v.iter().all(|i| i.lang == ProofLang::from_tag(Some("fr-CA"))), "{v:?}"); + assert_eq!(check_in("Bonjour!", ProofLang::from_tag(Some("fr-FR"))).len(), 1, "France still wants the space"); + } + + #[test] + fn high_punctuation_stays_quiet() { + quiet("Bonjour\u{202F}! Tu viens\u{202F}? Il pleut\u{202F}; note\u{A0}: rien."); + quiet("Bonjour\u{A0}! Quoi\u{202F}?! Ah\u{2009}!"); + quiet("Il est 10:30."); + quiet("Voir https://exemple.org:8080/a?b=c pour la suite."); + quiet("Écrire à contact@exemple.org!"); + quiet("Super :) et ;-) aussi :D"); + quiet("a:b et x;y"); + quiet("Le fichier C:\\dossier est là."); + } + + #[test] + fn missing_space_inside_guillemets() { + assert_eq!(found("Il dit «oui» hier."), [("«".to_string(), "«\u{A0}".to_string()), ("»".to_string(), "\u{A0}»".to_string())]); + quiet("Il dit «\u{A0}oui\u{A0}» hier."); + quiet("Il dit «\u{202F}oui\u{202F}» hier."); + } + + #[test] + fn straight_quotes_become_guillemets() { + one("Il a dit \"bonjour\" hier.", "\"bonjour\"", "«\u{A0}bonjour\u{A0}»"); + one("\"Bonne idée\", dit-elle.", "\"Bonne idée\"", "«\u{A0}Bonne idée\u{A0}»"); + // Inch marks and unbalanced quotes are left alone. + quiet("Un écran de 27\" et un de 32\" ici."); + quiet("Il a dit \"bonjour hier."); + } + + #[test] + fn repeated_words_are_flagged() { + one("Le le chat dort.", "Le le", "Le"); + one("Je pense que que tu viens.", "que que", "que"); + one("Il mange la la pomme.", "la la", "la"); + } + + #[test] + fn deliberate_repetitions_stay_quiet() { + quiet("Nous nous lavons et vous vous trompez."); + quiet("Si si, très très bien."); + quiet("Ah ah, bla bla."); + quiet("Je crois que qu’il viendra."); + quiet("Le livre de De Gaulle."); + quiet("Nous partons à Bora Bora."); + quiet("Ils ont dit bye bye. Tchin tchin\u{202F}!"); + } + + #[test] + fn sentence_capitals_are_flagged() { + one("Il pleut. il fait froid.", "il", "Il"); + one("Il pleut\u{202F}! il fait froid.", "il", "Il"); + one("Tu viens\u{202F}? on part.", "on", "On"); + one("Il pleut. l’orage arrive.", "l’orage", "L’orage"); + one("Il pleut. «\u{A0}tout va bien\u{A0}», dit-il.", "tout", "Tout"); + } + + #[test] + fn sentence_capitals_stay_quiet() { + quiet("Voir p. 12 et cf. la note, etc. puis la suite."); + quiet("M. le maire est là."); + quiet("Il attend… puis part. Il attend... puis part."); + quiet("«\u{A0}Tu viens\u{202F}?\u{A0}» demanda-t-elle."); + quiet("Ah\u{202F}! que c’est beau."); + quiet("Quoi\u{202F}? dit-il."); + quiet("Tu viens\u{202F}? demanda le voisin."); + quiet("— Tu viens\u{202F}? reprends-tu, inquiet."); + quiet("– Viens\u{202F}! on part."); + quiet("Voir www.exemple.fr pour la suite."); + quiet("En 2026. ensuite rien."); + quiet("il commence en minuscule."); + } + + #[test] + fn missing_elisions_are_flagged() { + one("Il refuse de avoir peur.", "de avoir", "d’avoir"); + one("Je ai faim.", "Je ai", "J’ai"); + one("Voici le homme.", "le homme", "l’homme"); + one("Je sais que il viendra.", "que il", "qu’il"); + one("Le arbre est grand.", "Le arbre", "L’arbre"); + one("Il ne a pas faim.", "ne a", "n’a"); + one("Bon, je y vais.", "je y", "j’y"); + one("Il se est levé.", "se est", "s’est"); + one("Voici la école.", "la école", "l’école"); + one("Il me aime.", "me aime", "m’aime"); + } + + #[test] + fn aspirated_h_and_other_exceptions_stay_quiet() { + for w in [ + "hibou", + "héros", + "haricot", + "hasard", + "haut", + "hall", + "hamac", + "hangar", + "hockey", + "homard", + "huit", + "hérisson", + "hors-jeu", + "onze", + "oui", + "yaourt", + ] { + quiet(&format!("Voici le {w}.")); + } + // hiérarchie has an aspirated h, despite the mute h in hiéroglyphe: + // https://www.academie-francaise.fr/le-hieroglyphe-pour-lhieroglyphe + for w in ["honte", "hache", "harpe", "hausse", "hiérarchie"] { + quiet(&format!("Voici la {w}.")); + } + quiet("Une question de hiérarchie."); + quiet("Le onze de France. À la une du journal."); + quiet("Il vient de Orléans."); + quiet("De a à z, le a et le b."); + quiet("Le y et le x sont des variables. La dérivée de y."); + quiet("Je mange de la viande."); + } + + #[test] + fn hostile_input_never_panics() { + for t in ["", " ", "!", " !", "«", "»", " »", "« ", "\"", "\"\"", ". a", "de", "de ", "\u{202F}?", "é!é", "« » « »", ":", " :", "a :)"] + { + let _ = check(t); + } + // A long run of marks in one token: linear, not one token scan per mark. + let run = format!("mot{}", "?".repeat(200_000)); + assert_eq!(check(&run).len(), 1); + } +} diff --git a/crates/proof/src/fr/mod.rs b/crates/proof/src/fr/mod.rs new file mode 100644 index 000000000..541fcac0d --- /dev/null +++ b/crates/proof/src/fr/mod.rs @@ -0,0 +1,181 @@ +//! French proofing through the shared Language pack cache. +//! +//! A French locale remains [`ProofLang::Pack`]: dictionary selection, installation, reloads, +//! memory limits and hyphenation use the same paths as other packs. This module adds French +//! elisions, accent-oriented suggestions and conservative typography, including Quebec spacing. +//! No dictionary or hyphenation data is bundled here. + +pub mod grammar; +mod suggest; +#[cfg(test)] +#[allow(clippy::unwrap_used)] +pub(crate) mod test_dict; + +use crate::affix::AffixDict; +use crate::{Options, ProofLang}; + +pub use suggest::suggest_in; + +/// Words after whose elision a user-dictionary entry is accepted. Ordinary words and their +/// elisions are always decided by the dictionary (`l'arbre`, but not `l'arbres`). +const ELISIONS: &[&str] = &["l", "d", "j", "m", "t", "s", "n", "c", "qu", "jusqu", "lorsqu", "puisqu", "quoiqu", "presqu", "quelqu"]; + +/// Check French spelling in the installed France/generic pack, with the default options. +/// Regional callers should use [`crate::is_correct_in`] with their full locale. +pub fn is_correct(word: &str) -> bool { + crate::is_correct_in(word, ProofLang::from_tag(Some("fr-FR"))) +} + +/// French suggestions from the installed France/generic pack, best first, at most `max`. +/// Regional callers should use [`crate::suggest_in`] with their full locale. +pub fn suggest(word: &str, max: usize) -> Vec { + crate::suggest_in(word, max, ProofLang::from_tag(Some("fr-FR"))) +} + +/// Check against a supplied dictionary, without changing any installed pack or cache. +pub fn is_correct_with(d: &AffixDict, word: &str) -> bool { + check(d, &crate::composed(word), &Options::default()) +} + +pub(crate) fn check(d: &AffixDict, word: &str, opts: &Options) -> bool { + let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true }; + // A regional dictionary may store the curly apostrophe without ICONV. Honor that entry + // before the French ASCII-apostrophe fallback. + if known(d, w) { + return true; + } + let w = w.replace('’', "'"); + if known(d, &w) { + return true; + } + if let Some((head, tail)) = w.split_once('\'') + && ELISIONS.contains(&head.to_lowercase().as_str()) + && !tail.is_empty() + && !tail.contains('\'') + && crate::in_user_dictionary(&tail.to_lowercase()) + { + return true; + } + w.contains('-') && w.split('-').all(|p| !p.is_empty() && (p.chars().count() <= 1 || known(d, p))) +} + +fn known(d: &AffixDict, word: &str) -> bool { + d.check(word) || crate::in_user_dictionary(&word.to_lowercase()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn original_french_words_and_elisions() { + let d = test_dict::dictionary(); + for word in [ + "maison", + "maisons", + "école", + "écoles", + "grand", + "grande", + "grands", + "grandes", + "aimer", + "aime", + "aimons", + "aiment", + "suis", + "est", + "sommes", + "sont", + "a", + "avons", + "ont", + "Paris", + "Montréal", + "œuvre", + "sœur", + "aujourd'hui", + "aujourd’hui", + "peut-être", + "arc-en-ciel", + "km", + "Maison", + "MAISON", + "SNCF", + "2026", + "très", + "rivière", + "l'arbre", + "l’arbre", + "L'arbre", + "d'enfants", + "l'homme", + "qu'elle", + "qu’il", + "j'aime", + "j’ai", + "d'avoir", + "l'école", + "jusqu'à", + "maison-école", + "est-il", + "a-t-il", + ] { + assert!(is_correct_with(&d, word), "{word}"); + } + for word in [ + "maisonn", + "ecole", + "arbrre", + "beaucoups", + "aimmer", + "soeur", + "Km", + "grandd", + "aim", + "l'hasard", + "l'chat", + "l'arbres", + "l'arbrre", + "qu'maison", + "maison-ecole", + "maison--école", + ] { + assert!(!is_correct_with(&d, word), "{word}"); + } + assert!(is_correct_with(&d, "e\u{301}cole")); + assert!(!is_correct_with(&d, "maison\u{338}"), "a trailing mark is part of the word"); + } + + #[test] + fn explicit_curly_apostrophe_without_iconv() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\naujourd’hui\n", &[]).unwrap(); + assert!(is_correct_with(&d, "aujourd’hui")); + assert!(!is_correct_with(&d, "aujourd’huui")); + } + + #[test] + fn user_word_after_an_elision() { + let d = test_dict::dictionary(); + let word = "zxqmotadaptfr"; + assert!(!is_correct_with(&d, word) && !is_correct_with(&d, &format!("l'{word}"))); + crate::add_word(word); + assert!(is_correct_with(&d, word) && is_correct_with(&d, &format!("l'{word}")) && is_correct_with(&d, &format!("d’{word}"))); + assert!(!is_correct_with(&d, &format!("chat'{word}"))); + crate::remove_word(word); + } + + #[test] + fn options_and_missing_dictionary() { + let d = test_dict::dictionary(); + let defaults = Options::default(); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + assert!(check(&d, "SNCF", &defaults) && check(&d, "mp3x", &defaults)); + assert!(!check(&d, "SNCF", &strict) && !check(&d, "mp3x", &strict)); + assert!(check(&d, "MAISON", &strict) && check(&d, "2026", &strict)); + for word in ["école", "l'arbre", "xqzv", "SNCF", ""] { + assert!(check(&AffixDict::default(), word, &strict)); + } + } +} diff --git a/crates/proof/src/fr/suggest.rs b/crates/proof/src/fr/suggest.rs new file mode 100644 index 000000000..86477d33f --- /dev/null +++ b/crates/proof/src/fr/suggest.rs @@ -0,0 +1,81 @@ +//! French tuning of the shared, bounded affix suggestion engine. + +use crate::affix::AffixDict; +use crate::affix_suggest::{Split, Tuning}; + +/// Short words often run into the next one (`et la`, `de Paris`, `pour moi`). +const LEADING_WORDS: &[&str] = &[ + "à", "a", "au", "aux", "de", "du", "des", "le", "la", "les", "un", "une", "et", "ou", "où", "en", "y", "ce", "se", "ne", "je", "tu", "il", "on", + "nous", "vous", "ils", "elle", "par", "pour", "sur", "sous", "dans", "avec", "sans", "chez", "vers", "entre", "très", "plus", "moins", "bien", + "pas", "que", "qui", "mais", "donc", "car", "si", "son", "sa", "ses", "mon", "ma", "mes", "ton", "ta", "tes", "est", "tout", "tous", "toute", + "comme", "quand", "puis", "encore", "déjà", "jamais", "toujours", +]; +/// Short words often run into the one before (`dis-moi` without the hyphen, `là-bas`, or a +/// dropped space: `pourmoi` → `pour moi`). +const TRAILING_WORDS: &[&str] = + &["moi", "toi", "lui", "elle", "nous", "vous", "eux", "le", "la", "les", "en", "y", "là", "ci", "il", "on", "ils", "elles", "est", "et"]; + +const TUNING: Tuning = Tuning { + split: Split::Near { leading: LEADING_WORDS, trailing: TRAILING_WORDS }, + alphabet: "esaitnrulodcpmévqfbghjàxèêyçôîâûùœ", + ..Tuning::DEFAULT +}; + +/// French suggestions from a supplied dictionary; keeps the writer's apostrophe and case. +/// Dictionary OCONV remains active for every other output conversion. +pub fn suggest_in(d: &AffixDict, word: &str, max: usize) -> Vec { + let word = crate::composed(word); + let curly = word.contains('’'); + let apostrophe = if curly { '’' } else { '\'' }; + let mut out = Vec::new(); + for text in crate::affix_suggest::suggest_in(d, &word, max, &TUNING) { + let styled: String = text.chars().map(|c| if c == '\'' || c == '’' { apostrophe } else { c }).collect(); + // A pack may spell only one apostrophe and have no ICONV. Keep its valid output + // rather than turn it back into the spelling mistake the caller is correcting. + let text = if styled.split(' ').all(|p| d.can_suggest_cased(p, p)) { styled } else { text }; + if text != word && !out.contains(&text) { + out.push(text); + } + } + out +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn french_accents_confusions_splits_and_elisions() { + let d = super::super::test_dict::dictionary(); + for (typo, expected) in [ + ("ecole", "école"), + ("riviere", "rivière"), + ("soeur", "sœur"), + ("maisonn", "maison"), + ("pourmoi", "pour moi"), + ("l'arbrre", "l'arbre"), + ("l’arbrre", "l’arbre"), + ] { + let got = suggest_in(&d, typo, 8); + assert!(got.iter().any(|s| s == expected), "{typo}: {got:?}"); + } + assert!(suggest_in(&d, "e\u{301}cole", 4).iter().all(|s| s != "école")); + assert!(suggest_in(&d, "ecole", 0).is_empty()); + assert!(suggest_in(&d, &"a".repeat(41), 4).is_empty()); + } + + #[test] + fn curly_dictionary_without_iconv_remains_suggestible() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\naujourd’hui\n", &[]).unwrap(); + assert!(suggest_in(&d, "aujourd’huui", 4).iter().any(|s| s == "aujourd’hui")); + assert!(suggest_in(&d, "aujourd'hui", 4).iter().any(|s| s == "aujourd’hui")); + } + + #[test] + fn output_conversion_preserves_typed_apostrophe() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV ’ '\nOCONV 1\nOCONV ' ’\n", "1\nl'arbre\n", &[]).unwrap(); + assert!(suggest_in(&d, "l'arbrre", 4).iter().any(|s| s == "l'arbre")); + assert!(suggest_in(&d, "l’arbrre", 4).iter().any(|s| s == "l’arbre")); + } +} diff --git a/crates/proof/src/fr/test_dict.rs b/crates/proof/src/fr/test_dict.rs new file mode 100644 index 000000000..23637254c --- /dev/null +++ b/crates/proof/src/fr/test_dict.rs @@ -0,0 +1,234 @@ +//! A tiny synthetic French Hunspell dictionary for tests (written for these tests; not taken +//! from any real dictionary). Real dictionaries are installed on the system, never bundled. +//! +//! It exercises what the Dicollecte French dictionary uses: two-character flags (`FLAG long`), +//! stems that need an affix (`NEEDAFFIX`, with an empty suffix for the singular), elisions as +//! prefixes licensed by a suffix's continuation (`l'arbre` but not `l'arbres`), irregular verbs +//! through `FULLSTRIP` (`être` → `suis`), `KEEPCASE` (`km`), `NOSUGGEST` (`bah`), `ICONV` (`’` +//! read as `'`) and the +//! suggestion tables (`TRY`, `MAP` for accents, `REP` for common confusions). + +pub const AFF: &str = "SET UTF-8 +FLAG long +NEEDAFFIX () +KEEPCASE || +NOSUGGEST -- +FULLSTRIP +ICONV 1 +ICONV ’ ' +TRY esaitnrulodcpmévqfbghjàxèêyçôîâûùœ +MAP 6 +MAP aàâä +MAP eéèêë +MAP iîï +MAP oôö +MAP uùûü +MAP cç +REP 3 +REP oe œ +REP ph f +REP f ph +PFX L' Y 1 +PFX L' 0 l' [aàâeéèêiîoôuœh] +PFX D' Y 1 +PFX D' 0 d' [aàâeéèêiîoôuœh] +PFX Q' Y 1 +PFX Q' 0 qu' [aàâeéèêiîoôuœh] +PFX J' Y 1 +PFX J' 0 j' [aàâeéèêiîoôuœhy] +SFX S. Y 2 +SFX S. 0 0/L'D'Q' [^sxz] +SFX S. 0 s/D'Q' [^sxz] +SFX S= Y 2 +SFX S= 0 0 [^sxz] +SFX S= 0 s [^sxz] +SFX F. Y 4 +SFX F. 0 0/L'D'Q' . +SFX F. 0 e/L'D'Q' . +SFX F. 0 s/D'Q' . +SFX F. 0 es/D'Q' . +SFX Er Y 6 +SFX Er er er/D' er +SFX Er er e/J' er +SFX Er er es er +SFX Er er ons er +SFX Er er ez er +SFX Er er ent er +SFX Et N 7 +SFX Et être être être +SFX Et être suis être +SFX Et être es être +SFX Et être est être +SFX Et être sommes être +SFX Et être êtes être +SFX Et être sont être +SFX Av N 7 +SFX Av avoir avoir/D' avoir +SFX Av avoir ai/J' avoir +SFX Av avoir as avoir +SFX Av avoir a avoir +SFX Av avoir avons avoir +SFX Av avoir avez avoir +SFX Av avoir ont/Q' avoir +"; + +/// Stems (the count line first, as in Hunspell files). +pub const DIC: &str = "150 +maison/S.() +arbre/S.() +enfant/S.() +ami/S.() +amie/S.() +homme/S.() +école/S.() +café/S.() +chat/S.() +rivière/S.() +ville/S.() +heure/S.() +histoire/S.() +sœur/S.() +œuvre/S.() +photo/S.() +téléphone/S.() +année/S.() +vie/S.() +livre/S.() +table/S.() +porte/S.() +femme/S.() +garçon/S.() +fille/S.() +père/S.() +mère/S.() +langue/S.() +mot/S.() +texte/S.() +phrase/S.() +exemple/S.() +jour/S.() +nuit/S.() +monde/S.() +hasard/S=() +haricot/S=() +grand/F.() +petit/F.() +noir/F.() +aimer/Er() +parler/Er() +chanter/Er() +arriver/Er() +être/Et() +avoir/Av() +il/Q' +ils/Q' +elle/Q' +elles/Q' +on/Q'L' +un/D'Q'L' +une/D'Q'L' +en/Q'J' +y/J' +à/Q' +jusqu'à +aujourd'hui/D' +peut-être +arc-en-ciel +km/|| +bah/-- +Paris +France +Montréal +le +la +les +de +des +du +et +à +je +tu +nous +vous +que +qui +ne +pas +se +me +te +ce +ça +avec +pour +dans +sur +sous +très +près +beaucoup +si +oui +non +mais +ou +où +dit +demanda +quand +comme +plus +bien +tout +toujours +jamais +ici +là +alors +donc +aussi +encore +déjà +mon +ma +mes +son +sa +ses +notre +votre +leur +cette +ces +au +aux +par +entre +vers +chez +sans +moi +toi +lui +rien +chose +temps +fois +eau +voici +voilà +bonjour +merci +beau +belle +la +etc +M +p +"; + +/// The original synthetic fixture, parsed independently of the installed pack cache. +pub fn dictionary() -> crate::affix::AffixDict { + crate::affix::AffixDict::from_hunspell(AFF, DIC, &[]).unwrap() +} diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 966ded72f..948d0e25b 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -754,9 +754,13 @@ mod tests { /// spec's default `~/.local/share` applies, and with no home there is no folder. #[test] fn xdg_data_home_follows_the_spec() { - let home = Path::new("/home/a"); - assert_eq!(xdg_data_home(Some("/data".into()), Some(home)), Some(PathBuf::from("/data"))); - assert_eq!(xdg_data_home(Some("/data".into()), None), Some(PathBuf::from("/data"))); + // On Windows a rooted path without a drive is not absolute. Keep the same + // absolute/relative contract on each platform instead of changing the loader. + let (home, data) = if cfg!(windows) { (r"C:\Users\a", r"C:\data") } else { ("/home/a", "/data") }; + let (home, data) = (Path::new(home), Path::new(data)); + assert!(data.is_absolute()); + assert_eq!(xdg_data_home(Some(data.as_os_str().into()), Some(home)), Some(data.to_path_buf())); + assert_eq!(xdg_data_home(Some(data.as_os_str().into()), None), Some(data.to_path_buf())); for xdg in [None, Some(""), Some("relative/data")] { assert_eq!(xdg_data_home(xdg.map(Into::into), Some(home)), Some(home.join(".local/share")), "{xdg:?}"); assert_eq!(xdg_data_home(xdg.map(Into::into), None), None, "{xdg:?}"); @@ -1010,7 +1014,12 @@ mod tests { assert!(load(&aff, &big).unwrap_err().contains("too large")); // A directory, and on Unix an endless device, are not dictionary files. (No clock: a // reader that read `/dev/zero` would never return, so the error itself is the proof.) - assert!(load(&aff, &dir).unwrap_err().contains("not a regular file")); + // Windows rejects directories in File::open, before the regular-file guard. The + // rejection is required everywhere; its diagnostic is platform-dependent. + let directory_error = load(&aff, &dir).unwrap_err(); + assert!(directory_error.contains(&dir.display().to_string()), "{directory_error}"); + #[cfg(unix)] + assert!(directory_error.contains("not a regular file"), "{directory_error}"); #[cfg(unix)] assert!(load(&aff, Path::new("/dev/zero")).unwrap_err().contains("not a regular file")); // Turkish reads through the same reader. diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 3df6cd017..f0f33d007 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -42,6 +42,9 @@ //! existing English fallback. //! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed //! spelling, MIT); no spelling or grammar marks yet. +//! - **French** (a [`ProofLang::Pack`] locale; module [`fr`]): the pack dictionary with French +//! elisions and accent-oriented suggestions, plus conservative typography and basic checks. +//! France and Canada retain their regional tags; Quebec uses its own punctuation spacing. //! - **Any other language** ([`ProofLang::Pack`], module [`packs`]): spelling and suggestions //! from its Hunspell dictionary (an installed Language pack, the dictionaries folder or the //! system's) with default tuning, hyphenation from its pack, no grammar rules; without a @@ -73,6 +76,7 @@ pub mod dict; pub mod en_variants; mod english; pub mod es; +pub mod fr; pub mod hr; pub mod hunspell; pub mod hyphen; @@ -976,6 +980,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec { match lang { ProofLang::En => grammar_en(text, capital_first), ProofLang::EnGb | ProofLang::EnAu | ProofLang::EnCa => grammar_en(text, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect(), + ProofLang::Pack(locale) if locale.primary() == "fr" => fr::grammar::check_in(text, lang), // `pt-pt` shares Portuguese's rules, its issues in its own language. _ => language(lang.built_in()) .map_or_else(Vec::new, |l| (l.grammar)(text, lang, capital_first).into_iter().map(|i| Issue { lang, ..i }).collect()), diff --git a/crates/proof/src/pack_lang.rs b/crates/proof/src/pack_lang.rs index 7b4d19315..8b6cea902 100644 --- a/crates/proof/src/pack_lang.rs +++ b/crates/proof/src/pack_lang.rs @@ -1,6 +1,7 @@ //! Proofing for any language by its locale ([`crate::ProofLang::Pack`]): its Hunspell dictionary (the //! dictionaries folder, then an installed Language pack, then the system's), read by the shared -//! affix engine, with default suggestion tuning and no grammar rules. +//! affix engine. Most use default suggestion tuning and no grammar rules; French locales add +//! the spelling tuning and conservative typography in [`crate::fr`] without a separate cache. //! //! The loaded dictionaries share a memory budget ([`MAX_LOADED_BYTES`], estimated from their //! files); the one used least recently goes first, but never one used during the current @@ -320,6 +321,9 @@ pub(crate) fn is_correct(word: &str, locale: PackLocale, opts: &crate::Options) if other_script(w, &scripts) { return true; } + if locale.primary() == "fr" { + return crate::fr::check(&d, w, opts); + } let known = |w: &str| d.check(w) || crate::in_user_dictionary(&w.to_lowercase()); if w.contains('-') && !known(w) { return w.split('-').all(|p| p.chars().count() <= 1 || known(p)); @@ -333,7 +337,7 @@ pub(crate) fn suggest(word: &str, max: usize, locale: PackLocale) -> Vec if d.is_empty() { return Vec::new(); } - crate::affix_suggest::suggest_in(&d, word, max, &Tuning::DEFAULT) + if locale.primary() == "fr" { crate::fr::suggest_in(&d, word, max) } else { crate::affix_suggest::suggest_in(&d, word, max, &Tuning::DEFAULT) } } /// The pack locales whose hyphenation patterns serve a [`crate::ProofLang::Pack`] language: the locale, diff --git a/crates/proof/tests/affix_compatibility.rs b/crates/proof/tests/affix_compatibility.rs new file mode 100644 index 000000000..6d09e5f90 --- /dev/null +++ b/crates/proof/tests/affix_compatibility.rs @@ -0,0 +1,92 @@ +//! Frozen public output from the pre-French shared affix engine at upstream +//! e6c5a701ddff5be16e266028168c70658aae39d9, using original synthetic fixtures. +//! Do not regenerate expectations from the new implementation: they establish the baseline. +//! Legacy WCAFF1 bytes remain frozen, but are explicitly refused by the WCAFF2 engine. + +use craft_proof::affix::{AffixDict, capitalize}; +use std::fmt::Write; + +#[path = "fixtures/affix_compatibility_cases.rs"] +mod fixtures; + +fn snapshot(case: &fixtures::Case, dict: &AffixDict) -> String { + let mut out = format!("{}\nlen={} rules={}\n", case.name, dict.len(), dict.rule_count()); + writeln!(out, "try={:?} key={:?} map={:?} rep={:?} comments={:?}", dict.try_chars(), dict.key(), dict.map(), dict.rep(), dict.comments()) + .unwrap(); + let mut stems: Vec<_> = dict.stems().collect(); + stems.sort_unstable(); + for stem in &stems { + let mut forms = dict.forms(stem); + forms.sort_unstable(); + forms.dedup(); + writeln!(out, "forms {stem:?}: {forms:?}").unwrap(); + } + let mut probes: Vec = case.probes.iter().map(|word| (*word).to_string()).collect(); + for stem in stems { + probes.extend(dict.forms(stem)); + } + let originals = probes.clone(); + for word in originals { + probes.extend([word.to_uppercase(), word.to_lowercase(), capitalize(&word)]); + } + probes.sort_unstable(); + probes.dedup(); + for word in probes { + writeln!( + out, + "{word:?}: contains={} check={} suggest={} raw_suggest={}", + dict.contains(&word), + dict.check(&word), + dict.can_suggest(&word), + dict.contains_suggestible(&word) + ) + .unwrap(); + } + out +} + +#[test] +fn non_french_affix_outputs_match_pre_french_baseline() { + let expected = include_str!("fixtures/affix_compatibility.expected"); + let mut actual = String::new(); + for case in fixtures::cases() { + let dict = AffixDict::from_hunspell(&case.aff, &case.dic, &["original compatibility fixture"]).unwrap(); + let result = snapshot(&case, &dict); + let restored = AffixDict::from_bytes(&dict.to_bytes().unwrap()).unwrap(); + assert_eq!(snapshot(&case, &restored), result, "current compact round trip: {}", case.name); + actual.push_str(&result); + } + // craft-libs 55332dd predates this adaptation and supports a prefix plus a two-suffix + // chain (native test affix::tests::a_prefix_with_a_suffix_chain). The old Indonesian + // fixture's bermain + kan + lah therefore became a word, in these three cases only. + // Keep the frozen historical file untouched and assert every explicit transition once. + let mut expected = expected.to_string(); + let mut transitions = 0; + for word in ["BERMAINKANLAH", "Bermainkanlah", "bermainkanlah"] { + let before = format!("{word:?}: contains=false check=false suggest=false raw_suggest=false\n"); + let after = if word == "bermainkanlah" { + format!("{word:?}: contains=true check=true suggest=true raw_suggest=true\n") + } else { + format!("{word:?}: contains=false check=true suggest=true raw_suggest=false\n") + }; + assert_eq!(expected.matches(&before).count(), 1, "one historical observable: {word}"); + expected = expected.replacen(&before, &after, 1); + transitions += 1; + } + assert_eq!(transitions, 3, "the documented upstream exception has exactly three observables"); + assert_eq!(actual, expected, "shared engine output changed beyond the explicit upstream chain exception"); +} + +#[test] +fn legacy_compact_data_is_refused_without_reinterpreting_homonyms() { + let encoded = include_str!("fixtures/affix_compatibility.compact"); + let cases = fixtures::cases(); + let lines: Vec<_> = encoded.lines().collect(); + assert_eq!(lines.len(), cases.len()); + for (case, hex) in cases.iter().zip(lines) { + assert_eq!(hex.len() % 2, 0); + let bytes: Vec<_> = + hex.as_bytes().as_chunks::<2>().0.iter().map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()).collect(); + assert!(AffixDict::from_bytes(&bytes).is_err(), "WCAFF1 must be regenerated as WCAFF2: {}", case.name); + } +} diff --git a/crates/proof/tests/fixtures/affix_compatibility.compact b/crates/proof/tests/fixtures/affix_compatibility.compact new file mode 100644 index 000000000..5c8c627de --- /dev/null +++ b/crates/proof/tests/fixtures/affix_compatibility.compact @@ -0,0 +1,7 @@ +4d8bc10a825014446b7bbf62a07dd62744e5ae120b2ca2e02aafb8bceac54b295d067e91db76ea7f2516d16666ce30138c47ae3ba41e8c95a35cf884c89caf1c4b282789531ce4112756d1cadf809591a47e6a4bb39107aef2d64d5924e44f3dd42fd8ac4d09ca823c778d101b0c7011853e2d1be696536cf7e9ee57280dcd8dfe2dd4f97b88306faa2a0753b0f0274bea046c6f19dfd989a8a34dec70485d6b3e7c157553da617a03 +1d8c410e82301045bb9e534ce2deea11a8d8c48d10304196058b990418534aa25ec7852770e9aa17b3b27c79ffbf6a9768bd8515b2a32b8da6c796879bf1d4504ffe811dddfdec2c9c8a1a8d259ec33bbcc2077456a8439aee8f5556a418be90eb33e658e3069dc535941193888c462ae485d5a2a768ffa71244c32379968902d19a290e4174e6699dcc41d0308f17962aa67f +1dc7410ac2301005d0ace7141fdc1b3d42aa64a9a15d5497d392ca40eac818416f6fe8eaf1c65388f1483ba8c9439e5c30ebfae22a9314a93f2cf2ad1fcb94e20d09771c60823d0dada15521d06d1d2e6064308dd7fe3c90eb75657233bfd97789dcc2967db3c864ea03fd01 +1d8a3d0a02311406b77ea7f8c0dee8114437651453ac220a51a33cc88fc408e62c9e451bf55e1bb6199861baf94cca298d10135f39188753f43793f9c88e73c1859ff9912cade4060a5b4c102cc6a4abea410b7687b21f425b0f83df0b86bae57aa1a9f1ce9cbf6fa1a8b9e714bd50b51551f1ffd860444b3d +1d8c310e82401444b1dd53fcc45ef408282cb11008ab414b346a3641310413ed0cad95072034f450488c46f10a7f6fe4aeddcc9bcc0b4606a503d28538e15bbe0f2358c5bb4398f2258f787a860d3fa5c7644d268607d8881ba1ae3f1c9ba6e504ae6f0296c471d9ccb62d3605cc8947e780052ca00ff8c1067a842952fd89b8caae6e8c6878c716df3a56a4f3c01abfe222ed998ea5222f6c552e64d64426e7a794d53ae612fc00 +2dc84b0e8230100050d6738a49dc8b1e0114589800a11864398522c3a768a951120f2f212edf2b4e5e181e618793e13b6b1ab09ac60759963cb05db0e18f7d1905795622c9aa56dc6bb804253edfcad8e54b73dd4016a45877d8419c886b140522c718d2f08629967840a90cee41acf636f7a45dff3ffe3603b5ab8b243b0b702455e47ae08cc4daf552700cb534f37a31fc00 +35c7b98e82600045e1a9790a92e967dc97d20df71515b51345455114c1ed6d2616c6c2cad8d8fe2f36e67269cec9a7e5328a1296be65db31e7e6666cc9137bbd1dbba66e5aa67b9667e6c9f51c435295813c961b72e8d31f4887746a024da82934a50cc8a066d08c9a43736a012d281332a925b4a456d08ab2208b5a436b6a036d281bb2a92db4a576d08e722087da437bca855cca833cea001da82374a44ed0893a4367ea025da80c94a1b25096ca41392a0fe5a90254a01448a18a50912a4125aa0c95a90a54a1aa5095aa4135aa0ed5a906d4a09a50936a412daa0db5a90ed4a15448a5ba5097ea413daa0ff5290dd2a80134a086d0901a41232a0485a83014a62250848a42512a06c5a83814a71250824a42492a05a5a83494a6c41ff8197df77d0ffcf0fd08fcf4fd0cfcf6fd0e7cf37d0b7cf57d0dfcf2fdfa586b76f2aaf4657bae63ff8a97f475301cddfe0d8523d1583c914ca533d95cbea0144be54ab5566f345bed8edaedf5b5c17034d6275363365f98cb95b5ded8db9db377bdc3f174be883f71153771170ff1142ff196fe01 diff --git a/crates/proof/tests/fixtures/affix_compatibility.expected b/crates/proof/tests/fixtures/affix_compatibility.expected new file mode 100644 index 000000000..8fe3a9929 --- /dev/null +++ b/crates/proof/tests/fixtures/affix_compatibility.expected @@ -0,0 +1,497 @@ +pl: conditional strip, prefix cross products, diacritics +len=4 rules=5 +try="aeoiułkr" key="" map=[['a', 'ą'], ['o', 'ó', 'u']] rep=[("ż", "rz"), ("u", "ó")] comments=["original compatibility fixture"] +forms "Warszawa": ["Warszawa", "Warszawą"] +forms "kot": ["kot", "kotem", "koty", "niekot", "niekotem", "niekoty"] +forms "krowa": ["krowa", "krową"] +forms "piesek": ["piesek", "piesekem", "pieseky", "pieska"] +"": contains=false check=false suggest=false raw_suggest=false +"KOT": contains=false check=true suggest=true raw_suggest=false +"KOTEM": contains=false check=true suggest=true raw_suggest=false +"KOTY": contains=false check=true suggest=true raw_suggest=false +"KOTĄ": contains=false check=false suggest=false raw_suggest=false +"KROWA": contains=false check=true suggest=true raw_suggest=false +"KROWY": contains=false check=false suggest=false raw_suggest=false +"KROWĄ": contains=false check=true suggest=true raw_suggest=false +"Kot": contains=false check=true suggest=true raw_suggest=false +"Kotem": contains=false check=true suggest=true raw_suggest=false +"Koty": contains=false check=true suggest=true raw_suggest=false +"Kotą": contains=false check=false suggest=false raw_suggest=false +"Krowa": contains=false check=true suggest=true raw_suggest=false +"Krowy": contains=false check=false suggest=false raw_suggest=false +"Krową": contains=false check=true suggest=true raw_suggest=false +"NIE": contains=false check=false suggest=false raw_suggest=false +"NIEKOT": contains=false check=true suggest=true raw_suggest=false +"NIEKOTEM": contains=false check=true suggest=true raw_suggest=false +"NIEKOTY": contains=false check=true suggest=true raw_suggest=false +"NIEPIESKA": contains=false check=false suggest=false raw_suggest=false +"Nie": contains=false check=false suggest=false raw_suggest=false +"Niekot": contains=false check=true suggest=true raw_suggest=false +"Niekotem": contains=false check=true suggest=true raw_suggest=false +"Niekoty": contains=false check=true suggest=true raw_suggest=false +"Niepieska": contains=false check=false suggest=false raw_suggest=false +"PIESEK": contains=false check=true suggest=true raw_suggest=false +"PIESEKA": contains=false check=false suggest=false raw_suggest=false +"PIESEKEM": contains=false check=true suggest=true raw_suggest=false +"PIESEKY": contains=false check=true suggest=true raw_suggest=false +"PIESKA": contains=false check=true suggest=true raw_suggest=false +"Piesek": contains=false check=true suggest=true raw_suggest=false +"Pieseka": contains=false check=false suggest=false raw_suggest=false +"Piesekem": contains=false check=true suggest=true raw_suggest=false +"Pieseky": contains=false check=true suggest=true raw_suggest=false +"Pieska": contains=false check=true suggest=true raw_suggest=false +"WARSZAWA": contains=false check=true suggest=true raw_suggest=false +"WARSZAWĄ": contains=false check=true suggest=true raw_suggest=false +"Warszawa": contains=true check=true suggest=true raw_suggest=true +"Warszawą": contains=true check=true suggest=true raw_suggest=true +"kot": contains=true check=true suggest=true raw_suggest=true +"kotem": contains=true check=true suggest=true raw_suggest=true +"koty": contains=true check=true suggest=true raw_suggest=true +"kotą": contains=false check=false suggest=false raw_suggest=false +"krowa": contains=true check=true suggest=true raw_suggest=true +"krowy": contains=false check=false suggest=false raw_suggest=false +"krową": contains=true check=true suggest=true raw_suggest=true +"nie": contains=false check=false suggest=false raw_suggest=false +"niekot": contains=true check=true suggest=true raw_suggest=true +"niekotem": contains=true check=true suggest=true raw_suggest=true +"niekoty": contains=true check=true suggest=true raw_suggest=true +"niepieska": contains=false check=false suggest=false raw_suggest=false +"piesek": contains=true check=true suggest=true raw_suggest=true +"pieseka": contains=false check=false suggest=false raw_suggest=false +"piesekem": contains=true check=true suggest=true raw_suggest=true +"pieseky": contains=true check=true suggest=true raw_suggest=true +"pieska": contains=true check=true suggest=true raw_suggest=true +"warszawa": contains=false check=false suggest=false raw_suggest=false +"warszawą": contains=false check=false suggest=false raw_suggest=false +pt: UTF-8 flags, forbidden words, two suffixes +len=4 rules=3 +try="aeiouçãõ" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "bonito": ["bonita", "bonitas", "bonito", "bonitos"] +forms "casa": ["casa", "casas"] +forms "fazer": ["fazer", "refazer"] +forms "imundo": [] +"BONITA": contains=false check=true suggest=true raw_suggest=false +"BONITAS": contains=false check=true suggest=true raw_suggest=false +"BONITASS": contains=false check=false suggest=false raw_suggest=false +"BONITO": contains=false check=true suggest=true raw_suggest=false +"BONITOS": contains=false check=true suggest=true raw_suggest=false +"Bonita": contains=false check=true suggest=true raw_suggest=false +"Bonitas": contains=false check=true suggest=true raw_suggest=false +"Bonitass": contains=false check=false suggest=false raw_suggest=false +"Bonito": contains=false check=true suggest=true raw_suggest=false +"Bonitos": contains=false check=true suggest=true raw_suggest=false +"CASA": contains=false check=true suggest=true raw_suggest=false +"CASAA": contains=false check=false suggest=false raw_suggest=false +"CASAS": contains=false check=true suggest=true raw_suggest=false +"Casa": contains=false check=true suggest=true raw_suggest=false +"Casaa": contains=false check=false suggest=false raw_suggest=false +"Casas": contains=false check=true suggest=true raw_suggest=false +"FAZER": contains=false check=true suggest=true raw_suggest=false +"Fazer": contains=false check=true suggest=true raw_suggest=false +"IMUNDO": contains=false check=false suggest=false raw_suggest=false +"IMUNDOS": contains=false check=false suggest=false raw_suggest=false +"Imundo": contains=false check=false suggest=false raw_suggest=false +"Imundos": contains=false check=false suggest=false raw_suggest=false +"RECASAS": contains=false check=false suggest=false raw_suggest=false +"REFAZER": contains=false check=true suggest=true raw_suggest=false +"Recasas": contains=false check=false suggest=false raw_suggest=false +"Refazer": contains=false check=true suggest=true raw_suggest=false +"bonita": contains=true check=true suggest=true raw_suggest=true +"bonitas": contains=true check=true suggest=true raw_suggest=true +"bonitass": contains=false check=false suggest=false raw_suggest=false +"bonito": contains=true check=true suggest=true raw_suggest=true +"bonitos": contains=true check=true suggest=true raw_suggest=true +"casa": contains=true check=true suggest=true raw_suggest=true +"casaa": contains=false check=false suggest=false raw_suggest=false +"casas": contains=true check=true suggest=true raw_suggest=true +"fazer": contains=true check=true suggest=true raw_suggest=true +"imundo": contains=false check=false suggest=false raw_suggest=false +"imundos": contains=false check=false suggest=false raw_suggest=false +"recasas": contains=false check=false suggest=false raw_suggest=false +"refazer": contains=true check=true suggest=true raw_suggest=true +it: stripping, non-crossing suffix, proper-name casing +len=4 rules=3 +try="" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "Roma": ["Roma"] +forms "casa": ["casa", "case", "ricasa"] +forms "fare": ["fare", "rifare"] +forms "libro": ["libri", "libro"] +"": contains=false check=false suggest=false raw_suggest=false +"CASA": contains=false check=true suggest=true raw_suggest=false +"CASE": contains=false check=true suggest=true raw_suggest=false +"Casa": contains=false check=true suggest=true raw_suggest=false +"Case": contains=false check=true suggest=true raw_suggest=false +"FARE": contains=false check=true suggest=true raw_suggest=false +"Fare": contains=false check=true suggest=true raw_suggest=false +"LIBRI": contains=false check=true suggest=true raw_suggest=false +"LIBRO": contains=false check=true suggest=true raw_suggest=false +"LIBROS": contains=false check=false suggest=false raw_suggest=false +"Libri": contains=false check=true suggest=true raw_suggest=false +"Libro": contains=false check=true suggest=true raw_suggest=false +"Libros": contains=false check=false suggest=false raw_suggest=false +"R ILIBRI": contains=false check=false suggest=false raw_suggest=false +"R ilibri": contains=false check=false suggest=false raw_suggest=false +"RICAS A": contains=false check=false suggest=false raw_suggest=false +"RICASA": contains=false check=true suggest=true raw_suggest=false +"RICASE": contains=false check=false suggest=false raw_suggest=false +"RIFARE": contains=false check=true suggest=true raw_suggest=false +"ROMA": contains=false check=true suggest=true raw_suggest=false +"Ricas a": contains=false check=false suggest=false raw_suggest=false +"Ricasa": contains=false check=true suggest=true raw_suggest=false +"Ricase": contains=false check=false suggest=false raw_suggest=false +"Rifare": contains=false check=true suggest=true raw_suggest=false +"Roma": contains=true check=true suggest=true raw_suggest=true +"casa": contains=true check=true suggest=true raw_suggest=true +"case": contains=true check=true suggest=true raw_suggest=true +"fare": contains=true check=true suggest=true raw_suggest=true +"libri": contains=true check=true suggest=true raw_suggest=true +"libro": contains=true check=true suggest=true raw_suggest=true +"libros": contains=false check=false suggest=false raw_suggest=false +"r ilibri": contains=false check=false suggest=false raw_suggest=false +"ricas a": contains=false check=false suggest=false raw_suggest=false +"ricasa": contains=true check=true suggest=true raw_suggest=true +"ricase": contains=false check=false suggest=false raw_suggest=false +"rifare": contains=true check=true suggest=true raw_suggest=true +"roma": contains=false check=false suggest=false raw_suggest=false +cs: negated conditions and shared Unicode stems +len=4 rules=3 +try="" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "mladý": ["mladý", "nemladý"] +forms "strom": ["nestrom", "nestromy", "strom", "stromy"] +forms "y": ["y"] +forms "žena": ["žena", "ženě"] +"MLADÝ": contains=false check=true suggest=true raw_suggest=false +"MLADÝY": contains=false check=false suggest=false raw_suggest=false +"Mladý": contains=false check=true suggest=true raw_suggest=false +"Mladýy": contains=false check=false suggest=false raw_suggest=false +"NEMLADÝ": contains=false check=true suggest=true raw_suggest=false +"NESTROM": contains=false check=true suggest=true raw_suggest=false +"NESTROMY": contains=false check=true suggest=true raw_suggest=false +"NEŽENĚ": contains=false check=false suggest=false raw_suggest=false +"Nemladý": contains=false check=true suggest=true raw_suggest=false +"Nestrom": contains=false check=true suggest=true raw_suggest=false +"Nestromy": contains=false check=true suggest=true raw_suggest=false +"Neženě": contains=false check=false suggest=false raw_suggest=false +"STROM": contains=false check=true suggest=true raw_suggest=false +"STROMY": contains=false check=true suggest=true raw_suggest=false +"Strom": contains=false check=true suggest=true raw_suggest=false +"Stromy": contains=false check=true suggest=true raw_suggest=false +"Y": contains=false check=true suggest=true raw_suggest=false +"YY": contains=false check=false suggest=false raw_suggest=false +"Yy": contains=false check=false suggest=false raw_suggest=false +"mladý": contains=true check=true suggest=true raw_suggest=true +"mladýy": contains=false check=false suggest=false raw_suggest=false +"nemladý": contains=true check=true suggest=true raw_suggest=true +"nestrom": contains=true check=true suggest=true raw_suggest=true +"nestromy": contains=true check=true suggest=true raw_suggest=true +"neženě": contains=false check=false suggest=false raw_suggest=false +"strom": contains=true check=true suggest=true raw_suggest=true +"stromy": contains=true check=true suggest=true raw_suggest=true +"y": contains=true check=true suggest=true raw_suggest=true +"yy": contains=false check=false suggest=false raw_suggest=false +"ŽENA": contains=false check=true suggest=true raw_suggest=false +"ŽENY": contains=false check=false suggest=false raw_suggest=false +"ŽENĚ": contains=false check=true suggest=true raw_suggest=false +"Žena": contains=false check=true suggest=true raw_suggest=false +"Ženy": contains=false check=false suggest=false raw_suggest=false +"Ženě": contains=false check=true suggest=true raw_suggest=false +"žena": contains=true check=true suggest=true raw_suggest=true +"ženy": contains=false check=false suggest=false raw_suggest=false +"ženě": contains=true check=true suggest=true raw_suggest=true +ru: Cyrillic flags and suggestions exclusion +len=4 rules=2 +try="" key="" map=[['е', 'ё']] rep=[] comments=["original compatibility fixture"] +forms "дом": ["дом", "домы"] +forms "запрет": [] +forms "кот": ["кот", "коты", "некот", "некоты"] +forms "тайна": ["тайна", "тайнаы"] +"ДОМ": contains=false check=true suggest=true raw_suggest=false +"ДОМЫ": contains=false check=true suggest=true raw_suggest=false +"Дом": contains=false check=true suggest=true raw_suggest=false +"Домы": contains=false check=true suggest=true raw_suggest=false +"ЗАПРЕТ": contains=false check=false suggest=false raw_suggest=false +"ЗАПРЕТЫ": contains=false check=false suggest=false raw_suggest=false +"Запрет": contains=false check=false suggest=false raw_suggest=false +"Запреты": contains=false check=false suggest=false raw_suggest=false +"КОТ": contains=false check=true suggest=true raw_suggest=false +"КОТЫ": contains=false check=true suggest=true raw_suggest=false +"КОт": contains=false check=false suggest=false raw_suggest=false +"Кот": contains=false check=true suggest=true raw_suggest=false +"Коты": contains=false check=true suggest=true raw_suggest=false +"НЕДОМЫ": contains=false check=false suggest=false raw_suggest=false +"НЕКОТ": contains=false check=true suggest=true raw_suggest=false +"НЕКОТЫ": contains=false check=true suggest=true raw_suggest=false +"Недомы": contains=false check=false suggest=false raw_suggest=false +"Некот": contains=false check=true suggest=true raw_suggest=false +"Некоты": contains=false check=true suggest=true raw_suggest=false +"ТАЙНА": contains=false check=true suggest=false raw_suggest=false +"ТАЙНАЫ": contains=false check=true suggest=false raw_suggest=false +"Тайна": contains=false check=true suggest=false raw_suggest=false +"Тайнаы": contains=false check=true suggest=false raw_suggest=false +"дом": contains=true check=true suggest=true raw_suggest=true +"домы": contains=true check=true suggest=true raw_suggest=true +"запрет": contains=false check=false suggest=false raw_suggest=false +"запреты": contains=false check=false suggest=false raw_suggest=false +"кОт": contains=false check=false suggest=false raw_suggest=false +"кот": contains=true check=true suggest=true raw_suggest=true +"коты": contains=true check=true suggest=true raw_suggest=true +"недомы": contains=false check=false suggest=false raw_suggest=false +"некот": contains=true check=true suggest=true raw_suggest=true +"некоты": contains=true check=true suggest=true raw_suggest=true +"тайна": contains=true check=true suggest=false raw_suggest=false +"тайнаы": contains=true check=true suggest=false raw_suggest=false +id: continuation and NOSUGGEST through derivations +len=3 rules=3 +try="abcdeikn" key="qwerty|asdf" map=[] rep=[("dj", "j")] comments=["original compatibility fixture"] +forms "baca": ["baca", "bacakan", "bacakanlah"] +forms "main": ["bermain", "bermainkan", "main", "mainkan", "mainkanlah"] +forms "rahasia": ["rahasia", "rahasiakan", "rahasiakanlah"] +"BACA": contains=false check=true suggest=true raw_suggest=false +"BACAKAN": contains=false check=true suggest=true raw_suggest=false +"BACAKANLAH": contains=false check=true suggest=true raw_suggest=false +"BERBACA": contains=false check=false suggest=false raw_suggest=false +"BERMAIN": contains=false check=true suggest=true raw_suggest=false +"BERMAINKAN": contains=false check=true suggest=true raw_suggest=false +"BERMAINKANLAH": contains=false check=false suggest=false raw_suggest=false +"Baca": contains=false check=true suggest=true raw_suggest=false +"Bacakan": contains=false check=true suggest=true raw_suggest=false +"Bacakanlah": contains=false check=true suggest=true raw_suggest=false +"Berbaca": contains=false check=false suggest=false raw_suggest=false +"Bermain": contains=false check=true suggest=true raw_suggest=false +"Bermainkan": contains=false check=true suggest=true raw_suggest=false +"Bermainkanlah": contains=false check=false suggest=false raw_suggest=false +"MAIN": contains=false check=true suggest=true raw_suggest=false +"MAINKAN": contains=false check=true suggest=true raw_suggest=false +"MAINKANLAH": contains=false check=true suggest=true raw_suggest=false +"MAINKANLAHLAH": contains=false check=false suggest=false raw_suggest=false +"MAINLAH": contains=false check=false suggest=false raw_suggest=false +"Main": contains=false check=true suggest=true raw_suggest=false +"Mainkan": contains=false check=true suggest=true raw_suggest=false +"Mainkanlah": contains=false check=true suggest=true raw_suggest=false +"Mainkanlahlah": contains=false check=false suggest=false raw_suggest=false +"Mainlah": contains=false check=false suggest=false raw_suggest=false +"RAHASIA": contains=false check=true suggest=false raw_suggest=false +"RAHASIAKAN": contains=false check=true suggest=false raw_suggest=false +"RAHASIAKANLAH": contains=false check=true suggest=false raw_suggest=false +"Rahasia": contains=false check=true suggest=false raw_suggest=false +"Rahasiakan": contains=false check=true suggest=false raw_suggest=false +"Rahasiakanlah": contains=false check=true suggest=false raw_suggest=false +"baca": contains=true check=true suggest=true raw_suggest=true +"bacakan": contains=true check=true suggest=true raw_suggest=true +"bacakanlah": contains=true check=true suggest=true raw_suggest=true +"berbaca": contains=false check=false suggest=false raw_suggest=false +"bermain": contains=true check=true suggest=true raw_suggest=true +"bermainkan": contains=true check=true suggest=true raw_suggest=true +"bermainkanlah": contains=false check=false suggest=false raw_suggest=false +"main": contains=true check=true suggest=true raw_suggest=true +"mainkan": contains=true check=true suggest=true raw_suggest=true +"mainkanlah": contains=true check=true suggest=true raw_suggest=true +"mainkanlahlah": contains=false check=false suggest=false raw_suggest=false +"mainlah": contains=false check=false suggest=false raw_suggest=false +"rahasia": contains=true check=true suggest=false raw_suggest=false +"rahasiakan": contains=true check=true suggest=false raw_suggest=false +"rahasiakanlah": contains=true check=true suggest=false raw_suggest=false +pt-sized: 70 distinct flags across the old u128 boundary +len=2 rules=70 +try="" key="" map=[] rep=[] comments=["original compatibility fixture"] +forms "outro": ["outro", "outroõ"] +forms "verbo": ["verbo", "verbo1", "verbo2", "verbo3", "verbo4", "verbo5", "verbo6", "verbo7", "verbo8", "verbo9", "verboA", "verboB", "verboC", "verboD", "verboE", "verboF", "verboG", "verboH", "verboI", "verboJ", "verboK", "verboL", "verboM", "verboN", "verboO", "verboP", "verboQ", "verboR", "verboS", "verboT", "verboU", "verboV", "verboW", "verboX", "verboY", "verboZ", "verboa", "verbob", "verboc", "verbod", "verboe", "verbof", "verbog", "verboh", "verboi", "verboj", "verbok", "verbol", "verbom", "verbon", "verboo", "verbop", "verboq", "verbor", "verbos", "verbot", "verbou", "verbov", "verbow", "verbox", "verboy", "verboz", "verboá", "verboã", "verboç", "verboé", "verboí", "verboó", "verboõ", "verboú"] +"OUTRO": contains=false check=true suggest=true raw_suggest=false +"OUTROA": contains=false check=false suggest=false raw_suggest=false +"OUTROÕ": contains=false check=true suggest=true raw_suggest=false +"Outro": contains=false check=true suggest=true raw_suggest=false +"Outroa": contains=false check=false suggest=false raw_suggest=false +"Outroõ": contains=false check=true suggest=true raw_suggest=false +"VERBO": contains=false check=true suggest=true raw_suggest=false +"VERBO0": contains=false check=false suggest=false raw_suggest=false +"VERBO1": contains=false check=true suggest=true raw_suggest=false +"VERBO2": contains=false check=true suggest=true raw_suggest=false +"VERBO3": contains=false check=true suggest=true raw_suggest=false +"VERBO4": contains=false check=true suggest=true raw_suggest=false +"VERBO5": contains=false check=true suggest=true raw_suggest=false +"VERBO6": contains=false check=true suggest=true raw_suggest=false +"VERBO7": contains=false check=true suggest=true raw_suggest=false +"VERBO8": contains=false check=true suggest=true raw_suggest=false +"VERBO9": contains=false check=true suggest=true raw_suggest=false +"VERBOA": contains=false check=true suggest=true raw_suggest=false +"VERBOB": contains=false check=true suggest=true raw_suggest=false +"VERBOC": contains=false check=true suggest=true raw_suggest=false +"VERBOD": contains=false check=true suggest=true raw_suggest=false +"VERBOE": contains=false check=true suggest=true raw_suggest=false +"VERBOF": contains=false check=true suggest=true raw_suggest=false +"VERBOG": contains=false check=true suggest=true raw_suggest=false +"VERBOH": contains=false check=true suggest=true raw_suggest=false +"VERBOI": contains=false check=true suggest=true raw_suggest=false +"VERBOJ": contains=false check=true suggest=true raw_suggest=false +"VERBOK": contains=false check=true suggest=true raw_suggest=false +"VERBOL": contains=false check=true suggest=true raw_suggest=false +"VERBOM": contains=false check=true suggest=true raw_suggest=false +"VERBON": contains=false check=true suggest=true raw_suggest=false +"VERBOO": contains=false check=true suggest=true raw_suggest=false +"VERBOP": contains=false check=true suggest=true raw_suggest=false +"VERBOQ": contains=false check=true suggest=true raw_suggest=false +"VERBOR": contains=false check=true suggest=true raw_suggest=false +"VERBOS": contains=false check=true suggest=true raw_suggest=false +"VERBOSS": contains=false check=false suggest=false raw_suggest=false +"VERBOT": contains=false check=true suggest=true raw_suggest=false +"VERBOU": contains=false check=true suggest=true raw_suggest=false +"VERBOV": contains=false check=true suggest=true raw_suggest=false +"VERBOW": contains=false check=true suggest=true raw_suggest=false +"VERBOX": contains=false check=true suggest=true raw_suggest=false +"VERBOY": contains=false check=true suggest=true raw_suggest=false +"VERBOZ": contains=false check=true suggest=true raw_suggest=false +"VERBOÁ": contains=false check=true suggest=true raw_suggest=false +"VERBOÃ": contains=false check=true suggest=true raw_suggest=false +"VERBOÇ": contains=false check=true suggest=true raw_suggest=false +"VERBOÉ": contains=false check=true suggest=true raw_suggest=false +"VERBOÍ": contains=false check=true suggest=true raw_suggest=false +"VERBOÓ": contains=false check=true suggest=true raw_suggest=false +"VERBOÕ": contains=false check=true suggest=true raw_suggest=false +"VERBOÚ": contains=false check=true suggest=true raw_suggest=false +"Verbo": contains=false check=true suggest=true raw_suggest=false +"Verbo0": contains=false check=false suggest=false raw_suggest=false +"Verbo1": contains=false check=true suggest=true raw_suggest=false +"Verbo2": contains=false check=true suggest=true raw_suggest=false +"Verbo3": contains=false check=true suggest=true raw_suggest=false +"Verbo4": contains=false check=true suggest=true raw_suggest=false +"Verbo5": contains=false check=true suggest=true raw_suggest=false +"Verbo6": contains=false check=true suggest=true raw_suggest=false +"Verbo7": contains=false check=true suggest=true raw_suggest=false +"Verbo8": contains=false check=true suggest=true raw_suggest=false +"Verbo9": contains=false check=true suggest=true raw_suggest=false +"VerboA": contains=false check=false suggest=false raw_suggest=false +"VerboB": contains=false check=false suggest=false raw_suggest=false +"VerboC": contains=false check=false suggest=false raw_suggest=false +"VerboD": contains=false check=false suggest=false raw_suggest=false +"VerboE": contains=false check=false suggest=false raw_suggest=false +"VerboF": contains=false check=false suggest=false raw_suggest=false +"VerboG": contains=false check=false suggest=false raw_suggest=false +"VerboH": contains=false check=false suggest=false raw_suggest=false +"VerboI": contains=false check=false suggest=false raw_suggest=false +"VerboJ": contains=false check=false suggest=false raw_suggest=false +"VerboK": contains=false check=false suggest=false raw_suggest=false +"VerboL": contains=false check=false suggest=false raw_suggest=false +"VerboM": contains=false check=false suggest=false raw_suggest=false +"VerboN": contains=false check=false suggest=false raw_suggest=false +"VerboO": contains=false check=false suggest=false raw_suggest=false +"VerboP": contains=false check=false suggest=false raw_suggest=false +"VerboQ": contains=false check=false suggest=false raw_suggest=false +"VerboR": contains=false check=false suggest=false raw_suggest=false +"VerboS": contains=false check=false suggest=false raw_suggest=false +"VerboT": contains=false check=false suggest=false raw_suggest=false +"VerboU": contains=false check=false suggest=false raw_suggest=false +"VerboV": contains=false check=false suggest=false raw_suggest=false +"VerboW": contains=false check=false suggest=false raw_suggest=false +"VerboX": contains=false check=false suggest=false raw_suggest=false +"VerboY": contains=false check=false suggest=false raw_suggest=false +"VerboZ": contains=false check=false suggest=false raw_suggest=false +"Verboa": contains=false check=true suggest=true raw_suggest=false +"Verbob": contains=false check=true suggest=true raw_suggest=false +"Verboc": contains=false check=true suggest=true raw_suggest=false +"Verbod": contains=false check=true suggest=true raw_suggest=false +"Verboe": contains=false check=true suggest=true raw_suggest=false +"Verbof": contains=false check=true suggest=true raw_suggest=false +"Verbog": contains=false check=true suggest=true raw_suggest=false +"Verboh": contains=false check=true suggest=true raw_suggest=false +"Verboi": contains=false check=true suggest=true raw_suggest=false +"Verboj": contains=false check=true suggest=true raw_suggest=false +"Verbok": contains=false check=true suggest=true raw_suggest=false +"Verbol": contains=false check=true suggest=true raw_suggest=false +"Verbom": contains=false check=true suggest=true raw_suggest=false +"Verbon": contains=false check=true suggest=true raw_suggest=false +"Verboo": contains=false check=true suggest=true raw_suggest=false +"Verbop": contains=false check=true suggest=true raw_suggest=false +"Verboq": contains=false check=true suggest=true raw_suggest=false +"Verbor": contains=false check=true suggest=true raw_suggest=false +"Verbos": contains=false check=true suggest=true raw_suggest=false +"Verbot": contains=false check=true suggest=true raw_suggest=false +"Verbou": contains=false check=true suggest=true raw_suggest=false +"Verbov": contains=false check=true suggest=true raw_suggest=false +"Verbow": contains=false check=true suggest=true raw_suggest=false +"Verbox": contains=false check=true suggest=true raw_suggest=false +"Verboy": contains=false check=true suggest=true raw_suggest=false +"Verboz": contains=false check=true suggest=true raw_suggest=false +"Verboß": contains=false check=false suggest=false raw_suggest=false +"Verboá": contains=false check=true suggest=true raw_suggest=false +"Verboã": contains=false check=true suggest=true raw_suggest=false +"Verboç": contains=false check=true suggest=true raw_suggest=false +"Verboé": contains=false check=true suggest=true raw_suggest=false +"Verboí": contains=false check=true suggest=true raw_suggest=false +"Verboó": contains=false check=true suggest=true raw_suggest=false +"Verboõ": contains=false check=true suggest=true raw_suggest=false +"Verboú": contains=false check=true suggest=true raw_suggest=false +"outro": contains=true check=true suggest=true raw_suggest=true +"outroa": contains=false check=false suggest=false raw_suggest=false +"outroõ": contains=true check=true suggest=true raw_suggest=true +"verbo": contains=true check=true suggest=true raw_suggest=true +"verbo0": contains=false check=false suggest=false raw_suggest=false +"verbo1": contains=true check=true suggest=true raw_suggest=true +"verbo2": contains=true check=true suggest=true raw_suggest=true +"verbo3": contains=true check=true suggest=true raw_suggest=true +"verbo4": contains=true check=true suggest=true raw_suggest=true +"verbo5": contains=true check=true suggest=true raw_suggest=true +"verbo6": contains=true check=true suggest=true raw_suggest=true +"verbo7": contains=true check=true suggest=true raw_suggest=true +"verbo8": contains=true check=true suggest=true raw_suggest=true +"verbo9": contains=true check=true suggest=true raw_suggest=true +"verboA": contains=true check=true suggest=true raw_suggest=true +"verboB": contains=true check=true suggest=true raw_suggest=true +"verboC": contains=true check=true suggest=true raw_suggest=true +"verboD": contains=true check=true suggest=true raw_suggest=true +"verboE": contains=true check=true suggest=true raw_suggest=true +"verboF": contains=true check=true suggest=true raw_suggest=true +"verboG": contains=true check=true suggest=true raw_suggest=true +"verboH": contains=true check=true suggest=true raw_suggest=true +"verboI": contains=true check=true suggest=true raw_suggest=true +"verboJ": contains=true check=true suggest=true raw_suggest=true +"verboK": contains=true check=true suggest=true raw_suggest=true +"verboL": contains=true check=true suggest=true raw_suggest=true +"verboM": contains=true check=true suggest=true raw_suggest=true +"verboN": contains=true check=true suggest=true raw_suggest=true +"verboO": contains=true check=true suggest=true raw_suggest=true +"verboP": contains=true check=true suggest=true raw_suggest=true +"verboQ": contains=true check=true suggest=true raw_suggest=true +"verboR": contains=true check=true suggest=true raw_suggest=true +"verboS": contains=true check=true suggest=true raw_suggest=true +"verboT": contains=true check=true suggest=true raw_suggest=true +"verboU": contains=true check=true suggest=true raw_suggest=true +"verboV": contains=true check=true suggest=true raw_suggest=true +"verboW": contains=true check=true suggest=true raw_suggest=true +"verboX": contains=true check=true suggest=true raw_suggest=true +"verboY": contains=true check=true suggest=true raw_suggest=true +"verboZ": contains=true check=true suggest=true raw_suggest=true +"verboa": contains=true check=true suggest=true raw_suggest=true +"verbob": contains=true check=true suggest=true raw_suggest=true +"verboc": contains=true check=true suggest=true raw_suggest=true +"verbod": contains=true check=true suggest=true raw_suggest=true +"verboe": contains=true check=true suggest=true raw_suggest=true +"verbof": contains=true check=true suggest=true raw_suggest=true +"verbog": contains=true check=true suggest=true raw_suggest=true +"verboh": contains=true check=true suggest=true raw_suggest=true +"verboi": contains=true check=true suggest=true raw_suggest=true +"verboj": contains=true check=true suggest=true raw_suggest=true +"verbok": contains=true check=true suggest=true raw_suggest=true +"verbol": contains=true check=true suggest=true raw_suggest=true +"verbom": contains=true check=true suggest=true raw_suggest=true +"verbon": contains=true check=true suggest=true raw_suggest=true +"verboo": contains=true check=true suggest=true raw_suggest=true +"verbop": contains=true check=true suggest=true raw_suggest=true +"verboq": contains=true check=true suggest=true raw_suggest=true +"verbor": contains=true check=true suggest=true raw_suggest=true +"verbos": contains=true check=true suggest=true raw_suggest=true +"verbot": contains=true check=true suggest=true raw_suggest=true +"verbou": contains=true check=true suggest=true raw_suggest=true +"verbov": contains=true check=true suggest=true raw_suggest=true +"verbow": contains=true check=true suggest=true raw_suggest=true +"verbox": contains=true check=true suggest=true raw_suggest=true +"verboy": contains=true check=true suggest=true raw_suggest=true +"verboz": contains=true check=true suggest=true raw_suggest=true +"verboß": contains=false check=false suggest=false raw_suggest=false +"verboá": contains=true check=true suggest=true raw_suggest=true +"verboã": contains=true check=true suggest=true raw_suggest=true +"verboç": contains=true check=true suggest=true raw_suggest=true +"verboé": contains=true check=true suggest=true raw_suggest=true +"verboí": contains=true check=true suggest=true raw_suggest=true +"verboó": contains=true check=true suggest=true raw_suggest=true +"verboõ": contains=true check=true suggest=true raw_suggest=true +"verboú": contains=true check=true suggest=true raw_suggest=true diff --git a/crates/proof/tests/fixtures/affix_compatibility_cases.rs b/crates/proof/tests/fixtures/affix_compatibility_cases.rs new file mode 100644 index 000000000..d0c2973b4 --- /dev/null +++ b/crates/proof/tests/fixtures/affix_compatibility_cases.rs @@ -0,0 +1,61 @@ +// Original, deliberately small fixtures for behavior supported before French proofing. +// These are not extracts from installed dictionaries and do not claim full language coverage. +pub struct Case { + pub name: &'static str, + pub aff: String, + pub dic: String, + pub probes: Vec<&'static str>, +} + +pub fn cases() -> Vec { + let mut out = vec![ + Case { + name: "pl: conditional strip, prefix cross products, diacritics", + aff: "SET UTF-8\nTRY aeoiułkr\nPFX b Y 1\nPFX b 0 nie .\nSFX a Y 3\nSFX a 0 y [^y]\nSFX a ek ka ek\nSFX a 0 em .\nSFX c N 1\nSFX c a ą a\nMAP 2\nMAP aą\nMAP oóu\nREP 2\nREP ż rz\nREP u ó\n".into(), + dic: "4\nkot/ab\npiesek/a\nkrowa/c\nWarszawa/c\n".into(), + probes: vec!["kot", "koty", "kotem", "niekoty", "pieska", "pieseka", "niepieska", "krową", "krowy", "Warszawą", "warszawą", "kotą", "nie", ""], + }, + Case { + name: "pt: UTF-8 flags, forbidden words, two suffixes", + aff: "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ý\nTRY aeiouçãõ\nSFX A Y 1\nSFX A o a/B o\nSFX B Y 1\nSFX B 0 s .\nPFX P Y 1\nPFX P 0 re .\n".into(), + dic: "4\nbonito/AB\ncasa/B\nfazer/P\nimundo/ýB\n".into(), + probes: vec!["bonito", "bonita", "bonitos", "bonitas", "bonitass", "casa", "casas", "casaa", "refazer", "recasas", "imundo", "imundos", "Imundo", "IMUNDO"], + }, + Case { + name: "it: stripping, non-crossing suffix, proper-name casing", + aff: "SET UTF-8\nSFX A Y 1\nSFX A o i o\nSFX B N 1\nSFX B a e a\nPFX P Y 1\nPFX P 0 ri .\n".into(), + dic: "4\nlibro/A\ncasa/BP\nfare/P\nRoma\n".into(), + probes: vec!["libri", "libros", "case", "ricas a", "ricasa", "ricase", "rifare", "r ilibri", "Roma", "roma", "ROMA", ""], + }, + Case { + name: "cs: negated conditions and shared Unicode stems", + aff: "SET UTF-8\nPFX N Y 1\nPFX N 0 ne .\nSFX S Y 1\nSFX S 0 y [^y]\nSFX E N 1\nSFX E a ě a\n".into(), + dic: "4\nstrom/NS\nžena/E\nmladý/N\ny/S\n".into(), + probes: vec!["stromy", "nestromy", "nestrom", "ženě", "ženy", "neženě", "nemladý", "mladýy", "yy", "Ženě", "STROMY"], + }, + Case { + name: "ru: Cyrillic flags and suggestions exclusion", + aff: "SET UTF-8\nFLAG UTF-8\nNOSUGGEST Н\nFORBIDDENWORD Ф\nSFX Я Y 1\nSFX Я 0 ы .\nPFX П Y 1\nPFX П 0 не .\nMAP 1\nMAP её\n".into(), + dic: "4\nкот/ЯП\nдом/Я\nтайна/НЯ\nзапрет/ФЯ\n".into(), + probes: vec!["кот", "коты", "некоты", "домы", "недомы", "тайна", "тайнаы", "ТАЙНАЫ", "запрет", "запреты", "КОТЫ", "кОт"], + }, + Case { + name: "id: continuation and NOSUGGEST through derivations", + aff: "SET UTF-8\nNOSUGGEST N\nKEY qwerty|asdf\nTRY abcdeikn\nREP 1\nREP dj j\nPFX P Y 1\nPFX P 0 ber .\nSFX A Y 1\nSFX A 0 kan/B .\nSFX B Y 1\nSFX B 0 lah .\n".into(), + dic: "3\nmain/PA\nbaca/A\nrahasia/NA\n".into(), + probes: vec!["main", "bermain", "mainkan", "mainkanlah", "bermainkanlah", "mainlah", "mainkanlahlah", "bacakanlah", "berbaca", "rahasia", "rahasiakan", "RAHASIAKANLAH"], + }, + ]; + let flags: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); + let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); + for flag in &flags { + aff.push_str(&format!("SFX {flag} N 1\nSFX {flag} 0 {flag} .\n")); + } + out.push(Case { + name: "pt-sized: 70 distinct flags across the old u128 boundary", + aff, + dic: format!("2\nverbo/{}\noutro/õ\n", flags.iter().collect::()), + probes: vec!["verboa", "verboz", "verboA", "verboZ", "verbo0", "verbo9", "verboç", "verboõ", "verboß", "outroõ", "outroa"], + }); + out +} diff --git a/crates/proof/tests/french_dictionary.rs b/crates/proof/tests/french_dictionary.rs new file mode 100644 index 000000000..5612a0e61 --- /dev/null +++ b/crates/proof/tests/french_dictionary.rs @@ -0,0 +1,107 @@ +//! Explicit manual real-dictionary gate. Verify the pinned runtime inputs as documented in +//! crates/proof/README.md before running with `--ignored`. Missing data is a failure, +//! never a silently successful test that checked nothing. + +#![cfg(not(target_arch = "wasm32"))] +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use craft_proof::{IssueKind, Options, ProofLang, fr}; + +#[test] +#[ignore = "requires a real French dictionary in WORDCRAFT_DICTIONARIES"] +fn real_french_dictionary_spelling_suggestions_and_dispatch() { + let directory = std::env::var_os("WORDCRAFT_DICTIONARIES").expect("set WORDCRAFT_DICTIONARIES to the test dictionary folder"); + craft_proof::set_dictionaries_folder(Some(std::path::PathBuf::from(&directory))); + let french = ProofLang::from_tag(Some("fr-FR")); + let installed = craft_proof::pack_dictionary_file(french).expect("the real French dictionary must be installed"); + assert_eq!(installed.parent(), Some(std::path::Path::new(&directory)), "do not silently use another system dictionary"); + let aff = installed.with_extension("aff"); + let dictionary = craft_proof::hunspell::load(&aff, &installed).expect("the real French dictionary loads"); + assert!(dictionary.len() > 50_000, "expected Dicollecte, not the synthetic fixture: {}", dictionary.len()); + for word in [ + "maison", + "maisons", + "Paris", + "mangeons", + "finissaient", + "étaient", + "l'arbre", + "l’arbre", + "qu'elles", + "école", + "œuvre", + "aujourd'hui", + "chevaux", + "heureuse", + "va-t-il", + ] { + assert!(fr::is_correct(word), "correct French word: {word}"); + } + for (typo, expected) in [("maisonn", "maison"), ("ecole", "école"), ("l’ecole", "l’école"), ("documant", "document")] { + assert!(!fr::is_correct(typo), "misspelling: {typo}"); + let suggestions = fr::suggest(typo, 8); + assert!(suggestions.iter().any(|s| s == expected), "{typo} -> {suggestions:?}, expected {expected}"); + } + let text = "Une maisonn près de l’école."; + for lang in [ProofLang::from_tag(Some("fr-FR")), ProofLang::from_tag(Some("fr-CA"))] { + let issues = craft_proof::check_text(text, &[(0..text.len(), lang)]); + assert_eq!(issues.len(), 1, "{lang:?}: {issues:?}"); + let issue = &issues[0]; + assert_eq!(&text[issue.start..issue.end], "maisonn"); + assert_eq!(issue.lang, lang); + assert_eq!(issue.message, "Faute d’orthographe possible"); + } + check_combining_marks_and_compounds(&dictionary); + assert!(craft_proof::dictionary_load_error(ProofLang::from_tag(Some("fr-FR"))).is_none()); +} + +fn check_combining_marks_and_compounds(dictionary: &craft_proof::affix::AffixDict) { + let spelling = |text: &str, spans: &[(std::ops::Range, ProofLang)]| { + craft_proof::check_text_with(text, spans, &Options::default(), false, false) + .into_iter() + .filter(|i| i.kind == IssueKind::Spelling) + .map(|i| (i.start, i.end, i.lang)) + .collect::>() + }; + let mut failures = Vec::new(); + for lang in [ProofLang::from_tag(Some("fr-FR")), ProofLang::from_tag(Some("fr-CA"))] { + for text in + ["école", "e\u{301}cole", "l’école", "l’e\u{301}cole", "café", "cafe\u{301}", "porte-monnaie", "va-t-il", "a-t-il", "maison-école"] + { + assert!( + fr::is_correct_with(dictionary, text) || ["va-t-il", "a-t-il", "maison-école"].contains(&text), + "real dictionary recognizes {text:?}" + ); + let issues = spelling(text, &[(0..text.len(), lang)]); + if !issues.is_empty() { + failures.push(format!("correct {lang:?} {text:?}: {issues:?}")); + } + } + for text in ["porte--monnaie", "va--t-il", "maison--école", "e\u{301}colllle", "cafffe\u{301}"] { + assert!(!fr::is_correct_with(dictionary, text), "the explicit dictionary entry must not already accept {text:?}"); + let issues = spelling(text, &[(0..text.len(), lang)]); + let expected = vec![(0, text.len(), lang)]; + if issues != expected { + failures.push(format!("typo {lang:?} {text:?}: {issues:?}, wanted {expected:?}")); + } + } + let text = "e\u{301}cole cafe\u{301} xyzxyz porte--monnaie"; + let foreign = text.find("xyzxyz").unwrap(); + let compound = text.find("porte").unwrap(); + let spans = [(0..foreign, lang), (foreign..compound, ProofLang::Other), (compound..text.len(), lang)]; + let issues = spelling(text, &spans); + if issues != [(compound, text.len(), lang)] { + failures.push(format!("mixed {lang:?}: {issues:?}")); + } + assert!(spelling(text, &[(0..text.len(), ProofLang::Other)]).is_empty()); + // An English word remains in its own dictionary next to the original French bytes. + let text = "e\u{301}cole hello cafe\u{301}"; + let en = text.find("hello").unwrap(); + let spans = [(0..en, lang), (en..en + 5, ProofLang::En), (en + 5..text.len(), lang)]; + let issues = spelling(text, &spans); + if !issues.is_empty() { + failures.push(format!("English boundary {lang:?}: {issues:?}")); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} diff --git a/crates/proof/tests/french_packs.rs b/crates/proof/tests/french_packs.rs new file mode 100644 index 000000000..64ad46748 --- /dev/null +++ b/crates/proof/tests/french_packs.rs @@ -0,0 +1,124 @@ +//! French uses ordinary Language packs and keeps each regional locale at the public API. +#![allow(clippy::unwrap_used, clippy::expect_used)] + +use craft_proof::affix; +use craft_proof::{IssueKind, Options, ProofLang, check_text_with, is_correct_in_with, suggest_in}; +use std::path::{Path, PathBuf}; + +#[path = "../src/fr/test_dict.rs"] +mod fixture; + +struct Tools(PathBuf); +impl Drop for Tools { + fn drop(&mut self) { + craft_proof::set_dictionaries_folder(None); + craft_proof::packs::set_packs_dir(None); + craft_proof::remove_word("zxqrechargefr"); + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn pair(dir: &Path, name: &str, aff: &str, dic: &str) { + std::fs::create_dir_all(dir).unwrap(); + std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); + std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap(); +} + +#[test] +fn french_packs_regions_reload_and_original_ranges() { + assert!(!fixture::dictionary().is_empty()); + let root = std::env::temp_dir().join(format!("craft-fr-packs-{}", std::process::id())); + let _tools = Tools(root.clone()); + let pack = root.join("packs/fr"); + pair(&pack, "spelling", fixture::AFF, fixture::DIC); + std::fs::write(pack.join("pack.toml"), "locale = 'fr'\nname = 'Français'\nlicence = 'MIT'\n").unwrap(); + std::fs::write(pack.join("hyphenation.pat"), "ma1ison\n").unwrap(); + let folder = root.join("dictionaries"); + std::fs::create_dir_all(&folder).unwrap(); + craft_proof::packs::set_packs_dir(Some(root.join("packs"))); + craft_proof::set_dictionaries_folder(Some(folder.clone())); + let defaults = Options::default(); + let strict = Options { ignore_uppercase: false, ignore_numbers: false, ..defaults }; + check_locales(defaults, strict); + check_typography(strict); + let fr = ProofLang::from_tag(Some("fr-FR")); + let ca = ProofLang::from_tag(Some("fr-CA")); + check_user_word(fr, ca, strict); + check_overrides(folder, fr, ca, strict); +} + +fn check_locales(defaults: Options, strict: Options) { + for tag in ["fr", "fr-FR", "fr-CA", "fr-BE", "fr-CA-u-nu-latn", "fr-u-ca-gregory"] { + let lang = ProofLang::from_tag(Some(tag)); + assert!(matches!(lang, ProofLang::Pack(_)), "{tag}"); + assert!(craft_proof::has_dictionary(Some(tag)), "{tag}"); + assert!(craft_proof::pack_dictionary_file(lang).unwrap().ends_with("fr/spelling.dic"), "{tag}"); + for word in ["école", "e\u{301}cole", "l'arbre", "l’arbre", "maison-école"] { + assert!(is_correct_in_with(word, lang, &strict), "{tag}: {word}"); + } + assert!(!is_correct_in_with("maison--école", lang, &strict)); + assert!(!is_correct_in_with("l'arbres", lang, &strict)); + assert!(is_correct_in_with("SNCF", lang, &defaults)); + assert!(!is_correct_in_with("SNCF", lang, &strict)); + assert!(is_correct_in_with("mp3x", lang, &defaults)); + assert!(!is_correct_in_with("mp3x", lang, &strict)); + assert!(suggest_in("ecole", 6, lang).iter().any(|s| s == "école")); + let text = "🙂 maison\u{338} école"; + let issues = check_text_with(text, &[(0..text.len(), lang)], &strict, false, false); + assert_eq!(issues.len(), 1, "{tag}: {issues:?}"); + assert_eq!(&text[issues[0].start..issues[0].end], "maison\u{338}"); + assert_eq!(issues[0].lang, lang); + assert_eq!(issues[0].kind, IssueKind::Spelling); + assert_eq!(issues[0].message, "Faute d’orthographe possible"); + } +} + +fn check_typography(strict: Options) { + let fr = ProofLang::from_tag(Some("fr-FR")); + let ca = ProofLang::from_tag(Some("fr-CA")); + for lang in [fr, ca] { + assert_eq!(craft_proof::hyphen::hyphen_points_in("maison", &craft_proof::hyphen::Limits::default(), lang), [2]); + } + for tag in ["fr-FR", "fr-BE", "fr-u-ca-gregory", "fr-FR-u-rg-cazzzz"] { + let issues = craft_proof::check_grammar_in("Bonjour!", ProofLang::from_tag(Some(tag))); + assert_eq!(issues.len(), 1, "{tag}: {issues:?}"); + assert_eq!(issues[0].suggestions, ["\u{202f}!"]); + } + assert!(craft_proof::check_grammar_in("Bonjour!", ca).is_empty()); + assert!(craft_proof::check_grammar_in("Bonjour!", ProofLang::from_tag(Some("fr-Latn-CA"))).is_empty()); + assert!(craft_proof::check_grammar_in("Bonjour!", ProofLang::from_tag(Some("nl-NL"))).is_empty()); + assert!(craft_proof::fr::grammar::check_in("Bonjour!", ProofLang::En).is_empty()); + let text = "Bonjour! Salut!"; + let issues = check_text_with(text, &[(0..8, fr), (8..text.len(), ca)], &strict, true, false); + let spacing: Vec<_> = issues.iter().filter(|i| i.kind == IssueKind::Grammar && i.message.contains("fine insécable")).collect(); + assert_eq!(spacing.len(), 1, "{issues:?}"); + assert_eq!(spacing[0].lang, fr); +} + +fn check_user_word(fr: ProofLang, ca: ProofLang, strict: Options) { + craft_proof::add_word("zxqrechargefr"); + for lang in [fr, ca] { + assert!(is_correct_in_with("l’zxqrechargefr", lang, &strict)); + } + craft_proof::remove_word("zxqrechargefr"); + assert!(!is_correct_in_with("l’zxqrechargefr", ca, &strict)); +} + +fn check_overrides(folder: PathBuf, fr: ProofLang, ca: ProofLang, strict: Options) { + // Regional overrides keep the generic pack fallback and the shared reload generation. + pair(&folder, "fr_FR", "SET UTF-8\n", "1\nhexagonefixture\n"); + pair(&folder, "fr_CA", "SET UTF-8\n", "1\nquebecfixture\n"); + let generation = craft_proof::dictionary_generation(); + craft_proof::set_dictionaries_folder(Some(folder.clone())); + assert!(craft_proof::dictionary_generation() > generation); + assert!(is_correct_in_with("hexagonefixture", fr, &strict)); + assert!(!is_correct_in_with("hexagonefixture", ca, &strict)); + assert!(is_correct_in_with("quebecfixture", ca, &strict)); + assert!(!is_correct_in_with("quebecfixture", fr, &strict)); + assert!(craft_proof::pack_dictionary_file(ca).unwrap().ends_with("fr_CA.dic")); + std::fs::remove_file(folder.join("fr_CA.dic")).unwrap(); + std::fs::remove_file(folder.join("fr_CA.aff")).unwrap(); + craft_proof::set_dictionaries_folder(Some(folder)); + assert!(is_correct_in_with("école", ca, &strict), "removed regional override falls back to fr pack"); + assert!(craft_proof::pack_dictionary_file(ca).unwrap().ends_with("fr/spelling.dic")); +}