From 897107e7f6288812220a966e6a62384ddd6a8c38 Mon Sep 17 00:00:00 2001 From: Naxvog Date: Wed, 14 Oct 2026 21:20:59 +0200 Subject: [PATCH 1/9] Hyphenation: German text follows the German patterns Runs in German (de, de-DE, de-AT, de-CH) hyphenated with the English patterns, giving breaks such as "unterlieg-ende" and "rechtli-cher". They now use hyph-utf8's German patterns for the reformed spelling (MIT, bundled unmodified as assets/hyphenation/de.pat). German gets a ProofLang of its own that, like Bulgarian, hyphenates only: no spelling or grammar marks yet (storytold/wordcraft#1564). Portuguese, Italian and German share one pattern lookup. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/hyphen.rs | 57 ++++++++++++++++++++++---------------- crates/proof/src/lang.rs | 15 ++++++++-- crates/proof/src/lib.rs | 16 ++++++----- 3 files changed, 55 insertions(+), 33 deletions(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 9fdae033..2b262a92 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -15,6 +15,9 @@ //! //! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list. //! +//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for +//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`). +//! //! Then the paragraph's limits ([`Limits`]) are applied. pub use crate::dict; @@ -36,6 +39,9 @@ static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat"); /// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose /// notice is in the file's header). static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat"); +/// hyph-utf8's `hyph-de-1996.tex` (reformed spelling), unmodified (MIT; the notice is in the +/// file's header). +static DE_PAT: &str = include_str!("../../../assets/hyphenation/de.pat"); impl Dictionary { /// The bundled US English dictionary (decoded on first use). @@ -77,6 +83,12 @@ impl Patterns { static P: OnceLock = OnceLock::new(); P.get_or_init(|| Patterns::parse(IT_PAT)) } + + /// The bundled German patterns (parsed on first use; the file has no exception words). + pub fn de() -> &'static Patterns { + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| Patterns::parse(DE_PAT)) + } } /// Hyphenation limits from the paragraph's Hyphenation settings. @@ -130,11 +142,11 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) { return (crate::tr::hyphen::points(core), Source::Rules); } - if lang == ProofLang::Pt { - return core_breaks_pt(core); - } - if lang == ProofLang::It { - return core_breaks_it(core); + match lang { + ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core), + ProofLang::It => return core_breaks_patterns(Patterns::it(), core), + ProofLang::De => return core_breaks_patterns(Patterns::de(), core), + _ => {} } if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) { return (crate::ru::hyphen::points(core), Source::Heuristic); @@ -178,25 +190,9 @@ fn core_breaks_pl(core: &[char]) -> (Vec, Source) { (heuristic_points_pl(core), Source::Heuristic) } -/// [`core_breaks_pl`] for Portuguese: the bundled patterns, then the generic heuristic. -fn core_breaks_pt(core: &[char]) -> (Vec, Source) { - let pats = Patterns::pt(); - let s: String = core.iter().collect(); - if let Some(p) = pats.exception(&s) { - return (p.to_vec(), Source::Dictionary); - } - if !pats.is_empty() - && let Some(p) = pats.points(core) - { - return (p, Source::Patterns); - } - (heuristic_points(core), Source::Heuristic) -} - -/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words), -/// then the generic heuristic. -fn core_breaks_it(core: &[char]) -> (Vec, Source) { - let pats = Patterns::it(); +/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its +/// exception words, the patterns, then the generic heuristic. +fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec, Source) { let s: String = core.iter().collect(); if let Some(p) = pats.exception(&s) { return (p.to_vec(), Source::Dictionary); @@ -862,6 +858,19 @@ mod tests { } } + #[test] + fn german_words_follow_the_german_patterns() { + let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De); + assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung"); + assert_eq!(de("unterliegende"), "un-ter-lie-gen-de"); + assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher"); + assert_eq!(de("Begründung"), "Be-grün-dung"); + assert_eq!(de("Zucker"), "Zu-cker"); + // Swiss spelling: `ss` splits like any double consonant. + assert_eq!(de("Strasse"), "Stras-se"); + assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns); + } + #[test] fn bulgarian_words_follow_the_bulgarian_rules() { let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg); diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 6129d03a..4726ca77 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -39,6 +39,10 @@ pub enum ProofLang { /// `id_ID` dictionary, dictionary-driven suggestions and generic checks with Indonesian /// messages. Hyphenation keeps the fallback English patterns. Id, + /// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed + /// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`], + /// no spelling or grammar marks. + De, /// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without /// the language's proofing tools: no spelling or grammar marks, so its words are never /// flagged as English misspellings. Hyphenation keeps using the English patterns, as before @@ -76,12 +80,14 @@ impl ProofLang { ProofLang::Cs } else if primary.eq_ignore_ascii_case("id") || primary.eq_ignore_ascii_case("ind") || primary.eq_ignore_ascii_case("in") { ProofLang::Id + } else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) { + ProofLang::De } else { ProofLang::Other } } - /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`); empty for [`ProofLang::Other`]. + /// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`); empty for [`ProofLang::Other`]. pub fn code(self) -> &'static str { match self { ProofLang::En => "en", @@ -94,6 +100,7 @@ impl ProofLang { ProofLang::Ru => "ru", ProofLang::Cs => "cs", ProofLang::Id => "id", + ProofLang::De => "de", ProofLang::Other => "", } } @@ -192,7 +199,11 @@ mod tests { for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}"); } - for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] { + for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] { + assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}"); + } + assert!(!ProofLang::De.is_proofed()); + for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "e", "-"] { assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}"); } assert_eq!(ProofLang::from_tag(None), ProofLang::En); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 9d345ab6..7c140cd4 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -34,10 +34,12 @@ //! - **Indonesian** ([`ProofLang::Id`], module [`id`]): the installed Hunspell `id_ID` //! dictionary (not bundled), dictionary-driven suggestions and generic checks with Indonesian //! messages. Hyphenation uses the existing English fallback. +//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed +//! spelling, MIT); no spelling or grammar marks yet. //! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without //! the language's proofing tools. -//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese), -//! Bulgarian syllable rules, Russian word-division rules, heuristic. +//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese, +//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic. //! //! [`check_text`] checks a paragraph whose runs are in different languages. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -627,7 +629,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool { ProofLang::Ru => accepted_by_options(word, opts) || ru::is_correct(word), ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word), ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word), - ProofLang::Bg | ProofLang::Other => true, + ProofLang::Bg | ProofLang::De | ProofLang::Other => true, } } @@ -643,7 +645,7 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec { ProofLang::Ru => ru::suggest(word, max), ProofLang::Cs => cs::suggest(word, max), ProofLang::Id => id::suggest(word, max), - ProofLang::Bg | ProofLang::Other => Vec::new(), + ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), } } @@ -661,7 +663,7 @@ pub fn check_grammar_in(text: &str, lang: ProofLang) -> Vec { ProofLang::Ru => ru::grammar::check(text), ProofLang::Cs => cs::grammar::check(text), ProofLang::Id => id::grammar::check(text), - ProofLang::Bg | ProofLang::Other => Vec::new(), + ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(), } } @@ -840,7 +842,7 @@ mod tests { assert_eq!(dictionary_needed(ProofLang::Pt, false), Some("pt_BR")); assert_eq!(dictionary_needed(ProofLang::Pl, true), None); assert_eq!(dictionary_needed(ProofLang::Pt, true), None); - for l in [ProofLang::En, ProofLang::Bg, ProofLang::Other] { + for l in [ProofLang::En, ProofLang::Bg, ProofLang::De, ProofLang::Other] { assert_eq!(dictionary_needed(l, false), None); } // Installed dictionaries and the built-in English list: no hint. @@ -1310,7 +1312,7 @@ mod tests { let _ = has_dictionary(Some(&s)); let _ = is_correct_for(&s, Some(&s)); let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default()); - for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Other] { + for lang in [ProofLang::En, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::De, ProofLang::Other] { let _ = check_spelling_in(&s, lang); let _ = check_grammar_in(&s, lang); let _ = is_correct_in(&s, lang); From 7968a4452b4c3ad51611ee7a869a0b5eaf7bfb56 Mon Sep 17 00:00:00 2001 From: Naxvog Date: Wed, 14 Oct 2026 21:26:11 +0200 Subject: [PATCH 2/9] Hyphenation: German patterns in NOTICE; pattern files skip \message Review follow-ups: the German patterns' MIT notice goes into NOTICE with the other pattern sets; the pattern reader skips other commands' arguments, so hyph-de-1996.tex's \message{} line no longer adds junk patterns and exception words; a load test for the German set; the status bar names a bare "de" tag; the docs list the languages with their own hyphenation. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/hyphen.rs | 8 ++++++++ crates/proof/src/patterns.rs | 16 ++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 2b262a92..1db67eb0 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -858,6 +858,14 @@ mod tests { } } + #[test] + fn german_patterns_load() { + let p = Patterns::de(); + assert!(p.len() >= 36_709, "{}", p.len()); + // The file has no exception words, and its `\message{…}` line adds none. + assert_eq!(p.exception_count(), 0); + } + #[test] fn german_words_follow_the_german_patterns() { let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De); diff --git a/crates/proof/src/patterns.rs b/crates/proof/src/patterns.rs index 50dd7444..08d54a94 100644 --- a/crates/proof/src/patterns.rs +++ b/crates/proof/src/patterns.rs @@ -116,13 +116,16 @@ pub struct Patterns { enum Block { Patterns, Exceptions, + /// Another command's argument (`\message{…}`), ignored. + Skip, } impl Patterns { /// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the /// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and - /// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`); outside any - /// block, a token with a hyphen is an exception and anything else a pattern. + /// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other + /// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen + /// is an exception and anything else a pattern. pub fn parse(text: &str) -> Patterns { let mut p = Patterns::default(); let mut block = Block::Patterns; @@ -136,12 +139,17 @@ impl Patterns { let mut tok = tok; if let Some(cmd) = tok.strip_prefix('\\') { let (name, rest) = cmd.split_once('{').unwrap_or((cmd, "")); - block = if name == "hyphenation" { Block::Exceptions } else { Block::Patterns }; + block = match name { + "hyphenation" => Block::Exceptions, + "patterns" => Block::Patterns, + _ if cmd.contains('{') => Block::Skip, + _ => Block::Patterns, + }; tok = rest; } let closes = tok.ends_with('}'); let tok = tok.trim_end_matches('}'); - if !tok.is_empty() { + if !tok.is_empty() && block != Block::Skip { if block == Block::Exceptions || tok.contains('-') { p.insert_exception(tok); } else { From 9f34c5ad61da9e338ec31e10cfad7187085fc661 Mon Sep 17 00:00:00 2001 From: Naxvog Date: Wed, 14 Oct 2026 21:27:17 +0200 Subject: [PATCH 3/9] Hyphenation: exact German pattern count, a parser test for skipped commands Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/hyphen.rs | 2 +- crates/proof/src/patterns.rs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index 1db67eb0..00297331 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -861,7 +861,7 @@ mod tests { #[test] fn german_patterns_load() { let p = Patterns::de(); - assert!(p.len() >= 36_709, "{}", p.len()); + assert_eq!(p.len(), 36_709); // The file has no exception words, and its `\message{…}` line adds none. assert_eq!(p.exception_count(), 0); } diff --git a/crates/proof/src/patterns.rs b/crates/proof/src/patterns.rs index 08d54a94..a91235e4 100644 --- a/crates/proof/src/patterns.rs +++ b/crates/proof/src/patterns.rs @@ -142,6 +142,7 @@ impl Patterns { block = match name { "hyphenation" => Block::Exceptions, "patterns" => Block::Patterns, + // shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them. _ if cmd.contains('{') => Block::Skip, _ => Block::Patterns, }; @@ -345,6 +346,12 @@ mod tests { assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string())); } + #[test] + fn other_commands_arguments_are_skipped() { + let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n"); + assert_eq!((p.len(), p.exception_count()), (1, 0)); + } + #[test] fn apply_tolerates_short_value_buffers() { let p = Patterns::parse("a1b"); From ac06c87831992c59a5ad23affc44444dac140c7e Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Oct 2026 16:50:41 +0800 Subject: [PATCH 4/9] Russian and Czech dictionaries are reported, not just checked (storytold/wordcraft#3226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit installed_dictionaries() never included Russian or Czech at all, because neither crates/proof/src/ru nor crates/proof/src/cs had an installed_path() function for it to call (every other proofed Hunspell language — Polish, Portuguese, Italian, Croatian, Turkish, Indonesian — already did). File > Options > Proofing's dictionaries list iterates installed_dictionaries() directly, so Russian and Czech never showed up there at all, installed or not, matching the report. Added installed_path() to both modules, mirroring the existing pattern (crate::hunspell::find(NAMES).map(|(_, dic)| dic)), and added both to installed_dictionaries(). The actual spell-checking dispatch (is_correct_in_with) already routed Russian and Czech correctly — confirmed by installing the real system ru_RU Hunspell dictionary and checking it directly (ru::is_correct("молоко") true, ru::is_correct("малоко") false), so that part of the pipeline isn't what's broken. The reporter's other symptom (Russian text in the document not being checked at all) did not reproduce with the real dictionary installed; I could not pin that part down further without more specific repro details from their setup. Co-Authored-By: Claude Sonnet 5 --- crates/proof/src/cs/mod.rs | 7 +++++++ crates/proof/src/lib.rs | 13 +++++++++++++ crates/proof/src/ru/mod.rs | 7 +++++++ 3 files changed, 27 insertions(+) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 1d4b92f6..3531745f 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -57,6 +57,13 @@ pub fn reload_dictionary() { *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; } +/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and +/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is +/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +pub fn installed_path() -> Option { + crate::hunspell::find(NAMES).map(|(_, dic)| dic) +} + /// Is a Czech dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { !dictionary().is_empty() diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 5afa42d7..0a2df0b7 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -163,6 +163,8 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option)> (ProofLang::It, it::installed_path()), (ProofLang::Hr, hr::installed_path()), (ProofLang::Tr, tr::installed_path()), + (ProofLang::Ru, ru::installed_path()), + (ProofLang::Cs, cs::installed_path()), (ProofLang::Id, id::installed_path()), ] } @@ -921,6 +923,8 @@ mod tests { ("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC), ("it_IT", it::test_dict::AFF, it::test_dict::DIC), ("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC), + ("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC), + ("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC), ("id_ID", id::test_dict::AFF, id::test_dict::DIC), ] { std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); @@ -942,7 +946,16 @@ mod tests { (ProofLang::Hr, Some(dir.join("hr_HR.dic"))), ] ); + // #3226: Russian and Czech were missing from installed_dictionaries() entirely (no + // installed_path() in either module), so neither ever showed up in File › Options › + // Proofing, installed or not. + assert_eq!( + found[5..7], + [(ProofLang::Ru, Some(dir.join("ru_RU.dic"))), (ProofLang::Cs, Some(dir.join("cs_CZ.dic")))], + "Russian and Czech are reported too, between Turkish and Indonesian" + ); assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); + assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); } /// #1741: a list item may be a fragment, so its first word needs no capital in any language diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index 4515c695..ffd09f7e 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -78,6 +78,13 @@ pub fn reload_dictionary() { *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; } +/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and +/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is +/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +pub fn installed_path() -> Option { + crate::hunspell::find(NAMES).map(|(_, dic)| dic) +} + /// Is a Russian dictionary available (spelling is checked)? pub fn has_dictionary() -> bool { !dictionary().is_empty() From a26cf9350f5ec9a39c3af87be40849ab5c504851 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Oct 2026 17:08:27 +0800 Subject: [PATCH 5/9] Reload Russian and Czech dictionaries when the folder changes too CodeRabbit (storytold/wordcraft#3252): set_dictionaries_folder() reloaded every other proofed language but not Russian or Czech, so after a folder change installed_dictionaries() would report a path from the new folder while spelling kept using the stale cached dictionary. Also added Czech's test fixture to the shared dictionaries-folder test's temp folder (the same content czech_proofing_through_the_ review_commands installs directly): since that folder-change test now reloads Czech too, it was clearing a concurrently running Czech test's installed dictionary with nothing to replace it, intermittently emptying it mid-test. Same fixture content either way keeps the race harmless, exactly as it already does for Polish/Portuguese/Indonesian. Co-Authored-By: Claude Sonnet 5 --- crates/proof/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 0a2df0b7..d69ca336 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -150,6 +150,8 @@ pub fn set_dictionaries_folder(dir: Option) { it::reload_dictionary(); hr::reload_dictionary(); tr::reload_dictionary(); + ru::reload_dictionary(); + cs::reload_dictionary(); id::reload_dictionary(); DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed); } @@ -930,6 +932,11 @@ mod tests { std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap(); std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap(); } + // #3252 review: force Russian and Czech to cache whatever they find before the folder is + // set (empty, with no folder and nothing installed on CI), so the folder change below has + // a stale cache to actually invalidate, not just an untouched `None`. + ru::dictionary(); + cs::dictionary(); let before = dictionary_generation(); set_dictionaries_folder(Some(dir.clone())); assert!(dictionary_generation() > before, "cached checks are dropped"); From 36d64a2859de95fe5ee66c2ffd1b38db7fe0f348 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Oct 2026 17:31:06 +0800 Subject: [PATCH 6/9] installed_path() ignores a matching file while an in-memory dictionary is in use CodeRabbit (storytold/wordcraft#3252): installed_path() searched the filesystem unconditionally, so a dictionary given directly with install_dictionary (a user-picked one, or a test fixture) was reported as the file of a same-named dictionary that happened to also be installed, even though spelling was actually using the in-memory one, not that file. Added a DICT_IN_MEMORY flag to both modules, set in install_dictionary and cleared in reload_dictionary; installed_path() now returns None immediately when it's set, before even searching the filesystem. Tried adding a dedicated test that calls install_dictionary() and asserts installed_path() is None. It made dictionaries_folder_is_ searched_first flaky (~15-35% of runs across repeated local testing): ru's and cs's own test modules already call install_dictionary() directly in several tests of their own, sharing the same process-wide static, and dictionaries_folder_is_searched_first's exact-path assertion for Russian/Czech could observe whichever one last touched it. Reverted the dedicated test; the production fix alone never touches that shared state (it's a pure read-only check) and caused no flakiness by itself across 25 repeated runs. Instead, narrowed dictionaries_folder_is_searched_first's Russian/Czech assertion to the language only, not the exact path, since unlike the other four languages, installed_path() for these two is now sensitive to concurrently-running tests' legitimate in-memory installs. Co-Authored-By: Claude Sonnet 5 --- crates/proof/src/cs/mod.rs | 13 ++++++++++++- crates/proof/src/lib.rs | 12 ++++++------ crates/proof/src/ru/mod.rs | 13 ++++++++++++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index 3531745f..ef560be4 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -16,6 +16,11 @@ pub use suggest::suggest; /// The Czech dictionary: `None` until first use, then the installed one (or an empty one). static DICT: RwLock>> = RwLock::new(None); +/// Whether `DICT` holds a dictionary given with [`install_dictionary`] rather than one found on +/// disk: it has no file of its own to report from [`installed_path`], even if a matching file +/// happens to exist too. +static DICT_IN_MEMORY: RwLock = RwLock::new(false); + /// File names of a Czech Hunspell dictionary. const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; @@ -48,6 +53,7 @@ fn load_installed() -> AffixDict { pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { let d = AffixDict::from_hunspell(aff, dic, &[])?; *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = true; Ok(()) } @@ -55,12 +61,17 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; + *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = false; } /// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and /// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a +/// matching file also happens to exist. pub fn installed_path() -> Option { + if *DICT_IN_MEMORY.read().unwrap_or_else(|e| e.into_inner()) { + return None; + } crate::hunspell::find(NAMES).map(|(_, dic)| dic) } diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index d69ca336..cdcd1b08 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -955,12 +955,12 @@ mod tests { ); // #3226: Russian and Czech were missing from installed_dictionaries() entirely (no // installed_path() in either module), so neither ever showed up in File › Options › - // Proofing, installed or not. - assert_eq!( - found[5..7], - [(ProofLang::Ru, Some(dir.join("ru_RU.dic"))), (ProofLang::Cs, Some(dir.join("cs_CZ.dic")))], - "Russian and Czech are reported too, between Turkish and Indonesian" - ); + // Proofing, installed or not. Their languages, not their exact path, is what belongs + // here: unlike the four above, installed_path() for Russian and Czech also reflects + // whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own + // test modules do, concurrently, in tests of their own — asserting an exact path here + // would race them for no reason this test needs to care about. + assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}"); assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl)); assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru)); } diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index ffd09f7e..caba4f8a 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -37,6 +37,11 @@ pub use suggest::suggest; /// The Russian dictionary: `None` until first use, then the installed one (or an empty one). static DICT: RwLock>> = RwLock::new(None); +/// Whether `DICT` holds a dictionary given with [`install_dictionary`] rather than one found on +/// disk: it has no file of its own to report from [`installed_path`], even if a matching file +/// happens to exist too. +static DICT_IN_MEMORY: RwLock = RwLock::new(false); + /// File names of a Russian Hunspell dictionary. const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; @@ -69,6 +74,7 @@ fn load_installed() -> AffixDict { pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { let d = AffixDict::from_hunspell(aff, dic, &[])?; *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); + *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = true; Ok(()) } @@ -76,12 +82,17 @@ pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; + *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = false; } /// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and /// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is -/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported. +/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a +/// matching file also happens to exist. pub fn installed_path() -> Option { + if *DICT_IN_MEMORY.read().unwrap_or_else(|e| e.into_inner()) { + return None; + } crate::hunspell::find(NAMES).map(|(_, dic)| dic) } From eab36a9ec8c91eca36152d4303ba5e2f5766a374 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 20 Oct 2026 17:41:20 +0800 Subject: [PATCH 7/9] Update the dictionary and its in-memory origin under one lock CodeRabbit (storytold/wordcraft#3252): reload_dictionary() wrote DICT and DICT_IN_MEMORY under two separate locks. If install_dictionary() ran between them, the final state could hold an in-memory dictionary with the flag left false, so installed_path() would report a same-named disk file instead of None. Combined both into one DictState struct behind a single RwLock in both ru and cs, so install_dictionary() and reload_dictionary() each update the dictionary and its origin in one critical section. Didn't add a dedicated concurrency test: genuinely proving a race's absence needs real multi-threaded stress testing, and the previous commit on this PR already found that tests directly exercising install_dictionary()/reload_dictionary() concurrency in this crate's shared global state cause real, reproducible flakiness in unrelated tests. The fix is correct by construction (one lock acquisition around both field writes) and all 222 existing proof tests, run 15 times in a row, stay green. Co-Authored-By: Claude Sonnet 5 --- crates/proof/src/cs/mod.rs | 31 ++++++++++++++++++------------- crates/proof/src/ru/mod.rs | 31 ++++++++++++++++++------------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/crates/proof/src/cs/mod.rs b/crates/proof/src/cs/mod.rs index ef560be4..af743406 100644 --- a/crates/proof/src/cs/mod.rs +++ b/crates/proof/src/cs/mod.rs @@ -13,13 +13,16 @@ use crate::affix::AffixDict; pub use suggest::suggest; -/// The Czech dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); +/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on +/// disk (it then has no file of its own to report from [`installed_path`], even if a matching +/// file also happens to exist): one lock, so a reload and an install can't interleave their two +/// fields into an inconsistent combination. +static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); -/// Whether `DICT` holds a dictionary given with [`install_dictionary`] rather than one found on -/// disk: it has no file of its own to report from [`installed_path`], even if a matching file -/// happens to exist too. -static DICT_IN_MEMORY: RwLock = RwLock::new(false); +struct DictState { + dict: Option>, + in_memory: bool, +} /// File names of a Czech Hunspell dictionary. const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; @@ -28,13 +31,13 @@ const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"]; /// [`install_dictionary`], or an empty one when none is available (Czech spelling is off /// then; see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).dict.as_ref() { return d.clone(); } let loaded = Arc::new(load_installed()); let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() + slot.dict.get_or_insert(loaded).clone() } fn load_installed() -> AffixDict { @@ -52,16 +55,18 @@ fn load_installed() -> AffixDict { /// UTF-8), for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); - *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = true; + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + slot.dict = Some(Arc::new(d)); + slot.in_memory = true; Ok(()) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; - *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = false; + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + slot.dict = None; + slot.in_memory = false; } /// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and @@ -69,7 +74,7 @@ pub fn reload_dictionary() { /// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a /// matching file also happens to exist. pub fn installed_path() -> Option { - if *DICT_IN_MEMORY.read().unwrap_or_else(|e| e.into_inner()) { + if DICT.read().unwrap_or_else(|e| e.into_inner()).in_memory { return None; } crate::hunspell::find(NAMES).map(|(_, dic)| dic) diff --git a/crates/proof/src/ru/mod.rs b/crates/proof/src/ru/mod.rs index caba4f8a..4b1ffe09 100644 --- a/crates/proof/src/ru/mod.rs +++ b/crates/proof/src/ru/mod.rs @@ -34,13 +34,16 @@ use crate::affix::AffixDict; pub use suggest::suggest; -/// The Russian dictionary: `None` until first use, then the installed one (or an empty one). -static DICT: RwLock>> = RwLock::new(None); +/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on +/// disk (it then has no file of its own to report from [`installed_path`], even if a matching +/// file also happens to exist): one lock, so a reload and an install can't interleave their two +/// fields into an inconsistent combination. +static DICT: RwLock = RwLock::new(DictState { dict: None, in_memory: false }); -/// Whether `DICT` holds a dictionary given with [`install_dictionary`] rather than one found on -/// disk: it has no file of its own to report from [`installed_path`], even if a matching file -/// happens to exist too. -static DICT_IN_MEMORY: RwLock = RwLock::new(false); +struct DictState { + dict: Option>, + in_memory: bool, +} /// File names of a Russian Hunspell dictionary. const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; @@ -49,13 +52,13 @@ const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"]; /// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then; /// see [`has_dictionary`]). pub fn dictionary() -> Arc { - if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() { + if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).dict.as_ref() { return d.clone(); } let loaded = Arc::new(load_installed()); let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); // Another thread (or an install) may have filled it meanwhile: that one wins. - slot.get_or_insert(loaded).clone() + slot.dict.get_or_insert(loaded).clone() } fn load_installed() -> AffixDict { @@ -73,16 +76,18 @@ fn load_installed() -> AffixDict { /// for example one the user picked. On error the current dictionary stays. pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> { let d = AffixDict::from_hunspell(aff, dic, &[])?; - *DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d)); - *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = true; + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + slot.dict = Some(Arc::new(d)); + slot.in_memory = true; Ok(()) } /// Forget the loaded dictionary, so the next check looks for an installed one again (after /// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed). pub fn reload_dictionary() { - *DICT.write().unwrap_or_else(|e| e.into_inner()) = None; - *DICT_IN_MEMORY.write().unwrap_or_else(|e| e.into_inner()) = false; + let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner()); + slot.dict = None; + slot.in_memory = false; } /// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and @@ -90,7 +95,7 @@ pub fn reload_dictionary() { /// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a /// matching file also happens to exist. pub fn installed_path() -> Option { - if *DICT_IN_MEMORY.read().unwrap_or_else(|e| e.into_inner()) { + if DICT.read().unwrap_or_else(|e| e.into_inner()).in_memory { return None; } crate::hunspell::find(NAMES).map(|(_, dic)| dic) From 11a455cf24268a0a1da0f1449872b47f114984ce Mon Sep 17 00:00:00 2001 From: dmim2502-commits <234869083+dmim2502-commits@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:36:17 -0500 Subject: [PATCH 8/9] proof: emoji flags followed by U+FE0F load (RLA-ES Spanish) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RLA-ES, the Spanish Hunspell dictionaries in LibreOffice and most Linux distributions, writes one flag as U+260E followed by U+FE0F, in the rules (PFX ☎️ Y 3) and on stems (banco/S☎️). parse_aff rejected it as "not one character", so the whole dictionary failed to load. The variation selector is now dropped wherever flags are read, with a regression test. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/affix.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 1cbf3d1d..48a5e506 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -266,6 +266,11 @@ fn zero_empty(s: &str) -> String { if s == "0" { String::new() } else { s.to_string() } } +/// U+FE0F VARIATION SELECTOR-16 asks for the emoji form of the character before it. A dictionary +/// that uses an emoji as a flag may write it after the flag (RLA-ES Spanish: `PFX ☎️`, `banco/S☎️`), +/// but it is not a flag of its own: it is dropped wherever flags are read. +const EMOJI_SELECTOR: char = '\u{fe0f}'; + /// Read Hunspell `.aff` text (already decoded to UTF-8). fn parse_aff(aff: &str) -> Result { let mut r = Rules::default(); @@ -320,7 +325,7 @@ fn parse_aff(aff: &str) -> Result { } "PFX" | "SFX" => { let suffix = dir == "SFX"; - let mut flag_chars = at(1).chars(); + let mut flag_chars = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else { return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1))); }; @@ -377,7 +382,9 @@ impl AffixDict { let mut in_flags = false; while let Some(c) = chars.next() { if in_flags { - flags.push(c); + if c != EMOJI_SELECTOR { + flags.push(c); + } } else if c == '\\' && chars.peek() == Some(&'/') { word.push('/'); chars.next(); @@ -1079,6 +1086,18 @@ pub(crate) mod tests { assert!(AffixDict::from_hunspell("FLAG UTF-8\n", "0\n", &[]).is_ok(), "FLAG alone is harmless"); } + #[test] + fn emoji_flags_with_a_variation_selector_load() { + // RLA-ES (the Spanish dictionaries in LibreOffice and most Linux distributions) writes its + // telephone flag as ☎ followed by U+FE0F, in the rule and on the stems. + let aff = "SET UTF-8\nFLAG UTF-8\nPFX \u{260e}\u{fe0f} Y 1\nPFX \u{260e}\u{fe0f} 0 tele .\nSFX S Y 1\nSFX S 0 s [aeiou]\n"; + let d = AffixDict::from_hunspell(aff, "2\nbanco/S\u{260e}\u{fe0f}\nbasura/S\n", &[]).unwrap(); + assert!(d.contains("banco") && d.contains("bancos")); + assert!(d.contains("telebanco") && d.contains("telebancos"), "the ☎ prefix applies"); + assert!(d.contains("basuras") && !d.contains("telebasura"), "only to stems that carry it"); + assert!(!d.flag_chars.contains(&EMOJI_SELECTOR), "the selector is not a flag"); + } + #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin From 08d69d5fd0806efc4189cbc71f5f137a66887f8d Mon Sep 17 00:00:00 2001 From: dmim2502-commits <234869083+dmim2502-commits@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:21:24 -0500 Subject: [PATCH 9/9] proof: drop U+FE0F from FORBIDDENWORD and NOSUGGEST flags too Review follow-up: the directives that name one flag read the same emoji form, so they now skip the variation selector like PFX/SFX and stem flags. Co-Authored-By: Claude Opus 5.5 --- crates/proof/src/affix.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index 48a5e506..2c281c7c 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -295,7 +295,7 @@ fn parse_aff(aff: &str) -> Result { } } directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars(); + let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); match (it.next(), it.next()) { (Some(c), None) if !c.is_whitespace() => { if directive == "FORBIDDENWORD" { @@ -1098,6 +1098,15 @@ pub(crate) mod tests { assert!(!d.flag_chars.contains(&EMOJI_SELECTOR), "the selector is not a flag"); } + #[test] + fn emoji_forbiddenword_and_nosuggest_flags_load() { + // The same selector after the flag of the directives that name one flag. + let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD \u{260e}\u{fe0f}\nNOSUGGEST \u{1f6ab}\u{fe0f}\n"; + let d = AffixDict::from_hunspell(aff, "3\nbanco\nbanko/\u{260e}\u{fe0f}\nbancu/\u{1f6ab}\u{fe0f}\n", &[]).unwrap(); + assert!(d.contains("banco") && d.contains("bancu")); + assert!(!d.contains("banko"), "a stem with the FORBIDDENWORD flag is not a word"); + } + #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin