Merge origin/main into spanish-proofing

Resolve conflicts with German hyphenation (ProofLang::De), Russian and Czech
in installed_dictionaries(), and storytold/wordcraft#2164's U+FE0F flag fix: affix.rs takes
main's fix and drops this branch's duplicate. The Danish, Dutch, Romanian and
Slovenian catalogs gain the Spanish (Latin America) and (Uruguay) names; the
language menu tests get room for the four Spanish entries.
This commit is contained in:
Federico Olivares committed 2026-10-22 02:46:30 -03:00
commit d80c136e55
7 files changed
+191 -81

No files matched your search

+29 -30
View File
@@ -11,8 +11,7 @@
//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text
//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB).
//!
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`;
//! a variation selector after one, as in the Spanish dictionaries' `☎️`, belongs to it),
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),
//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross
//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,
@@ -262,23 +261,16 @@ struct Rules {
/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.
type RawRule = (char, bool, String, String, String, String);
/// Is `c` a Unicode variation selector (U+FE00–U+FE0F)? It only picks how the character before
/// it is drawn, so in a flag field it belongs to that flag: the RLA-ES Spanish dictionaries write
/// a flag as `☎️` (`☎` and U+FE0F), which Hunspell reads as `☎` (#3360).
fn is_variation_selector(c: char) -> bool {
('\u{FE00}'..='\u{FE0F}').contains(&c)
}
/// The flag characters of a flag field, without [variation selectors](is_variation_selector).
fn flag_chars(field: &str) -> impl Iterator<Item = char> + '_ {
field.chars().filter(|&c| !is_variation_selector(c))
}
/// `0` means empty in Hunspell's strip and append fields.
fn zero_empty(s: &str) -> String {
if s == "0" { String::new() } else { s.to_string() }
}
/// U+FE0F VARIATION SELECTOR-16 asks for the emoji form of the character before it. A dictionary
/// that uses an emoji as a flag may write it after the flag (RLA-ES Spanish: `PFX ☎️`, `banco/S☎️`),
/// but it is not a flag of its own: it is dropped wherever flags are read.
const EMOJI_SELECTOR: char = '\u{fe0f}';
/// Read Hunspell `.aff` text (already decoded to UTF-8).
fn parse_aff(aff: &str) -> Result<Rules, String> {
let mut r = Rules::default();
@@ -303,7 +295,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
}
}
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
let mut it = flag_chars(at(1));
let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
match (it.next(), it.next()) {
(Some(c), None) if !c.is_whitespace() => {
if directive == "FORBIDDENWORD" {
@@ -333,8 +325,8 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
}
"PFX" | "SFX" => {
let suffix = dir == "SFX";
let mut flag_field = flag_chars(at(1));
let (Some(flag), None) = (flag_field.next(), flag_field.next()) else {
let mut flag_chars = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else {
return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1)));
};
// Only a header ignores trailing fields; a rule condition beginning with # is literal.
@@ -354,7 +346,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
let cond = if f.len() >= 5 { at(4) } else { "." };
parse_cond(cond).map_err(|e| format!("line {}: {e}", n + 1))?;
let c = cross.get(&(suffix, flag)).copied().unwrap_or(false);
let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), flag_chars(cont).collect(), cond.to_string());
let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), cont.to_string(), cond.to_string());
if suffix {
r.suffixes.push(rule);
} else {
@@ -390,7 +382,7 @@ impl AffixDict {
let mut in_flags = false;
while let Some(c) = chars.next() {
if in_flags {
if !is_variation_selector(c) {
if c != EMOJI_SELECTOR {
flags.push(c);
}
} else if c == '\\' && chars.peek() == Some(&'/') {
@@ -1094,18 +1086,25 @@ pub(crate) mod tests {
assert!(AffixDict::from_hunspell("FLAG UTF-8\n", "0\n", &[]).is_ok(), "FLAG alone is harmless");
}
/// #3360: the RLA-ES Spanish dictionaries write one flag as `☎️` (`☎` and the variation
/// selector U+FE0F) in the `.aff` and in the `.dic`; it is the flag `☎`, as Hunspell reads it,
/// not a two-character flag that refuses the whole dictionary.
#[test]
fn a_variation_selector_belongs_to_its_flag() {
let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ✖\u{FE0F}\nSFX \u{260E}\u{FE0F} Y 1\nSFX \u{260E}\u{FE0F} 0 s/A\u{FE0F} .\nSFX A Y 1\nSFX A 0 x .\n";
let d = AffixDict::from_hunspell(aff, "3\ncasa/\u{260E}\u{FE0F}\nmesa/\u{260E}\nzq/✖\u{FE0F}\n", &[]).unwrap();
assert!(d.contains("casa") && d.contains("casas") && d.contains("casasx"));
assert!(d.contains("mesas"), "the bare flag is the same flag");
assert!(!d.contains("zq"), "FORBIDDENWORD with a selector still forbids");
assert_eq!(d.flag_chars.len(), 3, "{:?}", d.flag_chars);
assert!(AffixDict::from_hunspell("SFX ab Y 1\nSFX ab 0 s .\n", "0\n", &[]).is_err(), "two flag characters are still refused");
fn emoji_flags_with_a_variation_selector_load() {
// RLA-ES (the Spanish dictionaries in LibreOffice and most Linux distributions) writes its
// telephone flag as ☎ followed by U+FE0F, in the rule and on the stems.
let aff = "SET UTF-8\nFLAG UTF-8\nPFX \u{260e}\u{fe0f} Y 1\nPFX \u{260e}\u{fe0f} 0 tele .\nSFX S Y 1\nSFX S 0 s [aeiou]\n";
let d = AffixDict::from_hunspell(aff, "2\nbanco/S\u{260e}\u{fe0f}\nbasura/S\n", &[]).unwrap();
assert!(d.contains("banco") && d.contains("bancos"));
assert!(d.contains("telebanco") && d.contains("telebancos"), "the ☎ prefix applies");
assert!(d.contains("basuras") && !d.contains("telebasura"), "only to stems that carry it");
assert!(!d.flag_chars.contains(&EMOJI_SELECTOR), "the selector is not a flag");
}
#[test]
fn emoji_forbiddenword_and_nosuggest_flags_load() {
// The same selector after the flag of the directives that name one flag.
let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD \u{260e}\u{fe0f}\nNOSUGGEST \u{1f6ab}\u{fe0f}\n";
let d = AffixDict::from_hunspell(aff, "3\nbanco\nbanko/\u{260e}\u{fe0f}\nbancu/\u{1f6ab}\u{fe0f}\n", &[]).unwrap();
assert!(d.contains("banco") && d.contains("bancu"));
assert!(!d.contains("banko"), "a stem with the FORBIDDENWORD flag is not a word");
}
#[test]
+29 -6
View File
@@ -13,8 +13,16 @@ use crate::affix::AffixDict;
pub use suggest::suggest;
/// The Czech dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
/// fields into an inconsistent combination.
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
struct DictState {
dict: Option<Arc<AffixDict>>,
in_memory: bool,
}
/// File names of a Czech Hunspell dictionary.
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
@@ -23,13 +31,13 @@ const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
/// then; see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() {
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).dict.as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
slot.dict.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
@@ -47,14 +55,29 @@ fn load_installed() -> AffixDict {
/// UTF-8), for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d));
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(|e| e.into_inner()) = None;
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
slot.dict = None;
slot.in_memory = false;
}
/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
/// matching file also happens to exist.
pub fn installed_path() -> Option<std::path::PathBuf> {
if DICT.read().unwrap_or_else(|e| e.into_inner()).in_memory {
return None;
}
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
}
/// Is a Czech dictionary available (spelling is checked)?
+41 -24
View File
@@ -15,6 +15,9 @@
//!
//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list.
//!
//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for
//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`).
//!
//! Then the paragraph's limits ([`Limits`]) are applied.
pub use crate::dict;
@@ -36,6 +39,9 @@ static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat");
/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose
/// notice is in the file's header).
static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat");
/// hyph-utf8's `hyph-de-1996.tex` (reformed spelling), unmodified (MIT; the notice is in the
/// file's header).
static DE_PAT: &str = include_str!("../../../assets/hyphenation/de.pat");
impl Dictionary {
/// The bundled US English dictionary (decoded on first use).
@@ -77,6 +83,12 @@ impl Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(IT_PAT))
}
/// The bundled German patterns (parsed on first use; the file has no exception words).
pub fn de() -> &'static Patterns {
static P: OnceLock<Patterns> = OnceLock::new();
P.get_or_init(|| Patterns::parse(DE_PAT))
}
}
/// Hyphenation limits from the paragraph's Hyphenation settings.
@@ -130,11 +142,11 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
return (crate::tr::hyphen::points(core), Source::Rules);
}
if lang == ProofLang::Pt {
return core_breaks_pt(core);
}
if lang == ProofLang::It {
return core_breaks_it(core);
match lang {
ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core),
ProofLang::It => return core_breaks_patterns(Patterns::it(), core),
ProofLang::De => return core_breaks_patterns(Patterns::de(), core),
_ => {}
}
if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) {
return (crate::ru::hyphen::points(core), Source::Heuristic);
@@ -178,25 +190,9 @@ fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
(heuristic_points_pl(core), Source::Heuristic)
}
/// [`core_breaks_pl`] for Portuguese: the bundled patterns, then the generic heuristic.
fn core_breaks_pt(core: &[char]) -> (Vec<usize>, Source) {
let pats = Patterns::pt();
let s: String = core.iter().collect();
if let Some(p) = pats.exception(&s) {
return (p.to_vec(), Source::Dictionary);
}
if !pats.is_empty()
&& let Some(p) = pats.points(core)
{
return (p, Source::Patterns);
}
(heuristic_points(core), Source::Heuristic)
}
/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words),
/// then the generic heuristic.
fn core_breaks_it(core: &[char]) -> (Vec<usize>, Source) {
let pats = Patterns::it();
/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its
/// exception words, the patterns, then the generic heuristic.
fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec<usize>, Source) {
let s: String = core.iter().collect();
if let Some(p) = pats.exception(&s) {
return (p.to_vec(), Source::Dictionary);
@@ -862,6 +858,27 @@ mod tests {
}
}
#[test]
fn german_patterns_load() {
let p = Patterns::de();
assert_eq!(p.len(), 36_709);
// The file has no exception words, and its `\message{…}` line adds none.
assert_eq!(p.exception_count(), 0);
}
#[test]
fn german_words_follow_the_german_patterns() {
let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De);
assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung");
assert_eq!(de("unterliegende"), "un-ter-lie-gen-de");
assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher");
assert_eq!(de("Begründung"), "Be-grün-dung");
assert_eq!(de("Zucker"), "Zu-cker");
// Swiss spelling: `ss` splits like any double consonant.
assert_eq!(de("Strasse"), "Stras-se");
assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns);
}
#[test]
fn bulgarian_words_follow_the_bulgarian_rules() {
let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg);
+13 -2
View File
@@ -65,6 +65,10 @@ pub enum ProofLang {
/// dictionary (else the neighbouring Rioplatense `es_AR`, a Latin American or a generic one),
/// and the `vos` forms of verbs (`tenés`, `vení`, `decime`) accepted ([`crate::es::voseo`]).
EsUy,
/// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
/// no spelling or grammar marks.
De,
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
/// the language's proofing tools: no spelling or grammar marks, so its words are never
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
@@ -121,6 +125,8 @@ impl ProofLang {
ProofLang::Id
} else if primary.eq_ignore_ascii_case("es") || primary.eq_ignore_ascii_case("spa") {
ProofLang::spanish(tag)
} else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) {
ProofLang::De
} else {
ProofLang::Other
}
@@ -160,7 +166,7 @@ impl ProofLang {
matches!(self, ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy)
}
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `es`); empty for [`ProofLang::Other`].
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`, `es`); empty for [`ProofLang::Other`].
pub fn code(self) -> &'static str {
match self {
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
@@ -174,6 +180,7 @@ impl ProofLang {
ProofLang::Cs => "cs",
ProofLang::Id => "id",
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es",
ProofLang::De => "de",
ProofLang::Other => "",
}
}
@@ -327,7 +334,11 @@ mod tests {
for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}");
}
for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] {
for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}");
}
assert!(!ProofLang::De.is_proofed());
for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] {
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}");
}
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
+31 -9
View File
@@ -39,10 +39,12 @@
//! `es_UY`, with fallbacks; not bundled), dictionary-driven suggestions, Uruguayan `vos` forms,
//! and generic checks plus the opening `¿`/`¡` with Spanish messages. Hyphenation uses the
//! existing English fallback.
//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed
//! spelling, MIT); no spelling or grammar marks yet.
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
//! the language's proofing tools.
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese),
//! Bulgarian syllable rules, Russian word-division rules, heuristic.
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese,
//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic.
//!
//! [`check_text`] checks a paragraph whose runs are in different languages.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
@@ -157,6 +159,8 @@ pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
it::reload_dictionary();
hr::reload_dictionary();
tr::reload_dictionary();
ru::reload_dictionary();
cs::reload_dictionary();
id::reload_dictionary();
es::reload_dictionary();
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
@@ -171,6 +175,8 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)>
(ProofLang::It, it::installed_path()),
(ProofLang::Hr, hr::installed_path()),
(ProofLang::Tr, tr::installed_path()),
(ProofLang::Ru, ru::installed_path()),
(ProofLang::Cs, cs::installed_path()),
(ProofLang::Id, id::installed_path()),
(ProofLang::Es, es::installed_path(ProofLang::Es)),
(ProofLang::EsMx, es::installed_path(ProofLang::EsMx)),
@@ -738,7 +744,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word),
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word),
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => accepted_by_options(word, opts) || es::is_correct(word, lang),
ProofLang::Bg | ProofLang::Other => true,
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
}
}
@@ -755,7 +761,7 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
ProofLang::Cs => cs::suggest(word, max),
ProofLang::Id => id::suggest(word, max),
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang),
ProofLang::Bg | ProofLang::Other => Vec::new(),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
}
}
@@ -784,7 +790,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
ProofLang::Id => id::grammar::check(text),
// Spanish has no sentence-capital rule either.
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang),
ProofLang::Bg | ProofLang::Other => Vec::new(),
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
}
}
@@ -978,7 +984,7 @@ mod tests {
assert_eq!(dictionary_needed(l, false), Some(name));
assert_eq!(dictionary_needed(l, true), None);
}
for l in [ProofLang::En, ProofLang::Bg, ProofLang::Other] {
for l in [ProofLang::En, ProofLang::Bg, ProofLang::De, ProofLang::Other] {
assert_eq!(dictionary_needed(l, false), None);
}
// Installed dictionaries and the built-in English list: no hint.
@@ -1005,6 +1011,8 @@ mod tests {
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC),
("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC),
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
("es_ES", es::test_dict::AFF, es::test_dict::DIC),
("es_MX", es::test_dict::AFF, es::test_dict::DIC_AMERICAS),
@@ -1014,14 +1022,19 @@ mod tests {
std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap();
std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap();
}
// #3252 review: force Russian and Czech to cache whatever they find before the folder is
// set (empty, with no folder and nothing installed on CI), so the folder change below has
// a stale cache to actually invalidate, not just an untouched `None`.
ru::dictionary();
cs::dictionary();
let before = dictionary_generation();
set_dictionaries_folder(Some(dir.clone()));
assert!(dictionary_generation() > before, "cached checks are dropped");
// Turkish has no test dictionary to put in the folder; Indonesian and the Spanishes are listed after it.
// Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it.
let found = installed_dictionaries();
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
assert_eq!(
found[5..],
found[7..],
[
(ProofLang::Id, Some(dir.join("id_ID.dic"))),
(ProofLang::Es, Some(dir.join("es_ES.dic"))),
@@ -1039,8 +1052,17 @@ mod tests {
(ProofLang::Hr, Some(dir.join("hr_HR.dic"))),
]
);
// #3226: Russian and Czech were missing from installed_dictionaries() entirely (no
// installed_path() in either module), so neither ever showed up in File › Options ›
// Proofing, installed or not. Their languages, not their exact path, is what belongs
// here: unlike the four above, installed_path() for Russian and Czech also reflects
// whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own
// test modules do, concurrently, in tests of their own — asserting an exact path here
// would race them for no reason this test needs to care about.
assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::<Vec<_>>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}");
assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl));
assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy));
assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru));
}
/// #1741: a list item may be a fragment, so its first word needs no capital in any language
@@ -1704,7 +1726,7 @@ mod tests {
let _ = has_dictionary(Some(&s));
let _ = is_correct_for(&s, Some(&s));
let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default());
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other] {
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::De, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other] {
let _ = check_spelling_in(&s, lang);
let _ = check_grammar_in(&s, lang);
let _ = is_correct_in(&s, lang);
+19 -4
View File
@@ -116,13 +116,16 @@ pub struct Patterns {
enum Block {
Patterns,
Exceptions,
/// Another command's argument (`\message{…}`), ignored.
Skip,
}
impl Patterns {
/// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the
/// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`); outside any
/// block, a token with a hyphen is an exception and anything else a pattern.
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other
/// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen
/// is an exception and anything else a pattern.
pub fn parse(text: &str) -> Patterns {
let mut p = Patterns::default();
let mut block = Block::Patterns;
@@ -136,12 +139,18 @@ impl Patterns {
let mut tok = tok;
if let Some(cmd) = tok.strip_prefix('\\') {
let (name, rest) = cmd.split_once('{').unwrap_or((cmd, ""));
block = if name == "hyphenation" { Block::Exceptions } else { Block::Patterns };
block = match name {
"hyphenation" => Block::Exceptions,
"patterns" => Block::Patterns,
// shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them.
_ if cmd.contains('{') => Block::Skip,
_ => Block::Patterns,
};
tok = rest;
}
let closes = tok.ends_with('}');
let tok = tok.trim_end_matches('}');
if !tok.is_empty() {
if !tok.is_empty() && block != Block::Skip {
if block == Block::Exceptions || tok.contains('-') {
p.insert_exception(tok);
} else {
@@ -337,6 +346,12 @@ mod tests {
assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string()));
}
#[test]
fn other_commands_arguments_are_skipped() {
let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n");
assert_eq!((p.len(), p.exception_count()), (1, 0));
}
#[test]
fn apply_tolerates_short_value_buffers() {
let p = Patterns::parse("a1b");
+29 -6
View File
@@ -34,8 +34,16 @@ use crate::affix::AffixDict;
pub use suggest::suggest;
/// The Russian dictionary: `None` until first use, then the installed one (or an empty one).
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
/// fields into an inconsistent combination.
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
struct DictState {
dict: Option<Arc<AffixDict>>,
in_memory: bool,
}
/// File names of a Russian Hunspell dictionary.
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
@@ -44,13 +52,13 @@ const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then;
/// see [`has_dictionary`]).
pub fn dictionary() -> Arc<AffixDict> {
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() {
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).dict.as_ref() {
return d.clone();
}
let loaded = Arc::new(load_installed());
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
// Another thread (or an install) may have filled it meanwhile: that one wins.
slot.get_or_insert(loaded).clone()
slot.dict.get_or_insert(loaded).clone()
}
fn load_installed() -> AffixDict {
@@ -68,14 +76,29 @@ fn load_installed() -> AffixDict {
/// for example one the user picked. On error the current dictionary stays.
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
let d = AffixDict::from_hunspell(aff, dic, &[])?;
*DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d));
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
slot.dict = Some(Arc::new(d));
slot.in_memory = true;
Ok(())
}
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
pub fn reload_dictionary() {
*DICT.write().unwrap_or_else(|e| e.into_inner()) = None;
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
slot.dict = None;
slot.in_memory = false;
}
/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
/// matching file also happens to exist.
pub fn installed_path() -> Option<std::path::PathBuf> {
if DICT.read().unwrap_or_else(|e| e.into_inner()).in_memory {
return None;
}
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
}
/// Is a Russian dictionary available (spelling is checked)?