mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-11 13:02:25 +00:00
Merge origin/main into spanish-proofing
Resolve conflicts with German hyphenation (ProofLang::De), Russian and Czech in installed_dictionaries(), and storytold/wordcraft#2164's U+FE0F flag fix: affix.rs takes main's fix and drops this branch's duplicate. The Danish, Dutch, Romanian and Slovenian catalogs gain the Spanish (Latin America) and (Uruguay) names; the language menu tests get room for the four Spanish entries.
This commit is contained in:
commit
d80c136e55
7 files changed
+191
-81
No files matched your search
+29
-30
@@ -11,8 +11,7 @@
|
||||
//! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text
|
||||
//! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB).
|
||||
//!
|
||||
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`;
|
||||
//! a variation selector after one, as in the Spanish dictionaries' `☎️`, belongs to it),
|
||||
//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),
|
||||
//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross
|
||||
//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),
|
||||
//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,
|
||||
@@ -262,23 +261,16 @@ struct Rules {
|
||||
/// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.
|
||||
type RawRule = (char, bool, String, String, String, String);
|
||||
|
||||
/// Is `c` a Unicode variation selector (U+FE00–U+FE0F)? It only picks how the character before
|
||||
/// it is drawn, so in a flag field it belongs to that flag: the RLA-ES Spanish dictionaries write
|
||||
/// a flag as `☎️` (`☎` and U+FE0F), which Hunspell reads as `☎` (#3360).
|
||||
fn is_variation_selector(c: char) -> bool {
|
||||
('\u{FE00}'..='\u{FE0F}').contains(&c)
|
||||
}
|
||||
|
||||
/// The flag characters of a flag field, without [variation selectors](is_variation_selector).
|
||||
fn flag_chars(field: &str) -> impl Iterator<Item = char> + '_ {
|
||||
field.chars().filter(|&c| !is_variation_selector(c))
|
||||
}
|
||||
|
||||
/// `0` means empty in Hunspell's strip and append fields.
|
||||
fn zero_empty(s: &str) -> String {
|
||||
if s == "0" { String::new() } else { s.to_string() }
|
||||
}
|
||||
|
||||
/// U+FE0F VARIATION SELECTOR-16 asks for the emoji form of the character before it. A dictionary
|
||||
/// that uses an emoji as a flag may write it after the flag (RLA-ES Spanish: `PFX ☎️`, `banco/S☎️`),
|
||||
/// but it is not a flag of its own: it is dropped wherever flags are read.
|
||||
const EMOJI_SELECTOR: char = '\u{fe0f}';
|
||||
|
||||
/// Read Hunspell `.aff` text (already decoded to UTF-8).
|
||||
fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
let mut r = Rules::default();
|
||||
@@ -303,7 +295,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
}
|
||||
}
|
||||
directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {
|
||||
let mut it = flag_chars(at(1));
|
||||
let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
|
||||
match (it.next(), it.next()) {
|
||||
(Some(c), None) if !c.is_whitespace() => {
|
||||
if directive == "FORBIDDENWORD" {
|
||||
@@ -333,8 +325,8 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
}
|
||||
"PFX" | "SFX" => {
|
||||
let suffix = dir == "SFX";
|
||||
let mut flag_field = flag_chars(at(1));
|
||||
let (Some(flag), None) = (flag_field.next(), flag_field.next()) else {
|
||||
let mut flag_chars = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);
|
||||
let (Some(flag), None) = (flag_chars.next(), flag_chars.next()) else {
|
||||
return Err(format!("line {}: flag `{}` is not one character", n + 1, at(1)));
|
||||
};
|
||||
// Only a header ignores trailing fields; a rule condition beginning with # is literal.
|
||||
@@ -354,7 +346,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> {
|
||||
let cond = if f.len() >= 5 { at(4) } else { "." };
|
||||
parse_cond(cond).map_err(|e| format!("line {}: {e}", n + 1))?;
|
||||
let c = cross.get(&(suffix, flag)).copied().unwrap_or(false);
|
||||
let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), flag_chars(cont).collect(), cond.to_string());
|
||||
let rule: RawRule = (flag, c, zero_empty(at(2)), zero_empty(add), cont.to_string(), cond.to_string());
|
||||
if suffix {
|
||||
r.suffixes.push(rule);
|
||||
} else {
|
||||
@@ -390,7 +382,7 @@ impl AffixDict {
|
||||
let mut in_flags = false;
|
||||
while let Some(c) = chars.next() {
|
||||
if in_flags {
|
||||
if !is_variation_selector(c) {
|
||||
if c != EMOJI_SELECTOR {
|
||||
flags.push(c);
|
||||
}
|
||||
} else if c == '\\' && chars.peek() == Some(&'/') {
|
||||
@@ -1094,18 +1086,25 @@ pub(crate) mod tests {
|
||||
assert!(AffixDict::from_hunspell("FLAG UTF-8\n", "0\n", &[]).is_ok(), "FLAG alone is harmless");
|
||||
}
|
||||
|
||||
/// #3360: the RLA-ES Spanish dictionaries write one flag as `☎️` (`☎` and the variation
|
||||
/// selector U+FE0F) in the `.aff` and in the `.dic`; it is the flag `☎`, as Hunspell reads it,
|
||||
/// not a two-character flag that refuses the whole dictionary.
|
||||
#[test]
|
||||
fn a_variation_selector_belongs_to_its_flag() {
|
||||
let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD ✖\u{FE0F}\nSFX \u{260E}\u{FE0F} Y 1\nSFX \u{260E}\u{FE0F} 0 s/A\u{FE0F} .\nSFX A Y 1\nSFX A 0 x .\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\ncasa/\u{260E}\u{FE0F}\nmesa/\u{260E}\nzq/✖\u{FE0F}\n", &[]).unwrap();
|
||||
assert!(d.contains("casa") && d.contains("casas") && d.contains("casasx"));
|
||||
assert!(d.contains("mesas"), "the bare flag is the same flag");
|
||||
assert!(!d.contains("zq"), "FORBIDDENWORD with a selector still forbids");
|
||||
assert_eq!(d.flag_chars.len(), 3, "{:?}", d.flag_chars);
|
||||
assert!(AffixDict::from_hunspell("SFX ab Y 1\nSFX ab 0 s .\n", "0\n", &[]).is_err(), "two flag characters are still refused");
|
||||
fn emoji_flags_with_a_variation_selector_load() {
|
||||
// RLA-ES (the Spanish dictionaries in LibreOffice and most Linux distributions) writes its
|
||||
// telephone flag as ☎ followed by U+FE0F, in the rule and on the stems.
|
||||
let aff = "SET UTF-8\nFLAG UTF-8\nPFX \u{260e}\u{fe0f} Y 1\nPFX \u{260e}\u{fe0f} 0 tele .\nSFX S Y 1\nSFX S 0 s [aeiou]\n";
|
||||
let d = AffixDict::from_hunspell(aff, "2\nbanco/S\u{260e}\u{fe0f}\nbasura/S\n", &[]).unwrap();
|
||||
assert!(d.contains("banco") && d.contains("bancos"));
|
||||
assert!(d.contains("telebanco") && d.contains("telebancos"), "the ☎ prefix applies");
|
||||
assert!(d.contains("basuras") && !d.contains("telebasura"), "only to stems that carry it");
|
||||
assert!(!d.flag_chars.contains(&EMOJI_SELECTOR), "the selector is not a flag");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn emoji_forbiddenword_and_nosuggest_flags_load() {
|
||||
// The same selector after the flag of the directives that name one flag.
|
||||
let aff = "SET UTF-8\nFLAG UTF-8\nFORBIDDENWORD \u{260e}\u{fe0f}\nNOSUGGEST \u{1f6ab}\u{fe0f}\n";
|
||||
let d = AffixDict::from_hunspell(aff, "3\nbanco\nbanko/\u{260e}\u{fe0f}\nbancu/\u{1f6ab}\u{fe0f}\n", &[]).unwrap();
|
||||
assert!(d.contains("banco") && d.contains("bancu"));
|
||||
assert!(!d.contains("banko"), "a stem with the FORBIDDENWORD flag is not a word");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -13,8 +13,16 @@ use crate::affix::AffixDict;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Czech dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
/// The Czech dictionary and whether it came from [`install_dictionary`] rather than a file on
|
||||
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
|
||||
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
|
||||
/// fields into an inconsistent combination.
|
||||
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
|
||||
|
||||
struct DictState {
|
||||
dict: Option<Arc<AffixDict>>,
|
||||
in_memory: bool,
|
||||
}
|
||||
|
||||
/// File names of a Czech Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
|
||||
@@ -23,13 +31,13 @@ const NAMES: &[&str] = &["cs_CZ", "cs-CZ", "cs"];
|
||||
/// [`install_dictionary`], or an empty one when none is available (Czech spelling is off
|
||||
/// then; see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
slot.dict.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
@@ -47,14 +55,29 @@ fn load_installed() -> AffixDict {
|
||||
/// UTF-8), for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d));
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Czech dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
|
||||
/// matching file also happens to exist.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
if DICT.read().unwrap_or_else(|e| e.into_inner()).in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
}
|
||||
|
||||
/// Is a Czech dictionary available (spelling is checked)?
|
||||
|
||||
+41
-24
@@ -15,6 +15,9 @@
|
||||
//!
|
||||
//! **Bulgarian**: the Bulgarian syllable rules ([`crate::bg`]); no pattern file or word list.
|
||||
//!
|
||||
//! **German**: the TeX German patterns for the reformed spelling ([`Patterns::de`]), also for
|
||||
//! Swiss text, whose `ss` the patterns split like any double consonant (`Stras-se`).
|
||||
//!
|
||||
//! Then the paragraph's limits ([`Limits`]) are applied.
|
||||
|
||||
pub use crate::dict;
|
||||
@@ -36,6 +39,9 @@ static PT_PAT: &str = include_str!("../../../assets/hyphenation/pt.pat");
|
||||
/// hyph-utf8's `hyph-it.tex`, unmodified (offered under MIT or LPPL; WordCraft uses MIT, whose
|
||||
/// notice is in the file's header).
|
||||
static IT_PAT: &str = include_str!("../../../assets/hyphenation/it.pat");
|
||||
/// hyph-utf8's `hyph-de-1996.tex` (reformed spelling), unmodified (MIT; the notice is in the
|
||||
/// file's header).
|
||||
static DE_PAT: &str = include_str!("../../../assets/hyphenation/de.pat");
|
||||
|
||||
impl Dictionary {
|
||||
/// The bundled US English dictionary (decoded on first use).
|
||||
@@ -77,6 +83,12 @@ impl Patterns {
|
||||
static P: OnceLock<Patterns> = OnceLock::new();
|
||||
P.get_or_init(|| Patterns::parse(IT_PAT))
|
||||
}
|
||||
|
||||
/// The bundled German patterns (parsed on first use; the file has no exception words).
|
||||
pub fn de() -> &'static Patterns {
|
||||
static P: OnceLock<Patterns> = OnceLock::new();
|
||||
P.get_or_init(|| Patterns::parse(DE_PAT))
|
||||
}
|
||||
}
|
||||
|
||||
/// Hyphenation limits from the paragraph's Hyphenation settings.
|
||||
@@ -130,11 +142,11 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
|
||||
return (crate::tr::hyphen::points(core), Source::Rules);
|
||||
}
|
||||
if lang == ProofLang::Pt {
|
||||
return core_breaks_pt(core);
|
||||
}
|
||||
if lang == ProofLang::It {
|
||||
return core_breaks_it(core);
|
||||
match lang {
|
||||
ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core),
|
||||
ProofLang::It => return core_breaks_patterns(Patterns::it(), core),
|
||||
ProofLang::De => return core_breaks_patterns(Patterns::de(), core),
|
||||
_ => {}
|
||||
}
|
||||
if lang == ProofLang::Ru && core.iter().all(|&c| crate::ru::is_cyrillic(c)) {
|
||||
return (crate::ru::hyphen::points(core), Source::Heuristic);
|
||||
@@ -178,25 +190,9 @@ fn core_breaks_pl(core: &[char]) -> (Vec<usize>, Source) {
|
||||
(heuristic_points_pl(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// [`core_breaks_pl`] for Portuguese: the bundled patterns, then the generic heuristic.
|
||||
fn core_breaks_pt(core: &[char]) -> (Vec<usize>, Source) {
|
||||
let pats = Patterns::pt();
|
||||
let s: String = core.iter().collect();
|
||||
if let Some(p) = pats.exception(&s) {
|
||||
return (p.to_vec(), Source::Dictionary);
|
||||
}
|
||||
if !pats.is_empty()
|
||||
&& let Some(p) = pats.points(core)
|
||||
{
|
||||
return (p, Source::Patterns);
|
||||
}
|
||||
(heuristic_points(core), Source::Heuristic)
|
||||
}
|
||||
|
||||
/// [`core_breaks_pt`] for Italian: the bundled patterns (the file has no exception words),
|
||||
/// then the generic heuristic.
|
||||
fn core_breaks_it(core: &[char]) -> (Vec<usize>, Source) {
|
||||
let pats = Patterns::it();
|
||||
/// [`core_breaks_pl`] for a language with bundled patterns (Portuguese, Italian, German): its
|
||||
/// exception words, the patterns, then the generic heuristic.
|
||||
fn core_breaks_patterns(pats: &Patterns, core: &[char]) -> (Vec<usize>, Source) {
|
||||
let s: String = core.iter().collect();
|
||||
if let Some(p) = pats.exception(&s) {
|
||||
return (p.to_vec(), Source::Dictionary);
|
||||
@@ -862,6 +858,27 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn german_patterns_load() {
|
||||
let p = Patterns::de();
|
||||
assert_eq!(p.len(), 36_709);
|
||||
// The file has no exception words, and its `\message{…}` line adds none.
|
||||
assert_eq!(p.exception_count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn german_words_follow_the_german_patterns() {
|
||||
let de = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::De);
|
||||
assert_eq!(de("Silbentrennung"), "Sil-ben-tren-nung");
|
||||
assert_eq!(de("unterliegende"), "un-ter-lie-gen-de");
|
||||
assert_eq!(de("öffentlich-rechtlicher"), "öf-fent-lich-recht-li-cher");
|
||||
assert_eq!(de("Begründung"), "Be-grün-dung");
|
||||
assert_eq!(de("Zucker"), "Zu-cker");
|
||||
// Swiss spelling: `ss` splits like any double consonant.
|
||||
assert_eq!(de("Strasse"), "Stras-se");
|
||||
assert_eq!(word_breaks_in("Unterlagen", ProofLang::De).1, Source::Patterns);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulgarian_words_follow_the_bulgarian_rules() {
|
||||
let bg = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Bg);
|
||||
|
||||
@@ -65,6 +65,10 @@ pub enum ProofLang {
|
||||
/// dictionary (else the neighbouring Rioplatense `es_AR`, a Latin American or a generic one),
|
||||
/// and the `vos` forms of verbs (`tenés`, `vení`, `decime`) accepted ([`crate::es::voseo`]).
|
||||
EsUy,
|
||||
/// German (`de`, `de-DE`, `de-AT`, `de-CH`…): hyphenation by the German patterns (reformed
|
||||
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
|
||||
/// no spelling or grammar marks.
|
||||
De,
|
||||
/// A language without proofing tools here (`fr-FR`, `ja-JP`, `x-none`…). Like Word without
|
||||
/// the language's proofing tools: no spelling or grammar marks, so its words are never
|
||||
/// flagged as English misspellings. Hyphenation keeps using the English patterns, as before
|
||||
@@ -121,6 +125,8 @@ impl ProofLang {
|
||||
ProofLang::Id
|
||||
} else if primary.eq_ignore_ascii_case("es") || primary.eq_ignore_ascii_case("spa") {
|
||||
ProofLang::spanish(tag)
|
||||
} else if ["de", "deu", "ger"].iter().any(|c| primary.eq_ignore_ascii_case(c)) {
|
||||
ProofLang::De
|
||||
} else {
|
||||
ProofLang::Other
|
||||
}
|
||||
@@ -160,7 +166,7 @@ impl ProofLang {
|
||||
matches!(self, ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy)
|
||||
}
|
||||
|
||||
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `es`); empty for [`ProofLang::Other`].
|
||||
/// The primary language subtag (`en`, `pl`, `bg`, `tr`, `ru`, `cs`, `id`, `de`, `es`); empty for [`ProofLang::Other`].
|
||||
pub fn code(self) -> &'static str {
|
||||
match self {
|
||||
ProofLang::En | ProofLang::EnGb | ProofLang::EnCa => "en",
|
||||
@@ -174,6 +180,7 @@ impl ProofLang {
|
||||
ProofLang::Cs => "cs",
|
||||
ProofLang::Id => "id",
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => "es",
|
||||
ProofLang::De => "de",
|
||||
ProofLang::Other => "",
|
||||
}
|
||||
}
|
||||
@@ -327,7 +334,11 @@ mod tests {
|
||||
for t in ["tr", "tr-TR", "tr-CY", "TR_tr", "tur"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Tr, "{t}");
|
||||
}
|
||||
for t in ["fr-FR", "de", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] {
|
||||
for t in ["de", "de-DE", "de-CH", "DE_at", "deu", "ger"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::De, "{t}");
|
||||
}
|
||||
assert!(!ProofLang::De.is_proofed());
|
||||
for t in ["fr-FR", "dex", "x-none", "ja-JP", "plx", "bgx", "trx", "csx", "idx", "inx", "esx", "spax", "e", "-"] {
|
||||
assert_eq!(ProofLang::from_tag(Some(t)), ProofLang::Other, "{t}");
|
||||
}
|
||||
assert_eq!(ProofLang::from_tag(None), ProofLang::En);
|
||||
|
||||
+31
-9
@@ -39,10 +39,12 @@
|
||||
//! `es_UY`, with fallbacks; not bundled), dictionary-driven suggestions, Uruguayan `vos` forms,
|
||||
//! and generic checks plus the opening `¿`/`¡` with Spanish messages. Hyphenation uses the
|
||||
//! existing English fallback.
|
||||
//! - **German** ([`ProofLang::De`]): hyphenation by `hyph-utf8`'s German patterns (reformed
|
||||
//! spelling, MIT); no spelling or grammar marks yet.
|
||||
//! - **Other languages** ([`ProofLang::Other`]): no spelling or grammar marks, like Word without
|
||||
//! the language's proofing tools.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese),
|
||||
//! Bulgarian syllable rules, Russian word-division rules, heuristic.
|
||||
//! - **Hyphenation** ([`hyphen`]): dictionary, Liang patterns (English, Polish, Portuguese,
|
||||
//! Italian, German), Bulgarian syllable rules, Russian word-division rules, heuristic.
|
||||
//!
|
||||
//! [`check_text`] checks a paragraph whose runs are in different languages.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
@@ -157,6 +159,8 @@ pub fn set_dictionaries_folder(dir: Option<std::path::PathBuf>) {
|
||||
it::reload_dictionary();
|
||||
hr::reload_dictionary();
|
||||
tr::reload_dictionary();
|
||||
ru::reload_dictionary();
|
||||
cs::reload_dictionary();
|
||||
id::reload_dictionary();
|
||||
es::reload_dictionary();
|
||||
DICTIONARY_GENERATION.fetch_add(1, Ordering::Relaxed);
|
||||
@@ -171,6 +175,8 @@ pub fn installed_dictionaries() -> Vec<(ProofLang, Option<std::path::PathBuf>)>
|
||||
(ProofLang::It, it::installed_path()),
|
||||
(ProofLang::Hr, hr::installed_path()),
|
||||
(ProofLang::Tr, tr::installed_path()),
|
||||
(ProofLang::Ru, ru::installed_path()),
|
||||
(ProofLang::Cs, cs::installed_path()),
|
||||
(ProofLang::Id, id::installed_path()),
|
||||
(ProofLang::Es, es::installed_path(ProofLang::Es)),
|
||||
(ProofLang::EsMx, es::installed_path(ProofLang::EsMx)),
|
||||
@@ -738,7 +744,7 @@ pub fn is_correct_in_with(word: &str, lang: ProofLang, opts: &Options) -> bool {
|
||||
ProofLang::Cs => accepted_by_options(word, opts) || cs::is_correct(word),
|
||||
ProofLang::Id => accepted_by_options(word, opts) || id::is_correct(word),
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => accepted_by_options(word, opts) || es::is_correct(word, lang),
|
||||
ProofLang::Bg | ProofLang::Other => true,
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => true,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -755,7 +761,7 @@ pub fn suggest_in(word: &str, max: usize, lang: ProofLang) -> Vec<String> {
|
||||
ProofLang::Cs => cs::suggest(word, max),
|
||||
ProofLang::Id => id::suggest(word, max),
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::suggest(word, max, lang),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -784,7 +790,7 @@ fn grammar_in(text: &str, lang: ProofLang, capital_first: bool) -> Vec<Issue> {
|
||||
ProofLang::Id => id::grammar::check(text),
|
||||
// Spanish has no sentence-capital rule either.
|
||||
ProofLang::Es | ProofLang::EsMx | ProofLang::Es419 | ProofLang::EsUy => es::grammar::check(text, lang),
|
||||
ProofLang::Bg | ProofLang::Other => Vec::new(),
|
||||
ProofLang::Bg | ProofLang::De | ProofLang::Other => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -978,7 +984,7 @@ mod tests {
|
||||
assert_eq!(dictionary_needed(l, false), Some(name));
|
||||
assert_eq!(dictionary_needed(l, true), None);
|
||||
}
|
||||
for l in [ProofLang::En, ProofLang::Bg, ProofLang::Other] {
|
||||
for l in [ProofLang::En, ProofLang::Bg, ProofLang::De, ProofLang::Other] {
|
||||
assert_eq!(dictionary_needed(l, false), None);
|
||||
}
|
||||
// Installed dictionaries and the built-in English list: no hint.
|
||||
@@ -1005,6 +1011,8 @@ mod tests {
|
||||
("pt_BR", pt::test_dict::AFF, pt::test_dict::DIC),
|
||||
("it_IT", it::test_dict::AFF, it::test_dict::DIC),
|
||||
("hr_HR", hr::test_dict::AFF, hr::test_dict::DIC),
|
||||
("ru_RU", ru::test_dict::AFF, ru::test_dict::DIC),
|
||||
("cs_CZ", cs::test_dict::AFF, cs::test_dict::DIC),
|
||||
("id_ID", id::test_dict::AFF, id::test_dict::DIC),
|
||||
("es_ES", es::test_dict::AFF, es::test_dict::DIC),
|
||||
("es_MX", es::test_dict::AFF, es::test_dict::DIC_AMERICAS),
|
||||
@@ -1014,14 +1022,19 @@ mod tests {
|
||||
std::fs::write(dir.join(format!("{name}.aff")), aff).unwrap();
|
||||
std::fs::write(dir.join(format!("{name}.dic")), dic).unwrap();
|
||||
}
|
||||
// #3252 review: force Russian and Czech to cache whatever they find before the folder is
|
||||
// set (empty, with no folder and nothing installed on CI), so the folder change below has
|
||||
// a stale cache to actually invalidate, not just an untouched `None`.
|
||||
ru::dictionary();
|
||||
cs::dictionary();
|
||||
let before = dictionary_generation();
|
||||
set_dictionaries_folder(Some(dir.clone()));
|
||||
assert!(dictionary_generation() > before, "cached checks are dropped");
|
||||
// Turkish has no test dictionary to put in the folder; Indonesian and the Spanishes are listed after it.
|
||||
// Turkish has no test dictionary to put in the folder; Russian, Czech, Indonesian and the Spanishes follow it.
|
||||
let found = installed_dictionaries();
|
||||
assert_eq!(found.get(4).map(|(lang, _)| *lang), Some(ProofLang::Tr));
|
||||
assert_eq!(
|
||||
found[5..],
|
||||
found[7..],
|
||||
[
|
||||
(ProofLang::Id, Some(dir.join("id_ID.dic"))),
|
||||
(ProofLang::Es, Some(dir.join("es_ES.dic"))),
|
||||
@@ -1039,8 +1052,17 @@ mod tests {
|
||||
(ProofLang::Hr, Some(dir.join("hr_HR.dic"))),
|
||||
]
|
||||
);
|
||||
// #3226: Russian and Czech were missing from installed_dictionaries() entirely (no
|
||||
// installed_path() in either module), so neither ever showed up in File › Options ›
|
||||
// Proofing, installed or not. Their languages, not their exact path, is what belongs
|
||||
// here: unlike the four above, installed_path() for Russian and Czech also reflects
|
||||
// whether install_dictionary() gave one directly (#3252 review), which ru's and cs's own
|
||||
// test modules do, concurrently, in tests of their own — asserting an exact path here
|
||||
// would race them for no reason this test needs to care about.
|
||||
assert_eq!(found[5..7].iter().map(|(lang, _)| *lang).collect::<Vec<_>>(), [ProofLang::Ru, ProofLang::Cs], "{found:?}");
|
||||
assert!(is_correct_in("chrząszcz", ProofLang::Pl) && !is_correct_in("chrzonszcz", ProofLang::Pl));
|
||||
assert!(is_correct_in("tenés", ProofLang::EsUy) && !is_correct_in("prueva", ProofLang::EsUy));
|
||||
assert!(is_correct_in("дом", ProofLang::Ru) && !is_correct_in("дум", ProofLang::Ru));
|
||||
}
|
||||
|
||||
/// #1741: a list item may be a fragment, so its first word needs no capital in any language
|
||||
@@ -1704,7 +1726,7 @@ mod tests {
|
||||
let _ = has_dictionary(Some(&s));
|
||||
let _ = is_correct_for(&s, Some(&s));
|
||||
let _ = hyphen::hyphen_points(&s, &hyphen::Limits::default());
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other] {
|
||||
for lang in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa, ProofLang::Pl, ProofLang::Pt, ProofLang::It, ProofLang::Hr, ProofLang::Tr, ProofLang::Ru, ProofLang::Cs, ProofLang::Id, ProofLang::De, ProofLang::Es, ProofLang::EsMx, ProofLang::Es419, ProofLang::EsUy, ProofLang::Other] {
|
||||
let _ = check_spelling_in(&s, lang);
|
||||
let _ = check_grammar_in(&s, lang);
|
||||
let _ = is_correct_in(&s, lang);
|
||||
|
||||
@@ -116,13 +116,16 @@ pub struct Patterns {
|
||||
enum Block {
|
||||
Patterns,
|
||||
Exceptions,
|
||||
/// Another command's argument (`\message{…}`), ignored.
|
||||
Skip,
|
||||
}
|
||||
|
||||
impl Patterns {
|
||||
/// Parse patterns: whitespace-separated, `%` comments to the end of the line (`#` too at the
|
||||
/// start of a line). A whole TeX pattern file works as is: `\patterns{…}` holds patterns and
|
||||
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`); outside any
|
||||
/// block, a token with a hyphen is an exception and anything else a pattern.
|
||||
/// `\hyphenation{…}` exception words written with their hyphens (`ni-gdy`), and other
|
||||
/// commands' arguments (`\message{…}`) are skipped; outside any block, a token with a hyphen
|
||||
/// is an exception and anything else a pattern.
|
||||
pub fn parse(text: &str) -> Patterns {
|
||||
let mut p = Patterns::default();
|
||||
let mut block = Block::Patterns;
|
||||
@@ -136,12 +139,18 @@ impl Patterns {
|
||||
let mut tok = tok;
|
||||
if let Some(cmd) = tok.strip_prefix('\\') {
|
||||
let (name, rest) = cmd.split_once('{').unwrap_or((cmd, ""));
|
||||
block = if name == "hyphenation" { Block::Exceptions } else { Block::Patterns };
|
||||
block = match name {
|
||||
"hyphenation" => Block::Exceptions,
|
||||
"patterns" => Block::Patterns,
|
||||
// shortcut: the argument ends at the first `}`, so nested braces would leak; no bundled file has them.
|
||||
_ if cmd.contains('{') => Block::Skip,
|
||||
_ => Block::Patterns,
|
||||
};
|
||||
tok = rest;
|
||||
}
|
||||
let closes = tok.ends_with('}');
|
||||
let tok = tok.trim_end_matches('}');
|
||||
if !tok.is_empty() {
|
||||
if !tok.is_empty() && block != Block::Skip {
|
||||
if block == Block::Exceptions || tok.contains('-') {
|
||||
p.insert_exception(tok);
|
||||
} else {
|
||||
@@ -337,6 +346,12 @@ mod tests {
|
||||
assert!(p.to_tex().contains(&".ab4cdefghijklm5n".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_commands_arguments_are_skipped() {
|
||||
let p = Patterns::parse("\\message{Some Patterns `x-y' 2024-02-28}\n\\patterns{%\na1b\n}\n\\endinput\n");
|
||||
assert_eq!((p.len(), p.exception_count()), (1, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_tolerates_short_value_buffers() {
|
||||
let p = Patterns::parse("a1b");
|
||||
|
||||
@@ -34,8 +34,16 @@ use crate::affix::AffixDict;
|
||||
|
||||
pub use suggest::suggest;
|
||||
|
||||
/// The Russian dictionary: `None` until first use, then the installed one (or an empty one).
|
||||
static DICT: RwLock<Option<Arc<AffixDict>>> = RwLock::new(None);
|
||||
/// The Russian dictionary and whether it came from [`install_dictionary`] rather than a file on
|
||||
/// disk (it then has no file of its own to report from [`installed_path`], even if a matching
|
||||
/// file also happens to exist): one lock, so a reload and an install can't interleave their two
|
||||
/// fields into an inconsistent combination.
|
||||
static DICT: RwLock<DictState> = RwLock::new(DictState { dict: None, in_memory: false });
|
||||
|
||||
struct DictState {
|
||||
dict: Option<Arc<AffixDict>>,
|
||||
in_memory: bool,
|
||||
}
|
||||
|
||||
/// File names of a Russian Hunspell dictionary.
|
||||
const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
|
||||
@@ -44,13 +52,13 @@ const NAMES: &[&str] = &["ru_RU", "ru-RU", "ru"];
|
||||
/// [`install_dictionary`], or an empty one when none is available (Russian spelling is off then;
|
||||
/// see [`has_dictionary`]).
|
||||
pub fn dictionary() -> Arc<AffixDict> {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).as_ref() {
|
||||
if let Some(d) = DICT.read().unwrap_or_else(|e| e.into_inner()).dict.as_ref() {
|
||||
return d.clone();
|
||||
}
|
||||
let loaded = Arc::new(load_installed());
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
// Another thread (or an install) may have filled it meanwhile: that one wins.
|
||||
slot.get_or_insert(loaded).clone()
|
||||
slot.dict.get_or_insert(loaded).clone()
|
||||
}
|
||||
|
||||
fn load_installed() -> AffixDict {
|
||||
@@ -68,14 +76,29 @@ fn load_installed() -> AffixDict {
|
||||
/// for example one the user picked. On error the current dictionary stays.
|
||||
pub fn install_dictionary(aff: &str, dic: &str) -> Result<(), String> {
|
||||
let d = AffixDict::from_hunspell(aff, dic, &[])?;
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = Some(Arc::new(d));
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
slot.dict = Some(Arc::new(d));
|
||||
slot.in_memory = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Forget the loaded dictionary, so the next check looks for an installed one again (after
|
||||
/// [`crate::hunspell::set_dictionary_dir`] or a dictionary was installed).
|
||||
pub fn reload_dictionary() {
|
||||
*DICT.write().unwrap_or_else(|e| e.into_inner()) = None;
|
||||
let mut slot = DICT.write().unwrap_or_else(|e| e.into_inner());
|
||||
slot.dict = None;
|
||||
slot.in_memory = false;
|
||||
}
|
||||
|
||||
/// The `.dic` file of the installed Russian dictionary, searching the dictionaries folder and
|
||||
/// the system locations now (File › Options › Proofing shows it, #3226); `None` when there is
|
||||
/// none. A dictionary given with [`install_dictionary`] has no file and isn't reported, even if a
|
||||
/// matching file also happens to exist.
|
||||
pub fn installed_path() -> Option<std::path::PathBuf> {
|
||||
if DICT.read().unwrap_or_else(|e| e.into_inner()).in_memory {
|
||||
return None;
|
||||
}
|
||||
crate::hunspell::find(NAMES).map(|(_, dic)| dic)
|
||||
}
|
||||
|
||||
/// Is a Russian dictionary available (spelling is checked)?
|
||||
|
||||
Reference in new issue
Block a user