diff --git a/NOTICE b/NOTICE index 1c7ac9c9..1073f48d 100644 --- a/NOTICE +++ b/NOTICE @@ -25,6 +25,33 @@ crates/proof/data/en-words.dic: public domain, the words of the Moby Hyphenator Ward (Project Gutenberg eBook #3204), without its break points. crates/proof/data/en-extra.txt: MIT OR Apache-2.0, WordCraft contributors' own list of modern English words (a copy of craft-i18n's language-data/en-us/en-extra.txt). +craft-proof Language data, embedded at build time from craft-i18n's language-data/ when + CRAFT_I18N_DIR is set (CRAFT_PROOF_EMBED picks the locales; by default English and the German, + Polish, Portuguese, Italian and Russian data). Each file keeps its licence header; the + licences are listed in craft-i18n's ATTRIBUTION.md, and a pattern file's notice ships with any + application that embeds it. The Portuguese hyphenation patterns (language-data/pt/ + hyphenation.pat, hyph-utf8 hyph-pt 1.4) are under the BSD 3-clause licence: + Copyright (C) 1987, 1994, 1996, 2015 Pedro J. de Rezende, 1996, 2015 J. Joao Dias Almeida, + 2024 Leonardo Araujo and Aline Benevides. + Redistribution and use in source and binary forms, with or without modification, are + permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + * Neither the name of the University of Campinas, of the University of Minho nor the names of + its contributors may be used to endorse or promote products derived from this software + without specific prior written permission. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND + FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PEDRO J. DE REZENDE OR J.JOAO + DIAS ALMEIDA BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. crates/text/assets/hyphen/en-us.pat: CC0-1.0, trained from en-us.dic by DesignCraft's trainer. Hyphenation patterns for other languages, embedded through the hypher crate only when the matching craft-text feature is enabled (none are by default). They come from hyph-utf8, diff --git a/README.md b/README.md index 740258af..8f9f68f1 100644 --- a/README.md +++ b/README.md @@ -88,8 +88,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "/language-data/en-us/en-us.dic data/en-words.dic +//! cargo run -p craft-proof --example en_words -- /language-data/en-us/en-us.dic crates/proof/data/en-words.dic //! ``` //! //! The input is craft-i18n's `en-us.dic`: the Moby Hyphenator II list by Grady Ward (Project diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs index c9b8d339..63d6f289 100644 --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -55,15 +55,17 @@ const MAX_INPUT_BYTES: usize = 4 * MAX_WORD_BYTES; const MAX_CONV_PAIRS: usize = 1 << 16; /// Longest text on either side of an `ICONV` or `OCONV` pair, in bytes. const MAX_CONV_BYTES: usize = 64; -/// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a -/// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or -/// skipped (`.dic`, though its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). -pub const MAX_FLAGS_PER_ENTRY: usize = 1024; +/// Most flags one `.dic` entry or one continuation class field may carry. LibreOffice's Turkish +/// dictionary has 920 entries over 1,024 flags (`abart`, `savaş`: up to 3,926), so the cap sits +/// well above that; a longer field is hostile and is refused (`.aff`) or skipped (`.dic`, though +/// its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). What all the flags of a +/// dictionary may cost together is bounded by `MAX_SET_DATA`. +pub const MAX_FLAGS_PER_ENTRY: usize = 8192; /// Most flags one stem may carry once its homonyms (the same word on several `.dic` lines) are /// merged, and so most flags a stem line of the compact format may carry. Larger than /// [`MAX_FLAGS_PER_ENTRY`] because merged sets aren't `.dic` fields; a homonym that would take the /// set past it adds only its `FORBIDDENWORD` and `NOSUGGEST` flags, as an exclusion is never lost. -pub const MAX_FLAGS_PER_STEM: usize = 8 * MAX_FLAGS_PER_ENTRY; +pub const MAX_FLAGS_PER_STEM: usize = 4 * MAX_FLAGS_PER_ENTRY; /// Most `AF` aliases, and most bytes of alias text in all: each alias is read once and shared by /// the stems using it, so these bound the flags an `AF` table can make a dictionary hold /// (the largest Language pack's table: 1,596 aliases, 19 KB). @@ -72,6 +74,7 @@ const MAX_ALIAS_BYTES: usize = 4 << 20; /// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup /// tests each distinct continuation set it meets at most once, so this bounds its cost. const MAX_SET_DATA: usize = 1 << 23; + /// Most distinct flags a dictionary may use: each gets a `u16` id. const MAX_DISTINCT_FLAGS: usize = 1 << 16; diff --git a/crates/proof/src/affix_suggest.rs b/crates/proof/src/affix_suggest.rs index b20dfb93..3c9b06bc 100644 --- a/crates/proof/src/affix_suggest.rs +++ b/crates/proof/src/affix_suggest.rs @@ -201,25 +201,32 @@ fn alternatives(c: char, map: &[Vec]) -> Vec { } /// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`, -/// letter by letter), with its cost, each written after `buf`. -fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { - if out.len() >= MAX_MAP_VARIANTS { +/// letter by letter), with its cost: all one-change variants first, then two, and so on, so the +/// [`MAX_MAP_VARIANTS`] cap drops the variants with the most changes. +fn map_variants(chars: &[char], alts: &[Vec], t: &Tuning, out: &mut Vec<(String, u32)>) { + let mut buf = Vec::with_capacity(chars.len()); + for n in 1..=t.max_map_changes.min(chars.len()) { + exact_map_variants(chars, alts, t, n, 0, &mut buf, out); + } +} + +/// [`map_variants`] with exactly `left` more changes, each written after `buf`. +fn exact_map_variants(chars: &[char], alts: &[Vec], t: &Tuning, left: usize, cost: u32, buf: &mut Vec, out: &mut Vec<(String, u32)>) { + if out.len() >= MAX_MAP_VARIANTS || left > chars.len() { return; } let Some((&c, rest)) = chars.split_first() else { - if changes > 0 { - out.push((buf.iter().collect(), cost)); - } + out.push((buf.iter().collect(), cost)); return; }; let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l)); buf.push(c); - map_variants(rest, later, t, changes, cost, buf, out); + exact_map_variants(rest, later, t, left, cost, buf, out); buf.pop(); - if changes < t.max_map_changes { + if left > 0 { for &a in here { buf.push(a); - map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out); + exact_map_variants(rest, later, t, left - 1, cost + (t.map_cost)(c, a), buf, out); buf.pop(); } } @@ -368,7 +375,7 @@ fn add_case(col: &mut Collector, word: &str, input: &str) { fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec], t: &Tuning) { let alts: Vec> = chars.iter().map(|&c| alternatives(c, map)).collect(); let mut variants = Vec::new(); - map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants); + map_variants(chars, &alts, t, &mut variants); variants.sort_by_key(|v| v.1); for (v, cost) in &variants { col.offer(v, *cost); @@ -535,6 +542,22 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec]) { mod tests { use super::*; + /// craft-libs#487 review: MAP variants are made by number of changes, so when a long word has + /// more than [`MAX_MAP_VARIANTS`] of them the cap drops the most-changed ones, never a single + /// slip (here the first letter's, which depth-first order reached last). + #[test] + fn the_map_cap_keeps_the_fewest_changes() { + let chars: Vec = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".chars().collect(); + let alts = vec!["áàâäãåā".chars().collect::>(); chars.len()]; + let mut out = Vec::new(); + map_variants(&chars, &alts, &Tuning::DEFAULT, &mut out); + assert_eq!(out.len(), MAX_MAP_VARIANTS); + let changed = |v: &str| v.chars().filter(|&c| c != 'a').count(); + assert_eq!(out.iter().filter(|(v, _)| changed(v) == 1).count(), 30 * 7, "every single slip is kept"); + assert!(out.iter().any(|(v, _)| v.starts_with('á') && changed(v) == 1)); + assert!(out.windows(2).all(|w| changed(&w[0].0) <= changed(&w[1].0)), "fewest changes first"); + } + /// craft-libs#490 review: a suggestion recased to capitals keeps `ß` (`STRAßE`, right with or /// without CHECKSHARPS); under CHECKSHARPS the SS form comes first, and a KEEPCASE `ß` word may /// be capitalized or written in capitals with SS. Expected lists are the hunspell 1.7.5 CLI's diff --git a/crates/proof/src/cs/grammar.rs b/crates/proof/src/cs/grammar.rs index 26a14fcd..8a2590e6 100644 --- a/crates/proof/src/cs/grammar.rs +++ b/crates/proof/src/cs/grammar.rs @@ -15,11 +15,19 @@ pub fn check(text: &str) -> Vec { issues } +/// Words Czech repeats on purpose (`ne ne`, `ano ano`, `už už`, `to to`). +const ALLOWED_REPEATS: &[&str] = &["ne", "ano", "už", "to", "jo", "no", "tak", "pryč", "dál", "pomalu", "honem"]; + fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec) { for pair in ws.windows(2) { let [(a0, b0), (a1, b1)] = pair else { continue }; let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue }; - if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() { + if !gap.is_empty() + && gap.chars().all(char::is_whitespace) + && w0.chars().all(char::is_alphabetic) + && w0.to_lowercase() == w1.to_lowercase() + && !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str()) + { v.push(issue(*a0, *b1, format!("Opakované slovo: „{w1}“"), vec![w0.to_string()])); } } @@ -66,6 +74,15 @@ fn spaces(text: &str, v: &mut Vec) { mod tests { use super::*; + /// craft-libs#495 review: intentional repeats aren't marked; others still are. + #[test] + fn intentional_repeats_are_fine() { + for text in ["Ne ne, to nechci.", "Ano ano, přijdu.", "Už už odcházel.", "To to bylo krásné."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + assert_eq!(check("Byl byl doma.").len(), 1); + } + #[test] fn repeated_words_and_spaces_have_czech_messages() { let text = "Město město , škola."; diff --git a/crates/proof/src/hr/grammar.rs b/crates/proof/src/hr/grammar.rs index 517c5623..2605ae88 100644 --- a/crates/proof/src/hr/grammar.rs +++ b/crates/proof/src/hr/grammar.rs @@ -98,7 +98,7 @@ fn spaces(text: &str, v: &mut Vec) { // ` ...` is an ellipsis, ` .5` a number. '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. - ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + ':' | '!' | '?' => !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), _ => false, }; if flag { diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs index 8e83104a..274af9eb 100644 --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -105,16 +105,7 @@ fn search_tiers(lo_exts: Option<&[String]>) -> (Vec, Vec) { system.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy())); } } - // If LibreOffice is installed on another drive (e.g. D:\Program Files\LibreOffice), check drives C..Z - for drive in b'C'..=b'Z' { - let drive_char = drive as char; - for pf in ["Program Files", "Program Files (x86)"] { - let lo_base = PathBuf::from(format!(r"{drive_char}:\{pf}\LibreOffice\share\extensions")); - if lo_base.is_dir() { - system.extend(lo_dirs(&lo_base.to_string_lossy())); - } - } - } + system.extend(lo_drive_dirs(lo_exts, || LO_DRIVE_BASES.get_or_init(|| lo_drive_bases(Path::is_dir)))); } else if cfg!(target_os = "macos") { if let Some(h) = &home { // An app started from Finder or the Dock doesn't see `WORDCRAFT_DICTIONARIES`; this @@ -144,6 +135,26 @@ fn search_tiers(lo_exts: Option<&[String]>) -> (Vec, Vec) { (dirs, system) } +/// LibreOffice's language folders (`:\Program Files\LibreOffice\share\extensions\`) +/// on the drives C to Z where LibreOffice is installed (`bases`). No drive is probed without +/// `lo_exts` (no LibreOffice folders are wanted then). +fn lo_drive_dirs<'a>(lo_exts: Option<&[String]>, bases: impl FnOnce() -> &'a [PathBuf]) -> Vec { + let Some(exts) = lo_exts else { return Vec::new() }; + bases().iter().flat_map(|base| exts.iter().map(|e| base.join(e))).collect() +} + +/// The `LibreOffice\share\extensions` folders on drives C to Z (48 probes). +fn lo_drive_bases(is_dir: impl Fn(&Path) -> bool) -> Vec { + ('C'..='Z') + .flat_map(|drive| ["Program Files", "Program Files (x86)"].map(|pf| PathBuf::from(format!(r"{drive}:\{pf}\LibreOffice\share\extensions")))) + .filter(|base| is_dir(base)) + .collect() +} + +/// [`lo_drive_bases`], probed once per process. +// ponytail: a LibreOffice installed while the app runs is seen after a restart. +static LO_DRIVE_BASES: std::sync::OnceLock> = std::sync::OnceLock::new(); + /// The user's data folder on Linux and the BSDs: `$XDG_DATA_HOME` when it is an absolute path (the /// XDG Base Directory spec ignores a relative one), else `~/.local/share`. fn xdg_data_home(xdg: Option, home: Option<&Path>) -> Option { @@ -610,6 +621,31 @@ fn decode_utf8_dic(dic: &[u8]) -> Result { mod tests { use super::*; + /// CodeRabbit on craft-libs#489: without LibreOffice extension names no drive is probed; with + /// them, only drives where LibreOffice is found give folders. + #[test] + fn libreoffice_drives_are_probed_only_when_wanted() { + let probes = std::cell::Cell::new(0); + let count = |_: &Path| { + probes.set(probes.get() + 1); + false + }; + // craft-libs#495 review: the drives are probed once, not on every lookup. + let cache = std::sync::OnceLock::new(); + let cached = || cache.get_or_init(|| lo_drive_bases(count)).as_slice(); + assert!(lo_drive_dirs(None, cached).is_empty()); + assert_eq!(probes.get(), 0); + let exts = ["dict-tr".to_string()]; + assert!(lo_drive_dirs(Some(&exts), cached).is_empty()); + assert!(lo_drive_dirs(Some(&exts), cached).is_empty()); + assert_eq!(probes.get(), 48); + let found = lo_drive_bases(|p: &Path| p.to_string_lossy().starts_with(r"D:\Program Files\")); + assert_eq!( + lo_drive_dirs(Some(&exts), || found.as_slice()), + [PathBuf::from(r"D:\Program Files\LibreOffice\share\extensions").join("dict-tr")] + ); + } + /// Another region's Language pack never beats the system's dictionary of the right region /// (`pt-pt` pack vs system `pt_BR`; `de-ch` vs `de_DE`; `en-gb` vs `en_US`), but beats the /// system's other names; the right region's pack beats the system; the dictionaries folder diff --git a/crates/proof/src/hyphen.rs b/crates/proof/src/hyphen.rs index e3c89daa..6bc804b2 100644 --- a/crates/proof/src/hyphen.rs +++ b/crates/proof/src/hyphen.rs @@ -148,8 +148,16 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec, Source) { if lang == ProofLang::Bg && core.iter().all(|&c| crate::bg::is_letter(c)) { return (crate::bg::points(core), Source::Rules); } - if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) { - return (crate::tr::hyphen::points(core), Source::Rules); + // A suffix written after an apostrophe (`Ankara'dan`, `Türkiye'nin`) is hyphenated part by + // part; no break goes next to the apostrophe ([`word_breaks_in`] drops those). + if lang == ProofLang::Tr && core.iter().all(|&c| c == '\'' || crate::tr::hyphen::is_letter(c)) { + let mut out = Vec::new(); + let mut start = 0; + for part in core.split(|&c| c == '\'') { + out.extend(crate::tr::hyphen::points(part).into_iter().map(|p| p + start)); + start += part.len() + 1; + } + return (out, Source::Rules); } match lang { ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core), @@ -526,6 +534,18 @@ fn heuristic_points_pl(c: &[char]) -> Vec { mod tests { use super::*; + /// CodeRabbit on craft-libs#484: a Turkish word with a suffix after an apostrophe is + /// hyphenated by the Turkish rules, part by part, never next to the apostrophe. + #[test] + fn turkish_apostrophe_suffixes_follow_turkish_rules() { + let tr = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Tr); + assert_eq!(tr("Ankara'dan"), "An-ka-ra'dan"); + assert_eq!(tr("Türkiye'nin"), "Tür-ki-ye'nin"); + assert_eq!(tr("İstanbul'daki"), "İs-tan-bul'da-ki"); + assert_eq!(word_breaks_in("Ankara'dan", ProofLang::Tr).1, Source::Rules); + assert_eq!(tr("Ankara"), "An-ka-ra", "no apostrophe: as before"); + } + fn h(w: &str) -> String { hyphenate_word(w, &Limits::default()) } diff --git a/crates/proof/src/it/grammar.rs b/crates/proof/src/it/grammar.rs index 46302936..57c62e28 100644 --- a/crates/proof/src/it/grammar.rs +++ b/crates/proof/src/it/grammar.rs @@ -44,6 +44,9 @@ const NOT_FEMININE_IN_A: &[&str] = &["acrobata", "autodidatta", "alfa", "omega", /// Abbreviations written with a final period after which a sentence goes on in lower case /// (lower case, without the period). const ABBREVIATIONS: &[&str] = &[ + "pp", + "tab", + "dr", "ecc", "es", "dott", @@ -193,7 +196,7 @@ fn spaces(text: &str, v: &mut Vec) { // ` ...` is an ellipsis, ` .5` a number. '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. - ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + ':' | '!' | '?' => !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), _ => false, }; if flag { @@ -210,13 +213,15 @@ fn spaces(text: &str, v: &mut Vec) { fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec) { let mut after_end = capital_first; - for (a, b) in ws { + for (i, (a, b)) in ws.iter().enumerate() { let Some(w) = text.get(*a..*b) else { continue }; if after_end && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) { let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default(); v.push(issue(*a, *b, "Maiuscola a inizio frase".into(), vec![fixed])); } - after_end = ends_sentence(text.get(*b..).unwrap_or(""), w); + // Only the gap before the next word: the rest of the paragraph never ends a sentence. + let next = ws.get(i + 1).map_or(text.len(), |n| n.0); + after_end = ends_sentence(text.get(*b..next).unwrap_or(""), w); } } @@ -384,6 +389,20 @@ mod tests { (a.to_string(), b.to_string()) } + /// CodeRabbit on craft-libs#484: a sentence after the first one starts with a capital too. + #[test] + fn later_sentences_start_with_a_capital() { + assert!(fixes("Vado a Roma. il cane dorme.").contains(&pair("il", "Il"))); + assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì"))); + assert!(!fixes("Vado a Roma. Il cane dorme.").iter().any(|(_, b)| b == "Il")); + for t in ["Il sig. rossi arriva.", "Vedi pp. dieci e oltre.", "Nella tab. due.", "Il dr. bianchi arriva.", "Mele, pere ecc. sono frutti."] { + assert!( + !fixes(t).iter().any(|(_, b)| b.starts_with(char::is_uppercase) && b != "Il" && b != "Vedi" && b != "Nella" && b != "Mele"), + "after an abbreviation: {t}" + ); + } + } + #[test] fn po_takes_an_apostrophe() { assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]); diff --git a/crates/proof/src/it/mod.rs b/crates/proof/src/it/mod.rs index ecc4aad6..a742eaa2 100644 --- a/crates/proof/src/it/mod.rs +++ b/crates/proof/src/it/mod.rs @@ -95,8 +95,9 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) { return true; } + // The whole word first, as Hunspell does: a hyphenated entry or user word. if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } if let Some((head, tail)) = w.split_once(['\'', '’']) && PROCLITICS.contains(&head.to_lowercase().as_str()) @@ -139,6 +140,14 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// craft-libs#487 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn without_a_dictionary_nothing_is_flagged() { let none = AffixDict::default(); diff --git a/crates/proof/src/lang.rs b/crates/proof/src/lang.rs index 64ad003e..993e4030 100644 --- a/crates/proof/src/lang.rs +++ b/crates/proof/src/lang.rs @@ -23,9 +23,8 @@ pub enum ProofLang { /// Polish (`pl`, `pl-PL`): the installed Hunspell dictionary (when there is one), the Polish grammar rules and the Polish /// hyphenation patterns. Pl, - /// Bulgarian (`bg`, `bg-BG`): hyphenation by the Bulgarian syllable rules ([`crate::bg`]). - /// There is no Bulgarian dictionary or grammar yet, so, as for [`ProofLang::Other`], no - /// spelling or grammar marks. + /// Bulgarian (`bg`, `bg-BG`): hyphenation by the Bulgarian syllable rules ([`crate::bg`]), and + /// spelling when a Bulgarian Hunspell dictionary or `bg` pack is found. No grammar rules yet. Bg, /// Portuguese (`pt`, `pt-BR`; `pt-PT` is [`ProofLang::Pack`] `pt-pt`, with its own dictionary /// and these grammar rules and patterns): the installed Hunspell dictionary (VERO and @@ -69,8 +68,8 @@ pub enum ProofLang { EsUy, /// German (`de`, `de-DE`, `de-AT`…; `de-CH` and `de-LI` are [`ProofLang::Pack`] `de-ch`, with /// their own dictionary and these patterns): hyphenation by the German patterns (reformed - /// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`], - /// no spelling or grammar marks. + /// spelling), and spelling when a German Hunspell dictionary or `de` pack is found. No grammar + /// rules yet. De, /// Any other language, by its tag (`nl`, `fr-FR`, `sr-Latn`): spelling and suggestions from /// its Hunspell dictionary (the dictionaries folder, an installed Language pack, the system's) @@ -112,7 +111,7 @@ impl ProofLang { /// `x-none`, `und`, `zxx`, `mul`, `mis` and malformed tags are [`ProofLang::Other`]. pub fn from_tag(tag: Option<&str>) -> ProofLang { let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En }; - let primary = tag.split(['-', '_']).next().unwrap_or(""); + let primary = tag.split(['-', '_', '.', '@']).next().unwrap_or(""); if primary.eq_ignore_ascii_case("en") || primary.eq_ignore_ascii_case("eng") { ProofLang::english(tag) } else if primary.eq_ignore_ascii_case("pl") || primary.eq_ignore_ascii_case("pol") { @@ -148,6 +147,9 @@ impl ProofLang { } else { ProofLang::De } + } else if tag.split_once('@').is_some_and(|(_, m)| m.eq_ignore_ascii_case("latin")) { + // POSIX `sr_RS@latin`: the Latin-script language (`sr-latn`). + PackLocale::new(&format!("{primary}-latn")).map_or(ProofLang::Other, ProofLang::Pack) } else { PackLocale::new(tag).map_or(ProofLang::Other, ProofLang::Pack) } @@ -443,7 +445,13 @@ mod tests { #[test] fn tags_resolve_primary_languages() { - assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"], ProofLang::Pl); + assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol", "pl@euro", "pl.UTF-8"], ProofLang::Pl); + // craft-libs#482 review: a POSIX charset or modifier with no region. + assert_tags(&["en.UTF-8", "en@euro"], ProofLang::En); + assert_tags(&["de.UTF-8"], ProofLang::De); + let sr_latn = ProofLang::Pack(PackLocale::new("sr-latn").unwrap()); + assert_tags(&["sr@latin", "sr_RS@latin", "sr_RS.UTF-8@latin", "sr-Latn"], sr_latn); + assert_tags(&["sr", "sr@cyrillic"], ProofLang::Pack(PackLocale::new("sr").unwrap())); assert_tags(&["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", "pt-AO", "por"], ProofLang::Pt); let pt_pt = ProofLang::Pack(PackLocale::new("pt-pt").unwrap()); assert_tags(&[" pt-PT ", "pt_PT.UTF-8", "por-PT"], pt_pt); @@ -499,8 +507,11 @@ mod tests { assert_tags(&["de-CH", "de_CH.UTF-8", "de-LI"], de_ch); assert_eq!(de_ch.built_in(), ProofLang::De); assert_tags(&["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"], ProofLang::Ru); - assert!(!ProofLang::De.is_proofed()); - assert!(!ProofLang::Bg.is_proofed()); + // German and Bulgarian are proofed when a dictionary is found (craft-libs#485 review: an + // installed hunspell-de-de must not fail this test), so only their locales are fixed here. + assert_eq!(ProofLang::De.spelling_locale(), PackLocale::new("de-de")); + assert_eq!(ProofLang::Bg.spelling_locale(), PackLocale::new("bg-bg")); + assert!(!ProofLang::PROOFED.contains(&ProofLang::De) && !ProofLang::PROOFED.contains(&ProofLang::Bg)); assert_eq!(ProofLang::Bg.code(), "bg"); assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed()); assert!(ProofLang::Tr.is_proofed()); diff --git a/crates/proof/src/lib.rs b/crates/proof/src/lib.rs index 21229a47..5b58768c 100644 --- a/crates/proof/src/lib.rs +++ b/crates/proof/src/lib.rs @@ -675,6 +675,13 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool { || in_user_dictionary(&w.to_lowercase()) } +/// Is the space at byte `space` of `text` the first of a score or ratio's `3 : 1`, which keeps +/// its spaces (the grammar rules' space-before-punctuation check)? +pub(crate) fn score_colon(text: &str, space: usize) -> bool { + text.get(..space).and_then(|t| t.chars().next_back()).is_some_and(|c| c.is_ascii_digit()) + && text.get(space..).is_some_and(|t| t.starts_with(" : ") && t.get(3..).and_then(|r| r.chars().next()).is_some_and(|c| c.is_ascii_digit())) +} + /// [`is_correct`] with the File › Options › Proofing choices: English with no dialect, so a word /// any English spells so is right (`color` and `colour`; American English's dictionary when it /// isn't SCOWL: [`english_source`]), without the dialect policy. @@ -1930,6 +1937,45 @@ mod tests { let _ = std::fs::remove_dir_all(&root); } + /// craft-libs#485 review: a licence file that is too big is skipped alone; the pack's other + /// licences and the next pack's are still shown. + #[test] + fn an_oversized_licence_skips_only_itself() { + crate::hold_dictionary_tests(); + set_dictionaries_folder(Some(shared_test_folder())); + let root = std::env::temp_dir().join(format!("wc-proof-licence-skip-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + for l in ["qa", "qb"] { + let dir = write_pack(&root, l, &[]); + std::fs::write(dir.join("LICENCES").join("A-HUGE"), vec![b'x'; (4 << 20) + 1]).unwrap(); + std::fs::write(dir.join("LICENCES").join("COPYING"), format!("licence {l}")).unwrap(); + } + packs::set_packs_dir(Some(root.clone())); + let texts = packs::licence_texts(); + let shown: Vec<&str> = texts.iter().filter(|(t, _)| t.contains("(q")).map(|(_, x)| x.as_str()).collect(); + assert_eq!(shown, ["licence qa", "licence qb"]); + packs::set_packs_dir(None); + set_dictionaries_folder(Some(shared_test_folder())); + let _ = std::fs::remove_dir_all(&root); + } + + /// craft-libs#483 and #488 reviews: a score keeps its spaces (`3 : 1`) in every language's + /// space-before-punctuation rule, and in Polish and Russian ` ,,` (a typed opening quote) is + /// left to the doubled-punctuation rule alone. + #[test] + fn scores_and_typed_quotes_keep_their_spaces() { + let space_marks = |text: &str, lang| check_grammar_in(text, lang).into_iter().filter(|i| text.get(i.start..i.start + 1) == Some(" ")).count(); + for lang in [ProofLang::Ru, ProofLang::Pl, ProofLang::Hr, ProofLang::It, ProofLang::Pt] { + assert_eq!(space_marks("Wynik 3 : 1 dzisiaj.", lang), 0, "{lang:?}"); + assert_eq!(space_marks("Wynik 3 : dzisiaj.", lang), 1, "{lang:?}: not a score"); + } + for (text, lang) in [("Powiedział ,,tak” wczoraj.", ProofLang::Pl), ("Он сказал ,,да” вчера.", ProofLang::Ru)] { + let issues = check_grammar_in(text, lang); + assert_eq!(space_marks(text, lang), 0, "{issues:?}"); + assert!(issues.iter().any(|i| text.get(i.start..i.end) == Some(",,")), "the quote rule still marks it: {issues:?}"); + } + } + /// Pack languages for the cache tests: `qa`, `qb`… each knowing `word`. fn many_packs(root: &std::path::Path, n: usize) -> Vec { (0..n) diff --git a/crates/proof/src/no_data.rs b/crates/proof/src/no_data.rs index f4aa52c3..2f429985 100644 --- a/crates/proof/src/no_data.rs +++ b/crates/proof/src/no_data.rs @@ -64,7 +64,15 @@ fn nothing_is_embedded() { fn english_spelling_still_marks_misspellings() { crate::hold_dictionary_tests(); for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] { - assert!(matches!(crate::english_source(l), Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); + let source = crate::english_source(l); + assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}"); + // Any English dictionary marks junk and knows `the`; the word-by-word results below are + // the built-in list's, so an installed dictionary (with `teh`, without `because`) is + // not held to them (craft-libs#482 and #487 reviews). + assert!(!crate::is_correct_in("zxqvbnwk", l) && crate::is_correct_in("the", l), "{l:?}"); + if !matches!(source, Some(EnglishSource::CoreList)) { + continue; + } assert!(!crate::is_correct_in("becuase", l) && !crate::is_correct_in("teh", l), "{l:?}"); assert!(crate::suggest_in("teh", 5, l).contains(&"the".to_string()), "{l:?}"); assert!(crate::suggest_in("becuase", 5, l).contains(&"because".to_string()), "{l:?}"); @@ -73,11 +81,15 @@ fn english_spelling_still_marks_misspellings() { } assert_eq!(crate::check_spelling_in("Becuase the house of the people", l).len(), 1, "{l:?}"); } - assert!( - !crate::is_correct("becuase") && !crate::is_correct_with("becuase", &Options::default()) && !crate::is_correct_for("becuase", Some("en-GB")) - ); - assert!(crate::suggest("becuase", 5).contains(&"because".to_string())); - assert_eq!(crate::check_spelling("Becuase house").len(), 1); + if matches!(crate::english_source(ProofLang::En), Some(EnglishSource::CoreList)) { + assert!( + !crate::is_correct("becuase") + && !crate::is_correct_with("becuase", &Options::default()) + && !crate::is_correct_for("becuase", Some("en-GB")) + ); + assert!(crate::suggest("becuase", 5).contains(&"because".to_string())); + assert_eq!(crate::check_spelling("Becuase house").len(), 1); + } // The dialect variants are Language data: none without it. for w in ["colour", "color"] { assert_eq!(en_variants::preferred(w, ProofLang::En), None); diff --git a/crates/proof/src/packs.rs b/crates/proof/src/packs.rs index c4cee539..2fb291e9 100644 --- a/crates/proof/src/packs.rs +++ b/crates/proof/src/packs.rs @@ -389,8 +389,8 @@ fn licence_files(pack: &Path, left: &mut u64) -> Vec<(String, String)> { let path = dir.join(&n); let size = std::fs::symlink_metadata(&path).ok().filter(std::fs::Metadata::is_file)?.len(); if size > (*left).min(MAX_LICENCE) { - // Over what is left: none of the rest is read either. - *left = 0; + // Too big, or over what is left: this file is skipped, smaller ones still fit. + log::warn!("Language pack licence {} is {size} bytes; not shown", path.display()); return None; } let bytes = read_capped(&path, size.min(MAX_LICENCE)).ok()?; diff --git a/crates/proof/src/pl/grammar.rs b/crates/proof/src/pl/grammar.rs index 4289f0c7..d514429e 100644 --- a/crates/proof/src/pl/grammar.rs +++ b/crates/proof/src/pl/grammar.rs @@ -288,13 +288,16 @@ fn spaces(text: &str, v: &mut Vec) { let Some(p) = it.next() else { continue }; let after = it.next(); let flag = match p { - ',' => true, + // ` ,,` opens a typed quote (the doubled-punctuation rule suggests `„`). + ',' => after != Some(','), // `;)` and `;-)` are emoticons. ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), // ` ...` is an ellipsis, ` .5` a number. '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. - ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?')), + ':' | '!' | '?' => { + !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?')) + } _ => false, }; if flag { diff --git a/crates/proof/src/pt/grammar.rs b/crates/proof/src/pt/grammar.rs index 6b72e322..303929de 100644 --- a/crates/proof/src/pt/grammar.rs +++ b/crates/proof/src/pt/grammar.rs @@ -63,7 +63,9 @@ fn ends_sentence(gap: &str, prev: &str) -> bool { return false; } let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING); - rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c)) + // A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes + // on in lower case. + rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c)) } /// Grammar issues in a paragraph of Portuguese text, in order. @@ -115,7 +117,7 @@ fn spaces(text: &str, v: &mut Vec) { // ` ...` is an ellipsis, ` .5` a number. '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. - ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), + ':' | '!' | '?' => !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')), _ => false, }; if flag { @@ -220,6 +222,9 @@ mod tests { clean("Zhou ficou em 5. lugar no séc. XX."); clean("Esperei... e ele apareceu."); clean("— Vai — disse ele. — Amanhã eu volto."); + // craft-libs#488 review: no space after the dash. + clean("—Vai? —disse ele."); + clean("— Vai? — disse ele."); clean("Ele disse: \"Volto amanhã\". E saiu."); // A list item may start in lower case. clean("maçãs,"); diff --git a/crates/proof/src/pt/mod.rs b/crates/proof/src/pt/mod.rs index 661bb998..00e80265 100644 --- a/crates/proof/src/pt/mod.rs +++ b/crates/proof/src/pt/mod.rs @@ -71,8 +71,9 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool { let Some(w) = crate::spelling::word_to_check(d, word, opts) else { return true; }; + // The whole word first, as Hunspell does: a hyphenated entry or user word. if w.contains('-') { - return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); + return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p)); } known(d, w) } @@ -96,6 +97,14 @@ mod tests { assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty()); } + /// craft-libs#487 review: a hyphenated dictionary entry is a word even when its parts aren't. + #[test] + fn a_hyphenated_entry_is_looked_up_whole() { + let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap(); + assert!(is_correct_with(&d, "tic-tac")); + assert!(!is_correct_with(&d, "tic-tacc")); + } + #[test] fn without_a_dictionary_nothing_is_flagged() { let none = AffixDict::default(); diff --git a/crates/proof/src/ru/extra.rs b/crates/proof/src/ru/extra.rs index 26551e0d..a8567b29 100644 --- a/crates/proof/src/ru/extra.rs +++ b/crates/proof/src/ru/extra.rs @@ -133,7 +133,14 @@ fn forms(base: &str, class: &str) -> Vec { } "adj" => { if let Some(stem) = base.strip_suffix("ый").or_else(|| base.strip_suffix("ой")) { - let mut v = with(stem, &["ого", "ому", "ым", "ом", "ая", "ой", "ую", "ое", "ые", "ых", "ыми"]); + // `ы` is written `и` after г, к, х and ж, ш, ч, щ (`дорогие`, `большими`). + let i = is_velar(last(stem)) || is_hushing(last(stem)); + let ends: &[&str] = if i { + &["ого", "ому", "им", "ом", "ая", "ой", "ую", "ое", "ие", "их", "ими"] + } else { + &["ого", "ому", "ым", "ом", "ая", "ой", "ую", "ое", "ые", "ых", "ыми"] + }; + let mut v = with(stem, ends); v.push(base.to_string()); return v; } @@ -156,6 +163,16 @@ fn forms(base: &str, class: &str) -> Vec { mod tests { use super::*; + /// craft-libs#495 review: `-ой` adjectives after a velar or hushing letter take `и`. + #[test] + fn oi_adjectives_spell_i_after_velars_and_hushers() { + let f = forms("дорогой", "adj"); + assert!(f.contains(&"дорогие".to_string()) && f.contains(&"дорогими".to_string()) && !f.iter().any(|w| w.contains('ы')), "{f:?}"); + let f = forms("большой", "adj"); + assert!(f.contains(&"больших".to_string()) && !f.iter().any(|w| w.contains('ы')), "{f:?}"); + assert!(forms("молодой", "adj").contains(&"молодые".to_string())); + } + #[cfg_attr( not(craft_proof_embed = "ru"), ignore = "needs the ru Language data (CRAFT_I18N_DIR; ru-extra.txt is embedded in every build that has it)" diff --git a/crates/proof/src/ru/grammar.rs b/crates/proof/src/ru/grammar.rs index 70b9a1e6..ff8711fb 100644 --- a/crates/proof/src/ru/grammar.rs +++ b/crates/proof/src/ru/grammar.rs @@ -264,6 +264,10 @@ const NO_COMMA_AFTER: &[&str] = &[ "затем", "тем", "того", + // `тем более что`, `при условии что`, `в то время как`. + "более", + "условии", + "время", "ввиду", "вследствие", "несмотря", @@ -483,13 +487,16 @@ fn spaces(text: &str, v: &mut Vec) { let Some(p) = it.next() else { continue }; let after = it.next(); let flag = match p { - ',' => true, + // ` ,,` opens a typed quote (the doubled-punctuation rule suggests `„`). + ',' => after != Some(','), // `;)` and `;-)` are emoticons. ';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')), // ` ...` is an ellipsis, ` .5` a number. '.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), // ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space. - ':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '»' | '”' | '’' | '!' | '?')), + ':' | '!' | '?' => { + !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '»' | '”' | '’' | '!' | '?')) + } _ => false, }; if flag { @@ -687,6 +694,10 @@ mod tests { clean("Расскажи хотя бы что-нибудь, за что тебя хвалят."); clean("Что ты делаешь? Который час?"); clean("Кто-то что-то сказал."); + // craft-libs#495 review: the second half of a compound conjunction. + clean("Он устал, тем более что шёл долго."); + clean("Приду, при условии что успею."); + clean("Он читал, в то время как она спала."); } #[test] diff --git a/crates/proof/src/spelling.rs b/crates/proof/src/spelling.rs index aaf25956..c80f42fe 100644 --- a/crates/proof/src/spelling.rs +++ b/crates/proof/src/spelling.rs @@ -24,7 +24,9 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options) if d.is_empty() { return None; } - let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); + // A trailing combining mark with no precomposed form (`x\u{338}`) is part of the word. + let edge = |c: char| !c.is_alphanumeric() && c != '\'' && c != '’'; + let w = word.trim_start_matches(edge).trim_end_matches(|c| edge(c) && !unicode_normalization::char::is_combining_mark(c)); let w = w.trim_matches(['\'', '’']); let n = w.chars().count(); if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { @@ -56,4 +58,13 @@ mod tests { assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B")); assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms"); } + + /// craft-libs#487 review: a trailing combining mark stays on the word it marks. + #[test] + fn trailing_combining_marks_are_kept() { + let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap(); + let defaults = Options::default(); + assert_eq!(word_to_check(&d, "kox\u{338}", &defaults), Some("kox\u{338}")); + assert_eq!(word_to_check(&d, "(kox\u{338}).", &defaults), Some("kox\u{338}")); + } } diff --git a/crates/proof/src/tr/dict.rs b/crates/proof/src/tr/dict.rs index 8cbacf02..117d5378 100644 --- a/crates/proof/src/tr/dict.rs +++ b/crates/proof/src/tr/dict.rs @@ -1,29 +1,16 @@ -//! Turkish Hunspell dictionary parser with `FLAG num` support. +//! The Turkish Hunspell dictionary (`FLAG num`: thousands of numeric suffix flags). //! -//! Turkish uses `FLAG num`, storing thousands of numerical suffix flags (0..6465). -//! This engine parses `/usr/share/hunspell/tr_TR.aff` and `.dic` safely without panics. +//! Read by the shared affix engine ([`AffixDict`]), so Turkish follows Hunspell as every other +//! language does: homonyms on separate lines keep each one's flags, a `.dic` entry ends before +//! its morphological fields (`kitap/1 po:noun`), and an affix applies only where its condition +//! matches the stem (`SFX 1 0 lar [^e]` never makes `kalelar`). -use std::collections::HashMap; +use crate::affix::AffixDict; -/// A suffix rule for Turkish. -#[derive(Clone, Debug)] -pub struct TrSuffixRule { - pub strip: Box, - pub add: Box, - pub cond: Box, -} - -/// Dedicated Turkish Hunspell dictionary engine. +/// The Turkish dictionary. #[derive(Clone, Debug, Default)] pub struct TrDict { - /// Stem -> list of numerical flags it carries. - stems: HashMap>, - /// Flag number -> list of suffix rules. - rules: HashMap>, - /// Suffix 'add' string -> list of (flag, rule_index). - suffix_index: HashMap>, - /// Longest suffix 'add' string in characters. - max_suffix_len: usize, + dict: AffixDict, } impl crate::slot::Dictionary for TrDict { @@ -43,102 +30,47 @@ impl crate::slot::Dictionary for TrDict { impl TrDict { /// Returns true if empty (no dictionary loaded). pub fn is_empty(&self) -> bool { - self.stems.is_empty() + self.dict.is_empty() } /// Load from `.aff` and `.dic` contents. pub fn from_hunspell(aff: &str, dic: &str) -> Result { - let mut rules: HashMap> = HashMap::new(); - let mut max_suffix_len = 0; - - for line in aff.lines() { - let parts: Vec<&str> = line.split_whitespace().collect(); - // SFX [] - if parts.len() >= 4 && parts[0] == "SFX" { - let Ok(flag) = parts[1].parse::() else { continue }; - // Skip header line: `SFX Y/N ` - if parts.len() == 4 && (parts[2] == "Y" || parts[2] == "N") && parts[3].parse::().is_ok() { - continue; - } - let strip = if parts[2] == "0" { "" } else { parts[2] }; - let add = if parts[3] == "0" { "" } else { parts[3] }; - let cond = if parts.len() > 4 { parts[4] } else { "." }; - - max_suffix_len = max_suffix_len.max(add.chars().count()); - - rules.entry(flag).or_default().push(TrSuffixRule { strip: strip.into(), add: add.into(), cond: cond.into() }); - } - } - - let mut suffix_index: HashMap> = HashMap::new(); - for (&flag, r_list) in &rules { - for (idx, r) in r_list.iter().enumerate() { - if let Ok(idx_u16) = u16::try_from(idx) { - suffix_index.entry(r.add.to_string()).or_default().push((flag, idx_u16)); - } - } - } - - let mut stems: HashMap> = HashMap::new(); - for (i, line) in dic.lines().enumerate() { - if i == 0 && line.trim().parse::().is_ok() { - continue; - } - let line = line.trim(); - if line.is_empty() { - continue; - } - if let Some((word, flags_part)) = line.split_once('/') { - let flags: Vec = flags_part.split(',').filter_map(|f| f.parse::().ok()).collect(); - stems.insert(word.to_string(), flags.into_boxed_slice()); - } else { - stems.insert(line.to_string(), Box::new([])); - } - } - - Ok(TrDict { stems, rules, suffix_index, max_suffix_len }) + Ok(TrDict { dict: AffixDict::from_hunspell(aff, dic, &[])? }) } - /// Check if word is correct (stem or affixed). + /// Is `word` exactly (case and all) a stem or an affixed form of one? pub fn contains(&self, word: &str) -> bool { - // Longer input is never a dictionary word, and isn't scanned (#3625). - if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES { - return false; + self.dict.contains(word) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// CodeRabbit on craft-libs#484/#489: suffix conditions, homonyms and morphological fields, + /// each answer as hunspell 1.7 gives it on the same files. + #[test] + fn conditions_homonyms_and_fields_follow_hunspell() { + let aff = "SET UTF-8\nFLAG num\nSFX 1 Y 2\nSFX 1 0 lar [^e]\nSFX 1 0 ler e\nSFX 2 Y 1\nSFX 2 0 de .\n"; + let d = TrDict::from_hunspell(aff, "4\nkitap/1\nkale/1\nkale/2\nsonra po:adv\n").unwrap(); + for (w, right) in + [("kitaplar", true), ("kaleler", true), ("kalelar", false), ("kitapler", false), ("kalede", true), ("sonra", true), ("kitap", true)] + { + assert_eq!(d.contains(w), right, "{w}"); } - - // Direct stem hit - if self.stems.contains_key(word) { - return true; - } - - // Try stripping suffixes matching the tail - let word_len = word.chars().count(); - let char_indices: Vec<(usize, char)> = word.char_indices().collect(); - - // Check each possible suffix length up to max_suffix_len - for cut_chars in 0..=self.max_suffix_len.min(word_len) { - let byte_idx = if cut_chars == 0 { word.len() } else { char_indices[word_len - cut_chars].0 }; - - let add = &word[byte_idx..]; - let base = &word[..byte_idx]; - - if let Some(matches) = self.suffix_index.get(add) { - for &(flag, rule_idx) in matches { - let Some(rule_list) = self.rules.get(&flag) else { continue }; - let Some(rule) = rule_list.get(rule_idx as usize) else { continue }; - - // Check condition on root - let mut candidate = String::with_capacity(base.len() + rule.strip.len()); - candidate.push_str(base); - candidate.push_str(&rule.strip); - - if self.stems.get(&candidate).is_some_and(|f_list| f_list.contains(&flag)) { - return true; - } - } - } - } - - false + let d = TrDict::from_hunspell(aff, "1\nkitap/1,2\tpo:noun\n").unwrap(); + assert!(d.contains("kitaplar") && d.contains("kitapde"), "both flags before the field"); + } + + /// #495 review: LibreOffice's `tr_TR.dic` has 920 entries with more than 1,024 flags + /// (`abart`, `savaş`, `inan`: up to 3,926); such a stem and its forms are words. + #[test] + fn a_stem_with_thousands_of_flags_is_a_word() { + let aff = "SET UTF-8\nFLAG num\nSFX 1 Y 1\nSFX 1 0 de .\nSFX 3999 Y 1\nSFX 3999 0 lar .\n"; + let flags = (1..=4000).map(|i| i.to_string()).collect::>().join(","); + let d = TrDict::from_hunspell(aff, &format!("2\nabart/{flags}\nkitap\n")).unwrap(); + assert!(d.contains("abart") && d.contains("abartde") && d.contains("abartlar") && d.contains("kitap")); + assert!(!d.contains("kitaplar")); } } diff --git a/crates/proof/src/tr/grammar.rs b/crates/proof/src/tr/grammar.rs index b17e89cb..81ba93c2 100644 --- a/crates/proof/src/tr/grammar.rs +++ b/crates/proof/src/tr/grammar.rs @@ -10,6 +10,7 @@ //! `ne ... ne` conjunctions should not have commas immediately before or after them. //! - **Doubled punctuation** (`..`, `,,`, `;;`). +use super::suggest::turkish_to_lower; use crate::{Issue, IssueKind, ProofLang, words}; /// Turkish reduplications and emphatic word pairings that are intentionally repeated. @@ -120,15 +121,15 @@ const ALLOWED_REPEATS: &[&str] = &[ /// Turkish abbreviations that end with a period and after which lower case may continue. const ABBREVIATIONS: &[&str] = &[ "vb", "vs", "vd", "bkz", "bknz", "ör", "örnek", "mad", "md", "s", "sf", "sn", "dk", "sa", "yy", "m", "km", "cm", "mm", "gr", "kg", "mg", "dr", - "prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "a.ş", "ltd", "şti", "t.c", - "tbmm", "meb", "möt", "mö", "ms", + "prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "ltd", "şti", "tbmm", "meb", + "möt", "mö", "ms", ]; /// Conjunctions that must not be preceded by a comma. const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"]; /// Conjunctions that must not be followed by a comma. -const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "ne", "ya", "gerek", "ister", "de", "da"]; +const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "gerek", "ister"]; /// Check grammar and punctuation in a Turkish text block. pub fn check(text: &str) -> Vec { @@ -152,9 +153,11 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { None => continue, }; let only_spaces = gap.chars().all(|c| c == ' ' || c == '\t'); - let lower1 = w1.to_lowercase(); - let lower2 = w2.to_lowercase(); - if only_spaces && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) { + let lower1 = turkish_to_lower(w1); + let lower2 = turkish_to_lower(w2); + // Words, not numbers or codes (`1 1`, `A1 A1`); `Ankara'da Ankara'da` is one. + let word = w1.chars().any(char::is_alphabetic) && !w1.chars().any(|c| c.is_numeric()); + if only_spaces && word && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) { v.push(Issue { start: pair[1].0, end: pair[1].1, @@ -172,9 +175,30 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { let (idx, c) = chars[i]; if matches!(c, ',' | '.' | '?' | '!' | ';' | ':') && i > 0 && chars[i - 1].1 == ' ' { let prev_sp = chars[i - 1].0; - // Leave emoticons and ellipses alone - let is_dot_in_dots = c == '.' && (i + 1 < chars.len() && chars[i + 1].1 == '.'); - if !is_dot_in_dots { + // Leave ellipses (` ...`), numbers (` .5`), emoticons (` :)`, ` ;-)`, ` :D`) and + // scores (`3 : 1`) alone. + let after = chars.get(i + 1).map(|&(_, a)| a); + let ratio = c == ':' + && i >= 2 + && chars.get(i - 2).is_some_and(|&(_, b)| b.is_ascii_digit()) + && after == Some(' ') + && chars.get(i + 2).is_some_and(|&(_, d)| d.is_ascii_digit()); + // `:)`, `;(`, `:-)` are emoticons; `:D` and `;P` only when a space or the end follows + // (`;Dün` is a stray space before a word). + let ends = |k: usize| chars.get(k).is_none_or(|&(_, e)| e.is_whitespace()); + let emoticon = match after { + Some(')' | '(') => true, + Some('-') => matches!(chars.get(i + 2), Some(&(_, ')' | '('))), + Some('D' | 'P') => ends(i + 2), + _ => false, + }; + let flag = !ratio + && match c { + ',' | '!' | '?' => true, + ':' | ';' => !emoticon, + _ => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()), + }; + if flag { v.push(Issue { start: prev_sp, end: idx + c.len_utf8(), @@ -205,7 +229,7 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { // 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`) for (i, &(w_start, w_end)) in ws.iter().enumerate() { let word = match text.get(w_start..w_end) { - Some(w) => w.to_lowercase(), + Some(w) => turkish_to_lower(w), None => continue, }; // Check comma before conjunction @@ -268,9 +292,18 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec { let prev_word = text.get(ws[i - 1].0..prev_end).unwrap_or(""); if let Some(gap) = text.get(prev_end..w_start) { let trimmed = gap.trim(); + // A space must follow the mark (`rapor.txt`, `www.example.com` are one token), and a + // number before a period is an ordinal (`5. sınıf`, `21. yüzyıl`). + let spaced = gap.rfind(['.', '?', '!']).and_then(|m| gap.get(m + 1..)).is_some_and(|rest| rest.contains(char::is_whitespace)); + let ordinal = trimmed.ends_with('.') && prev_word.chars().any(|c| c.is_ascii_digit()); + // An initial (`A.Ş.`, `T.C.`, `M. Kemal`) ends no sentence. + let initial = trimmed.starts_with('.') && prev_word.chars().count() == 1; if (trimmed.ends_with('.') || trimmed.ends_with('?') || trimmed.ends_with('!')) + && spaced + && !ordinal + && !initial && !trimmed.contains("...") - && !ABBREVIATIONS.contains(&prev_word.to_lowercase().as_str()) + && !ABBREVIATIONS.contains(&turkish_to_lower(prev_word).as_str()) && let Some(w) = text.get(w_start..w_end) && w.chars().next().is_some_and(char::is_lowercase) { @@ -310,6 +343,31 @@ pub fn turkish_capitalize(s: &str) -> String { mod tests { use super::*; + /// CodeRabbit on craft-libs#484/#489: emoticons, decimals and scores aren't stray spaces; + /// ordinals and file or web names don't end a sentence. + #[test] + fn emoticons_numbers_ordinals_and_names_are_left_alone() { + for ok in [ + "Harika :)", + "Tamam ;)", + "Süper :D", + "Oran .5 oldu.", + "Maç 3 : 1 bitti.", + "Ali 5. sınıfta okuyor.", + "Bina 21. yüzyıldan kalma.", + "Dosyayı rapor.txt olarak kaydet.", + "Adres www.example.com olarak yazıldı.", + ] { + assert!(check(ok).is_empty(), "{ok}: {:?}", check(ok)); + } + assert_eq!(check("Gel , bak.").len(), 1, "a real stray space is still flagged"); + for bad in ["Not :bu", "Ne ?Bu", "Peki ;Dün", "Evet !"] { + assert!(check(bad).iter().any(|i| i.message.starts_with("Noktalama")), "{bad}: {:?}", check(bad)); + } + assert!(check("Harika :-) dedi.").is_empty() && check("Peki ;P").is_empty()); + assert_eq!(check("Geldi. bu iyi.").len(), 1, "a real sentence start is still flagged"); + } + #[test] fn clean_turkish_text_produces_no_issues() { let text = "Bu bir test cümlesidir. Kitap ve defter aldım. Çocuk yavaş yavaş yürüdü."; @@ -343,10 +401,39 @@ mod tests { assert_eq!(issues.len(), 1); assert!(issues[0].message.contains("virgül konulmaz")); - let text2 = "Bana kalırsa bu mesele de, haklı olan taraf bizdik."; + let text2 = "Elma ve, armut."; let issues2 = check(text2); assert_eq!(issues2.len(), 1); - assert!(issues2[0].message.contains("'de' bağlacından sonra virgül konulmaz")); + assert!(issues2[0].message.contains("'ve' bağlacından sonra virgül konulmaz")); + } + + /// `de`/`da`, `ne` and `ya` often take a comma after them in correct Turkish, so no mark + /// (craft-libs#488 review). + #[test] + fn a_comma_after_de_ne_or_ya_is_fine() { + for text in ["Ali de, Veli de geldi.", "Ne, sen mi geldin?", "Ya, bilmiyorum.", "Bu mesele de, haklı olan bizdik."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + } + + /// craft-libs#495 review: initials (`A.Ş.`, `T.C.`) and `İst.` end no sentence, and Turkish + /// case folding finds `Işık ışık`; a word with an apostrophe is still a repeat. + #[test] + fn turkish_case_and_initials() { + for text in ["Şirket A.Ş. bugün açıldı.", "O T.C. vatandaşı.", "Adres İst. caddesi."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } + for text in ["Işık ışık yandı.", "Ankara'da Ankara'da kaldık."] { + assert_eq!(check(text).iter().filter(|i| i.message == "Yinelenen sözcük").count(), 1, "{text}"); + } + } + + /// Repeated numbers are not repeated words (craft-libs#488 review). + #[test] + fn repeated_numbers_are_not_flagged() { + for text in ["Skor 1 1 bitti.", "Toplam 100 100 değil.", "Oda A1 A1 değil."] { + assert!(check(text).is_empty(), "{text}: {:?}", check(text)); + } } #[test] diff --git a/crates/proof/src/tr/hyphen.rs b/crates/proof/src/tr/hyphen.rs index a8a48108..1479a53e 100644 --- a/crates/proof/src/tr/hyphen.rs +++ b/crates/proof/src/tr/hyphen.rs @@ -3,12 +3,13 @@ //! Turkish words are split at syllables according to Turkish Language Association (TDK) rules. //! Because Turkish syllable boundaries are fully regular, they can be determined directly by rule //! without needing external pattern files or word lists: -//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü` (both lowercase and uppercase). -//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `ö-ğren-ci`). +//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü`, and `â`, `î`, `û` (`hi-kâ-ye`, +//! `dük-kân`), lowercase and uppercase. +//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `a-ra-ba`). //! - Of two consonants between two vowels, the first stays with the preceding syllable and the -//! second joins the next (`alt-lık`, `sev-gi`, `tür-kçe`). +//! second joins the next (`öğ-ren-ci`, `sev-gi`, `kar-deş`). //! - Of three consonants between two vowels, the first two stay with the preceding syllable and the -//! third joins the next (`türk-çe`, `sürp-riz`). +//! third joins the next (`türk-çe`, `alt-lık`, `sürp-riz`). //! - In compound words, syllable division follows pronunciation across the boundary (`ba-şöğ-ret-men`, //! `il-ko-kul`), which matches the general vowel/consonant distribution rules. //! @@ -17,7 +18,7 @@ /// Is `c` a Turkish vowel? pub fn is_vowel(c: char) -> bool { - matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü') + matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'â' | 'î' | 'û' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü' | 'Â' | 'Î' | 'Û') } /// Is `c` a Turkish letter? @@ -86,6 +87,18 @@ mod tests { assert_eq!(split("başöğretmen"), "ba-şöğ-ret-men"); } + /// CodeRabbit on craft-libs#484: `â`, `î` and `û` are vowels, and the module doc's examples + /// are the ones the rules give. + #[test] + fn circumflex_vowels_and_doc_examples() { + assert_eq!(split("dükkân"), "dük-kân"); + assert_eq!(split("hikâye"), "hi-kâ-ye"); + assert_eq!(split("kâğıt"), "kâ-ğıt"); + assert_eq!(split("millî"), "mil-lî"); + assert_eq!(split("kardeş"), "kar-deş"); + assert_eq!(split("sürpriz"), "sürp-riz"); + } + #[test] fn short_or_non_words_stay_unsplit() { assert_eq!(split(""), ""); diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index 967148d8..5472fbe2 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -78,12 +78,8 @@ fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool { } let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’'); let w = w.trim_matches(['\'', '’']); - let n = w.chars().count(); - if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) { - return true; - } - // Check user dictionary first - if crate::in_user_dictionary(&suggest::turkish_to_lower(w)) || crate::in_user_dictionary(&w.to_lowercase()) { + // One letter, no letters, digits, capitals (Ignore words in UPPERCASE) or the user dictionary. + if crate::accepted_by_options(w, opts) || crate::in_user_dictionary(&suggest::turkish_to_lower(w)) { return true; } @@ -161,4 +157,15 @@ mod tests { assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct"); assert!(is_correct("kitapp2"), "direct checks keep the defaults"); } + + /// Ignore words in UPPERCASE reaches Turkish's own entry points too, so `NATO` and `TBMM` are + /// right by default and checked when the option is off (craft-libs#495 review). + #[test] + fn turkish_ignores_capitals_unless_told_otherwise() { + test_dict::install(); + let check_caps = crate::Options { ignore_uppercase: false, ..crate::Options::default() }; + assert!(is_correct("TBMMX") && is_correct_with_options("TBMMX", &crate::Options::default())); + assert!(!is_correct_with_options("TBMMX", &check_caps)); + assert!(is_correct_with_options("KİTAP", &check_caps), "a word in Turkish capitals is still looked up"); + } } diff --git a/crates/proof/src/tr/suggest.rs b/crates/proof/src/tr/suggest.rs index 08874420..0bb705b8 100644 --- a/crates/proof/src/tr/suggest.rs +++ b/crates/proof/src/tr/suggest.rs @@ -28,12 +28,9 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[ ("sarmısak", "sarımsak"), ("makina", "makine"), ("ünvan", "unvan"), - ("rastlantı", "rastlantı"), ("labaratuvar", "laboratuvar"), ("laboratuar", "laboratuvar"), - ("kravat", "kravat"), ("gardolap", "gardırop"), - ("egzoz", "egzoz"), ("egzozt", "egzoz"), ("egsoz", "egzoz"), ("gelcen", "geleceksin"), @@ -73,7 +70,7 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { // 1. Check direct common replacement table for &(wrong, right) in COMMON_REPLACEMENTS { if lower == wrong && seen.insert(right.to_string()) { - suggestions.push(right.to_string()); + suggestions.push(match_casing(word, right)); if suggestions.len() >= max { return suggestions; } @@ -88,7 +85,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { if !d.is_empty() { let variants = generate_diacritic_variants(&lower); for cand in variants { - if cand != lower && d.contains(&cand) && seen.insert(cand.clone()) { + if cand != lower + && let Some(cand) = in_dictionary(d, &cand) + && seen.insert(cand.clone()) + { suggestions.push(match_casing(word, &cand)); if suggestions.len() >= max { return suggestions; @@ -124,7 +124,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { let mut swapped = chars.clone(); swapped.swap(i, i + 1); let cand: String = swapped.into_iter().collect(); - if d.contains(&cand) && seen.insert(cand.clone()) { + if let Some(cand) = in_dictionary(d, &cand) + && seen.insert(cand.clone()) + { suggestions.push(match_casing(word, &cand)); if suggestions.len() >= max { return suggestions; @@ -144,7 +146,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { let mut deleted = chars.clone(); deleted.remove(i); let cand: String = deleted.into_iter().collect(); - if d.contains(&cand) && seen.insert(cand.clone()) { + if let Some(cand) = in_dictionary(d, &cand) + && seen.insert(cand.clone()) + { suggestions.push(match_casing(word, &cand)); if suggestions.len() >= max { return suggestions; @@ -161,7 +165,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { *target = c; } let cand: String = substituted.into_iter().collect(); - if d.contains(&cand) && seen.insert(cand.clone()) { + if let Some(cand) = in_dictionary(d, &cand) + && seen.insert(cand.clone()) + { suggestions.push(match_casing(word, &cand)); if suggestions.len() >= max { return suggestions; @@ -177,7 +183,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec { let mut inserted = chars.clone(); inserted.insert(i, c); let cand: String = inserted.into_iter().collect(); - if d.contains(&cand) && seen.insert(cand.clone()) { + if let Some(cand) = in_dictionary(d, &cand) + && seen.insert(cand.clone()) + { suggestions.push(match_casing(word, &cand)); if suggestions.len() >= max { return suggestions; @@ -258,10 +266,34 @@ pub fn turkish_to_lower(s: &str) -> String { out } +/// Turkish uppercase with dotted/dotless I support (`i` -> `İ`, `ı` -> `I`). +pub fn turkish_to_upper(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + match c { + 'i' => out.push('İ'), + 'ı' => out.push('I'), + other => out.extend(other.to_uppercase()), + } + } + out +} + +/// `cand` (lower case) as the dictionary writes it: itself, or capitalized for a name +/// (`İstanbul`), which a lower-case lookup doesn't find. +fn in_dictionary(d: &TrDict, cand: &str) -> Option { + if d.contains(cand) { + return Some(cand.to_string()); + } + let cap = super::grammar::turkish_capitalize(cand); + d.contains(&cap).then_some(cap) +} + /// Match original casing on suggestion. fn match_casing(orig: &str, cand: &str) -> String { - if orig.chars().all(char::is_uppercase) { - cand.to_uppercase() + let mut letters = orig.chars().filter(|c| c.is_alphabetic()).peekable(); + if letters.peek().is_some() && letters.all(char::is_uppercase) { + turkish_to_upper(cand) } else if orig.chars().next().is_some_and(char::is_uppercase) { super::grammar::turkish_capitalize(cand) } else { @@ -273,6 +305,19 @@ fn match_casing(orig: &str, cand: &str) -> String { mod tests { use super::*; + /// CodeRabbit on craft-libs#484/#489: a suggestion in capitals uses Turkish capitals + /// (`İ`, not `I`), and no table entry suggests the word itself. + #[test] + fn capitals_are_turkish_and_no_entry_suggests_itself() { + assert_eq!(match_casing("KİTAPP", "kitap"), "KİTAP"); + assert_eq!(match_casing("ISTANBUL", "istanbul"), "İSTANBUL"); + assert_eq!(match_casing("KAPI", "kapı"), "KAPI"); + assert_eq!(turkish_to_upper("ıiçşğüö"), "IİÇŞĞÜÖ"); + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nistanbul\n").unwrap(); + assert!(suggest_with(&d, "KİTAPP", 3).contains(&"KİTAP".to_string())); + assert!(COMMON_REPLACEMENTS.iter().all(|(a, b)| a != b)); + } + #[test] fn common_replacements_are_suggested() { let sugs = suggest("yalnış", 3); @@ -302,11 +347,24 @@ mod tests { assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed()); } + /// craft-libs#495 review: a name in the dictionary (`İstanbul`, `Ankara`) is found by the + /// diacritic and typo steps from a lower-case slip, in its dictionary case. + #[test] + fn capitalized_stems_are_suggested() { + let d = TrDict::from_hunspell("SET UTF-8\n", "2\nİstanbul\nAnkara\n").unwrap(); + assert!(suggest_with(&d, "istanbu", 3).contains(&"İstanbul".to_string()), "{:?}", suggest_with(&d, "istanbu", 3)); + assert!(suggest_with(&d, "ankraa", 3).contains(&"Ankara".to_string()), "{:?}", suggest_with(&d, "ankraa", 3)); + assert!(suggest_with(&d, "Istanbul", 3).contains(&"İstanbul".to_string()), "{:?}", suggest_with(&d, "Istanbul", 3)); + } + #[test] fn two_words_get_only_the_tables_replacement() { let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap(); assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]); - assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]); + // craft-libs#487 review: the table keeps the word's case. + assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["Birkaç".to_string()]); + assert_eq!(suggest_with(&d, "BİR KAÇ", 3), vec!["BİRKAÇ".to_string()]); + assert_eq!(suggest_with(&d, "Herşey", 3), vec!["Her şey".to_string()]); assert!(suggest_with(&d, "sorna sorna", 3).is_empty()); }