mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-08 19:42:12 +00:00
Merge pull request #495 from benandez/p6-coderabbit
craft-proof: fixes from CodeRabbit on the ported code
This commit is contained in:
27 files changed
+589
-210
No files matched your search
@@ -25,6 +25,33 @@ crates/proof/data/en-words.dic: public domain, the words of the Moby Hyphenator
|
||||
Ward (Project Gutenberg eBook #3204), without its break points.
|
||||
crates/proof/data/en-extra.txt: MIT OR Apache-2.0, WordCraft contributors' own list of modern
|
||||
English words (a copy of craft-i18n's language-data/en-us/en-extra.txt).
|
||||
craft-proof Language data, embedded at build time from craft-i18n's language-data/ when
|
||||
CRAFT_I18N_DIR is set (CRAFT_PROOF_EMBED picks the locales; by default English and the German,
|
||||
Polish, Portuguese, Italian and Russian data). Each file keeps its licence header; the
|
||||
licences are listed in craft-i18n's ATTRIBUTION.md, and a pattern file's notice ships with any
|
||||
application that embeds it. The Portuguese hyphenation patterns (language-data/pt/
|
||||
hyphenation.pat, hyph-utf8 hyph-pt 1.4) are under the BSD 3-clause licence:
|
||||
Copyright (C) 1987, 1994, 1996, 2015 Pedro J. de Rezende, 1996, 2015 J. Joao Dias Almeida,
|
||||
2024 Leonardo Araujo and Aline Benevides.
|
||||
Redistribution and use in source and binary forms, with or without modification, are
|
||||
permitted provided that the following conditions are met:
|
||||
* Redistributions of source code must retain the above copyright notice, this list of
|
||||
conditions and the following disclaimer.
|
||||
* Redistributions in binary form must reproduce the above copyright notice, this list of
|
||||
conditions and the following disclaimer in the documentation and/or other materials
|
||||
provided with the distribution.
|
||||
* Neither the name of the University of Campinas, of the University of Minho nor the names of
|
||||
its contributors may be used to endorse or promote products derived from this software
|
||||
without specific prior written permission.
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR
|
||||
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
|
||||
FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL PEDRO J. DE REZENDE OR J.JOAO
|
||||
DIAS ALMEIDA BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
|
||||
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
crates/text/assets/hyphen/en-us.pat: CC0-1.0, trained from en-us.dic by DesignCraft's trainer.
|
||||
Hyphenation patterns for other languages, embedded through the hypher crate only when the
|
||||
matching craft-text feature is enabled (none are by default). They come from hyph-utf8,
|
||||
|
||||
@@ -88,8 +88,10 @@ craft-<name> = { git = "https://github.com/storytold/craft-libs", rev = "<commit
|
||||
- [`craft-print`](crates/print/README.md): PDF submission to CUPS and Windows, printer discovery,
|
||||
duplex/copies/colour and driver preferences. PDF generation stays in each app.
|
||||
- [`craft-proof`](crates/proof/README.md): proofing per language (spelling, suggestions, grammar,
|
||||
hyphenation, a Hunspell reader, user words). The word lists and patterns are craft-i18n's
|
||||
`language-data/`, embedded at build time from `CRAFT_I18N_DIR`.
|
||||
hyphenation, a Hunspell reader, user words, Language packs). The word lists and patterns are
|
||||
craft-i18n's `language-data/`, embedded at build time from `CRAFT_I18N_DIR`: English and the
|
||||
carry-over locales by default, the set `CRAFT_PROOF_EMBED` names otherwise (`en` for English
|
||||
only); other languages come from installed Language packs.
|
||||
|
||||
- [`craft-quit`](crates/quit/src/lib.rs): native macOS quit, egui close handling and an optional
|
||||
app menu. Unsaved-document decisions and save prompts stay in each app.
|
||||
|
||||
@@ -29,6 +29,7 @@ use craft_proof::{ProofLang, check_text_with, suggest_in, Options};
|
||||
use craft_proof::hyphen::{hyphen_points_in, Limits};
|
||||
|
||||
// A paragraph whose runs are in different languages (byte ranges of `text`).
|
||||
let text = "The colour of the hous.";
|
||||
let issues = check_text_with(text, &[(0..text.len(), ProofLang::EnGb)], &Options::default(), true, false);
|
||||
let fixes = suggest_in("colour", 5, ProofLang::En);
|
||||
let breaks = hyphen_points_in("hyphenation", &Limits::default(), ProofLang::En);
|
||||
@@ -166,8 +167,8 @@ English spelling uses the first of these that is there; `english_source(lang)` s
|
||||
4. **System Hunspell:** the English's own pair in the system locations. Never another English's:
|
||||
an `en_US` dictionary would mark British spellings, so en-GB and en-CA without their own use
|
||||
the built-in list, which has both spellings. Each name is looked for in every folder before the
|
||||
next name; a file that can't be read is skipped for the next name or folder; one file is read
|
||||
once for every English that uses it. `COMPOUNDRULE`, `ONLYINCOMPOUND` and `COMPOUNDMIN`
|
||||
next name; a file that can't be read is skipped for the next name or folder, up to the first eight
|
||||
candidates (the rest aren't tried); one file is read once for every English that uses it. `COMPOUNDRULE`, `ONLYINCOMPOUND` and `COMPOUNDMIN`
|
||||
(ordinals such as `21st`) are read; flag-based compounding (`COMPOUNDFLAG`…) is ignored for
|
||||
English rather than refused; `OCONV` writes suggestions as the dictionary says (`don’t`), as for
|
||||
every language. None on the web.
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//! README): every word of the public-domain Moby Hyphenator II list, without its break points.
|
||||
//!
|
||||
//! ```sh
|
||||
//! cargo run -p craft-proof --example en_words -- <craft-i18n>/language-data/en-us/en-us.dic data/en-words.dic
|
||||
//! cargo run -p craft-proof --example en_words -- <craft-i18n>/language-data/en-us/en-us.dic crates/proof/data/en-words.dic
|
||||
//! ```
|
||||
//!
|
||||
//! The input is craft-i18n's `en-us.dic`: the Moby Hyphenator II list by Grady Ward (Project
|
||||
|
||||
@@ -55,15 +55,17 @@ const MAX_INPUT_BYTES: usize = 4 * MAX_WORD_BYTES;
|
||||
const MAX_CONV_PAIRS: usize = 1 << 16;
|
||||
/// Longest text on either side of an `ICONV` or `OCONV` pair, in bytes.
|
||||
const MAX_CONV_BYTES: usize = 64;
|
||||
/// Most flags one `.dic` entry or one continuation class field may carry. Real dictionaries use a
|
||||
/// few hundred at most (Belarusian: 264); a longer field is hostile and is refused (`.aff`) or
|
||||
/// skipped (`.dic`, though its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word).
|
||||
pub const MAX_FLAGS_PER_ENTRY: usize = 1024;
|
||||
/// Most flags one `.dic` entry or one continuation class field may carry. LibreOffice's Turkish
|
||||
/// dictionary has 920 entries over 1,024 flags (`abart`, `savaş`: up to 3,926), so the cap sits
|
||||
/// well above that; a longer field is hostile and is refused (`.aff`) or skipped (`.dic`, though
|
||||
/// its `FORBIDDENWORD` and `NOSUGGEST` flags still apply to the word). What all the flags of a
|
||||
/// dictionary may cost together is bounded by `MAX_SET_DATA`.
|
||||
pub const MAX_FLAGS_PER_ENTRY: usize = 8192;
|
||||
/// Most flags one stem may carry once its homonyms (the same word on several `.dic` lines) are
|
||||
/// merged, and so most flags a stem line of the compact format may carry. Larger than
|
||||
/// [`MAX_FLAGS_PER_ENTRY`] because merged sets aren't `.dic` fields; a homonym that would take the
|
||||
/// set past it adds only its `FORBIDDENWORD` and `NOSUGGEST` flags, as an exclusion is never lost.
|
||||
pub const MAX_FLAGS_PER_STEM: usize = 8 * MAX_FLAGS_PER_ENTRY;
|
||||
pub const MAX_FLAGS_PER_STEM: usize = 4 * MAX_FLAGS_PER_ENTRY;
|
||||
/// Most `AF` aliases, and most bytes of alias text in all: each alias is read once and shared by
|
||||
/// the stems using it, so these bound the flags an `AF` table can make a dictionary hold
|
||||
/// (the largest Language pack's table: 1,596 aliases, 19 KB).
|
||||
@@ -72,6 +74,7 @@ const MAX_ALIAS_BYTES: usize = 4 << 20;
|
||||
/// Most flags all distinct flag sets (stems' and continuation classes') may hold together: a lookup
|
||||
/// tests each distinct continuation set it meets at most once, so this bounds its cost.
|
||||
const MAX_SET_DATA: usize = 1 << 23;
|
||||
|
||||
/// Most distinct flags a dictionary may use: each gets a `u16` id.
|
||||
const MAX_DISTINCT_FLAGS: usize = 1 << 16;
|
||||
|
||||
|
||||
@@ -201,25 +201,32 @@ fn alternatives(c: char, map: &[Vec<char>]) -> Vec<char> {
|
||||
}
|
||||
|
||||
/// Every way of changing up to `t.max_map_changes` letters of `chars` to a MAP relative (`alts`,
|
||||
/// letter by letter), with its cost, each written after `buf`.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, changes: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS {
|
||||
/// letter by letter), with its cost: all one-change variants first, then two, and so on, so the
|
||||
/// [`MAX_MAP_VARIANTS`] cap drops the variants with the most changes.
|
||||
fn map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, out: &mut Vec<(String, u32)>) {
|
||||
let mut buf = Vec::with_capacity(chars.len());
|
||||
for n in 1..=t.max_map_changes.min(chars.len()) {
|
||||
exact_map_variants(chars, alts, t, n, 0, &mut buf, out);
|
||||
}
|
||||
}
|
||||
|
||||
/// [`map_variants`] with exactly `left` more changes, each written after `buf`.
|
||||
fn exact_map_variants(chars: &[char], alts: &[Vec<char>], t: &Tuning, left: usize, cost: u32, buf: &mut Vec<char>, out: &mut Vec<(String, u32)>) {
|
||||
if out.len() >= MAX_MAP_VARIANTS || left > chars.len() {
|
||||
return;
|
||||
}
|
||||
let Some((&c, rest)) = chars.split_first() else {
|
||||
if changes > 0 {
|
||||
out.push((buf.iter().collect(), cost));
|
||||
}
|
||||
out.push((buf.iter().collect(), cost));
|
||||
return;
|
||||
};
|
||||
let (here, later) = alts.split_first().map_or((&[][..], &[][..]), |(h, l)| (h.as_slice(), l));
|
||||
buf.push(c);
|
||||
map_variants(rest, later, t, changes, cost, buf, out);
|
||||
exact_map_variants(rest, later, t, left, cost, buf, out);
|
||||
buf.pop();
|
||||
if changes < t.max_map_changes {
|
||||
if left > 0 {
|
||||
for &a in here {
|
||||
buf.push(a);
|
||||
map_variants(rest, later, t, changes + 1, cost + (t.map_cost)(c, a), buf, out);
|
||||
exact_map_variants(rest, later, t, left - 1, cost + (t.map_cost)(c, a), buf, out);
|
||||
buf.pop();
|
||||
}
|
||||
}
|
||||
@@ -368,7 +375,7 @@ fn add_case(col: &mut Collector, word: &str, input: &str) {
|
||||
fn add_map_slips(col: &mut Collector, chars: &[char], map: &[Vec<char>], t: &Tuning) {
|
||||
let alts: Vec<Vec<char>> = chars.iter().map(|&c| alternatives(c, map)).collect();
|
||||
let mut variants = Vec::new();
|
||||
map_variants(chars, &alts, t, 0, 0, &mut Vec::with_capacity(chars.len()), &mut variants);
|
||||
map_variants(chars, &alts, t, &mut variants);
|
||||
variants.sort_by_key(|v| v.1);
|
||||
for (v, cost) in &variants {
|
||||
col.offer(v, *cost);
|
||||
@@ -535,6 +542,22 @@ fn scan(col: &mut Collector, chars: &[char], map: &[Vec<char>]) {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// craft-libs#487 review: MAP variants are made by number of changes, so when a long word has
|
||||
/// more than [`MAX_MAP_VARIANTS`] of them the cap drops the most-changed ones, never a single
|
||||
/// slip (here the first letter's, which depth-first order reached last).
|
||||
#[test]
|
||||
fn the_map_cap_keeps_the_fewest_changes() {
|
||||
let chars: Vec<char> = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".chars().collect();
|
||||
let alts = vec!["áàâäãåā".chars().collect::<Vec<char>>(); chars.len()];
|
||||
let mut out = Vec::new();
|
||||
map_variants(&chars, &alts, &Tuning::DEFAULT, &mut out);
|
||||
assert_eq!(out.len(), MAX_MAP_VARIANTS);
|
||||
let changed = |v: &str| v.chars().filter(|&c| c != 'a').count();
|
||||
assert_eq!(out.iter().filter(|(v, _)| changed(v) == 1).count(), 30 * 7, "every single slip is kept");
|
||||
assert!(out.iter().any(|(v, _)| v.starts_with('á') && changed(v) == 1));
|
||||
assert!(out.windows(2).all(|w| changed(&w[0].0) <= changed(&w[1].0)), "fewest changes first");
|
||||
}
|
||||
|
||||
/// craft-libs#490 review: a suggestion recased to capitals keeps `ß` (`STRAßE`, right with or
|
||||
/// without CHECKSHARPS); under CHECKSHARPS the SS form comes first, and a KEEPCASE `ß` word may
|
||||
/// be capitalized or written in capitals with SS. Expected lists are the hunspell 1.7.5 CLI's
|
||||
|
||||
@@ -15,11 +15,19 @@ pub fn check(text: &str) -> Vec<Issue> {
|
||||
issues
|
||||
}
|
||||
|
||||
/// Words Czech repeats on purpose (`ne ne`, `ano ano`, `už už`, `to to`).
|
||||
const ALLOWED_REPEATS: &[&str] = &["ne", "ano", "už", "to", "jo", "no", "tak", "pryč", "dál", "pomalu", "honem"];
|
||||
|
||||
fn repeated_words(text: &str, ws: &[(usize, usize)], v: &mut Vec<Issue>) {
|
||||
for pair in ws.windows(2) {
|
||||
let [(a0, b0), (a1, b1)] = pair else { continue };
|
||||
let (Some(w0), Some(w1), Some(gap)) = (text.get(*a0..*b0), text.get(*a1..*b1), text.get(*b0..*a1)) else { continue };
|
||||
if !gap.is_empty() && gap.chars().all(char::is_whitespace) && w0.chars().all(char::is_alphabetic) && w0.to_lowercase() == w1.to_lowercase() {
|
||||
if !gap.is_empty()
|
||||
&& gap.chars().all(char::is_whitespace)
|
||||
&& w0.chars().all(char::is_alphabetic)
|
||||
&& w0.to_lowercase() == w1.to_lowercase()
|
||||
&& !ALLOWED_REPEATS.contains(&w0.to_lowercase().as_str())
|
||||
{
|
||||
v.push(issue(*a0, *b1, format!("Opakované slovo: „{w1}“"), vec![w0.to_string()]));
|
||||
}
|
||||
}
|
||||
@@ -66,6 +74,15 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// craft-libs#495 review: intentional repeats aren't marked; others still are.
|
||||
#[test]
|
||||
fn intentional_repeats_are_fine() {
|
||||
for text in ["Ne ne, to nechci.", "Ano ano, přijdu.", "Už už odcházel.", "To to bylo krásné."] {
|
||||
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
|
||||
}
|
||||
assert_eq!(check("Byl byl doma.").len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_words_and_spaces_have_czech_messages() {
|
||||
let text = "Město město , škola.";
|
||||
|
||||
@@ -98,7 +98,7 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
':' | '!' | '?' => !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
|
||||
@@ -105,16 +105,7 @@ fn search_tiers(lo_exts: Option<&[String]>) -> (Vec<PathBuf>, Vec<PathBuf>) {
|
||||
system.extend(lo_dirs(&base.join("LibreOffice").join("share").join("extensions").to_string_lossy()));
|
||||
}
|
||||
}
|
||||
// If LibreOffice is installed on another drive (e.g. D:\Program Files\LibreOffice), check drives C..Z
|
||||
for drive in b'C'..=b'Z' {
|
||||
let drive_char = drive as char;
|
||||
for pf in ["Program Files", "Program Files (x86)"] {
|
||||
let lo_base = PathBuf::from(format!(r"{drive_char}:\{pf}\LibreOffice\share\extensions"));
|
||||
if lo_base.is_dir() {
|
||||
system.extend(lo_dirs(&lo_base.to_string_lossy()));
|
||||
}
|
||||
}
|
||||
}
|
||||
system.extend(lo_drive_dirs(lo_exts, || LO_DRIVE_BASES.get_or_init(|| lo_drive_bases(Path::is_dir))));
|
||||
} else if cfg!(target_os = "macos") {
|
||||
if let Some(h) = &home {
|
||||
// An app started from Finder or the Dock doesn't see `WORDCRAFT_DICTIONARIES`; this
|
||||
@@ -144,6 +135,26 @@ fn search_tiers(lo_exts: Option<&[String]>) -> (Vec<PathBuf>, Vec<PathBuf>) {
|
||||
(dirs, system)
|
||||
}
|
||||
|
||||
/// LibreOffice's language folders (`<drive>:\Program Files\LibreOffice\share\extensions\<ext>`)
|
||||
/// on the drives C to Z where LibreOffice is installed (`bases`). No drive is probed without
|
||||
/// `lo_exts` (no LibreOffice folders are wanted then).
|
||||
fn lo_drive_dirs<'a>(lo_exts: Option<&[String]>, bases: impl FnOnce() -> &'a [PathBuf]) -> Vec<PathBuf> {
|
||||
let Some(exts) = lo_exts else { return Vec::new() };
|
||||
bases().iter().flat_map(|base| exts.iter().map(|e| base.join(e))).collect()
|
||||
}
|
||||
|
||||
/// The `LibreOffice\share\extensions` folders on drives C to Z (48 probes).
|
||||
fn lo_drive_bases(is_dir: impl Fn(&Path) -> bool) -> Vec<PathBuf> {
|
||||
('C'..='Z')
|
||||
.flat_map(|drive| ["Program Files", "Program Files (x86)"].map(|pf| PathBuf::from(format!(r"{drive}:\{pf}\LibreOffice\share\extensions"))))
|
||||
.filter(|base| is_dir(base))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// [`lo_drive_bases`], probed once per process.
|
||||
// ponytail: a LibreOffice installed while the app runs is seen after a restart.
|
||||
static LO_DRIVE_BASES: std::sync::OnceLock<Vec<PathBuf>> = std::sync::OnceLock::new();
|
||||
|
||||
/// The user's data folder on Linux and the BSDs: `$XDG_DATA_HOME` when it is an absolute path (the
|
||||
/// XDG Base Directory spec ignores a relative one), else `~/.local/share`.
|
||||
fn xdg_data_home(xdg: Option<std::ffi::OsString>, home: Option<&Path>) -> Option<PathBuf> {
|
||||
@@ -610,6 +621,31 @@ fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// CodeRabbit on craft-libs#489: without LibreOffice extension names no drive is probed; with
|
||||
/// them, only drives where LibreOffice is found give folders.
|
||||
#[test]
|
||||
fn libreoffice_drives_are_probed_only_when_wanted() {
|
||||
let probes = std::cell::Cell::new(0);
|
||||
let count = |_: &Path| {
|
||||
probes.set(probes.get() + 1);
|
||||
false
|
||||
};
|
||||
// craft-libs#495 review: the drives are probed once, not on every lookup.
|
||||
let cache = std::sync::OnceLock::new();
|
||||
let cached = || cache.get_or_init(|| lo_drive_bases(count)).as_slice();
|
||||
assert!(lo_drive_dirs(None, cached).is_empty());
|
||||
assert_eq!(probes.get(), 0);
|
||||
let exts = ["dict-tr".to_string()];
|
||||
assert!(lo_drive_dirs(Some(&exts), cached).is_empty());
|
||||
assert!(lo_drive_dirs(Some(&exts), cached).is_empty());
|
||||
assert_eq!(probes.get(), 48);
|
||||
let found = lo_drive_bases(|p: &Path| p.to_string_lossy().starts_with(r"D:\Program Files\"));
|
||||
assert_eq!(
|
||||
lo_drive_dirs(Some(&exts), || found.as_slice()),
|
||||
[PathBuf::from(r"D:\Program Files\LibreOffice\share\extensions").join("dict-tr")]
|
||||
);
|
||||
}
|
||||
|
||||
/// Another region's Language pack never beats the system's dictionary of the right region
|
||||
/// (`pt-pt` pack vs system `pt_BR`; `de-ch` vs `de_DE`; `en-gb` vs `en_US`), but beats the
|
||||
/// system's other names; the right region's pack beats the system; the dictionaries folder
|
||||
|
||||
@@ -148,8 +148,16 @@ fn core_breaks(core: &[char], lang: ProofLang) -> (Vec<usize>, Source) {
|
||||
if lang == ProofLang::Bg && core.iter().all(|&c| crate::bg::is_letter(c)) {
|
||||
return (crate::bg::points(core), Source::Rules);
|
||||
}
|
||||
if lang == ProofLang::Tr && core.iter().all(|&c| crate::tr::hyphen::is_letter(c)) {
|
||||
return (crate::tr::hyphen::points(core), Source::Rules);
|
||||
// A suffix written after an apostrophe (`Ankara'dan`, `Türkiye'nin`) is hyphenated part by
|
||||
// part; no break goes next to the apostrophe ([`word_breaks_in`] drops those).
|
||||
if lang == ProofLang::Tr && core.iter().all(|&c| c == '\'' || crate::tr::hyphen::is_letter(c)) {
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
for part in core.split(|&c| c == '\'') {
|
||||
out.extend(crate::tr::hyphen::points(part).into_iter().map(|p| p + start));
|
||||
start += part.len() + 1;
|
||||
}
|
||||
return (out, Source::Rules);
|
||||
}
|
||||
match lang {
|
||||
ProofLang::Pt => return core_breaks_patterns(Patterns::pt(), core),
|
||||
@@ -526,6 +534,18 @@ fn heuristic_points_pl(c: &[char]) -> Vec<usize> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// CodeRabbit on craft-libs#484: a Turkish word with a suffix after an apostrophe is
|
||||
/// hyphenated by the Turkish rules, part by part, never next to the apostrophe.
|
||||
#[test]
|
||||
fn turkish_apostrophe_suffixes_follow_turkish_rules() {
|
||||
let tr = |w: &str| hyphenate_word_in(w, &Limits::default(), ProofLang::Tr);
|
||||
assert_eq!(tr("Ankara'dan"), "An-ka-ra'dan");
|
||||
assert_eq!(tr("Türkiye'nin"), "Tür-ki-ye'nin");
|
||||
assert_eq!(tr("İstanbul'daki"), "İs-tan-bul'da-ki");
|
||||
assert_eq!(word_breaks_in("Ankara'dan", ProofLang::Tr).1, Source::Rules);
|
||||
assert_eq!(tr("Ankara"), "An-ka-ra", "no apostrophe: as before");
|
||||
}
|
||||
|
||||
fn h(w: &str) -> String {
|
||||
hyphenate_word(w, &Limits::default())
|
||||
}
|
||||
|
||||
@@ -44,6 +44,9 @@ const NOT_FEMININE_IN_A: &[&str] = &["acrobata", "autodidatta", "alfa", "omega",
|
||||
/// Abbreviations written with a final period after which a sentence goes on in lower case
|
||||
/// (lower case, without the period).
|
||||
const ABBREVIATIONS: &[&str] = &[
|
||||
"pp",
|
||||
"tab",
|
||||
"dr",
|
||||
"ecc",
|
||||
"es",
|
||||
"dott",
|
||||
@@ -193,7 +196,7 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
':' | '!' | '?' => !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
@@ -210,13 +213,15 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
|
||||
fn sentence_capitals(text: &str, ws: &[(usize, usize)], capital_first: bool, v: &mut Vec<Issue>) {
|
||||
let mut after_end = capital_first;
|
||||
for (a, b) in ws {
|
||||
for (i, (a, b)) in ws.iter().enumerate() {
|
||||
let Some(w) = text.get(*a..*b) else { continue };
|
||||
if after_end && w.chars().next().is_some_and(char::is_lowercase) && w.chars().all(char::is_alphabetic) {
|
||||
let fixed: String = w.chars().next().map(|f| f.to_uppercase().chain(w.chars().skip(1)).collect()).unwrap_or_default();
|
||||
v.push(issue(*a, *b, "Maiuscola a inizio frase".into(), vec![fixed]));
|
||||
}
|
||||
after_end = ends_sentence(text.get(*b..).unwrap_or(""), w);
|
||||
// Only the gap before the next word: the rest of the paragraph never ends a sentence.
|
||||
let next = ws.get(i + 1).map_or(text.len(), |n| n.0);
|
||||
after_end = ends_sentence(text.get(*b..next).unwrap_or(""), w);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -384,6 +389,20 @@ mod tests {
|
||||
(a.to_string(), b.to_string())
|
||||
}
|
||||
|
||||
/// CodeRabbit on craft-libs#484: a sentence after the first one starts with a capital too.
|
||||
#[test]
|
||||
fn later_sentences_start_with_a_capital() {
|
||||
assert!(fixes("Vado a Roma. il cane dorme.").contains(&pair("il", "Il")));
|
||||
assert!(fixes("Vado a Roma? sì.").contains(&pair("sì", "Sì")));
|
||||
assert!(!fixes("Vado a Roma. Il cane dorme.").iter().any(|(_, b)| b == "Il"));
|
||||
for t in ["Il sig. rossi arriva.", "Vedi pp. dieci e oltre.", "Nella tab. due.", "Il dr. bianchi arriva.", "Mele, pere ecc. sono frutti."] {
|
||||
assert!(
|
||||
!fixes(t).iter().any(|(_, b)| b.starts_with(char::is_uppercase) && b != "Il" && b != "Vedi" && b != "Nella" && b != "Mele"),
|
||||
"after an abbreviation: {t}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn po_takes_an_apostrophe() {
|
||||
assert_eq!(fixes("Dammi un pò di pane"), [pair("pò", "po'")]);
|
||||
|
||||
@@ -95,8 +95,9 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
if TRUNCATIONS.contains(&lower.as_str()) || left_to_grammar(&lower) {
|
||||
return true;
|
||||
}
|
||||
// The whole word first, as Hunspell does: a hyphenated entry or user word.
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
if let Some((head, tail)) = w.split_once(['\'', '’'])
|
||||
&& PROCLITICS.contains(&head.to_lowercase().as_str())
|
||||
@@ -139,6 +140,14 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// craft-libs#487 review: a hyphenated dictionary entry is a word even when its parts aren't.
|
||||
#[test]
|
||||
fn a_hyphenated_entry_is_looked_up_whole() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
|
||||
assert!(is_correct_with(&d, "tic-tac"));
|
||||
assert!(!is_correct_with(&d, "tic-tacc"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_dictionary_nothing_is_flagged() {
|
||||
let none = AffixDict::default();
|
||||
|
||||
@@ -23,9 +23,8 @@ pub enum ProofLang {
|
||||
/// Polish (`pl`, `pl-PL`): the installed Hunspell dictionary (when there is one), the Polish grammar rules and the Polish
|
||||
/// hyphenation patterns.
|
||||
Pl,
|
||||
/// Bulgarian (`bg`, `bg-BG`): hyphenation by the Bulgarian syllable rules ([`crate::bg`]).
|
||||
/// There is no Bulgarian dictionary or grammar yet, so, as for [`ProofLang::Other`], no
|
||||
/// spelling or grammar marks.
|
||||
/// Bulgarian (`bg`, `bg-BG`): hyphenation by the Bulgarian syllable rules ([`crate::bg`]), and
|
||||
/// spelling when a Bulgarian Hunspell dictionary or `bg` pack is found. No grammar rules yet.
|
||||
Bg,
|
||||
/// Portuguese (`pt`, `pt-BR`; `pt-PT` is [`ProofLang::Pack`] `pt-pt`, with its own dictionary
|
||||
/// and these grammar rules and patterns): the installed Hunspell dictionary (VERO and
|
||||
@@ -69,8 +68,8 @@ pub enum ProofLang {
|
||||
EsUy,
|
||||
/// German (`de`, `de-DE`, `de-AT`…; `de-CH` and `de-LI` are [`ProofLang::Pack`] `de-ch`, with
|
||||
/// their own dictionary and these patterns): hyphenation by the German patterns (reformed
|
||||
/// spelling). There is no German dictionary or grammar yet, so, as for [`ProofLang::Other`],
|
||||
/// no spelling or grammar marks.
|
||||
/// spelling), and spelling when a German Hunspell dictionary or `de` pack is found. No grammar
|
||||
/// rules yet.
|
||||
De,
|
||||
/// Any other language, by its tag (`nl`, `fr-FR`, `sr-Latn`): spelling and suggestions from
|
||||
/// its Hunspell dictionary (the dictionaries folder, an installed Language pack, the system's)
|
||||
@@ -112,7 +111,7 @@ impl ProofLang {
|
||||
/// `x-none`, `und`, `zxx`, `mul`, `mis` and malformed tags are [`ProofLang::Other`].
|
||||
pub fn from_tag(tag: Option<&str>) -> ProofLang {
|
||||
let Some(tag) = tag.map(str::trim).filter(|t| !t.is_empty()) else { return ProofLang::En };
|
||||
let primary = tag.split(['-', '_']).next().unwrap_or("");
|
||||
let primary = tag.split(['-', '_', '.', '@']).next().unwrap_or("");
|
||||
if primary.eq_ignore_ascii_case("en") || primary.eq_ignore_ascii_case("eng") {
|
||||
ProofLang::english(tag)
|
||||
} else if primary.eq_ignore_ascii_case("pl") || primary.eq_ignore_ascii_case("pol") {
|
||||
@@ -148,6 +147,9 @@ impl ProofLang {
|
||||
} else {
|
||||
ProofLang::De
|
||||
}
|
||||
} else if tag.split_once('@').is_some_and(|(_, m)| m.eq_ignore_ascii_case("latin")) {
|
||||
// POSIX `sr_RS@latin`: the Latin-script language (`sr-latn`).
|
||||
PackLocale::new(&format!("{primary}-latn")).map_or(ProofLang::Other, ProofLang::Pack)
|
||||
} else {
|
||||
PackLocale::new(tag).map_or(ProofLang::Other, ProofLang::Pack)
|
||||
}
|
||||
@@ -443,7 +445,13 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn tags_resolve_primary_languages() {
|
||||
assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol"], ProofLang::Pl);
|
||||
assert_tags(&["pl", "pl-PL", "PL-pl", "pl_PL", "pl_PL.UTF-8", " pl-PL ", "pol", "pl@euro", "pl.UTF-8"], ProofLang::Pl);
|
||||
// craft-libs#482 review: a POSIX charset or modifier with no region.
|
||||
assert_tags(&["en.UTF-8", "en@euro"], ProofLang::En);
|
||||
assert_tags(&["de.UTF-8"], ProofLang::De);
|
||||
let sr_latn = ProofLang::Pack(PackLocale::new("sr-latn").unwrap());
|
||||
assert_tags(&["sr@latin", "sr_RS@latin", "sr_RS.UTF-8@latin", "sr-Latn"], sr_latn);
|
||||
assert_tags(&["sr", "sr@cyrillic"], ProofLang::Pack(PackLocale::new("sr").unwrap()));
|
||||
assert_tags(&["pt", "pt-BR", "PT_br", "pt-BR.UTF-8", "pt-AO", "por"], ProofLang::Pt);
|
||||
let pt_pt = ProofLang::Pack(PackLocale::new("pt-pt").unwrap());
|
||||
assert_tags(&[" pt-PT ", "pt_PT.UTF-8", "por-PT"], pt_pt);
|
||||
@@ -499,8 +507,11 @@ mod tests {
|
||||
assert_tags(&["de-CH", "de_CH.UTF-8", "de-LI"], de_ch);
|
||||
assert_eq!(de_ch.built_in(), ProofLang::De);
|
||||
assert_tags(&["ru", "ru-RU", "RU-ru", "ru_RU", "ru_RU.UTF-8", " ru-RU ", "rus"], ProofLang::Ru);
|
||||
assert!(!ProofLang::De.is_proofed());
|
||||
assert!(!ProofLang::Bg.is_proofed());
|
||||
// German and Bulgarian are proofed when a dictionary is found (craft-libs#485 review: an
|
||||
// installed hunspell-de-de must not fail this test), so only their locales are fixed here.
|
||||
assert_eq!(ProofLang::De.spelling_locale(), PackLocale::new("de-de"));
|
||||
assert_eq!(ProofLang::Bg.spelling_locale(), PackLocale::new("bg-bg"));
|
||||
assert!(!ProofLang::PROOFED.contains(&ProofLang::De) && !ProofLang::PROOFED.contains(&ProofLang::Bg));
|
||||
assert_eq!(ProofLang::Bg.code(), "bg");
|
||||
assert!(ProofLang::Pt.is_proofed() && !ProofLang::Other.is_proofed());
|
||||
assert!(ProofLang::Tr.is_proofed());
|
||||
|
||||
@@ -675,6 +675,13 @@ pub(crate) fn accepted_by_options(word: &str, opts: &Options) -> bool {
|
||||
|| in_user_dictionary(&w.to_lowercase())
|
||||
}
|
||||
|
||||
/// Is the space at byte `space` of `text` the first of a score or ratio's `3 : 1`, which keeps
|
||||
/// its spaces (the grammar rules' space-before-punctuation check)?
|
||||
pub(crate) fn score_colon(text: &str, space: usize) -> bool {
|
||||
text.get(..space).and_then(|t| t.chars().next_back()).is_some_and(|c| c.is_ascii_digit())
|
||||
&& text.get(space..).is_some_and(|t| t.starts_with(" : ") && t.get(3..).and_then(|r| r.chars().next()).is_some_and(|c| c.is_ascii_digit()))
|
||||
}
|
||||
|
||||
/// [`is_correct`] with the File › Options › Proofing choices: English with no dialect, so a word
|
||||
/// any English spells so is right (`color` and `colour`; American English's dictionary when it
|
||||
/// isn't SCOWL: [`english_source`]), without the dialect policy.
|
||||
@@ -1930,6 +1937,45 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// craft-libs#485 review: a licence file that is too big is skipped alone; the pack's other
|
||||
/// licences and the next pack's are still shown.
|
||||
#[test]
|
||||
fn an_oversized_licence_skips_only_itself() {
|
||||
crate::hold_dictionary_tests();
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let root = std::env::temp_dir().join(format!("wc-proof-licence-skip-{}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
for l in ["qa", "qb"] {
|
||||
let dir = write_pack(&root, l, &[]);
|
||||
std::fs::write(dir.join("LICENCES").join("A-HUGE"), vec![b'x'; (4 << 20) + 1]).unwrap();
|
||||
std::fs::write(dir.join("LICENCES").join("COPYING"), format!("licence {l}")).unwrap();
|
||||
}
|
||||
packs::set_packs_dir(Some(root.clone()));
|
||||
let texts = packs::licence_texts();
|
||||
let shown: Vec<&str> = texts.iter().filter(|(t, _)| t.contains("(q")).map(|(_, x)| x.as_str()).collect();
|
||||
assert_eq!(shown, ["licence qa", "licence qb"]);
|
||||
packs::set_packs_dir(None);
|
||||
set_dictionaries_folder(Some(shared_test_folder()));
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// craft-libs#483 and #488 reviews: a score keeps its spaces (`3 : 1`) in every language's
|
||||
/// space-before-punctuation rule, and in Polish and Russian ` ,,` (a typed opening quote) is
|
||||
/// left to the doubled-punctuation rule alone.
|
||||
#[test]
|
||||
fn scores_and_typed_quotes_keep_their_spaces() {
|
||||
let space_marks = |text: &str, lang| check_grammar_in(text, lang).into_iter().filter(|i| text.get(i.start..i.start + 1) == Some(" ")).count();
|
||||
for lang in [ProofLang::Ru, ProofLang::Pl, ProofLang::Hr, ProofLang::It, ProofLang::Pt] {
|
||||
assert_eq!(space_marks("Wynik 3 : 1 dzisiaj.", lang), 0, "{lang:?}");
|
||||
assert_eq!(space_marks("Wynik 3 : dzisiaj.", lang), 1, "{lang:?}: not a score");
|
||||
}
|
||||
for (text, lang) in [("Powiedział ,,tak” wczoraj.", ProofLang::Pl), ("Он сказал ,,да” вчера.", ProofLang::Ru)] {
|
||||
let issues = check_grammar_in(text, lang);
|
||||
assert_eq!(space_marks(text, lang), 0, "{issues:?}");
|
||||
assert!(issues.iter().any(|i| text.get(i.start..i.end) == Some(",,")), "the quote rule still marks it: {issues:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Pack languages for the cache tests: `qa`, `qb`… each knowing `word<locale>`.
|
||||
fn many_packs(root: &std::path::Path, n: usize) -> Vec<ProofLang> {
|
||||
(0..n)
|
||||
|
||||
@@ -64,7 +64,15 @@ fn nothing_is_embedded() {
|
||||
fn english_spelling_still_marks_misspellings() {
|
||||
crate::hold_dictionary_tests();
|
||||
for l in [ProofLang::En, ProofLang::EnGb, ProofLang::EnCa] {
|
||||
assert!(matches!(crate::english_source(l), Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}");
|
||||
let source = crate::english_source(l);
|
||||
assert!(matches!(source, Some(EnglishSource::Hunspell(_) | EnglishSource::CoreList)), "{l:?}");
|
||||
// Any English dictionary marks junk and knows `the`; the word-by-word results below are
|
||||
// the built-in list's, so an installed dictionary (with `teh`, without `because`) is
|
||||
// not held to them (craft-libs#482 and #487 reviews).
|
||||
assert!(!crate::is_correct_in("zxqvbnwk", l) && crate::is_correct_in("the", l), "{l:?}");
|
||||
if !matches!(source, Some(EnglishSource::CoreList)) {
|
||||
continue;
|
||||
}
|
||||
assert!(!crate::is_correct_in("becuase", l) && !crate::is_correct_in("teh", l), "{l:?}");
|
||||
assert!(crate::suggest_in("teh", 5, l).contains(&"the".to_string()), "{l:?}");
|
||||
assert!(crate::suggest_in("becuase", 5, l).contains(&"because".to_string()), "{l:?}");
|
||||
@@ -73,11 +81,15 @@ fn english_spelling_still_marks_misspellings() {
|
||||
}
|
||||
assert_eq!(crate::check_spelling_in("Becuase the house of the people", l).len(), 1, "{l:?}");
|
||||
}
|
||||
assert!(
|
||||
!crate::is_correct("becuase") && !crate::is_correct_with("becuase", &Options::default()) && !crate::is_correct_for("becuase", Some("en-GB"))
|
||||
);
|
||||
assert!(crate::suggest("becuase", 5).contains(&"because".to_string()));
|
||||
assert_eq!(crate::check_spelling("Becuase house").len(), 1);
|
||||
if matches!(crate::english_source(ProofLang::En), Some(EnglishSource::CoreList)) {
|
||||
assert!(
|
||||
!crate::is_correct("becuase")
|
||||
&& !crate::is_correct_with("becuase", &Options::default())
|
||||
&& !crate::is_correct_for("becuase", Some("en-GB"))
|
||||
);
|
||||
assert!(crate::suggest("becuase", 5).contains(&"because".to_string()));
|
||||
assert_eq!(crate::check_spelling("Becuase house").len(), 1);
|
||||
}
|
||||
// The dialect variants are Language data: none without it.
|
||||
for w in ["colour", "color"] {
|
||||
assert_eq!(en_variants::preferred(w, ProofLang::En), None);
|
||||
|
||||
@@ -389,8 +389,8 @@ fn licence_files(pack: &Path, left: &mut u64) -> Vec<(String, String)> {
|
||||
let path = dir.join(&n);
|
||||
let size = std::fs::symlink_metadata(&path).ok().filter(std::fs::Metadata::is_file)?.len();
|
||||
if size > (*left).min(MAX_LICENCE) {
|
||||
// Over what is left: none of the rest is read either.
|
||||
*left = 0;
|
||||
// Too big, or over what is left: this file is skipped, smaller ones still fit.
|
||||
log::warn!("Language pack licence {} is {size} bytes; not shown", path.display());
|
||||
return None;
|
||||
}
|
||||
let bytes = read_capped(&path, size.min(MAX_LICENCE)).ok()?;
|
||||
|
||||
@@ -288,13 +288,16 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
let Some(p) = it.next() else { continue };
|
||||
let after = it.next();
|
||||
let flag = match p {
|
||||
',' => true,
|
||||
// ` ,,` opens a typed quote (the doubled-punctuation rule suggests `„`).
|
||||
',' => after != Some(','),
|
||||
// `;)` and `;-)` are emoticons.
|
||||
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?')),
|
||||
':' | '!' | '?' => {
|
||||
!crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '»' | '’' | '!' | '?'))
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
|
||||
@@ -63,7 +63,9 @@ fn ends_sentence(gap: &str, prev: &str) -> bool {
|
||||
return false;
|
||||
}
|
||||
let rest = t.trim_start_matches(['.', '!', '?']).trim_start_matches(CLOSING);
|
||||
rest.starts_with(char::is_whitespace) && rest.trim_start().chars().all(|c| OPENING.contains(&c))
|
||||
// A dialogue dash after the mark brings back the narration (`—Vai? —disse ele.`), which goes
|
||||
// on in lower case.
|
||||
rest.starts_with(char::is_whitespace) && !rest.contains('—') && rest.trim_start().chars().all(|c| OPENING.contains(&c))
|
||||
}
|
||||
|
||||
/// Grammar issues in a paragraph of Portuguese text, in order.
|
||||
@@ -115,7 +117,7 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
':' | '!' | '?' => !crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '”' | '’' | '!' | '?')),
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
@@ -220,6 +222,9 @@ mod tests {
|
||||
clean("Zhou ficou em 5. lugar no séc. XX.");
|
||||
clean("Esperei... e ele apareceu.");
|
||||
clean("— Vai — disse ele. — Amanhã eu volto.");
|
||||
// craft-libs#488 review: no space after the dash.
|
||||
clean("—Vai? —disse ele.");
|
||||
clean("— Vai? — disse ele.");
|
||||
clean("Ele disse: \"Volto amanhã\". E saiu.");
|
||||
// A list item may start in lower case.
|
||||
clean("maçãs,");
|
||||
|
||||
@@ -71,8 +71,9 @@ fn check(d: &AffixDict, word: &str, opts: &crate::Options) -> bool {
|
||||
let Some(w) = crate::spelling::word_to_check(d, word, opts) else {
|
||||
return true;
|
||||
};
|
||||
// The whole word first, as Hunspell does: a hyphenated entry or user word.
|
||||
if w.contains('-') {
|
||||
return w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
return known(d, w) || w.split('-').all(|p| p.chars().count() <= 1 || known(d, p));
|
||||
}
|
||||
known(d, w)
|
||||
}
|
||||
@@ -96,6 +97,14 @@ mod tests {
|
||||
assert!(!d.rep().is_empty() && !d.map().is_empty() && !d.try_chars().is_empty());
|
||||
}
|
||||
|
||||
/// craft-libs#487 review: a hyphenated dictionary entry is a word even when its parts aren't.
|
||||
#[test]
|
||||
fn a_hyphenated_entry_is_looked_up_whole() {
|
||||
let d = AffixDict::from_hunspell("SET UTF-8\n", "1\ntic-tac\n", &[]).unwrap();
|
||||
assert!(is_correct_with(&d, "tic-tac"));
|
||||
assert!(!is_correct_with(&d, "tic-tacc"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_dictionary_nothing_is_flagged() {
|
||||
let none = AffixDict::default();
|
||||
|
||||
@@ -133,7 +133,14 @@ fn forms(base: &str, class: &str) -> Vec<String> {
|
||||
}
|
||||
"adj" => {
|
||||
if let Some(stem) = base.strip_suffix("ый").or_else(|| base.strip_suffix("ой")) {
|
||||
let mut v = with(stem, &["ого", "ому", "ым", "ом", "ая", "ой", "ую", "ое", "ые", "ых", "ыми"]);
|
||||
// `ы` is written `и` after г, к, х and ж, ш, ч, щ (`дорогие`, `большими`).
|
||||
let i = is_velar(last(stem)) || is_hushing(last(stem));
|
||||
let ends: &[&str] = if i {
|
||||
&["ого", "ому", "им", "ом", "ая", "ой", "ую", "ое", "ие", "их", "ими"]
|
||||
} else {
|
||||
&["ого", "ому", "ым", "ом", "ая", "ой", "ую", "ое", "ые", "ых", "ыми"]
|
||||
};
|
||||
let mut v = with(stem, ends);
|
||||
v.push(base.to_string());
|
||||
return v;
|
||||
}
|
||||
@@ -156,6 +163,16 @@ fn forms(base: &str, class: &str) -> Vec<String> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// craft-libs#495 review: `-ой` adjectives after a velar or hushing letter take `и`.
|
||||
#[test]
|
||||
fn oi_adjectives_spell_i_after_velars_and_hushers() {
|
||||
let f = forms("дорогой", "adj");
|
||||
assert!(f.contains(&"дорогие".to_string()) && f.contains(&"дорогими".to_string()) && !f.iter().any(|w| w.contains('ы')), "{f:?}");
|
||||
let f = forms("большой", "adj");
|
||||
assert!(f.contains(&"больших".to_string()) && !f.iter().any(|w| w.contains('ы')), "{f:?}");
|
||||
assert!(forms("молодой", "adj").contains(&"молодые".to_string()));
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
not(craft_proof_embed = "ru"),
|
||||
ignore = "needs the ru Language data (CRAFT_I18N_DIR; ru-extra.txt is embedded in every build that has it)"
|
||||
|
||||
@@ -264,6 +264,10 @@ const NO_COMMA_AFTER: &[&str] = &[
|
||||
"затем",
|
||||
"тем",
|
||||
"того",
|
||||
// `тем более что`, `при условии что`, `в то время как`.
|
||||
"более",
|
||||
"условии",
|
||||
"время",
|
||||
"ввиду",
|
||||
"вследствие",
|
||||
"несмотря",
|
||||
@@ -483,13 +487,16 @@ fn spaces(text: &str, v: &mut Vec<Issue>) {
|
||||
let Some(p) = it.next() else { continue };
|
||||
let after = it.next();
|
||||
let flag = match p {
|
||||
',' => true,
|
||||
// ` ,,` opens a typed quote (the doubled-punctuation rule suggests `„`).
|
||||
',' => after != Some(','),
|
||||
// `;)` and `;-)` are emoticons.
|
||||
';' => !matches!(after, Some(')' | '(' | '-' | 'P' | 'D')),
|
||||
// ` ...` is an ellipsis, ` .5` a number.
|
||||
'.' => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
// ` :)`, ` :-)`, ` :D` and similar are emoticons; ` ?!` is still a stray space.
|
||||
':' | '!' | '?' => after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '»' | '”' | '’' | '!' | '?')),
|
||||
':' | '!' | '?' => {
|
||||
!crate::score_colon(text, i) && after.is_none_or(|a| a.is_whitespace() || matches!(a, '"' | '»' | '”' | '’' | '!' | '?'))
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if flag {
|
||||
@@ -687,6 +694,10 @@ mod tests {
|
||||
clean("Расскажи хотя бы что-нибудь, за что тебя хвалят.");
|
||||
clean("Что ты делаешь? Который час?");
|
||||
clean("Кто-то что-то сказал.");
|
||||
// craft-libs#495 review: the second half of a compound conjunction.
|
||||
clean("Он устал, тем более что шёл долго.");
|
||||
clean("Приду, при условии что успею.");
|
||||
clean("Он читал, в то время как она спала.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -24,7 +24,9 @@ pub(crate) fn word_to_check_in<'a>(d: &AffixDict, word: &'a str, opts: &Options)
|
||||
if d.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
// A trailing combining mark with no precomposed form (`x\u{338}`) is part of the word.
|
||||
let edge = |c: char| !c.is_alphanumeric() && c != '\'' && c != '’';
|
||||
let w = word.trim_start_matches(edge).trim_end_matches(|c| edge(c) && !unicode_normalization::char::is_combining_mark(c));
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
@@ -56,4 +58,13 @@ mod tests {
|
||||
assert_eq!(word_to_check(&d, "B2B", &strict), Some("B2B"));
|
||||
assert_eq!(word_to_check(&d, "PRZEDSIĘBIORSTWO", &defaults), Some("PRZEDSIĘBIORSTWO"), "only short capitals are acronyms");
|
||||
}
|
||||
|
||||
/// craft-libs#487 review: a trailing combining mark stays on the word it marks.
|
||||
#[test]
|
||||
fn trailing_combining_marks_are_kept() {
|
||||
let d = AffixDict::from_hunspell("", "1\nkot\n", &[]).unwrap();
|
||||
let defaults = Options::default();
|
||||
assert_eq!(word_to_check(&d, "kox\u{338}", &defaults), Some("kox\u{338}"));
|
||||
assert_eq!(word_to_check(&d, "(kox\u{338}).", &defaults), Some("kox\u{338}"));
|
||||
}
|
||||
}
|
||||
+42
-110
@@ -1,29 +1,16 @@
|
||||
//! Turkish Hunspell dictionary parser with `FLAG num` support.
|
||||
//! The Turkish Hunspell dictionary (`FLAG num`: thousands of numeric suffix flags).
|
||||
//!
|
||||
//! Turkish uses `FLAG num`, storing thousands of numerical suffix flags (0..6465).
|
||||
//! This engine parses `/usr/share/hunspell/tr_TR.aff` and `.dic` safely without panics.
|
||||
//! Read by the shared affix engine ([`AffixDict`]), so Turkish follows Hunspell as every other
|
||||
//! language does: homonyms on separate lines keep each one's flags, a `.dic` entry ends before
|
||||
//! its morphological fields (`kitap/1 po:noun`), and an affix applies only where its condition
|
||||
//! matches the stem (`SFX 1 0 lar [^e]` never makes `kalelar`).
|
||||
|
||||
use std::collections::HashMap;
|
||||
use crate::affix::AffixDict;
|
||||
|
||||
/// A suffix rule for Turkish.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct TrSuffixRule {
|
||||
pub strip: Box<str>,
|
||||
pub add: Box<str>,
|
||||
pub cond: Box<str>,
|
||||
}
|
||||
|
||||
/// Dedicated Turkish Hunspell dictionary engine.
|
||||
/// The Turkish dictionary.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct TrDict {
|
||||
/// Stem -> list of numerical flags it carries.
|
||||
stems: HashMap<String, Box<[u16]>>,
|
||||
/// Flag number -> list of suffix rules.
|
||||
rules: HashMap<u16, Vec<TrSuffixRule>>,
|
||||
/// Suffix 'add' string -> list of (flag, rule_index).
|
||||
suffix_index: HashMap<String, Vec<(u16, u16)>>,
|
||||
/// Longest suffix 'add' string in characters.
|
||||
max_suffix_len: usize,
|
||||
dict: AffixDict,
|
||||
}
|
||||
|
||||
impl crate::slot::Dictionary for TrDict {
|
||||
@@ -43,102 +30,47 @@ impl crate::slot::Dictionary for TrDict {
|
||||
impl TrDict {
|
||||
/// Returns true if empty (no dictionary loaded).
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.stems.is_empty()
|
||||
self.dict.is_empty()
|
||||
}
|
||||
|
||||
/// Load from `.aff` and `.dic` contents.
|
||||
pub fn from_hunspell(aff: &str, dic: &str) -> Result<Self, String> {
|
||||
let mut rules: HashMap<u16, Vec<TrSuffixRule>> = HashMap::new();
|
||||
let mut max_suffix_len = 0;
|
||||
|
||||
for line in aff.lines() {
|
||||
let parts: Vec<&str> = line.split_whitespace().collect();
|
||||
// SFX <flag_num> <strip> <add> [<cond>]
|
||||
if parts.len() >= 4 && parts[0] == "SFX" {
|
||||
let Ok(flag) = parts[1].parse::<u16>() else { continue };
|
||||
// Skip header line: `SFX <flag> Y/N <count>`
|
||||
if parts.len() == 4 && (parts[2] == "Y" || parts[2] == "N") && parts[3].parse::<usize>().is_ok() {
|
||||
continue;
|
||||
}
|
||||
let strip = if parts[2] == "0" { "" } else { parts[2] };
|
||||
let add = if parts[3] == "0" { "" } else { parts[3] };
|
||||
let cond = if parts.len() > 4 { parts[4] } else { "." };
|
||||
|
||||
max_suffix_len = max_suffix_len.max(add.chars().count());
|
||||
|
||||
rules.entry(flag).or_default().push(TrSuffixRule { strip: strip.into(), add: add.into(), cond: cond.into() });
|
||||
}
|
||||
}
|
||||
|
||||
let mut suffix_index: HashMap<String, Vec<(u16, u16)>> = HashMap::new();
|
||||
for (&flag, r_list) in &rules {
|
||||
for (idx, r) in r_list.iter().enumerate() {
|
||||
if let Ok(idx_u16) = u16::try_from(idx) {
|
||||
suffix_index.entry(r.add.to_string()).or_default().push((flag, idx_u16));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut stems: HashMap<String, Box<[u16]>> = HashMap::new();
|
||||
for (i, line) in dic.lines().enumerate() {
|
||||
if i == 0 && line.trim().parse::<usize>().is_ok() {
|
||||
continue;
|
||||
}
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Some((word, flags_part)) = line.split_once('/') {
|
||||
let flags: Vec<u16> = flags_part.split(',').filter_map(|f| f.parse::<u16>().ok()).collect();
|
||||
stems.insert(word.to_string(), flags.into_boxed_slice());
|
||||
} else {
|
||||
stems.insert(line.to_string(), Box::new([]));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(TrDict { stems, rules, suffix_index, max_suffix_len })
|
||||
Ok(TrDict { dict: AffixDict::from_hunspell(aff, dic, &[])? })
|
||||
}
|
||||
|
||||
/// Check if word is correct (stem or affixed).
|
||||
/// Is `word` exactly (case and all) a stem or an affixed form of one?
|
||||
pub fn contains(&self, word: &str) -> bool {
|
||||
// Longer input is never a dictionary word, and isn't scanned (#3625).
|
||||
if self.is_empty() || word.is_empty() || word.len() > crate::affix::MAX_WORD_BYTES {
|
||||
return false;
|
||||
self.dict.contains(word)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// CodeRabbit on craft-libs#484/#489: suffix conditions, homonyms and morphological fields,
|
||||
/// each answer as hunspell 1.7 gives it on the same files.
|
||||
#[test]
|
||||
fn conditions_homonyms_and_fields_follow_hunspell() {
|
||||
let aff = "SET UTF-8\nFLAG num\nSFX 1 Y 2\nSFX 1 0 lar [^e]\nSFX 1 0 ler e\nSFX 2 Y 1\nSFX 2 0 de .\n";
|
||||
let d = TrDict::from_hunspell(aff, "4\nkitap/1\nkale/1\nkale/2\nsonra po:adv\n").unwrap();
|
||||
for (w, right) in
|
||||
[("kitaplar", true), ("kaleler", true), ("kalelar", false), ("kitapler", false), ("kalede", true), ("sonra", true), ("kitap", true)]
|
||||
{
|
||||
assert_eq!(d.contains(w), right, "{w}");
|
||||
}
|
||||
|
||||
// Direct stem hit
|
||||
if self.stems.contains_key(word) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Try stripping suffixes matching the tail
|
||||
let word_len = word.chars().count();
|
||||
let char_indices: Vec<(usize, char)> = word.char_indices().collect();
|
||||
|
||||
// Check each possible suffix length up to max_suffix_len
|
||||
for cut_chars in 0..=self.max_suffix_len.min(word_len) {
|
||||
let byte_idx = if cut_chars == 0 { word.len() } else { char_indices[word_len - cut_chars].0 };
|
||||
|
||||
let add = &word[byte_idx..];
|
||||
let base = &word[..byte_idx];
|
||||
|
||||
if let Some(matches) = self.suffix_index.get(add) {
|
||||
for &(flag, rule_idx) in matches {
|
||||
let Some(rule_list) = self.rules.get(&flag) else { continue };
|
||||
let Some(rule) = rule_list.get(rule_idx as usize) else { continue };
|
||||
|
||||
// Check condition on root
|
||||
let mut candidate = String::with_capacity(base.len() + rule.strip.len());
|
||||
candidate.push_str(base);
|
||||
candidate.push_str(&rule.strip);
|
||||
|
||||
if self.stems.get(&candidate).is_some_and(|f_list| f_list.contains(&flag)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
let d = TrDict::from_hunspell(aff, "1\nkitap/1,2\tpo:noun\n").unwrap();
|
||||
assert!(d.contains("kitaplar") && d.contains("kitapde"), "both flags before the field");
|
||||
}
|
||||
|
||||
/// #495 review: LibreOffice's `tr_TR.dic` has 920 entries with more than 1,024 flags
|
||||
/// (`abart`, `savaş`, `inan`: up to 3,926); such a stem and its forms are words.
|
||||
#[test]
|
||||
fn a_stem_with_thousands_of_flags_is_a_word() {
|
||||
let aff = "SET UTF-8\nFLAG num\nSFX 1 Y 1\nSFX 1 0 de .\nSFX 3999 Y 1\nSFX 3999 0 lar .\n";
|
||||
let flags = (1..=4000).map(|i| i.to_string()).collect::<Vec<_>>().join(",");
|
||||
let d = TrDict::from_hunspell(aff, &format!("2\nabart/{flags}\nkitap\n")).unwrap();
|
||||
assert!(d.contains("abart") && d.contains("abartde") && d.contains("abartlar") && d.contains("kitap"));
|
||||
assert!(!d.contains("kitaplar"));
|
||||
}
|
||||
}
|
||||
+100
-13
@@ -10,6 +10,7 @@
|
||||
//! `ne ... ne` conjunctions should not have commas immediately before or after them.
|
||||
//! - **Doubled punctuation** (`..`, `,,`, `;;`).
|
||||
|
||||
use super::suggest::turkish_to_lower;
|
||||
use crate::{Issue, IssueKind, ProofLang, words};
|
||||
|
||||
/// Turkish reduplications and emphatic word pairings that are intentionally repeated.
|
||||
@@ -120,15 +121,15 @@ const ALLOWED_REPEATS: &[&str] = &[
|
||||
/// Turkish abbreviations that end with a period and after which lower case may continue.
|
||||
const ABBREVIATIONS: &[&str] = &[
|
||||
"vb", "vs", "vd", "bkz", "bknz", "ör", "örnek", "mad", "md", "s", "sf", "sn", "dk", "sa", "yy", "m", "km", "cm", "mm", "gr", "kg", "mg", "dr",
|
||||
"prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "a.ş", "ltd", "şti", "t.c",
|
||||
"tbmm", "meb", "möt", "mö", "ms",
|
||||
"prof", "doç", "yrd", "av", "ist", "ank", "izm", "cad", "sok", "mah", "apt", "no", "tel", "faks", "tl", "kr", "krş", "ltd", "şti", "tbmm", "meb",
|
||||
"möt", "mö", "ms",
|
||||
];
|
||||
|
||||
/// Conjunctions that must not be preceded by a comma.
|
||||
const CONJUNCTIONS_NO_PRE_COMMA: &[&str] = &["ve", "veya", "yahut", "velev", "veyahut", "de", "da"];
|
||||
|
||||
/// Conjunctions that must not be followed by a comma.
|
||||
const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "ne", "ya", "gerek", "ister", "de", "da"];
|
||||
const CONJUNCTIONS_NO_POST_COMMA: &[&str] = &["ve", "veya", "yahut", "veyahut", "hem", "gerek", "ister"];
|
||||
|
||||
/// Check grammar and punctuation in a Turkish text block.
|
||||
pub fn check(text: &str) -> Vec<Issue> {
|
||||
@@ -152,9 +153,11 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
|
||||
None => continue,
|
||||
};
|
||||
let only_spaces = gap.chars().all(|c| c == ' ' || c == '\t');
|
||||
let lower1 = w1.to_lowercase();
|
||||
let lower2 = w2.to_lowercase();
|
||||
if only_spaces && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) {
|
||||
let lower1 = turkish_to_lower(w1);
|
||||
let lower2 = turkish_to_lower(w2);
|
||||
// Words, not numbers or codes (`1 1`, `A1 A1`); `Ankara'da Ankara'da` is one.
|
||||
let word = w1.chars().any(char::is_alphabetic) && !w1.chars().any(|c| c.is_numeric());
|
||||
if only_spaces && word && lower1 == lower2 && !ALLOWED_REPEATS.contains(&lower1.as_str()) {
|
||||
v.push(Issue {
|
||||
start: pair[1].0,
|
||||
end: pair[1].1,
|
||||
@@ -172,9 +175,30 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
|
||||
let (idx, c) = chars[i];
|
||||
if matches!(c, ',' | '.' | '?' | '!' | ';' | ':') && i > 0 && chars[i - 1].1 == ' ' {
|
||||
let prev_sp = chars[i - 1].0;
|
||||
// Leave emoticons and ellipses alone
|
||||
let is_dot_in_dots = c == '.' && (i + 1 < chars.len() && chars[i + 1].1 == '.');
|
||||
if !is_dot_in_dots {
|
||||
// Leave ellipses (` ...`), numbers (` .5`), emoticons (` :)`, ` ;-)`, ` :D`) and
|
||||
// scores (`3 : 1`) alone.
|
||||
let after = chars.get(i + 1).map(|&(_, a)| a);
|
||||
let ratio = c == ':'
|
||||
&& i >= 2
|
||||
&& chars.get(i - 2).is_some_and(|&(_, b)| b.is_ascii_digit())
|
||||
&& after == Some(' ')
|
||||
&& chars.get(i + 2).is_some_and(|&(_, d)| d.is_ascii_digit());
|
||||
// `:)`, `;(`, `:-)` are emoticons; `:D` and `;P` only when a space or the end follows
|
||||
// (`;Dün` is a stray space before a word).
|
||||
let ends = |k: usize| chars.get(k).is_none_or(|&(_, e)| e.is_whitespace());
|
||||
let emoticon = match after {
|
||||
Some(')' | '(') => true,
|
||||
Some('-') => matches!(chars.get(i + 2), Some(&(_, ')' | '('))),
|
||||
Some('D' | 'P') => ends(i + 2),
|
||||
_ => false,
|
||||
};
|
||||
let flag = !ratio
|
||||
&& match c {
|
||||
',' | '!' | '?' => true,
|
||||
':' | ';' => !emoticon,
|
||||
_ => !matches!(after, Some('.')) && !after.is_some_and(|a| a.is_ascii_digit()),
|
||||
};
|
||||
if flag {
|
||||
v.push(Issue {
|
||||
start: prev_sp,
|
||||
end: idx + c.len_utf8(),
|
||||
@@ -205,7 +229,7 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
|
||||
// 3. Comma before or after conjunctions (`ali, ve veli` or `ve, veli`)
|
||||
for (i, &(w_start, w_end)) in ws.iter().enumerate() {
|
||||
let word = match text.get(w_start..w_end) {
|
||||
Some(w) => w.to_lowercase(),
|
||||
Some(w) => turkish_to_lower(w),
|
||||
None => continue,
|
||||
};
|
||||
// Check comma before conjunction
|
||||
@@ -268,9 +292,18 @@ pub fn check_with(text: &str, capital_first: bool) -> Vec<Issue> {
|
||||
let prev_word = text.get(ws[i - 1].0..prev_end).unwrap_or("");
|
||||
if let Some(gap) = text.get(prev_end..w_start) {
|
||||
let trimmed = gap.trim();
|
||||
// A space must follow the mark (`rapor.txt`, `www.example.com` are one token), and a
|
||||
// number before a period is an ordinal (`5. sınıf`, `21. yüzyıl`).
|
||||
let spaced = gap.rfind(['.', '?', '!']).and_then(|m| gap.get(m + 1..)).is_some_and(|rest| rest.contains(char::is_whitespace));
|
||||
let ordinal = trimmed.ends_with('.') && prev_word.chars().any(|c| c.is_ascii_digit());
|
||||
// An initial (`A.Ş.`, `T.C.`, `M. Kemal`) ends no sentence.
|
||||
let initial = trimmed.starts_with('.') && prev_word.chars().count() == 1;
|
||||
if (trimmed.ends_with('.') || trimmed.ends_with('?') || trimmed.ends_with('!'))
|
||||
&& spaced
|
||||
&& !ordinal
|
||||
&& !initial
|
||||
&& !trimmed.contains("...")
|
||||
&& !ABBREVIATIONS.contains(&prev_word.to_lowercase().as_str())
|
||||
&& !ABBREVIATIONS.contains(&turkish_to_lower(prev_word).as_str())
|
||||
&& let Some(w) = text.get(w_start..w_end)
|
||||
&& w.chars().next().is_some_and(char::is_lowercase)
|
||||
{
|
||||
@@ -310,6 +343,31 @@ pub fn turkish_capitalize(s: &str) -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// CodeRabbit on craft-libs#484/#489: emoticons, decimals and scores aren't stray spaces;
|
||||
/// ordinals and file or web names don't end a sentence.
|
||||
#[test]
|
||||
fn emoticons_numbers_ordinals_and_names_are_left_alone() {
|
||||
for ok in [
|
||||
"Harika :)",
|
||||
"Tamam ;)",
|
||||
"Süper :D",
|
||||
"Oran .5 oldu.",
|
||||
"Maç 3 : 1 bitti.",
|
||||
"Ali 5. sınıfta okuyor.",
|
||||
"Bina 21. yüzyıldan kalma.",
|
||||
"Dosyayı rapor.txt olarak kaydet.",
|
||||
"Adres www.example.com olarak yazıldı.",
|
||||
] {
|
||||
assert!(check(ok).is_empty(), "{ok}: {:?}", check(ok));
|
||||
}
|
||||
assert_eq!(check("Gel , bak.").len(), 1, "a real stray space is still flagged");
|
||||
for bad in ["Not :bu", "Ne ?Bu", "Peki ;Dün", "Evet !"] {
|
||||
assert!(check(bad).iter().any(|i| i.message.starts_with("Noktalama")), "{bad}: {:?}", check(bad));
|
||||
}
|
||||
assert!(check("Harika :-) dedi.").is_empty() && check("Peki ;P").is_empty());
|
||||
assert_eq!(check("Geldi. bu iyi.").len(), 1, "a real sentence start is still flagged");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clean_turkish_text_produces_no_issues() {
|
||||
let text = "Bu bir test cümlesidir. Kitap ve defter aldım. Çocuk yavaş yavaş yürüdü.";
|
||||
@@ -343,10 +401,39 @@ mod tests {
|
||||
assert_eq!(issues.len(), 1);
|
||||
assert!(issues[0].message.contains("virgül konulmaz"));
|
||||
|
||||
let text2 = "Bana kalırsa bu mesele de, haklı olan taraf bizdik.";
|
||||
let text2 = "Elma ve, armut.";
|
||||
let issues2 = check(text2);
|
||||
assert_eq!(issues2.len(), 1);
|
||||
assert!(issues2[0].message.contains("'de' bağlacından sonra virgül konulmaz"));
|
||||
assert!(issues2[0].message.contains("'ve' bağlacından sonra virgül konulmaz"));
|
||||
}
|
||||
|
||||
/// `de`/`da`, `ne` and `ya` often take a comma after them in correct Turkish, so no mark
|
||||
/// (craft-libs#488 review).
|
||||
#[test]
|
||||
fn a_comma_after_de_ne_or_ya_is_fine() {
|
||||
for text in ["Ali de, Veli de geldi.", "Ne, sen mi geldin?", "Ya, bilmiyorum.", "Bu mesele de, haklı olan bizdik."] {
|
||||
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
|
||||
}
|
||||
}
|
||||
|
||||
/// craft-libs#495 review: initials (`A.Ş.`, `T.C.`) and `İst.` end no sentence, and Turkish
|
||||
/// case folding finds `Işık ışık`; a word with an apostrophe is still a repeat.
|
||||
#[test]
|
||||
fn turkish_case_and_initials() {
|
||||
for text in ["Şirket A.Ş. bugün açıldı.", "O T.C. vatandaşı.", "Adres İst. caddesi."] {
|
||||
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
|
||||
}
|
||||
for text in ["Işık ışık yandı.", "Ankara'da Ankara'da kaldık."] {
|
||||
assert_eq!(check(text).iter().filter(|i| i.message == "Yinelenen sözcük").count(), 1, "{text}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Repeated numbers are not repeated words (craft-libs#488 review).
|
||||
#[test]
|
||||
fn repeated_numbers_are_not_flagged() {
|
||||
for text in ["Skor 1 1 bitti.", "Toplam 100 100 değil.", "Oda A1 A1 değil."] {
|
||||
assert!(check(text).is_empty(), "{text}: {:?}", check(text));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3,12 +3,13 @@
|
||||
//! Turkish words are split at syllables according to Turkish Language Association (TDK) rules.
|
||||
//! Because Turkish syllable boundaries are fully regular, they can be determined directly by rule
|
||||
//! without needing external pattern files or word lists:
|
||||
//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü` (both lowercase and uppercase).
|
||||
//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `ö-ğren-ci`).
|
||||
//! - Turkish vowels: `a`, `e`, `ı`, `i`, `o`, `ö`, `u`, `ü`, and `â`, `î`, `û` (`hi-kâ-ye`,
|
||||
//! `dük-kân`), lowercase and uppercase.
|
||||
//! - A single consonant between two vowels joins the following syllable (`ba-lık`, `a-ra-ba`).
|
||||
//! - Of two consonants between two vowels, the first stays with the preceding syllable and the
|
||||
//! second joins the next (`alt-lık`, `sev-gi`, `tür-kçe`).
|
||||
//! second joins the next (`öğ-ren-ci`, `sev-gi`, `kar-deş`).
|
||||
//! - Of three consonants between two vowels, the first two stay with the preceding syllable and the
|
||||
//! third joins the next (`türk-çe`, `sürp-riz`).
|
||||
//! third joins the next (`türk-çe`, `alt-lık`, `sürp-riz`).
|
||||
//! - In compound words, syllable division follows pronunciation across the boundary (`ba-şöğ-ret-men`,
|
||||
//! `il-ko-kul`), which matches the general vowel/consonant distribution rules.
|
||||
//!
|
||||
@@ -17,7 +18,7 @@
|
||||
|
||||
/// Is `c` a Turkish vowel?
|
||||
pub fn is_vowel(c: char) -> bool {
|
||||
matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü')
|
||||
matches!(c, 'a' | 'e' | 'ı' | 'i' | 'o' | 'ö' | 'u' | 'ü' | 'â' | 'î' | 'û' | 'A' | 'E' | 'I' | 'İ' | 'O' | 'Ö' | 'U' | 'Ü' | 'Â' | 'Î' | 'Û')
|
||||
}
|
||||
|
||||
/// Is `c` a Turkish letter?
|
||||
@@ -86,6 +87,18 @@ mod tests {
|
||||
assert_eq!(split("başöğretmen"), "ba-şöğ-ret-men");
|
||||
}
|
||||
|
||||
/// CodeRabbit on craft-libs#484: `â`, `î` and `û` are vowels, and the module doc's examples
|
||||
/// are the ones the rules give.
|
||||
#[test]
|
||||
fn circumflex_vowels_and_doc_examples() {
|
||||
assert_eq!(split("dükkân"), "dük-kân");
|
||||
assert_eq!(split("hikâye"), "hi-kâ-ye");
|
||||
assert_eq!(split("kâğıt"), "kâ-ğıt");
|
||||
assert_eq!(split("millî"), "mil-lî");
|
||||
assert_eq!(split("kardeş"), "kar-deş");
|
||||
assert_eq!(split("sürpriz"), "sürp-riz");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_or_non_words_stay_unsplit() {
|
||||
assert_eq!(split(""), "");
|
||||
|
||||
@@ -78,12 +78,8 @@ fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool {
|
||||
}
|
||||
let w = word.trim_matches(|c: char| !c.is_alphanumeric() && c != '\'' && c != '’');
|
||||
let w = w.trim_matches(['\'', '’']);
|
||||
let n = w.chars().count();
|
||||
if n <= 1 || (opts.ignore_numbers && w.chars().any(|c| c.is_ascii_digit())) || !w.chars().any(char::is_alphabetic) {
|
||||
return true;
|
||||
}
|
||||
// Check user dictionary first
|
||||
if crate::in_user_dictionary(&suggest::turkish_to_lower(w)) || crate::in_user_dictionary(&w.to_lowercase()) {
|
||||
// One letter, no letters, digits, capitals (Ignore words in UPPERCASE) or the user dictionary.
|
||||
if crate::accepted_by_options(w, opts) || crate::in_user_dictionary(&suggest::turkish_to_lower(w)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -161,4 +157,15 @@ mod tests {
|
||||
assert!(crate::is_correct_in_with("2026", lang, &check_digits), "numbers alone stay correct");
|
||||
assert!(is_correct("kitapp2"), "direct checks keep the defaults");
|
||||
}
|
||||
|
||||
/// Ignore words in UPPERCASE reaches Turkish's own entry points too, so `NATO` and `TBMM` are
|
||||
/// right by default and checked when the option is off (craft-libs#495 review).
|
||||
#[test]
|
||||
fn turkish_ignores_capitals_unless_told_otherwise() {
|
||||
test_dict::install();
|
||||
let check_caps = crate::Options { ignore_uppercase: false, ..crate::Options::default() };
|
||||
assert!(is_correct("TBMMX") && is_correct_with_options("TBMMX", &crate::Options::default()));
|
||||
assert!(!is_correct_with_options("TBMMX", &check_caps));
|
||||
assert!(is_correct_with_options("KİTAP", &check_caps), "a word in Turkish capitals is still looked up");
|
||||
}
|
||||
}
|
||||
@@ -28,12 +28,9 @@ const COMMON_REPLACEMENTS: &[(&str, &str)] = &[
|
||||
("sarmısak", "sarımsak"),
|
||||
("makina", "makine"),
|
||||
("ünvan", "unvan"),
|
||||
("rastlantı", "rastlantı"),
|
||||
("labaratuvar", "laboratuvar"),
|
||||
("laboratuar", "laboratuvar"),
|
||||
("kravat", "kravat"),
|
||||
("gardolap", "gardırop"),
|
||||
("egzoz", "egzoz"),
|
||||
("egzozt", "egzoz"),
|
||||
("egsoz", "egzoz"),
|
||||
("gelcen", "geleceksin"),
|
||||
@@ -73,7 +70,7 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
// 1. Check direct common replacement table
|
||||
for &(wrong, right) in COMMON_REPLACEMENTS {
|
||||
if lower == wrong && seen.insert(right.to_string()) {
|
||||
suggestions.push(right.to_string());
|
||||
suggestions.push(match_casing(word, right));
|
||||
if suggestions.len() >= max {
|
||||
return suggestions;
|
||||
}
|
||||
@@ -88,7 +85,10 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
if !d.is_empty() {
|
||||
let variants = generate_diacritic_variants(&lower);
|
||||
for cand in variants {
|
||||
if cand != lower && d.contains(&cand) && seen.insert(cand.clone()) {
|
||||
if cand != lower
|
||||
&& let Some(cand) = in_dictionary(d, &cand)
|
||||
&& seen.insert(cand.clone())
|
||||
{
|
||||
suggestions.push(match_casing(word, &cand));
|
||||
if suggestions.len() >= max {
|
||||
return suggestions;
|
||||
@@ -124,7 +124,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
let mut swapped = chars.clone();
|
||||
swapped.swap(i, i + 1);
|
||||
let cand: String = swapped.into_iter().collect();
|
||||
if d.contains(&cand) && seen.insert(cand.clone()) {
|
||||
if let Some(cand) = in_dictionary(d, &cand)
|
||||
&& seen.insert(cand.clone())
|
||||
{
|
||||
suggestions.push(match_casing(word, &cand));
|
||||
if suggestions.len() >= max {
|
||||
return suggestions;
|
||||
@@ -144,7 +146,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
let mut deleted = chars.clone();
|
||||
deleted.remove(i);
|
||||
let cand: String = deleted.into_iter().collect();
|
||||
if d.contains(&cand) && seen.insert(cand.clone()) {
|
||||
if let Some(cand) = in_dictionary(d, &cand)
|
||||
&& seen.insert(cand.clone())
|
||||
{
|
||||
suggestions.push(match_casing(word, &cand));
|
||||
if suggestions.len() >= max {
|
||||
return suggestions;
|
||||
@@ -161,7 +165,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
*target = c;
|
||||
}
|
||||
let cand: String = substituted.into_iter().collect();
|
||||
if d.contains(&cand) && seen.insert(cand.clone()) {
|
||||
if let Some(cand) = in_dictionary(d, &cand)
|
||||
&& seen.insert(cand.clone())
|
||||
{
|
||||
suggestions.push(match_casing(word, &cand));
|
||||
if suggestions.len() >= max {
|
||||
return suggestions;
|
||||
@@ -177,7 +183,9 @@ pub fn suggest_with(d: &TrDict, word: &str, max: usize) -> Vec<String> {
|
||||
let mut inserted = chars.clone();
|
||||
inserted.insert(i, c);
|
||||
let cand: String = inserted.into_iter().collect();
|
||||
if d.contains(&cand) && seen.insert(cand.clone()) {
|
||||
if let Some(cand) = in_dictionary(d, &cand)
|
||||
&& seen.insert(cand.clone())
|
||||
{
|
||||
suggestions.push(match_casing(word, &cand));
|
||||
if suggestions.len() >= max {
|
||||
return suggestions;
|
||||
@@ -258,10 +266,34 @@ pub fn turkish_to_lower(s: &str) -> String {
|
||||
out
|
||||
}
|
||||
|
||||
/// Turkish uppercase with dotted/dotless I support (`i` -> `İ`, `ı` -> `I`).
|
||||
pub fn turkish_to_upper(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'i' => out.push('İ'),
|
||||
'ı' => out.push('I'),
|
||||
other => out.extend(other.to_uppercase()),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// `cand` (lower case) as the dictionary writes it: itself, or capitalized for a name
|
||||
/// (`İstanbul`), which a lower-case lookup doesn't find.
|
||||
fn in_dictionary(d: &TrDict, cand: &str) -> Option<String> {
|
||||
if d.contains(cand) {
|
||||
return Some(cand.to_string());
|
||||
}
|
||||
let cap = super::grammar::turkish_capitalize(cand);
|
||||
d.contains(&cap).then_some(cap)
|
||||
}
|
||||
|
||||
/// Match original casing on suggestion.
|
||||
fn match_casing(orig: &str, cand: &str) -> String {
|
||||
if orig.chars().all(char::is_uppercase) {
|
||||
cand.to_uppercase()
|
||||
let mut letters = orig.chars().filter(|c| c.is_alphabetic()).peekable();
|
||||
if letters.peek().is_some() && letters.all(char::is_uppercase) {
|
||||
turkish_to_upper(cand)
|
||||
} else if orig.chars().next().is_some_and(char::is_uppercase) {
|
||||
super::grammar::turkish_capitalize(cand)
|
||||
} else {
|
||||
@@ -273,6 +305,19 @@ fn match_casing(orig: &str, cand: &str) -> String {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// CodeRabbit on craft-libs#484/#489: a suggestion in capitals uses Turkish capitals
|
||||
/// (`İ`, not `I`), and no table entry suggests the word itself.
|
||||
#[test]
|
||||
fn capitals_are_turkish_and_no_entry_suggests_itself() {
|
||||
assert_eq!(match_casing("KİTAPP", "kitap"), "KİTAP");
|
||||
assert_eq!(match_casing("ISTANBUL", "istanbul"), "İSTANBUL");
|
||||
assert_eq!(match_casing("KAPI", "kapı"), "KAPI");
|
||||
assert_eq!(turkish_to_upper("ıiçşğüö"), "IİÇŞĞÜÖ");
|
||||
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nkitap\nistanbul\n").unwrap();
|
||||
assert!(suggest_with(&d, "KİTAPP", 3).contains(&"KİTAP".to_string()));
|
||||
assert!(COMMON_REPLACEMENTS.iter().all(|(a, b)| a != b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn common_replacements_are_suggested() {
|
||||
let sugs = suggest("yalnış", 3);
|
||||
@@ -302,11 +347,24 @@ mod tests {
|
||||
assert!(t.elapsed().as_secs_f64() < 10.0, "{:?}", t.elapsed());
|
||||
}
|
||||
|
||||
/// craft-libs#495 review: a name in the dictionary (`İstanbul`, `Ankara`) is found by the
|
||||
/// diacritic and typo steps from a lower-case slip, in its dictionary case.
|
||||
#[test]
|
||||
fn capitalized_stems_are_suggested() {
|
||||
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nİstanbul\nAnkara\n").unwrap();
|
||||
assert!(suggest_with(&d, "istanbu", 3).contains(&"İstanbul".to_string()), "{:?}", suggest_with(&d, "istanbu", 3));
|
||||
assert!(suggest_with(&d, "ankraa", 3).contains(&"Ankara".to_string()), "{:?}", suggest_with(&d, "ankraa", 3));
|
||||
assert!(suggest_with(&d, "Istanbul", 3).contains(&"İstanbul".to_string()), "{:?}", suggest_with(&d, "Istanbul", 3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_words_get_only_the_tables_replacement() {
|
||||
let d = TrDict::from_hunspell("SET UTF-8\n", "2\nbirkaç\nsonra\n").unwrap();
|
||||
assert_eq!(suggest_with(&d, "bir kaç", 3), vec!["birkaç".to_string()]);
|
||||
assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["birkaç".to_string()]);
|
||||
// craft-libs#487 review: the table keeps the word's case.
|
||||
assert_eq!(suggest_with(&d, "Bir kaç", 3), vec!["Birkaç".to_string()]);
|
||||
assert_eq!(suggest_with(&d, "BİR KAÇ", 3), vec!["BİRKAÇ".to_string()]);
|
||||
assert_eq!(suggest_with(&d, "Herşey", 3), vec!["Her şey".to_string()]);
|
||||
assert!(suggest_with(&d, "sorna sorna", 3).is_empty());
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user